2024-09-13T13:56:42.644 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T13:56:42.670 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T13:56:42.771 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903545 branch: wip-aclamk-cbt-combined description: upgrade/quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: akupczyk@redhat.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7903545' kernel: kdb: 1 sha1: distro ktype: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: wip-aclamk-cbt-combined ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - PG_AVAILABILITY sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 workunit: branch: wip-aclamk-cbt-combined sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 owner: scheduled_akupczyk@teuthology priority: 160 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 6545 sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 682/13280 suite: upgrade suite_branch: wip-aclamk-cbt-combined suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 targets: smithi099.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAVdixK/l6qLT0+musCmZC63chbllCoJzRgDU6ClKpd19s1dv6ezsSHRlcqxkpNABJEddKYfwmOKIn5ICtGWVH0= smithi177.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG5E2tTGU34xSxAkpbPjoErIkv99TInAH7BAxrFvOaGhUMAoSzebytOjJlw24/1ep/HNTdLCy7KJMOcLVjfyTzI= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_10:00:18 tube: smithi user: akupczyk verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-13T13:56:42.771 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa; will attempt to use it 2024-09-13T13:56:42.772 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks 2024-09-13T13:56:42.772 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T13:56:42.773 INFO:teuthology.task.internal:Checking packages... 2024-09-13T13:56:42.795 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'd58f73069a8157a9ace12a0247ee3e8497707ea9' 2024-09-13T13:56:42.795 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T13:56:42.795 INFO:teuthology.packaging:ref: None 2024-09-13T13:56:42.795 INFO:teuthology.packaging:tag: None 2024-09-13T13:56:42.795 INFO:teuthology.packaging:branch: wip-aclamk-cbt-combined 2024-09-13T13:56:42.796 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T13:56:42.796 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=wip-aclamk-cbt-combined 2024-09-13T13:56:42.949 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4697-gd58f7306-1jammy 2024-09-13T13:56:42.950 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T13:56:42.956 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T13:56:42.956 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T13:56:42.974 INFO:teuthology.task.internal:Saving configuration 2024-09-13T13:56:42.995 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T13:56:43.001 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T13:56:43.034 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi099.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903545', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 13:51:10.204152', 'locked_by': 'scheduled_akupczyk@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIADWpO+H4t88Ygr626OkxJZv5czd0mFKzOf7WW5DLC4+'} 2024-09-13T13:56:43.065 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi177.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903545', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 13:51:10.202268', 'locked_by': 'scheduled_akupczyk@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEDqiy2oKZX9xjGi/3lPPYx//LLSJoB2D7ZxErYgzlWp'} 2024-09-13T13:56:43.065 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T13:56:43.072 INFO:teuthology.task.internal:roles: ubuntu@smithi099.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-13T13:56:43.072 INFO:teuthology.task.internal:roles: ubuntu@smithi177.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-13T13:56:43.072 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T13:56:43.144 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fd439f87be0>, signals=[15]) 2024-09-13T13:56:43.144 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T13:56:43.150 INFO:teuthology.task.internal:Opening connections... 2024-09-13T13:56:43.150 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi099.front.sepia.ceph.com 2024-09-13T13:56:43.152 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:56:43.238 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi177.front.sepia.ceph.com 2024-09-13T13:56:43.239 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:56:43.323 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T13:56:43.330 DEBUG:teuthology.orchestra.run.smithi099:> uname -m 2024-09-13T13:56:43.335 INFO:teuthology.orchestra.run.smithi099.stdout:x86_64 2024-09-13T13:56:43.335 DEBUG:teuthology.orchestra.run.smithi099:> cat /etc/os-release 2024-09-13T13:56:43.386 INFO:teuthology.orchestra.run.smithi099.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-09-13T13:56:43.386 INFO:teuthology.orchestra.run.smithi099.stdout:NAME="Ubuntu" 2024-09-13T13:56:43.386 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION_ID="22.04" 2024-09-13T13:56:43.386 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-09-13T13:56:43.386 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION_CODENAME=jammy 2024-09-13T13:56:43.386 INFO:teuthology.orchestra.run.smithi099.stdout:ID=ubuntu 2024-09-13T13:56:43.386 INFO:teuthology.orchestra.run.smithi099.stdout:ID_LIKE=debian 2024-09-13T13:56:43.387 INFO:teuthology.orchestra.run.smithi099.stdout:HOME_URL="https://www.ubuntu.com/" 2024-09-13T13:56:43.387 INFO:teuthology.orchestra.run.smithi099.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-09-13T13:56:43.387 INFO:teuthology.orchestra.run.smithi099.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-09-13T13:56:43.387 INFO:teuthology.orchestra.run.smithi099.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-09-13T13:56:43.387 INFO:teuthology.orchestra.run.smithi099.stdout:UBUNTU_CODENAME=jammy 2024-09-13T13:56:43.387 INFO:teuthology.lock.ops:Updating smithi099.front.sepia.ceph.com on lock server 2024-09-13T13:56:43.413 DEBUG:teuthology.orchestra.run.smithi177:> uname -m 2024-09-13T13:56:43.418 INFO:teuthology.orchestra.run.smithi177.stdout:x86_64 2024-09-13T13:56:43.419 DEBUG:teuthology.orchestra.run.smithi177:> cat /etc/os-release 2024-09-13T13:56:43.464 INFO:teuthology.orchestra.run.smithi177.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-09-13T13:56:43.464 INFO:teuthology.orchestra.run.smithi177.stdout:NAME="Ubuntu" 2024-09-13T13:56:43.464 INFO:teuthology.orchestra.run.smithi177.stdout:VERSION_ID="22.04" 2024-09-13T13:56:43.464 INFO:teuthology.orchestra.run.smithi177.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-09-13T13:56:43.465 INFO:teuthology.orchestra.run.smithi177.stdout:VERSION_CODENAME=jammy 2024-09-13T13:56:43.465 INFO:teuthology.orchestra.run.smithi177.stdout:ID=ubuntu 2024-09-13T13:56:43.465 INFO:teuthology.orchestra.run.smithi177.stdout:ID_LIKE=debian 2024-09-13T13:56:43.465 INFO:teuthology.orchestra.run.smithi177.stdout:HOME_URL="https://www.ubuntu.com/" 2024-09-13T13:56:43.465 INFO:teuthology.orchestra.run.smithi177.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-09-13T13:56:43.465 INFO:teuthology.orchestra.run.smithi177.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-09-13T13:56:43.465 INFO:teuthology.orchestra.run.smithi177.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-09-13T13:56:43.465 INFO:teuthology.orchestra.run.smithi177.stdout:UBUNTU_CODENAME=jammy 2024-09-13T13:56:43.465 INFO:teuthology.lock.ops:Updating smithi177.front.sepia.ceph.com on lock server 2024-09-13T13:56:43.496 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T13:56:43.505 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T13:56:43.510 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T13:56:43.510 DEBUG:teuthology.orchestra.run.smithi099:> test '!' -e /home/ubuntu/cephtest 2024-09-13T13:56:43.512 DEBUG:teuthology.orchestra.run.smithi177:> test '!' -e /home/ubuntu/cephtest 2024-09-13T13:56:43.517 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T13:56:43.523 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T13:56:43.523 DEBUG:teuthology.orchestra.run.smithi099:> test -z $(ls -A /var/lib/ceph) 2024-09-13T13:56:43.559 DEBUG:teuthology.orchestra.run.smithi177:> test -z $(ls -A /var/lib/ceph) 2024-09-13T13:56:43.591 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T13:56:43.661 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T13:56:43.675 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T13:56:43.675 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T13:56:43.675 DEBUG:teuthology.orchestra.run.smithi099:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T13:56:43.675 DEBUG:teuthology.orchestra.run.smithi177:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T13:56:43.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:56:43.680 DEBUG:teuthology.orchestra.run.smithi099:> uname -r 2024-09-13T13:56:43.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:56:43.681 DEBUG:teuthology.orchestra.run.smithi177:> uname -r 2024-09-13T13:56:43.726 INFO:teuthology.orchestra.run.smithi099.stdout:5.15.0-56-generic 2024-09-13T13:56:43.727 INFO:teuthology.task.kernel:Running kernel on smithi099: 5.15.0-56-generic 2024-09-13T13:56:43.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo apt-get clean 2024-09-13T13:56:43.728 INFO:teuthology.orchestra.run.smithi177.stdout:5.15.0-56-generic 2024-09-13T13:56:43.728 INFO:teuthology.task.kernel:Running kernel on smithi177: 5.15.0-56-generic 2024-09-13T13:56:43.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo apt-get clean 2024-09-13T13:56:43.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo apt-get update 2024-09-13T13:56:43.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo apt-get update 2024-09-13T13:56:43.999 INFO:teuthology.orchestra.run.smithi099.stdout:Get:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-09-13T13:56:44.114 INFO:teuthology.orchestra.run.smithi177.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T13:56:44.115 INFO:teuthology.orchestra.run.smithi177.stdout:Get:2 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-09-13T13:56:44.115 INFO:teuthology.orchestra.run.smithi099.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T13:56:44.181 INFO:teuthology.orchestra.run.smithi177.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-09-13T13:56:44.198 INFO:teuthology.orchestra.run.smithi099.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-09-13T13:56:44.220 INFO:teuthology.orchestra.run.smithi099.stdout:Get:4 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [539 kB] 2024-09-13T13:56:44.318 INFO:teuthology.orchestra.run.smithi099.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,806 kB] 2024-09-13T13:56:44.356 INFO:teuthology.orchestra.run.smithi099.stdout:Get:6 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [295 kB] 2024-09-13T13:56:44.358 INFO:teuthology.orchestra.run.smithi099.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.3 kB] 2024-09-13T13:56:44.359 INFO:teuthology.orchestra.run.smithi099.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.7 kB] 2024-09-13T13:56:44.359 INFO:teuthology.orchestra.run.smithi099.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,377 kB] 2024-09-13T13:56:44.383 INFO:teuthology.orchestra.run.smithi099.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [409 kB] 2024-09-13T13:56:44.387 INFO:teuthology.orchestra.run.smithi099.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [584 B] 2024-09-13T13:56:44.387 INFO:teuthology.orchestra.run.smithi099.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [627 kB] 2024-09-13T13:56:44.393 INFO:teuthology.orchestra.run.smithi099.stdout:Get:13 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [902 kB] 2024-09-13T13:56:44.401 INFO:teuthology.orchestra.run.smithi099.stdout:Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [176 kB] 2024-09-13T13:56:44.403 INFO:teuthology.orchestra.run.smithi099.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [19.2 kB] 2024-09-13T13:56:44.403 INFO:teuthology.orchestra.run.smithi099.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-09-13T13:56:44.404 INFO:teuthology.orchestra.run.smithi099.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-09-13T13:56:44.404 INFO:teuthology.orchestra.run.smithi099.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-09-13T13:56:44.408 INFO:teuthology.orchestra.run.smithi099.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-09-13T13:56:44.536 INFO:teuthology.orchestra.run.smithi177.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-09-13T13:56:44.553 INFO:teuthology.orchestra.run.smithi099.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-09-13T13:56:44.557 INFO:teuthology.orchestra.run.smithi177.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [539 kB] 2024-09-13T13:56:44.676 INFO:teuthology.orchestra.run.smithi177.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,023 kB] 2024-09-13T13:56:44.692 INFO:teuthology.orchestra.run.smithi099.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [696 kB] 2024-09-13T13:56:44.865 INFO:teuthology.orchestra.run.smithi177.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,806 kB] 2024-09-13T13:56:44.887 INFO:teuthology.orchestra.run.smithi099.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,023 kB] 2024-09-13T13:56:44.995 INFO:teuthology.orchestra.run.smithi177.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [696 kB] 2024-09-13T13:56:45.039 INFO:teuthology.orchestra.run.smithi177.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [352 kB] 2024-09-13T13:56:45.041 INFO:teuthology.orchestra.run.smithi177.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [295 kB] 2024-09-13T13:56:45.050 INFO:teuthology.orchestra.run.smithi177.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.8 kB] 2024-09-13T13:56:45.051 INFO:teuthology.orchestra.run.smithi099.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [352 kB] 2024-09-13T13:56:45.051 INFO:teuthology.orchestra.run.smithi177.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [39.3 kB] 2024-09-13T13:56:45.052 INFO:teuthology.orchestra.run.smithi177.stdout:Get:13 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.3 kB] 2024-09-13T13:56:45.053 INFO:teuthology.orchestra.run.smithi177.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,437 kB] 2024-09-13T13:56:45.053 INFO:teuthology.orchestra.run.smithi177.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,377 kB] 2024-09-13T13:56:45.064 INFO:teuthology.orchestra.run.smithi099.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.8 kB] 2024-09-13T13:56:45.065 INFO:teuthology.orchestra.run.smithi099.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,437 kB] 2024-09-13T13:56:45.126 INFO:teuthology.orchestra.run.smithi177.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [419 kB] 2024-09-13T13:56:45.136 INFO:teuthology.orchestra.run.smithi099.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [39.3 kB] 2024-09-13T13:56:45.137 INFO:teuthology.orchestra.run.smithi099.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [419 kB] 2024-09-13T13:56:45.138 INFO:teuthology.orchestra.run.smithi177.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [616 B] 2024-09-13T13:56:45.138 INFO:teuthology.orchestra.run.smithi177.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [732 kB] 2024-09-13T13:56:45.140 INFO:teuthology.orchestra.run.smithi177.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.7 kB] 2024-09-13T13:56:45.151 INFO:teuthology.orchestra.run.smithi099.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [616 B] 2024-09-13T13:56:45.151 INFO:teuthology.orchestra.run.smithi099.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [732 kB] 2024-09-13T13:56:45.173 INFO:teuthology.orchestra.run.smithi099.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,124 kB] 2024-09-13T13:56:45.173 INFO:teuthology.orchestra.run.smithi177.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,124 kB] 2024-09-13T13:56:45.174 INFO:teuthology.orchestra.run.smithi177.stdout:Get:21 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [409 kB] 2024-09-13T13:56:45.182 INFO:teuthology.orchestra.run.smithi177.stdout:Get:22 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [584 B] 2024-09-13T13:56:45.183 INFO:teuthology.orchestra.run.smithi177.stdout:Get:23 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [627 kB] 2024-09-13T13:56:45.195 INFO:teuthology.orchestra.run.smithi177.stdout:Get:24 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [902 kB] 2024-09-13T13:56:45.197 INFO:teuthology.orchestra.run.smithi177.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [261 kB] 2024-09-13T13:56:45.202 INFO:teuthology.orchestra.run.smithi177.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [26.1 kB] 2024-09-13T13:56:45.203 INFO:teuthology.orchestra.run.smithi177.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-09-13T13:56:45.204 INFO:teuthology.orchestra.run.smithi177.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,752 B] 2024-09-13T13:56:45.205 INFO:teuthology.orchestra.run.smithi177.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-09-13T13:56:45.207 INFO:teuthology.orchestra.run.smithi099.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [261 kB] 2024-09-13T13:56:45.210 INFO:teuthology.orchestra.run.smithi177.stdout:Get:30 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [176 kB] 2024-09-13T13:56:45.212 INFO:teuthology.orchestra.run.smithi177.stdout:Get:31 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [19.2 kB] 2024-09-13T13:56:45.213 INFO:teuthology.orchestra.run.smithi177.stdout:Get:32 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-09-13T13:56:45.213 INFO:teuthology.orchestra.run.smithi177.stdout:Get:33 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-09-13T13:56:45.213 INFO:teuthology.orchestra.run.smithi099.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [26.1 kB] 2024-09-13T13:56:45.215 INFO:teuthology.orchestra.run.smithi099.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,752 B] 2024-09-13T13:56:45.216 INFO:teuthology.orchestra.run.smithi099.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-09-13T13:56:45.244 INFO:teuthology.orchestra.run.smithi177.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-09-13T13:56:45.244 INFO:teuthology.orchestra.run.smithi177.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.8 kB] 2024-09-13T13:56:45.245 INFO:teuthology.orchestra.run.smithi177.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.9 kB] 2024-09-13T13:56:45.287 INFO:teuthology.orchestra.run.smithi177.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.1 kB] 2024-09-13T13:56:45.288 INFO:teuthology.orchestra.run.smithi177.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-09-13T13:56:45.288 INFO:teuthology.orchestra.run.smithi177.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-09-13T13:56:45.288 INFO:teuthology.orchestra.run.smithi177.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.3 kB] 2024-09-13T13:56:45.289 INFO:teuthology.orchestra.run.smithi177.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-09-13T13:56:45.289 INFO:teuthology.orchestra.run.smithi177.stdout:Get:42 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-09-13T13:56:45.290 INFO:teuthology.orchestra.run.smithi177.stdout:Get:43 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-09-13T13:56:45.290 INFO:teuthology.orchestra.run.smithi099.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-09-13T13:56:45.290 INFO:teuthology.orchestra.run.smithi099.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-09-13T13:56:45.290 INFO:teuthology.orchestra.run.smithi099.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.8 kB] 2024-09-13T13:56:45.291 INFO:teuthology.orchestra.run.smithi099.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.9 kB] 2024-09-13T13:56:45.292 INFO:teuthology.orchestra.run.smithi099.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.1 kB] 2024-09-13T13:56:45.297 INFO:teuthology.orchestra.run.smithi099.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-09-13T13:56:45.297 INFO:teuthology.orchestra.run.smithi099.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.3 kB] 2024-09-13T13:56:45.298 INFO:teuthology.orchestra.run.smithi099.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-09-13T13:56:45.325 INFO:teuthology.orchestra.run.smithi177.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-09-13T13:56:45.378 INFO:teuthology.orchestra.run.smithi099.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-09-13T13:56:45.379 INFO:teuthology.orchestra.run.smithi099.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-09-13T13:56:50.638 INFO:teuthology.orchestra.run.smithi099.stdout:Fetched 16.0 MB in 2s (7,467 kB/s) 2024-09-13T13:56:51.282 INFO:teuthology.orchestra.run.smithi177.stdout:Fetched 16.0 MB in 3s (5,793 kB/s) 2024-09-13T13:56:52.115 INFO:teuthology.orchestra.run.smithi099.stdout:Reading package lists... 2024-09-13T13:56:52.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-13T13:56:52.185 INFO:teuthology.orchestra.run.smithi099.stdout:Reading package lists... 2024-09-13T13:56:52.369 INFO:teuthology.orchestra.run.smithi099.stdout:Building dependency tree... 2024-09-13T13:56:52.369 INFO:teuthology.orchestra.run.smithi099.stdout:Reading state information... 2024-09-13T13:56:52.537 INFO:teuthology.orchestra.run.smithi099.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T13:56:52.537 INFO:teuthology.orchestra.run.smithi099.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T13:56:52.538 INFO:teuthology.orchestra.run.smithi099.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T13:56:52.538 INFO:teuthology.orchestra.run.smithi099.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T13:56:52.538 INFO:teuthology.orchestra.run.smithi099.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-09-13T13:56:52.538 INFO:teuthology.orchestra.run.smithi099.stdout: python2.7-minimal 2024-09-13T13:56:52.538 INFO:teuthology.orchestra.run.smithi099.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T13:56:52.539 INFO:teuthology.orchestra.run.smithi099.stdout:The following additional packages will be installed: 2024-09-13T13:56:52.539 INFO:teuthology.orchestra.run.smithi099.stdout: linux-generic linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-13T13:56:52.539 INFO:teuthology.orchestra.run.smithi099.stdout: linux-headers-generic linux-image-5.15.0-121-generic 2024-09-13T13:56:52.540 INFO:teuthology.orchestra.run.smithi099.stdout: linux-modules-5.15.0-121-generic linux-modules-extra-5.15.0-121-generic 2024-09-13T13:56:52.541 INFO:teuthology.orchestra.run.smithi099.stdout:Suggested packages: 2024-09-13T13:56:52.541 INFO:teuthology.orchestra.run.smithi099.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-09-13T13:56:52.541 INFO:teuthology.orchestra.run.smithi099.stdout:Recommended packages: 2024-09-13T13:56:52.541 INFO:teuthology.orchestra.run.smithi099.stdout: thermald 2024-09-13T13:56:52.606 INFO:teuthology.orchestra.run.smithi099.stdout:The following NEW packages will be installed: 2024-09-13T13:56:52.607 INFO:teuthology.orchestra.run.smithi099.stdout: linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-13T13:56:52.607 INFO:teuthology.orchestra.run.smithi099.stdout: linux-image-5.15.0-121-generic linux-modules-5.15.0-121-generic 2024-09-13T13:56:52.607 INFO:teuthology.orchestra.run.smithi099.stdout: linux-modules-extra-5.15.0-121-generic 2024-09-13T13:56:52.608 INFO:teuthology.orchestra.run.smithi099.stdout:The following packages will be upgraded: 2024-09-13T13:56:52.608 INFO:teuthology.orchestra.run.smithi099.stdout: linux-generic linux-headers-generic linux-image-generic 2024-09-13T13:56:52.660 INFO:teuthology.orchestra.run.smithi099.stdout:3 upgraded, 5 newly installed, 0 to remove and 344 not upgraded. 2024-09-13T13:56:52.660 INFO:teuthology.orchestra.run.smithi099.stdout:Need to get 113 MB of archives. 2024-09-13T13:56:52.661 INFO:teuthology.orchestra.run.smithi099.stdout:After this operation, 584 MB of additional disk space will be used. 2024-09-13T13:56:52.661 INFO:teuthology.orchestra.run.smithi099.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-121-generic amd64 5.15.0-121.131 [22.7 MB] 2024-09-13T13:56:52.706 INFO:teuthology.orchestra.run.smithi177.stdout:Reading package lists... 2024-09-13T13:56:52.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-13T13:56:52.778 INFO:teuthology.orchestra.run.smithi177.stdout:Reading package lists... 2024-09-13T13:56:52.965 INFO:teuthology.orchestra.run.smithi177.stdout:Building dependency tree... 2024-09-13T13:56:52.965 INFO:teuthology.orchestra.run.smithi177.stdout:Reading state information... 2024-09-13T13:56:53.081 INFO:teuthology.orchestra.run.smithi099.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-121-generic amd64 5.15.0-121.131 [11.5 MB] 2024-09-13T13:56:53.133 INFO:teuthology.orchestra.run.smithi177.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T13:56:53.133 INFO:teuthology.orchestra.run.smithi177.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T13:56:53.133 INFO:teuthology.orchestra.run.smithi177.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T13:56:53.134 INFO:teuthology.orchestra.run.smithi177.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T13:56:53.134 INFO:teuthology.orchestra.run.smithi177.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-09-13T13:56:53.134 INFO:teuthology.orchestra.run.smithi177.stdout: python2.7-minimal 2024-09-13T13:56:53.134 INFO:teuthology.orchestra.run.smithi177.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T13:56:53.135 INFO:teuthology.orchestra.run.smithi177.stdout:The following additional packages will be installed: 2024-09-13T13:56:53.135 INFO:teuthology.orchestra.run.smithi177.stdout: linux-generic linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-13T13:56:53.135 INFO:teuthology.orchestra.run.smithi177.stdout: linux-headers-generic linux-image-5.15.0-121-generic 2024-09-13T13:56:53.136 INFO:teuthology.orchestra.run.smithi177.stdout: linux-modules-5.15.0-121-generic linux-modules-extra-5.15.0-121-generic 2024-09-13T13:56:53.137 INFO:teuthology.orchestra.run.smithi177.stdout:Suggested packages: 2024-09-13T13:56:53.137 INFO:teuthology.orchestra.run.smithi177.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-09-13T13:56:53.137 INFO:teuthology.orchestra.run.smithi177.stdout:Recommended packages: 2024-09-13T13:56:53.137 INFO:teuthology.orchestra.run.smithi177.stdout: thermald 2024-09-13T13:56:53.175 INFO:teuthology.orchestra.run.smithi099.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-121-generic amd64 5.15.0-121.131 [63.9 MB] 2024-09-13T13:56:53.176 INFO:teuthology.orchestra.run.smithi177.stdout:The following NEW packages will be installed: 2024-09-13T13:56:53.176 INFO:teuthology.orchestra.run.smithi177.stdout: linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-13T13:56:53.176 INFO:teuthology.orchestra.run.smithi177.stdout: linux-image-5.15.0-121-generic linux-modules-5.15.0-121-generic 2024-09-13T13:56:53.176 INFO:teuthology.orchestra.run.smithi177.stdout: linux-modules-extra-5.15.0-121-generic 2024-09-13T13:56:53.177 INFO:teuthology.orchestra.run.smithi177.stdout:The following packages will be upgraded: 2024-09-13T13:56:53.178 INFO:teuthology.orchestra.run.smithi177.stdout: linux-generic linux-headers-generic linux-image-generic 2024-09-13T13:56:53.358 INFO:teuthology.orchestra.run.smithi177.stdout:3 upgraded, 5 newly installed, 0 to remove and 344 not upgraded. 2024-09-13T13:56:53.358 INFO:teuthology.orchestra.run.smithi177.stdout:Need to get 113 MB of archives. 2024-09-13T13:56:53.358 INFO:teuthology.orchestra.run.smithi177.stdout:After this operation, 584 MB of additional disk space will be used. 2024-09-13T13:56:53.358 INFO:teuthology.orchestra.run.smithi177.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-121-generic amd64 5.15.0-121.131 [22.7 MB] 2024-09-13T13:56:53.979 INFO:teuthology.orchestra.run.smithi099.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.121.121 [1,694 B] 2024-09-13T13:56:53.979 INFO:teuthology.orchestra.run.smithi099.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.121.121 [2,464 B] 2024-09-13T13:56:53.979 INFO:teuthology.orchestra.run.smithi099.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121 all 5.15.0-121.131 [12.3 MB] 2024-09-13T13:56:54.108 INFO:teuthology.orchestra.run.smithi099.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121-generic amd64 5.15.0-121.131 [2,827 kB] 2024-09-13T13:56:54.129 INFO:teuthology.orchestra.run.smithi099.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.121.121 [2,316 B] 2024-09-13T13:56:54.512 INFO:teuthology.orchestra.run.smithi099.stdout:Fetched 113 MB in 2s (74.8 MB/s) 2024-09-13T13:56:54.692 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package linux-modules-5.15.0-121-generic. 2024-09-13T13:56:54.875 INFO:teuthology.orchestra.run.smithi177.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-121-generic amd64 5.15.0-121.131 [11.5 MB] 2024-09-13T13:56:55.224 INFO:teuthology.orchestra.run.smithi177.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-121-generic amd64 5.15.0-121.131 [63.9 MB] 2024-09-13T13:56:56.905 INFO:teuthology.orchestra.run.smithi099.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-09-13T13:56:56.910 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../0-linux-modules-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T13:56:57.122 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:56:57.452 INFO:teuthology.orchestra.run.smithi177.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.121.121 [1,694 B] 2024-09-13T13:56:57.452 INFO:teuthology.orchestra.run.smithi177.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.121.121 [2,464 B] 2024-09-13T13:56:57.452 INFO:teuthology.orchestra.run.smithi177.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121 all 5.15.0-121.131 [12.3 MB] 2024-09-13T13:56:57.790 INFO:teuthology.orchestra.run.smithi177.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121-generic amd64 5.15.0-121.131 [2,827 kB] 2024-09-13T13:56:57.869 INFO:teuthology.orchestra.run.smithi177.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.121.121 [2,316 B] 2024-09-13T13:56:58.265 INFO:teuthology.orchestra.run.smithi177.stdout:Fetched 113 MB in 5s (24.2 MB/s) 2024-09-13T13:56:58.448 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package linux-modules-5.15.0-121-generic. 2024-09-13T13:56:59.239 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package linux-image-5.15.0-121-generic. 2024-09-13T13:56:59.260 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../1-linux-image-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T13:56:59.369 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:56:59.876 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-121-generic. 2024-09-13T13:56:59.896 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T13:56:59.934 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:00.429 INFO:teuthology.orchestra.run.smithi177.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-09-13T13:57:00.436 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../0-linux-modules-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T13:57:00.520 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:02.595 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package linux-image-5.15.0-121-generic. 2024-09-13T13:57:02.617 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../1-linux-image-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T13:57:02.713 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:03.182 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-121-generic. 2024-09-13T13:57:03.202 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T13:57:03.240 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:06.572 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../3-linux-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T13:57:06.724 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking linux-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T13:57:07.015 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T13:57:07.111 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking linux-image-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T13:57:07.439 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package linux-headers-5.15.0-121. 2024-09-13T13:57:07.461 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../5-linux-headers-5.15.0-121_5.15.0-121.131_all.deb ... 2024-09-13T13:57:07.505 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-13T13:57:09.620 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../3-linux-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T13:57:09.748 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking linux-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T13:57:10.065 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T13:57:10.184 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking linux-image-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T13:57:10.487 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package linux-headers-5.15.0-121. 2024-09-13T13:57:10.509 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../5-linux-headers-5.15.0-121_5.15.0-121.131_all.deb ... 2024-09-13T13:57:10.553 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-13T13:57:13.549 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package linux-headers-5.15.0-121-generic. 2024-09-13T13:57:13.587 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../6-linux-headers-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T13:57:13.632 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:15.840 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package linux-headers-5.15.0-121-generic. 2024-09-13T13:57:15.878 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../6-linux-headers-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T13:57:15.915 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:16.353 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T13:57:16.475 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking linux-headers-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T13:57:16.808 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-13T13:57:16.955 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:17.081 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up linux-headers-generic (5.15.0.121.121) ... 2024-09-13T13:57:17.200 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:18.554 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T13:57:18.561 INFO:teuthology.orchestra.run.smithi099.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-09-13T13:57:18.561 INFO:teuthology.orchestra.run.smithi099.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-09-13T13:57:18.561 INFO:teuthology.orchestra.run.smithi099.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-121-generic 2024-09-13T13:57:18.561 INFO:teuthology.orchestra.run.smithi099.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-121-generic 2024-09-13T13:57:18.677 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking linux-headers-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T13:57:18.719 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:19.028 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-13T13:57:19.149 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:19.275 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up linux-headers-generic (5.15.0.121.121) ... 2024-09-13T13:57:20.120 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:20.870 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:21.669 INFO:teuthology.orchestra.run.smithi177.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-09-13T13:57:21.669 INFO:teuthology.orchestra.run.smithi177.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-09-13T13:57:21.669 INFO:teuthology.orchestra.run.smithi177.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-121-generic 2024-09-13T13:57:21.669 INFO:teuthology.orchestra.run.smithi177.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-121-generic 2024-09-13T13:57:21.832 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:22.288 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up linux-image-generic (5.15.0.121.121) ... 2024-09-13T13:57:22.414 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up linux-generic (5.15.0.121.121) ... 2024-09-13T13:57:22.565 INFO:teuthology.orchestra.run.smithi099.stdout:Processing triggers for linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:22.640 INFO:teuthology.orchestra.run.smithi099.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-09-13T13:57:22.641 INFO:teuthology.orchestra.run.smithi099.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-121-generic 2024-09-13T13:57:23.249 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:24.642 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up linux-image-generic (5.15.0.121.121) ... 2024-09-13T13:57:24.768 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up linux-generic (5.15.0.121.121) ... 2024-09-13T13:57:24.886 INFO:teuthology.orchestra.run.smithi177.stdout:Processing triggers for linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T13:57:24.946 INFO:teuthology.orchestra.run.smithi177.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-09-13T13:57:24.946 INFO:teuthology.orchestra.run.smithi177.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-121-generic 2024-09-13T13:57:37.245 INFO:teuthology.orchestra.run.smithi099.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-09-13T13:57:37.245 INFO:teuthology.orchestra.run.smithi099.stdout:Sourcing file `/etc/default/grub' 2024-09-13T13:57:37.267 INFO:teuthology.orchestra.run.smithi099.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-13T13:57:37.284 INFO:teuthology.orchestra.run.smithi099.stdout:Generating grub configuration file ... 2024-09-13T13:57:37.567 INFO:teuthology.orchestra.run.smithi099.stdout:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-13T13:57:37.584 INFO:teuthology.orchestra.run.smithi099.stdout:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-13T13:57:37.915 INFO:teuthology.orchestra.run.smithi099.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-13T13:57:37.921 INFO:teuthology.orchestra.run.smithi099.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-13T13:57:37.992 INFO:teuthology.orchestra.run.smithi099.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-13T13:57:37.998 INFO:teuthology.orchestra.run.smithi099.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-13T13:57:38.157 INFO:teuthology.orchestra.run.smithi099.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-13T13:57:38.158 INFO:teuthology.orchestra.run.smithi099.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-09-13T13:57:38.158 INFO:teuthology.orchestra.run.smithi099.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-13T13:57:38.189 INFO:teuthology.orchestra.run.smithi099.stdout:done 2024-09-13T13:57:38.630 DEBUG:teuthology.orchestra.run.smithi099:> dpkg -s linux-image-generic 2024-09-13T13:57:38.653 INFO:teuthology.orchestra.run.smithi099.stdout:Package: linux-image-generic 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Status: install ok installed 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Priority: optional 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Section: kernel 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Installed-Size: 21 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Maintainer: Ubuntu Kernel Team 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Architecture: amd64 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Source: linux-meta 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Version: 5.15.0.121.121 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Recommends: thermald 2024-09-13T13:57:38.654 INFO:teuthology.orchestra.run.smithi099.stdout:Description: Generic Linux kernel image 2024-09-13T13:57:38.655 INFO:teuthology.orchestra.run.smithi099.stdout: This package will always depend on the latest generic kernel image 2024-09-13T13:57:38.655 INFO:teuthology.orchestra.run.smithi099.stdout: available. 2024-09-13T13:57:38.655 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-121-generic 2024-09-13T13:57:38.655 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T13:57:38.655 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T13:57:38.655 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-13T13:57:38.655 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi099.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T13:57:38.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo apt-get clean 2024-09-13T13:57:38.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo apt-get update 2024-09-13T13:57:38.970 INFO:teuthology.orchestra.run.smithi177.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-09-13T13:57:38.970 INFO:teuthology.orchestra.run.smithi177.stdout:Sourcing file `/etc/default/grub' 2024-09-13T13:57:38.987 INFO:teuthology.orchestra.run.smithi177.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-13T13:57:39.000 INFO:teuthology.orchestra.run.smithi177.stdout:Generating grub configuration file ... 2024-09-13T13:57:39.088 INFO:teuthology.orchestra.run.smithi099.stdout:Hit:1 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-13T13:57:39.088 INFO:teuthology.orchestra.run.smithi099.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T13:57:39.160 INFO:teuthology.orchestra.run.smithi099.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-13T13:57:39.242 INFO:teuthology.orchestra.run.smithi099.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-13T13:57:39.272 INFO:teuthology.orchestra.run.smithi177.stdout:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-13T13:57:39.293 INFO:teuthology.orchestra.run.smithi177.stdout:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-13T13:57:39.567 INFO:teuthology.orchestra.run.smithi177.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-13T13:57:39.573 INFO:teuthology.orchestra.run.smithi177.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-13T13:57:39.654 INFO:teuthology.orchestra.run.smithi177.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-13T13:57:39.660 INFO:teuthology.orchestra.run.smithi177.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-13T13:57:39.830 INFO:teuthology.orchestra.run.smithi177.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-13T13:57:39.830 INFO:teuthology.orchestra.run.smithi177.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-09-13T13:57:39.830 INFO:teuthology.orchestra.run.smithi177.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-13T13:57:39.865 INFO:teuthology.orchestra.run.smithi177.stdout:done 2024-09-13T13:57:40.364 DEBUG:teuthology.orchestra.run.smithi177:> dpkg -s linux-image-generic 2024-09-13T13:57:40.387 INFO:teuthology.orchestra.run.smithi177.stdout:Package: linux-image-generic 2024-09-13T13:57:40.387 INFO:teuthology.orchestra.run.smithi177.stdout:Status: install ok installed 2024-09-13T13:57:40.387 INFO:teuthology.orchestra.run.smithi177.stdout:Priority: optional 2024-09-13T13:57:40.387 INFO:teuthology.orchestra.run.smithi177.stdout:Section: kernel 2024-09-13T13:57:40.387 INFO:teuthology.orchestra.run.smithi177.stdout:Installed-Size: 21 2024-09-13T13:57:40.387 INFO:teuthology.orchestra.run.smithi177.stdout:Maintainer: Ubuntu Kernel Team 2024-09-13T13:57:40.387 INFO:teuthology.orchestra.run.smithi177.stdout:Architecture: amd64 2024-09-13T13:57:40.387 INFO:teuthology.orchestra.run.smithi177.stdout:Source: linux-meta 2024-09-13T13:57:40.387 INFO:teuthology.orchestra.run.smithi177.stdout:Version: 5.15.0.121.121 2024-09-13T13:57:40.388 INFO:teuthology.orchestra.run.smithi177.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-13T13:57:40.388 INFO:teuthology.orchestra.run.smithi177.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-13T13:57:40.388 INFO:teuthology.orchestra.run.smithi177.stdout:Recommends: thermald 2024-09-13T13:57:40.388 INFO:teuthology.orchestra.run.smithi177.stdout:Description: Generic Linux kernel image 2024-09-13T13:57:40.388 INFO:teuthology.orchestra.run.smithi177.stdout: This package will always depend on the latest generic kernel image 2024-09-13T13:57:40.388 INFO:teuthology.orchestra.run.smithi177.stdout: available. 2024-09-13T13:57:40.389 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-121-generic 2024-09-13T13:57:40.389 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T13:57:40.389 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T13:57:40.389 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-13T13:57:40.389 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi177.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T13:57:40.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo apt-get clean 2024-09-13T13:57:40.498 INFO:teuthology.orchestra.run.smithi099.stdout:Reading package lists... 2024-09-13T13:57:40.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-13T13:57:40.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo apt-get update 2024-09-13T13:57:40.582 INFO:teuthology.orchestra.run.smithi099.stdout:Reading package lists... 2024-09-13T13:57:40.677 INFO:teuthology.orchestra.run.smithi177.stdout:Hit:1 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-13T13:57:40.791 INFO:teuthology.orchestra.run.smithi177.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T13:57:40.792 INFO:teuthology.orchestra.run.smithi099.stdout:Building dependency tree... 2024-09-13T13:57:40.792 INFO:teuthology.orchestra.run.smithi099.stdout:Reading state information... 2024-09-13T13:57:40.872 INFO:teuthology.orchestra.run.smithi177.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-13T13:57:40.953 INFO:teuthology.orchestra.run.smithi177.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-13T13:57:40.975 INFO:teuthology.orchestra.run.smithi099.stdout:linux-image-generic is already the newest version (5.15.0.121.121). 2024-09-13T13:57:40.975 INFO:teuthology.orchestra.run.smithi099.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T13:57:40.975 INFO:teuthology.orchestra.run.smithi099.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T13:57:40.975 INFO:teuthology.orchestra.run.smithi099.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T13:57:40.975 INFO:teuthology.orchestra.run.smithi099.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T13:57:40.976 INFO:teuthology.orchestra.run.smithi099.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-13T13:57:40.976 INFO:teuthology.orchestra.run.smithi099.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-13T13:57:40.976 INFO:teuthology.orchestra.run.smithi099.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T13:57:41.014 INFO:teuthology.orchestra.run.smithi099.stdout:0 upgraded, 0 newly installed, 0 to remove and 344 not upgraded. 2024-09-13T13:57:41.016 DEBUG:teuthology.orchestra.run.smithi099:> dpkg -s linux-image-generic 2024-09-13T13:57:41.031 INFO:teuthology.orchestra.run.smithi099.stdout:Package: linux-image-generic 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Status: install ok installed 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Priority: optional 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Section: kernel 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Installed-Size: 21 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Maintainer: Ubuntu Kernel Team 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Architecture: amd64 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Source: linux-meta 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Version: 5.15.0.121.121 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-13T13:57:41.032 INFO:teuthology.orchestra.run.smithi099.stdout:Recommends: thermald 2024-09-13T13:57:41.033 INFO:teuthology.orchestra.run.smithi099.stdout:Description: Generic Linux kernel image 2024-09-13T13:57:41.033 INFO:teuthology.orchestra.run.smithi099.stdout: This package will always depend on the latest generic kernel image 2024-09-13T13:57:41.033 INFO:teuthology.orchestra.run.smithi099.stdout: available. 2024-09-13T13:57:41.033 DEBUG:teuthology.orchestra.run.smithi099:> mktemp 2024-09-13T13:57:41.077 INFO:teuthology.orchestra.run.smithi099.stdout:/tmp/tmp.9N2kBYkaDH 2024-09-13T13:57:41.078 DEBUG:teuthology.orchestra.run.smithi099:> sudo cp /boot/grub/grub.cfg /tmp/tmp.9N2kBYkaDH 2024-09-13T13:57:41.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 0666 /tmp/tmp.9N2kBYkaDH 2024-09-13T13:57:41.248 DEBUG:teuthology.orchestra.remote:smithi099:/tmp/tmp.9N2kBYkaDH is 10KB 2024-09-13T13:57:41.261 DEBUG:teuthology.orchestra.run.smithi099:> rm -fr /tmp/tmp.9N2kBYkaDH 2024-09-13T13:57:41.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-09-13T13:57:41.321 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T13:57:41.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-09-13T13:57:41.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-09-13T13:57:41.391 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-121-generic 2024-09-13T13:57:41.391 DEBUG:teuthology.orchestra.run.smithi099:> sudo update-grub 2024-09-13T13:57:42.097 INFO:teuthology.orchestra.run.smithi177.stdout:Reading package lists... 2024-09-13T13:57:42.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-13T13:57:42.180 INFO:teuthology.orchestra.run.smithi177.stdout:Reading package lists... 2024-09-13T13:57:42.402 INFO:teuthology.orchestra.run.smithi177.stdout:Building dependency tree... 2024-09-13T13:57:42.403 INFO:teuthology.orchestra.run.smithi177.stdout:Reading state information... 2024-09-13T13:57:42.592 INFO:teuthology.orchestra.run.smithi177.stdout:linux-image-generic is already the newest version (5.15.0.121.121). 2024-09-13T13:57:42.592 INFO:teuthology.orchestra.run.smithi177.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T13:57:42.593 INFO:teuthology.orchestra.run.smithi177.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T13:57:42.593 INFO:teuthology.orchestra.run.smithi177.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T13:57:42.593 INFO:teuthology.orchestra.run.smithi177.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T13:57:42.593 INFO:teuthology.orchestra.run.smithi177.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-13T13:57:42.593 INFO:teuthology.orchestra.run.smithi177.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-13T13:57:42.593 INFO:teuthology.orchestra.run.smithi177.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T13:57:42.637 INFO:teuthology.orchestra.run.smithi177.stdout:0 upgraded, 0 newly installed, 0 to remove and 344 not upgraded. 2024-09-13T13:57:42.639 DEBUG:teuthology.orchestra.run.smithi177:> dpkg -s linux-image-generic 2024-09-13T13:57:42.642 INFO:teuthology.orchestra.run.smithi099.stderr:Sourcing file `/etc/default/grub' 2024-09-13T13:57:42.643 INFO:teuthology.orchestra.run.smithi099.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-13T13:57:42.646 INFO:teuthology.orchestra.run.smithi099.stderr:Generating grub configuration file ... 2024-09-13T13:57:42.655 INFO:teuthology.orchestra.run.smithi177.stdout:Package: linux-image-generic 2024-09-13T13:57:42.655 INFO:teuthology.orchestra.run.smithi177.stdout:Status: install ok installed 2024-09-13T13:57:42.655 INFO:teuthology.orchestra.run.smithi177.stdout:Priority: optional 2024-09-13T13:57:42.655 INFO:teuthology.orchestra.run.smithi177.stdout:Section: kernel 2024-09-13T13:57:42.655 INFO:teuthology.orchestra.run.smithi177.stdout:Installed-Size: 21 2024-09-13T13:57:42.655 INFO:teuthology.orchestra.run.smithi177.stdout:Maintainer: Ubuntu Kernel Team 2024-09-13T13:57:42.655 INFO:teuthology.orchestra.run.smithi177.stdout:Architecture: amd64 2024-09-13T13:57:42.656 INFO:teuthology.orchestra.run.smithi177.stdout:Source: linux-meta 2024-09-13T13:57:42.656 INFO:teuthology.orchestra.run.smithi177.stdout:Version: 5.15.0.121.121 2024-09-13T13:57:42.656 INFO:teuthology.orchestra.run.smithi177.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-13T13:57:42.656 INFO:teuthology.orchestra.run.smithi177.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-13T13:57:42.656 INFO:teuthology.orchestra.run.smithi177.stdout:Recommends: thermald 2024-09-13T13:57:42.656 INFO:teuthology.orchestra.run.smithi177.stdout:Description: Generic Linux kernel image 2024-09-13T13:57:42.656 INFO:teuthology.orchestra.run.smithi177.stdout: This package will always depend on the latest generic kernel image 2024-09-13T13:57:42.656 INFO:teuthology.orchestra.run.smithi177.stdout: available. 2024-09-13T13:57:42.656 DEBUG:teuthology.orchestra.run.smithi177:> mktemp 2024-09-13T13:57:42.703 INFO:teuthology.orchestra.run.smithi177.stdout:/tmp/tmp.IzvfhfP2Q8 2024-09-13T13:57:42.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo cp /boot/grub/grub.cfg /tmp/tmp.IzvfhfP2Q8 2024-09-13T13:57:42.760 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod 0666 /tmp/tmp.IzvfhfP2Q8 2024-09-13T13:57:42.872 DEBUG:teuthology.orchestra.remote:smithi177:/tmp/tmp.IzvfhfP2Q8 is 10KB 2024-09-13T13:57:42.887 DEBUG:teuthology.orchestra.run.smithi177:> rm -fr /tmp/tmp.IzvfhfP2Q8 2024-09-13T13:57:42.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-09-13T13:57:42.903 INFO:teuthology.orchestra.run.smithi099.stderr:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-13T13:57:42.909 INFO:teuthology.orchestra.run.smithi099.stderr:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-13T13:57:42.947 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T13:57:42.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-09-13T13:57:42.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-09-13T13:57:43.014 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-121-generic 2024-09-13T13:57:43.014 DEBUG:teuthology.orchestra.run.smithi177:> sudo update-grub 2024-09-13T13:57:43.155 INFO:teuthology.orchestra.run.smithi099.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-13T13:57:43.160 INFO:teuthology.orchestra.run.smithi099.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-13T13:57:43.204 INFO:teuthology.orchestra.run.smithi099.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-13T13:57:43.210 INFO:teuthology.orchestra.run.smithi099.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-13T13:57:43.359 INFO:teuthology.orchestra.run.smithi099.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-13T13:57:43.359 INFO:teuthology.orchestra.run.smithi099.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-09-13T13:57:43.359 INFO:teuthology.orchestra.run.smithi099.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-13T13:57:43.370 INFO:teuthology.orchestra.run.smithi099.stderr:done 2024-09-13T13:57:43.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo shutdown -r now 2024-09-13T13:57:44.234 INFO:teuthology.orchestra.run.smithi177.stderr:Sourcing file `/etc/default/grub' 2024-09-13T13:57:44.235 INFO:teuthology.orchestra.run.smithi177.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-13T13:57:44.237 INFO:teuthology.orchestra.run.smithi177.stderr:Generating grub configuration file ... 2024-09-13T13:57:44.432 INFO:teuthology.orchestra.run.smithi177.stderr:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-13T13:57:44.438 INFO:teuthology.orchestra.run.smithi177.stderr:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-13T13:57:44.670 INFO:teuthology.orchestra.run.smithi177.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-13T13:57:44.676 INFO:teuthology.orchestra.run.smithi177.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-13T13:57:44.716 INFO:teuthology.orchestra.run.smithi177.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-13T13:57:44.721 INFO:teuthology.orchestra.run.smithi177.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-13T13:57:44.852 INFO:teuthology.orchestra.run.smithi177.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-13T13:57:44.852 INFO:teuthology.orchestra.run.smithi177.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-09-13T13:57:44.852 INFO:teuthology.orchestra.run.smithi177.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-13T13:57:44.863 INFO:teuthology.orchestra.run.smithi177.stderr:done 2024-09-13T13:57:44.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo shutdown -r now 2024-09-13T13:58:13.409 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-13T13:58:13.409 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-09-13T13:58:13.410 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:58:14.869 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-13T13:58:14.869 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-09-13T13:58:14.870 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:58:31.741 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.99 2024-09-13T13:58:33.277 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.177 2024-09-13T13:58:40.749 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-09-13T13:58:40.749 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:58:42.279 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-09-13T13:58:42.279 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:58:43.841 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.99 2024-09-13T13:58:45.341 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.177 2024-09-13T13:58:55.846 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-09-13T13:58:55.847 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:58:57.342 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-09-13T13:58:57.343 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:59:15.773 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.177 2024-09-13T13:59:27.633 DEBUG:teuthology.orchestra.run.smithi099:> true 2024-09-13T13:59:28.251 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-09-13T13:59:28.251 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-121-generic"... 2024-09-13T13:59:28.252 DEBUG:teuthology.orchestra.run.smithi099:> uname -r 2024-09-13T13:59:28.298 INFO:teuthology.orchestra.run.smithi099.stdout:5.15.0-121-generic 2024-09-13T13:59:28.298 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-121-generic vs 5.15.0-121-generic 2024-09-13T13:59:28.298 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T13:59:28.298 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-09-13T13:59:29.299 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-13T13:59:29.299 DEBUG:teuthology.orchestra.run.smithi099:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T13:59:29.370 INFO:teuthology.orchestra.run.smithi099.stdout:ttyS1 2024-09-13T13:59:29.395 DEBUG:teuthology.parallel:result is None 2024-09-13T13:59:30.774 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-09-13T13:59:30.774 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T13:59:31.092 DEBUG:teuthology.orchestra.run.smithi177:> true 2024-09-13T13:59:31.731 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-09-13T13:59:31.731 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-121-generic"... 2024-09-13T13:59:31.731 DEBUG:teuthology.orchestra.run.smithi177:> uname -r 2024-09-13T13:59:31.777 INFO:teuthology.orchestra.run.smithi177.stdout:5.15.0-121-generic 2024-09-13T13:59:31.777 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-121-generic vs 5.15.0-121-generic 2024-09-13T13:59:31.777 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T13:59:31.777 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-09-13T13:59:32.778 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-13T13:59:32.778 DEBUG:teuthology.orchestra.run.smithi177:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T13:59:32.849 INFO:teuthology.orchestra.run.smithi177.stdout:ttyS1 2024-09-13T13:59:32.875 DEBUG:teuthology.parallel:result is None 2024-09-13T13:59:32.875 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T13:59:32.883 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T13:59:32.883 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T13:59:32.887 DEBUG:teuthology.orchestra.run.smithi177:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T13:59:32.892 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T13:59:32.900 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T13:59:32.914 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T13:59:32.915 DEBUG:teuthology.orchestra.run.smithi099:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T13:59:32.934 DEBUG:teuthology.orchestra.run.smithi177:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T13:59:33.008 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T13:59:33.016 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T13:59:33.016 DEBUG:teuthology.orchestra.run.smithi099:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T13:59:33.019 DEBUG:teuthology.orchestra.run.smithi177:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T13:59:33.041 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T13:59:33.042 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T13:59:33.048 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T13:59:33.050 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T13:59:33.051 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T13:59:33.058 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T13:59:33.058 DEBUG:teuthology.orchestra.run.smithi099:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T13:59:33.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T13:59:33.109 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T13:59:33.119 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T13:59:33.120 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T13:59:33.151 DEBUG:teuthology.orchestra.run.smithi177:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T13:59:33.157 DEBUG:teuthology.orchestra.run.smithi099:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T13:59:33.203 DEBUG:teuthology.orchestra.run.smithi099:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T13:59:33.249 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T13:59:33.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T13:59:33.317 DEBUG:teuthology.orchestra.run.smithi177:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T13:59:33.323 DEBUG:teuthology.orchestra.run.smithi177:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T13:59:33.370 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T13:59:33.370 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T13:59:33.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo service rsyslog restart 2024-09-13T13:59:33.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo service rsyslog restart 2024-09-13T13:59:33.514 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T13:59:33.522 INFO:teuthology.task.internal:Starting timer... 2024-09-13T13:59:33.523 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T13:59:33.535 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T13:59:33.545 DEBUG:teuthology.task.selinux:Excluding smithi099: OS 'ubuntu' does not support SELinux 2024-09-13T13:59:33.545 DEBUG:teuthology.task.selinux:Excluding smithi177: OS 'ubuntu' does not support SELinux 2024-09-13T13:59:33.545 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T13:59:33.545 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2024-09-13T13:59:33.545 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T13:59:33.546 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T13:59:33.557 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-13T13:59:33.561 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-13T13:59:33.716 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T13:59:33.727 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T13:59:33.728 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi099.front.sepia.ceph.com,smithi177.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T14:07:01.303 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi099.front.sepia.ceph.com'), Remote(name='ubuntu@smithi177.front.sepia.ceph.com')] 2024-09-13T14:07:01.304 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-09-13T14:07:01.305 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:07:01.399 DEBUG:teuthology.orchestra.run.smithi099:> true 2024-09-13T14:07:01.480 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-09-13T14:07:01.481 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-09-13T14:07:01.481 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:07:01.585 DEBUG:teuthology.orchestra.run.smithi177:> true 2024-09-13T14:07:01.664 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-09-13T14:07:01.665 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T14:07:01.677 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T14:07:01.677 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T14:07:01.678 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:07:01.680 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T14:07:01.680 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:07:01.708 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-09-13T14:07:01.708 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: Command line: ntpd -gq 2024-09-13T14:07:01.709 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: ---------------------------------------------------- 2024-09-13T14:07:01.709 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: ntp-4 is maintained by Network Time Foundation, 2024-09-13T14:07:01.709 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-09-13T14:07:01.709 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: corporation. Support and training for ntp-4 are 2024-09-13T14:07:01.709 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: available at https://www.nwtime.org/support 2024-09-13T14:07:01.709 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: ---------------------------------------------------- 2024-09-13T14:07:01.710 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: proto: precision = 0.044 usec (-24) 2024-09-13T14:07:01.711 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: basedate set to 2022-02-04 2024-09-13T14:07:01.711 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: gps base set to 2022-02-06 (week 2196) 2024-09-13T14:07:01.711 INFO:teuthology.orchestra.run.smithi099.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-13T14:07:01.711 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-13T14:07:01.711 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: restrict ::: KOD does nothing without LIMITED. 2024-09-13T14:07:01.712 INFO:teuthology.orchestra.run.smithi099.stderr:restrict ::: KOD does nothing without LIMITED. 2024-09-13T14:07:01.712 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: Listen and drop on 0 v6wildcard [::]:123 2024-09-13T14:07:01.713 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-09-13T14:07:01.713 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: Listen normally on 2 lo 127.0.0.1:123 2024-09-13T14:07:01.713 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: Listen normally on 3 enp3s0f1 172.21.15.99:123 2024-09-13T14:07:01.713 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: Listen normally on 4 lo [::1]:123 2024-09-13T14:07:01.713 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d803%5]:123 2024-09-13T14:07:01.713 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:01 ntpd[18521]: Listening on routing socket on fd #22 for interface updates 2024-09-13T14:07:01.733 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-09-13T14:07:01.733 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: Command line: ntpd -gq 2024-09-13T14:07:01.733 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: ---------------------------------------------------- 2024-09-13T14:07:01.733 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: ntp-4 is maintained by Network Time Foundation, 2024-09-13T14:07:01.733 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-09-13T14:07:01.733 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: corporation. Support and training for ntp-4 are 2024-09-13T14:07:01.733 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: available at https://www.nwtime.org/support 2024-09-13T14:07:01.733 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: ---------------------------------------------------- 2024-09-13T14:07:01.734 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: proto: precision = 0.060 usec (-24) 2024-09-13T14:07:01.734 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: basedate set to 2022-02-04 2024-09-13T14:07:01.734 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: gps base set to 2022-02-06 (week 2196) 2024-09-13T14:07:01.734 INFO:teuthology.orchestra.run.smithi177.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-13T14:07:01.735 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-13T14:07:01.735 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: restrict ::: KOD does nothing without LIMITED. 2024-09-13T14:07:01.735 INFO:teuthology.orchestra.run.smithi177.stderr:restrict ::: KOD does nothing without LIMITED. 2024-09-13T14:07:01.736 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: Listen and drop on 0 v6wildcard [::]:123 2024-09-13T14:07:01.736 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-09-13T14:07:01.736 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: Listen normally on 2 lo 127.0.0.1:123 2024-09-13T14:07:01.736 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: Listen normally on 3 enp3s0f1 172.21.15.177:123 2024-09-13T14:07:01.736 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: Listen normally on 4 lo [::1]:123 2024-09-13T14:07:01.736 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:feea:b04b%5]:123 2024-09-13T14:07:01.736 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:01 ntpd[18506]: Listening on routing socket on fd #22 for interface updates 2024-09-13T14:07:02.709 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:02.709 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:02.709 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:02.709 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:02.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:02.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:02.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:02.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:02.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:02.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:02.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:02.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:02 ntpd[18521]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:02.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:02 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:02.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:02 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:02.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:02 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:02.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:02 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:02.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:02 ntpd[18506]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:02.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:02 ntpd[18506]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:03.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:03 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:03.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:03 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:03.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:03 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:03.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:03 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:03.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:03 ntpd[18506]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:03.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:03 ntpd[18506]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:04.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:04.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:04.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:04.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:04.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:04.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:04.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:04.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:04.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:04.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:04.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:04.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:04 ntpd[18521]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:04.733 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:04 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:04.733 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:04 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:04.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:04 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:04.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:04 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:04.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:04 ntpd[18506]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:04.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:04 ntpd[18506]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:05.709 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:05 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:05.709 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:05 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:05.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:05 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:05.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:05 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:05.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:05 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:05.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:05 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:05.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:05 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:05.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:05 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:05.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:05 ntpd[18506]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:05.736 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:05 ntpd[18506]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:06.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:06.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:06.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:06.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:06.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:06.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:06.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:06.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:06.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:06.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:06.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:06.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:06.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:06.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:06.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:06.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:06.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:06.712 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:06 ntpd[18521]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:06.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:06.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:06.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:06.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:06.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:06.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:06.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:06.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:06.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:06.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:06.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:06.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:06 ntpd[18506]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:07.733 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:07 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:07.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:07 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:07.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:07 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:07.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:07 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:07.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:07 ntpd[18506]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:07.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:07 ntpd[18506]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:08.709 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:08.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:08.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:08.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:08.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:08.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:08.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:08.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:08.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:08.710 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:08.711 INFO:teuthology.orchestra.run.smithi099.stdout:13 Sep 14:07:08 ntpd[18521]: ntpd: time slew -0.001969 s 2024-09-13T14:07:08.711 INFO:teuthology.orchestra.run.smithi099.stdout:ntpd: time slew -0.001969s 2024-09-13T14:07:08.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-09-13T14:07:08.711 INFO:teuthology.orchestra.run.smithi099.stderr:13 Sep 14:07:08 ntpd[18521]: can't open /var/log/ntpstats/loopstats.20240913: Permission denied 2024-09-13T14:07:08.734 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T14:07:08.735 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T14:07:08.736 INFO:teuthology.orchestra.run.smithi177.stdout:13 Sep 14:07:08 ntpd[18506]: ntpd: time slew -0.001048 s 2024-09-13T14:07:08.736 INFO:teuthology.orchestra.run.smithi177.stdout:ntpd: time slew -0.001048s 2024-09-13T14:07:08.737 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-09-13T14:07:08.737 INFO:teuthology.orchestra.run.smithi177.stderr:13 Sep 14:07:08 ntpd[18506]: can't open /var/log/ntpstats/loopstats.20240913: Permission denied 2024-09-13T14:07:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: remote refid st t when poll reach delay offset jitter 2024-09-13T14:07:08.753 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================== 2024-09-13T14:07:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T14:07:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T14:07:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T14:07:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T14:07:08.781 INFO:teuthology.orchestra.run.smithi177.stdout: remote refid st t when poll reach delay offset jitter 2024-09-13T14:07:08.781 INFO:teuthology.orchestra.run.smithi177.stdout:============================================================================== 2024-09-13T14:07:08.782 INFO:teuthology.orchestra.run.smithi177.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T14:07:08.782 INFO:teuthology.orchestra.run.smithi177.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T14:07:08.782 INFO:teuthology.orchestra.run.smithi177.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T14:07:08.782 INFO:teuthology.orchestra.run.smithi177.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T14:07:08.782 INFO:teuthology.run_tasks:Running task install... 2024-09-13T14:07:08.791 DEBUG:teuthology.task.install:project ceph 2024-09-13T14:07:08.791 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}} 2024-09-13T14:07:08.792 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'} 2024-09-13T14:07:08.792 INFO:teuthology.task.install:Using flavor: default 2024-09-13T14:07:08.797 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-13T14:07:08.797 INFO:teuthology.task.install:extra packages: [] 2024-09-13T14:07:08.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo apt-key list | grep Ceph 2024-09-13T14:07:08.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo apt-key list | grep Ceph 2024-09-13T14:07:08.860 INFO:teuthology.orchestra.run.smithi099.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-09-13T14:07:08.879 INFO:teuthology.orchestra.run.smithi177.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-09-13T14:07:08.889 INFO:teuthology.orchestra.run.smithi099.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-09-13T14:07:08.889 INFO:teuthology.orchestra.run.smithi099.stdout:uid [ unknown] Ceph.com (release key) 2024-09-13T14:07:08.891 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-09-13T14:07:08.891 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:07:08.891 INFO:teuthology.packaging:ref: None 2024-09-13T14:07:08.891 INFO:teuthology.packaging:tag: None 2024-09-13T14:07:08.891 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:07:08.891 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:07:08.891 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-13T14:07:08.905 INFO:teuthology.orchestra.run.smithi177.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-09-13T14:07:08.905 INFO:teuthology.orchestra.run.smithi177.stdout:uid [ unknown] Ceph.com (release key) 2024-09-13T14:07:08.905 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-09-13T14:07:08.905 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:07:08.905 INFO:teuthology.packaging:ref: None 2024-09-13T14:07:08.905 INFO:teuthology.packaging:tag: None 2024-09-13T14:07:08.905 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:07:08.905 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:07:08.906 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-13T14:07:09.043 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default/ 2024-09-13T14:07:09.043 INFO:teuthology.task.install.deb:Package version is 17.2.7-1jammy 2024-09-13T14:07:09.059 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default/ 2024-09-13T14:07:09.059 INFO:teuthology.task.install.deb:Package version is 17.2.7-1jammy 2024-09-13T14:07:09.111 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:07:09.111 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-09-13T14:07:09.113 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:07:09.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-09-13T14:07:09.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo apt-get update 2024-09-13T14:07:09.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo apt-get update 2024-09-13T14:07:09.268 INFO:teuthology.orchestra.run.smithi099.stdout:Ign:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy InRelease 2024-09-13T14:07:09.295 INFO:teuthology.orchestra.run.smithi099.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release [7,641 B] 2024-09-13T14:07:09.295 INFO:teuthology.orchestra.run.smithi177.stdout:Ign:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy InRelease 2024-09-13T14:07:09.296 INFO:teuthology.orchestra.run.smithi177.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release [7,641 B] 2024-09-13T14:07:09.296 INFO:teuthology.orchestra.run.smithi099.stdout:Ign:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release.gpg 2024-09-13T14:07:09.315 INFO:teuthology.orchestra.run.smithi177.stdout:Hit:3 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-13T14:07:09.326 INFO:teuthology.orchestra.run.smithi099.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,331 B] 2024-09-13T14:07:09.326 INFO:teuthology.orchestra.run.smithi177.stdout:Ign:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release.gpg 2024-09-13T14:07:09.327 INFO:teuthology.orchestra.run.smithi177.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,331 B] 2024-09-13T14:07:09.416 INFO:teuthology.orchestra.run.smithi177.stdout:Get:6 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.3 kB] 2024-09-13T14:07:09.416 INFO:teuthology.orchestra.run.smithi099.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.3 kB] 2024-09-13T14:07:09.416 INFO:teuthology.orchestra.run.smithi177.stdout:Hit:7 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T14:07:09.425 INFO:teuthology.orchestra.run.smithi099.stdout:Hit:6 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T14:07:09.435 INFO:teuthology.orchestra.run.smithi099.stdout:Hit:7 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-13T14:07:09.498 INFO:teuthology.orchestra.run.smithi177.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-13T14:07:09.499 INFO:teuthology.orchestra.run.smithi099.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-13T14:07:09.580 INFO:teuthology.orchestra.run.smithi177.stdout:Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-13T14:07:09.582 INFO:teuthology.orchestra.run.smithi099.stdout:Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-13T14:07:09.719 INFO:teuthology.orchestra.run.smithi177.stdout:Fetched 28.3 kB in 0s (68.8 kB/s) 2024-09-13T14:07:09.796 INFO:teuthology.orchestra.run.smithi099.stdout:Fetched 28.3 kB in 0s (57.7 kB/s) 2024-09-13T14:07:10.761 INFO:teuthology.orchestra.run.smithi177.stdout:Reading package lists... 2024-09-13T14:07:10.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.7-1jammy cephadm=17.2.7-1jammy ceph-mds=17.2.7-1jammy ceph-mgr=17.2.7-1jammy ceph-common=17.2.7-1jammy ceph-fuse=17.2.7-1jammy ceph-test=17.2.7-1jammy radosgw=17.2.7-1jammy python3-rados=17.2.7-1jammy python3-rgw=17.2.7-1jammy python3-cephfs=17.2.7-1jammy python3-rbd=17.2.7-1jammy libcephfs2=17.2.7-1jammy libcephfs-dev=17.2.7-1jammy librados2=17.2.7-1jammy librbd1=17.2.7-1jammy rbd-fuse=17.2.7-1jammy 2024-09-13T14:07:10.822 INFO:teuthology.orchestra.run.smithi099.stdout:Reading package lists... 2024-09-13T14:07:10.832 INFO:teuthology.orchestra.run.smithi177.stdout:Reading package lists... 2024-09-13T14:07:10.838 DEBUG:teuthology.orchestra.run.smithi099:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.7-1jammy cephadm=17.2.7-1jammy ceph-mds=17.2.7-1jammy ceph-mgr=17.2.7-1jammy ceph-common=17.2.7-1jammy ceph-fuse=17.2.7-1jammy ceph-test=17.2.7-1jammy radosgw=17.2.7-1jammy python3-rados=17.2.7-1jammy python3-rgw=17.2.7-1jammy python3-cephfs=17.2.7-1jammy python3-rbd=17.2.7-1jammy libcephfs2=17.2.7-1jammy libcephfs-dev=17.2.7-1jammy librados2=17.2.7-1jammy librbd1=17.2.7-1jammy rbd-fuse=17.2.7-1jammy 2024-09-13T14:07:10.901 INFO:teuthology.orchestra.run.smithi099.stdout:Reading package lists... 2024-09-13T14:07:11.025 INFO:teuthology.orchestra.run.smithi177.stdout:Building dependency tree... 2024-09-13T14:07:11.025 INFO:teuthology.orchestra.run.smithi177.stdout:Reading state information... 2024-09-13T14:07:11.119 INFO:teuthology.orchestra.run.smithi099.stdout:Building dependency tree... 2024-09-13T14:07:11.120 INFO:teuthology.orchestra.run.smithi099.stdout:Reading state information... 2024-09-13T14:07:11.217 INFO:teuthology.orchestra.run.smithi177.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T14:07:11.217 INFO:teuthology.orchestra.run.smithi177.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T14:07:11.217 INFO:teuthology.orchestra.run.smithi177.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T14:07:11.217 INFO:teuthology.orchestra.run.smithi177.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T14:07:11.218 INFO:teuthology.orchestra.run.smithi177.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-13T14:07:11.218 INFO:teuthology.orchestra.run.smithi177.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-13T14:07:11.218 INFO:teuthology.orchestra.run.smithi177.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T14:07:11.219 INFO:teuthology.orchestra.run.smithi177.stdout:The following additional packages will be installed: 2024-09-13T14:07:11.219 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-09-13T14:07:11.219 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-09-13T14:07:11.219 INFO:teuthology.orchestra.run.smithi177.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-09-13T14:07:11.219 INFO:teuthology.orchestra.run.smithi177.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-09-13T14:07:11.220 INFO:teuthology.orchestra.run.smithi177.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-09-13T14:07:11.220 INFO:teuthology.orchestra.run.smithi177.stdout: python-babel-localedata python-pastedeploy-tpl python3-babel python3-bcrypt 2024-09-13T14:07:11.220 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-09-13T14:07:11.221 INFO:teuthology.orchestra.run.smithi177.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-09-13T14:07:11.222 INFO:teuthology.orchestra.run.smithi177.stdout:Suggested packages: 2024-09-13T14:07:11.222 INFO:teuthology.orchestra.run.smithi177.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-09-13T14:07:11.222 INFO:teuthology.orchestra.run.smithi177.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-09-13T14:07:11.222 INFO:teuthology.orchestra.run.smithi177.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-09-13T14:07:11.222 INFO:teuthology.orchestra.run.smithi177.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-09-13T14:07:11.222 INFO:teuthology.orchestra.run.smithi177.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-09-13T14:07:11.222 INFO:teuthology.orchestra.run.smithi177.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-09-13T14:07:11.222 INFO:teuthology.orchestra.run.smithi177.stdout: python3-watchdog 2024-09-13T14:07:11.222 INFO:teuthology.orchestra.run.smithi177.stdout:Recommended packages: 2024-09-13T14:07:11.223 INFO:teuthology.orchestra.run.smithi177.stdout: btrfs-tools 2024-09-13T14:07:11.319 INFO:teuthology.orchestra.run.smithi099.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T14:07:11.319 INFO:teuthology.orchestra.run.smithi099.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T14:07:11.320 INFO:teuthology.orchestra.run.smithi099.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T14:07:11.320 INFO:teuthology.orchestra.run.smithi099.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T14:07:11.320 INFO:teuthology.orchestra.run.smithi099.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-13T14:07:11.320 INFO:teuthology.orchestra.run.smithi099.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-13T14:07:11.320 INFO:teuthology.orchestra.run.smithi099.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T14:07:11.321 INFO:teuthology.orchestra.run.smithi099.stdout:The following additional packages will be installed: 2024-09-13T14:07:11.321 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-09-13T14:07:11.321 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-09-13T14:07:11.321 INFO:teuthology.orchestra.run.smithi099.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-09-13T14:07:11.321 INFO:teuthology.orchestra.run.smithi099.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python-babel-localedata python-pastedeploy-tpl python3-babel python3-bcrypt 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-09-13T14:07:11.323 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi099.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi099.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi099.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi099.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi177.stdout:The following NEW packages will be installed: 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi177.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi177.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi177.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-09-13T14:07:11.324 INFO:teuthology.orchestra.run.smithi177.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-09-13T14:07:11.325 INFO:teuthology.orchestra.run.smithi099.stdout:Suggested packages: 2024-09-13T14:07:11.325 INFO:teuthology.orchestra.run.smithi099.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-09-13T14:07:11.325 INFO:teuthology.orchestra.run.smithi099.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-09-13T14:07:11.325 INFO:teuthology.orchestra.run.smithi099.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-09-13T14:07:11.325 INFO:teuthology.orchestra.run.smithi099.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-09-13T14:07:11.325 INFO:teuthology.orchestra.run.smithi099.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-09-13T14:07:11.325 INFO:teuthology.orchestra.run.smithi099.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-09-13T14:07:11.326 INFO:teuthology.orchestra.run.smithi099.stdout: python3-watchdog 2024-09-13T14:07:11.326 INFO:teuthology.orchestra.run.smithi099.stdout:Recommended packages: 2024-09-13T14:07:11.326 INFO:teuthology.orchestra.run.smithi099.stdout: btrfs-tools 2024-09-13T14:07:11.326 INFO:teuthology.orchestra.run.smithi177.stdout: libthrift-0.16.0 nvme-cli python-babel-localedata python-pastedeploy-tpl 2024-09-13T14:07:11.326 INFO:teuthology.orchestra.run.smithi177.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-09-13T14:07:11.326 INFO:teuthology.orchestra.run.smithi177.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-09-13T14:07:11.326 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-13T14:07:11.326 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-13T14:07:11.326 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-09-13T14:07:11.327 INFO:teuthology.orchestra.run.smithi177.stdout: radosgw rbd-fuse socat xmlstarlet 2024-09-13T14:07:11.390 INFO:teuthology.orchestra.run.smithi177.stdout:0 upgraded, 107 newly installed, 0 to remove and 331 not upgraded. 2024-09-13T14:07:11.390 INFO:teuthology.orchestra.run.smithi177.stdout:Need to get 187 MB of archives. 2024-09-13T14:07:11.390 INFO:teuthology.orchestra.run.smithi177.stdout:After this operation, 815 MB of additional disk space will be used. 2024-09-13T14:07:11.390 INFO:teuthology.orchestra.run.smithi177.stdout:Get:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 17.2.7-1jammy [3,198 kB] 2024-09-13T14:07:11.426 INFO:teuthology.orchestra.run.smithi099.stdout:The following NEW packages will be installed: 2024-09-13T14:07:11.426 INFO:teuthology.orchestra.run.smithi099.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-09-13T14:07:11.426 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: libthrift-0.16.0 nvme-cli python-babel-localedata python-pastedeploy-tpl 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-13T14:07:11.427 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-13T14:07:11.428 INFO:teuthology.orchestra.run.smithi099.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-13T14:07:11.428 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-09-13T14:07:11.428 INFO:teuthology.orchestra.run.smithi099.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-09-13T14:07:11.428 INFO:teuthology.orchestra.run.smithi099.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-09-13T14:07:11.428 INFO:teuthology.orchestra.run.smithi099.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-09-13T14:07:11.428 INFO:teuthology.orchestra.run.smithi099.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-09-13T14:07:11.428 INFO:teuthology.orchestra.run.smithi099.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-09-13T14:07:11.428 INFO:teuthology.orchestra.run.smithi099.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-09-13T14:07:11.428 INFO:teuthology.orchestra.run.smithi099.stdout: radosgw rbd-fuse socat xmlstarlet 2024-09-13T14:07:11.465 INFO:teuthology.orchestra.run.smithi177.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 17.2.7-1jammy [3,067 kB] 2024-09-13T14:07:11.469 INFO:teuthology.orchestra.run.smithi099.stdout:0 upgraded, 107 newly installed, 0 to remove and 331 not upgraded. 2024-09-13T14:07:11.469 INFO:teuthology.orchestra.run.smithi099.stdout:Need to get 187 MB of archives. 2024-09-13T14:07:11.469 INFO:teuthology.orchestra.run.smithi099.stdout:After this operation, 815 MB of additional disk space will be used. 2024-09-13T14:07:11.469 INFO:teuthology.orchestra.run.smithi099.stdout:Get:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 17.2.7-1jammy [3,198 kB] 2024-09-13T14:07:11.497 INFO:teuthology.orchestra.run.smithi099.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 17.2.7-1jammy [3,067 kB] 2024-09-13T14:07:11.513 INFO:teuthology.orchestra.run.smithi177.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-09-13T14:07:11.532 INFO:teuthology.orchestra.run.smithi177.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 17.2.7-1jammy [632 kB] 2024-09-13T14:07:11.532 INFO:teuthology.orchestra.run.smithi099.stdout:Get:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 17.2.7-1jammy [632 kB] 2024-09-13T14:07:11.568 INFO:teuthology.orchestra.run.smithi099.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 17.2.7-1jammy [344 kB] 2024-09-13T14:07:11.568 INFO:teuthology.orchestra.run.smithi177.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 17.2.7-1jammy [344 kB] 2024-09-13T14:07:11.598 INFO:teuthology.orchestra.run.smithi099.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 17.2.7-1jammy [23.1 kB] 2024-09-13T14:07:11.599 INFO:teuthology.orchestra.run.smithi177.stdout:Get:6 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 17.2.7-1jammy [23.1 kB] 2024-09-13T14:07:11.616 INFO:teuthology.orchestra.run.smithi099.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-09-13T14:07:11.627 INFO:teuthology.orchestra.run.smithi099.stdout:Get:7 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 17.2.7-1jammy [164 kB] 2024-09-13T14:07:11.627 INFO:teuthology.orchestra.run.smithi177.stdout:Get:7 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 17.2.7-1jammy [164 kB] 2024-09-13T14:07:11.650 INFO:teuthology.orchestra.run.smithi099.stdout:Get:8 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 17.2.7-1jammy [47.7 kB] 2024-09-13T14:07:11.651 INFO:teuthology.orchestra.run.smithi177.stdout:Get:8 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 17.2.7-1jammy [47.7 kB] 2024-09-13T14:07:11.734 INFO:teuthology.orchestra.run.smithi177.stdout:Get:9 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 17.2.7-1jammy [313 kB] 2024-09-13T14:07:11.735 INFO:teuthology.orchestra.run.smithi099.stdout:Get:9 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 17.2.7-1jammy [313 kB] 2024-09-13T14:07:11.777 INFO:teuthology.orchestra.run.smithi177.stdout:Get:10 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 17.2.7-1jammy [5,086 kB] 2024-09-13T14:07:11.777 INFO:teuthology.orchestra.run.smithi099.stdout:Get:10 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 17.2.7-1jammy [5,086 kB] 2024-09-13T14:07:11.793 INFO:teuthology.orchestra.run.smithi177.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-09-13T14:07:11.876 INFO:teuthology.orchestra.run.smithi177.stdout:Get:12 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 17.2.7-1jammy [102 kB] 2024-09-13T14:07:11.876 INFO:teuthology.orchestra.run.smithi099.stdout:Get:11 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 17.2.7-1jammy [102 kB] 2024-09-13T14:07:11.883 INFO:teuthology.orchestra.run.smithi177.stdout:Get:13 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 17.2.7-1jammy [415 kB] 2024-09-13T14:07:11.884 INFO:teuthology.orchestra.run.smithi099.stdout:Get:12 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 17.2.7-1jammy [415 kB] 2024-09-13T14:07:11.896 INFO:teuthology.orchestra.run.smithi099.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-09-13T14:07:11.898 INFO:teuthology.orchestra.run.smithi177.stdout:Get:14 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 17.2.7-1jammy [25.3 MB] 2024-09-13T14:07:11.899 INFO:teuthology.orchestra.run.smithi099.stdout:Get:14 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 17.2.7-1jammy [25.3 MB] 2024-09-13T14:07:11.965 INFO:teuthology.orchestra.run.smithi177.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-09-13T14:07:11.973 INFO:teuthology.orchestra.run.smithi177.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-09-13T14:07:11.986 INFO:teuthology.orchestra.run.smithi177.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-09-13T14:07:12.069 INFO:teuthology.orchestra.run.smithi099.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-09-13T14:07:12.077 INFO:teuthology.orchestra.run.smithi099.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-09-13T14:07:12.089 INFO:teuthology.orchestra.run.smithi099.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-09-13T14:07:12.129 INFO:teuthology.orchestra.run.smithi177.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-09-13T14:07:12.134 INFO:teuthology.orchestra.run.smithi177.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-09-13T14:07:12.157 INFO:teuthology.orchestra.run.smithi177.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-09-13T14:07:12.231 INFO:teuthology.orchestra.run.smithi099.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-09-13T14:07:12.236 INFO:teuthology.orchestra.run.smithi099.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-09-13T14:07:12.262 INFO:teuthology.orchestra.run.smithi099.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-09-13T14:07:12.278 INFO:teuthology.orchestra.run.smithi177.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-09-13T14:07:12.285 INFO:teuthology.orchestra.run.smithi177.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-09-13T14:07:12.306 INFO:teuthology.orchestra.run.smithi177.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-09-13T14:07:12.313 INFO:teuthology.orchestra.run.smithi177.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-09-13T14:07:12.314 INFO:teuthology.orchestra.run.smithi177.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-09-13T14:07:12.316 INFO:teuthology.orchestra.run.smithi177.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-09-13T14:07:12.317 INFO:teuthology.orchestra.run.smithi177.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-09-13T14:07:12.318 INFO:teuthology.orchestra.run.smithi177.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-09-13T14:07:12.320 INFO:teuthology.orchestra.run.smithi177.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-09-13T14:07:12.374 INFO:teuthology.orchestra.run.smithi177.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-09-13T14:07:12.421 INFO:teuthology.orchestra.run.smithi177.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-09-13T14:07:12.421 INFO:teuthology.orchestra.run.smithi177.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-09-13T14:07:12.428 INFO:teuthology.orchestra.run.smithi099.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-09-13T14:07:12.472 INFO:teuthology.orchestra.run.smithi177.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-09-13T14:07:12.472 INFO:teuthology.orchestra.run.smithi177.stdout:Get:34 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 17.2.7-1jammy [6,378 kB] 2024-09-13T14:07:12.472 INFO:teuthology.orchestra.run.smithi177.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-09-13T14:07:12.473 INFO:teuthology.orchestra.run.smithi177.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-09-13T14:07:12.480 INFO:teuthology.orchestra.run.smithi177.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-09-13T14:07:12.481 INFO:teuthology.orchestra.run.smithi177.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-09-13T14:07:12.482 INFO:teuthology.orchestra.run.smithi177.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-09-13T14:07:12.483 INFO:teuthology.orchestra.run.smithi177.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-09-13T14:07:12.505 INFO:teuthology.orchestra.run.smithi099.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-09-13T14:07:12.506 INFO:teuthology.orchestra.run.smithi099.stdout:Get:23 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 17.2.7-1jammy [6,378 kB] 2024-09-13T14:07:12.515 INFO:teuthology.orchestra.run.smithi177.stdout:Get:41 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 17.2.7-1jammy [224 kB] 2024-09-13T14:07:12.522 INFO:teuthology.orchestra.run.smithi099.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-09-13T14:07:12.522 INFO:teuthology.orchestra.run.smithi177.stdout:Get:42 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 17.2.7-1jammy [115 kB] 2024-09-13T14:07:12.529 INFO:teuthology.orchestra.run.smithi099.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-09-13T14:07:12.530 INFO:teuthology.orchestra.run.smithi099.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-09-13T14:07:12.531 INFO:teuthology.orchestra.run.smithi099.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-09-13T14:07:12.532 INFO:teuthology.orchestra.run.smithi099.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-09-13T14:07:12.533 INFO:teuthology.orchestra.run.smithi099.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-09-13T14:07:12.535 INFO:teuthology.orchestra.run.smithi099.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-09-13T14:07:12.536 INFO:teuthology.orchestra.run.smithi099.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-09-13T14:07:12.536 INFO:teuthology.orchestra.run.smithi177.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-09-13T14:07:12.544 INFO:teuthology.orchestra.run.smithi177.stdout:Get:44 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 17.2.7-1jammy [1,429 kB] 2024-09-13T14:07:12.549 INFO:teuthology.orchestra.run.smithi099.stdout:Get:32 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 17.2.7-1jammy [224 kB] 2024-09-13T14:07:12.551 INFO:teuthology.orchestra.run.smithi099.stdout:Get:33 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 17.2.7-1jammy [115 kB] 2024-09-13T14:07:12.552 INFO:teuthology.orchestra.run.smithi099.stdout:Get:34 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 17.2.7-1jammy [1,429 kB] 2024-09-13T14:07:12.568 INFO:teuthology.orchestra.run.smithi177.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-09-13T14:07:12.569 INFO:teuthology.orchestra.run.smithi177.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2024-09-13T14:07:12.571 INFO:teuthology.orchestra.run.smithi177.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-09-13T14:07:12.572 INFO:teuthology.orchestra.run.smithi177.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-09-13T14:07:12.573 INFO:teuthology.orchestra.run.smithi099.stdout:Get:35 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 17.2.7-1jammy [8,201 kB] 2024-09-13T14:07:12.573 INFO:teuthology.orchestra.run.smithi177.stdout:Get:49 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 17.2.7-1jammy [8,201 kB] 2024-09-13T14:07:12.574 INFO:teuthology.orchestra.run.smithi177.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-09-13T14:07:12.595 INFO:teuthology.orchestra.run.smithi099.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-09-13T14:07:12.618 INFO:teuthology.orchestra.run.smithi099.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-09-13T14:07:12.619 INFO:teuthology.orchestra.run.smithi099.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-09-13T14:07:12.620 INFO:teuthology.orchestra.run.smithi099.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-09-13T14:07:12.621 INFO:teuthology.orchestra.run.smithi099.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-09-13T14:07:12.626 INFO:teuthology.orchestra.run.smithi099.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-09-13T14:07:12.626 INFO:teuthology.orchestra.run.smithi177.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-09-13T14:07:12.627 INFO:teuthology.orchestra.run.smithi099.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-09-13T14:07:12.627 INFO:teuthology.orchestra.run.smithi177.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-09-13T14:07:12.628 INFO:teuthology.orchestra.run.smithi177.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-09-13T14:07:12.629 INFO:teuthology.orchestra.run.smithi177.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-09-13T14:07:12.680 INFO:teuthology.orchestra.run.smithi099.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-09-13T14:07:12.680 INFO:teuthology.orchestra.run.smithi099.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-09-13T14:07:12.682 INFO:teuthology.orchestra.run.smithi099.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-09-13T14:07:12.699 INFO:teuthology.orchestra.run.smithi177.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-09-13T14:07:12.703 INFO:teuthology.orchestra.run.smithi177.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-09-13T14:07:12.705 INFO:teuthology.orchestra.run.smithi177.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-09-13T14:07:12.707 INFO:teuthology.orchestra.run.smithi177.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-09-13T14:07:12.709 INFO:teuthology.orchestra.run.smithi177.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-09-13T14:07:12.765 INFO:teuthology.orchestra.run.smithi099.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-09-13T14:07:12.765 INFO:teuthology.orchestra.run.smithi099.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2024-09-13T14:07:12.767 INFO:teuthology.orchestra.run.smithi099.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-09-13T14:07:12.769 INFO:teuthology.orchestra.run.smithi099.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-09-13T14:07:12.770 INFO:teuthology.orchestra.run.smithi099.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-09-13T14:07:12.780 INFO:teuthology.orchestra.run.smithi099.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-09-13T14:07:12.781 INFO:teuthology.orchestra.run.smithi177.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-09-13T14:07:12.792 INFO:teuthology.orchestra.run.smithi099.stdout:Get:52 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 17.2.7-1jammy [28.4 MB] 2024-09-13T14:07:12.804 INFO:teuthology.orchestra.run.smithi177.stdout:Get:61 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 17.2.7-1jammy [28.4 MB] 2024-09-13T14:07:12.849 INFO:teuthology.orchestra.run.smithi099.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-09-13T14:07:12.850 INFO:teuthology.orchestra.run.smithi099.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-09-13T14:07:12.851 INFO:teuthology.orchestra.run.smithi099.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-09-13T14:07:12.854 INFO:teuthology.orchestra.run.smithi099.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-09-13T14:07:12.901 INFO:teuthology.orchestra.run.smithi177.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-09-13T14:07:12.902 INFO:teuthology.orchestra.run.smithi177.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-09-13T14:07:12.930 INFO:teuthology.orchestra.run.smithi099.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-09-13T14:07:12.932 INFO:teuthology.orchestra.run.smithi099.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-09-13T14:07:12.935 INFO:teuthology.orchestra.run.smithi099.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-09-13T14:07:12.936 INFO:teuthology.orchestra.run.smithi099.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-09-13T14:07:12.941 INFO:teuthology.orchestra.run.smithi099.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-09-13T14:07:12.946 INFO:teuthology.orchestra.run.smithi177.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-09-13T14:07:12.947 INFO:teuthology.orchestra.run.smithi177.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-09-13T14:07:12.949 INFO:teuthology.orchestra.run.smithi177.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-09-13T14:07:12.983 INFO:teuthology.orchestra.run.smithi177.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-09-13T14:07:12.987 INFO:teuthology.orchestra.run.smithi177.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-09-13T14:07:12.987 INFO:teuthology.orchestra.run.smithi177.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-09-13T14:07:13.047 INFO:teuthology.orchestra.run.smithi177.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-09-13T14:07:13.047 INFO:teuthology.orchestra.run.smithi177.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-09-13T14:07:13.100 INFO:teuthology.orchestra.run.smithi099.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-09-13T14:07:13.102 INFO:teuthology.orchestra.run.smithi099.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-09-13T14:07:13.104 INFO:teuthology.orchestra.run.smithi099.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-09-13T14:07:13.105 INFO:teuthology.orchestra.run.smithi099.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-09-13T14:07:13.107 INFO:teuthology.orchestra.run.smithi099.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-09-13T14:07:13.108 INFO:teuthology.orchestra.run.smithi177.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-09-13T14:07:13.109 INFO:teuthology.orchestra.run.smithi177.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-09-13T14:07:13.110 INFO:teuthology.orchestra.run.smithi177.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-09-13T14:07:13.111 INFO:teuthology.orchestra.run.smithi177.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-09-13T14:07:13.111 INFO:teuthology.orchestra.run.smithi177.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-09-13T14:07:13.113 INFO:teuthology.orchestra.run.smithi177.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-09-13T14:07:13.121 INFO:teuthology.orchestra.run.smithi177.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-09-13T14:07:13.142 INFO:teuthology.orchestra.run.smithi099.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-09-13T14:07:13.178 INFO:teuthology.orchestra.run.smithi099.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-09-13T14:07:13.178 INFO:teuthology.orchestra.run.smithi099.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-09-13T14:07:13.190 INFO:teuthology.orchestra.run.smithi177.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-09-13T14:07:13.194 INFO:teuthology.orchestra.run.smithi177.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-09-13T14:07:13.195 INFO:teuthology.orchestra.run.smithi177.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-09-13T14:07:13.209 INFO:teuthology.orchestra.run.smithi099.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-09-13T14:07:13.209 INFO:teuthology.orchestra.run.smithi099.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-09-13T14:07:13.267 INFO:teuthology.orchestra.run.smithi099.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-09-13T14:07:13.268 INFO:teuthology.orchestra.run.smithi099.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-09-13T14:07:13.269 INFO:teuthology.orchestra.run.smithi099.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-09-13T14:07:13.270 INFO:teuthology.orchestra.run.smithi099.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-09-13T14:07:13.270 INFO:teuthology.orchestra.run.smithi099.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-09-13T14:07:13.271 INFO:teuthology.orchestra.run.smithi177.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-09-13T14:07:13.271 INFO:teuthology.orchestra.run.smithi099.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-09-13T14:07:13.277 INFO:teuthology.orchestra.run.smithi177.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2024-09-13T14:07:13.278 INFO:teuthology.orchestra.run.smithi099.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-09-13T14:07:13.288 INFO:teuthology.orchestra.run.smithi177.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-09-13T14:07:13.289 INFO:teuthology.orchestra.run.smithi177.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-09-13T14:07:13.290 INFO:teuthology.orchestra.run.smithi177.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-09-13T14:07:13.291 INFO:teuthology.orchestra.run.smithi177.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-09-13T14:07:13.295 INFO:teuthology.orchestra.run.smithi177.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-09-13T14:07:13.351 INFO:teuthology.orchestra.run.smithi099.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-09-13T14:07:13.354 INFO:teuthology.orchestra.run.smithi099.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-09-13T14:07:13.355 INFO:teuthology.orchestra.run.smithi099.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-09-13T14:07:13.356 INFO:teuthology.orchestra.run.smithi177.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-09-13T14:07:13.377 INFO:teuthology.orchestra.run.smithi177.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-09-13T14:07:13.378 INFO:teuthology.orchestra.run.smithi177.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-09-13T14:07:13.435 INFO:teuthology.orchestra.run.smithi099.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-09-13T14:07:13.442 INFO:teuthology.orchestra.run.smithi177.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-09-13T14:07:13.487 INFO:teuthology.orchestra.run.smithi099.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2024-09-13T14:07:13.497 INFO:teuthology.orchestra.run.smithi177.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-09-13T14:07:13.556 INFO:teuthology.orchestra.run.smithi099.stdout:Get:84 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 17.2.7-1jammy [3,690 B] 2024-09-13T14:07:13.556 INFO:teuthology.orchestra.run.smithi099.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-09-13T14:07:13.556 INFO:teuthology.orchestra.run.smithi099.stdout:Get:86 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 17.2.7-1jammy [783 kB] 2024-09-13T14:07:13.556 INFO:teuthology.orchestra.run.smithi099.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-09-13T14:07:13.558 INFO:teuthology.orchestra.run.smithi099.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-09-13T14:07:13.558 INFO:teuthology.orchestra.run.smithi099.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-09-13T14:07:13.562 INFO:teuthology.orchestra.run.smithi099.stdout:Get:90 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 17.2.7-1jammy [2,120 kB] 2024-09-13T14:07:13.562 INFO:teuthology.orchestra.run.smithi099.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-09-13T14:07:13.564 INFO:teuthology.orchestra.run.smithi099.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi177.stdout:Get:94 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 17.2.7-1jammy [3,690 B] 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi177.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-09-13T14:07:13.569 INFO:teuthology.orchestra.run.smithi177.stdout:Get:96 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 17.2.7-1jammy [783 kB] 2024-09-13T14:07:13.576 INFO:teuthology.orchestra.run.smithi177.stdout:Get:97 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 17.2.7-1jammy [2,120 kB] 2024-09-13T14:07:13.578 INFO:teuthology.orchestra.run.smithi099.stdout:Get:93 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 17.2.7-1jammy [80.2 kB] 2024-09-13T14:07:13.579 INFO:teuthology.orchestra.run.smithi099.stdout:Get:94 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 17.2.7-1jammy [118 kB] 2024-09-13T14:07:13.580 INFO:teuthology.orchestra.run.smithi099.stdout:Get:95 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 17.2.7-1jammy [1,824 kB] 2024-09-13T14:07:13.581 INFO:teuthology.orchestra.run.smithi099.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-09-13T14:07:13.582 INFO:teuthology.orchestra.run.smithi099.stdout:Get:97 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-09-13T14:07:13.592 INFO:teuthology.orchestra.run.smithi177.stdout:Get:98 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 17.2.7-1jammy [80.2 kB] 2024-09-13T14:07:13.593 INFO:teuthology.orchestra.run.smithi177.stdout:Get:99 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 17.2.7-1jammy [118 kB] 2024-09-13T14:07:13.594 INFO:teuthology.orchestra.run.smithi177.stdout:Get:100 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 17.2.7-1jammy [1,824 kB] 2024-09-13T14:07:13.598 INFO:teuthology.orchestra.run.smithi099.stdout:Get:98 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-09-13T14:07:13.605 INFO:teuthology.orchestra.run.smithi099.stdout:Get:99 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 17.2.7-1jammy [8,615 kB] 2024-09-13T14:07:13.608 INFO:teuthology.orchestra.run.smithi177.stdout:Get:101 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 17.2.7-1jammy [8,615 kB] 2024-09-13T14:07:13.679 INFO:teuthology.orchestra.run.smithi099.stdout:Get:100 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-09-13T14:07:13.681 INFO:teuthology.orchestra.run.smithi099.stdout:Get:101 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-09-13T14:07:13.799 INFO:teuthology.orchestra.run.smithi099.stdout:Get:102 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 17.2.7-1jammy [3,814 B] 2024-09-13T14:07:13.799 INFO:teuthology.orchestra.run.smithi099.stdout:Get:103 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 17.2.7-1jammy [52.7 MB] 2024-09-13T14:07:13.799 INFO:teuthology.orchestra.run.smithi177.stdout:Get:102 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 17.2.7-1jammy [3,814 B] 2024-09-13T14:07:13.799 INFO:teuthology.orchestra.run.smithi177.stdout:Get:103 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 17.2.7-1jammy [52.7 MB] 2024-09-13T14:07:14.912 INFO:teuthology.orchestra.run.smithi099.stdout:Get:104 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 17.2.7-1jammy [240 kB] 2024-09-13T14:07:14.915 INFO:teuthology.orchestra.run.smithi099.stdout:Get:105 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 17.2.7-1jammy [22.4 kB] 2024-09-13T14:07:14.915 INFO:teuthology.orchestra.run.smithi099.stdout:Get:106 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 17.2.7-1jammy [15.4 MB] 2024-09-13T14:07:14.946 INFO:teuthology.orchestra.run.smithi177.stdout:Get:104 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 17.2.7-1jammy [240 kB] 2024-09-13T14:07:14.950 INFO:teuthology.orchestra.run.smithi177.stdout:Get:105 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 17.2.7-1jammy [22.4 kB] 2024-09-13T14:07:14.950 INFO:teuthology.orchestra.run.smithi177.stdout:Get:106 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 17.2.7-1jammy [15.4 MB] 2024-09-13T14:07:15.121 INFO:teuthology.orchestra.run.smithi099.stdout:Get:107 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 17.2.7-1jammy [75.8 kB] 2024-09-13T14:07:15.171 INFO:teuthology.orchestra.run.smithi177.stdout:Get:107 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 17.2.7-1jammy [75.8 kB] 2024-09-13T14:07:15.696 INFO:teuthology.orchestra.run.smithi099.stdout:Fetched 187 MB in 4s (50.6 MB/s) 2024-09-13T14:07:15.741 INFO:teuthology.orchestra.run.smithi177.stdout:Fetched 187 MB in 4s (48.6 MB/s) 2024-09-13T14:07:15.804 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libfmt8:amd64. 2024-09-13T14:07:15.854 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libfmt8:amd64. 2024-09-13T14:07:15.900 INFO:teuthology.orchestra.run.smithi099.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165990 files and directories currently installed.) 2024-09-13T14:07:15.905 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-09-13T14:07:15.950 INFO:teuthology.orchestra.run.smithi177.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165990 files and directories currently installed.) 2024-09-13T14:07:15.955 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-09-13T14:07:15.986 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-13T14:07:16.035 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-13T14:07:16.297 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-09-13T14:07:16.323 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-09-13T14:07:16.332 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-09-13T14:07:16.355 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-13T14:07:16.357 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-09-13T14:07:16.397 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-13T14:07:16.700 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package librados2. 2024-09-13T14:07:16.718 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package librados2. 2024-09-13T14:07:16.727 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../002-librados2_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:16.744 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../002-librados2_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:16.767 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking librados2 (17.2.7-1jammy) ... 2024-09-13T14:07:16.784 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking librados2 (17.2.7-1jammy) ... 2024-09-13T14:07:17.211 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package librbd1. 2024-09-13T14:07:17.212 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package librbd1. 2024-09-13T14:07:17.238 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../003-librbd1_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:17.238 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../003-librbd1_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:17.270 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking librbd1 (17.2.7-1jammy) ... 2024-09-13T14:07:17.277 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking librbd1 (17.2.7-1jammy) ... 2024-09-13T14:07:17.672 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libcephfs2. 2024-09-13T14:07:17.699 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../004-libcephfs2_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:17.706 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libcephfs2. 2024-09-13T14:07:17.733 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../004-libcephfs2_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:17.738 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libcephfs2 (17.2.7-1jammy) ... 2024-09-13T14:07:17.773 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libcephfs2 (17.2.7-1jammy) ... 2024-09-13T14:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-rados. 2024-09-13T14:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../005-python3-rados_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:18.051 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-rados. 2024-09-13T14:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-rados (17.2.7-1jammy) ... 2024-09-13T14:07:18.074 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../005-python3-rados_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:18.118 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-rados (17.2.7-1jammy) ... 2024-09-13T14:07:18.385 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-09-13T14:07:18.412 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../006-python3-ceph-argparse_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:18.420 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-09-13T14:07:18.447 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../006-python3-ceph-argparse_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:18.452 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-13T14:07:18.479 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-13T14:07:18.704 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-cephfs. 2024-09-13T14:07:18.706 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-cephfs. 2024-09-13T14:07:18.730 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../007-python3-cephfs_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:18.732 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../007-python3-cephfs_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:18.770 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-cephfs (17.2.7-1jammy) ... 2024-09-13T14:07:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-cephfs (17.2.7-1jammy) ... 2024-09-13T14:07:19.057 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-ceph-common. 2024-09-13T14:07:19.083 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../008-python3-ceph-common_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:19.084 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-ceph-common. 2024-09-13T14:07:19.110 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../008-python3-ceph-common_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:19.123 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-ceph-common (17.2.7-1jammy) ... 2024-09-13T14:07:19.151 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-ceph-common (17.2.7-1jammy) ... 2024-09-13T14:07:19.418 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-wcwidth. 2024-09-13T14:07:19.445 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../009-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-09-13T14:07:19.462 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-wcwidth. 2024-09-13T14:07:19.485 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-13T14:07:19.488 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../009-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-09-13T14:07:19.529 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-13T14:07:19.804 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-prettytable. 2024-09-13T14:07:19.831 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../010-python3-prettytable_2.5.0-2_all.deb ... 2024-09-13T14:07:19.848 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-prettytable. 2024-09-13T14:07:19.870 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-09-13T14:07:19.875 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../010-python3-prettytable_2.5.0-2_all.deb ... 2024-09-13T14:07:19.968 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-09-13T14:07:20.123 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-rbd. 2024-09-13T14:07:20.126 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-rbd. 2024-09-13T14:07:20.138 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../011-python3-rbd_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:20.141 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../011-python3-rbd_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:20.173 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-rbd (17.2.7-1jammy) ... 2024-09-13T14:07:20.176 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-rbd (17.2.7-1jammy) ... 2024-09-13T14:07:20.513 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-09-13T14:07:20.539 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../012-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-09-13T14:07:20.551 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-09-13T14:07:20.578 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../012-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-09-13T14:07:20.579 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-13T14:07:20.617 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-13T14:07:20.934 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-09-13T14:07:20.961 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-09-13T14:07:20.970 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-09-13T14:07:20.997 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-09-13T14:07:21.000 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-13T14:07:21.037 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-13T14:07:21.312 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-09-13T14:07:21.339 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../014-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-09-13T14:07:21.379 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-13T14:07:21.381 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-09-13T14:07:21.408 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../014-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-09-13T14:07:21.448 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-13T14:07:21.827 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-09-13T14:07:21.841 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-09-13T14:07:21.856 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../015-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T14:07:21.868 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../015-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T14:07:21.901 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:07:21.916 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:07:22.229 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-09-13T14:07:22.244 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-09-13T14:07:22.245 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../016-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T14:07:22.271 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../016-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T14:07:22.287 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:07:22.302 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-09-13T14:07:22.640 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-09-13T14:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../017-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T14:07:22.667 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../017-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T14:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:07:22.716 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:07:23.042 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-09-13T14:07:23.068 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../018-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-09-13T14:07:23.068 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-09-13T14:07:23.095 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../018-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-09-13T14:07:23.109 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-13T14:07:23.135 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-13T14:07:23.395 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package librgw2. 2024-09-13T14:07:23.396 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package librgw2. 2024-09-13T14:07:23.412 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../019-librgw2_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:23.421 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../019-librgw2_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:23.446 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking librgw2 (17.2.7-1jammy) ... 2024-09-13T14:07:23.453 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking librgw2 (17.2.7-1jammy) ... 2024-09-13T14:07:23.914 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-rgw. 2024-09-13T14:07:23.929 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../020-python3-rgw_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:23.972 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-rgw (17.2.7-1jammy) ... 2024-09-13T14:07:24.107 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-rgw. 2024-09-13T14:07:24.122 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../020-python3-rgw_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:24.165 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-rgw (17.2.7-1jammy) ... 2024-09-13T14:07:24.309 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package liboath0:amd64. 2024-09-13T14:07:24.335 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-09-13T14:07:24.384 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-09-13T14:07:24.520 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package liboath0:amd64. 2024-09-13T14:07:24.547 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-09-13T14:07:24.586 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-09-13T14:07:24.662 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libradosstriper1. 2024-09-13T14:07:24.677 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../022-libradosstriper1_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:24.712 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libradosstriper1 (17.2.7-1jammy) ... 2024-09-13T14:07:24.889 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libradosstriper1. 2024-09-13T14:07:24.916 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../022-libradosstriper1_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:24.956 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libradosstriper1 (17.2.7-1jammy) ... 2024-09-13T14:07:25.149 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-common. 2024-09-13T14:07:25.174 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../023-ceph-common_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:25.215 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-common (17.2.7-1jammy) ... 2024-09-13T14:07:25.309 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-common. 2024-09-13T14:07:25.336 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../023-ceph-common_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:25.367 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-common (17.2.7-1jammy) ... 2024-09-13T14:07:26.663 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-base. 2024-09-13T14:07:26.689 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../024-ceph-base_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:26.723 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-base. 2024-09-13T14:07:26.739 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-base (17.2.7-1jammy) ... 2024-09-13T14:07:26.749 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../024-ceph-base_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:26.799 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-base (17.2.7-1jammy) ... 2024-09-13T14:07:27.401 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-openssl. 2024-09-13T14:07:27.427 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../025-python3-openssl_21.0.0-1_all.deb ... 2024-09-13T14:07:27.467 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-09-13T14:07:27.468 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-openssl. 2024-09-13T14:07:27.495 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../025-python3-openssl_21.0.0-1_all.deb ... 2024-09-13T14:07:27.535 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-09-13T14:07:27.720 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-09-13T14:07:27.735 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:27.778 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-13T14:07:27.779 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-09-13T14:07:27.795 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:27.829 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-13T14:07:28.133 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-bcrypt. 2024-09-13T14:07:28.159 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../027-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-09-13T14:07:28.199 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-09-13T14:07:28.225 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-bcrypt. 2024-09-13T14:07:28.252 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../027-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-09-13T14:07:28.292 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-09-13T14:07:28.519 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-09-13T14:07:28.546 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../028-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-09-13T14:07:28.586 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-09-13T14:07:28.595 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-09-13T14:07:28.621 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../028-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-09-13T14:07:28.661 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-09-13T14:07:28.872 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-cheroot. 2024-09-13T14:07:28.898 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../029-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-09-13T14:07:28.939 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-13T14:07:29.031 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-cheroot. 2024-09-13T14:07:29.057 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../029-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-09-13T14:07:29.097 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-13T14:07:29.250 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-09-13T14:07:29.277 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../030-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-09-13T14:07:29.325 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-09-13T14:07:29.393 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-09-13T14:07:29.418 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../030-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-09-13T14:07:29.450 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-09-13T14:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-jaraco.text. 2024-09-13T14:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../031-python3-jaraco.text_3.6.0-2_all.deb ... 2024-09-13T14:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-09-13T14:07:29.728 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-jaraco.text. 2024-09-13T14:07:29.755 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../031-python3-jaraco.text_3.6.0-2_all.deb ... 2024-09-13T14:07:29.794 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-09-13T14:07:29.973 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-09-13T14:07:29.999 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../032-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-09-13T14:07:30.039 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-09-13T14:07:30.097 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-09-13T14:07:30.124 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../032-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-09-13T14:07:30.164 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-09-13T14:07:30.343 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-tempora. 2024-09-13T14:07:30.369 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../033-python3-tempora_4.1.2-1_all.deb ... 2024-09-13T14:07:30.409 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-09-13T14:07:30.450 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-tempora. 2024-09-13T14:07:30.477 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../033-python3-tempora_4.1.2-1_all.deb ... 2024-09-13T14:07:30.516 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-09-13T14:07:30.679 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-portend. 2024-09-13T14:07:30.694 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../034-python3-portend_3.0.0-1_all.deb ... 2024-09-13T14:07:30.737 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-09-13T14:07:30.786 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-portend. 2024-09-13T14:07:30.801 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../034-python3-portend_3.0.0-1_all.deb ... 2024-09-13T14:07:30.836 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-09-13T14:07:31.040 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-09-13T14:07:31.066 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../035-python3-zc.lockfile_2.0-1_all.deb ... 2024-09-13T14:07:31.106 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-09-13T14:07:31.130 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-09-13T14:07:31.157 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../035-python3-zc.lockfile_2.0-1_all.deb ... 2024-09-13T14:07:31.197 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-09-13T14:07:31.410 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-cherrypy3. 2024-09-13T14:07:31.436 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../036-python3-cherrypy3_18.6.1-4_all.deb ... 2024-09-13T14:07:31.476 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-09-13T14:07:31.500 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-cherrypy3. 2024-09-13T14:07:31.515 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../036-python3-cherrypy3_18.6.1-4_all.deb ... 2024-09-13T14:07:31.558 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-09-13T14:07:31.847 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-natsort. 2024-09-13T14:07:31.873 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../037-python3-natsort_8.0.2-1_all.deb ... 2024-09-13T14:07:31.886 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-natsort. 2024-09-13T14:07:31.913 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-09-13T14:07:31.913 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../037-python3-natsort_8.0.2-1_all.deb ... 2024-09-13T14:07:31.953 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-09-13T14:07:32.216 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-logutils. 2024-09-13T14:07:32.243 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../038-python3-logutils_0.3.3-8_all.deb ... 2024-09-13T14:07:32.283 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-09-13T14:07:32.289 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-logutils. 2024-09-13T14:07:32.316 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../038-python3-logutils_0.3.3-8_all.deb ... 2024-09-13T14:07:32.355 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-09-13T14:07:32.603 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-markupsafe. 2024-09-13T14:07:32.629 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../039-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-09-13T14:07:32.658 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-markupsafe. 2024-09-13T14:07:32.669 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-09-13T14:07:32.685 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../039-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-09-13T14:07:32.725 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-09-13T14:07:32.922 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-mako. 2024-09-13T14:07:32.949 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../040-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-09-13T14:07:32.989 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-13T14:07:33.002 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-mako. 2024-09-13T14:07:33.018 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../040-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-09-13T14:07:33.052 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-13T14:07:33.301 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-simplegeneric. 2024-09-13T14:07:33.327 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../041-python3-simplegeneric_0.8.1-3_all.deb ... 2024-09-13T14:07:33.347 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-simplegeneric. 2024-09-13T14:07:33.367 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-09-13T14:07:33.374 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../041-python3-simplegeneric_0.8.1-3_all.deb ... 2024-09-13T14:07:33.405 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-09-13T14:07:33.687 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-singledispatch. 2024-09-13T14:07:33.708 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-singledispatch. 2024-09-13T14:07:33.713 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../042-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-09-13T14:07:33.735 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../042-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-09-13T14:07:33.753 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-09-13T14:07:33.775 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-09-13T14:07:34.040 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-webob. 2024-09-13T14:07:34.066 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../043-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2024-09-13T14:07:34.106 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-13T14:07:34.219 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-webob. 2024-09-13T14:07:34.246 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../043-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2024-09-13T14:07:34.319 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-13T14:07:34.418 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-waitress. 2024-09-13T14:07:34.444 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../044-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-09-13T14:07:34.479 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-13T14:07:34.681 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-waitress. 2024-09-13T14:07:34.707 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../044-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-09-13T14:07:34.750 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-13T14:07:34.771 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-tempita. 2024-09-13T14:07:34.797 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../045-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-09-13T14:07:34.837 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-09-13T14:07:35.041 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-tempita. 2024-09-13T14:07:35.068 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../045-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-09-13T14:07:35.108 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-09-13T14:07:35.361 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-paste. 2024-09-13T14:07:35.387 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../046-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-09-13T14:07:35.419 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-09-13T14:07:35.449 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-paste. 2024-09-13T14:07:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../046-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-09-13T14:07:35.515 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-09-13T14:07:35.747 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-09-13T14:07:35.773 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../047-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-09-13T14:07:35.813 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-09-13T14:07:35.836 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-09-13T14:07:35.862 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../047-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-09-13T14:07:35.902 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-09-13T14:07:36.099 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-pastedeploy. 2024-09-13T14:07:36.114 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../048-python3-pastedeploy_2.1.1-1_all.deb ... 2024-09-13T14:07:36.149 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-09-13T14:07:36.188 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-pastedeploy. 2024-09-13T14:07:36.214 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../048-python3-pastedeploy_2.1.1-1_all.deb ... 2024-09-13T14:07:36.255 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-09-13T14:07:36.444 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-webtest. 2024-09-13T14:07:36.471 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../049-python3-webtest_2.0.35-1_all.deb ... 2024-09-13T14:07:36.511 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-09-13T14:07:36.558 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-webtest. 2024-09-13T14:07:36.584 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../049-python3-webtest_2.0.35-1_all.deb ... 2024-09-13T14:07:36.624 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-09-13T14:07:36.838 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-pecan. 2024-09-13T14:07:36.865 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../050-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-09-13T14:07:36.902 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-pecan. 2024-09-13T14:07:36.905 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-09-13T14:07:36.918 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../050-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-09-13T14:07:36.952 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-09-13T14:07:37.268 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-certifi. 2024-09-13T14:07:37.282 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-certifi. 2024-09-13T14:07:37.294 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../051-python3-certifi_2020.6.20-1_all.deb ... 2024-09-13T14:07:37.298 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../051-python3-certifi_2020.6.20-1_all.deb ... 2024-09-13T14:07:37.332 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-09-13T14:07:37.334 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-09-13T14:07:37.652 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-idna. 2024-09-13T14:07:37.654 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-idna. 2024-09-13T14:07:37.679 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../052-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-09-13T14:07:37.681 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../052-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-09-13T14:07:37.719 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-09-13T14:07:37.720 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-09-13T14:07:37.990 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-urllib3. 2024-09-13T14:07:38.005 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../053-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-09-13T14:07:38.030 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-urllib3. 2024-09-13T14:07:38.039 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-13T14:07:38.057 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../053-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-09-13T14:07:38.097 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-13T14:07:38.334 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-requests. 2024-09-13T14:07:38.349 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../054-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-09-13T14:07:38.384 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-13T14:07:38.450 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-requests. 2024-09-13T14:07:38.476 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../054-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-09-13T14:07:38.516 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-13T14:07:38.654 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-werkzeug. 2024-09-13T14:07:38.680 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../055-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-09-13T14:07:38.720 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-13T14:07:38.820 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-werkzeug. 2024-09-13T14:07:38.848 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../055-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-09-13T14:07:38.886 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-13T14:07:39.040 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-09-13T14:07:39.055 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:39.090 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-13T14:07:39.223 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-09-13T14:07:39.249 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:39.293 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-13T14:07:39.384 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-mgr. 2024-09-13T14:07:39.411 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../057-ceph-mgr_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:39.517 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-mgr (17.2.7-1jammy) ... 2024-09-13T14:07:39.610 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-mgr. 2024-09-13T14:07:39.636 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../057-ceph-mgr_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:39.668 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-mgr (17.2.7-1jammy) ... 2024-09-13T14:07:40.046 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-mon. 2024-09-13T14:07:40.063 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-mon. 2024-09-13T14:07:40.073 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../058-ceph-mon_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:40.089 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../058-ceph-mon_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:40.112 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-mon (17.2.7-1jammy) ... 2024-09-13T14:07:40.129 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-mon (17.2.7-1jammy) ... 2024-09-13T14:07:40.891 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-osd. 2024-09-13T14:07:40.906 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../059-ceph-osd_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:40.940 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-osd (17.2.7-1jammy) ... 2024-09-13T14:07:40.958 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-osd. 2024-09-13T14:07:40.984 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../059-ceph-osd_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:41.024 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-osd (17.2.7-1jammy) ... 2024-09-13T14:07:42.519 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph. 2024-09-13T14:07:42.534 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../060-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:42.577 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph (17.2.7-1jammy) ... 2024-09-13T14:07:42.694 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph. 2024-09-13T14:07:42.710 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../060-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:42.753 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph (17.2.7-1jammy) ... 2024-09-13T14:07:42.905 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-fuse. 2024-09-13T14:07:42.931 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../061-ceph-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:42.971 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-fuse (17.2.7-1jammy) ... 2024-09-13T14:07:43.089 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-fuse. 2024-09-13T14:07:43.115 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../061-ceph-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:43.155 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-fuse (17.2.7-1jammy) ... 2024-09-13T14:07:43.291 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-mds. 2024-09-13T14:07:43.317 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../062-ceph-mds_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:43.350 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-mds (17.2.7-1jammy) ... 2024-09-13T14:07:43.526 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-mds. 2024-09-13T14:07:43.552 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../062-ceph-mds_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:43.592 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-mds (17.2.7-1jammy) ... 2024-09-13T14:07:43.727 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package cephadm. 2024-09-13T14:07:43.754 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../063-cephadm_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:43.819 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking cephadm (17.2.7-1jammy) ... 2024-09-13T14:07:43.962 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package cephadm. 2024-09-13T14:07:43.989 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../063-cephadm_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:44.029 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking cephadm (17.2.7-1jammy) ... 2024-09-13T14:07:44.088 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python-babel-localedata. 2024-09-13T14:07:44.104 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-09-13T14:07:44.146 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-13T14:07:44.298 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python-babel-localedata. 2024-09-13T14:07:44.313 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-09-13T14:07:44.356 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-13T14:07:45.976 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-babel. 2024-09-13T14:07:46.004 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-09-13T14:07:46.034 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-09-13T14:07:46.287 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-babel. 2024-09-13T14:07:46.312 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-jinja2. 2024-09-13T14:07:46.314 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-09-13T14:07:46.328 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-09-13T14:07:46.353 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-09-13T14:07:46.363 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-13T14:07:46.665 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-jinja2. 2024-09-13T14:07:46.691 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-09-13T14:07:46.699 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-09-13T14:07:46.726 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:46.731 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-13T14:07:46.757 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-13T14:07:47.043 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-09-13T14:07:47.052 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-repoze.lru. 2024-09-13T14:07:47.070 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:47.079 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../068-python3-repoze.lru_0.7-2_all.deb ... 2024-09-13T14:07:47.110 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-13T14:07:47.119 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-09-13T14:07:47.421 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-routes. 2024-09-13T14:07:47.430 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-repoze.lru. 2024-09-13T14:07:47.449 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../069-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-09-13T14:07:47.456 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../068-python3-repoze.lru_0.7-2_all.deb ... 2024-09-13T14:07:47.488 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-09-13T14:07:47.488 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-09-13T14:07:47.766 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-routes. 2024-09-13T14:07:47.793 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../069-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-09-13T14:07:47.833 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-09-13T14:07:47.849 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-09-13T14:07:47.877 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../070-ceph-mgr-dashboard_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:47.916 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-13T14:07:48.145 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-09-13T14:07:48.171 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../070-ceph-mgr-dashboard_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:48.211 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-13T14:07:48.294 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-09-13T14:07:48.310 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../071-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-09-13T14:07:48.344 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-13T14:07:48.623 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-09-13T14:07:48.650 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../071-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-09-13T14:07:48.690 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-13T14:07:48.815 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-joblib. 2024-09-13T14:07:48.842 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../072-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-09-13T14:07:48.881 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-09-13T14:07:49.177 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-joblib. 2024-09-13T14:07:49.201 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-09-13T14:07:49.204 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../072-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-09-13T14:07:49.216 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../073-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-09-13T14:07:49.243 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-09-13T14:07:49.251 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-09-13T14:07:49.554 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-sklearn. 2024-09-13T14:07:49.581 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../074-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-09-13T14:07:49.606 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-09-13T14:07:49.620 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-13T14:07:49.632 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../073-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-09-13T14:07:49.672 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-09-13T14:07:49.975 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-sklearn. 2024-09-13T14:07:50.002 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../074-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-09-13T14:07:50.042 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-13T14:07:50.338 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-09-13T14:07:50.365 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../075-ceph-mgr-diskprediction-local_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:50.403 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-13T14:07:50.751 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-09-13T14:07:50.778 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../075-ceph-mgr-diskprediction-local_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:50.817 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-13T14:07:51.315 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-cachetools. 2024-09-13T14:07:51.330 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../076-python3-cachetools_5.0.0-1_all.deb ... 2024-09-13T14:07:51.365 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-09-13T14:07:51.692 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-pyasn1. 2024-09-13T14:07:51.719 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../077-python3-pyasn1_0.4.8-1_all.deb ... 2024-09-13T14:07:51.754 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-cachetools. 2024-09-13T14:07:51.758 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-09-13T14:07:51.781 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../076-python3-cachetools_5.0.0-1_all.deb ... 2024-09-13T14:07:51.820 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-09-13T14:07:52.112 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-09-13T14:07:52.140 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../078-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-09-13T14:07:52.148 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-pyasn1. 2024-09-13T14:07:52.175 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../077-python3-pyasn1_0.4.8-1_all.deb ... 2024-09-13T14:07:52.178 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-09-13T14:07:52.215 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-09-13T14:07:52.490 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-rsa. 2024-09-13T14:07:52.517 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../079-python3-rsa_4.8-1_all.deb ... 2024-09-13T14:07:52.535 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-09-13T14:07:52.556 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-rsa (4.8-1) ... 2024-09-13T14:07:52.562 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../078-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-09-13T14:07:52.593 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-09-13T14:07:52.835 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-google-auth. 2024-09-13T14:07:52.862 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../080-python3-google-auth_1.5.1-3_all.deb ... 2024-09-13T14:07:52.871 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-rsa. 2024-09-13T14:07:52.887 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../079-python3-rsa_4.8-1_all.deb ... 2024-09-13T14:07:52.892 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-09-13T14:07:52.921 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-rsa (4.8-1) ... 2024-09-13T14:07:53.188 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-09-13T14:07:53.215 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../081-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-09-13T14:07:53.225 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-google-auth. 2024-09-13T14:07:53.252 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../080-python3-google-auth_1.5.1-3_all.deb ... 2024-09-13T14:07:53.254 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-13T14:07:53.291 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-09-13T14:07:53.565 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-websocket. 2024-09-13T14:07:53.592 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../082-python3-websocket_1.2.3-1_all.deb ... 2024-09-13T14:07:53.611 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-09-13T14:07:53.631 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-09-13T14:07:53.638 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../081-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-09-13T14:07:53.678 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-13T14:07:53.989 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-websocket. 2024-09-13T14:07:54.016 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../082-python3-websocket_1.2.3-1_all.deb ... 2024-09-13T14:07:54.045 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-kubernetes. 2024-09-13T14:07:54.056 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-09-13T14:07:54.073 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../083-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-09-13T14:07:54.111 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-13T14:07:54.428 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-kubernetes. 2024-09-13T14:07:54.455 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../083-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-09-13T14:07:54.494 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-13T14:07:54.724 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-09-13T14:07:54.741 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../084-ceph-mgr-k8sevents_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:54.783 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-13T14:07:55.117 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-09-13T14:07:55.144 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../084-ceph-mgr-k8sevents_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:55.175 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-13T14:07:55.194 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libonig5:amd64. 2024-09-13T14:07:55.222 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../085-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-09-13T14:07:55.331 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-13T14:07:55.520 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libonig5:amd64. 2024-09-13T14:07:55.547 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../085-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-09-13T14:07:55.586 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-13T14:07:55.680 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libjq1:amd64. 2024-09-13T14:07:55.707 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../086-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-09-13T14:07:55.747 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-13T14:07:55.923 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libjq1:amd64. 2024-09-13T14:07:55.950 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../086-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-09-13T14:07:55.989 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-13T14:07:55.990 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package jq. 2024-09-13T14:07:56.006 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../087-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-09-13T14:07:56.049 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-09-13T14:07:56.301 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package socat. 2024-09-13T14:07:56.318 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../088-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-09-13T14:07:56.368 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-09-13T14:07:56.431 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package jq. 2024-09-13T14:07:56.458 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../087-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-09-13T14:07:56.500 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-09-13T14:07:56.655 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package xmlstarlet. 2024-09-13T14:07:56.670 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../089-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-09-13T14:07:56.711 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package socat. 2024-09-13T14:07:56.712 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-09-13T14:07:56.738 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../088-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-09-13T14:07:56.778 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-09-13T14:07:57.050 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-test. 2024-09-13T14:07:57.066 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../090-ceph-test_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:57.090 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package xmlstarlet. 2024-09-13T14:07:57.100 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-test (17.2.7-1jammy) ... 2024-09-13T14:07:57.117 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../089-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-09-13T14:07:57.156 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-09-13T14:07:57.494 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-test. 2024-09-13T14:07:57.521 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../090-ceph-test_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:57.560 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-test (17.2.7-1jammy) ... 2024-09-13T14:07:59.529 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package ceph-volume. 2024-09-13T14:07:59.544 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../091-ceph-volume_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:59.589 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking ceph-volume (17.2.7-1jammy) ... 2024-09-13T14:07:59.924 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package libcephfs-dev. 2024-09-13T14:07:59.939 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package ceph-volume. 2024-09-13T14:07:59.940 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../092-libcephfs-dev_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:07:59.955 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../091-ceph-volume_17.2.7-1jammy_all.deb ... 2024-09-13T14:07:59.982 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking libcephfs-dev (17.2.7-1jammy) ... 2024-09-13T14:07:59.997 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking ceph-volume (17.2.7-1jammy) ... 2024-09-13T14:08:00.327 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package libcephfs-dev. 2024-09-13T14:08:00.343 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../092-libcephfs-dev_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:08:00.369 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package nvme-cli. 2024-09-13T14:08:00.385 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking libcephfs-dev (17.2.7-1jammy) ... 2024-09-13T14:08:00.396 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../093-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2024-09-13T14:08:00.435 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2024-09-13T14:08:00.714 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package nvme-cli. 2024-09-13T14:08:00.741 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../093-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2024-09-13T14:08:00.772 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2024-09-13T14:08:00.841 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-iniconfig. 2024-09-13T14:08:00.869 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../094-python3-iniconfig_1.1.1-2_all.deb ... 2024-09-13T14:08:00.907 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-09-13T14:08:01.172 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-iniconfig. 2024-09-13T14:08:01.186 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-pastescript. 2024-09-13T14:08:01.199 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../094-python3-iniconfig_1.1.1-2_all.deb ... 2024-09-13T14:08:01.213 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../095-python3-pastescript_2.0.2-4_all.deb ... 2024-09-13T14:08:01.238 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-09-13T14:08:01.252 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-09-13T14:08:01.530 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-pluggy. 2024-09-13T14:08:01.549 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-pastescript. 2024-09-13T14:08:01.558 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../096-python3-pluggy_0.13.0-7.1_all.deb ... 2024-09-13T14:08:01.577 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../095-python3-pastescript_2.0.2-4_all.deb ... 2024-09-13T14:08:01.588 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-09-13T14:08:01.633 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-09-13T14:08:01.891 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-psutil. 2024-09-13T14:08:01.919 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../097-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-09-13T14:08:01.953 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-pluggy. 2024-09-13T14:08:01.957 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-09-13T14:08:01.981 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../096-python3-pluggy_0.13.0-7.1_all.deb ... 2024-09-13T14:08:02.011 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-09-13T14:08:02.303 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-py. 2024-09-13T14:08:02.330 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../098-python3-py_1.10.0-1_all.deb ... 2024-09-13T14:08:02.339 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-psutil. 2024-09-13T14:08:02.366 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../097-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-09-13T14:08:02.368 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-py (1.10.0-1) ... 2024-09-13T14:08:02.406 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-09-13T14:08:02.664 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-pygments. 2024-09-13T14:08:02.692 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../099-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-09-13T14:08:02.730 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-09-13T14:08:02.734 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-py. 2024-09-13T14:08:02.761 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../098-python3-py_1.10.0-1_all.deb ... 2024-09-13T14:08:02.800 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-py (1.10.0-1) ... 2024-09-13T14:08:03.129 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-pygments. 2024-09-13T14:08:03.157 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../099-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-09-13T14:08:03.196 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-09-13T14:08:03.201 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-pyinotify. 2024-09-13T14:08:03.229 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../100-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-09-13T14:08:03.293 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-09-13T14:08:03.603 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-toml. 2024-09-13T14:08:03.631 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../101-python3-toml_0.10.2-1_all.deb ... 2024-09-13T14:08:03.641 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-pyinotify. 2024-09-13T14:08:03.669 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../100-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-09-13T14:08:03.670 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-09-13T14:08:03.708 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-09-13T14:08:03.947 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-pytest. 2024-09-13T14:08:03.963 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../102-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-09-13T14:08:03.997 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-09-13T14:08:04.002 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-toml. 2024-09-13T14:08:04.030 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../101-python3-toml_0.10.2-1_all.deb ... 2024-09-13T14:08:04.069 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-09-13T14:08:04.334 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package python3-simplejson. 2024-09-13T14:08:04.362 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../103-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-09-13T14:08:04.372 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-pytest. 2024-09-13T14:08:04.400 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../102-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-09-13T14:08:04.400 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-09-13T14:08:04.438 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-09-13T14:08:04.688 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-09-13T14:08:04.716 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../104-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-09-13T14:08:04.755 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-09-13T14:08:04.759 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package python3-simplejson. 2024-09-13T14:08:04.786 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../103-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-09-13T14:08:04.825 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-09-13T14:08:05.078 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-09-13T14:08:05.094 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../104-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-09-13T14:08:05.136 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-09-13T14:08:05.442 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package radosgw. 2024-09-13T14:08:05.470 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../105-radosgw_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:08:05.500 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking radosgw (17.2.7-1jammy) ... 2024-09-13T14:08:05.924 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package radosgw. 2024-09-13T14:08:05.952 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../105-radosgw_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:08:06.098 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking radosgw (17.2.7-1jammy) ... 2024-09-13T14:08:06.520 INFO:teuthology.orchestra.run.smithi177.stdout:Selecting previously unselected package rbd-fuse. 2024-09-13T14:08:06.536 INFO:teuthology.orchestra.run.smithi177.stdout:Preparing to unpack .../106-rbd-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:08:06.578 INFO:teuthology.orchestra.run.smithi177.stdout:Unpacking rbd-fuse (17.2.7-1jammy) ... 2024-09-13T14:08:06.881 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-09-13T14:08:07.140 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-13T14:08:07.143 INFO:teuthology.orchestra.run.smithi099.stdout:Selecting previously unselected package rbd-fuse. 2024-09-13T14:08:07.159 INFO:teuthology.orchestra.run.smithi099.stdout:Preparing to unpack .../106-rbd-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-13T14:08:07.201 INFO:teuthology.orchestra.run.smithi099.stdout:Unpacking rbd-fuse (17.2.7-1jammy) ... 2024-09-13T14:08:07.242 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2024-09-13T14:08:07.487 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-09-13T14:08:07.497 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-09-13T14:08:07.765 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-13T14:08:07.788 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-09-13T14:08:07.891 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2024-09-13T14:08:08.151 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-09-13T14:08:08.201 INFO:teuthology.orchestra.run.smithi177.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-09-13T14:08:08.268 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up cephadm (17.2.7-1jammy) ... 2024-09-13T14:08:08.480 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-09-13T14:08:08.786 INFO:teuthology.orchestra.run.smithi177.stdout:Adding system user cephadm....done 2024-09-13T14:08:08.899 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-13T14:08:08.906 INFO:teuthology.orchestra.run.smithi099.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-09-13T14:08:09.059 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up cephadm (17.2.7-1jammy) ... 2024-09-13T14:08:09.167 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-09-13T14:08:09.410 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-09-13T14:08:09.645 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-09-13T14:08:09.669 INFO:teuthology.orchestra.run.smithi099.stdout:Adding system user cephadm....done 2024-09-13T14:08:09.904 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-09-13T14:08:09.908 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-13T14:08:10.034 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-py (1.10.0-1) ... 2024-09-13T14:08:10.332 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-09-13T14:08:10.469 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-09-13T14:08:10.703 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-09-13T14:08:10.767 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-09-13T14:08:10.955 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-09-13T14:08:11.017 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-09-13T14:08:11.205 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-09-13T14:08:11.318 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-09-13T14:08:11.332 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-py (1.10.0-1) ... 2024-09-13T14:08:11.553 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-09-13T14:08:11.633 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-09-13T14:08:11.812 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-09-13T14:08:11.969 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-09-13T14:08:12.063 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-13T14:08:12.220 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-09-13T14:08:12.323 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-13T14:08:12.449 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-09-13T14:08:12.496 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-09-13T14:08:12.699 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-13T14:08:12.755 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-09-13T14:08:12.809 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-09-13T14:08:13.031 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-09-13T14:08:13.160 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-13T14:08:13.261 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-09-13T14:08:13.291 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-13T14:08:13.520 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-09-13T14:08:13.625 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-13T14:08:13.743 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-09-13T14:08:13.779 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-09-13T14:08:13.977 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-13T14:08:14.089 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-09-13T14:08:14.095 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-09-13T14:08:14.463 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-13T14:08:14.589 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-09-13T14:08:14.758 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-09-13T14:08:14.865 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-09-13T14:08:15.025 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-09-13T14:08:15.141 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-09-13T14:08:15.151 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-09-13T14:08:15.277 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-13T14:08:15.417 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-09-13T14:08:15.579 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-09-13T14:08:15.829 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-09-13T14:08:16.095 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-09-13T14:08:16.214 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-13T14:08:16.362 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-09-13T14:08:16.488 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-09-13T14:08:16.498 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-13T14:08:16.614 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-13T14:08:16.817 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-09-13T14:08:16.915 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-09-13T14:08:17.051 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-09-13T14:08:17.166 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-09-13T14:08:17.177 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-ceph-common (17.2.7-1jammy) ... 2024-09-13T14:08:17.470 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-09-13T14:08:17.543 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-13T14:08:17.804 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-13T14:08:17.836 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-13T14:08:18.188 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-09-13T14:08:18.447 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-09-13T14:08:18.573 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-ceph-common (17.2.7-1jammy) ... 2024-09-13T14:08:18.874 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-09-13T14:08:19.010 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:08:19.144 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-toml (0.10.2-1) ... 2024-09-13T14:08:19.192 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-13T14:08:19.403 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-13T14:08:19.529 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-09-13T14:08:19.655 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-13T14:08:19.948 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-09-13T14:08:20.182 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-09-13T14:08:20.348 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:08:20.408 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:08:20.482 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-toml (0.10.2-1) ... 2024-09-13T14:08:20.534 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-13T14:08:20.733 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-13T14:08:20.775 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-09-13T14:08:20.851 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-09-13T14:08:20.977 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-13T14:08:21.104 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-09-13T14:08:21.303 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-09-13T14:08:21.355 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-09-13T14:08:21.563 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-09-13T14:08:21.614 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-13T14:08:21.723 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-09-13T14:08:21.822 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:08:21.948 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-13T14:08:21.975 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-09-13T14:08:22.074 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-09-13T14:08:22.200 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-09-13T14:08:22.376 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-09-13T14:08:22.460 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-13T14:08:22.585 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-09-13T14:08:22.635 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-09-13T14:08:22.879 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-13T14:08:22.903 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-13T14:08:23.004 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-13T14:08:23.029 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-09-13T14:08:23.288 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-09-13T14:08:23.289 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-13T14:08:23.548 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-09-13T14:08:23.581 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-09-13T14:08:23.807 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-13T14:08:23.816 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up librados2 (17.2.7-1jammy) ... 2024-09-13T14:08:23.917 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-09-13T14:08:23.917 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-09-13T14:08:24.185 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-13T14:08:24.302 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-13T14:08:24.324 INFO:teuthology.orchestra.run.smithi177.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-09-13T14:08:24.569 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-13T14:08:24.603 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-13T14:08:24.695 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-13T14:08:24.821 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-09-13T14:08:24.905 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-09-13T14:08:25.156 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up librados2 (17.2.7-1jammy) ... 2024-09-13T14:08:25.223 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-09-13T14:08:25.282 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-09-13T14:08:25.482 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-portend (3.0.0-1) ... 2024-09-13T14:08:25.681 INFO:teuthology.orchestra.run.smithi099.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-09-13T14:08:25.725 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:08:25.826 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libcephfs2 (17.2.7-1jammy) ... 2024-09-13T14:08:25.951 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-13T14:08:25.952 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-13T14:08:26.077 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-13T14:08:26.204 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-09-13T14:08:26.278 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libradosstriper1 (17.2.7-1jammy) ... 2024-09-13T14:08:26.420 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-09-13T14:08:26.588 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-09-13T14:08:26.747 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-13T14:08:26.848 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-portend (3.0.0-1) ... 2024-09-13T14:08:27.023 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up librbd1 (17.2.7-1jammy) ... 2024-09-13T14:08:27.099 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T14:08:27.140 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-09-13T14:08:27.216 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libcephfs2 (17.2.7-1jammy) ... 2024-09-13T14:08:27.241 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-rsa (4.8-1) ... 2024-09-13T14:08:27.351 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-13T14:08:27.492 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-09-13T14:08:27.660 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libradosstriper1 (17.2.7-1jammy) ... 2024-09-13T14:08:27.743 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-fuse (17.2.7-1jammy) ... 2024-09-13T14:08:27.786 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-09-13T14:08:27.955 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-13T14:08:27.955 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-13T14:08:28.088 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-13T14:08:28.364 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up librbd1 (17.2.7-1jammy) ... 2024-09-13T14:08:28.445 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libcephfs-dev (17.2.7-1jammy) ... 2024-09-13T14:08:28.493 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-09-13T14:08:28.570 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-13T14:08:28.616 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-rsa (4.8-1) ... 2024-09-13T14:08:28.838 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-09-13T14:08:28.892 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-09-13T14:08:29.151 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-fuse (17.2.7-1jammy) ... 2024-09-13T14:08:29.223 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-09-13T14:08:29.354 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-13T14:08:29.354 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-13T14:08:29.516 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-09-13T14:08:29.844 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libcephfs-dev (17.2.7-1jammy) ... 2024-09-13T14:08:29.929 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-13T14:08:29.970 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-13T14:08:30.055 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-rados (17.2.7-1jammy) ... 2024-09-13T14:08:30.206 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up librgw2 (17.2.7-1jammy) ... 2024-09-13T14:08:30.238 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-09-13T14:08:30.332 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-rbd (17.2.7-1jammy) ... 2024-09-13T14:08:30.458 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up rbd-fuse (17.2.7-1jammy) ... 2024-09-13T14:08:30.575 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-09-13T14:08:30.606 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-09-13T14:08:30.835 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-rgw (17.2.7-1jammy) ... 2024-09-13T14:08:30.899 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-09-13T14:08:31.228 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-cephfs (17.2.7-1jammy) ... 2024-09-13T14:08:31.295 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-13T14:08:31.354 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-13T14:08:31.421 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-rados (17.2.7-1jammy) ... 2024-09-13T14:08:31.548 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up librgw2 (17.2.7-1jammy) ... 2024-09-13T14:08:31.649 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-rbd (17.2.7-1jammy) ... 2024-09-13T14:08:31.750 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up rbd-fuse (17.2.7-1jammy) ... 2024-09-13T14:08:31.868 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-09-13T14:08:32.153 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-rgw (17.2.7-1jammy) ... 2024-09-13T14:08:32.279 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-cephfs (17.2.7-1jammy) ... 2024-09-13T14:08:32.430 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-13T14:08:32.574 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-common (17.2.7-1jammy) ... 2024-09-13T14:08:33.059 INFO:teuthology.orchestra.run.smithi177.stdout:Adding group ceph....done 2024-09-13T14:08:33.452 INFO:teuthology.orchestra.run.smithi177.stdout:Adding system user ceph....done 2024-09-13T14:08:33.621 INFO:teuthology.orchestra.run.smithi177.stdout:Setting system user ceph properties....done 2024-09-13T14:08:33.639 INFO:teuthology.orchestra.run.smithi177.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-09-13T14:08:33.642 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-common (17.2.7-1jammy) ... 2024-09-13T14:08:33.772 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-09-13T14:08:34.072 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-09-13T14:08:34.169 INFO:teuthology.orchestra.run.smithi099.stdout:Adding group ceph....done 2024-09-13T14:08:34.545 INFO:teuthology.orchestra.run.smithi099.stdout:Adding system user ceph....done 2024-09-13T14:08:34.576 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-test (17.2.7-1jammy) ... 2024-09-13T14:08:34.702 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up radosgw (17.2.7-1jammy) ... 2024-09-13T14:08:34.823 INFO:teuthology.orchestra.run.smithi099.stdout:Setting system user ceph properties....done 2024-09-13T14:08:34.844 INFO:teuthology.orchestra.run.smithi099.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-09-13T14:08:34.978 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-09-13T14:08:35.293 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-09-13T14:08:35.308 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:08:35.308 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:08:35.761 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-test (17.2.7-1jammy) ... 2024-09-13T14:08:35.887 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up radosgw (17.2.7-1jammy) ... 2024-09-13T14:08:35.896 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-base (17.2.7-1jammy) ... 2024-09-13T14:08:36.139 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:08:36.139 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:08:36.246 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-09-13T14:08:36.723 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-mds (17.2.7-1jammy) ... 2024-09-13T14:08:36.763 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-base (17.2.7-1jammy) ... 2024-09-13T14:08:36.942 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-13T14:08:36.942 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-13T14:08:37.084 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-09-13T14:08:37.433 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-mgr (17.2.7-1jammy) ... 2024-09-13T14:08:37.641 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-13T14:08:37.641 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-13T14:08:37.774 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-mds (17.2.7-1jammy) ... 2024-09-13T14:08:37.976 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-13T14:08:37.976 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-13T14:08:38.100 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-osd (17.2.7-1jammy) ... 2024-09-13T14:08:38.372 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-13T14:08:38.372 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-13T14:08:38.434 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-mgr (17.2.7-1jammy) ... 2024-09-13T14:08:38.641 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-13T14:08:38.641 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-13T14:08:38.869 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-13T14:08:38.995 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-13T14:08:39.110 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-osd (17.2.7-1jammy) ... 2024-09-13T14:08:39.146 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-mon (17.2.7-1jammy) ... 2024-09-13T14:08:39.356 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-13T14:08:39.356 INFO:teuthology.orchestra.run.smithi177.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-13T14:08:39.375 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-13T14:08:39.375 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-13T14:08:39.805 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-13T14:08:39.820 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-13T14:08:39.940 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph (17.2.7-1jammy) ... 2024-09-13T14:08:39.946 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-13T14:08:40.066 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-13T14:08:40.192 INFO:teuthology.orchestra.run.smithi177.stdout:Setting up ceph-volume (17.2.7-1jammy) ... 2024-09-13T14:08:40.214 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-mon (17.2.7-1jammy) ... 2024-09-13T14:08:40.293 INFO:teuthology.orchestra.run.smithi177.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-13T14:08:40.408 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-13T14:08:40.408 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-13T14:08:40.866 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-13T14:08:41.009 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph (17.2.7-1jammy) ... 2024-09-13T14:08:41.134 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-13T14:08:41.285 INFO:teuthology.orchestra.run.smithi099.stdout:Setting up ceph-volume (17.2.7-1jammy) ... 2024-09-13T14:08:41.302 INFO:teuthology.orchestra.run.smithi177.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-09-13T14:08:41.413 INFO:teuthology.orchestra.run.smithi099.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-13T14:08:41.828 INFO:teuthology.orchestra.run.smithi177.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-13T14:08:41.834 DEBUG:teuthology.parallel:result is None 2024-09-13T14:08:42.581 INFO:teuthology.orchestra.run.smithi099.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-09-13T14:08:43.073 INFO:teuthology.orchestra.run.smithi099.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-13T14:08:43.077 DEBUG:teuthology.parallel:result is None 2024-09-13T14:08:43.078 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:08:43.078 INFO:teuthology.packaging:ref: None 2024-09-13T14:08:43.079 INFO:teuthology.packaging:tag: None 2024-09-13T14:08:43.079 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:08:43.079 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:08:43.079 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-13T14:08:43.233 DEBUG:teuthology.orchestra.run.smithi099:> dpkg-query -W -f '${Version}' ceph 2024-09-13T14:08:43.251 INFO:teuthology.orchestra.run.smithi099.stdout:17.2.7-1jammy 2024-09-13T14:08:43.252 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1jammy 2024-09-13T14:08:43.252 INFO:teuthology.task.install:The correct ceph version 17.2.7-1jammy is installed. 2024-09-13T14:08:43.254 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:08:43.255 INFO:teuthology.packaging:ref: None 2024-09-13T14:08:43.255 INFO:teuthology.packaging:tag: None 2024-09-13T14:08:43.255 INFO:teuthology.packaging:branch: quincy 2024-09-13T14:08:43.255 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:08:43.255 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-13T14:08:43.404 DEBUG:teuthology.orchestra.run.smithi177:> dpkg-query -W -f '${Version}' ceph 2024-09-13T14:08:43.423 INFO:teuthology.orchestra.run.smithi177.stdout:17.2.7-1jammy 2024-09-13T14:08:43.423 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1jammy 2024-09-13T14:08:43.423 INFO:teuthology.task.install:The correct ceph version 17.2.7-1jammy is installed. 2024-09-13T14:08:43.425 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-13T14:08:43.425 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:08:43.425 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T14:08:43.439 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:08:43.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T14:08:43.479 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-13T14:08:43.479 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:08:43.479 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T14:08:43.493 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T14:08:43.549 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:08:43.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T14:08:43.562 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T14:08:43.619 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-13T14:08:43.620 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:08:43.620 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T14:08:43.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T14:08:43.689 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:08:43.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T14:08:43.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T14:08:43.758 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-13T14:08:43.758 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:08:43.759 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T14:08:43.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T14:08:43.825 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:08:43.825 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T14:08:43.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T14:08:43.893 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T14:08:43.992 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'} 2024-09-13T14:08:43.992 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:08:43.993 INFO:tasks.cephadm:Cluster fsid is af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:08:43.993 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T14:08:43.993 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.99', 'mon.c': '[v2:172.21.15.99:3301,v1:172.21.15.99:6790]', 'mon.b': '172.21.15.177'} 2024-09-13T14:08:43.993 INFO:tasks.cephadm:First mon is mon.a on smithi099 2024-09-13T14:08:43.993 INFO:tasks.cephadm:First mgr is y 2024-09-13T14:08:43.993 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T14:08:43.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo hostname $(hostname -s) 2024-09-13T14:08:44.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo hostname $(hostname -s) 2024-09-13T14:08:44.021 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-13T14:08:44.022 DEBUG:teuthology.orchestra.run.smithi099:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:44.282 INFO:teuthology.orchestra.run.smithi099.stdout:-rw-rw-r-- 1 ubuntu ubuntu 378585 Sep 13 14:08 /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:44.282 DEBUG:teuthology.orchestra.run.smithi177:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:44.512 INFO:teuthology.orchestra.run.smithi177.stdout:-rw-rw-r-- 1 ubuntu ubuntu 378585 Sep 13 14:08 /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:44.512 DEBUG:teuthology.orchestra.run.smithi099:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:44.519 DEBUG:teuthology.orchestra.run.smithi177:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T14:08:44.536 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-13T14:08:44.537 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-13T14:08:44.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-13T14:08:44.729 INFO:teuthology.orchestra.run.smithi099.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:08:44.737 INFO:teuthology.orchestra.run.smithi177.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:09:59.943 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:09:59.943 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-13T14:09:59.943 INFO:teuthology.orchestra.run.smithi099.stdout: "image_id": "a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c", 2024-09-13T14:09:59.943 INFO:teuthology.orchestra.run.smithi099.stdout: "repo_digests": [ 2024-09-13T14:09:59.943 INFO:teuthology.orchestra.run.smithi099.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be" 2024-09-13T14:09:59.943 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:09:59.943 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:10:07.829 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T14:10:07.829 INFO:teuthology.orchestra.run.smithi177.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-13T14:10:07.829 INFO:teuthology.orchestra.run.smithi177.stdout: "image_id": "a994b00deb3a4a8eeb484a456d3f3207aae0401a31d251b199b2c9dbc1c4036c", 2024-09-13T14:10:07.829 INFO:teuthology.orchestra.run.smithi177.stdout: "repo_digests": [ 2024-09-13T14:10:07.829 INFO:teuthology.orchestra.run.smithi177.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be" 2024-09-13T14:10:07.829 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:10:07.829 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T14:10:07.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo mkdir -p /etc/ceph 2024-09-13T14:10:07.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo mkdir -p /etc/ceph 2024-09-13T14:10:07.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 777 /etc/ceph 2024-09-13T14:10:07.937 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod 777 /etc/ceph 2024-09-13T14:10:07.952 INFO:tasks.cephadm:Writing seed config... 2024-09-13T14:10:07.953 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-13T14:10:07.953 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-13T14:10:07.953 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T14:10:07.953 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T14:10:07.953 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-13T14:10:07.953 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-09-13T14:10:07.953 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T14:10:07.954 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T14:10:07.954 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T14:10:07.954 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T14:10:07.954 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T14:10:07.954 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:10:07.954 DEBUG:teuthology.orchestra.run.smithi099:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T14:10:07.985 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = af9d2c22-71d9-11ef-bceb-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T14:10:07.985 DEBUG:teuthology.orchestra.run.smithi099:mon.a> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.a.service 2024-09-13T14:10:08.031 DEBUG:teuthology.orchestra.run.smithi099:mgr.y> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.y.service 2024-09-13T14:10:08.077 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T14:10:08.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.99 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:10:08.296 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:10:08.296 INFO:teuthology.orchestra.run.smithi099.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'af9d2c22-71d9-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.99', '--skip-admin-label'] 2024-09-13T14:10:08.296 INFO:teuthology.orchestra.run.smithi099.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-13T14:10:08.297 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying podman|docker is present... 2024-09-13T14:10:08.297 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying lvm2 is present... 2024-09-13T14:10:08.297 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying time synchronization is in place... 2024-09-13T14:10:08.302 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T14:10:08.302 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T14:10:08.305 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T14:10:08.305 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-09-13T14:10:08.309 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-09-13T14:10:08.309 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-09-13T14:10:08.312 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-09-13T14:10:08.313 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-09-13T14:10:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-09-13T14:10:08.317 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout masked 2024-09-13T14:10:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-09-13T14:10:08.321 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-09-13T14:10:08.325 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-09-13T14:10:08.325 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-09-13T14:10:08.327 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-09-13T14:10:08.328 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-09-13T14:10:08.331 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout enabled 2024-09-13T14:10:08.335 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout active 2024-09-13T14:10:08.335 INFO:teuthology.orchestra.run.smithi099.stdout:Unit ntp.service is enabled and running 2024-09-13T14:10:08.335 INFO:teuthology.orchestra.run.smithi099.stdout:Repeating the final host check... 2024-09-13T14:10:08.335 INFO:teuthology.orchestra.run.smithi099.stdout:docker (/usr/bin/docker) is present 2024-09-13T14:10:08.336 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl is present 2024-09-13T14:10:08.336 INFO:teuthology.orchestra.run.smithi099.stdout:lvcreate is present 2024-09-13T14:10:08.339 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T14:10:08.339 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T14:10:08.344 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T14:10:08.344 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-09-13T14:10:08.347 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-09-13T14:10:08.347 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-09-13T14:10:08.352 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-09-13T14:10:08.352 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-09-13T14:10:08.355 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-09-13T14:10:08.355 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout masked 2024-09-13T14:10:08.359 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-09-13T14:10:08.359 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-09-13T14:10:08.362 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-09-13T14:10:08.362 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-09-13T14:10:08.367 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-09-13T14:10:08.367 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout inactive 2024-09-13T14:10:08.372 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout enabled 2024-09-13T14:10:08.376 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stdout active 2024-09-13T14:10:08.376 INFO:teuthology.orchestra.run.smithi099.stdout:Unit ntp.service is enabled and running 2024-09-13T14:10:08.376 INFO:teuthology.orchestra.run.smithi099.stdout:Host looks OK 2024-09-13T14:10:08.376 INFO:teuthology.orchestra.run.smithi099.stdout:Cluster fsid: af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:08.376 INFO:teuthology.orchestra.run.smithi099.stdout:Acquiring lock 140375662331216 on /run/cephadm/af9d2c22-71d9-11ef-bceb-c7b262605968.lock 2024-09-13T14:10:08.377 INFO:teuthology.orchestra.run.smithi099.stdout:Lock 140375662331216 acquired on /run/cephadm/af9d2c22-71d9-11ef-bceb-c7b262605968.lock 2024-09-13T14:10:08.377 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying IP 172.21.15.99 port 3300 ... 2024-09-13T14:10:08.377 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying IP 172.21.15.99 port 6789 ... 2024-09-13T14:10:08.377 INFO:teuthology.orchestra.run.smithi099.stdout:Base mon IP(s) is [172.21.15.99:3300, 172.21.15.99:6789], mon addrv is [v2:172.21.15.99:3300,v1:172.21.15.99:6789] 2024-09-13T14:10:08.379 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 2024-09-13T14:10:08.379 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.99 metric 100 2024-09-13T14:10:08.380 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout 158.69.67.47 via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.99 metric 100 2024-09-13T14:10:08.380 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2024-09-13T14:10:08.380 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.99 metric 100 2024-09-13T14:10:08.380 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout 172.21.0.1 dev enp3s0f1 proto dhcp scope link src 172.21.15.99 metric 100 2024-09-13T14:10:08.380 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout 172.21.0.2 dev enp3s0f1 proto dhcp scope link src 172.21.15.99 metric 100 2024-09-13T14:10:08.380 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.99 metric 100 2024-09-13T14:10:08.381 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T14:10:08.381 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-09-13T14:10:08.381 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1779sec pref medium 2024-09-13T14:10:08.383 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T14:10:08.383 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-13T14:10:08.383 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T14:10:08.384 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-13T14:10:08.384 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d803/64 scope link 2024-09-13T14:10:08.384 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T14:10:08.385 INFO:teuthology.orchestra.run.smithi099.stdout:Mon IP `172.21.15.99` is in CIDR network `172.21.0.0/20` 2024-09-13T14:10:08.385 INFO:teuthology.orchestra.run.smithi099.stdout:Mon IP `172.21.15.99` is in CIDR network `172.21.0.0/20` 2024-09-13T14:10:08.385 INFO:teuthology.orchestra.run.smithi099.stdout:Mon IP `172.21.15.99` is in CIDR network `172.21.0.1/32` 2024-09-13T14:10:08.385 INFO:teuthology.orchestra.run.smithi099.stdout:Mon IP `172.21.15.99` is in CIDR network `172.21.0.1/32` 2024-09-13T14:10:08.385 INFO:teuthology.orchestra.run.smithi099.stdout:Mon IP `172.21.15.99` is in CIDR network `172.21.0.2/32` 2024-09-13T14:10:08.385 INFO:teuthology.orchestra.run.smithi099.stdout:Mon IP `172.21.15.99` is in CIDR network `172.21.0.2/32` 2024-09-13T14:10:08.385 INFO:teuthology.orchestra.run.smithi099.stdout:Mon IP `172.21.15.99` is in CIDR network `172.21.15.254/32` 2024-09-13T14:10:08.385 INFO:teuthology.orchestra.run.smithi099.stdout:Mon IP `172.21.15.99` is in CIDR network `172.21.15.254/32` 2024-09-13T14:10:08.385 INFO:teuthology.orchestra.run.smithi099.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.0.1/32', '172.21.0.1/32', '172.21.0.2/32', '172.21.0.2/32', '172.21.15.254/32', '172.21.15.254/32'] 2024-09-13T14:10:08.385 INFO:teuthology.orchestra.run.smithi099.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T14:10:08.386 INFO:teuthology.orchestra.run.smithi099.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T14:10:08.615 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/docker: stdout quincy: Pulling from ceph-ci/ceph 2024-09-13T14:10:08.615 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/docker: stdout Digest: sha256:20745bd452f31fe48c67c092910f9b2c550f9dc62d239ff3e2953415a85590be 2024-09-13T14:10:08.615 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/docker: stdout Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:10:08.615 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/docker: stdout quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T14:10:10.211 INFO:teuthology.orchestra.run.smithi099.stdout:ceph: stdout ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T14:10:10.211 INFO:teuthology.orchestra.run.smithi099.stdout:Ceph version: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T14:10:10.211 INFO:teuthology.orchestra.run.smithi099.stdout:Extracting ceph user uid/gid from container image... 2024-09-13T14:10:11.583 INFO:teuthology.orchestra.run.smithi099.stdout:stat: stdout 167 167 2024-09-13T14:10:11.583 INFO:teuthology.orchestra.run.smithi099.stdout:Creating initial keys... 2024-09-13T14:10:12.930 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-authtool: stdout AQDER+Rm3ckyFxAACjquRy6rqK3ri+n3aRvpuw== 2024-09-13T14:10:14.394 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-authtool: stdout AQDFR+RmXWexMhAA6WCgvw8Ccrqde9tXrtqMOQ== 2024-09-13T14:10:15.799 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-authtool: stdout AQDHR+RmpY7GDxAAZBm/W7LsB+SpoozPmrZu5w== 2024-09-13T14:10:15.800 INFO:teuthology.orchestra.run.smithi099.stdout:Creating initial monmap... 2024-09-13T14:10:17.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T14:10:17.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-13T14:10:17.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:17.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T14:10:17.289 INFO:teuthology.orchestra.run.smithi099.stdout:monmaptool for a [v2:172.21.15.99:3300,v1:172.21.15.99:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T14:10:17.289 INFO:teuthology.orchestra.run.smithi099.stdout:setting min_mon_release = octopus 2024-09-13T14:10:17.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: set fsid to af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:17.289 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T14:10:17.289 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:10:17.289 INFO:teuthology.orchestra.run.smithi099.stdout:Creating mon... 2024-09-13T14:10:18.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.138+0000 7f12f995d900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:10:18.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.138+0000 7f12f995d900 1 imported monmap: 2024-09-13T14:10:18.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr epoch 0 2024-09-13T14:10:18.760 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr last_changed 2024-09-13T14:10:16.718964+0000 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr min_mon_release 15 (octopus) 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr election_strategy: 1 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.138+0000 7f12f995d900 0 /usr/bin/ceph-mon: set fsid to af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: RocksDB version: 6.15.5 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Compile date Sep 12 2024 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: DB SUMMARY 2024-09-13T14:10:18.761 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: DB Session ID: ZOYAVO8HN5SVSN0R6DNT 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.error_if_exists: 0 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.create_if_missing: 1 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.env: 0x561bee87d440 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.fs: Posix File System 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.info_log: 0x561bf0104660 2024-09-13T14:10:18.762 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.statistics: (nil) 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.use_fsync: 0 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:10:18.763 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.db_log_dir: 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.write_buffer_manager: 0x561bf014de90 2024-09-13T14:10:18.764 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:10:18.765 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:10:18.766 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:10:18.766 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:10:18.766 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:10:18.766 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:10:18.766 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:10:18.766 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:10:18.766 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.unordered_write: 0 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.row_cache: None 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.wal_filter: None 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.preserve_deletes: 0 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.two_write_queues: 0 2024-09-13T14:10:18.767 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:10:18.768 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.atomic_flush: 0 2024-09-13T14:10:18.768 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:10:18.768 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:10:18.768 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:10:18.768 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T14:10:18.768 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:10:18.768 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:10:18.768 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:10:18.768 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:10:18.769 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_open_files: -1 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Compression algorithms supported: 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: kZSTD supported: 0 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: kXpressCompression supported: 0 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T14:10:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: kZlibCompression supported: 1 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: [db/db_impl/db_impl_open.cc:281] Creating manifest 1 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.142+0000 7f12f995d900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.merge_operator: 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_filter: None 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:10:18.771 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561bf00a4ee0) 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks: 1 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr pin_top_level_index_and_filter: 1 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr index_type: 0 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr data_block_index_type: 0 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr index_shortening: 1 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr hash_index_allow_collision: 1 2024-09-13T14:10:18.772 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr checksum: 1 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr no_block_cache: 0 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr block_cache: 0x561bf00dd1f0 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr block_cache_name: BinnedLRUCache 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr block_cache_options: 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr capacity : 536870912 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr num_shard_bits : 4 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr strict_capacity_limit : 0 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr high_pri_pool_ratio: 0.000 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr block_cache_compressed: (nil) 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr persistent_cache: (nil) 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr block_size: 4096 2024-09-13T14:10:18.773 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr block_size_deviation: 10 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr block_restart_interval: 16 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr index_block_restart_interval: 1 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr metadata_block_size: 4096 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr partition_filters: 0 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr use_delta_encoding: 1 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr whole_key_filtering: 1 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr verify_compression: 0 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr read_amp_bytes_per_bit: 0 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr format_version: 4 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr enable_index_compression: 1 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr block_align: 0 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.774 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compression: NoCompression 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.num_levels: 7 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:10:18.775 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:10:18.776 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:10:18.777 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:10:18.777 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:10:18.777 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:10:18.777 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:10:18.777 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:10:18.777 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:10:18.777 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:10:18.777 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:10:18.777 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:10:18.777 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:10:18.778 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:10:18.778 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:10:18.778 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:10:18.778 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:10:18.778 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:10:18.778 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:10:18.778 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:10:18.778 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:10:18.778 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:10:18.778 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:10:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.bloom_locality: 0 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.ttl: 2592000 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.enable_blob_files: false 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.min_blob_size: 0 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:10:18.780 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 3 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x561bf010e700 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: DB pointer 0x561bf00e0000 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f10eb640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f10eb640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:10:18.781 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ** DB Stats ** 2024-09-13T14:10:18.782 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:10:18.782 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:10:18.782 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:10:18.782 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:10:18.787 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:10:18.788 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-13T14:10:18.789 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:10:18.790 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:10:18.791 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:10:18.791 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:10:18.791 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.791 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:10:18.791 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T14:10:18.791 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.146+0000 7f12f995d900 4 rocksdb: [db/db_impl/db_impl.cc:446] Shutdown: canceling all background work 2024-09-13T14:10:18.791 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.150+0000 7f12f995d900 4 rocksdb: [db/db_impl/db_impl.cc:625] Shutdown complete 2024-09-13T14:10:18.791 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T14:10:18.150+0000 7f12f995d900 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2024-09-13T14:10:18.791 INFO:teuthology.orchestra.run.smithi099.stdout:create mon.a on 2024-09-13T14:10:19.013 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2024-09-13T14:10:19.210 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2024-09-13T14:10:19.427 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968.target -> /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968.target. 2024-09-13T14:10:19.427 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968.target -> /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968.target. 2024-09-13T14:10:19.654 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.a 2024-09-13T14:10:19.654 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to reset failed state of unit ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-13T14:10:19.851 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968.target.wants/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.a.service -> /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service. 2024-09-13T14:10:19.855 INFO:teuthology.orchestra.run.smithi099.stdout:firewalld does not appear to be present 2024-09-13T14:10:19.855 INFO:teuthology.orchestra.run.smithi099.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T14:10:19.855 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mon to start... 2024-09-13T14:10:19.856 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mon... 2024-09-13T14:10:20.001 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:19 smithi099 systemd[1]: Started Ceph mon.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:10:21.544 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:21 smithi099 bash[22171]: cluster 2024-09-13T14:10:21.364047+0000 mon.a (mon.0) 0 : cluster [INF] mkfs af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:21.544 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:21 smithi099 bash[22171]: cluster 2024-09-13T14:10:21.358378+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:10:22.107 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout cluster: 2024-09-13T14:10:22.107 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout id: af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:22.107 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-13T14:10:22.107 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:22.107 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout services: 2024-09-13T14:10:22.107 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.171973s) 2024-09-13T14:10:22.107 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-13T14:10:22.107 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-13T14:10:22.108 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:22.108 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout data: 2024-09-13T14:10:22.108 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-13T14:10:22.108 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-13T14:10:22.108 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-13T14:10:22.108 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout pgs: 2024-09-13T14:10:22.108 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:22.108 INFO:teuthology.orchestra.run.smithi099.stdout:mon is available 2024-09-13T14:10:22.108 INFO:teuthology.orchestra.run.smithi099.stdout:Assimilating anything we can from ceph.conf... 2024-09-13T14:10:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:22 smithi099 bash[22171]: cluster 2024-09-13T14:10:21.363149+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:10:22.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:22 smithi099 bash[22171]: cluster 2024-09-13T14:10:21.363319+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:10:22.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:22 smithi099 bash[22171]: cluster 2024-09-13T14:10:21.365384+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2024-09-13T14:10:22.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:22 smithi099 bash[22171]: cluster 2024-09-13T14:10:21.367576+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-09-13T14:10:22.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:22 smithi099 bash[22171]: cluster 2024-09-13T14:10:21.367983+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2024-09-13T14:10:22.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:22 smithi099 bash[22171]: audit 2024-09-13T14:10:21.535092+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.99:0/507132356' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T14:10:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:23 smithi099 bash[22171]: audit 2024-09-13T14:10:23.327322+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.99:0/3853616090' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T14:10:23.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:23 smithi099 bash[22171]: audit 2024-09-13T14:10:23.333596+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.99:0/3853616090' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout fsid = af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.99:3300,v1:172.21.15.99:6789] 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T14:10:23.838 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T14:10:23.839 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:23.839 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T14:10:23.839 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T14:10:23.839 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T14:10:23.839 INFO:teuthology.orchestra.run.smithi099.stdout:Generating new minimal ceph.conf... 2024-09-13T14:10:25.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:25 smithi099 bash[22171]: audit 2024-09-13T14:10:24.967936+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.99:0/1232548818' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:10:25.543 INFO:teuthology.orchestra.run.smithi099.stdout:Restarting the monitor... 2024-09-13T14:10:25.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:25 smithi099 systemd[1]: Stopping Ceph mon.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:10:25.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:25 smithi099 bash[22171]: debug 2024-09-13T14:10:25.682+0000 7f7f88d88640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T14:10:25.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:25 smithi099 bash[22171]: debug 2024-09-13T14:10:25.682+0000 7f7f88d88640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T14:10:26.477 INFO:teuthology.orchestra.run.smithi099.stdout:Setting public_network to 172.21.15.254/32,172.21.0.2/32,172.21.0.0/20,172.21.0.1/32 in mon config section 2024-09-13T14:10:26.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:26 smithi099 bash[22552]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:10:26.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:26 smithi099 bash[22615]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:10:26.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:26 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T14:10:26.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:26 smithi099 systemd[1]: Stopped Ceph mon.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:10:26.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:26 smithi099 systemd[1]: Started Ceph mon.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:10:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.898+0000 7fea5162e900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:10:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.898+0000 7fea5162e900 0 ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T14:10:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.898+0000 7fea5162e900 0 pidfile_write: ignore empty --pid-file 2024-09-13T14:10:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 0 load: jerasure load: lrc 2024-09-13T14:10:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: RocksDB version: 6.15.5 2024-09-13T14:10:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:10:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Compile date Sep 12 2024 2024-09-13T14:10:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: DB SUMMARY 2024-09-13T14:10:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: DB Session ID: F7SRCW92B00IGL3O0QLY 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: CURRENT file: CURRENT 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83675 ; 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.error_if_exists: 0 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.create_if_missing: 0 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.env: 0x563fc5824440 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.fs: Posix File System 2024-09-13T14:10:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.info_log: 0x563fc6582ee0 2024-09-13T14:10:28.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:10:28.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.statistics: (nil) 2024-09-13T14:10:28.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.use_fsync: 0 2024-09-13T14:10:28.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T14:10:28.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:10:28.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.db_log_dir: 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:10:28.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.write_buffer_manager: 0x563fc66407e0 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:10:28.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.unordered_write: 0 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.row_cache: None 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.wal_filter: None 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.preserve_deletes: 0 2024-09-13T14:10:28.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.two_write_queues: 0 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.atomic_flush: 0 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:10:28.079 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:10:28.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:10:28.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T14:10:28.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T14:10:28.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T14:10:28.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:10:28.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:10:28.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:10:28.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_open_files: -1 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Compression algorithms supported: 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: kZSTD supported: 0 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: kXpressCompression supported: 0 2024-09-13T14:10:28.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T14:10:28.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:10:28.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T14:10:28.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: kZlibCompression supported: 1 2024-09-13T14:10:28.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:10:28.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T14:10:28.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:10:28.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-13T14:10:28.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:10:28.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:10:28.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.merge_operator: 2024-09-13T14:10:28.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_filter: None 2024-09-13T14:10:28.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:10:28.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:10:28.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:10:28.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:10:28.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563fc652ba38) 2024-09-13T14:10:28.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: cache_index_and_filter_blocks: 1 2024-09-13T14:10:28.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:10:28.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:10:28.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: pin_top_level_index_and_filter: 1 2024-09-13T14:10:28.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: index_type: 0 2024-09-13T14:10:28.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: data_block_index_type: 0 2024-09-13T14:10:28.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: index_shortening: 1 2024-09-13T14:10:28.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:10:28.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: hash_index_allow_collision: 1 2024-09-13T14:10:28.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: checksum: 1 2024-09-13T14:10:28.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: no_block_cache: 0 2024-09-13T14:10:28.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: block_cache: 0x563fc655f350 2024-09-13T14:10:28.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: block_cache_name: BinnedLRUCache 2024-09-13T14:10:28.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: block_cache_options: 2024-09-13T14:10:28.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: capacity : 536870912 2024-09-13T14:10:28.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: num_shard_bits : 4 2024-09-13T14:10:28.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: strict_capacity_limit : 0 2024-09-13T14:10:28.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: high_pri_pool_ratio: 0.000 2024-09-13T14:10:28.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: block_cache_compressed: (nil) 2024-09-13T14:10:28.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: persistent_cache: (nil) 2024-09-13T14:10:28.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: block_size: 4096 2024-09-13T14:10:28.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: block_size_deviation: 10 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: block_restart_interval: 16 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: index_block_restart_interval: 1 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: metadata_block_size: 4096 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: partition_filters: 0 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: use_delta_encoding: 1 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: whole_key_filtering: 1 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: verify_compression: 0 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: read_amp_bytes_per_bit: 0 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: format_version: 4 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: enable_index_compression: 1 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: block_align: 0 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:10:28.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compression: NoCompression 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.num_levels: 7 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:10:28.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:10:28.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:10:28.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:10:28.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:10:28.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:10:28.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:10:28.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:10:28.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:10:28.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:10:28.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:10:28.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:10:28.092 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:10:28.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:10:28.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:10:28.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.bloom_locality: 0 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.ttl: 2592000 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.enable_blob_files: false 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.min_blob_size: 0 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:10:28.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.902+0000 7fea5162e900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea5162e900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726236627910157, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea5162e900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea5162e900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea5162e900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726236627911166, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80248, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78490, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72704, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726236627, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "9efea801-3da2-4bdd-8ce2-edd00894fe79", "db_session_id": "F7SRCW92B00IGL3O0QLY"}} 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea5162e900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea5162e900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726236627912309, "job": 1, "event": "recovery_finished"} 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea5162e900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea5162e900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x563fc6594700 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea5162e900 4 rocksdb: DB pointer 0x563fc6562000 2024-09-13T14:10:28.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea473fa640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.906+0000 7fea473fa640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ** DB Stats ** 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ** Compaction Stats [default] ** 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: L0 2/0 80.15 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 120.1 0.00 0.00 1 0.001 0 0 2024-09-13T14:10:28.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Sum 2/0 80.15 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 120.1 0.00 0.00 1 0.001 0 0 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 120.1 0.00 0.00 1 0.001 0 0 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ** Compaction Stats [default] ** 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 120.1 0.00 0.00 1 0.001 0 0 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Cumulative compaction: 0.00 GB write, 15.70 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:10:28.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Interval compaction: 0.00 GB write, 15.70 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ** Compaction Stats [default] ** 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: L0 2/0 80.15 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 120.1 0.00 0.00 1 0.001 0 0 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Sum 2/0 80.15 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 120.1 0.00 0.00 1 0.001 0 0 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ** Compaction Stats [default] ** 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 120.1 0.00 0.00 1 0.001 0 0 2024-09-13T14:10:28.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Cumulative compaction: 0.00 GB write, 15.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.910+0000 7fea5162e900 0 starting mon.a rank 0 at public addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] at bind addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.910+0000 7fea5162e900 1 mon.a@-1(???) e1 preinit fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.910+0000 7fea5162e900 0 mon.a@-1(???).mds e1 new map 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.910+0000 7fea5162e900 0 mon.a@-1(???).mds e1 print_map 2024-09-13T14:10:28.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: e1 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: legacy client fscid: -1 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: No filesystems configured 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.910+0000 7fea5162e900 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.910+0000 7fea5162e900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.910+0000 7fea5162e900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.910+0000 7fea5162e900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: debug 2024-09-13T14:10:27.910+0000 7fea5162e900 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: cluster 2024-09-13T14:10:27.942383+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: cluster 2024-09-13T14:10:27.942457+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:10:28.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: cluster 2024-09-13T14:10:27.942897+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2024-09-13T14:10:28.103 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: cluster 2024-09-13T14:10:27.942932+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-09-13T14:10:28.103 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:27 smithi099 bash[22656]: cluster 2024-09-13T14:10:27.943402+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2024-09-13T14:10:28.754 INFO:teuthology.orchestra.run.smithi099.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-13T14:10:28.754 INFO:teuthology.orchestra.run.smithi099.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:10:28.754 INFO:teuthology.orchestra.run.smithi099.stdout:Creating mgr... 2024-09-13T14:10:28.754 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-13T14:10:28.754 INFO:teuthology.orchestra.run.smithi099.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-13T14:10:28.986 INFO:teuthology.orchestra.run.smithi099.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.y 2024-09-13T14:10:28.987 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Failed to reset failed state of unit ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-13T14:10:29.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:10:29.186 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:10:29.186 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:29 smithi099 bash[22656]: audit 2024-09-13T14:10:28.122484+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.99:0/3950915463' entity='client.admin' 2024-09-13T14:10:29.186 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:10:29.187 INFO:teuthology.orchestra.run.smithi099.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968.target.wants/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.y.service -> /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service. 2024-09-13T14:10:29.196 INFO:teuthology.orchestra.run.smithi099.stdout:firewalld does not appear to be present 2024-09-13T14:10:29.196 INFO:teuthology.orchestra.run.smithi099.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T14:10:29.196 INFO:teuthology.orchestra.run.smithi099.stdout:firewalld does not appear to be present 2024-09-13T14:10:29.196 INFO:teuthology.orchestra.run.smithi099.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-13T14:10:29.196 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mgr to start... 2024-09-13T14:10:29.196 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mgr... 2024-09-13T14:10:29.572 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:29 smithi099 systemd[1]: Started Ceph mgr.y for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:10:30.954 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:30 smithi099 bash[22909]: debug 2024-09-13T14:10:30.822+0000 7f065942bec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:10:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:30 smithi099 bash[22656]: audit 2024-09-13T14:10:30.945031+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.99:0/1524216587' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:10:31.606 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:31.606 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:31.606 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsid": "af9d2c22-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:10:31.606 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:10:31.606 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:10:31.606 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:10:31.606 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:10:31.609 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:31.609 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:10:31.610 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:10:31.610 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:10:31.610 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:31.610 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:10:31.610 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:10:31.610 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:31.610 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_age": 3, 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:10:31.611 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:10:31.612 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:10:31.616 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:31.616 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:10:31.616 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:10:31.617 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:31.618 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:10:21.364420+0000", 2024-09-13T14:10:31.619 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:31.619 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:31.619 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:10:31.619 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:31.619 INFO:teuthology.orchestra.run.smithi099.stdout:mgr not available, waiting (1/15)... 2024-09-13T14:10:32.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:32 smithi099 bash[22909]: debug 2024-09-13T14:10:31.998+0000 7f065942bec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:10:32.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:32 smithi099 bash[22909]: debug 2024-09-13T14:10:32.062+0000 7f065942bec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:10:32.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:32 smithi099 bash[22909]: debug 2024-09-13T14:10:32.206+0000 7f065942bec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:10:33.114 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:32 smithi099 bash[22909]: debug 2024-09-13T14:10:32.826+0000 7f065942bec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:10:33.114 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:32 smithi099 bash[22909]: debug 2024-09-13T14:10:32.890+0000 7f065942bec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:10:33.114 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:32 smithi099 bash[22909]: debug 2024-09-13T14:10:32.958+0000 7f065942bec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:10:33.475 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:33 smithi099 bash[22909]: debug 2024-09-13T14:10:33.110+0000 7f065942bec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:10:33.475 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:33 smithi099 bash[22909]: debug 2024-09-13T14:10:33.182+0000 7f065942bec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:10:33.475 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:33 smithi099 bash[22909]: debug 2024-09-13T14:10:33.250+0000 7f065942bec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:10:33.475 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:33 smithi099 bash[22909]: debug 2024-09-13T14:10:33.470+0000 7f065942bec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:10:33.779 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:33 smithi099 bash[22909]: debug 2024-09-13T14:10:33.534+0000 7f065942bec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:10:33.779 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:33 smithi099 bash[22909]: debug 2024-09-13T14:10:33.702+0000 7f065942bec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:10:34.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:33 smithi099 bash[22909]: debug 2024-09-13T14:10:33.774+0000 7f065942bec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:10:34.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:34 smithi099 bash[22909]: debug 2024-09-13T14:10:34.002+0000 7f065942bec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:10:34.573 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:34 smithi099 bash[22909]: debug 2024-09-13T14:10:34.162+0000 7f065942bec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:10:34.573 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:34 smithi099 bash[22909]: debug 2024-09-13T14:10:34.238+0000 7f065942bec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:10:34.573 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:34 smithi099 bash[22909]: debug 2024-09-13T14:10:34.390+0000 7f065942bec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:10:35.231 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:34 smithi099 bash[22656]: audit 2024-09-13T14:10:34.913836+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.99:0/3951200406' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:10:35.231 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22909]: debug 2024-09-13T14:10:35.154+0000 7f065942bec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:10:35.496 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22909]: debug 2024-09-13T14:10:35.226+0000 7f065942bec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:10:35.496 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22909]: debug 2024-09-13T14:10:35.418+0000 7f065942bec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:10:35.496 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22909]: debug 2024-09-13T14:10:35.490+0000 7f065942bec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:10:35.530 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:35.530 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:35.530 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsid": "af9d2c22-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:10:35.530 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:10:35.530 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:10:35.532 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:10:35.532 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:10:35.532 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:35.532 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:10:35.532 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:10:35.532 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:10:35.532 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:35.532 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:10:35.533 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:10:35.533 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:35.533 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-09-13T14:10:35.533 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:10:35.533 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:35.533 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:10:35.534 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:10:35.534 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:35.534 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:10:35.535 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:35.535 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:10:35.535 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:10:35.535 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:10:35.535 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:10:35.535 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:10:35.535 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:10:35.538 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:35.538 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:10:35.538 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:10:35.538 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:10:35.538 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T14:10:35.539 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:10:21.364420+0000", 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:35.540 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:10:35.541 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:35.541 INFO:teuthology.orchestra.run.smithi099.stdout:mgr not available, waiting (2/15)... 2024-09-13T14:10:35.964 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22909]: debug 2024-09-13T14:10:35.802+0000 7f065942bec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:10:35.964 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22909]: debug 2024-09-13T14:10:35.878+0000 7f065942bec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:10:35.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: cluster 2024-09-13T14:10:35.885753+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2024-09-13T14:10:35.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: cluster 2024-09-13T14:10:35.892233+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00671368s) 2024-09-13T14:10:35.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.893754+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:10:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.893954+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:10:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.894104+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:10:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.894211+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:10:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.894294+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:10:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.894378+0000 mon.a (mon.0) 16 : audit [DBG] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:10:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.894497+0000 mon.a (mon.0) 17 : audit [DBG] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.895983+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:10:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: cluster 2024-09-13T14:10:35.919987+0000 mon.a (mon.0) 19 : cluster [INF] Manager daemon y is now available 2024-09-13T14:10:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.931245+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:10:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.933367+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:10:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.936505+0000 mon.a (mon.0) 22 : audit [INF] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' 2024-09-13T14:10:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.939063+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' 2024-09-13T14:10:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:35 smithi099 bash[22656]: audit 2024-09-13T14:10:35.941590+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.14100 172.21.15.99:0/2960786965' entity='mgr.y' 2024-09-13T14:10:38.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:37 smithi099 bash[22656]: cluster 2024-09-13T14:10:36.901248+0000 mon.a (mon.0) 25 : cluster [DBG] mgrmap e3: y(active, since 1.01573s) 2024-09-13T14:10:39.722 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:39.722 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:39.722 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsid": "af9d2c22-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:10:39.722 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:10:39.722 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:10:39.722 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:10:39.722 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:10:39.723 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:39.724 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:10:39.724 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:39.724 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:10:39.724 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:10:39.724 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:10:39.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:10:39.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:10:39.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:10:39.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:39.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:10:39.727 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:39.728 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:10:21.364420+0000", 2024-09-13T14:10:39.729 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:10:39.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:10:39.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:10:39.730 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:39.730 INFO:teuthology.orchestra.run.smithi099.stdout:mgr is available 2024-09-13T14:10:40.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:39 smithi099 bash[22656]: cluster 2024-09-13T14:10:38.898610+0000 mon.a (mon.0) 26 : cluster [DBG] mgrmap e4: y(active, since 3s) 2024-09-13T14:10:40.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:39 smithi099 bash[22656]: audit 2024-09-13T14:10:39.140279+0000 mon.a (mon.0) 27 : audit [DBG] from='client.? 172.21.15.99:0/1626034098' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:10:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:41 smithi099 bash[22656]: audit 2024-09-13T14:10:41.055986+0000 mon.a (mon.0) 28 : audit [INF] from='client.? 172.21.15.99:0/3618948778' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T14:10:41.636 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:41.636 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T14:10:41.636 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout fsid = af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:41.636 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-13T14:10:41.636 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T14:10:41.636 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T14:10:41.637 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T14:10:41.637 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T14:10:41.637 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:41.637 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T14:10:41.637 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T14:10:41.637 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 2024-09-13T14:10:41.637 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T14:10:41.637 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T14:10:41.638 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T14:10:41.638 INFO:teuthology.orchestra.run.smithi099.stdout:Enabling cephadm module... 2024-09-13T14:10:43.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:43 smithi099 bash[22656]: audit 2024-09-13T14:10:42.980081+0000 mon.a (mon.0) 29 : audit [INF] from='client.? 172.21.15.99:0/4130649879' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T14:10:43.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:43 smithi099 bash[22909]: ignoring --setuser ceph since I am not root 2024-09-13T14:10:43.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:43 smithi099 bash[22909]: ignoring --setgroup ceph since I am not root 2024-09-13T14:10:43.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:43 smithi099 bash[22909]: debug 2024-09-13T14:10:43.173+0000 7fc445ad9ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:10:44.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:44 smithi099 bash[22656]: audit 2024-09-13T14:10:43.036125+0000 mon.a (mon.0) 30 : audit [INF] from='client.? 172.21.15.99:0/4130649879' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T14:10:44.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:44 smithi099 bash[22656]: cluster 2024-09-13T14:10:43.036164+0000 mon.a (mon.0) 31 : cluster [DBG] mgrmap e5: y(active, since 7s) 2024-09-13T14:10:44.823 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:44 smithi099 bash[22909]: debug 2024-09-13T14:10:44.349+0000 7fc445ad9ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:10:44.823 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:44 smithi099 bash[22909]: debug 2024-09-13T14:10:44.413+0000 7fc445ad9ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:10:44.823 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:44 smithi099 bash[22909]: debug 2024-09-13T14:10:44.569+0000 7fc445ad9ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:10:45.199 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:45 smithi099 bash[22656]: audit 2024-09-13T14:10:44.908724+0000 mon.a (mon.0) 32 : audit [DBG] from='client.? 172.21.15.99:0/4182843626' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T14:10:45.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:45.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-13T14:10:45.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:10:45.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T14:10:45.465 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T14:10:45.466 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:45.466 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for the mgr to restart... 2024-09-13T14:10:45.466 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mgr epoch 5... 2024-09-13T14:10:45.492 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:45 smithi099 bash[22909]: debug 2024-09-13T14:10:45.197+0000 7fc445ad9ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:10:45.492 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:45 smithi099 bash[22909]: debug 2024-09-13T14:10:45.265+0000 7fc445ad9ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:10:45.492 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:45 smithi099 bash[22909]: debug 2024-09-13T14:10:45.329+0000 7fc445ad9ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:10:45.823 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:45 smithi099 bash[22909]: debug 2024-09-13T14:10:45.489+0000 7fc445ad9ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:10:45.823 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:45 smithi099 bash[22909]: debug 2024-09-13T14:10:45.569+0000 7fc445ad9ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:10:45.823 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:45 smithi099 bash[22909]: debug 2024-09-13T14:10:45.633+0000 7fc445ad9ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:10:46.166 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:45 smithi099 bash[22909]: debug 2024-09-13T14:10:45.857+0000 7fc445ad9ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:10:46.166 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:45 smithi099 bash[22909]: debug 2024-09-13T14:10:45.925+0000 7fc445ad9ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:10:46.166 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:46 smithi099 bash[22909]: debug 2024-09-13T14:10:46.089+0000 7fc445ad9ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:10:46.166 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:46 smithi099 bash[22909]: debug 2024-09-13T14:10:46.165+0000 7fc445ad9ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:10:46.573 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:46 smithi099 bash[22909]: debug 2024-09-13T14:10:46.397+0000 7fc445ad9ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:10:47.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:46 smithi099 bash[22909]: debug 2024-09-13T14:10:46.573+0000 7fc445ad9ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:10:47.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:46 smithi099 bash[22909]: debug 2024-09-13T14:10:46.649+0000 7fc445ad9ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:10:47.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:46 smithi099 bash[22909]: debug 2024-09-13T14:10:46.797+0000 7fc445ad9ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:10:47.817 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:47 smithi099 bash[22909]: debug 2024-09-13T14:10:47.549+0000 7fc445ad9ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:10:47.817 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:47 smithi099 bash[22909]: debug 2024-09-13T14:10:47.621+0000 7fc445ad9ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:10:47.817 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:47 smithi099 bash[22909]: debug 2024-09-13T14:10:47.813+0000 7fc445ad9ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:10:48.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:47 smithi099 bash[22909]: debug 2024-09-13T14:10:47.885+0000 7fc445ad9ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:10:48.573 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22909]: debug 2024-09-13T14:10:48.197+0000 7fc445ad9ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:10:48.573 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22909]: debug 2024-09-13T14:10:48.273+0000 7fc445ad9ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:10:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: cluster 2024-09-13T14:10:48.295282+0000 mon.a (mon.0) 33 : cluster [INF] Active manager daemon y restarted 2024-09-13T14:10:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: cluster 2024-09-13T14:10:48.295780+0000 mon.a (mon.0) 34 : cluster [INF] Activating manager daemon y 2024-09-13T14:10:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: cluster 2024-09-13T14:10:48.302115+0000 mon.a (mon.0) 35 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2024-09-13T14:10:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: cluster 2024-09-13T14:10:48.302340+0000 mon.a (mon.0) 36 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00672266s) 2024-09-13T14:10:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: audit 2024-09-13T14:10:48.304917+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:10:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: audit 2024-09-13T14:10:48.306411+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:10:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: audit 2024-09-13T14:10:48.307269+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:10:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: audit 2024-09-13T14:10:48.307572+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:10:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: audit 2024-09-13T14:10:48.307869+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:10:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: cluster 2024-09-13T14:10:48.332308+0000 mon.a (mon.0) 42 : cluster [INF] Manager daemon y is now available 2024-09-13T14:10:48.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:48 smithi099 bash[22656]: audit 2024-09-13T14:10:48.348109+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:10:49.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:49 smithi099 bash[22656]: cephadm 2024-09-13T14:10:48.342854+0000 mgr.y (mgr.14118) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2024-09-13T14:10:49.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:49 smithi099 bash[22656]: audit 2024-09-13T14:10:48.358657+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:10:49.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:49 smithi099 bash[22656]: audit 2024-09-13T14:10:48.363006+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:49.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:49 smithi099 bash[22656]: audit 2024-09-13T14:10:48.365715+0000 mon.a (mon.0) 46 : audit [DBG] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:49.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:49 smithi099 bash[22656]: audit 2024-09-13T14:10:48.373506+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:10:49.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:49 smithi099 bash[22656]: audit 2024-09-13T14:10:48.375983+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:10:49.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:49 smithi099 bash[22656]: cluster 2024-09-13T14:10:49.307351+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e7: y(active, since 1.01173s) 2024-09-13T14:10:49.888 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:10:49.888 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-13T14:10:49.888 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T14:10:49.888 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:10:49.888 INFO:teuthology.orchestra.run.smithi099.stdout:mgr epoch 5 is available 2024-09-13T14:10:49.889 INFO:teuthology.orchestra.run.smithi099.stdout:Setting orchestrator backend to cephadm... 2024-09-13T14:10:50.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: debug 2024-09-13T14:10:50.089+0000 7fc3f7ea9640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:10:50] ENGINE Error in HTTPServer.serve 2024-09-13T14:10:50.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: Traceback (most recent call last): 2024-09-13T14:10:50.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:10:50.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: self._connections.run(self.expiration_interval) 2024-09-13T14:10:50.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:10:50.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: self._run(expiration_interval) 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: s = self.context.wrap_socket( 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: return self.sslsocket_class._create( 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: self.do_handshake() 2024-09-13T14:10:50.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:10:50.326 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: self._sslobj.do_handshake() 2024-09-13T14:10:50.326 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22909]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:10:50.703 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22656]: audit 2024-09-13T14:10:49.305670+0000 mgr.y (mgr.14118) 2 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T14:10:50.703 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22656]: audit 2024-09-13T14:10:49.317948+0000 mgr.y (mgr.14118) 3 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T14:10:50.703 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22656]: cephadm 2024-09-13T14:10:49.970923+0000 mgr.y (mgr.14118) 4 : cephadm [INF] [13/Sep/2024:14:10:49] ENGINE Bus STARTING 2024-09-13T14:10:50.703 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22656]: audit 2024-09-13T14:10:50.096185+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:10:50.703 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:50 smithi099 bash[22656]: audit 2024-09-13T14:10:50.097201+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: cephadm 2024-09-13T14:10:50.090383+0000 mgr.y (mgr.14118) 5 : cephadm [INF] [13/Sep/2024:14:10:50] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:10:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: cephadm 2024-09-13T14:10:50.090466+0000 mgr.y (mgr.14118) 6 : cephadm [INF] [13/Sep/2024:14:10:50] ENGINE Bus STARTED 2024-09-13T14:10:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: cephadm 2024-09-13T14:10:50.092621+0000 mgr.y (mgr.14118) 7 : cephadm [ERR] [13/Sep/2024:14:10:50] ENGINE Error in HTTPServer.serve 2024-09-13T14:10:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: Traceback (most recent call last): 2024-09-13T14:10:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:10:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: self._connections.run(self.expiration_interval) 2024-09-13T14:10:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:10:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: self._run(expiration_interval) 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: s = self.context.wrap_socket( 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: return self.sslsocket_class._create( 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: self.do_handshake() 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: self._sslobj.do_handshake() 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:10:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: cluster 2024-09-13T14:10:51.099346+0000 mon.a (mon.0) 52 : cluster [DBG] mgrmap e8: y(active, since 2s) 2024-09-13T14:10:51.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: audit 2024-09-13T14:10:51.207542+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:10:51.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:51 smithi099 bash[22656]: audit 2024-09-13T14:10:51.219151+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:10:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:52 smithi099 bash[22656]: audit 2024-09-13T14:10:51.198816+0000 mgr.y (mgr.14118) 8 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:53.513 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-13T14:10:53.513 INFO:teuthology.orchestra.run.smithi099.stdout:Generating ssh key... 2024-09-13T14:10:53.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:53 smithi099 bash[22656]: audit 2024-09-13T14:10:52.982614+0000 mgr.y (mgr.14118) 9 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:55.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: Generating public/private rsa key pair. 2024-09-13T14:10:55.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: Your identification has been saved in /tmp/tmprjex3ihh/key 2024-09-13T14:10:55.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: Your public key has been saved in /tmp/tmprjex3ihh/key.pub 2024-09-13T14:10:55.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: The key fingerprint is: 2024-09-13T14:10:55.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: SHA256:b/R3f6cnkPY7gLY7a5DXfyWoDKwC6+dkoBWfg1cYdOI ceph-af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:55.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: The key's randomart image is: 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: +---[RSA 3072]----+ 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: | .+ . | 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: | . = | 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: | . E . | 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: | + o | 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: | + = .S..o o | 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: | o.o . =o+.B . .| 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: |. oo . *o+.=..o| 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: | .oo . .* .=.*| 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: | ..o.. .o+ oB+| 2024-09-13T14:10:55.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:10:54 smithi099 bash[22909]: +----[SHA256]-----+ 2024-09-13T14:10:56.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:55 smithi099 bash[22656]: audit 2024-09-13T14:10:54.773724+0000 mgr.y (mgr.14118) 10 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:56.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:55 smithi099 bash[22656]: cephadm 2024-09-13T14:10:54.774152+0000 mgr.y (mgr.14118) 11 : cephadm [INF] Generating ssh key... 2024-09-13T14:10:56.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:55 smithi099 bash[22656]: audit 2024-09-13T14:10:54.962611+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:10:56.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:55 smithi099 bash[22656]: audit 2024-09-13T14:10:54.967852+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:10:57.382 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLcLgw5rNr7VDrhy1WHMBFTQ92IK8lK2XC9fDogu7+oGIj27rXjYw+LgmiWr4/3w6mvvHoXuZfKLShCREXRwGWNilitqjSua5mwePLHKvAKU6677l8OB84GOSaId5afgWPUTRWZ7wjXwabXpPydH0bN/hQ6arEmnrIBs7thDpTp6ipX5ulIWS4hz87YB4QFSMXUPcBUYO797Jfj6z/6SIOOrQTkv02tr11ze+aWeLouu2gNufIHPjPIPfDRV5Wx7WaJDS+006dx+R0KXucw3KSUy/HH87Z+mXO5EGswOuB/LurpMTlUSukzp5dMN6t5g7pi8zbs1a/BDmAwfWN4U+FBaMCReog3PV4glnaxMNKOCAdUJb1y7ILK43rpwNdZWyCYNaCXswy2wSUzPqry8rwfv3MK3/O3222ysX97iCkFV6Lsy/l1YBuBHlzSbyB0OtgE23d02Yu/YetBZwyVJPM+pMLrggETIENIktZivFl/tAQBzdwUHsHSNMUpNQDLsc= ceph-af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:10:57.383 INFO:teuthology.orchestra.run.smithi099.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T14:10:57.383 INFO:teuthology.orchestra.run.smithi099.stdout:Adding key to root@localhost authorized_keys... 2024-09-13T14:10:57.383 INFO:teuthology.orchestra.run.smithi099.stdout:Adding host smithi099... 2024-09-13T14:10:57.474 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:57 smithi099 bash[22656]: audit 2024-09-13T14:10:56.798361+0000 mgr.y (mgr.14118) 12 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:10:59.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:10:59 smithi099 bash[22656]: audit 2024-09-13T14:10:58.779133+0000 mgr.y (mgr.14118) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi099", "addr": "172.21.15.99", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:00.156 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:00 smithi099 bash[22656]: cephadm 2024-09-13T14:10:59.759084+0000 mgr.y (mgr.14118) 14 : cephadm [INF] Deploying cephadm binary to smithi099 2024-09-13T14:11:02.607 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout Added host 'smithi099' with addr '172.21.15.99' 2024-09-13T14:11:02.607 INFO:teuthology.orchestra.run.smithi099.stdout:Deploying unmanaged mon service... 2024-09-13T14:11:03.017 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:02 smithi099 bash[22656]: audit 2024-09-13T14:11:01.870589+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:11:03.017 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:02 smithi099 bash[22656]: cephadm 2024-09-13T14:11:01.871049+0000 mgr.y (mgr.14118) 15 : cephadm [INF] Added host smithi099 2024-09-13T14:11:03.018 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:02 smithi099 bash[22656]: audit 2024-09-13T14:11:01.871444+0000 mon.a (mon.0) 58 : audit [DBG] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:05.526 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-13T14:11:05.526 INFO:teuthology.orchestra.run.smithi099.stdout:Deploying unmanaged mgr service... 2024-09-13T14:11:06.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:05 smithi099 bash[22656]: audit 2024-09-13T14:11:04.643885+0000 mgr.y (mgr.14118) 16 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:06.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:05 smithi099 bash[22656]: cephadm 2024-09-13T14:11:04.645870+0000 mgr.y (mgr.14118) 17 : cephadm [INF] Saving service mon spec with placement count:5 2024-09-13T14:11:06.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:05 smithi099 bash[22656]: audit 2024-09-13T14:11:04.650682+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:11:06.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:05 smithi099 bash[22656]: audit 2024-09-13T14:11:05.236962+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:11:06.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:05 smithi099 bash[22656]: audit 2024-09-13T14:11:05.512472+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:11:07.527 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-13T14:11:08.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:07 smithi099 bash[22656]: audit 2024-09-13T14:11:06.995790+0000 mgr.y (mgr.14118) 18 : audit [DBG] from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:08.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:07 smithi099 bash[22656]: cephadm 2024-09-13T14:11:06.996763+0000 mgr.y (mgr.14118) 19 : cephadm [INF] Saving service mgr spec with placement count:2 2024-09-13T14:11:08.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:08 smithi099 bash[22656]: audit 2024-09-13T14:11:07.001017+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:11:10.206 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:09 smithi099 bash[22656]: cluster 2024-09-13T14:11:08.308833+0000 mgr.y (mgr.14118) 20 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:10.206 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:09 smithi099 bash[22656]: audit 2024-09-13T14:11:08.827234+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.99:0/2852907532' entity='client.admin' 2024-09-13T14:11:11.331 INFO:teuthology.orchestra.run.smithi099.stdout:Enabling the dashboard module... 2024-09-13T14:11:11.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:11 smithi099 bash[22656]: cluster 2024-09-13T14:11:10.309166+0000 mgr.y (mgr.14118) 21 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:11 smithi099 bash[22656]: audit 2024-09-13T14:11:10.403896+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:11:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:11 smithi099 bash[22656]: audit 2024-09-13T14:11:10.703461+0000 mon.a (mon.0) 65 : audit [INF] from='client.? 172.21.15.99:0/1851294079' entity='client.admin' 2024-09-13T14:11:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:11 smithi099 bash[22656]: audit 2024-09-13T14:11:10.916046+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.14118 172.21.15.99:0/3572345601' entity='mgr.y' 2024-09-13T14:11:13.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:13 smithi099 bash[22656]: cluster 2024-09-13T14:11:12.309464+0000 mgr.y (mgr.14118) 22 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:13.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:13 smithi099 bash[22656]: audit 2024-09-13T14:11:12.849674+0000 mon.a (mon.0) 67 : audit [INF] from='client.? 172.21.15.99:0/2650395146' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T14:11:13.824 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:13 smithi099 bash[22909]: ignoring --setuser ceph since I am not root 2024-09-13T14:11:13.824 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:13 smithi099 bash[22909]: ignoring --setgroup ceph since I am not root 2024-09-13T14:11:13.824 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:13 smithi099 bash[22909]: debug 2024-09-13T14:11:13.568+0000 7f9d8f292ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:11:14.738 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:14 smithi099 bash[22656]: audit 2024-09-13T14:11:13.416353+0000 mon.a (mon.0) 68 : audit [INF] from='client.? 172.21.15.99:0/2650395146' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T14:11:14.738 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:14 smithi099 bash[22656]: cluster 2024-09-13T14:11:13.416408+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e9: y(active, since 25s) 2024-09-13T14:11:15.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:14 smithi099 bash[22909]: debug 2024-09-13T14:11:14.736+0000 7f9d8f292ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:11:15.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:14 smithi099 bash[22909]: debug 2024-09-13T14:11:14.796+0000 7f9d8f292ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:11:15.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:14 smithi099 bash[22909]: debug 2024-09-13T14:11:14.952+0000 7f9d8f292ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:11:15.678 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:15 smithi099 bash[22909]: debug 2024-09-13T14:11:15.608+0000 7f9d8f292ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:11:15.678 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:15 smithi099 bash[22656]: audit 2024-09-13T14:11:15.387971+0000 mon.a (mon.0) 70 : audit [DBG] from='client.? 172.21.15.99:0/3933132669' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T14:11:15.973 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:15 smithi099 bash[22909]: debug 2024-09-13T14:11:15.676+0000 7f9d8f292ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:11:15.973 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:15 smithi099 bash[22909]: debug 2024-09-13T14:11:15.740+0000 7f9d8f292ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:11:15.973 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:15 smithi099 bash[22909]: debug 2024-09-13T14:11:15.896+0000 7f9d8f292ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:11:16.028 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:11:16.028 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-13T14:11:16.028 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:11:16.028 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T14:11:16.028 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T14:11:16.028 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:11:16.028 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for the mgr to restart... 2024-09-13T14:11:16.028 INFO:teuthology.orchestra.run.smithi099.stdout:Waiting for mgr epoch 9... 2024-09-13T14:11:16.267 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:15 smithi099 bash[22909]: debug 2024-09-13T14:11:15.972+0000 7f9d8f292ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:11:16.267 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:16 smithi099 bash[22909]: debug 2024-09-13T14:11:16.036+0000 7f9d8f292ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:11:16.571 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:16 smithi099 bash[22909]: debug 2024-09-13T14:11:16.264+0000 7f9d8f292ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:11:16.572 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:16 smithi099 bash[22909]: debug 2024-09-13T14:11:16.328+0000 7f9d8f292ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:11:16.572 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:16 smithi099 bash[22909]: debug 2024-09-13T14:11:16.496+0000 7f9d8f292ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:11:16.823 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:16 smithi099 bash[22909]: debug 2024-09-13T14:11:16.568+0000 7f9d8f292ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:11:16.823 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:16 smithi099 bash[22909]: debug 2024-09-13T14:11:16.800+0000 7f9d8f292ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:11:17.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:16 smithi099 bash[22909]: debug 2024-09-13T14:11:16.956+0000 7f9d8f292ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:11:17.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:17 smithi099 bash[22909]: debug 2024-09-13T14:11:17.036+0000 7f9d8f292ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:11:17.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:17 smithi099 bash[22909]: debug 2024-09-13T14:11:17.196+0000 7f9d8f292ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:11:18.224 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:17 smithi099 bash[22909]: debug 2024-09-13T14:11:17.952+0000 7f9d8f292ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:11:18.224 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22909]: debug 2024-09-13T14:11:18.028+0000 7f9d8f292ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:11:18.529 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22909]: debug 2024-09-13T14:11:18.220+0000 7f9d8f292ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:11:18.529 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22909]: debug 2024-09-13T14:11:18.292+0000 7f9d8f292ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:11:18.823 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22909]: debug 2024-09-13T14:11:18.612+0000 7f9d8f292ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:11:18.823 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22909]: debug 2024-09-13T14:11:18.692+0000 7f9d8f292ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:11:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: cluster 2024-09-13T14:11:18.714870+0000 mon.a (mon.0) 71 : cluster [INF] Active manager daemon y restarted 2024-09-13T14:11:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: cluster 2024-09-13T14:11:18.715106+0000 mon.a (mon.0) 72 : cluster [INF] Activating manager daemon y 2024-09-13T14:11:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: cluster 2024-09-13T14:11:18.718193+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2024-09-13T14:11:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: cluster 2024-09-13T14:11:18.718299+0000 mon.a (mon.0) 74 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00330286s) 2024-09-13T14:11:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: audit 2024-09-13T14:11:18.719073+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:11:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: audit 2024-09-13T14:11:18.719783+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:11:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: audit 2024-09-13T14:11:18.720406+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:11:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: audit 2024-09-13T14:11:18.720535+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:11:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: audit 2024-09-13T14:11:18.720658+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:11:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: cluster 2024-09-13T14:11:18.726364+0000 mon.a (mon.0) 80 : cluster [INF] Manager daemon y is now available 2024-09-13T14:11:18.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: audit 2024-09-13T14:11:18.743098+0000 mon.a (mon.0) 81 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:18.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: audit 2024-09-13T14:11:18.760665+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:11:18.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:18 smithi099 bash[22656]: audit 2024-09-13T14:11:18.762264+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:11:20.267 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:11:20.267 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-13T14:11:20.267 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T14:11:20.267 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:11:20.267 INFO:teuthology.orchestra.run.smithi099.stdout:mgr epoch 9 is available 2024-09-13T14:11:20.267 INFO:teuthology.orchestra.run.smithi099.stdout:Generating a dashboard self-signed certificate... 2024-09-13T14:11:20.574 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: debug 2024-09-13T14:11:20.184+0000 7f9d3fddf640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:11:20] ENGINE Error in HTTPServer.serve 2024-09-13T14:11:20.574 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: Traceback (most recent call last): 2024-09-13T14:11:20.574 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:11:20.574 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: self._connections.run(self.expiration_interval) 2024-09-13T14:11:20.574 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:11:20.574 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: self._run(expiration_interval) 2024-09-13T14:11:20.574 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:11:20.575 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:11:20.575 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:11:20.575 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:11:20.575 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:11:20.575 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: s = self.context.wrap_socket( 2024-09-13T14:11:20.575 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:11:20.575 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: return self.sslsocket_class._create( 2024-09-13T14:11:20.575 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:11:20.575 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: self.do_handshake() 2024-09-13T14:11:20.575 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:11:20.576 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: self._sslobj.do_handshake() 2024-09-13T14:11:20.576 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22909]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:11:21.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: cluster 2024-09-13T14:11:19.726267+0000 mon.a (mon.0) 84 : cluster [DBG] mgrmap e11: y(active, since 1.01126s) 2024-09-13T14:11:21.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: cephadm 2024-09-13T14:11:20.063951+0000 mgr.y (mgr.14150) 3 : cephadm [INF] [13/Sep/2024:14:11:20] ENGINE Bus STARTING 2024-09-13T14:11:21.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: cephadm 2024-09-13T14:11:20.184116+0000 mgr.y (mgr.14150) 4 : cephadm [INF] [13/Sep/2024:14:11:20] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:11:21.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: cephadm 2024-09-13T14:11:20.184218+0000 mgr.y (mgr.14150) 5 : cephadm [INF] [13/Sep/2024:14:11:20] ENGINE Bus STARTED 2024-09-13T14:11:21.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: cephadm 2024-09-13T14:11:20.186061+0000 mgr.y (mgr.14150) 6 : cephadm [ERR] [13/Sep/2024:14:11:20] ENGINE Error in HTTPServer.serve 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: Traceback (most recent call last): 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: self._connections.run(self.expiration_interval) 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: self._run(expiration_interval) 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: s = self.context.wrap_socket( 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:11:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: return self.sslsocket_class._create( 2024-09-13T14:11:21.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:11:21.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: self.do_handshake() 2024-09-13T14:11:21.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:11:21.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: self._sslobj.do_handshake() 2024-09-13T14:11:21.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:11:21.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:20 smithi099 bash[22656]: audit 2024-09-13T14:11:20.187455+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:22.324 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-13T14:11:22.324 INFO:teuthology.orchestra.run.smithi099.stdout:Creating initial admin user... 2024-09-13T14:11:23.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:22 smithi099 bash[22656]: audit 2024-09-13T14:11:21.606969+0000 mgr.y (mgr.14150) 7 : audit [DBG] from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:23.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:22 smithi099 bash[22656]: cluster 2024-09-13T14:11:21.727248+0000 mon.a (mon.0) 86 : cluster [DBG] mgrmap e12: y(active, since 3s) 2024-09-13T14:11:23.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:22 smithi099 bash[22656]: audit 2024-09-13T14:11:21.761159+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:23.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:22 smithi099 bash[22656]: audit 2024-09-13T14:11:21.765554+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:24.440 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$Q0.5cENPLGKaDuSzx9TT8.51NGIuNfgrmnTBegSrq69Ws3cs94YV6", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726236683, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T14:11:24.440 INFO:teuthology.orchestra.run.smithi099.stdout:Fetching dashboard port number... 2024-09-13T14:11:25.277 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:24 smithi099 bash[22656]: audit 2024-09-13T14:11:23.617998+0000 mgr.y (mgr.14150) 8 : audit [DBG] from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:25.278 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:24 smithi099 bash[22656]: audit 2024-09-13T14:11:23.837497+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:26.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:25 smithi099 bash[22656]: cluster 2024-09-13T14:11:24.842067+0000 mon.a (mon.0) 90 : cluster [DBG] mgrmap e13: y(active, since 6s) 2024-09-13T14:11:26.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:25 smithi099 bash[22656]: audit 2024-09-13T14:11:25.147875+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:26.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:25 smithi099 bash[22656]: audit 2024-09-13T14:11:25.647537+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:26.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:25 smithi099 bash[22656]: audit 2024-09-13T14:11:25.752232+0000 mon.a (mon.0) 93 : audit [DBG] from='client.? 172.21.15.99:0/3820981268' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T14:11:26.386 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stdout 8443 2024-09-13T14:11:26.387 INFO:teuthology.orchestra.run.smithi099.stdout:firewalld does not appear to be present 2024-09-13T14:11:26.387 INFO:teuthology.orchestra.run.smithi099.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T14:11:26.390 INFO:teuthology.orchestra.run.smithi099.stdout:Ceph Dashboard is now available at: 2024-09-13T14:11:26.390 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:11:26.390 INFO:teuthology.orchestra.run.smithi099.stdout: URL: https://smithi099.front.sepia.ceph.com:8443/ 2024-09-13T14:11:26.390 INFO:teuthology.orchestra.run.smithi099.stdout: User: admin 2024-09-13T14:11:26.390 INFO:teuthology.orchestra.run.smithi099.stdout: Password: mzsfpz49nt 2024-09-13T14:11:26.390 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:11:26.390 INFO:teuthology.orchestra.run.smithi099.stdout:Saving cluster configuration to /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config directory 2024-09-13T14:11:26.391 INFO:teuthology.orchestra.run.smithi099.stdout:Enabling autotune for osd_memory_target 2024-09-13T14:11:30.250 INFO:teuthology.orchestra.run.smithi099.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-13T14:11:30.250 INFO:teuthology.orchestra.run.smithi099.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-13T14:11:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:11:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:11:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:11:30.250 INFO:teuthology.orchestra.run.smithi099.stdout:Or, if you are only running a single cluster on this host: 2024-09-13T14:11:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:11:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-13T14:11:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:11:30.251 INFO:teuthology.orchestra.run.smithi099.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-13T14:11:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:11:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: ceph telemetry on 2024-09-13T14:11:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:11:30.251 INFO:teuthology.orchestra.run.smithi099.stdout:For more information see: 2024-09-13T14:11:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:11:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T14:11:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:11:30.251 INFO:teuthology.orchestra.run.smithi099.stdout:Bootstrap complete. 2024-09-13T14:11:30.289 INFO:tasks.cephadm:Fetching config... 2024-09-13T14:11:30.289 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:11:30.289 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T14:11:30.295 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T14:11:30.295 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:11:30.295 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T14:11:30.341 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T14:11:30.341 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:11:30.341 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-13T14:11:30.396 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T14:11:30.397 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:11:30.397 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T14:11:30.445 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T14:11:30.446 DEBUG:teuthology.orchestra.run.smithi099:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLcLgw5rNr7VDrhy1WHMBFTQ92IK8lK2XC9fDogu7+oGIj27rXjYw+LgmiWr4/3w6mvvHoXuZfKLShCREXRwGWNilitqjSua5mwePLHKvAKU6677l8OB84GOSaId5afgWPUTRWZ7wjXwabXpPydH0bN/hQ6arEmnrIBs7thDpTp6ipX5ulIWS4hz87YB4QFSMXUPcBUYO797Jfj6z/6SIOOrQTkv02tr11ze+aWeLouu2gNufIHPjPIPfDRV5Wx7WaJDS+006dx+R0KXucw3KSUy/HH87Z+mXO5EGswOuB/LurpMTlUSukzp5dMN6t5g7pi8zbs1a/BDmAwfWN4U+FBaMCReog3PV4glnaxMNKOCAdUJb1y7ILK43rpwNdZWyCYNaCXswy2wSUzPqry8rwfv3MK3/O3222ysX97iCkFV6Lsy/l1YBuBHlzSbyB0OtgE23d02Yu/YetBZwyVJPM+pMLrggETIENIktZivFl/tAQBzdwUHsHSNMUpNQDLsc= ceph-af9d2c22-71d9-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T14:11:30.510 INFO:teuthology.orchestra.run.smithi099.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLcLgw5rNr7VDrhy1WHMBFTQ92IK8lK2XC9fDogu7+oGIj27rXjYw+LgmiWr4/3w6mvvHoXuZfKLShCREXRwGWNilitqjSua5mwePLHKvAKU6677l8OB84GOSaId5afgWPUTRWZ7wjXwabXpPydH0bN/hQ6arEmnrIBs7thDpTp6ipX5ulIWS4hz87YB4QFSMXUPcBUYO797Jfj6z/6SIOOrQTkv02tr11ze+aWeLouu2gNufIHPjPIPfDRV5Wx7WaJDS+006dx+R0KXucw3KSUy/HH87Z+mXO5EGswOuB/LurpMTlUSukzp5dMN6t5g7pi8zbs1a/BDmAwfWN4U+FBaMCReog3PV4glnaxMNKOCAdUJb1y7ILK43rpwNdZWyCYNaCXswy2wSUzPqry8rwfv3MK3/O3222ysX97iCkFV6Lsy/l1YBuBHlzSbyB0OtgE23d02Yu/YetBZwyVJPM+pMLrggETIENIktZivFl/tAQBzdwUHsHSNMUpNQDLsc= ceph-af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:11:30.519 DEBUG:teuthology.orchestra.run.smithi177:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLcLgw5rNr7VDrhy1WHMBFTQ92IK8lK2XC9fDogu7+oGIj27rXjYw+LgmiWr4/3w6mvvHoXuZfKLShCREXRwGWNilitqjSua5mwePLHKvAKU6677l8OB84GOSaId5afgWPUTRWZ7wjXwabXpPydH0bN/hQ6arEmnrIBs7thDpTp6ipX5ulIWS4hz87YB4QFSMXUPcBUYO797Jfj6z/6SIOOrQTkv02tr11ze+aWeLouu2gNufIHPjPIPfDRV5Wx7WaJDS+006dx+R0KXucw3KSUy/HH87Z+mXO5EGswOuB/LurpMTlUSukzp5dMN6t5g7pi8zbs1a/BDmAwfWN4U+FBaMCReog3PV4glnaxMNKOCAdUJb1y7ILK43rpwNdZWyCYNaCXswy2wSUzPqry8rwfv3MK3/O3222ysX97iCkFV6Lsy/l1YBuBHlzSbyB0OtgE23d02Yu/YetBZwyVJPM+pMLrggETIENIktZivFl/tAQBzdwUHsHSNMUpNQDLsc= ceph-af9d2c22-71d9-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T14:11:30.544 INFO:teuthology.orchestra.run.smithi177.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLcLgw5rNr7VDrhy1WHMBFTQ92IK8lK2XC9fDogu7+oGIj27rXjYw+LgmiWr4/3w6mvvHoXuZfKLShCREXRwGWNilitqjSua5mwePLHKvAKU6677l8OB84GOSaId5afgWPUTRWZ7wjXwabXpPydH0bN/hQ6arEmnrIBs7thDpTp6ipX5ulIWS4hz87YB4QFSMXUPcBUYO797Jfj6z/6SIOOrQTkv02tr11ze+aWeLouu2gNufIHPjPIPfDRV5Wx7WaJDS+006dx+R0KXucw3KSUy/HH87Z+mXO5EGswOuB/LurpMTlUSukzp5dMN6t5g7pi8zbs1a/BDmAwfWN4U+FBaMCReog3PV4glnaxMNKOCAdUJb1y7ILK43rpwNdZWyCYNaCXswy2wSUzPqry8rwfv3MK3/O3222ysX97iCkFV6Lsy/l1YBuBHlzSbyB0OtgE23d02Yu/YetBZwyVJPM+pMLrggETIENIktZivFl/tAQBzdwUHsHSNMUpNQDLsc= ceph-af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:11:30.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T14:11:30.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:30 smithi099 bash[22656]: audit 2024-09-13T14:11:29.722833+0000 mon.a (mon.0) 94 : audit [INF] from='client.? 172.21.15.99:0/3132993467' entity='client.admin' 2024-09-13T14:11:36.442 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:37.310 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:37 smithi099 bash[22656]: audit 2024-09-13T14:11:36.048920+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:37.310 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:37 smithi099 bash[22656]: audit 2024-09-13T14:11:36.053338+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:37.310 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:37 smithi099 bash[22656]: audit 2024-09-13T14:11:36.054661+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi099", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:11:37.310 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:37 smithi099 bash[22656]: audit 2024-09-13T14:11:36.060327+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:37.310 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:37 smithi099 bash[22656]: audit 2024-09-13T14:11:36.089752+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:37.310 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:37 smithi099 bash[22656]: audit 2024-09-13T14:11:36.098328+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:39.822 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T14:11:39.823 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T14:11:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:40 smithi099 bash[22656]: cluster 2024-09-13T14:11:38.722283+0000 mgr.y (mgr.14150) 9 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:40 smithi099 bash[22656]: audit 2024-09-13T14:11:39.255877+0000 mon.a (mon.0) 101 : audit [INF] from='client.? 172.21.15.99:0/4275218877' entity='client.admin' 2024-09-13T14:11:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:40 smithi099 bash[22656]: audit 2024-09-13T14:11:39.269978+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:40 smithi099 bash[22656]: audit 2024-09-13T14:11:39.275136+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:42.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:42 smithi099 bash[22656]: cluster 2024-09-13T14:11:40.722698+0000 mgr.y (mgr.14150) 10 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:44.535 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:44.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:44 smithi099 bash[22656]: cluster 2024-09-13T14:11:42.723138+0000 mgr.y (mgr.14150) 11 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:46.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:46 smithi099 bash[22656]: cluster 2024-09-13T14:11:44.723577+0000 mgr.y (mgr.14150) 12 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:48.204 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi177 2024-09-13T14:11:48.205 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:11:48.205 DEBUG:teuthology.orchestra.run.smithi177:> dd of=/etc/ceph/ceph.conf 2024-09-13T14:11:48.211 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:11:48.211 DEBUG:teuthology.orchestra.run.smithi177:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:11:48.257 INFO:tasks.cephadm:Adding host smithi177 to orchestrator... 2024-09-13T14:11:48.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch host add smithi177 2024-09-13T14:11:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:48 smithi099 bash[22656]: cluster 2024-09-13T14:11:46.724001+0000 mgr.y (mgr.14150) 13 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:48 smithi099 bash[22656]: audit 2024-09-13T14:11:47.600244+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:48 smithi099 bash[22656]: audit 2024-09-13T14:11:47.601340+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:11:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:48 smithi099 bash[22656]: audit 2024-09-13T14:11:47.604044+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:11:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:48 smithi099 bash[22656]: audit 2024-09-13T14:11:47.604891+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:11:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:48 smithi099 bash[22656]: audit 2024-09-13T14:11:47.988488+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:48 smithi099 bash[22656]: audit 2024-09-13T14:11:47.991648+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:48 smithi099 bash[22656]: audit 2024-09-13T14:11:47.995414+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:11:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:49 smithi099 bash[22656]: audit 2024-09-13T14:11:47.597163+0000 mgr.y (mgr.14150) 14 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:49 smithi099 bash[22656]: cephadm 2024-09-13T14:11:47.605980+0000 mgr.y (mgr.14150) 15 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:11:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:49 smithi099 bash[22656]: cephadm 2024-09-13T14:11:47.697868+0000 mgr.y (mgr.14150) 16 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:11:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:49 smithi099 bash[22656]: cephadm 2024-09-13T14:11:47.785622+0000 mgr.y (mgr.14150) 17 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:11:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:49 smithi099 bash[22656]: cephadm 2024-09-13T14:11:47.883873+0000 mgr.y (mgr.14150) 18 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:11:50.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:50 smithi099 bash[22656]: cluster 2024-09-13T14:11:48.724410+0000 mgr.y (mgr.14150) 19 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:52.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:52 smithi099 bash[22656]: cluster 2024-09-13T14:11:50.724823+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:52.968 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:11:54.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:54 smithi099 bash[22656]: cluster 2024-09-13T14:11:52.725231+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:56.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:56 smithi099 bash[22656]: cluster 2024-09-13T14:11:54.725611+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:57.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:57 smithi099 bash[22656]: audit 2024-09-13T14:11:55.786410+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi177", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:11:58.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:58 smithi099 bash[22656]: cluster 2024-09-13T14:11:56.726004+0000 mgr.y (mgr.14150) 24 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:11:58.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:11:58 smithi099 bash[22656]: cephadm 2024-09-13T14:11:56.809453+0000 mgr.y (mgr.14150) 25 : cephadm [INF] Deploying cephadm binary to smithi177 2024-09-13T14:11:59.122 INFO:teuthology.orchestra.run.smithi099.stdout:Added host 'smithi177' with addr '172.21.15.177' 2024-09-13T14:11:59.749 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T14:12:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:00 smithi099 bash[22656]: cluster 2024-09-13T14:11:58.726328+0000 mgr.y (mgr.14150) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:00 smithi099 bash[22656]: audit 2024-09-13T14:11:59.120955+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:00 smithi099 bash[22656]: cephadm 2024-09-13T14:11:59.121765+0000 mgr.y (mgr.14150) 27 : cephadm [INF] Added host smithi177 2024-09-13T14:12:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:00 smithi099 bash[22656]: audit 2024-09-13T14:11:59.122593+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:00 smithi099 bash[22656]: audit 2024-09-13T14:11:59.382653+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:02.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:01 smithi099 bash[22656]: audit 2024-09-13T14:12:00.643101+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:02.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:01 smithi099 bash[22656]: cluster 2024-09-13T14:12:00.726737+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:02.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:01 smithi099 bash[22656]: audit 2024-09-13T14:12:01.392400+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:04.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:03 smithi099 bash[22656]: cluster 2024-09-13T14:12:02.727165+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:04.458 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:06.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:05 smithi099 bash[22656]: cluster 2024-09-13T14:12:04.727619+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:07.420 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:12:07.420 INFO:teuthology.orchestra.run.smithi099.stdout:[{"addr": "172.21.15.99", "hostname": "smithi099", "labels": [], "status": ""}, {"addr": "172.21.15.177", "hostname": "smithi177", "labels": [], "status": ""}] 2024-09-13T14:12:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:06.672111+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:06.679324+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:06.687614+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:06.693165+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:06.694538+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi177", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:06.696487+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:06.698082+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: cephadm 2024-09-13T14:12:06.700256+0000 mgr.y (mgr.14150) 31 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: cluster 2024-09-13T14:12:06.728031+0000 mgr.y (mgr.14150) 32 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: cephadm 2024-09-13T14:12:06.859211+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: cephadm 2024-09-13T14:12:07.000699+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: cephadm 2024-09-13T14:12:07.137619+0000 mgr.y (mgr.14150) 35 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:12:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:07.242786+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:07.250324+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:07.253754+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:07 smithi099 bash[22656]: audit 2024-09-13T14:12:07.420077+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:12:07.962 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T14:12:07.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T14:12:10.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:09 smithi099 bash[22656]: cluster 2024-09-13T14:12:08.728435+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:12.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:11 smithi099 bash[22656]: cluster 2024-09-13T14:12:10.728790+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:12.674 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:14.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:13 smithi099 bash[22656]: cluster 2024-09-13T14:12:12.729105+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:15.794 INFO:teuthology.orchestra.run.smithi099.stderr:adjusted tunables profile to default 2024-09-13T14:12:16.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:15 smithi099 bash[22656]: cluster 2024-09-13T14:12:14.729548+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:16.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:15 smithi099 bash[22656]: audit 2024-09-13T14:12:15.563865+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.99:0/4075393359' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T14:12:16.423 INFO:tasks.cephadm:Adding mon.a on smithi099 2024-09-13T14:12:16.423 INFO:tasks.cephadm:Adding mon.c on smithi099 2024-09-13T14:12:16.423 INFO:tasks.cephadm:Adding mon.b on smithi177 2024-09-13T14:12:16.424 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi099:172.21.15.99=a;smithi099:[v2:172.21.15.99:3301,v1:172.21.15.99:6790]=c;smithi177:172.21.15.177=b' 2024-09-13T14:12:17.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:16 smithi099 bash[22656]: audit 2024-09-13T14:12:15.794198+0000 mon.a (mon.0) 127 : audit [INF] from='client.? 172.21.15.99:0/4075393359' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T14:12:17.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:16 smithi099 bash[22656]: cluster 2024-09-13T14:12:15.794290+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:17.636 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:18.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:17 smithi099 bash[22656]: cluster 2024-09-13T14:12:16.729997+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:20.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:19 smithi099 bash[22656]: cluster 2024-09-13T14:12:18.730377+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:20.323 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled mon update... 2024-09-13T14:12:21.097 DEBUG:teuthology.orchestra.run.smithi099:mon.c> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.c.service 2024-09-13T14:12:21.100 DEBUG:teuthology.orchestra.run.smithi177:mon.b> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.b.service 2024-09-13T14:12:21.102 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:12:21.102 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:12:21.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:21 smithi099 bash[22656]: audit 2024-09-13T14:12:20.312124+0000 mgr.y (mgr.14150) 43 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi099:172.21.15.99=a;smithi099:[v2:172.21.15.99:3301,v1:172.21.15.99:6790]=c;smithi177:172.21.15.177=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:21.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:21 smithi099 bash[22656]: cephadm 2024-09-13T14:12:20.317253+0000 mgr.y (mgr.14150) 44 : cephadm [INF] Saving service mon spec with placement smithi099:172.21.15.99=a;smithi099:[v2:172.21.15.99:3301,v1:172.21.15.99:6790]=c;smithi177:172.21.15.177=b;count:3 2024-09-13T14:12:21.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:21 smithi099 bash[22656]: audit 2024-09-13T14:12:20.322704+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:21 smithi099 bash[22656]: audit 2024-09-13T14:12:20.324343+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:21 smithi099 bash[22656]: audit 2024-09-13T14:12:20.327887+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:21 smithi099 bash[22656]: audit 2024-09-13T14:12:20.329583+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:21 smithi099 bash[22656]: audit 2024-09-13T14:12:20.337986+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:21 smithi099 bash[22656]: audit 2024-09-13T14:12:20.341653+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:21 smithi099 bash[22656]: audit 2024-09-13T14:12:20.343179+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:21 smithi099 bash[22656]: cephadm 2024-09-13T14:12:20.345039+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Deploying daemon mon.b on smithi177 2024-09-13T14:12:22.342 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:22.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:22 smithi099 bash[22656]: cluster 2024-09-13T14:12:20.730802+0000 mgr.y (mgr.14150) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:24.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:24 smithi099 bash[22656]: cluster 2024-09-13T14:12:22.731239+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:26.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:25 smithi177 systemd[1]: Started Ceph mon.b for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:12:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-09-13T14:12:26.165 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","modified":"2024-09-13T14:10:16.718964Z","created":"2024-09-13T14:10:16.718964Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:3300","nonce":0},{"type":"v1","addr":"172.21.15.99:6789","nonce":0}]},"addr":"172.21.15.99:6789/0","public_addr":"172.21.15.99:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T14:12:26.166 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2024-09-13T14:12:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:26 smithi099 bash[22656]: cluster 2024-09-13T14:12:24.731587+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:26 smithi099 bash[22656]: audit 2024-09-13T14:12:25.847735+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:26 smithi099 bash[22656]: audit 2024-09-13T14:12:25.853904+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:26 smithi099 bash[22656]: audit 2024-09-13T14:12:25.860588+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:26 smithi099 bash[22656]: audit 2024-09-13T14:12:25.861755+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:26 smithi099 bash[22656]: audit 2024-09-13T14:12:25.863476+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:26 smithi099 bash[22656]: audit 2024-09-13T14:12:26.165524+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.177:0/1724937396' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:12:27.190 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.177+0000 7fae12584900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:12:27.190 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.177+0000 7fae12584900 0 ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T14:12:27.190 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.177+0000 7fae12584900 0 pidfile_write: ignore empty --pid-file 2024-09-13T14:12:27.190 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 0 load: jerasure load: lrc 2024-09-13T14:12:27.190 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: RocksDB version: 6.15.5 2024-09-13T14:12:27.190 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:12:27.190 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Compile date Sep 12 2024 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: DB SUMMARY 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: DB Session ID: LVFKHRG3X6CDLEB8KVIF 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: CURRENT file: CURRENT 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.error_if_exists: 0 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.create_if_missing: 0 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T14:12:27.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.env: 0x55e604b5e440 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.fs: Posix File System 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.info_log: 0x55e6068beee0 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.statistics: (nil) 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.use_fsync: 0 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:12:27.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.db_log_dir: 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:12:27.193 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.write_buffer_manager: 0x55e60697c7e0 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.unordered_write: 0 2024-09-13T14:12:27.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:12:27.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:12:27.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:12:27.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:12:27.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.row_cache: None 2024-09-13T14:12:27.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.wal_filter: None 2024-09-13T14:12:27.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:12:27.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:12:27.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.preserve_deletes: 0 2024-09-13T14:12:27.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.two_write_queues: 0 2024-09-13T14:12:27.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:12:27.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.atomic_flush: 0 2024-09-13T14:12:27.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:12:27.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:12:27.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_open_files: -1 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T14:12:27.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Compression algorithms supported: 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: kZSTD supported: 0 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: kXpressCompression supported: 0 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: kZlibCompression supported: 1 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.merge_operator: 2024-09-13T14:12:27.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_filter: None 2024-09-13T14:12:27.202 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:12:27.202 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:12:27.202 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:12:27.202 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:12:27.202 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e606867a38) 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cache_index_and_filter_blocks: 1 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: pin_top_level_index_and_filter: 1 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: index_type: 0 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: data_block_index_type: 0 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: index_shortening: 1 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: hash_index_allow_collision: 1 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: checksum: 1 2024-09-13T14:12:27.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: no_block_cache: 0 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: block_cache: 0x55e60689b1f0 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: block_cache_name: BinnedLRUCache 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: block_cache_options: 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: capacity : 536870912 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: num_shard_bits : 4 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: strict_capacity_limit : 0 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: high_pri_pool_ratio: 0.000 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: block_cache_compressed: (nil) 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: persistent_cache: (nil) 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: block_size: 4096 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: block_size_deviation: 10 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: block_restart_interval: 16 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: index_block_restart_interval: 1 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: metadata_block_size: 4096 2024-09-13T14:12:27.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: partition_filters: 0 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: use_delta_encoding: 1 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: whole_key_filtering: 1 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: verify_compression: 0 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: read_amp_bytes_per_bit: 0 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: format_version: 4 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: enable_index_compression: 1 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: block_align: 0 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compression: NoCompression 2024-09-13T14:12:27.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.num_levels: 7 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:12:27.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:12:27.207 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:12:27.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:12:27.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.bloom_locality: 0 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.ttl: 2592000 2024-09-13T14:12:27.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.enable_blob_files: false 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.min_blob_size: 0 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726236747188731, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.181+0000 7fae12584900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-13T14:12:27.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae12584900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae12584900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726236747189150, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726236747, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "73fd8e4e-c0fb-4e2d-ba08-bb478df7d478", "db_session_id": "LVFKHRG3X6CDLEB8KVIF"}} 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae12584900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae12584900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726236747190236, "job": 1, "event": "recovery_finished"} 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae12584900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae12584900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55e6068d0700 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae12584900 4 rocksdb: DB pointer 0x55e60689e000 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae08350640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae08350640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ** DB Stats ** 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:12:27.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ** Compaction Stats [default] ** 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ** Compaction Stats [default] ** 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:12:27.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Cumulative compaction: 0.00 GB write, 0.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Interval compaction: 0.00 GB write, 0.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ** Compaction Stats [default] ** 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:12:27.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:27.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ** Compaction Stats [default] ** 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:12:27.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Cumulative compaction: 0.00 GB write, 0.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae12584900 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae12584900 0 using public_addr v2:172.21.15.177:0/0 -> [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.185+0000 7fae12584900 0 starting mon.b rank -1 at public addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] at bind addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.189+0000 7fae12584900 1 mon.b@-1(???) e0 preinit fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.209+0000 7fae0b356640 0 mon.b@-1(synchronizing).mds e1 new map 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.209+0000 7fae0b356640 0 mon.b@-1(synchronizing).mds e1 print_map 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: e1 2024-09-13T14:12:27.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:12:27.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:12:27.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: legacy client fscid: -1 2024-09-13T14:12:27.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: 2024-09-13T14:12:27.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: No filesystems configured 2024-09-13T14:12:27.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.209+0000 7fae0b356640 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T14:12:27.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.209+0000 7fae0b356640 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.209+0000 7fae0b356640 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.209+0000 7fae0b356640 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.209+0000 7fae0b356640 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.209+0000 7fae0b356640 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.213+0000 7fae0b356640 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.213+0000 7fae0b356640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.213+0000 7fae0b356640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.213+0000 7fae0b356640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:11:55.786410+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi177", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:11:56.726004+0000 mgr.y (mgr.14150) 24 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cephadm 2024-09-13T14:11:56.809453+0000 mgr.y (mgr.14150) 25 : cephadm [INF] Deploying cephadm binary to smithi177 2024-09-13T14:12:27.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:11:58.726328+0000 mgr.y (mgr.14150) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:11:59.120955+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cephadm 2024-09-13T14:11:59.121765+0000 mgr.y (mgr.14150) 27 : cephadm [INF] Added host smithi177 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:11:59.122593+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:11:59.382653+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:00.643101+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:00.726737+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:01.392400+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:02.727165+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:04.727619+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:06.672111+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:06.679324+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:06.687614+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:06.693165+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:06.694538+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi177", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:06.696487+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:06.698082+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cephadm 2024-09-13T14:12:06.700256+0000 mgr.y (mgr.14150) 31 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:06.728031+0000 mgr.y (mgr.14150) 32 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cephadm 2024-09-13T14:12:06.859211+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cephadm 2024-09-13T14:12:07.000699+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cephadm 2024-09-13T14:12:07.137619+0000 mgr.y (mgr.14150) 35 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:07.242786+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:07.250324+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:07.253754+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:07.420077+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:12:27.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:08.728435+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:10.728790+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:12.729105+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:14.729548+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:15.563865+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.99:0/4075393359' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:15.794198+0000 mon.a (mon.0) 127 : audit [INF] from='client.? 172.21.15.99:0/4075393359' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:15.794290+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:16.729997+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:18.730377+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:20.312124+0000 mgr.y (mgr.14150) 43 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi099:172.21.15.99=a;smithi099:[v2:172.21.15.99:3301,v1:172.21.15.99:6790]=c;smithi177:172.21.15.177=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cephadm 2024-09-13T14:12:20.317253+0000 mgr.y (mgr.14150) 44 : cephadm [INF] Saving service mon spec with placement smithi099:172.21.15.99=a;smithi099:[v2:172.21.15.99:3301,v1:172.21.15.99:6790]=c;smithi177:172.21.15.177=b;count:3 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:20.322704+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:20.324343+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:27.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:20.327887+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:20.329583+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:20.337986+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:20.341653+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:20.343179+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cephadm 2024-09-13T14:12:20.345039+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Deploying daemon mon.b on smithi177 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:20.730802+0000 mgr.y (mgr.14150) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:22.731239+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: cluster 2024-09-13T14:12:24.731587+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:25.847735+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:25.853904+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:25.860588+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:25.861755+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:27.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:25.863476+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:27.372 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: audit 2024-09-13T14:12:26.165524+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.177:0/1724937396' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:12:27.372 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:27 smithi177 bash[24494]: debug 2024-09-13T14:12:27.213+0000 7fae0b356640 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T14:12:27.941 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:12:27.941 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:12:29.589 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:12:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:12:29.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:12:29.919 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:12:29.919 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:12:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:12:29.920 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:29 smithi099 systemd[1]: Started Ceph mon.c for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:12:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.842+0000 7f84b5ac6900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:12:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.842+0000 7f84b5ac6900 0 ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T14:12:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.842+0000 7f84b5ac6900 0 pidfile_write: ignore empty --pid-file 2024-09-13T14:12:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 0 load: jerasure load: lrc 2024-09-13T14:12:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: RocksDB version: 6.15.5 2024-09-13T14:12:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T14:12:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Compile date Sep 12 2024 2024-09-13T14:12:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: DB SUMMARY 2024-09-13T14:12:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: DB Session ID: I7XPMAVEV01X4EW0MXPK 2024-09-13T14:12:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: CURRENT file: CURRENT 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.error_if_exists: 0 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.create_if_missing: 0 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.env: 0x55a83cff2440 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.fs: Posix File System 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.info_log: 0x55a83eca0ee0 2024-09-13T14:12:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:12:31.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.statistics: (nil) 2024-09-13T14:12:31.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.use_fsync: 0 2024-09-13T14:12:31.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T14:12:31.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:12:31.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:12:31.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.db_log_dir: 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:12:31.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.write_buffer_manager: 0x55a83ed5e810 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:12:31.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.unordered_write: 0 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.row_cache: None 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.wal_filter: None 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.preserve_deletes: 0 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.two_write_queues: 0 2024-09-13T14:12:31.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.atomic_flush: 0 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T14:12:31.081 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_open_files: -1 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:12:31.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Compression algorithms supported: 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: kZSTD supported: 0 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: kXpressCompression supported: 0 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: kZlibCompression supported: 1 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.846+0000 7f84b5ac6900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-13T14:12:31.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T14:12:31.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:12:31.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.merge_operator: 2024-09-13T14:12:31.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_filter: None 2024-09-13T14:12:31.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:12:31.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a83ec49a38) 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cache_index_and_filter_blocks: 1 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: pin_top_level_index_and_filter: 1 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: index_type: 0 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: data_block_index_type: 0 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: index_shortening: 1 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: hash_index_allow_collision: 1 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: checksum: 1 2024-09-13T14:12:31.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: no_block_cache: 0 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: block_cache: 0x55a83ec7d1f0 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: block_cache_name: BinnedLRUCache 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: block_cache_options: 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: capacity : 536870912 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: num_shard_bits : 4 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: strict_capacity_limit : 0 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: high_pri_pool_ratio: 0.000 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: block_cache_compressed: (nil) 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: persistent_cache: (nil) 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: block_size: 4096 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: block_size_deviation: 10 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: block_restart_interval: 16 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: index_block_restart_interval: 1 2024-09-13T14:12:31.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: metadata_block_size: 4096 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: partition_filters: 0 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: use_delta_encoding: 1 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: whole_key_filtering: 1 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: verify_compression: 0 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: read_amp_bytes_per_bit: 0 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: format_version: 4 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: enable_index_compression: 1 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: block_align: 0 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compression: NoCompression 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:12:31.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.num_levels: 7 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:12:31.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:12:31.090 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:12:31.091 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:12:31.092 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.bloom_locality: 0 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.ttl: 2592000 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.enable_blob_files: false 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.min_blob_size: 0 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:12:31.093 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726236750855472, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726236750855850, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726236750, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "6623fa0e-6c65-4beb-a183-460ada02465e", "db_session_id": "I7XPMAVEV01X4EW0MXPK"}} 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726236750856279, "job": 1, "event": "recovery_finished"} 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:12:31.094 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55a83ecb2700 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 4 rocksdb: DB pointer 0x55a83ec80000 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84ab892640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84ab892640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ** DB Stats ** 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ** Compaction Stats [default] ** 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:12:31.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ** Compaction Stats [default] ** 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:12:31.096 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Cumulative compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Interval compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ** Compaction Stats [default] ** 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ** Compaction Stats [default] ** 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:12:31.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.1 0.00 0.00 1 0.000 0 0 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Cumulative compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.850+0000 7f84b5ac6900 0 using public_addrv [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] 2024-09-13T14:12:31.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.854+0000 7f84b5ac6900 0 starting mon.c rank -1 at public addrs [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] at bind addrs [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.854+0000 7f84b5ac6900 1 mon.c@-1(???) e0 preinit fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 0 mon.c@-1(synchronizing).mds e1 new map 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 0 mon.c@-1(synchronizing).mds e1 print_map 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: e1 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: legacy client fscid: -1 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: No filesystems configured 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T14:12:31.099 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.874+0000 7f84ae898640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:11:55.786410+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi177", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:11:56.726004+0000 mgr.y (mgr.14150) 24 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cephadm 2024-09-13T14:11:56.809453+0000 mgr.y (mgr.14150) 25 : cephadm [INF] Deploying cephadm binary to smithi177 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:11:58.726328+0000 mgr.y (mgr.14150) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:11:59.120955+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cephadm 2024-09-13T14:11:59.121765+0000 mgr.y (mgr.14150) 27 : cephadm [INF] Added host smithi177 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:11:59.122593+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:11:59.382653+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.100 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:00.643101+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:00.726737+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:01.392400+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:02.727165+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:04.727619+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:06.672111+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:06.679324+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:06.687614+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:06.693165+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:06.694538+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi177", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:06.696487+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:06.698082+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cephadm 2024-09-13T14:12:06.700256+0000 mgr.y (mgr.14150) 31 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:06.728031+0000 mgr.y (mgr.14150) 32 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cephadm 2024-09-13T14:12:06.859211+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cephadm 2024-09-13T14:12:07.000699+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cephadm 2024-09-13T14:12:07.137619+0000 mgr.y (mgr.14150) 35 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:07.242786+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:07.250324+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:07.253754+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:07.420077+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:08.728435+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:10.728790+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:12.729105+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:14.729548+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:15.563865+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.99:0/4075393359' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:15.794198+0000 mon.a (mon.0) 127 : audit [INF] from='client.? 172.21.15.99:0/4075393359' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:15.794290+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:31.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:16.729997+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:18.730377+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:20.312124+0000 mgr.y (mgr.14150) 43 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi099:172.21.15.99=a;smithi099:[v2:172.21.15.99:3301,v1:172.21.15.99:6790]=c;smithi177:172.21.15.177=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cephadm 2024-09-13T14:12:20.317253+0000 mgr.y (mgr.14150) 44 : cephadm [INF] Saving service mon spec with placement smithi099:172.21.15.99=a;smithi099:[v2:172.21.15.99:3301,v1:172.21.15.99:6790]=c;smithi177:172.21.15.177=b;count:3 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:20.322704+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:20.324343+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:20.327887+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:20.329583+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:20.337986+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:20.341653+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:20.343179+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cephadm 2024-09-13T14:12:20.345039+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Deploying daemon mon.b on smithi177 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:20.730802+0000 mgr.y (mgr.14150) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:22.731239+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.103 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: cluster 2024-09-13T14:12:24.731587+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:31.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:25.847735+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:25.853904+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:25.860588+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:31.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:25.861755+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:31.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:25.863476+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:31.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: audit 2024-09-13T14:12:26.165524+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.177:0/1724937396' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:12:31.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:30 smithi099 bash[32082]: debug 2024-09-13T14:12:30.878+0000 7f84ae898640 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T14:12:31.649 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:12:32.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:26.731985+0000 mgr.y (mgr.14150) 50 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:32.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:27.225603+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:27.228170+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:12:32.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:27.228398+0000 mon.a (mon.0) 144 : cluster [INF] mon.a calling monitor election 2024-09-13T14:12:32.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:27.250443+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:28.226470+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:28.732325+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:32.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:29.226521+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:29.227179+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-09-13T14:12:32.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:30.226714+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:30.732724+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:32.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:30.886510+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:32.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:31.226949+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:31.887640+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:32.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:32.227096+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:32.271770+0000 mon.a (mon.0) 153 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:12:32.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:32.282319+0000 mon.a (mon.0) 154 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:12:32.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:32.302775+0000 mon.a (mon.0) 155 : cluster [DBG] fsmap 2024-09-13T14:12:32.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:32.302828+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:32.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:32.303094+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e13: y(active, since 73s) 2024-09-13T14:12:32.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: cluster 2024-09-13T14:12:32.303329+0000 mon.a (mon.0) 158 : cluster [INF] overall HEALTH_OK 2024-09-13T14:12:32.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:32.309709+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:32.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:32.316740+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:32.618 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:32.324734+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:32.618 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:32 smithi177 bash[24494]: audit 2024-09-13T14:12:32.332912+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:32.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:26.731985+0000 mgr.y (mgr.14150) 50 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:27.225603+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:27.228170+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:27.228398+0000 mon.a (mon.0) 144 : cluster [INF] mon.a calling monitor election 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:27.250443+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:28.226470+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:28.732325+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:29.226521+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:29.227179+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:30.226714+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:30.732724+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:30.886510+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:31.226949+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:31.887640+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:32.227096+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:32.271770+0000 mon.a (mon.0) 153 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:32.282319+0000 mon.a (mon.0) 154 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:32.302775+0000 mon.a (mon.0) 155 : cluster [DBG] fsmap 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:32.302828+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:32.303094+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e13: y(active, since 73s) 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: cluster 2024-09-13T14:12:32.303329+0000 mon.a (mon.0) 158 : cluster [INF] overall HEALTH_OK 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:32.309709+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:32.316740+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:32.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:32.324734+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:32.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:32 smithi099 bash[22656]: audit 2024-09-13T14:12:32.332912+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:33.574 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:12:33 smithi099 bash[22909]: debug 2024-09-13T14:12:33.222+0000 7f9d737c6640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-13T14:12:38.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:32.733168+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:38.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:32.891961+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:32.894644+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:32.895118+0000 mon.a (mon.0) 166 : cluster [INF] mon.a calling monitor election 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:32.915321+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:32.915601+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:33.888007+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:34.733491+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:34.888136+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:35.888144+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:36.733931+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:36.888274+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:37.888487+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:37.915388+0000 mon.a (mon.0) 174 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:12:38.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:37.921051+0000 mon.a (mon.0) 175 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:12:38.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:37.922163+0000 mon.a (mon.0) 176 : cluster [DBG] fsmap 2024-09-13T14:12:38.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:37.922220+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:38.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:37.922486+0000 mon.a (mon.0) 178 : cluster [DBG] mgrmap e13: y(active, since 79s) 2024-09-13T14:12:38.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:37.922785+0000 mon.a (mon.0) 179 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T14:12:38.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:37.943427+0000 mon.a (mon.0) 180 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-13T14:12:38.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:37.943460+0000 mon.a (mon.0) 181 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T14:12:38.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: cluster 2024-09-13T14:12:37.943496+0000 mon.a (mon.0) 182 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] is down (out of quorum) 2024-09-13T14:12:38.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:37.948154+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:38.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:37.953108+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:38.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:37.958273+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:38.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:37.963783+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:38.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:37 smithi099 bash[22656]: audit 2024-09-13T14:12:37.969358+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:38.331 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 3 2024-09-13T14:12:38.332 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-09-13T14:12:38.332 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":3,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","modified":"2024-09-13T14:12:32.888208Z","created":"2024-09-13T14:10:16.718964Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:3300","nonce":0},{"type":"v1","addr":"172.21.15.99:6789","nonce":0}]},"addr":"172.21.15.99:6789/0","public_addr":"172.21.15.99:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:3300","nonce":0},{"type":"v1","addr":"172.21.15.177:6789","nonce":0}]},"addr":"172.21.15.177:6789/0","public_addr":"172.21.15.177:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:3301","nonce":0},{"type":"v1","addr":"172.21.15.99:6790","nonce":0}]},"addr":"172.21.15.99:6790/0","public_addr":"172.21.15.99:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T14:12:38.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:32.733168+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:38.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:32.891961+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T14:12:38.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:32.894644+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:12:38.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:32.895118+0000 mon.a (mon.0) 166 : cluster [INF] mon.a calling monitor election 2024-09-13T14:12:38.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:32.915321+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:38.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:32.915601+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:33.888007+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:34.733491+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:38.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:34.888136+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:35.888144+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:36.733931+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:38.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:36.888274+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:37.888487+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:37.915388+0000 mon.a (mon.0) 174 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:37.921051+0000 mon.a (mon.0) 175 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:37.922163+0000 mon.a (mon.0) 176 : cluster [DBG] fsmap 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:37.922220+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:37.922486+0000 mon.a (mon.0) 178 : cluster [DBG] mgrmap e13: y(active, since 79s) 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:37.922785+0000 mon.a (mon.0) 179 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:37.943427+0000 mon.a (mon.0) 180 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:37.943460+0000 mon.a (mon.0) 181 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: cluster 2024-09-13T14:12:37.943496+0000 mon.a (mon.0) 182 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] is down (out of quorum) 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:37.948154+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:37.953108+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:38.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:37.958273+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:38.348 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:37.963783+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:38.348 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:37 smithi177 bash[24494]: audit 2024-09-13T14:12:37.969358+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:38.963 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T14:12:38.964 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T14:12:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.331950+0000 mon.a (mon.0) 188 : audit [DBG] from='client.? 172.21.15.177:0/2044310959' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:12:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.369923+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.370880+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: cephadm 2024-09-13T14:12:38.371977+0000 mgr.y (mgr.14150) 56 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:12:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: cephadm 2024-09-13T14:12:38.372145+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:12:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.606817+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.610575+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.619029+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.623046+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.627613+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.659347+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.660266+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:12:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.660870+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:38 smithi099 bash[22656]: audit 2024-09-13T14:12:38.887807+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:39.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.331950+0000 mon.a (mon.0) 188 : audit [DBG] from='client.? 172.21.15.177:0/2044310959' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:12:39.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.369923+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:39.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.370880+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:39.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: cephadm 2024-09-13T14:12:38.371977+0000 mgr.y (mgr.14150) 56 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:12:39.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: cephadm 2024-09-13T14:12:38.372145+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:12:39.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.606817+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:39.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.610575+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:39.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.619029+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:39.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.623046+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:39.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.627613+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:39.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.659347+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:39.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.660266+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:12:39.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.660870+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:39.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:38 smithi177 bash[24494]: audit 2024-09-13T14:12:38.887807+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:40.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:26.731985+0000 mgr.y (mgr.14150) 50 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:40.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:27.225603+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:40.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:27.228170+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:12:40.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:27.228398+0000 mon.a (mon.0) 144 : cluster [INF] mon.a calling monitor election 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:27.250443+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:28.226470+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:28.732325+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:29.226521+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:29.227179+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:30.226714+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:30.732724+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:30.886510+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:31.226949+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:31.887640+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:32.227096+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:40.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:32.271770+0000 mon.a (mon.0) 153 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:12:40.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:32.282319+0000 mon.a (mon.0) 154 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:12:40.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:32.302775+0000 mon.a (mon.0) 155 : cluster [DBG] fsmap 2024-09-13T14:12:40.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:32.302828+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:40.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:32.303094+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e13: y(active, since 73s) 2024-09-13T14:12:40.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:32.303329+0000 mon.a (mon.0) 158 : cluster [INF] overall HEALTH_OK 2024-09-13T14:12:40.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:32.309709+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:32.316740+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:32.324734+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:32.332912+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:32.733168+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:32.891961+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:32.894644+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:32.895118+0000 mon.a (mon.0) 166 : cluster [INF] mon.a calling monitor election 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:32.915321+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:32.915601+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:33.888007+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:34.733491+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:34.888136+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:35.888144+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:36.733931+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:40.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:36.888274+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:37.888487+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:37.915388+0000 mon.a (mon.0) 174 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:37.921051+0000 mon.a (mon.0) 175 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:37.922163+0000 mon.a (mon.0) 176 : cluster [DBG] fsmap 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:37.922220+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:37.922486+0000 mon.a (mon.0) 178 : cluster [DBG] mgrmap e13: y(active, since 79s) 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:37.922785+0000 mon.a (mon.0) 179 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:37.943427+0000 mon.a (mon.0) 180 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:37.943460+0000 mon.a (mon.0) 181 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cluster 2024-09-13T14:12:37.943496+0000 mon.a (mon.0) 182 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] is down (out of quorum) 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:37.948154+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:37.953108+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:37.958273+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:37.963783+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:37.969358+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.331950+0000 mon.a (mon.0) 188 : audit [DBG] from='client.? 172.21.15.177:0/2044310959' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.369923+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.370880+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cephadm 2024-09-13T14:12:38.371977+0000 mgr.y (mgr.14150) 56 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: cephadm 2024-09-13T14:12:38.372145+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.606817+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.610575+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.619029+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.623046+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.627613+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:40.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.659347+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:40.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.660266+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:12:40.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.660870+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:40.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:39 smithi099 bash[32082]: audit 2024-09-13T14:12:38.887807+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:41.034 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:34.890176+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.931667+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.931782+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.932585+0000 mon.a (mon.0) 201 : cluster [INF] mon.a calling monitor election 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.964379+0000 mon.a (mon.0) 202 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.974589+0000 mon.a (mon.0) 203 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.975589+0000 mon.a (mon.0) 204 : cluster [DBG] fsmap 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.975642+0000 mon.a (mon.0) 205 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.975897+0000 mon.a (mon.0) 206 : cluster [DBG] mgrmap e13: y(active, since 81s) 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.976168+0000 mon.a (mon.0) 207 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.976202+0000 mon.a (mon.0) 208 : cluster [INF] Cluster is now healthy 2024-09-13T14:12:41.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[32082]: cluster 2024-09-13T14:12:39.983355+0000 mon.a (mon.0) 209 : cluster [INF] overall HEALTH_OK 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:34.890176+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.931667+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.931782+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.932585+0000 mon.a (mon.0) 201 : cluster [INF] mon.a calling monitor election 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.964379+0000 mon.a (mon.0) 202 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.974589+0000 mon.a (mon.0) 203 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.975589+0000 mon.a (mon.0) 204 : cluster [DBG] fsmap 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.975642+0000 mon.a (mon.0) 205 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.975897+0000 mon.a (mon.0) 206 : cluster [DBG] mgrmap e13: y(active, since 81s) 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.976168+0000 mon.a (mon.0) 207 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.976202+0000 mon.a (mon.0) 208 : cluster [INF] Cluster is now healthy 2024-09-13T14:12:41.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:40 smithi099 bash[22656]: cluster 2024-09-13T14:12:39.983355+0000 mon.a (mon.0) 209 : cluster [INF] overall HEALTH_OK 2024-09-13T14:12:41.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:34.890176+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T14:12:41.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.931667+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-09-13T14:12:41.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.931782+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-09-13T14:12:41.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.932585+0000 mon.a (mon.0) 201 : cluster [INF] mon.a calling monitor election 2024-09-13T14:12:41.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.964379+0000 mon.a (mon.0) 202 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:12:41.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.974589+0000 mon.a (mon.0) 203 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:12:41.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.975589+0000 mon.a (mon.0) 204 : cluster [DBG] fsmap 2024-09-13T14:12:41.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.975642+0000 mon.a (mon.0) 205 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:12:41.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.975897+0000 mon.a (mon.0) 206 : cluster [DBG] mgrmap e13: y(active, since 81s) 2024-09-13T14:12:41.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.976168+0000 mon.a (mon.0) 207 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:12:41.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.976202+0000 mon.a (mon.0) 208 : cluster [INF] Cluster is now healthy 2024-09-13T14:12:41.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:40 smithi177 bash[24494]: cluster 2024-09-13T14:12:39.983355+0000 mon.a (mon.0) 209 : cluster [INF] overall HEALTH_OK 2024-09-13T14:12:42.015 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[22656]: cluster 2024-09-13T14:12:40.734622+0000 mgr.y (mgr.14150) 63 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:42.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[22656]: audit 2024-09-13T14:12:40.736311+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:42.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[22656]: audit 2024-09-13T14:12:40.741355+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:42.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[22656]: cephadm 2024-09-13T14:12:40.742304+0000 mgr.y (mgr.14150) 64 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T14:12:42.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[22656]: audit 2024-09-13T14:12:40.742770+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:42.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[22656]: audit 2024-09-13T14:12:40.744302+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:12:42.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[22656]: audit 2024-09-13T14:12:40.745740+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:42.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[22656]: cephadm 2024-09-13T14:12:40.747322+0000 mgr.y (mgr.14150) 65 : cephadm [INF] Reconfiguring daemon mon.c on smithi099 2024-09-13T14:12:42.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[22656]: audit 2024-09-13T14:12:40.888696+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:42.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[32082]: cluster 2024-09-13T14:12:40.734622+0000 mgr.y (mgr.14150) 63 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:42.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[32082]: audit 2024-09-13T14:12:40.736311+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:42.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[32082]: audit 2024-09-13T14:12:40.741355+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:42.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[32082]: cephadm 2024-09-13T14:12:40.742304+0000 mgr.y (mgr.14150) 64 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T14:12:42.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[32082]: audit 2024-09-13T14:12:40.742770+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:42.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[32082]: audit 2024-09-13T14:12:40.744302+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:12:42.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[32082]: audit 2024-09-13T14:12:40.745740+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:42.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[32082]: cephadm 2024-09-13T14:12:40.747322+0000 mgr.y (mgr.14150) 65 : cephadm [INF] Reconfiguring daemon mon.c on smithi099 2024-09-13T14:12:42.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:41 smithi099 bash[32082]: audit 2024-09-13T14:12:40.888696+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:42.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:41 smithi177 bash[24494]: cluster 2024-09-13T14:12:40.734622+0000 mgr.y (mgr.14150) 63 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:42.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:41 smithi177 bash[24494]: audit 2024-09-13T14:12:40.736311+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:42.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:41 smithi177 bash[24494]: audit 2024-09-13T14:12:40.741355+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:42.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:41 smithi177 bash[24494]: cephadm 2024-09-13T14:12:40.742304+0000 mgr.y (mgr.14150) 64 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T14:12:42.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:41 smithi177 bash[24494]: audit 2024-09-13T14:12:40.742770+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:42.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:41 smithi177 bash[24494]: audit 2024-09-13T14:12:40.744302+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:12:42.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:41 smithi177 bash[24494]: audit 2024-09-13T14:12:40.745740+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:42.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:41 smithi177 bash[24494]: cephadm 2024-09-13T14:12:40.747322+0000 mgr.y (mgr.14150) 65 : cephadm [INF] Reconfiguring daemon mon.c on smithi099 2024-09-13T14:12:42.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:41 smithi177 bash[24494]: audit 2024-09-13T14:12:40.888696+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:12:43.685 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:12:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:43 smithi177 bash[24494]: audit 2024-09-13T14:12:42.609285+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:43.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:43 smithi177 bash[24494]: audit 2024-09-13T14:12:42.613515+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:43.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:43 smithi177 bash[24494]: cephadm 2024-09-13T14:12:42.614337+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T14:12:43.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:43 smithi177 bash[24494]: audit 2024-09-13T14:12:42.614674+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:43.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:43 smithi177 bash[24494]: audit 2024-09-13T14:12:42.615586+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:12:43.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:43 smithi177 bash[24494]: audit 2024-09-13T14:12:42.616383+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:43.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:43 smithi177 bash[24494]: cephadm 2024-09-13T14:12:42.617325+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring daemon mon.b on smithi177 2024-09-13T14:12:43.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:43 smithi177 bash[24494]: cluster 2024-09-13T14:12:42.734954+0000 mgr.y (mgr.14150) 68 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:44.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[22656]: audit 2024-09-13T14:12:42.609285+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:44.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[22656]: audit 2024-09-13T14:12:42.613515+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:44.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[22656]: cephadm 2024-09-13T14:12:42.614337+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T14:12:44.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[22656]: audit 2024-09-13T14:12:42.614674+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:44.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[22656]: audit 2024-09-13T14:12:42.615586+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:12:44.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[22656]: audit 2024-09-13T14:12:42.616383+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:44.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[22656]: cephadm 2024-09-13T14:12:42.617325+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring daemon mon.b on smithi177 2024-09-13T14:12:44.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[22656]: cluster 2024-09-13T14:12:42.734954+0000 mgr.y (mgr.14150) 68 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[32082]: audit 2024-09-13T14:12:42.609285+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[32082]: audit 2024-09-13T14:12:42.613515+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[32082]: cephadm 2024-09-13T14:12:42.614337+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T14:12:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[32082]: audit 2024-09-13T14:12:42.614674+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:12:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[32082]: audit 2024-09-13T14:12:42.615586+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:12:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[32082]: audit 2024-09-13T14:12:42.616383+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[32082]: cephadm 2024-09-13T14:12:42.617325+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring daemon mon.b on smithi177 2024-09-13T14:12:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:43 smithi099 bash[32082]: cluster 2024-09-13T14:12:42.734954+0000 mgr.y (mgr.14150) 68 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:45.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[32082]: audit 2024-09-13T14:12:44.586222+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:45.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[32082]: audit 2024-09-13T14:12:44.595345+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:45.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[32082]: audit 2024-09-13T14:12:44.597438+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:45.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[32082]: audit 2024-09-13T14:12:44.600534+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:45.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[32082]: audit 2024-09-13T14:12:44.602032+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:45.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[32082]: audit 2024-09-13T14:12:44.613410+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:45.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[32082]: cluster 2024-09-13T14:12:44.735341+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:45.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[22656]: audit 2024-09-13T14:12:44.586222+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:45.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[22656]: audit 2024-09-13T14:12:44.595345+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:45.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[22656]: audit 2024-09-13T14:12:44.597438+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:45.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[22656]: audit 2024-09-13T14:12:44.600534+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:45.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[22656]: audit 2024-09-13T14:12:44.602032+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:45.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[22656]: audit 2024-09-13T14:12:44.613410+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:45.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:45 smithi099 bash[22656]: cluster 2024-09-13T14:12:44.735341+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:45.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:45 smithi177 bash[24494]: audit 2024-09-13T14:12:44.586222+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:45.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:45 smithi177 bash[24494]: audit 2024-09-13T14:12:44.595345+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:45.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:45 smithi177 bash[24494]: audit 2024-09-13T14:12:44.597438+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:45.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:45 smithi177 bash[24494]: audit 2024-09-13T14:12:44.600534+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:45.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:45 smithi177 bash[24494]: audit 2024-09-13T14:12:44.602032+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:45.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:45 smithi177 bash[24494]: audit 2024-09-13T14:12:44.613410+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:45.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:45 smithi177 bash[24494]: cluster 2024-09-13T14:12:44.735341+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:46.640 INFO:teuthology.orchestra.run.smithi099.stdout:# minimal ceph.conf for af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:12:46.640 INFO:teuthology.orchestra.run.smithi099.stdout:[global] 2024-09-13T14:12:46.640 INFO:teuthology.orchestra.run.smithi099.stdout: fsid = af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:12:46.640 INFO:teuthology.orchestra.run.smithi099.stdout: mon_host = [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] 2024-09-13T14:12:47.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:46 smithi099 bash[22656]: audit 2024-09-13T14:12:46.640225+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.99:0/3965573815' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:47.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:46 smithi099 bash[32082]: audit 2024-09-13T14:12:46.640225+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.99:0/3965573815' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:47.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:46 smithi177 bash[24494]: audit 2024-09-13T14:12:46.640225+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.99:0/3965573815' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:47.295 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T14:12:47.296 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:12:47.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T14:12:47.309 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:12:47.309 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:47.367 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:12:47.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T14:12:47.382 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:12:47.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:12:47.438 INFO:tasks.cephadm:Adding mgr.y on smithi099 2024-09-13T14:12:47.438 INFO:tasks.cephadm:Adding mgr.x on smithi177 2024-09-13T14:12:47.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi099=y;smithi177=x' 2024-09-13T14:12:48.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:47 smithi099 bash[22656]: cluster 2024-09-13T14:12:46.735617+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:47 smithi099 bash[32082]: cluster 2024-09-13T14:12:46.735617+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:48.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:47 smithi177 bash[24494]: cluster 2024-09-13T14:12:46.735617+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:50.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:49 smithi099 bash[22656]: cluster 2024-09-13T14:12:48.735974+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:50.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:49 smithi099 bash[32082]: cluster 2024-09-13T14:12:48.735974+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:50.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:49 smithi177 bash[24494]: cluster 2024-09-13T14:12:48.735974+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:51.181 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:12:52.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:51 smithi099 bash[22656]: cluster 2024-09-13T14:12:50.736361+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:52.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:51 smithi099 bash[32082]: cluster 2024-09-13T14:12:50.736361+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:52.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:51 smithi177 bash[24494]: cluster 2024-09-13T14:12:50.736361+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:54.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:53 smithi099 bash[22656]: cluster 2024-09-13T14:12:52.736747+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:54.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:53 smithi099 bash[32082]: cluster 2024-09-13T14:12:52.736747+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:54.090 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled mgr update... 2024-09-13T14:12:54.101 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:53 smithi177 bash[24494]: cluster 2024-09-13T14:12:52.736747+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:54.934 DEBUG:teuthology.orchestra.run.smithi177:mgr.x> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.x.service 2024-09-13T14:12:54.937 DEBUG:tasks.cephadm:set 0 configs 2024-09-13T14:12:54.938 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T14:12:54.938 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:12:54.938 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T14:12:54.944 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T14:12:54.944 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_1 2024-09-13T14:12:54.994 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T14:12:54.994 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:12:54.994 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 878 Links: 1 2024-09-13T14:12:54.994 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:12:54.994 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-09-13 14:03:31.738451186 +0000 2024-09-13T14:12:54.994 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-09-13 14:03:31.482453472 +0000 2024-09-13T14:12:54.995 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-09-13 14:03:31.482453472 +0000 2024-09-13T14:12:54.995 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2024-09-13T14:12:54.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T14:12:55.049 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-09-13T14:12:55.049 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-09-13T14:12:55.049 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000475033 s, 1.1 MB/s 2024-09-13T14:12:55.050 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T14:12:55.098 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_2 2024-09-13T14:12:55.146 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T14:12:55.146 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:12:55.146 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 892 Links: 1 2024-09-13T14:12:55.146 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:12:55.146 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-09-13 14:03:32.222446863 +0000 2024-09-13T14:12:55.146 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-09-13 14:03:31.982449006 +0000 2024-09-13T14:12:55.146 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-09-13 14:03:31.982449006 +0000 2024-09-13T14:12:55.146 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2024-09-13T14:12:55.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T14:12:55.200 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-09-13T14:12:55.200 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-09-13T14:12:55.200 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000310566 s, 1.6 MB/s 2024-09-13T14:12:55.201 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T14:12:55.251 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_3 2024-09-13T14:12:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T14:12:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:12:55.298 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 909 Links: 1 2024-09-13T14:12:55.298 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:12:55.298 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-09-13 14:03:32.662442933 +0000 2024-09-13T14:12:55.298 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-09-13 14:03:32.430445005 +0000 2024-09-13T14:12:55.298 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-09-13 14:03:32.430445005 +0000 2024-09-13T14:12:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2024-09-13T14:12:55.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T14:12:55.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: audit 2024-09-13T14:12:54.079504+0000 mgr.y (mgr.14150) 74 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi099=y;smithi177=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:55.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: cephadm 2024-09-13T14:12:54.082464+0000 mgr.y (mgr.14150) 75 : cephadm [INF] Saving service mgr spec with placement smithi099=y;smithi177=x;count:2 2024-09-13T14:12:55.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: audit 2024-09-13T14:12:54.089916+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:55.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: audit 2024-09-13T14:12:54.091563+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:55.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: audit 2024-09-13T14:12:54.094673+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:55.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: audit 2024-09-13T14:12:54.096383+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: audit 2024-09-13T14:12:54.106349+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: audit 2024-09-13T14:12:54.109141+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:12:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: audit 2024-09-13T14:12:54.116211+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:12:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: audit 2024-09-13T14:12:54.117929+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:12:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: audit 2024-09-13T14:12:54.119863+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[22656]: cephadm 2024-09-13T14:12:54.121793+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Deploying daemon mgr.x on smithi177 2024-09-13T14:12:55.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: audit 2024-09-13T14:12:54.079504+0000 mgr.y (mgr.14150) 74 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi099=y;smithi177=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:55.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: cephadm 2024-09-13T14:12:54.082464+0000 mgr.y (mgr.14150) 75 : cephadm [INF] Saving service mgr spec with placement smithi099=y;smithi177=x;count:2 2024-09-13T14:12:55.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: audit 2024-09-13T14:12:54.089916+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: audit 2024-09-13T14:12:54.091563+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: audit 2024-09-13T14:12:54.094673+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: audit 2024-09-13T14:12:54.096383+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: audit 2024-09-13T14:12:54.106349+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: audit 2024-09-13T14:12:54.109141+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:12:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: audit 2024-09-13T14:12:54.116211+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:12:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: audit 2024-09-13T14:12:54.117929+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:12:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: audit 2024-09-13T14:12:54.119863+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:55 smithi099 bash[32082]: cephadm 2024-09-13T14:12:54.121793+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Deploying daemon mgr.x on smithi177 2024-09-13T14:12:55.337 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-09-13T14:12:55.337 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-09-13T14:12:55.337 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000288515 s, 1.8 MB/s 2024-09-13T14:12:55.338 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T14:12:55.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: audit 2024-09-13T14:12:54.079504+0000 mgr.y (mgr.14150) 74 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi099=y;smithi177=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:12:55.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: cephadm 2024-09-13T14:12:54.082464+0000 mgr.y (mgr.14150) 75 : cephadm [INF] Saving service mgr spec with placement smithi099=y;smithi177=x;count:2 2024-09-13T14:12:55.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: audit 2024-09-13T14:12:54.089916+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:55.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: audit 2024-09-13T14:12:54.091563+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:55.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: audit 2024-09-13T14:12:54.094673+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:55.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: audit 2024-09-13T14:12:54.096383+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:12:55.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: audit 2024-09-13T14:12:54.106349+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:55.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: audit 2024-09-13T14:12:54.109141+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:12:55.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: audit 2024-09-13T14:12:54.116211+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:12:55.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: audit 2024-09-13T14:12:54.117929+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:12:55.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: audit 2024-09-13T14:12:54.119863+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:12:55.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:55 smithi177 bash[24494]: cephadm 2024-09-13T14:12:54.121793+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Deploying daemon mgr.x on smithi177 2024-09-13T14:12:55.387 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_4 2024-09-13T14:12:55.433 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T14:12:55.433 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:12:55.433 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 929 Links: 1 2024-09-13T14:12:55.433 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:12:55.433 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-09-13 14:03:33.158438503 +0000 2024-09-13T14:12:55.433 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-09-13 14:03:32.910440718 +0000 2024-09-13T14:12:55.434 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-09-13 14:03:32.910440718 +0000 2024-09-13T14:12:55.434 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2024-09-13T14:12:55.434 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T14:12:55.487 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-09-13T14:12:55.487 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-09-13T14:12:55.487 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000283695 s, 1.8 MB/s 2024-09-13T14:12:55.488 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T14:12:55.538 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:12:55.539 DEBUG:teuthology.orchestra.run.smithi177:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T14:12:55.545 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T14:12:55.545 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_1 2024-09-13T14:12:55.594 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T14:12:55.594 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:12:55.594 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 5h/5d Inode: 877 Links: 1 2024-09-13T14:12:55.594 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:12:55.594 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2024-09-13 14:03:30.704513821 +0000 2024-09-13T14:12:55.594 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2024-09-13 14:03:30.444516144 +0000 2024-09-13T14:12:55.594 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2024-09-13 14:03:30.444516144 +0000 2024-09-13T14:12:55.594 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: - 2024-09-13T14:12:55.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T14:12:55.649 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2024-09-13T14:12:55.649 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2024-09-13T14:12:55.649 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000334028 s, 1.5 MB/s 2024-09-13T14:12:55.650 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T14:12:55.698 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_2 2024-09-13T14:12:55.749 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T14:12:55.749 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:12:55.749 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 5h/5d Inode: 896 Links: 1 2024-09-13T14:12:55.749 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:12:55.750 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2024-09-13 14:03:30.928511821 +0000 2024-09-13T14:12:55.750 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2024-09-13 14:03:30.924511856 +0000 2024-09-13T14:12:55.750 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2024-09-13 14:03:30.924511856 +0000 2024-09-13T14:12:55.750 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: - 2024-09-13T14:12:55.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T14:12:55.801 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2024-09-13T14:12:55.801 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2024-09-13T14:12:55.801 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000278894 s, 1.8 MB/s 2024-09-13T14:12:55.802 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T14:12:55.849 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_3 2024-09-13T14:12:55.896 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T14:12:55.897 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:12:55.897 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 5h/5d Inode: 912 Links: 1 2024-09-13T14:12:55.897 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:12:55.897 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2024-09-13 14:03:31.664505246 +0000 2024-09-13T14:12:55.897 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2024-09-13 14:03:31.408507533 +0000 2024-09-13T14:12:55.897 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2024-09-13 14:03:31.408507533 +0000 2024-09-13T14:12:55.897 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: - 2024-09-13T14:12:55.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T14:12:55.952 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2024-09-13T14:12:55.953 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2024-09-13T14:12:55.953 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000299402 s, 1.7 MB/s 2024-09-13T14:12:55.954 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T14:12:56.002 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_4 2024-09-13T14:12:56.049 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T14:12:56.049 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:12:56.050 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 5h/5d Inode: 933 Links: 1 2024-09-13T14:12:56.050 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:12:56.050 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2024-09-13 14:03:32.180500637 +0000 2024-09-13T14:12:56.050 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2024-09-13 14:03:31.896503174 +0000 2024-09-13T14:12:56.050 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2024-09-13 14:03:31.896503174 +0000 2024-09-13T14:12:56.050 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: - 2024-09-13T14:12:56.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T14:12:56.101 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2024-09-13T14:12:56.101 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2024-09-13T14:12:56.101 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000284254 s, 1.8 MB/s 2024-09-13T14:12:56.102 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T14:12:56.150 INFO:tasks.cephadm:Deploying osd.0 on smithi099 with /dev/vg_nvme/lv_4... 2024-09-13T14:12:56.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T14:12:56.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:56 smithi177 bash[24494]: cluster 2024-09-13T14:12:54.737150+0000 mgr.y (mgr.14150) 77 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:56.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:56 smithi099 bash[22656]: cluster 2024-09-13T14:12:54.737150+0000 mgr.y (mgr.14150) 77 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:56 smithi099 bash[32082]: cluster 2024-09-13T14:12:54.737150+0000 mgr.y (mgr.14150) 77 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:56.729 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:12:56.729 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:12:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:12:56.729 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:12:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:12:56.729 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:12:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:12:57.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:12:57.113 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:12:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:12:57.113 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:12:56 smithi177 systemd[1]: Started Ceph mgr.x for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:12:58.113 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:12:57 smithi177 bash[25434]: debug 2024-09-13T14:12:57.944+0000 7f083e415640 1 -- 172.21.15.177:0/4070512143 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ce6851a340 con 0x55ce68527000 2024-09-13T14:12:58.113 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:12:58 smithi177 bash[25434]: debug 2024-09-13T14:12:58.052+0000 7f0840c7dec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:12:58.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:57 smithi177 bash[24494]: cluster 2024-09-13T14:12:56.737644+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:58.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:57 smithi177 bash[24494]: audit 2024-09-13T14:12:56.865767+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:57 smithi177 bash[24494]: audit 2024-09-13T14:12:56.873171+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:57 smithi177 bash[24494]: audit 2024-09-13T14:12:56.880853+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:57 smithi177 bash[24494]: audit 2024-09-13T14:12:56.888082+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:57 smithi177 bash[24494]: audit 2024-09-13T14:12:56.944724+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:58.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[22656]: cluster 2024-09-13T14:12:56.737644+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:58.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[22656]: audit 2024-09-13T14:12:56.865767+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[22656]: audit 2024-09-13T14:12:56.873171+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[22656]: audit 2024-09-13T14:12:56.880853+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[22656]: audit 2024-09-13T14:12:56.888082+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[22656]: audit 2024-09-13T14:12:56.944724+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:58.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[32082]: cluster 2024-09-13T14:12:56.737644+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:12:58.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[32082]: audit 2024-09-13T14:12:56.865767+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[32082]: audit 2024-09-13T14:12:56.873171+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[32082]: audit 2024-09-13T14:12:56.880853+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[32082]: audit 2024-09-13T14:12:56.888082+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:58.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:57 smithi099 bash[32082]: audit 2024-09-13T14:12:56.944724+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:12:59.225 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:58 smithi177 bash[24494]: audit 2024-09-13T14:12:57.955636+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:59.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:58 smithi099 bash[22656]: audit 2024-09-13T14:12:57.955636+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:59.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:58 smithi099 bash[32082]: audit 2024-09-13T14:12:57.955636+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:12:59.613 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:12:59 smithi177 bash[25434]: debug 2024-09-13T14:12:59.224+0000 7f0840c7dec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:12:59.613 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:12:59 smithi177 bash[25434]: debug 2024-09-13T14:12:59.284+0000 7f0840c7dec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:12:59.613 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:12:59 smithi177 bash[25434]: debug 2024-09-13T14:12:59.432+0000 7f0840c7dec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:13:00.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:12:59 smithi099 bash[22656]: cluster 2024-09-13T14:12:58.738047+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:00.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:12:59 smithi099 bash[32082]: cluster 2024-09-13T14:12:58.738047+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:00.350 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:12:59 smithi177 bash[24494]: cluster 2024-09-13T14:12:58.738047+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:00.350 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:00 smithi177 bash[25434]: debug 2024-09-13T14:13:00.060+0000 7f0840c7dec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:13:00.350 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:00 smithi177 bash[25434]: debug 2024-09-13T14:13:00.124+0000 7f0840c7dec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:13:00.350 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:00 smithi177 bash[25434]: debug 2024-09-13T14:13:00.192+0000 7f0840c7dec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:13:00.613 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:00 smithi177 bash[25434]: debug 2024-09-13T14:13:00.348+0000 7f0840c7dec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:13:00.613 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:00 smithi177 bash[25434]: debug 2024-09-13T14:13:00.420+0000 7f0840c7dec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:13:00.613 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:00 smithi177 bash[25434]: debug 2024-09-13T14:13:00.488+0000 7f0840c7dec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:13:00.874 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:01.014 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:00 smithi177 bash[25434]: debug 2024-09-13T14:13:00.708+0000 7f0840c7dec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:13:01.014 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:00 smithi177 bash[25434]: debug 2024-09-13T14:13:00.772+0000 7f0840c7dec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:13:01.014 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:00 smithi177 bash[25434]: debug 2024-09-13T14:13:00.936+0000 7f0840c7dec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:13:01.363 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:01 smithi177 bash[25434]: debug 2024-09-13T14:13:01.012+0000 7f0840c7dec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:13:01.363 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:01 smithi177 bash[25434]: debug 2024-09-13T14:13:01.240+0000 7f0840c7dec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:13:01.862 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:01 smithi177 bash[25434]: debug 2024-09-13T14:13:01.396+0000 7f0840c7dec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:13:01.863 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:01 smithi177 bash[25434]: debug 2024-09-13T14:13:01.472+0000 7f0840c7dec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:13:01.863 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:01 smithi177 bash[25434]: debug 2024-09-13T14:13:01.624+0000 7f0840c7dec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:13:02.236 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:01 smithi099 bash[32082]: cluster 2024-09-13T14:13:00.738452+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:02.236 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:01 smithi099 bash[32082]: audit 2024-09-13T14:13:01.880063+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:02.236 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:01 smithi099 bash[32082]: audit 2024-09-13T14:13:01.889270+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:02.237 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:01 smithi099 bash[22656]: cluster 2024-09-13T14:13:00.738452+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:02.237 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:01 smithi099 bash[22656]: audit 2024-09-13T14:13:01.880063+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:02.237 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:01 smithi099 bash[22656]: audit 2024-09-13T14:13:01.889270+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:02.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:01 smithi177 bash[24494]: cluster 2024-09-13T14:13:00.738452+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:02.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:01 smithi177 bash[24494]: audit 2024-09-13T14:13:01.880063+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:02.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:01 smithi177 bash[24494]: audit 2024-09-13T14:13:01.889270+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:02.672 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[25434]: debug 2024-09-13T14:13:02.404+0000 7f0840c7dec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:13:02.672 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[25434]: debug 2024-09-13T14:13:02.476+0000 7f0840c7dec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:13:02.966 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[25434]: debug 2024-09-13T14:13:02.668+0000 7f0840c7dec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:13:02.966 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[25434]: debug 2024-09-13T14:13:02.740+0000 7f0840c7dec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:13:03.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[22656]: audit 2024-09-13T14:13:02.278261+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:03.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[22656]: audit 2024-09-13T14:13:02.279857+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:03.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[22656]: audit 2024-09-13T14:13:02.288495+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:03.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[22656]: cephadm 2024-09-13T14:13:02.325828+0000 mgr.y (mgr.14150) 81 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:13:03.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[22656]: audit 2024-09-13T14:13:02.326508+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[22656]: audit 2024-09-13T14:13:02.328057+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[22656]: audit 2024-09-13T14:13:02.329120+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[22656]: cephadm 2024-09-13T14:13:02.330242+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring daemon mgr.y on smithi099 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[32082]: audit 2024-09-13T14:13:02.278261+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[32082]: audit 2024-09-13T14:13:02.279857+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[32082]: audit 2024-09-13T14:13:02.288495+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[32082]: cephadm 2024-09-13T14:13:02.325828+0000 mgr.y (mgr.14150) 81 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[32082]: audit 2024-09-13T14:13:02.326508+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[32082]: audit 2024-09-13T14:13:02.328057+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[32082]: audit 2024-09-13T14:13:02.329120+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:03.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:02 smithi099 bash[32082]: cephadm 2024-09-13T14:13:02.330242+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring daemon mgr.y on smithi099 2024-09-13T14:13:03.363 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:03 smithi177 bash[25434]: debug 2024-09-13T14:13:03.052+0000 7f0840c7dec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:13:03.363 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:13:03 smithi177 bash[25434]: debug 2024-09-13T14:13:03.128+0000 7f0840c7dec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:13:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[24494]: audit 2024-09-13T14:13:02.278261+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[24494]: audit 2024-09-13T14:13:02.279857+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[24494]: audit 2024-09-13T14:13:02.288495+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[24494]: cephadm 2024-09-13T14:13:02.325828+0000 mgr.y (mgr.14150) 81 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:13:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[24494]: audit 2024-09-13T14:13:02.326508+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:13:03.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[24494]: audit 2024-09-13T14:13:02.328057+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:13:03.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[24494]: audit 2024-09-13T14:13:02.329120+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:03.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:02 smithi177 bash[24494]: cephadm 2024-09-13T14:13:02.330242+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring daemon mgr.y on smithi099 2024-09-13T14:13:04.223 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[22656]: cluster 2024-09-13T14:13:02.738976+0000 mgr.y (mgr.14150) 83 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:04.224 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[22656]: cluster 2024-09-13T14:13:03.153808+0000 mon.a (mon.0) 250 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:13:04.224 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[22656]: audit 2024-09-13T14:13:03.153909+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:13:04.224 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[22656]: audit 2024-09-13T14:13:03.154654+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:13:04.224 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[22656]: audit 2024-09-13T14:13:03.155740+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:13:04.224 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[22656]: audit 2024-09-13T14:13:03.156237+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:13:04.224 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[32082]: cluster 2024-09-13T14:13:02.738976+0000 mgr.y (mgr.14150) 83 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:04.224 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[32082]: cluster 2024-09-13T14:13:03.153808+0000 mon.a (mon.0) 250 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:13:04.224 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[32082]: audit 2024-09-13T14:13:03.153909+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:13:04.224 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[32082]: audit 2024-09-13T14:13:03.154654+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:13:04.224 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[32082]: audit 2024-09-13T14:13:03.155740+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:13:04.225 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:03 smithi099 bash[32082]: audit 2024-09-13T14:13:03.156237+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:13:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:03 smithi177 bash[24494]: cluster 2024-09-13T14:13:02.738976+0000 mgr.y (mgr.14150) 83 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:03 smithi177 bash[24494]: cluster 2024-09-13T14:13:03.153808+0000 mon.a (mon.0) 250 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:13:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:03 smithi177 bash[24494]: audit 2024-09-13T14:13:03.153909+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:13:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:03 smithi177 bash[24494]: audit 2024-09-13T14:13:03.154654+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:13:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:03 smithi177 bash[24494]: audit 2024-09-13T14:13:03.155740+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:13:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:03 smithi177 bash[24494]: audit 2024-09-13T14:13:03.156237+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.177:0/2240886731' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:13:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[22656]: cluster 2024-09-13T14:13:03.990541+0000 mon.a (mon.0) 251 : cluster [DBG] mgrmap e14: y(active, since 105s), standbys: x 2024-09-13T14:13:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[22656]: audit 2024-09-13T14:13:03.990777+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:13:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[22656]: audit 2024-09-13T14:13:04.119075+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[22656]: audit 2024-09-13T14:13:04.124551+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[22656]: audit 2024-09-13T14:13:04.126262+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[22656]: audit 2024-09-13T14:13:04.128434+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[22656]: audit 2024-09-13T14:13:04.129409+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[22656]: audit 2024-09-13T14:13:04.137810+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:05.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[32082]: cluster 2024-09-13T14:13:03.990541+0000 mon.a (mon.0) 251 : cluster [DBG] mgrmap e14: y(active, since 105s), standbys: x 2024-09-13T14:13:05.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[32082]: audit 2024-09-13T14:13:03.990777+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:13:05.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[32082]: audit 2024-09-13T14:13:04.119075+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:05.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[32082]: audit 2024-09-13T14:13:04.124551+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:05.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[32082]: audit 2024-09-13T14:13:04.126262+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:05.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[32082]: audit 2024-09-13T14:13:04.128434+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:05.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[32082]: audit 2024-09-13T14:13:04.129409+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:05.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:04 smithi099 bash[32082]: audit 2024-09-13T14:13:04.137810+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:05.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:04 smithi177 bash[24494]: cluster 2024-09-13T14:13:03.990541+0000 mon.a (mon.0) 251 : cluster [DBG] mgrmap e14: y(active, since 105s), standbys: x 2024-09-13T14:13:05.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:04 smithi177 bash[24494]: audit 2024-09-13T14:13:03.990777+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:13:05.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:04 smithi177 bash[24494]: audit 2024-09-13T14:13:04.119075+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:05.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:04 smithi177 bash[24494]: audit 2024-09-13T14:13:04.124551+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:05.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:04 smithi177 bash[24494]: audit 2024-09-13T14:13:04.126262+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:05.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:04 smithi177 bash[24494]: audit 2024-09-13T14:13:04.128434+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:05.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:04 smithi177 bash[24494]: audit 2024-09-13T14:13:04.129409+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:05.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:04 smithi177 bash[24494]: audit 2024-09-13T14:13:04.137810+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:13:06.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi099:vg_nvme/lv_4 2024-09-13T14:13:06.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:06 smithi099 bash[22656]: cluster 2024-09-13T14:13:04.739431+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:06.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:06 smithi099 bash[32082]: cluster 2024-09-13T14:13:04.739431+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:06 smithi177 bash[24494]: cluster 2024-09-13T14:13:04.739431+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:08.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:08 smithi099 bash[22656]: cluster 2024-09-13T14:13:06.739862+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:08.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:08 smithi099 bash[32082]: cluster 2024-09-13T14:13:06.739862+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:08.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:08 smithi177 bash[24494]: cluster 2024-09-13T14:13:06.739862+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:10.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:10 smithi099 bash[22656]: cluster 2024-09-13T14:13:08.740261+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:10.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:10 smithi099 bash[32082]: cluster 2024-09-13T14:13:08.740261+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:10 smithi177 bash[24494]: cluster 2024-09-13T14:13:08.740261+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:10.948 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:12.293 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:12 smithi099 bash[22656]: cluster 2024-09-13T14:13:10.740671+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:12.293 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:12 smithi099 bash[32082]: cluster 2024-09-13T14:13:10.740671+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:12.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:12 smithi177 bash[24494]: cluster 2024-09-13T14:13:10.740671+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:14.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:14 smithi099 bash[22656]: cluster 2024-09-13T14:13:12.741150+0000 mgr.y (mgr.14150) 88 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:14.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:14 smithi099 bash[22656]: audit 2024-09-13T14:13:13.866912+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:14.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:14 smithi099 bash[22656]: audit 2024-09-13T14:13:13.870895+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:14.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:14 smithi099 bash[22656]: audit 2024-09-13T14:13:13.872085+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:14.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:14 smithi099 bash[32082]: cluster 2024-09-13T14:13:12.741150+0000 mgr.y (mgr.14150) 88 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:14 smithi099 bash[32082]: audit 2024-09-13T14:13:13.866912+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:14 smithi099 bash[32082]: audit 2024-09-13T14:13:13.870895+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:14 smithi099 bash[32082]: audit 2024-09-13T14:13:13.872085+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:14.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:14 smithi177 bash[24494]: cluster 2024-09-13T14:13:12.741150+0000 mgr.y (mgr.14150) 88 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:14.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:14 smithi177 bash[24494]: audit 2024-09-13T14:13:13.866912+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:13:14.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:14 smithi177 bash[24494]: audit 2024-09-13T14:13:13.870895+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:13:14.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:14 smithi177 bash[24494]: audit 2024-09-13T14:13:13.872085+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:15.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:15 smithi099 bash[22656]: audit 2024-09-13T14:13:13.860333+0000 mgr.y (mgr.14150) 89 : audit [DBG] from='client.14202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:15.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:15 smithi099 bash[32082]: audit 2024-09-13T14:13:13.860333+0000 mgr.y (mgr.14150) 89 : audit [DBG] from='client.14202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:15 smithi177 bash[24494]: audit 2024-09-13T14:13:13.860333+0000 mgr.y (mgr.14150) 89 : audit [DBG] from='client.14202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:13:16.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:16 smithi099 bash[22656]: cluster 2024-09-13T14:13:14.741603+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:16.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:16 smithi099 bash[32082]: cluster 2024-09-13T14:13:14.741603+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:16.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:16 smithi177 bash[24494]: cluster 2024-09-13T14:13:14.741603+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:18.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:18 smithi099 bash[22656]: cluster 2024-09-13T14:13:16.742052+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:18.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:18 smithi099 bash[32082]: cluster 2024-09-13T14:13:16.742052+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:18 smithi177 bash[24494]: cluster 2024-09-13T14:13:16.742052+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:20.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:20 smithi099 bash[22656]: cluster 2024-09-13T14:13:18.742445+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:20.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:20 smithi099 bash[32082]: cluster 2024-09-13T14:13:18.742445+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:20.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:20 smithi177 bash[24494]: cluster 2024-09-13T14:13:18.742445+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:22.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:22 smithi099 bash[22656]: cluster 2024-09-13T14:13:20.742837+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:22.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:22 smithi099 bash[22656]: audit 2024-09-13T14:13:22.094371+0000 mon.a (mon.0) 262 : audit [INF] from='client.? 172.21.15.99:0/951079359' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0624feaa-339b-4468-be53-582a1f04352c"}]: dispatch 2024-09-13T14:13:22.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:22 smithi099 bash[22656]: audit 2024-09-13T14:13:22.100688+0000 mon.a (mon.0) 263 : audit [INF] from='client.? 172.21.15.99:0/951079359' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0624feaa-339b-4468-be53-582a1f04352c"}]': finished 2024-09-13T14:13:22.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:22 smithi099 bash[22656]: cluster 2024-09-13T14:13:22.100781+0000 mon.a (mon.0) 264 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:13:22.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:22 smithi099 bash[22656]: audit 2024-09-13T14:13:22.100900+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:22.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:22 smithi099 bash[32082]: cluster 2024-09-13T14:13:20.742837+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:22.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:22 smithi099 bash[32082]: audit 2024-09-13T14:13:22.094371+0000 mon.a (mon.0) 262 : audit [INF] from='client.? 172.21.15.99:0/951079359' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0624feaa-339b-4468-be53-582a1f04352c"}]: dispatch 2024-09-13T14:13:22.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:22 smithi099 bash[32082]: audit 2024-09-13T14:13:22.100688+0000 mon.a (mon.0) 263 : audit [INF] from='client.? 172.21.15.99:0/951079359' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0624feaa-339b-4468-be53-582a1f04352c"}]': finished 2024-09-13T14:13:22.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:22 smithi099 bash[32082]: cluster 2024-09-13T14:13:22.100781+0000 mon.a (mon.0) 264 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:13:22.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:22 smithi099 bash[32082]: audit 2024-09-13T14:13:22.100900+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:22 smithi177 bash[24494]: cluster 2024-09-13T14:13:20.742837+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:22.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:22 smithi177 bash[24494]: audit 2024-09-13T14:13:22.094371+0000 mon.a (mon.0) 262 : audit [INF] from='client.? 172.21.15.99:0/951079359' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0624feaa-339b-4468-be53-582a1f04352c"}]: dispatch 2024-09-13T14:13:22.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:22 smithi177 bash[24494]: audit 2024-09-13T14:13:22.100688+0000 mon.a (mon.0) 263 : audit [INF] from='client.? 172.21.15.99:0/951079359' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0624feaa-339b-4468-be53-582a1f04352c"}]': finished 2024-09-13T14:13:22.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:22 smithi177 bash[24494]: cluster 2024-09-13T14:13:22.100781+0000 mon.a (mon.0) 264 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:13:22.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:22 smithi177 bash[24494]: audit 2024-09-13T14:13:22.100900+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:23.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:23 smithi099 bash[22656]: audit 2024-09-13T14:13:22.699411+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.99:0/3264473391' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:23.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:23 smithi099 bash[32082]: audit 2024-09-13T14:13:22.699411+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.99:0/3264473391' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:23 smithi177 bash[24494]: audit 2024-09-13T14:13:22.699411+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.99:0/3264473391' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:13:24.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:24 smithi099 bash[22656]: cluster 2024-09-13T14:13:22.743151+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:24.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:24 smithi099 bash[32082]: cluster 2024-09-13T14:13:22.743151+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:24 smithi177 bash[24494]: cluster 2024-09-13T14:13:22.743151+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:26.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:26 smithi099 bash[22656]: cluster 2024-09-13T14:13:24.743595+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:26.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:26 smithi099 bash[32082]: cluster 2024-09-13T14:13:24.743595+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:26.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:26 smithi177 bash[24494]: cluster 2024-09-13T14:13:24.743595+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:28.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:28 smithi099 bash[22656]: cluster 2024-09-13T14:13:26.744117+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:28.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:28 smithi099 bash[32082]: cluster 2024-09-13T14:13:26.744117+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:28.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:28 smithi177 bash[24494]: cluster 2024-09-13T14:13:26.744117+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:30 smithi099 bash[22656]: cluster 2024-09-13T14:13:28.744624+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:30.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:30 smithi099 bash[32082]: cluster 2024-09-13T14:13:28.744624+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:30.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:30 smithi177 bash[24494]: cluster 2024-09-13T14:13:28.744624+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:32.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:32 smithi099 bash[22656]: cluster 2024-09-13T14:13:30.745074+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:32.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:32 smithi099 bash[32082]: cluster 2024-09-13T14:13:30.745074+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:32.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:32 smithi177 bash[24494]: cluster 2024-09-13T14:13:30.745074+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:34.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:34 smithi099 bash[22656]: cluster 2024-09-13T14:13:32.745598+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:34.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:34 smithi099 bash[32082]: cluster 2024-09-13T14:13:32.745598+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:34.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:34 smithi177 bash[24494]: cluster 2024-09-13T14:13:32.745598+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:35 smithi099 bash[32082]: audit 2024-09-13T14:13:34.341630+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:13:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:35 smithi099 bash[32082]: audit 2024-09-13T14:13:34.343271+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:35 smithi099 bash[32082]: cephadm 2024-09-13T14:13:34.344709+0000 mgr.y (mgr.14150) 100 : cephadm [INF] Deploying daemon osd.0 on smithi099 2024-09-13T14:13:35.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:35 smithi099 bash[22656]: audit 2024-09-13T14:13:34.341630+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:13:35.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:35 smithi099 bash[22656]: audit 2024-09-13T14:13:34.343271+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:35.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:35 smithi099 bash[22656]: cephadm 2024-09-13T14:13:34.344709+0000 mgr.y (mgr.14150) 100 : cephadm [INF] Deploying daemon osd.0 on smithi099 2024-09-13T14:13:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:35 smithi177 bash[24494]: audit 2024-09-13T14:13:34.341630+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:13:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:35 smithi177 bash[24494]: audit 2024-09-13T14:13:34.343271+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:35 smithi177 bash[24494]: cephadm 2024-09-13T14:13:34.344709+0000 mgr.y (mgr.14150) 100 : cephadm [INF] Deploying daemon osd.0 on smithi099 2024-09-13T14:13:36.459 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:36 smithi099 bash[22656]: cluster 2024-09-13T14:13:34.746106+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:36.459 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:36 smithi099 bash[32082]: cluster 2024-09-13T14:13:34.746106+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:36.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:36 smithi177 bash[24494]: cluster 2024-09-13T14:13:34.746106+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:38.477 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:13:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:13:38.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:38 smithi099 bash[22656]: cluster 2024-09-13T14:13:36.746580+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:38.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:13:38.478 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:38 smithi099 bash[32082]: cluster 2024-09-13T14:13:36.746580+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:38.478 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:13:38.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:38 smithi177 bash[24494]: cluster 2024-09-13T14:13:36.746580+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:38.822 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:13:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:13:38.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:13:38.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:13:39.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:39 smithi099 bash[22656]: audit 2024-09-13T14:13:38.726732+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:39.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:39 smithi099 bash[22656]: audit 2024-09-13T14:13:38.730619+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:39.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:39 smithi099 bash[22656]: audit 2024-09-13T14:13:38.737648+0000 mon.a (mon.0) 270 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:39.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:39 smithi099 bash[32082]: audit 2024-09-13T14:13:38.726732+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:39.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:39 smithi099 bash[32082]: audit 2024-09-13T14:13:38.730619+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:39.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:39 smithi099 bash[32082]: audit 2024-09-13T14:13:38.737648+0000 mon.a (mon.0) 270 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:39.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:39 smithi177 bash[24494]: audit 2024-09-13T14:13:38.726732+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:39.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:39 smithi177 bash[24494]: audit 2024-09-13T14:13:38.730619+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:39.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:39 smithi177 bash[24494]: audit 2024-09-13T14:13:38.737648+0000 mon.a (mon.0) 270 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:40.485 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:40 smithi099 bash[22656]: cluster 2024-09-13T14:13:38.746824+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:40.486 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:40 smithi099 bash[32082]: cluster 2024-09-13T14:13:38.746824+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:40.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:40 smithi177 bash[24494]: cluster 2024-09-13T14:13:38.746824+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:42.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:42 smithi099 bash[22656]: cluster 2024-09-13T14:13:40.747198+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:42.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:42 smithi099 bash[32082]: cluster 2024-09-13T14:13:40.747198+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:42.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:42 smithi177 bash[24494]: cluster 2024-09-13T14:13:40.747198+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:44.545 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[22656]: cluster 2024-09-13T14:13:42.747634+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:44.545 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[22656]: audit 2024-09-13T14:13:43.721302+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:44.545 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[22656]: audit 2024-09-13T14:13:43.727479+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:44.545 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[22656]: audit 2024-09-13T14:13:44.065799+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:44.546 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[22656]: audit 2024-09-13T14:13:44.066771+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:44.546 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[22656]: audit 2024-09-13T14:13:44.071909+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:44.546 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[32082]: cluster 2024-09-13T14:13:42.747634+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:44.546 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[32082]: audit 2024-09-13T14:13:43.721302+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:44.546 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[32082]: audit 2024-09-13T14:13:43.727479+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:44.546 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[32082]: audit 2024-09-13T14:13:44.065799+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:44.546 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[32082]: audit 2024-09-13T14:13:44.066771+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:44.546 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:44 smithi099 bash[32082]: audit 2024-09-13T14:13:44.071909+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:44.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:44 smithi177 bash[24494]: cluster 2024-09-13T14:13:42.747634+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:44.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:44 smithi177 bash[24494]: audit 2024-09-13T14:13:43.721302+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:44.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:44 smithi177 bash[24494]: audit 2024-09-13T14:13:43.727479+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:44.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:44 smithi177 bash[24494]: audit 2024-09-13T14:13:44.065799+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:44.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:44 smithi177 bash[24494]: audit 2024-09-13T14:13:44.066771+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:13:44.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:44 smithi177 bash[24494]: audit 2024-09-13T14:13:44.071909+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:46.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:46 smithi099 bash[22656]: cluster 2024-09-13T14:13:44.747893+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:46.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:46 smithi099 bash[22656]: audit 2024-09-13T14:13:45.418172+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/500907575,v1:172.21.15.99:6803/500907575]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:13:46.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:46 smithi099 bash[22656]: audit 2024-09-13T14:13:45.418521+0000 mon.a (mon.0) 276 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:13:46.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:46 smithi099 bash[32082]: cluster 2024-09-13T14:13:44.747893+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:46.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:46 smithi099 bash[32082]: audit 2024-09-13T14:13:45.418172+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/500907575,v1:172.21.15.99:6803/500907575]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:13:46.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:46 smithi099 bash[32082]: audit 2024-09-13T14:13:45.418521+0000 mon.a (mon.0) 276 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:46 smithi177 bash[24494]: cluster 2024-09-13T14:13:44.747893+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:46 smithi177 bash[24494]: audit 2024-09-13T14:13:45.418172+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/500907575,v1:172.21.15.99:6803/500907575]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:46 smithi177 bash[24494]: audit 2024-09-13T14:13:45.418521+0000 mon.a (mon.0) 276 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:13:47.511 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 0 on host 'smithi099' 2024-09-13T14:13:47.527 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:47 smithi099 bash[22656]: audit 2024-09-13T14:13:46.247193+0000 mon.a (mon.0) 277 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:13:47.527 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:47 smithi099 bash[22656]: cluster 2024-09-13T14:13:46.247256+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:13:47.527 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:47 smithi099 bash[22656]: audit 2024-09-13T14:13:46.247506+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:47.527 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:47 smithi099 bash[22656]: audit 2024-09-13T14:13:46.249830+0000 mon.c (mon.2) 6 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/500907575,v1:172.21.15.99:6803/500907575]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:13:47.527 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:47 smithi099 bash[22656]: audit 2024-09-13T14:13:46.250514+0000 mon.a (mon.0) 280 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:13:47.528 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:47 smithi099 bash[32082]: audit 2024-09-13T14:13:46.247193+0000 mon.a (mon.0) 277 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:13:47.528 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:47 smithi099 bash[32082]: cluster 2024-09-13T14:13:46.247256+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:13:47.528 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:47 smithi099 bash[32082]: audit 2024-09-13T14:13:46.247506+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:47.528 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:47 smithi099 bash[32082]: audit 2024-09-13T14:13:46.249830+0000 mon.c (mon.2) 6 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/500907575,v1:172.21.15.99:6803/500907575]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:13:47.528 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:47 smithi099 bash[32082]: audit 2024-09-13T14:13:46.250514+0000 mon.a (mon.0) 280 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:13:47.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:47 smithi177 bash[24494]: audit 2024-09-13T14:13:46.247193+0000 mon.a (mon.0) 277 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:13:47.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:47 smithi177 bash[24494]: cluster 2024-09-13T14:13:46.247256+0000 mon.a (mon.0) 278 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:13:47.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:47 smithi177 bash[24494]: audit 2024-09-13T14:13:46.247506+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:47.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:47 smithi177 bash[24494]: audit 2024-09-13T14:13:46.249830+0000 mon.c (mon.2) 6 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/500907575,v1:172.21.15.99:6803/500907575]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:13:47.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:47 smithi177 bash[24494]: audit 2024-09-13T14:13:46.250514+0000 mon.a (mon.0) 280 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:13:48.183 DEBUG:teuthology.orchestra.run.smithi099:osd.0> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T14:13:48.185 INFO:tasks.cephadm:Deploying osd.1 on smithi099 with /dev/vg_nvme/lv_3... 2024-09-13T14:13:48.185 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T14:13:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[22656]: cluster 2024-09-13T14:13:46.748252+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[22656]: audit 2024-09-13T14:13:47.259725+0000 mon.a (mon.0) 281 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:13:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[22656]: cluster 2024-09-13T14:13:47.259953+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:13:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[22656]: audit 2024-09-13T14:13:47.260441+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[22656]: audit 2024-09-13T14:13:47.267870+0000 mon.a (mon.0) 284 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[22656]: audit 2024-09-13T14:13:47.492471+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[22656]: audit 2024-09-13T14:13:47.499520+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:48.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[22656]: audit 2024-09-13T14:13:47.507342+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[32082]: cluster 2024-09-13T14:13:46.748252+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[32082]: audit 2024-09-13T14:13:47.259725+0000 mon.a (mon.0) 281 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:13:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[32082]: cluster 2024-09-13T14:13:47.259953+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:13:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[32082]: audit 2024-09-13T14:13:47.260441+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[32082]: audit 2024-09-13T14:13:47.267870+0000 mon.a (mon.0) 284 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[32082]: audit 2024-09-13T14:13:47.492471+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[32082]: audit 2024-09-13T14:13:47.499520+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:48.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:48 smithi099 bash[32082]: audit 2024-09-13T14:13:47.507342+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:48.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:48 smithi177 bash[24494]: cluster 2024-09-13T14:13:46.748252+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:13:48.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:48 smithi177 bash[24494]: audit 2024-09-13T14:13:47.259725+0000 mon.a (mon.0) 281 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:13:48.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:48 smithi177 bash[24494]: cluster 2024-09-13T14:13:47.259953+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:13:48.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:48 smithi177 bash[24494]: audit 2024-09-13T14:13:47.260441+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:48.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:48 smithi177 bash[24494]: audit 2024-09-13T14:13:47.267870+0000 mon.a (mon.0) 284 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:48.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:48 smithi177 bash[24494]: audit 2024-09-13T14:13:47.492471+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:13:48.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:48 smithi177 bash[24494]: audit 2024-09-13T14:13:47.499520+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:48.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:48 smithi177 bash[24494]: audit 2024-09-13T14:13:47.507342+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:13:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:49 smithi099 bash[22656]: cluster 2024-09-13T14:13:46.390502+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:13:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:49 smithi099 bash[22656]: cluster 2024-09-13T14:13:46.390600+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:13:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:49 smithi099 bash[22656]: cluster 2024-09-13T14:13:48.259753+0000 mon.a (mon.0) 288 : cluster [INF] osd.0 [v2:172.21.15.99:6802/500907575,v1:172.21.15.99:6803/500907575] boot 2024-09-13T14:13:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:49 smithi099 bash[22656]: cluster 2024-09-13T14:13:48.259816+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:13:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:49 smithi099 bash[22656]: audit 2024-09-13T14:13:48.260382+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:49 smithi099 bash[32082]: cluster 2024-09-13T14:13:46.390502+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:13:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:49 smithi099 bash[32082]: cluster 2024-09-13T14:13:46.390600+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:13:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:49 smithi099 bash[32082]: cluster 2024-09-13T14:13:48.259753+0000 mon.a (mon.0) 288 : cluster [INF] osd.0 [v2:172.21.15.99:6802/500907575,v1:172.21.15.99:6803/500907575] boot 2024-09-13T14:13:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:49 smithi099 bash[32082]: cluster 2024-09-13T14:13:48.259816+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:13:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:49 smithi099 bash[32082]: audit 2024-09-13T14:13:48.260382+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:49 smithi177 bash[24494]: cluster 2024-09-13T14:13:46.390502+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:13:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:49 smithi177 bash[24494]: cluster 2024-09-13T14:13:46.390600+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:13:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:49 smithi177 bash[24494]: cluster 2024-09-13T14:13:48.259753+0000 mon.a (mon.0) 288 : cluster [INF] osd.0 [v2:172.21.15.99:6802/500907575,v1:172.21.15.99:6803/500907575] boot 2024-09-13T14:13:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:49 smithi177 bash[24494]: cluster 2024-09-13T14:13:48.259816+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:13:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:49 smithi177 bash[24494]: audit 2024-09-13T14:13:48.260382+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:13:50.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:50 smithi099 bash[22656]: cluster 2024-09-13T14:13:48.748789+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:50.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:50 smithi099 bash[22656]: cluster 2024-09-13T14:13:49.278319+0000 mon.a (mon.0) 291 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:13:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:50 smithi099 bash[32082]: cluster 2024-09-13T14:13:48.748789+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:50 smithi099 bash[32082]: cluster 2024-09-13T14:13:49.278319+0000 mon.a (mon.0) 291 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:13:50.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:50 smithi177 bash[24494]: cluster 2024-09-13T14:13:48.748789+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:50.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:50 smithi177 bash[24494]: cluster 2024-09-13T14:13:49.278319+0000 mon.a (mon.0) 291 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:13:52.266 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:13:52.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:52 smithi099 bash[32082]: cluster 2024-09-13T14:13:50.749222+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:52.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:52 smithi099 bash[22656]: cluster 2024-09-13T14:13:50.749222+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:52 smithi177 bash[24494]: cluster 2024-09-13T14:13:50.749222+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:54.587 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:54 smithi099 bash[32082]: cluster 2024-09-13T14:13:52.749686+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:54.588 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:54 smithi099 bash[22656]: cluster 2024-09-13T14:13:52.749686+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:54 smithi177 bash[24494]: cluster 2024-09-13T14:13:52.749686+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:56.562 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:56 smithi099 bash[22656]: cluster 2024-09-13T14:13:54.750118+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:56.562 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:56 smithi099 bash[32082]: cluster 2024-09-13T14:13:54.750118+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:56 smithi177 bash[24494]: cluster 2024-09-13T14:13:54.750118+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:57.147 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:13:57.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi099:vg_nvme/lv_3 2024-09-13T14:13:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:13:58 smithi099 bash[22656]: cluster 2024-09-13T14:13:56.750585+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:13:58 smithi099 bash[32082]: cluster 2024-09-13T14:13:56.750585+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:13:58.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:13:58 smithi177 bash[24494]: cluster 2024-09-13T14:13:56.750585+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[22656]: cluster 2024-09-13T14:13:58.750937+0000 mgr.y (mgr.14150) 113 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[22656]: audit 2024-09-13T14:13:59.789171+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[22656]: audit 2024-09-13T14:13:59.795937+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[22656]: audit 2024-09-13T14:13:59.797397+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[22656]: audit 2024-09-13T14:13:59.805538+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[22656]: audit 2024-09-13T14:13:59.807734+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[22656]: audit 2024-09-13T14:13:59.809090+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:00.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[22656]: audit 2024-09-13T14:13:59.816333+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[32082]: cluster 2024-09-13T14:13:58.750937+0000 mgr.y (mgr.14150) 113 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[32082]: audit 2024-09-13T14:13:59.789171+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[32082]: audit 2024-09-13T14:13:59.795937+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[32082]: audit 2024-09-13T14:13:59.797397+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[32082]: audit 2024-09-13T14:13:59.805538+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[32082]: audit 2024-09-13T14:13:59.807734+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[32082]: audit 2024-09-13T14:13:59.809090+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:00 smithi099 bash[32082]: audit 2024-09-13T14:13:59.816333+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:00 smithi177 bash[24494]: cluster 2024-09-13T14:13:58.750937+0000 mgr.y (mgr.14150) 113 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:00.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:00 smithi177 bash[24494]: audit 2024-09-13T14:13:59.789171+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:00 smithi177 bash[24494]: audit 2024-09-13T14:13:59.795937+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:00 smithi177 bash[24494]: audit 2024-09-13T14:13:59.797397+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:00.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:00 smithi177 bash[24494]: audit 2024-09-13T14:13:59.805538+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:00.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:00 smithi177 bash[24494]: audit 2024-09-13T14:13:59.807734+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:00.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:00 smithi177 bash[24494]: audit 2024-09-13T14:13:59.809090+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:00.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:00 smithi177 bash[24494]: audit 2024-09-13T14:13:59.816333+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:01.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:01 smithi099 bash[22656]: cephadm 2024-09-13T14:13:59.782002+0000 mgr.y (mgr.14150) 114 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:14:01.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:01 smithi099 bash[22656]: cephadm 2024-09-13T14:13:59.798517+0000 mgr.y (mgr.14150) 115 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 16241M 2024-09-13T14:14:01.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:01 smithi099 bash[32082]: cephadm 2024-09-13T14:13:59.782002+0000 mgr.y (mgr.14150) 114 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:14:01.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:01 smithi099 bash[32082]: cephadm 2024-09-13T14:13:59.798517+0000 mgr.y (mgr.14150) 115 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 16241M 2024-09-13T14:14:01.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:01 smithi177 bash[24494]: cephadm 2024-09-13T14:13:59.782002+0000 mgr.y (mgr.14150) 114 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:14:01.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:01 smithi177 bash[24494]: cephadm 2024-09-13T14:13:59.798517+0000 mgr.y (mgr.14150) 115 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 16241M 2024-09-13T14:14:02.253 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:02.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:02 smithi099 bash[32082]: cluster 2024-09-13T14:14:00.751397+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:02.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:02 smithi099 bash[22656]: cluster 2024-09-13T14:14:00.751397+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:02.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:02 smithi177 bash[24494]: cluster 2024-09-13T14:14:00.751397+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:04.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:04 smithi099 bash[22656]: cluster 2024-09-13T14:14:02.751839+0000 mgr.y (mgr.14150) 117 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:04.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:04 smithi099 bash[32082]: cluster 2024-09-13T14:14:02.751839+0000 mgr.y (mgr.14150) 117 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:04.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:04 smithi177 bash[24494]: cluster 2024-09-13T14:14:02.751839+0000 mgr.y (mgr.14150) 117 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:06.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:06 smithi099 bash[22656]: cluster 2024-09-13T14:14:04.752312+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:06.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:06 smithi099 bash[22656]: audit 2024-09-13T14:14:05.506258+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:14:06.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:06 smithi099 bash[22656]: audit 2024-09-13T14:14:05.511455+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:14:06.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:06 smithi099 bash[22656]: audit 2024-09-13T14:14:05.512697+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:06.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:06 smithi099 bash[32082]: cluster 2024-09-13T14:14:04.752312+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:06.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:06 smithi099 bash[32082]: audit 2024-09-13T14:14:05.506258+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:14:06.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:06 smithi099 bash[32082]: audit 2024-09-13T14:14:05.511455+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:14:06.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:06 smithi099 bash[32082]: audit 2024-09-13T14:14:05.512697+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:06 smithi177 bash[24494]: cluster 2024-09-13T14:14:04.752312+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:06 smithi177 bash[24494]: audit 2024-09-13T14:14:05.506258+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:14:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:06 smithi177 bash[24494]: audit 2024-09-13T14:14:05.511455+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:14:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:06 smithi177 bash[24494]: audit 2024-09-13T14:14:05.512697+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:07.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:07 smithi099 bash[22656]: audit 2024-09-13T14:14:05.500253+0000 mgr.y (mgr.14150) 119 : audit [DBG] from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:07.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:07 smithi099 bash[32082]: audit 2024-09-13T14:14:05.500253+0000 mgr.y (mgr.14150) 119 : audit [DBG] from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:07 smithi177 bash[24494]: audit 2024-09-13T14:14:05.500253+0000 mgr.y (mgr.14150) 119 : audit [DBG] from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:08.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:08 smithi099 bash[22656]: cluster 2024-09-13T14:14:06.752745+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:08.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:08 smithi099 bash[32082]: cluster 2024-09-13T14:14:06.752745+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:08 smithi177 bash[24494]: cluster 2024-09-13T14:14:06.752745+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:10.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:10 smithi099 bash[22656]: cluster 2024-09-13T14:14:08.753244+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:10.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:10 smithi099 bash[32082]: cluster 2024-09-13T14:14:08.753244+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:10.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:10 smithi177 bash[24494]: cluster 2024-09-13T14:14:08.753244+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:12.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:12 smithi099 bash[32082]: cluster 2024-09-13T14:14:10.753814+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:12.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:12 smithi099 bash[22656]: cluster 2024-09-13T14:14:10.753814+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:12.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:12 smithi177 bash[24494]: cluster 2024-09-13T14:14:10.753814+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:14 smithi177 bash[24494]: cluster 2024-09-13T14:14:12.754285+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:14 smithi177 bash[24494]: audit 2024-09-13T14:14:13.767173+0000 mon.a (mon.0) 302 : audit [INF] from='client.? 172.21.15.99:0/1472260230' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7b63553-e49d-4636-b3e2-43b72556a1a6"}]: dispatch 2024-09-13T14:14:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:14 smithi177 bash[24494]: audit 2024-09-13T14:14:13.772995+0000 mon.a (mon.0) 303 : audit [INF] from='client.? 172.21.15.99:0/1472260230' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7b63553-e49d-4636-b3e2-43b72556a1a6"}]': finished 2024-09-13T14:14:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:14 smithi177 bash[24494]: cluster 2024-09-13T14:14:13.773046+0000 mon.a (mon.0) 304 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:14:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:14 smithi177 bash[24494]: audit 2024-09-13T14:14:13.773225+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:14.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:14 smithi099 bash[22656]: cluster 2024-09-13T14:14:12.754285+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:14.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:14 smithi099 bash[22656]: audit 2024-09-13T14:14:13.767173+0000 mon.a (mon.0) 302 : audit [INF] from='client.? 172.21.15.99:0/1472260230' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7b63553-e49d-4636-b3e2-43b72556a1a6"}]: dispatch 2024-09-13T14:14:14.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:14 smithi099 bash[22656]: audit 2024-09-13T14:14:13.772995+0000 mon.a (mon.0) 303 : audit [INF] from='client.? 172.21.15.99:0/1472260230' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7b63553-e49d-4636-b3e2-43b72556a1a6"}]': finished 2024-09-13T14:14:14.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:14 smithi099 bash[22656]: cluster 2024-09-13T14:14:13.773046+0000 mon.a (mon.0) 304 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:14:14.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:14 smithi099 bash[22656]: audit 2024-09-13T14:14:13.773225+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:14.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:14 smithi099 bash[32082]: cluster 2024-09-13T14:14:12.754285+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:14.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:14 smithi099 bash[32082]: audit 2024-09-13T14:14:13.767173+0000 mon.a (mon.0) 302 : audit [INF] from='client.? 172.21.15.99:0/1472260230' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b7b63553-e49d-4636-b3e2-43b72556a1a6"}]: dispatch 2024-09-13T14:14:14.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:14 smithi099 bash[32082]: audit 2024-09-13T14:14:13.772995+0000 mon.a (mon.0) 303 : audit [INF] from='client.? 172.21.15.99:0/1472260230' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b7b63553-e49d-4636-b3e2-43b72556a1a6"}]': finished 2024-09-13T14:14:14.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:14 smithi099 bash[32082]: cluster 2024-09-13T14:14:13.773046+0000 mon.a (mon.0) 304 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:14:14.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:14 smithi099 bash[32082]: audit 2024-09-13T14:14:13.773225+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:15 smithi177 bash[24494]: audit 2024-09-13T14:14:14.412604+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.99:0/960953408' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:14:15.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:15 smithi099 bash[22656]: audit 2024-09-13T14:14:14.412604+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.99:0/960953408' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:14:15.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:15 smithi099 bash[32082]: audit 2024-09-13T14:14:14.412604+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.99:0/960953408' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:14:16.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:16 smithi177 bash[24494]: cluster 2024-09-13T14:14:14.754741+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:16.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:16 smithi099 bash[22656]: cluster 2024-09-13T14:14:14.754741+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:16.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:16 smithi099 bash[32082]: cluster 2024-09-13T14:14:14.754741+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:18 smithi177 bash[24494]: cluster 2024-09-13T14:14:16.755150+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:18.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:18 smithi099 bash[22656]: cluster 2024-09-13T14:14:16.755150+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:18.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:18 smithi099 bash[32082]: cluster 2024-09-13T14:14:16.755150+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:20.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:20 smithi177 bash[24494]: cluster 2024-09-13T14:14:18.755590+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:20.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:20 smithi099 bash[22656]: cluster 2024-09-13T14:14:18.755590+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:20.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:20 smithi099 bash[32082]: cluster 2024-09-13T14:14:18.755590+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:22.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:22 smithi177 bash[24494]: cluster 2024-09-13T14:14:20.756037+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:22.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:22 smithi099 bash[22656]: cluster 2024-09-13T14:14:20.756037+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:22.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:22 smithi099 bash[32082]: cluster 2024-09-13T14:14:20.756037+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:24.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:24 smithi099 bash[22656]: cluster 2024-09-13T14:14:22.756452+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:24.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:24 smithi099 bash[32082]: cluster 2024-09-13T14:14:22.756452+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:24.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:24 smithi177 bash[24494]: cluster 2024-09-13T14:14:22.756452+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:26.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:26 smithi177 bash[24494]: cluster 2024-09-13T14:14:24.756908+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:26.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:26 smithi177 bash[24494]: audit 2024-09-13T14:14:26.184742+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:14:26.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:26 smithi177 bash[24494]: audit 2024-09-13T14:14:26.186565+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:26 smithi099 bash[22656]: cluster 2024-09-13T14:14:24.756908+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:26 smithi099 bash[22656]: audit 2024-09-13T14:14:26.184742+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:14:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:26 smithi099 bash[22656]: audit 2024-09-13T14:14:26.186565+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:26.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:26 smithi099 bash[32082]: cluster 2024-09-13T14:14:24.756908+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:26.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:26 smithi099 bash[32082]: audit 2024-09-13T14:14:26.184742+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:14:26.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:26 smithi099 bash[32082]: audit 2024-09-13T14:14:26.186565+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:27.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:27 smithi099 bash[22656]: cephadm 2024-09-13T14:14:26.188091+0000 mgr.y (mgr.14150) 130 : cephadm [INF] Deploying daemon osd.1 on smithi099 2024-09-13T14:14:27.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:27 smithi099 bash[32082]: cephadm 2024-09-13T14:14:26.188091+0000 mgr.y (mgr.14150) 130 : cephadm [INF] Deploying daemon osd.1 on smithi099 2024-09-13T14:14:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:27 smithi177 bash[24494]: cephadm 2024-09-13T14:14:26.188091+0000 mgr.y (mgr.14150) 130 : cephadm [INF] Deploying daemon osd.1 on smithi099 2024-09-13T14:14:28.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:28 smithi099 bash[22656]: cluster 2024-09-13T14:14:26.757317+0000 mgr.y (mgr.14150) 131 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:28.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:28 smithi099 bash[32082]: cluster 2024-09-13T14:14:26.757317+0000 mgr.y (mgr.14150) 131 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:28.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:28 smithi177 bash[24494]: cluster 2024-09-13T14:14:26.757317+0000 mgr.y (mgr.14150) 131 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:30.528 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:14:30 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:14:30.528 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:30 smithi099 bash[22656]: cluster 2024-09-13T14:14:28.757768+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:30.528 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:30 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:14:30.529 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:30 smithi099 bash[32082]: cluster 2024-09-13T14:14:28.757768+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:30.529 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:30 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:14:30.529 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:14:30 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:14:30.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:30 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:14:30.824 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:14:30 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:14:30.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:30 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:14:30.824 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:14:30 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:14:30.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:30 smithi177 bash[24494]: cluster 2024-09-13T14:14:28.757768+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:31.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:31 smithi099 bash[22656]: audit 2024-09-13T14:14:30.773255+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:31.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:31 smithi099 bash[22656]: audit 2024-09-13T14:14:30.777115+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:31.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:31 smithi099 bash[22656]: audit 2024-09-13T14:14:30.781599+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:31.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:31 smithi099 bash[32082]: audit 2024-09-13T14:14:30.773255+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:31.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:31 smithi099 bash[32082]: audit 2024-09-13T14:14:30.777115+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:31.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:31 smithi099 bash[32082]: audit 2024-09-13T14:14:30.781599+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:31 smithi177 bash[24494]: audit 2024-09-13T14:14:30.773255+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:31 smithi177 bash[24494]: audit 2024-09-13T14:14:30.777115+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:31.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:31 smithi177 bash[24494]: audit 2024-09-13T14:14:30.781599+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:32.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:32 smithi099 bash[22656]: cluster 2024-09-13T14:14:30.758109+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:32.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:32 smithi099 bash[32082]: cluster 2024-09-13T14:14:30.758109+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:32.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:32 smithi177 bash[24494]: cluster 2024-09-13T14:14:30.758109+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:34.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:34 smithi099 bash[22656]: cluster 2024-09-13T14:14:32.758482+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:34.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:34 smithi099 bash[32082]: cluster 2024-09-13T14:14:32.758482+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:34.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:34 smithi177 bash[24494]: cluster 2024-09-13T14:14:32.758482+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:36.652 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[22656]: cluster 2024-09-13T14:14:34.758884+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:36.653 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[22656]: audit 2024-09-13T14:14:35.825443+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:36.653 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[22656]: audit 2024-09-13T14:14:35.831596+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:36.653 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[22656]: audit 2024-09-13T14:14:35.832326+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:36.653 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[22656]: audit 2024-09-13T14:14:35.833152+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:36.653 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[22656]: audit 2024-09-13T14:14:35.839932+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:36.653 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[32082]: cluster 2024-09-13T14:14:34.758884+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:36.653 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[32082]: audit 2024-09-13T14:14:35.825443+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:36.653 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[32082]: audit 2024-09-13T14:14:35.831596+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:36.653 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[32082]: audit 2024-09-13T14:14:35.832326+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:36.654 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[32082]: audit 2024-09-13T14:14:35.833152+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:36.654 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:36 smithi099 bash[32082]: audit 2024-09-13T14:14:35.839932+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:36.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:36 smithi177 bash[24494]: cluster 2024-09-13T14:14:34.758884+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:36.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:36 smithi177 bash[24494]: audit 2024-09-13T14:14:35.825443+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:36.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:36 smithi177 bash[24494]: audit 2024-09-13T14:14:35.831596+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:36.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:36 smithi177 bash[24494]: audit 2024-09-13T14:14:35.832326+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:36.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:36 smithi177 bash[24494]: audit 2024-09-13T14:14:35.833152+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:36.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:36 smithi177 bash[24494]: audit 2024-09-13T14:14:35.839932+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:38.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:38 smithi099 bash[32082]: cluster 2024-09-13T14:14:36.759229+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:38.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:38 smithi099 bash[32082]: audit 2024-09-13T14:14:37.436706+0000 mon.a (mon.0) 316 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:14:38.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:38 smithi099 bash[22656]: cluster 2024-09-13T14:14:36.759229+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:38.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:38 smithi099 bash[22656]: audit 2024-09-13T14:14:37.436706+0000 mon.a (mon.0) 316 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:14:38.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:38 smithi177 bash[24494]: cluster 2024-09-13T14:14:36.759229+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:38.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:38 smithi177 bash[24494]: audit 2024-09-13T14:14:37.436706+0000 mon.a (mon.0) 316 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:14:39.517 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 1 on host 'smithi099' 2024-09-13T14:14:39.721 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:39 smithi099 bash[32082]: audit 2024-09-13T14:14:38.402867+0000 mon.a (mon.0) 317 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:14:39.722 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:39 smithi099 bash[32082]: cluster 2024-09-13T14:14:38.402933+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:14:39.722 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:39 smithi099 bash[32082]: audit 2024-09-13T14:14:38.403064+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:39.722 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:39 smithi099 bash[32082]: audit 2024-09-13T14:14:38.403750+0000 mon.a (mon.0) 320 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:14:39.722 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:39 smithi099 bash[22656]: audit 2024-09-13T14:14:38.402867+0000 mon.a (mon.0) 317 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:14:39.722 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:39 smithi099 bash[22656]: cluster 2024-09-13T14:14:38.402933+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:14:39.722 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:39 smithi099 bash[22656]: audit 2024-09-13T14:14:38.403064+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:39.722 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:39 smithi099 bash[22656]: audit 2024-09-13T14:14:38.403750+0000 mon.a (mon.0) 320 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:14:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:39 smithi177 bash[24494]: audit 2024-09-13T14:14:38.402867+0000 mon.a (mon.0) 317 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:14:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:39 smithi177 bash[24494]: cluster 2024-09-13T14:14:38.402933+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:14:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:39 smithi177 bash[24494]: audit 2024-09-13T14:14:38.403064+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:39 smithi177 bash[24494]: audit 2024-09-13T14:14:38.403750+0000 mon.a (mon.0) 320 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:14:40.225 DEBUG:teuthology.orchestra.run.smithi099:osd.1> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T14:14:40.228 INFO:tasks.cephadm:Deploying osd.2 on smithi099 with /dev/vg_nvme/lv_2... 2024-09-13T14:14:40.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T14:14:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[22656]: cluster 2024-09-13T14:14:38.759516+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[22656]: audit 2024-09-13T14:14:39.414585+0000 mon.a (mon.0) 321 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:14:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[22656]: cluster 2024-09-13T14:14:39.414705+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:14:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[22656]: audit 2024-09-13T14:14:39.415103+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[22656]: audit 2024-09-13T14:14:39.421940+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[22656]: audit 2024-09-13T14:14:39.504771+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[22656]: audit 2024-09-13T14:14:39.509921+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[22656]: audit 2024-09-13T14:14:39.514985+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:40.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[32082]: cluster 2024-09-13T14:14:38.759516+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:40.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[32082]: audit 2024-09-13T14:14:39.414585+0000 mon.a (mon.0) 321 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:14:40.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[32082]: cluster 2024-09-13T14:14:39.414705+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:14:40.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[32082]: audit 2024-09-13T14:14:39.415103+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:40.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[32082]: audit 2024-09-13T14:14:39.421940+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:40.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[32082]: audit 2024-09-13T14:14:39.504771+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:40.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[32082]: audit 2024-09-13T14:14:39.509921+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:40.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:40 smithi099 bash[32082]: audit 2024-09-13T14:14:39.514985+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:40.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:40 smithi177 bash[24494]: cluster 2024-09-13T14:14:38.759516+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:14:40.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:40 smithi177 bash[24494]: audit 2024-09-13T14:14:39.414585+0000 mon.a (mon.0) 321 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:14:40.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:40 smithi177 bash[24494]: cluster 2024-09-13T14:14:39.414705+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:14:40.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:40 smithi177 bash[24494]: audit 2024-09-13T14:14:39.415103+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:40.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:40 smithi177 bash[24494]: audit 2024-09-13T14:14:39.421940+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:40.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:40 smithi177 bash[24494]: audit 2024-09-13T14:14:39.504771+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:40.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:40 smithi177 bash[24494]: audit 2024-09-13T14:14:39.509921+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:40.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:40 smithi177 bash[24494]: audit 2024-09-13T14:14:39.514985+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:41.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:41 smithi099 bash[22656]: cluster 2024-09-13T14:14:38.398511+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:14:41.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:41 smithi099 bash[22656]: cluster 2024-09-13T14:14:38.398629+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:14:41.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:41 smithi099 bash[22656]: cluster 2024-09-13T14:14:40.416695+0000 mon.a (mon.0) 328 : cluster [INF] osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153] boot 2024-09-13T14:14:41.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:41 smithi099 bash[22656]: cluster 2024-09-13T14:14:40.416778+0000 mon.a (mon.0) 329 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:14:41.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:41 smithi099 bash[22656]: audit 2024-09-13T14:14:40.417418+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:41.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:41 smithi099 bash[32082]: cluster 2024-09-13T14:14:38.398511+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:14:41.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:41 smithi099 bash[32082]: cluster 2024-09-13T14:14:38.398629+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:14:41.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:41 smithi099 bash[32082]: cluster 2024-09-13T14:14:40.416695+0000 mon.a (mon.0) 328 : cluster [INF] osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153] boot 2024-09-13T14:14:41.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:41 smithi099 bash[32082]: cluster 2024-09-13T14:14:40.416778+0000 mon.a (mon.0) 329 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:14:41.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:41 smithi099 bash[32082]: audit 2024-09-13T14:14:40.417418+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:41 smithi177 bash[24494]: cluster 2024-09-13T14:14:38.398511+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:14:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:41 smithi177 bash[24494]: cluster 2024-09-13T14:14:38.398629+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:14:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:41 smithi177 bash[24494]: cluster 2024-09-13T14:14:40.416695+0000 mon.a (mon.0) 328 : cluster [INF] osd.1 [v2:172.21.15.99:6810/4008940153,v1:172.21.15.99:6811/4008940153] boot 2024-09-13T14:14:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:41 smithi177 bash[24494]: cluster 2024-09-13T14:14:40.416778+0000 mon.a (mon.0) 329 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:14:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:41 smithi177 bash[24494]: audit 2024-09-13T14:14:40.417418+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:14:42.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:42 smithi099 bash[22656]: cluster 2024-09-13T14:14:40.759985+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:42.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:42 smithi099 bash[22656]: cluster 2024-09-13T14:14:41.433452+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:14:42.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:42 smithi099 bash[32082]: cluster 2024-09-13T14:14:40.759985+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:42.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:42 smithi099 bash[32082]: cluster 2024-09-13T14:14:41.433452+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:14:42.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:42 smithi177 bash[24494]: cluster 2024-09-13T14:14:40.759985+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:42.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:42 smithi177 bash[24494]: cluster 2024-09-13T14:14:41.433452+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:14:44.270 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:44.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:44 smithi099 bash[22656]: cluster 2024-09-13T14:14:42.760437+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:44.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:44 smithi099 bash[32082]: cluster 2024-09-13T14:14:42.760437+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:44 smithi177 bash[24494]: cluster 2024-09-13T14:14:42.760437+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:46.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:46 smithi099 bash[32082]: cluster 2024-09-13T14:14:44.760940+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:46.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:46 smithi099 bash[22656]: cluster 2024-09-13T14:14:44.760940+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:46.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:46 smithi177 bash[24494]: cluster 2024-09-13T14:14:44.760940+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:48.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:48 smithi099 bash[22656]: cluster 2024-09-13T14:14:46.761403+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:48.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:48 smithi099 bash[32082]: cluster 2024-09-13T14:14:46.761403+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:48.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:48 smithi177 bash[24494]: cluster 2024-09-13T14:14:46.761403+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:50.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[22656]: cluster 2024-09-13T14:14:48.761728+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:50.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[22656]: audit 2024-09-13T14:14:49.625468+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[22656]: audit 2024-09-13T14:14:49.630794+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[22656]: audit 2024-09-13T14:14:49.632275+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:50.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[22656]: audit 2024-09-13T14:14:49.633312+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:50.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[22656]: audit 2024-09-13T14:14:49.638558+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[22656]: audit 2024-09-13T14:14:49.639968+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:50.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[22656]: audit 2024-09-13T14:14:49.640849+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:50.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[22656]: audit 2024-09-13T14:14:49.646484+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[32082]: cluster 2024-09-13T14:14:48.761728+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[32082]: audit 2024-09-13T14:14:49.625468+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[32082]: audit 2024-09-13T14:14:49.630794+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[32082]: audit 2024-09-13T14:14:49.632275+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[32082]: audit 2024-09-13T14:14:49.633312+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[32082]: audit 2024-09-13T14:14:49.638558+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[32082]: audit 2024-09-13T14:14:49.639968+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[32082]: audit 2024-09-13T14:14:49.640849+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:50 smithi099 bash[32082]: audit 2024-09-13T14:14:49.646484+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:50 smithi177 bash[24494]: cluster 2024-09-13T14:14:48.761728+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:50.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:50 smithi177 bash[24494]: audit 2024-09-13T14:14:49.625468+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:50 smithi177 bash[24494]: audit 2024-09-13T14:14:49.630794+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:50 smithi177 bash[24494]: audit 2024-09-13T14:14:49.632275+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:50.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:50 smithi177 bash[24494]: audit 2024-09-13T14:14:49.633312+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:14:50.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:50 smithi177 bash[24494]: audit 2024-09-13T14:14:49.638558+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:50.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:50 smithi177 bash[24494]: audit 2024-09-13T14:14:49.639968+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:14:50.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:50 smithi177 bash[24494]: audit 2024-09-13T14:14:49.640849+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:14:50.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:50 smithi177 bash[24494]: audit 2024-09-13T14:14:49.646484+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:14:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:51 smithi099 bash[22656]: cephadm 2024-09-13T14:14:49.619643+0000 mgr.y (mgr.14150) 143 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:14:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:51 smithi099 bash[22656]: cephadm 2024-09-13T14:14:49.634100+0000 mgr.y (mgr.14150) 144 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 8120M 2024-09-13T14:14:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:51 smithi099 bash[32082]: cephadm 2024-09-13T14:14:49.619643+0000 mgr.y (mgr.14150) 143 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:14:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:51 smithi099 bash[32082]: cephadm 2024-09-13T14:14:49.634100+0000 mgr.y (mgr.14150) 144 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 8120M 2024-09-13T14:14:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:51 smithi177 bash[24494]: cephadm 2024-09-13T14:14:49.619643+0000 mgr.y (mgr.14150) 143 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:14:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:51 smithi177 bash[24494]: cephadm 2024-09-13T14:14:49.634100+0000 mgr.y (mgr.14150) 144 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 8120M 2024-09-13T14:14:51.914 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:14:51.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi099:vg_nvme/lv_2 2024-09-13T14:14:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:52 smithi099 bash[22656]: cluster 2024-09-13T14:14:50.762048+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:52.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:52 smithi099 bash[32082]: cluster 2024-09-13T14:14:50.762048+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:52 smithi177 bash[24494]: cluster 2024-09-13T14:14:50.762048+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:54.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:54 smithi099 bash[22656]: cluster 2024-09-13T14:14:52.762546+0000 mgr.y (mgr.14150) 146 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:54.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:54 smithi099 bash[32082]: cluster 2024-09-13T14:14:52.762546+0000 mgr.y (mgr.14150) 146 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:54.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:54 smithi177 bash[24494]: cluster 2024-09-13T14:14:52.762546+0000 mgr.y (mgr.14150) 146 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:56.694 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:56.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:56 smithi099 bash[22656]: cluster 2024-09-13T14:14:54.763110+0000 mgr.y (mgr.14150) 147 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:56.796 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:56 smithi099 bash[32082]: cluster 2024-09-13T14:14:54.763110+0000 mgr.y (mgr.14150) 147 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:56.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:56 smithi177 bash[24494]: cluster 2024-09-13T14:14:54.763110+0000 mgr.y (mgr.14150) 147 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:58.718 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:14:58 smithi099 bash[22656]: cluster 2024-09-13T14:14:56.763542+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:58.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:14:58 smithi099 bash[32082]: cluster 2024-09-13T14:14:56.763542+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:14:58.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:14:58 smithi177 bash[24494]: cluster 2024-09-13T14:14:56.763542+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:00.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:00 smithi099 bash[32082]: cluster 2024-09-13T14:14:58.764105+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:00.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:00 smithi099 bash[32082]: audit 2024-09-13T14:15:00.012077+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:15:00.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:00 smithi099 bash[32082]: audit 2024-09-13T14:15:00.015615+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:15:00.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:00 smithi099 bash[32082]: audit 2024-09-13T14:15:00.016713+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:00.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:00 smithi099 bash[22656]: cluster 2024-09-13T14:14:58.764105+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:00.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:00 smithi099 bash[22656]: audit 2024-09-13T14:15:00.012077+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:15:00.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:00 smithi099 bash[22656]: audit 2024-09-13T14:15:00.015615+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:15:00.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:00 smithi099 bash[22656]: audit 2024-09-13T14:15:00.016713+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:00.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:00 smithi177 bash[24494]: cluster 2024-09-13T14:14:58.764105+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:00.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:00 smithi177 bash[24494]: audit 2024-09-13T14:15:00.012077+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:15:00.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:00 smithi177 bash[24494]: audit 2024-09-13T14:15:00.015615+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:15:00.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:00 smithi177 bash[24494]: audit 2024-09-13T14:15:00.016713+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:01.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:01 smithi099 bash[32082]: audit 2024-09-13T14:15:00.009456+0000 mgr.y (mgr.14150) 150 : audit [DBG] from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:01.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:01 smithi099 bash[22656]: audit 2024-09-13T14:15:00.009456+0000 mgr.y (mgr.14150) 150 : audit [DBG] from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:01.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:01 smithi177 bash[24494]: audit 2024-09-13T14:15:00.009456+0000 mgr.y (mgr.14150) 150 : audit [DBG] from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:02.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:02 smithi099 bash[32082]: cluster 2024-09-13T14:15:00.764638+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:02.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:02 smithi099 bash[22656]: cluster 2024-09-13T14:15:00.764638+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:02.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:02 smithi177 bash[24494]: cluster 2024-09-13T14:15:00.764638+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:04.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:04 smithi099 bash[32082]: cluster 2024-09-13T14:15:02.764970+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:04.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:04 smithi099 bash[22656]: cluster 2024-09-13T14:15:02.764970+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:04.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:04 smithi177 bash[24494]: cluster 2024-09-13T14:15:02.764970+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:06.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:06 smithi099 bash[22656]: cluster 2024-09-13T14:15:04.765385+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:06.807 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:06 smithi099 bash[32082]: cluster 2024-09-13T14:15:04.765385+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:06.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:06 smithi177 bash[24494]: cluster 2024-09-13T14:15:04.765385+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[22656]: cluster 2024-09-13T14:15:06.765826+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[22656]: audit 2024-09-13T14:15:08.472740+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.99:0/3093573405' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3b9eff6d-5f5c-4843-a1ff-815a902167ec"}]: dispatch 2024-09-13T14:15:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[22656]: audit 2024-09-13T14:15:08.473176+0000 mon.a (mon.0) 343 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3b9eff6d-5f5c-4843-a1ff-815a902167ec"}]: dispatch 2024-09-13T14:15:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[22656]: audit 2024-09-13T14:15:08.482429+0000 mon.a (mon.0) 344 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3b9eff6d-5f5c-4843-a1ff-815a902167ec"}]': finished 2024-09-13T14:15:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[22656]: cluster 2024-09-13T14:15:08.482616+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:15:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[22656]: audit 2024-09-13T14:15:08.482792+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:08.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[32082]: cluster 2024-09-13T14:15:06.765826+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:08.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[32082]: audit 2024-09-13T14:15:08.472740+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.99:0/3093573405' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3b9eff6d-5f5c-4843-a1ff-815a902167ec"}]: dispatch 2024-09-13T14:15:08.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[32082]: audit 2024-09-13T14:15:08.473176+0000 mon.a (mon.0) 343 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3b9eff6d-5f5c-4843-a1ff-815a902167ec"}]: dispatch 2024-09-13T14:15:08.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[32082]: audit 2024-09-13T14:15:08.482429+0000 mon.a (mon.0) 344 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3b9eff6d-5f5c-4843-a1ff-815a902167ec"}]': finished 2024-09-13T14:15:08.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[32082]: cluster 2024-09-13T14:15:08.482616+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:15:08.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:08 smithi099 bash[32082]: audit 2024-09-13T14:15:08.482792+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:08.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:08 smithi177 bash[24494]: cluster 2024-09-13T14:15:06.765826+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:08.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:08 smithi177 bash[24494]: audit 2024-09-13T14:15:08.472740+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.99:0/3093573405' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3b9eff6d-5f5c-4843-a1ff-815a902167ec"}]: dispatch 2024-09-13T14:15:08.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:08 smithi177 bash[24494]: audit 2024-09-13T14:15:08.473176+0000 mon.a (mon.0) 343 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3b9eff6d-5f5c-4843-a1ff-815a902167ec"}]: dispatch 2024-09-13T14:15:08.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:08 smithi177 bash[24494]: audit 2024-09-13T14:15:08.482429+0000 mon.a (mon.0) 344 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3b9eff6d-5f5c-4843-a1ff-815a902167ec"}]': finished 2024-09-13T14:15:08.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:08 smithi177 bash[24494]: cluster 2024-09-13T14:15:08.482616+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:15:08.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:08 smithi177 bash[24494]: audit 2024-09-13T14:15:08.482792+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:09.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:09 smithi099 bash[32082]: audit 2024-09-13T14:15:09.130810+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.99:0/2319599138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:15:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:09 smithi099 bash[22656]: audit 2024-09-13T14:15:09.130810+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.99:0/2319599138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:15:09.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:09 smithi177 bash[24494]: audit 2024-09-13T14:15:09.130810+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.99:0/2319599138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:15:10.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:10 smithi099 bash[22656]: cluster 2024-09-13T14:15:08.766318+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:10.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:10 smithi099 bash[32082]: cluster 2024-09-13T14:15:08.766318+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:10 smithi177 bash[24494]: cluster 2024-09-13T14:15:08.766318+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:12.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:12 smithi099 bash[32082]: cluster 2024-09-13T14:15:10.766790+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:12.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:12 smithi099 bash[22656]: cluster 2024-09-13T14:15:10.766790+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:12.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:12 smithi177 bash[24494]: cluster 2024-09-13T14:15:10.766790+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:14.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:14 smithi099 bash[22656]: cluster 2024-09-13T14:15:12.767231+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:14.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:14 smithi099 bash[32082]: cluster 2024-09-13T14:15:12.767231+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:14.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:14 smithi177 bash[24494]: cluster 2024-09-13T14:15:12.767231+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:15.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:15 smithi099 bash[22656]: cluster 2024-09-13T14:15:14.767685+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:15.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:15 smithi099 bash[32082]: cluster 2024-09-13T14:15:14.767685+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:15 smithi177 bash[24494]: cluster 2024-09-13T14:15:14.767685+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:18.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:17 smithi099 bash[22656]: cluster 2024-09-13T14:15:16.768132+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:17 smithi099 bash[32082]: cluster 2024-09-13T14:15:16.768132+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:18.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:17 smithi177 bash[24494]: cluster 2024-09-13T14:15:16.768132+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:20.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:19 smithi177 bash[24494]: cluster 2024-09-13T14:15:18.768577+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:20.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:19 smithi099 bash[22656]: cluster 2024-09-13T14:15:18.768577+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:20.249 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:19 smithi099 bash[32082]: cluster 2024-09-13T14:15:18.768577+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:21 smithi177 bash[24494]: cluster 2024-09-13T14:15:20.769007+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:21 smithi177 bash[24494]: audit 2024-09-13T14:15:20.953222+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:15:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:21 smithi177 bash[24494]: audit 2024-09-13T14:15:20.954416+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:21 smithi177 bash[24494]: cephadm 2024-09-13T14:15:20.955555+0000 mgr.y (mgr.14150) 162 : cephadm [INF] Deploying daemon osd.2 on smithi099 2024-09-13T14:15:22.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:21 smithi099 bash[32082]: cluster 2024-09-13T14:15:20.769007+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:22.209 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:21 smithi099 bash[32082]: audit 2024-09-13T14:15:20.953222+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:15:22.209 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:21 smithi099 bash[32082]: audit 2024-09-13T14:15:20.954416+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:22.209 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:21 smithi099 bash[32082]: cephadm 2024-09-13T14:15:20.955555+0000 mgr.y (mgr.14150) 162 : cephadm [INF] Deploying daemon osd.2 on smithi099 2024-09-13T14:15:22.209 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:21 smithi099 bash[22656]: cluster 2024-09-13T14:15:20.769007+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:22.209 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:21 smithi099 bash[22656]: audit 2024-09-13T14:15:20.953222+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:15:22.209 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:21 smithi099 bash[22656]: audit 2024-09-13T14:15:20.954416+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:22.209 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:21 smithi099 bash[22656]: cephadm 2024-09-13T14:15:20.955555+0000 mgr.y (mgr.14150) 162 : cephadm [INF] Deploying daemon osd.2 on smithi099 2024-09-13T14:15:24.102 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:23 smithi099 bash[22656]: cluster 2024-09-13T14:15:22.769443+0000 mgr.y (mgr.14150) 163 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:24.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:23 smithi099 bash[32082]: cluster 2024-09-13T14:15:22.769443+0000 mgr.y (mgr.14150) 163 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:24.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:23 smithi177 bash[24494]: cluster 2024-09-13T14:15:22.769443+0000 mgr.y (mgr.14150) 163 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:25.286 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:25 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:15:25.286 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:15:25 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:15:25.286 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:15:25 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:15:25.287 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:15:25 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:15:25.287 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:25 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:15:25.573 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:15:25 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:15:25.574 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:15:25 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:15:25.574 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:15:25 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:15:25.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:25 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:15:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:25 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:15:26.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:25 smithi099 bash[32082]: cluster 2024-09-13T14:15:24.769918+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:26.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:25 smithi099 bash[32082]: audit 2024-09-13T14:15:25.538258+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:26.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:25 smithi099 bash[32082]: audit 2024-09-13T14:15:25.541875+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:26.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:25 smithi099 bash[32082]: audit 2024-09-13T14:15:25.547910+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:26.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:25 smithi099 bash[22656]: cluster 2024-09-13T14:15:24.769918+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:26.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:25 smithi099 bash[22656]: audit 2024-09-13T14:15:25.538258+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:26.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:25 smithi099 bash[22656]: audit 2024-09-13T14:15:25.541875+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:26.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:25 smithi099 bash[22656]: audit 2024-09-13T14:15:25.547910+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:25 smithi177 bash[24494]: cluster 2024-09-13T14:15:24.769918+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:25 smithi177 bash[24494]: audit 2024-09-13T14:15:25.538258+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:25 smithi177 bash[24494]: audit 2024-09-13T14:15:25.541875+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:25 smithi177 bash[24494]: audit 2024-09-13T14:15:25.547910+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:28.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:27 smithi177 bash[24494]: cluster 2024-09-13T14:15:26.770212+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:28.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:27 smithi099 bash[22656]: cluster 2024-09-13T14:15:26.770212+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:28.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:27 smithi099 bash[32082]: cluster 2024-09-13T14:15:26.770212+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:30.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:29 smithi177 bash[24494]: cluster 2024-09-13T14:15:28.770491+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:29 smithi099 bash[22656]: cluster 2024-09-13T14:15:28.770491+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:29 smithi099 bash[32082]: cluster 2024-09-13T14:15:28.770491+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:32.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:31 smithi177 bash[24494]: cluster 2024-09-13T14:15:30.770902+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:32.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:31 smithi177 bash[24494]: audit 2024-09-13T14:15:31.594542+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:32.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:31 smithi177 bash[24494]: audit 2024-09-13T14:15:31.600547+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:31 smithi099 bash[22656]: cluster 2024-09-13T14:15:30.770902+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:31 smithi099 bash[22656]: audit 2024-09-13T14:15:31.594542+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:31 smithi099 bash[22656]: audit 2024-09-13T14:15:31.600547+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:32.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:31 smithi099 bash[32082]: cluster 2024-09-13T14:15:30.770902+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:32.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:31 smithi099 bash[32082]: audit 2024-09-13T14:15:31.594542+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:32.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:31 smithi099 bash[32082]: audit 2024-09-13T14:15:31.600547+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:33.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:32 smithi099 bash[22656]: audit 2024-09-13T14:15:31.905526+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:33.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:32 smithi099 bash[22656]: audit 2024-09-13T14:15:31.906315+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:33.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:32 smithi099 bash[22656]: audit 2024-09-13T14:15:31.911359+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:33.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:32 smithi099 bash[22656]: audit 2024-09-13T14:15:32.418568+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/4220960661,v1:172.21.15.99:6819/4220960661]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:15:33.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:32 smithi099 bash[22656]: audit 2024-09-13T14:15:32.418912+0000 mon.a (mon.0) 357 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:15:33.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:32 smithi099 bash[32082]: audit 2024-09-13T14:15:31.905526+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:33.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:32 smithi099 bash[32082]: audit 2024-09-13T14:15:31.906315+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:33.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:32 smithi099 bash[32082]: audit 2024-09-13T14:15:31.911359+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:33.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:32 smithi099 bash[32082]: audit 2024-09-13T14:15:32.418568+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/4220960661,v1:172.21.15.99:6819/4220960661]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:15:33.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:32 smithi099 bash[32082]: audit 2024-09-13T14:15:32.418912+0000 mon.a (mon.0) 357 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:15:33.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:32 smithi177 bash[24494]: audit 2024-09-13T14:15:31.905526+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:33.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:32 smithi177 bash[24494]: audit 2024-09-13T14:15:31.906315+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:33.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:32 smithi177 bash[24494]: audit 2024-09-13T14:15:31.911359+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:33.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:32 smithi177 bash[24494]: audit 2024-09-13T14:15:32.418568+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/4220960661,v1:172.21.15.99:6819/4220960661]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:15:33.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:32 smithi177 bash[24494]: audit 2024-09-13T14:15:32.418912+0000 mon.a (mon.0) 357 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:15:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[32082]: cluster 2024-09-13T14:15:32.771344+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[32082]: audit 2024-09-13T14:15:32.928928+0000 mon.a (mon.0) 358 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:15:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[32082]: cluster 2024-09-13T14:15:32.929123+0000 mon.a (mon.0) 359 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:15:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[32082]: audit 2024-09-13T14:15:32.929326+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[32082]: audit 2024-09-13T14:15:32.930527+0000 mon.c (mon.2) 11 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/4220960661,v1:172.21.15.99:6819/4220960661]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:15:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[32082]: audit 2024-09-13T14:15:32.931427+0000 mon.a (mon.0) 361 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:15:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[22656]: cluster 2024-09-13T14:15:32.771344+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[22656]: audit 2024-09-13T14:15:32.928928+0000 mon.a (mon.0) 358 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:15:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[22656]: cluster 2024-09-13T14:15:32.929123+0000 mon.a (mon.0) 359 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:15:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[22656]: audit 2024-09-13T14:15:32.929326+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[22656]: audit 2024-09-13T14:15:32.930527+0000 mon.c (mon.2) 11 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/4220960661,v1:172.21.15.99:6819/4220960661]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:15:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:33 smithi099 bash[22656]: audit 2024-09-13T14:15:32.931427+0000 mon.a (mon.0) 361 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:15:34.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:33 smithi177 bash[24494]: cluster 2024-09-13T14:15:32.771344+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:33 smithi177 bash[24494]: audit 2024-09-13T14:15:32.928928+0000 mon.a (mon.0) 358 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:15:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:33 smithi177 bash[24494]: cluster 2024-09-13T14:15:32.929123+0000 mon.a (mon.0) 359 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:15:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:33 smithi177 bash[24494]: audit 2024-09-13T14:15:32.929326+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:33 smithi177 bash[24494]: audit 2024-09-13T14:15:32.930527+0000 mon.c (mon.2) 11 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/4220960661,v1:172.21.15.99:6819/4220960661]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:15:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:33 smithi177 bash[24494]: audit 2024-09-13T14:15:32.931427+0000 mon.a (mon.0) 361 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:15:35.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:34 smithi099 bash[22656]: audit 2024-09-13T14:15:33.930642+0000 mon.a (mon.0) 362 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:15:35.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:34 smithi099 bash[22656]: cluster 2024-09-13T14:15:33.930774+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:15:35.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:34 smithi099 bash[22656]: audit 2024-09-13T14:15:33.933093+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:35.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:34 smithi099 bash[22656]: audit 2024-09-13T14:15:33.942570+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:35.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:34 smithi099 bash[32082]: audit 2024-09-13T14:15:33.930642+0000 mon.a (mon.0) 362 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:15:35.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:34 smithi099 bash[32082]: cluster 2024-09-13T14:15:33.930774+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:15:35.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:34 smithi099 bash[32082]: audit 2024-09-13T14:15:33.933093+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:35.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:34 smithi099 bash[32082]: audit 2024-09-13T14:15:33.942570+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:34 smithi177 bash[24494]: audit 2024-09-13T14:15:33.930642+0000 mon.a (mon.0) 362 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:15:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:34 smithi177 bash[24494]: cluster 2024-09-13T14:15:33.930774+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:15:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:34 smithi177 bash[24494]: audit 2024-09-13T14:15:33.933093+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:34 smithi177 bash[24494]: audit 2024-09-13T14:15:33.942570+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:35.585 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 2 on host 'smithi099' 2024-09-13T14:15:36.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[32082]: cluster 2024-09-13T14:15:33.420560+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:15:36.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[32082]: cluster 2024-09-13T14:15:33.420622+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:15:36.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[32082]: cluster 2024-09-13T14:15:34.771680+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:36.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[32082]: cluster 2024-09-13T14:15:34.946863+0000 mon.a (mon.0) 366 : cluster [INF] osd.2 [v2:172.21.15.99:6818/4220960661,v1:172.21.15.99:6819/4220960661] boot 2024-09-13T14:15:36.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[32082]: cluster 2024-09-13T14:15:34.946956+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:15:36.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[32082]: audit 2024-09-13T14:15:34.947671+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:36.255 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[32082]: audit 2024-09-13T14:15:35.561981+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:36.255 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[32082]: audit 2024-09-13T14:15:35.569114+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:36.255 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[32082]: audit 2024-09-13T14:15:35.579290+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:36.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[22656]: cluster 2024-09-13T14:15:33.420560+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:15:36.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[22656]: cluster 2024-09-13T14:15:33.420622+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:15:36.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[22656]: cluster 2024-09-13T14:15:34.771680+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:36.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[22656]: cluster 2024-09-13T14:15:34.946863+0000 mon.a (mon.0) 366 : cluster [INF] osd.2 [v2:172.21.15.99:6818/4220960661,v1:172.21.15.99:6819/4220960661] boot 2024-09-13T14:15:36.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[22656]: cluster 2024-09-13T14:15:34.946956+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:15:36.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[22656]: audit 2024-09-13T14:15:34.947671+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:36.255 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[22656]: audit 2024-09-13T14:15:35.561981+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:36.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[22656]: audit 2024-09-13T14:15:35.569114+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:36.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:35 smithi099 bash[22656]: audit 2024-09-13T14:15:35.579290+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:36.257 DEBUG:teuthology.orchestra.run.smithi099:osd.2> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T14:15:36.259 INFO:tasks.cephadm:Deploying osd.3 on smithi099 with /dev/vg_nvme/lv_1... 2024-09-13T14:15:36.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T14:15:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:35 smithi177 bash[24494]: cluster 2024-09-13T14:15:33.420560+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:15:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:35 smithi177 bash[24494]: cluster 2024-09-13T14:15:33.420622+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:15:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:35 smithi177 bash[24494]: cluster 2024-09-13T14:15:34.771680+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T14:15:36.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:35 smithi177 bash[24494]: cluster 2024-09-13T14:15:34.946863+0000 mon.a (mon.0) 366 : cluster [INF] osd.2 [v2:172.21.15.99:6818/4220960661,v1:172.21.15.99:6819/4220960661] boot 2024-09-13T14:15:36.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:35 smithi177 bash[24494]: cluster 2024-09-13T14:15:34.946956+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:15:36.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:35 smithi177 bash[24494]: audit 2024-09-13T14:15:34.947671+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:15:36.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:35 smithi177 bash[24494]: audit 2024-09-13T14:15:35.561981+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:36.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:35 smithi177 bash[24494]: audit 2024-09-13T14:15:35.569114+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:36.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:35 smithi177 bash[24494]: audit 2024-09-13T14:15:35.579290+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:37.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:37 smithi099 bash[22656]: cluster 2024-09-13T14:15:35.976224+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:15:37.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:37 smithi099 bash[22656]: audit 2024-09-13T14:15:36.857734+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:15:37.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:37 smithi099 bash[32082]: cluster 2024-09-13T14:15:35.976224+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:15:37.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:37 smithi099 bash[32082]: audit 2024-09-13T14:15:36.857734+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:15:37.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:37 smithi177 bash[24494]: cluster 2024-09-13T14:15:35.976224+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:15:37.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:37 smithi177 bash[24494]: audit 2024-09-13T14:15:36.857734+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T14:15:38.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:38 smithi177 bash[24494]: cluster 2024-09-13T14:15:36.772139+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v138: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:38.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:38 smithi177 bash[24494]: audit 2024-09-13T14:15:37.345135+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:15:38.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:38 smithi177 bash[24494]: cluster 2024-09-13T14:15:37.345177+0000 mon.a (mon.0) 375 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:15:38.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:38 smithi177 bash[24494]: audit 2024-09-13T14:15:37.346527+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:15:38.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:38 smithi099 bash[32082]: cluster 2024-09-13T14:15:36.772139+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v138: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:38.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:38 smithi099 bash[32082]: audit 2024-09-13T14:15:37.345135+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:15:38.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:38 smithi099 bash[32082]: cluster 2024-09-13T14:15:37.345177+0000 mon.a (mon.0) 375 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:15:38.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:38 smithi099 bash[32082]: audit 2024-09-13T14:15:37.346527+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:15:38.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:38 smithi099 bash[22656]: cluster 2024-09-13T14:15:36.772139+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v138: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:38.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:38 smithi099 bash[22656]: audit 2024-09-13T14:15:37.345135+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T14:15:38.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:38 smithi099 bash[22656]: cluster 2024-09-13T14:15:37.345177+0000 mon.a (mon.0) 375 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:15:38.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:38 smithi099 bash[22656]: audit 2024-09-13T14:15:37.346527+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:15:39.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:39 smithi099 bash[22656]: audit 2024-09-13T14:15:38.354865+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:15:39.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:39 smithi099 bash[22656]: cluster 2024-09-13T14:15:38.354970+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:15:39.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:39 smithi099 bash[32082]: audit 2024-09-13T14:15:38.354865+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:15:39.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:39 smithi099 bash[32082]: cluster 2024-09-13T14:15:38.354970+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:15:39.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:39 smithi177 bash[24494]: audit 2024-09-13T14:15:38.354865+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:15:39.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:39 smithi177 bash[24494]: cluster 2024-09-13T14:15:38.354970+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:15:40.373 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:40 smithi099 bash[32082]: cluster 2024-09-13T14:15:38.773271+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v141: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:40 smithi099 bash[32082]: cluster 2024-09-13T14:15:39.371720+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2024-09-13T14:15:40.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:40 smithi099 bash[32082]: audit 2024-09-13T14:15:39.705671+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:40.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:40 smithi099 bash[22656]: cluster 2024-09-13T14:15:38.773271+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v141: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:40.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:40 smithi099 bash[22656]: cluster 2024-09-13T14:15:39.371720+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2024-09-13T14:15:40.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:40 smithi099 bash[22656]: audit 2024-09-13T14:15:39.705671+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:40.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:40 smithi177 bash[24494]: cluster 2024-09-13T14:15:38.773271+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v141: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:40.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:40 smithi177 bash[24494]: cluster 2024-09-13T14:15:39.371720+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2024-09-13T14:15:40.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:40 smithi177 bash[24494]: audit 2024-09-13T14:15:39.705671+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:43.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[22656]: cluster 2024-09-13T14:15:40.773689+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:43.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[22656]: cluster 2024-09-13T14:15:41.375607+0000 mon.a (mon.0) 381 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-09-13T14:15:43.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[22656]: audit 2024-09-13T14:15:41.453805+0000 mon.a (mon.0) 382 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:15:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[22656]: audit 2024-09-13T14:15:42.344489+0000 mon.a (mon.0) 383 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:15:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[22656]: audit 2024-09-13T14:15:42.345056+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[22656]: audit 2024-09-13T14:15:42.345200+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[22656]: audit 2024-09-13T14:15:42.345354+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[22656]: audit 2024-09-13T14:15:42.347906+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[22656]: audit 2024-09-13T14:15:42.348113+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[22656]: audit 2024-09-13T14:15:42.348295+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:43.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[32082]: cluster 2024-09-13T14:15:40.773689+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:43.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[32082]: cluster 2024-09-13T14:15:41.375607+0000 mon.a (mon.0) 381 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-09-13T14:15:43.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[32082]: audit 2024-09-13T14:15:41.453805+0000 mon.a (mon.0) 382 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:15:43.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[32082]: audit 2024-09-13T14:15:42.344489+0000 mon.a (mon.0) 383 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:15:43.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[32082]: audit 2024-09-13T14:15:42.345056+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:43.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[32082]: audit 2024-09-13T14:15:42.345200+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:43.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[32082]: audit 2024-09-13T14:15:42.345354+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:43.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[32082]: audit 2024-09-13T14:15:42.347906+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:43.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[32082]: audit 2024-09-13T14:15:42.348113+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:43.329 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:43 smithi099 bash[32082]: audit 2024-09-13T14:15:42.348295+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:43 smithi177 bash[24494]: cluster 2024-09-13T14:15:40.773689+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:43 smithi177 bash[24494]: cluster 2024-09-13T14:15:41.375607+0000 mon.a (mon.0) 381 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-09-13T14:15:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:43 smithi177 bash[24494]: audit 2024-09-13T14:15:41.453805+0000 mon.a (mon.0) 382 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:15:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:43 smithi177 bash[24494]: audit 2024-09-13T14:15:42.344489+0000 mon.a (mon.0) 383 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:15:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:43 smithi177 bash[24494]: audit 2024-09-13T14:15:42.345056+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:43 smithi177 bash[24494]: audit 2024-09-13T14:15:42.345200+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:43 smithi177 bash[24494]: audit 2024-09-13T14:15:42.345354+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:43 smithi177 bash[24494]: audit 2024-09-13T14:15:42.347906+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:43 smithi177 bash[24494]: audit 2024-09-13T14:15:42.348113+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:43 smithi177 bash[24494]: audit 2024-09-13T14:15:42.348295+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:44.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[22656]: audit 2024-09-13T14:15:42.349049+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:15:44.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[22656]: cluster 2024-09-13T14:15:42.774217+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:44.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[22656]: audit 2024-09-13T14:15:43.216783+0000 mon.b (mon.1) 9 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:15:44.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[22656]: audit 2024-09-13T14:15:43.272073+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:44.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[22656]: audit 2024-09-13T14:15:43.275053+0000 mon.c (mon.2) 12 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:15:44.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[22656]: audit 2024-09-13T14:15:43.275377+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:44.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[22656]: audit 2024-09-13T14:15:43.275800+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:44.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[22656]: audit 2024-09-13T14:15:43.276233+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:44.490 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[22656]: audit 2024-09-13T14:15:44.185930+0000 mon.c (mon.2) 13 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:15:44.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[32082]: audit 2024-09-13T14:15:42.349049+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:15:44.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[32082]: cluster 2024-09-13T14:15:42.774217+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:44.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[32082]: audit 2024-09-13T14:15:43.216783+0000 mon.b (mon.1) 9 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:15:44.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[32082]: audit 2024-09-13T14:15:43.272073+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:44.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[32082]: audit 2024-09-13T14:15:43.275053+0000 mon.c (mon.2) 12 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:15:44.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[32082]: audit 2024-09-13T14:15:43.275377+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:44.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[32082]: audit 2024-09-13T14:15:43.275800+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:44.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[32082]: audit 2024-09-13T14:15:43.276233+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:44.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:44 smithi099 bash[32082]: audit 2024-09-13T14:15:44.185930+0000 mon.c (mon.2) 13 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:15:44.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:44 smithi177 bash[24494]: audit 2024-09-13T14:15:42.349049+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:15:44.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:44 smithi177 bash[24494]: cluster 2024-09-13T14:15:42.774217+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:44.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:44 smithi177 bash[24494]: audit 2024-09-13T14:15:43.216783+0000 mon.b (mon.1) 9 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:15:44.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:44 smithi177 bash[24494]: audit 2024-09-13T14:15:43.272073+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:44.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:44 smithi177 bash[24494]: audit 2024-09-13T14:15:43.275053+0000 mon.c (mon.2) 12 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:15:44.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:44 smithi177 bash[24494]: audit 2024-09-13T14:15:43.275377+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:44.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:44 smithi177 bash[24494]: audit 2024-09-13T14:15:43.275800+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:44.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:44 smithi177 bash[24494]: audit 2024-09-13T14:15:43.276233+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:44.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:44 smithi177 bash[24494]: audit 2024-09-13T14:15:44.185930+0000 mon.c (mon.2) 13 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:15:46.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:46 smithi099 bash[22656]: cluster 2024-09-13T14:15:44.774691+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:46.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:46 smithi099 bash[22656]: audit 2024-09-13T14:15:46.212501+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:46.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:46 smithi099 bash[22656]: audit 2024-09-13T14:15:46.219779+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:46.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:46 smithi099 bash[32082]: cluster 2024-09-13T14:15:44.774691+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:46.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:46 smithi099 bash[32082]: audit 2024-09-13T14:15:46.212501+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:46.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:46 smithi099 bash[32082]: audit 2024-09-13T14:15:46.219779+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:46.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:46 smithi177 bash[24494]: cluster 2024-09-13T14:15:44.774691+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:15:46.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:46 smithi177 bash[24494]: audit 2024-09-13T14:15:46.212501+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:46.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:46 smithi177 bash[24494]: audit 2024-09-13T14:15:46.219779+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:47.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[32082]: cephadm 2024-09-13T14:15:46.204892+0000 mgr.y (mgr.14150) 175 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:15:47.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[32082]: audit 2024-09-13T14:15:46.221983+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:47.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[32082]: audit 2024-09-13T14:15:46.223436+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[32082]: audit 2024-09-13T14:15:46.228733+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[32082]: cephadm 2024-09-13T14:15:46.230101+0000 mgr.y (mgr.14150) 176 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 5413M 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[32082]: audit 2024-09-13T14:15:46.236704+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[32082]: audit 2024-09-13T14:15:46.238869+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[32082]: audit 2024-09-13T14:15:46.239995+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[32082]: audit 2024-09-13T14:15:46.247280+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[22656]: cephadm 2024-09-13T14:15:46.204892+0000 mgr.y (mgr.14150) 175 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[22656]: audit 2024-09-13T14:15:46.221983+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[22656]: audit 2024-09-13T14:15:46.223436+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[22656]: audit 2024-09-13T14:15:46.228733+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:47.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[22656]: cephadm 2024-09-13T14:15:46.230101+0000 mgr.y (mgr.14150) 176 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 5413M 2024-09-13T14:15:47.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[22656]: audit 2024-09-13T14:15:46.236704+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:47.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[22656]: audit 2024-09-13T14:15:46.238869+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:47.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[22656]: audit 2024-09-13T14:15:46.239995+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:47.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:47 smithi099 bash[22656]: audit 2024-09-13T14:15:46.247280+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:47 smithi177 bash[24494]: cephadm 2024-09-13T14:15:46.204892+0000 mgr.y (mgr.14150) 175 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:15:47.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:47 smithi177 bash[24494]: audit 2024-09-13T14:15:46.221983+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:47.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:47 smithi177 bash[24494]: audit 2024-09-13T14:15:46.223436+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:47.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:47 smithi177 bash[24494]: audit 2024-09-13T14:15:46.228733+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:47.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:47 smithi177 bash[24494]: cephadm 2024-09-13T14:15:46.230101+0000 mgr.y (mgr.14150) 176 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 5413M 2024-09-13T14:15:47.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:47 smithi177 bash[24494]: audit 2024-09-13T14:15:46.236704+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:47.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:47 smithi177 bash[24494]: audit 2024-09-13T14:15:46.238869+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:47.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:47 smithi177 bash[24494]: audit 2024-09-13T14:15:46.239995+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:47.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:47 smithi177 bash[24494]: audit 2024-09-13T14:15:46.247280+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:15:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:48 smithi099 bash[32082]: cluster 2024-09-13T14:15:46.775240+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T14:15:48.658 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:48 smithi099 bash[22656]: cluster 2024-09-13T14:15:46.775240+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T14:15:48.659 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:48 smithi177 bash[24494]: cluster 2024-09-13T14:15:46.775240+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T14:15:48.663 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:15:48.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi099:vg_nvme/lv_1 2024-09-13T14:15:50.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:50 smithi099 bash[32082]: cluster 2024-09-13T14:15:48.775603+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T14:15:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:50 smithi099 bash[22656]: cluster 2024-09-13T14:15:48.775603+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T14:15:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:50 smithi177 bash[24494]: cluster 2024-09-13T14:15:48.775603+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T14:15:52.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:52 smithi099 bash[22656]: cluster 2024-09-13T14:15:50.776094+0000 mgr.y (mgr.14150) 179 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T14:15:52.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:52 smithi099 bash[32082]: cluster 2024-09-13T14:15:50.776094+0000 mgr.y (mgr.14150) 179 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T14:15:52.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:52 smithi177 bash[24494]: cluster 2024-09-13T14:15:50.776094+0000 mgr.y (mgr.14150) 179 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T14:15:53.444 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:54.552 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:54 smithi099 bash[22656]: cluster 2024-09-13T14:15:52.776616+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:15:54.552 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:54 smithi099 bash[32082]: cluster 2024-09-13T14:15:52.776616+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:15:54.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:54 smithi177 bash[24494]: cluster 2024-09-13T14:15:52.776616+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:15:56.536 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:56 smithi099 bash[22656]: cluster 2024-09-13T14:15:54.777088+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:15:56.536 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:56 smithi099 bash[32082]: cluster 2024-09-13T14:15:54.777088+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:15:56.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:56 smithi177 bash[24494]: cluster 2024-09-13T14:15:54.777088+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:15:57.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:57 smithi099 bash[22656]: audit 2024-09-13T14:15:56.879983+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:15:57.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:57 smithi099 bash[22656]: audit 2024-09-13T14:15:56.885546+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:15:57.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:57 smithi099 bash[22656]: audit 2024-09-13T14:15:56.887121+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:57.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:57 smithi099 bash[32082]: audit 2024-09-13T14:15:56.879983+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:15:57.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:57 smithi099 bash[32082]: audit 2024-09-13T14:15:56.885546+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:15:57.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:57 smithi099 bash[32082]: audit 2024-09-13T14:15:56.887121+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:57.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:57 smithi177 bash[24494]: audit 2024-09-13T14:15:56.879983+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:15:57.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:57 smithi177 bash[24494]: audit 2024-09-13T14:15:56.885546+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:15:57.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:57 smithi177 bash[24494]: audit 2024-09-13T14:15:56.887121+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:58 smithi099 bash[22656]: cluster 2024-09-13T14:15:56.777509+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:15:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:15:58 smithi099 bash[22656]: audit 2024-09-13T14:15:56.873782+0000 mgr.y (mgr.14150) 183 : audit [DBG] from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:58 smithi099 bash[32082]: cluster 2024-09-13T14:15:56.777509+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:15:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:15:58 smithi099 bash[32082]: audit 2024-09-13T14:15:56.873782+0000 mgr.y (mgr.14150) 183 : audit [DBG] from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:58.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:58 smithi177 bash[24494]: cluster 2024-09-13T14:15:56.777509+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:15:58.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:15:58 smithi177 bash[24494]: audit 2024-09-13T14:15:56.873782+0000 mgr.y (mgr.14150) 183 : audit [DBG] from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:00.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:00 smithi099 bash[22656]: cluster 2024-09-13T14:15:58.778032+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:00.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:00 smithi099 bash[32082]: cluster 2024-09-13T14:15:58.778032+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:00.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:00 smithi177 bash[24494]: cluster 2024-09-13T14:15:58.778032+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:02.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:02 smithi099 bash[22656]: cluster 2024-09-13T14:16:00.778547+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:02.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:02 smithi099 bash[32082]: cluster 2024-09-13T14:16:00.778547+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:02.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:02 smithi177 bash[24494]: cluster 2024-09-13T14:16:00.778547+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:04.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:04 smithi099 bash[32082]: cluster 2024-09-13T14:16:02.779059+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:04.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:04 smithi099 bash[22656]: cluster 2024-09-13T14:16:02.779059+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:04.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:04 smithi177 bash[24494]: cluster 2024-09-13T14:16:02.779059+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:05.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:05 smithi099 bash[22656]: audit 2024-09-13T14:16:05.083496+0000 mon.a (mon.0) 406 : audit [INF] from='client.? 172.21.15.99:0/1512948645' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "686c8d3c-9cab-4920-b02b-d243324f5808"}]: dispatch 2024-09-13T14:16:05.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:05 smithi099 bash[22656]: audit 2024-09-13T14:16:05.091868+0000 mon.a (mon.0) 407 : audit [INF] from='client.? 172.21.15.99:0/1512948645' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "686c8d3c-9cab-4920-b02b-d243324f5808"}]': finished 2024-09-13T14:16:05.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:05 smithi099 bash[22656]: cluster 2024-09-13T14:16:05.091913+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:16:05.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:05 smithi099 bash[22656]: audit 2024-09-13T14:16:05.092144+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:05 smithi099 bash[32082]: audit 2024-09-13T14:16:05.083496+0000 mon.a (mon.0) 406 : audit [INF] from='client.? 172.21.15.99:0/1512948645' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "686c8d3c-9cab-4920-b02b-d243324f5808"}]: dispatch 2024-09-13T14:16:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:05 smithi099 bash[32082]: audit 2024-09-13T14:16:05.091868+0000 mon.a (mon.0) 407 : audit [INF] from='client.? 172.21.15.99:0/1512948645' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "686c8d3c-9cab-4920-b02b-d243324f5808"}]': finished 2024-09-13T14:16:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:05 smithi099 bash[32082]: cluster 2024-09-13T14:16:05.091913+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:16:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:05 smithi099 bash[32082]: audit 2024-09-13T14:16:05.092144+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:05.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:05 smithi177 bash[24494]: audit 2024-09-13T14:16:05.083496+0000 mon.a (mon.0) 406 : audit [INF] from='client.? 172.21.15.99:0/1512948645' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "686c8d3c-9cab-4920-b02b-d243324f5808"}]: dispatch 2024-09-13T14:16:05.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:05 smithi177 bash[24494]: audit 2024-09-13T14:16:05.091868+0000 mon.a (mon.0) 407 : audit [INF] from='client.? 172.21.15.99:0/1512948645' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "686c8d3c-9cab-4920-b02b-d243324f5808"}]': finished 2024-09-13T14:16:05.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:05 smithi177 bash[24494]: cluster 2024-09-13T14:16:05.091913+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:16:05.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:05 smithi177 bash[24494]: audit 2024-09-13T14:16:05.092144+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:06.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:06 smithi099 bash[22656]: cluster 2024-09-13T14:16:04.779523+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:06.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:06 smithi099 bash[22656]: audit 2024-09-13T14:16:05.717616+0000 mon.a (mon.0) 410 : audit [DBG] from='client.? 172.21.15.99:0/3903620868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:06 smithi099 bash[32082]: cluster 2024-09-13T14:16:04.779523+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:06.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:06 smithi099 bash[32082]: audit 2024-09-13T14:16:05.717616+0000 mon.a (mon.0) 410 : audit [DBG] from='client.? 172.21.15.99:0/3903620868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:06.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:06 smithi177 bash[24494]: cluster 2024-09-13T14:16:04.779523+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:06.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:06 smithi177 bash[24494]: audit 2024-09-13T14:16:05.717616+0000 mon.a (mon.0) 410 : audit [DBG] from='client.? 172.21.15.99:0/3903620868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:08 smithi099 bash[22656]: cluster 2024-09-13T14:16:06.779860+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:08.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:08 smithi099 bash[32082]: cluster 2024-09-13T14:16:06.779860+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:08.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:08 smithi177 bash[24494]: cluster 2024-09-13T14:16:06.779860+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:10.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:10 smithi099 bash[32082]: cluster 2024-09-13T14:16:08.780356+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:10.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:10 smithi099 bash[22656]: cluster 2024-09-13T14:16:08.780356+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:10.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:10 smithi177 bash[24494]: cluster 2024-09-13T14:16:08.780356+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:12.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:12 smithi099 bash[32082]: cluster 2024-09-13T14:16:10.780870+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:12.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:12 smithi099 bash[22656]: cluster 2024-09-13T14:16:10.780870+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:12 smithi177 bash[24494]: cluster 2024-09-13T14:16:10.780870+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:14.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:14 smithi099 bash[32082]: cluster 2024-09-13T14:16:12.781432+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:14.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:14 smithi099 bash[22656]: cluster 2024-09-13T14:16:12.781432+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:14.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:14 smithi177 bash[24494]: cluster 2024-09-13T14:16:12.781432+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:16.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:16 smithi099 bash[32082]: cluster 2024-09-13T14:16:14.781971+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:16.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:16 smithi099 bash[22656]: cluster 2024-09-13T14:16:14.781971+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:16.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:16 smithi177 bash[24494]: cluster 2024-09-13T14:16:14.781971+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:18.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:18 smithi099 bash[22656]: cluster 2024-09-13T14:16:16.782456+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:18.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:18 smithi099 bash[22656]: audit 2024-09-13T14:16:17.938628+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:16:18.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:18 smithi099 bash[22656]: audit 2024-09-13T14:16:17.940352+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:18.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:18 smithi099 bash[32082]: cluster 2024-09-13T14:16:16.782456+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:18.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:18 smithi099 bash[32082]: audit 2024-09-13T14:16:17.938628+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:16:18.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:18 smithi099 bash[32082]: audit 2024-09-13T14:16:17.940352+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:18.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:18 smithi177 bash[24494]: cluster 2024-09-13T14:16:16.782456+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:18.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:18 smithi177 bash[24494]: audit 2024-09-13T14:16:17.938628+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:16:18.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:18 smithi177 bash[24494]: audit 2024-09-13T14:16:17.940352+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:19.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:19 smithi177 bash[24494]: cephadm 2024-09-13T14:16:17.941701+0000 mgr.y (mgr.14150) 194 : cephadm [INF] Deploying daemon osd.3 on smithi099 2024-09-13T14:16:19.734 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:19 smithi099 bash[22656]: cephadm 2024-09-13T14:16:17.941701+0000 mgr.y (mgr.14150) 194 : cephadm [INF] Deploying daemon osd.3 on smithi099 2024-09-13T14:16:19.735 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:19 smithi099 bash[32082]: cephadm 2024-09-13T14:16:17.941701+0000 mgr.y (mgr.14150) 194 : cephadm [INF] Deploying daemon osd.3 on smithi099 2024-09-13T14:16:20.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:20 smithi177 bash[24494]: cluster 2024-09-13T14:16:18.782980+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:20.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:20 smithi099 bash[22656]: cluster 2024-09-13T14:16:18.782980+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:20.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:20 smithi099 bash[32082]: cluster 2024-09-13T14:16:18.782980+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:22.558 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.558 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:22 smithi099 bash[22656]: cluster 2024-09-13T14:16:20.783526+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:22.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:22 smithi099 bash[32082]: cluster 2024-09-13T14:16:20.783526+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:22.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.559 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.560 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.560 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:22 smithi177 bash[24494]: cluster 2024-09-13T14:16:20.783526+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:22.825 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.825 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.825 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.826 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:22.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:22 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:16:23.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:23 smithi177 bash[24494]: audit 2024-09-13T14:16:22.831603+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:23.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:23 smithi177 bash[24494]: audit 2024-09-13T14:16:22.835831+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:23.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:23 smithi177 bash[24494]: audit 2024-09-13T14:16:22.841847+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:23.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:23 smithi099 bash[22656]: audit 2024-09-13T14:16:22.831603+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:23.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:23 smithi099 bash[22656]: audit 2024-09-13T14:16:22.835831+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:23.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:23 smithi099 bash[22656]: audit 2024-09-13T14:16:22.841847+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:23.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:23 smithi099 bash[32082]: audit 2024-09-13T14:16:22.831603+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:23.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:23 smithi099 bash[32082]: audit 2024-09-13T14:16:22.835831+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:23.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:23 smithi099 bash[32082]: audit 2024-09-13T14:16:22.841847+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:24.516 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:24 smithi099 bash[22656]: cluster 2024-09-13T14:16:22.784024+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:24.517 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:24 smithi099 bash[32082]: cluster 2024-09-13T14:16:22.784024+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:24.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:24 smithi177 bash[24494]: cluster 2024-09-13T14:16:22.784024+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:26 smithi177 bash[24494]: cluster 2024-09-13T14:16:24.784525+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:26.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:26 smithi099 bash[22656]: cluster 2024-09-13T14:16:24.784525+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:26.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:26 smithi099 bash[32082]: cluster 2024-09-13T14:16:24.784525+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:28.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:28 smithi177 bash[24494]: cluster 2024-09-13T14:16:26.785065+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:28.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:28 smithi099 bash[22656]: cluster 2024-09-13T14:16:26.785065+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:28.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:28 smithi099 bash[32082]: cluster 2024-09-13T14:16:26.785065+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[22656]: cluster 2024-09-13T14:16:28.785539+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[22656]: audit 2024-09-13T14:16:29.124784+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[22656]: audit 2024-09-13T14:16:29.129611+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[22656]: audit 2024-09-13T14:16:29.130406+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[22656]: audit 2024-09-13T14:16:29.131129+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[22656]: audit 2024-09-13T14:16:29.135694+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[22656]: audit 2024-09-13T14:16:30.001729+0000 mon.a (mon.0) 421 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:16:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[32082]: cluster 2024-09-13T14:16:28.785539+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[32082]: audit 2024-09-13T14:16:29.124784+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[32082]: audit 2024-09-13T14:16:29.129611+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[32082]: audit 2024-09-13T14:16:29.130406+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[32082]: audit 2024-09-13T14:16:29.131129+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[32082]: audit 2024-09-13T14:16:29.135694+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:30 smithi099 bash[32082]: audit 2024-09-13T14:16:30.001729+0000 mon.a (mon.0) 421 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:16:30.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:30 smithi177 bash[24494]: cluster 2024-09-13T14:16:28.785539+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:30.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:30 smithi177 bash[24494]: audit 2024-09-13T14:16:29.124784+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:30.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:30 smithi177 bash[24494]: audit 2024-09-13T14:16:29.129611+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:30.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:30 smithi177 bash[24494]: audit 2024-09-13T14:16:29.130406+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:30.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:30 smithi177 bash[24494]: audit 2024-09-13T14:16:29.131129+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:30.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:30 smithi177 bash[24494]: audit 2024-09-13T14:16:29.135694+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:30.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:30 smithi177 bash[24494]: audit 2024-09-13T14:16:30.001729+0000 mon.a (mon.0) 421 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:16:31.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:31 smithi099 bash[22656]: audit 2024-09-13T14:16:30.360520+0000 mon.a (mon.0) 422 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:16:31.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:31 smithi099 bash[22656]: cluster 2024-09-13T14:16:30.360624+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:16:31.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:31 smithi099 bash[22656]: audit 2024-09-13T14:16:30.360934+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:31.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:31 smithi099 bash[22656]: audit 2024-09-13T14:16:30.362475+0000 mon.a (mon.0) 425 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:16:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:31 smithi099 bash[32082]: audit 2024-09-13T14:16:30.360520+0000 mon.a (mon.0) 422 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:16:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:31 smithi099 bash[32082]: cluster 2024-09-13T14:16:30.360624+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:16:31.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:31 smithi099 bash[32082]: audit 2024-09-13T14:16:30.360934+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:31.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:31 smithi099 bash[32082]: audit 2024-09-13T14:16:30.362475+0000 mon.a (mon.0) 425 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:16:31.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:31 smithi177 bash[24494]: audit 2024-09-13T14:16:30.360520+0000 mon.a (mon.0) 422 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:16:31.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:31 smithi177 bash[24494]: cluster 2024-09-13T14:16:30.360624+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:16:31.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:31 smithi177 bash[24494]: audit 2024-09-13T14:16:30.360934+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:31.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:31 smithi177 bash[24494]: audit 2024-09-13T14:16:30.362475+0000 mon.a (mon.0) 425 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-09-13T14:16:32.545 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:32 smithi099 bash[32082]: cluster 2024-09-13T14:16:30.786121+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:32.545 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:32 smithi099 bash[32082]: audit 2024-09-13T14:16:31.363967+0000 mon.a (mon.0) 426 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:16:32.545 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:32 smithi099 bash[32082]: cluster 2024-09-13T14:16:31.364081+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2024-09-13T14:16:32.545 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:32 smithi099 bash[32082]: audit 2024-09-13T14:16:31.364952+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:32.545 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:32 smithi099 bash[32082]: audit 2024-09-13T14:16:31.375031+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:32.545 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:32 smithi099 bash[22656]: cluster 2024-09-13T14:16:30.786121+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:32.545 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:32 smithi099 bash[22656]: audit 2024-09-13T14:16:31.363967+0000 mon.a (mon.0) 426 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:16:32.546 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:32 smithi099 bash[22656]: cluster 2024-09-13T14:16:31.364081+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2024-09-13T14:16:32.546 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:32 smithi099 bash[22656]: audit 2024-09-13T14:16:31.364952+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:32.546 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:32 smithi099 bash[22656]: audit 2024-09-13T14:16:31.375031+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:32.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:32 smithi177 bash[24494]: cluster 2024-09-13T14:16:30.786121+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:32.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:32 smithi177 bash[24494]: audit 2024-09-13T14:16:31.363967+0000 mon.a (mon.0) 426 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-09-13T14:16:32.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:32 smithi177 bash[24494]: cluster 2024-09-13T14:16:31.364081+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2024-09-13T14:16:32.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:32 smithi177 bash[24494]: audit 2024-09-13T14:16:31.364952+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:32.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:32 smithi177 bash[24494]: audit 2024-09-13T14:16:31.375031+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:33.348 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 3 on host 'smithi099' 2024-09-13T14:16:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[22656]: cluster 2024-09-13T14:16:30.964662+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:16:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[22656]: cluster 2024-09-13T14:16:30.964750+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:16:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[22656]: cluster 2024-09-13T14:16:32.376022+0000 mon.a (mon.0) 430 : cluster [INF] osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921] boot 2024-09-13T14:16:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[22656]: cluster 2024-09-13T14:16:32.376054+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:16:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[22656]: audit 2024-09-13T14:16:32.376466+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[22656]: audit 2024-09-13T14:16:33.327343+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[22656]: audit 2024-09-13T14:16:33.334778+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[22656]: audit 2024-09-13T14:16:33.345525+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[32082]: cluster 2024-09-13T14:16:30.964662+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:16:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[32082]: cluster 2024-09-13T14:16:30.964750+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:16:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[32082]: cluster 2024-09-13T14:16:32.376022+0000 mon.a (mon.0) 430 : cluster [INF] osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921] boot 2024-09-13T14:16:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[32082]: cluster 2024-09-13T14:16:32.376054+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:16:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[32082]: audit 2024-09-13T14:16:32.376466+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[32082]: audit 2024-09-13T14:16:33.327343+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[32082]: audit 2024-09-13T14:16:33.334778+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:33 smithi099 bash[32082]: audit 2024-09-13T14:16:33.345525+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:33.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:33 smithi177 bash[24494]: cluster 2024-09-13T14:16:30.964662+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:16:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:33 smithi177 bash[24494]: cluster 2024-09-13T14:16:30.964750+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:16:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:33 smithi177 bash[24494]: cluster 2024-09-13T14:16:32.376022+0000 mon.a (mon.0) 430 : cluster [INF] osd.3 [v2:172.21.15.99:6826/1221827921,v1:172.21.15.99:6827/1221827921] boot 2024-09-13T14:16:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:33 smithi177 bash[24494]: cluster 2024-09-13T14:16:32.376054+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:16:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:33 smithi177 bash[24494]: audit 2024-09-13T14:16:32.376466+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:16:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:33 smithi177 bash[24494]: audit 2024-09-13T14:16:33.327343+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:33 smithi177 bash[24494]: audit 2024-09-13T14:16:33.334778+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:33 smithi177 bash[24494]: audit 2024-09-13T14:16:33.345525+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:34.091 DEBUG:teuthology.orchestra.run.smithi099:osd.3> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T14:16:34.133 INFO:tasks.cephadm:Deploying osd.4 on smithi177 with /dev/vg_nvme/lv_4... 2024-09-13T14:16:34.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T14:16:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:34 smithi177 bash[24494]: cluster 2024-09-13T14:16:32.786664+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:34 smithi177 bash[24494]: cluster 2024-09-13T14:16:33.392648+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2024-09-13T14:16:34.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:34 smithi099 bash[22656]: cluster 2024-09-13T14:16:32.786664+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:34.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:34 smithi099 bash[22656]: cluster 2024-09-13T14:16:33.392648+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2024-09-13T14:16:34.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:34 smithi099 bash[32082]: cluster 2024-09-13T14:16:32.786664+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T14:16:34.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:34 smithi099 bash[32082]: cluster 2024-09-13T14:16:33.392648+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2024-09-13T14:16:36.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:36 smithi099 bash[22656]: cluster 2024-09-13T14:16:34.787122+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:36.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:36 smithi099 bash[32082]: cluster 2024-09-13T14:16:34.787122+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:36.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:36 smithi177 bash[24494]: cluster 2024-09-13T14:16:34.787122+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:38.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:38 smithi099 bash[32082]: cluster 2024-09-13T14:16:36.787678+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:38.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:38 smithi099 bash[22656]: cluster 2024-09-13T14:16:36.787678+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:38.844 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:16:38.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:38 smithi177 bash[24494]: cluster 2024-09-13T14:16:36.787678+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:40.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:40 smithi177 bash[24494]: cluster 2024-09-13T14:16:38.788208+0000 mgr.y (mgr.14150) 205 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:40.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:40 smithi099 bash[32082]: cluster 2024-09-13T14:16:38.788208+0000 mgr.y (mgr.14150) 205 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:40.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:40 smithi099 bash[22656]: cluster 2024-09-13T14:16:38.788208+0000 mgr.y (mgr.14150) 205 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:42.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:42 smithi177 bash[24494]: cluster 2024-09-13T14:16:40.788755+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:42.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:42 smithi099 bash[22656]: cluster 2024-09-13T14:16:40.788755+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:42.728 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:42 smithi099 bash[32082]: cluster 2024-09-13T14:16:40.788755+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:42.813 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-09-13T14:16:42.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi177:vg_nvme/lv_4 2024-09-13T14:16:44.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: cluster 2024-09-13T14:16:42.789223+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:44.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: audit 2024-09-13T14:16:43.549731+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: audit 2024-09-13T14:16:43.559223+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: audit 2024-09-13T14:16:43.561034+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: audit 2024-09-13T14:16:43.562631+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: audit 2024-09-13T14:16:43.564193+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: audit 2024-09-13T14:16:43.565855+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: audit 2024-09-13T14:16:43.574947+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: audit 2024-09-13T14:16:43.577793+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:44.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: audit 2024-09-13T14:16:43.579601+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:44.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[22656]: audit 2024-09-13T14:16:43.589291+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: cluster 2024-09-13T14:16:42.789223+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:44.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: audit 2024-09-13T14:16:43.549731+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: audit 2024-09-13T14:16:43.559223+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: audit 2024-09-13T14:16:43.561034+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: audit 2024-09-13T14:16:43.562631+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: audit 2024-09-13T14:16:43.564193+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: audit 2024-09-13T14:16:43.565855+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: audit 2024-09-13T14:16:43.574947+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: audit 2024-09-13T14:16:43.577793+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:44.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: audit 2024-09-13T14:16:43.579601+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:44.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:44 smithi099 bash[32082]: audit 2024-09-13T14:16:43.589291+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: cluster 2024-09-13T14:16:42.789223+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: audit 2024-09-13T14:16:43.549731+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: audit 2024-09-13T14:16:43.559223+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: audit 2024-09-13T14:16:43.561034+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: audit 2024-09-13T14:16:43.562631+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: audit 2024-09-13T14:16:43.564193+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: audit 2024-09-13T14:16:43.565855+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: audit 2024-09-13T14:16:43.574947+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: audit 2024-09-13T14:16:43.577793+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: audit 2024-09-13T14:16:43.579601+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:44 smithi177 bash[24494]: audit 2024-09-13T14:16:43.589291+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:16:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:45 smithi099 bash[32082]: cephadm 2024-09-13T14:16:43.540442+0000 mgr.y (mgr.14150) 208 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:16:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:45 smithi099 bash[32082]: cephadm 2024-09-13T14:16:43.567136+0000 mgr.y (mgr.14150) 209 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 4060M 2024-09-13T14:16:45.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:45 smithi099 bash[22656]: cephadm 2024-09-13T14:16:43.540442+0000 mgr.y (mgr.14150) 208 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:16:45.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:45 smithi099 bash[22656]: cephadm 2024-09-13T14:16:43.567136+0000 mgr.y (mgr.14150) 209 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 4060M 2024-09-13T14:16:45.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:45 smithi177 bash[24494]: cephadm 2024-09-13T14:16:43.540442+0000 mgr.y (mgr.14150) 208 : cephadm [INF] Detected new or changed devices on smithi099 2024-09-13T14:16:45.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:45 smithi177 bash[24494]: cephadm 2024-09-13T14:16:43.567136+0000 mgr.y (mgr.14150) 209 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 4060M 2024-09-13T14:16:46.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:46 smithi099 bash[32082]: cluster 2024-09-13T14:16:44.789756+0000 mgr.y (mgr.14150) 210 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:46.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:46 smithi099 bash[22656]: cluster 2024-09-13T14:16:44.789756+0000 mgr.y (mgr.14150) 210 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:46.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:46 smithi177 bash[24494]: cluster 2024-09-13T14:16:44.789756+0000 mgr.y (mgr.14150) 210 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:47.565 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:16:48.691 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:48 smithi177 bash[24494]: cluster 2024-09-13T14:16:46.790429+0000 mgr.y (mgr.14150) 211 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:48 smithi099 bash[32082]: cluster 2024-09-13T14:16:46.790429+0000 mgr.y (mgr.14150) 211 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:48.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:48 smithi099 bash[22656]: cluster 2024-09-13T14:16:46.790429+0000 mgr.y (mgr.14150) 211 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:50 smithi099 bash[32082]: cluster 2024-09-13T14:16:48.790912+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:50.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:50 smithi099 bash[22656]: cluster 2024-09-13T14:16:48.790912+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:50.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:50 smithi177 bash[24494]: cluster 2024-09-13T14:16:48.790912+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:51.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:51 smithi099 bash[22656]: audit 2024-09-13T14:16:50.426955+0000 mgr.y (mgr.14150) 213 : audit [DBG] from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:51.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:51 smithi099 bash[22656]: audit 2024-09-13T14:16:50.438678+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:16:51.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:51 smithi099 bash[22656]: audit 2024-09-13T14:16:50.444936+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:16:51.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:51 smithi099 bash[22656]: audit 2024-09-13T14:16:50.446422+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:51.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:51 smithi099 bash[32082]: audit 2024-09-13T14:16:50.426955+0000 mgr.y (mgr.14150) 213 : audit [DBG] from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:51.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:51 smithi099 bash[32082]: audit 2024-09-13T14:16:50.438678+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:16:51.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:51 smithi099 bash[32082]: audit 2024-09-13T14:16:50.444936+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:16:51.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:51 smithi099 bash[32082]: audit 2024-09-13T14:16:50.446422+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:51.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:51 smithi177 bash[24494]: audit 2024-09-13T14:16:50.426955+0000 mgr.y (mgr.14150) 213 : audit [DBG] from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:51.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:51 smithi177 bash[24494]: audit 2024-09-13T14:16:50.438678+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:16:51.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:51 smithi177 bash[24494]: audit 2024-09-13T14:16:50.444936+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:16:51.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:51 smithi177 bash[24494]: audit 2024-09-13T14:16:50.446422+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:52.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:52 smithi099 bash[22656]: cluster 2024-09-13T14:16:50.791424+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:52.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:52 smithi099 bash[32082]: cluster 2024-09-13T14:16:50.791424+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:52.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:52 smithi177 bash[24494]: cluster 2024-09-13T14:16:50.791424+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:54.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:54 smithi099 bash[22656]: cluster 2024-09-13T14:16:52.791928+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:54.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:54 smithi099 bash[32082]: cluster 2024-09-13T14:16:52.791928+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:54.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:54 smithi177 bash[24494]: cluster 2024-09-13T14:16:52.791928+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:56.518 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:56 smithi177 bash[24494]: cluster 2024-09-13T14:16:54.792422+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:56.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:56 smithi099 bash[32082]: cluster 2024-09-13T14:16:54.792422+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:56.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:56 smithi099 bash[22656]: cluster 2024-09-13T14:16:54.792422+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:58.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:58 smithi099 bash[22656]: cluster 2024-09-13T14:16:56.792937+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:58.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:58 smithi099 bash[32082]: cluster 2024-09-13T14:16:56.792937+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:58.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:58 smithi177 bash[24494]: cluster 2024-09-13T14:16:56.792937+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:16:59.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[22656]: audit 2024-09-13T14:16:58.609636+0000 mon.a (mon.0) 450 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d56712b-a6fb-4089-9583-850beed92b37"}]: dispatch 2024-09-13T14:16:59.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[22656]: audit 2024-09-13T14:16:58.610018+0000 mon.b (mon.1) 10 : audit [INF] from='client.? 172.21.15.177:0/549369395' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d56712b-a6fb-4089-9583-850beed92b37"}]: dispatch 2024-09-13T14:16:59.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[22656]: audit 2024-09-13T14:16:58.618650+0000 mon.a (mon.0) 451 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d56712b-a6fb-4089-9583-850beed92b37"}]': finished 2024-09-13T14:16:59.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[22656]: cluster 2024-09-13T14:16:58.618766+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:16:59.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[22656]: audit 2024-09-13T14:16:58.619008+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:16:59.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[22656]: audit 2024-09-13T14:16:59.214023+0000 mon.b (mon.1) 11 : audit [DBG] from='client.? 172.21.15.177:0/12574166' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:59.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[32082]: audit 2024-09-13T14:16:58.609636+0000 mon.a (mon.0) 450 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d56712b-a6fb-4089-9583-850beed92b37"}]: dispatch 2024-09-13T14:16:59.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[32082]: audit 2024-09-13T14:16:58.610018+0000 mon.b (mon.1) 10 : audit [INF] from='client.? 172.21.15.177:0/549369395' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d56712b-a6fb-4089-9583-850beed92b37"}]: dispatch 2024-09-13T14:16:59.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[32082]: audit 2024-09-13T14:16:58.618650+0000 mon.a (mon.0) 451 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d56712b-a6fb-4089-9583-850beed92b37"}]': finished 2024-09-13T14:16:59.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[32082]: cluster 2024-09-13T14:16:58.618766+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:16:59.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[32082]: audit 2024-09-13T14:16:58.619008+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:16:59.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:16:59 smithi099 bash[32082]: audit 2024-09-13T14:16:59.214023+0000 mon.b (mon.1) 11 : audit [DBG] from='client.? 172.21.15.177:0/12574166' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:59.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:59 smithi177 bash[24494]: audit 2024-09-13T14:16:58.609636+0000 mon.a (mon.0) 450 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d56712b-a6fb-4089-9583-850beed92b37"}]: dispatch 2024-09-13T14:16:59.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:59 smithi177 bash[24494]: audit 2024-09-13T14:16:58.610018+0000 mon.b (mon.1) 10 : audit [INF] from='client.? 172.21.15.177:0/549369395' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d56712b-a6fb-4089-9583-850beed92b37"}]: dispatch 2024-09-13T14:16:59.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:59 smithi177 bash[24494]: audit 2024-09-13T14:16:58.618650+0000 mon.a (mon.0) 451 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d56712b-a6fb-4089-9583-850beed92b37"}]': finished 2024-09-13T14:16:59.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:59 smithi177 bash[24494]: cluster 2024-09-13T14:16:58.618766+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:16:59.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:59 smithi177 bash[24494]: audit 2024-09-13T14:16:58.619008+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:16:59.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:16:59 smithi177 bash[24494]: audit 2024-09-13T14:16:59.214023+0000 mon.b (mon.1) 11 : audit [DBG] from='client.? 172.21.15.177:0/12574166' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:00.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:00 smithi099 bash[22656]: cluster 2024-09-13T14:16:58.793420+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:00.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:00 smithi099 bash[32082]: cluster 2024-09-13T14:16:58.793420+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:00.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:00 smithi177 bash[24494]: cluster 2024-09-13T14:16:58.793420+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:02.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:02 smithi099 bash[22656]: cluster 2024-09-13T14:17:00.793957+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:02.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:02 smithi099 bash[32082]: cluster 2024-09-13T14:17:00.793957+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:02.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:02 smithi177 bash[24494]: cluster 2024-09-13T14:17:00.793957+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:04.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:04 smithi099 bash[22656]: cluster 2024-09-13T14:17:02.794566+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:04.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:04 smithi099 bash[32082]: cluster 2024-09-13T14:17:02.794566+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:04.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:04 smithi177 bash[24494]: cluster 2024-09-13T14:17:02.794566+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:06.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:06 smithi099 bash[22656]: cluster 2024-09-13T14:17:04.795202+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:06.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:06 smithi099 bash[32082]: cluster 2024-09-13T14:17:04.795202+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:06.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:06 smithi177 bash[24494]: cluster 2024-09-13T14:17:04.795202+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:08 smithi099 bash[22656]: cluster 2024-09-13T14:17:06.795760+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:08 smithi099 bash[32082]: cluster 2024-09-13T14:17:06.795760+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:08.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:08 smithi177 bash[24494]: cluster 2024-09-13T14:17:06.795760+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:10.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:10 smithi099 bash[22656]: cluster 2024-09-13T14:17:08.796218+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:10.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:10 smithi099 bash[32082]: cluster 2024-09-13T14:17:08.796218+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:10 smithi177 bash[24494]: cluster 2024-09-13T14:17:08.796218+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:11 smithi099 bash[22656]: audit 2024-09-13T14:17:10.787480+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:17:11.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:11 smithi099 bash[22656]: audit 2024-09-13T14:17:10.789182+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:11 smithi099 bash[32082]: audit 2024-09-13T14:17:10.787480+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:17:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:11 smithi099 bash[32082]: audit 2024-09-13T14:17:10.789182+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:11 smithi177 bash[24494]: audit 2024-09-13T14:17:10.787480+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:17:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:11 smithi177 bash[24494]: audit 2024-09-13T14:17:10.789182+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:12.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:12 smithi177 bash[24494]: cephadm 2024-09-13T14:17:10.790692+0000 mgr.y (mgr.14150) 224 : cephadm [INF] Deploying daemon osd.4 on smithi177 2024-09-13T14:17:12.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:12 smithi177 bash[24494]: cluster 2024-09-13T14:17:10.796751+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:12.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:12 smithi099 bash[32082]: cephadm 2024-09-13T14:17:10.790692+0000 mgr.y (mgr.14150) 224 : cephadm [INF] Deploying daemon osd.4 on smithi177 2024-09-13T14:17:12.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:12 smithi099 bash[32082]: cluster 2024-09-13T14:17:10.796751+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:12 smithi099 bash[22656]: cephadm 2024-09-13T14:17:10.790692+0000 mgr.y (mgr.14150) 224 : cephadm [INF] Deploying daemon osd.4 on smithi177 2024-09-13T14:17:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:12 smithi099 bash[22656]: cluster 2024-09-13T14:17:10.796751+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:14.746 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:14 smithi177 bash[24494]: cluster 2024-09-13T14:17:12.797243+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:14 smithi099 bash[32082]: cluster 2024-09-13T14:17:12.797243+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:14.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:14 smithi099 bash[22656]: cluster 2024-09-13T14:17:12.797243+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:15.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:14 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:17:15.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:14 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:17:15.083 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:17:14 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:17:15.084 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:17:14 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:17:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:15 smithi099 bash[32082]: audit 2024-09-13T14:17:15.119017+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:15 smithi099 bash[32082]: audit 2024-09-13T14:17:15.128504+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:15 smithi099 bash[32082]: audit 2024-09-13T14:17:15.136930+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:15.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:15 smithi099 bash[22656]: audit 2024-09-13T14:17:15.119017+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:15.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:15 smithi099 bash[22656]: audit 2024-09-13T14:17:15.128504+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:15.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:15 smithi099 bash[22656]: audit 2024-09-13T14:17:15.136930+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:15.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:15 smithi177 bash[24494]: audit 2024-09-13T14:17:15.119017+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:15.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:15 smithi177 bash[24494]: audit 2024-09-13T14:17:15.128504+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:15.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:15 smithi177 bash[24494]: audit 2024-09-13T14:17:15.136930+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:16 smithi099 bash[32082]: cluster 2024-09-13T14:17:14.797707+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:16.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:16 smithi099 bash[22656]: cluster 2024-09-13T14:17:14.797707+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:16.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:16 smithi177 bash[24494]: cluster 2024-09-13T14:17:14.797707+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:18 smithi099 bash[32082]: cluster 2024-09-13T14:17:16.798262+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:18 smithi099 bash[22656]: cluster 2024-09-13T14:17:16.798262+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:18.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:18 smithi177 bash[24494]: cluster 2024-09-13T14:17:16.798262+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:20 smithi099 bash[32082]: cluster 2024-09-13T14:17:18.798699+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:20 smithi099 bash[32082]: audit 2024-09-13T14:17:20.143056+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:20 smithi099 bash[32082]: audit 2024-09-13T14:17:20.151584+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:20.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:20 smithi099 bash[22656]: cluster 2024-09-13T14:17:18.798699+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:20.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:20 smithi099 bash[22656]: audit 2024-09-13T14:17:20.143056+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:20.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:20 smithi099 bash[22656]: audit 2024-09-13T14:17:20.151584+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:20.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:20 smithi177 bash[24494]: cluster 2024-09-13T14:17:18.798699+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:20.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:20 smithi177 bash[24494]: audit 2024-09-13T14:17:20.143056+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:20.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:20 smithi177 bash[24494]: audit 2024-09-13T14:17:20.151584+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:21.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:21 smithi177 bash[24494]: audit 2024-09-13T14:17:20.582960+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:21.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:21 smithi177 bash[24494]: audit 2024-09-13T14:17:20.584284+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:21.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:21 smithi177 bash[24494]: audit 2024-09-13T14:17:20.592312+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:21.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:21 smithi177 bash[24494]: audit 2024-09-13T14:17:21.501194+0000 mon.a (mon.0) 464 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:21.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:21 smithi177 bash[24494]: audit 2024-09-13T14:17:21.501796+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2146549575,v1:172.21.15.177:6801/2146549575]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:21 smithi099 bash[32082]: audit 2024-09-13T14:17:20.582960+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:21 smithi099 bash[32082]: audit 2024-09-13T14:17:20.584284+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:21 smithi099 bash[32082]: audit 2024-09-13T14:17:20.592312+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:21 smithi099 bash[32082]: audit 2024-09-13T14:17:21.501194+0000 mon.a (mon.0) 464 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:21 smithi099 bash[32082]: audit 2024-09-13T14:17:21.501796+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2146549575,v1:172.21.15.177:6801/2146549575]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:21 smithi099 bash[22656]: audit 2024-09-13T14:17:20.582960+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:21 smithi099 bash[22656]: audit 2024-09-13T14:17:20.584284+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:21 smithi099 bash[22656]: audit 2024-09-13T14:17:20.592312+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:21 smithi099 bash[22656]: audit 2024-09-13T14:17:21.501194+0000 mon.a (mon.0) 464 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:21 smithi099 bash[22656]: audit 2024-09-13T14:17:21.501796+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2146549575,v1:172.21.15.177:6801/2146549575]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[22656]: cluster 2024-09-13T14:17:20.799289+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[22656]: audit 2024-09-13T14:17:21.600875+0000 mon.a (mon.0) 465 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:17:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[22656]: cluster 2024-09-13T14:17:21.601059+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:17:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[22656]: audit 2024-09-13T14:17:21.601349+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:22.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[22656]: audit 2024-09-13T14:17:21.602968+0000 mon.a (mon.0) 468 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:17:22.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[22656]: audit 2024-09-13T14:17:21.603443+0000 mon.b (mon.1) 13 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2146549575,v1:172.21.15.177:6801/2146549575]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:17:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[32082]: cluster 2024-09-13T14:17:20.799289+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[32082]: audit 2024-09-13T14:17:21.600875+0000 mon.a (mon.0) 465 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:17:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[32082]: cluster 2024-09-13T14:17:21.601059+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:17:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[32082]: audit 2024-09-13T14:17:21.601349+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[32082]: audit 2024-09-13T14:17:21.602968+0000 mon.a (mon.0) 468 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:17:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:22 smithi099 bash[32082]: audit 2024-09-13T14:17:21.603443+0000 mon.b (mon.1) 13 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2146549575,v1:172.21.15.177:6801/2146549575]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:17:22.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:22 smithi177 bash[24494]: cluster 2024-09-13T14:17:20.799289+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:22.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:22 smithi177 bash[24494]: audit 2024-09-13T14:17:21.600875+0000 mon.a (mon.0) 465 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:17:22.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:22 smithi177 bash[24494]: cluster 2024-09-13T14:17:21.601059+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:17:22.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:22 smithi177 bash[24494]: audit 2024-09-13T14:17:21.601349+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:22.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:22 smithi177 bash[24494]: audit 2024-09-13T14:17:21.602968+0000 mon.a (mon.0) 468 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:17:22.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:22 smithi177 bash[24494]: audit 2024-09-13T14:17:21.603443+0000 mon.b (mon.1) 13 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2146549575,v1:172.21.15.177:6801/2146549575]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:17:23.826 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 4 on host 'smithi177' 2024-09-13T14:17:23.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:23 smithi177 bash[24494]: audit 2024-09-13T14:17:22.604373+0000 mon.a (mon.0) 469 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:17:23.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:23 smithi177 bash[24494]: cluster 2024-09-13T14:17:22.604561+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2024-09-13T14:17:23.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:23 smithi177 bash[24494]: audit 2024-09-13T14:17:22.604898+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:23.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:23 smithi177 bash[24494]: audit 2024-09-13T14:17:22.613445+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:23.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:23 smithi177 bash[24494]: cluster 2024-09-13T14:17:22.799895+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:23.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:23 smithi177 bash[24494]: cluster 2024-09-13T14:17:23.000232+0000 mon.a (mon.0) 473 : cluster [INF] osd.4 [v2:172.21.15.177:6800/2146549575,v1:172.21.15.177:6801/2146549575] boot 2024-09-13T14:17:23.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:23 smithi177 bash[24494]: cluster 2024-09-13T14:17:23.000360+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:17:23.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:23 smithi177 bash[24494]: audit 2024-09-13T14:17:23.000724+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:24.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[22656]: audit 2024-09-13T14:17:22.604373+0000 mon.a (mon.0) 469 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:17:24.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[22656]: cluster 2024-09-13T14:17:22.604561+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2024-09-13T14:17:24.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[22656]: audit 2024-09-13T14:17:22.604898+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:24.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[22656]: audit 2024-09-13T14:17:22.613445+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:24.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[22656]: cluster 2024-09-13T14:17:22.799895+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[22656]: cluster 2024-09-13T14:17:23.000232+0000 mon.a (mon.0) 473 : cluster [INF] osd.4 [v2:172.21.15.177:6800/2146549575,v1:172.21.15.177:6801/2146549575] boot 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[22656]: cluster 2024-09-13T14:17:23.000360+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[22656]: audit 2024-09-13T14:17:23.000724+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[32082]: audit 2024-09-13T14:17:22.604373+0000 mon.a (mon.0) 469 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[32082]: cluster 2024-09-13T14:17:22.604561+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[32082]: audit 2024-09-13T14:17:22.604898+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[32082]: audit 2024-09-13T14:17:22.613445+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[32082]: cluster 2024-09-13T14:17:22.799895+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[32082]: cluster 2024-09-13T14:17:23.000232+0000 mon.a (mon.0) 473 : cluster [INF] osd.4 [v2:172.21.15.177:6800/2146549575,v1:172.21.15.177:6801/2146549575] boot 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[32082]: cluster 2024-09-13T14:17:23.000360+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:17:24.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:23 smithi099 bash[32082]: audit 2024-09-13T14:17:23.000724+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:24.476 DEBUG:teuthology.orchestra.run.smithi177:osd.4> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T14:17:24.479 INFO:tasks.cephadm:Deploying osd.5 on smithi177 with /dev/vg_nvme/lv_3... 2024-09-13T14:17:24.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T14:17:24.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:24 smithi177 bash[24494]: cluster 2024-09-13T14:17:22.513498+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:17:24.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:24 smithi177 bash[24494]: cluster 2024-09-13T14:17:22.513608+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:17:24.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:24 smithi177 bash[24494]: audit 2024-09-13T14:17:23.808392+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:24.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:24 smithi177 bash[24494]: audit 2024-09-13T14:17:23.815903+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:24.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:24 smithi177 bash[24494]: audit 2024-09-13T14:17:23.823846+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:24.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:24 smithi177 bash[24494]: cluster 2024-09-13T14:17:24.002998+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:17:25.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[22656]: cluster 2024-09-13T14:17:22.513498+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:17:25.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[22656]: cluster 2024-09-13T14:17:22.513608+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:17:25.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[22656]: audit 2024-09-13T14:17:23.808392+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[22656]: audit 2024-09-13T14:17:23.815903+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[22656]: audit 2024-09-13T14:17:23.823846+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[22656]: cluster 2024-09-13T14:17:24.002998+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:17:25.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[32082]: cluster 2024-09-13T14:17:22.513498+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:17:25.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[32082]: cluster 2024-09-13T14:17:22.513608+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:17:25.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[32082]: audit 2024-09-13T14:17:23.808392+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:25.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[32082]: audit 2024-09-13T14:17:23.815903+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:25.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[32082]: audit 2024-09-13T14:17:23.823846+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:25.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:24 smithi099 bash[32082]: cluster 2024-09-13T14:17:24.002998+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:17:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:25 smithi099 bash[22656]: cluster 2024-09-13T14:17:24.800419+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:25 smithi099 bash[22656]: cluster 2024-09-13T14:17:25.003308+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:17:26.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:26 smithi099 bash[32082]: cluster 2024-09-13T14:17:24.800419+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:26.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:26 smithi099 bash[32082]: cluster 2024-09-13T14:17:25.003308+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:17:26.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:26 smithi177 bash[24494]: cluster 2024-09-13T14:17:24.800419+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:26.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:26 smithi177 bash[24494]: cluster 2024-09-13T14:17:25.003308+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:17:28.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:28 smithi099 bash[22656]: cluster 2024-09-13T14:17:26.801021+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-09-13T14:17:28.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:28 smithi099 bash[32082]: cluster 2024-09-13T14:17:26.801021+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-09-13T14:17:28.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:28 smithi177 bash[24494]: cluster 2024-09-13T14:17:26.801021+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 137 KiB/s, 0 objects/s recovering 2024-09-13T14:17:28.576 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:17:30.294 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:30 smithi177 bash[24494]: cluster 2024-09-13T14:17:28.801513+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-13T14:17:30.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:30 smithi099 bash[22656]: cluster 2024-09-13T14:17:28.801513+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-13T14:17:30.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:30 smithi099 bash[32082]: cluster 2024-09-13T14:17:28.801513+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-13T14:17:32.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:32 smithi099 bash[22656]: cluster 2024-09-13T14:17:30.802070+0000 mgr.y (mgr.14150) 235 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T14:17:32.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:32 smithi099 bash[32082]: cluster 2024-09-13T14:17:30.802070+0000 mgr.y (mgr.14150) 235 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T14:17:32.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:32 smithi177 bash[24494]: cluster 2024-09-13T14:17:30.802070+0000 mgr.y (mgr.14150) 235 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T14:17:33.299 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-09-13T14:17:33.325 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi177:vg_nvme/lv_3 2024-09-13T14:17:34.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:34 smithi099 bash[22656]: cluster 2024-09-13T14:17:32.802643+0000 mgr.y (mgr.14150) 236 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T14:17:34.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:34 smithi099 bash[32082]: cluster 2024-09-13T14:17:32.802643+0000 mgr.y (mgr.14150) 236 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T14:17:34.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:34 smithi177 bash[24494]: cluster 2024-09-13T14:17:32.802643+0000 mgr.y (mgr.14150) 236 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T14:17:36.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:36 smithi099 bash[22656]: cluster 2024-09-13T14:17:34.803261+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:17:36.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:36 smithi099 bash[32082]: cluster 2024-09-13T14:17:34.803261+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:17:36.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:36 smithi177 bash[24494]: cluster 2024-09-13T14:17:34.803261+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:17:37.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[22656]: cephadm 2024-09-13T14:17:36.174759+0000 mgr.y (mgr.14150) 238 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:17:37.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[22656]: audit 2024-09-13T14:17:36.184266+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[22656]: audit 2024-09-13T14:17:36.192670+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[22656]: audit 2024-09-13T14:17:36.194718+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:37.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[22656]: cephadm 2024-09-13T14:17:36.196251+0000 mgr.y (mgr.14150) 239 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 17265M 2024-09-13T14:17:37.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[22656]: audit 2024-09-13T14:17:36.204818+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[22656]: audit 2024-09-13T14:17:36.207192+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:37.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[22656]: audit 2024-09-13T14:17:36.209047+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:37.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[22656]: audit 2024-09-13T14:17:36.221035+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[32082]: cephadm 2024-09-13T14:17:36.174759+0000 mgr.y (mgr.14150) 238 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:17:37.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[32082]: audit 2024-09-13T14:17:36.184266+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[32082]: audit 2024-09-13T14:17:36.192670+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[32082]: audit 2024-09-13T14:17:36.194718+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:37.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[32082]: cephadm 2024-09-13T14:17:36.196251+0000 mgr.y (mgr.14150) 239 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 17265M 2024-09-13T14:17:37.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[32082]: audit 2024-09-13T14:17:36.204818+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[32082]: audit 2024-09-13T14:17:36.207192+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:37.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[32082]: audit 2024-09-13T14:17:36.209047+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:37.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:37 smithi099 bash[32082]: audit 2024-09-13T14:17:36.221035+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:37 smithi177 bash[24494]: cephadm 2024-09-13T14:17:36.174759+0000 mgr.y (mgr.14150) 238 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:17:37.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:37 smithi177 bash[24494]: audit 2024-09-13T14:17:36.184266+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:37 smithi177 bash[24494]: audit 2024-09-13T14:17:36.192670+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:37 smithi177 bash[24494]: audit 2024-09-13T14:17:36.194718+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:37.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:37 smithi177 bash[24494]: cephadm 2024-09-13T14:17:36.196251+0000 mgr.y (mgr.14150) 239 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 17265M 2024-09-13T14:17:37.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:37 smithi177 bash[24494]: audit 2024-09-13T14:17:36.204818+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:37.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:37 smithi177 bash[24494]: audit 2024-09-13T14:17:36.207192+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:37.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:37 smithi177 bash[24494]: audit 2024-09-13T14:17:36.209047+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:37.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:37 smithi177 bash[24494]: audit 2024-09-13T14:17:36.221035+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:17:38.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:38 smithi099 bash[22656]: cluster 2024-09-13T14:17:36.803912+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-13T14:17:38.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:38 smithi099 bash[32082]: cluster 2024-09-13T14:17:36.803912+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-13T14:17:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:38 smithi177 bash[24494]: cluster 2024-09-13T14:17:36.803912+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-09-13T14:17:39.122 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:17:40.461 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:40 smithi177 bash[24494]: cluster 2024-09-13T14:17:38.804455+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:40.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:40 smithi099 bash[22656]: cluster 2024-09-13T14:17:38.804455+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:40.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:40 smithi099 bash[32082]: cluster 2024-09-13T14:17:38.804455+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:42.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:42 smithi099 bash[22656]: cluster 2024-09-13T14:17:40.804992+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:42.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:42 smithi099 bash[22656]: audit 2024-09-13T14:17:42.176051+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:42.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:42 smithi099 bash[22656]: audit 2024-09-13T14:17:42.181871+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:42.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:42 smithi099 bash[22656]: audit 2024-09-13T14:17:42.183654+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:42.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:42 smithi099 bash[32082]: cluster 2024-09-13T14:17:40.804992+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:42.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:42 smithi099 bash[32082]: audit 2024-09-13T14:17:42.176051+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:42.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:42 smithi099 bash[32082]: audit 2024-09-13T14:17:42.181871+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:42.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:42 smithi099 bash[32082]: audit 2024-09-13T14:17:42.183654+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:42.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:42 smithi177 bash[24494]: cluster 2024-09-13T14:17:40.804992+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:42.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:42 smithi177 bash[24494]: audit 2024-09-13T14:17:42.176051+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:42.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:42 smithi177 bash[24494]: audit 2024-09-13T14:17:42.181871+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:42.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:42 smithi177 bash[24494]: audit 2024-09-13T14:17:42.183654+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:43.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:43 smithi099 bash[32082]: audit 2024-09-13T14:17:42.169852+0000 mgr.y (mgr.14150) 243 : audit [DBG] from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:43 smithi099 bash[22656]: audit 2024-09-13T14:17:42.169852+0000 mgr.y (mgr.14150) 243 : audit [DBG] from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:43.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:43 smithi177 bash[24494]: audit 2024-09-13T14:17:42.169852+0000 mgr.y (mgr.14150) 243 : audit [DBG] from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:44.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:44 smithi099 bash[32082]: cluster 2024-09-13T14:17:42.805508+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:44.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:44 smithi099 bash[22656]: cluster 2024-09-13T14:17:42.805508+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:44.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:44 smithi177 bash[24494]: cluster 2024-09-13T14:17:42.805508+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:46 smithi099 bash[32082]: cluster 2024-09-13T14:17:44.806038+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:46.630 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:46 smithi099 bash[22656]: cluster 2024-09-13T14:17:44.806038+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:46.630 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:46 smithi177 bash[24494]: cluster 2024-09-13T14:17:44.806038+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:48.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:48 smithi099 bash[32082]: cluster 2024-09-13T14:17:46.806568+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:48 smithi099 bash[22656]: cluster 2024-09-13T14:17:46.806568+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:48.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:48 smithi177 bash[24494]: cluster 2024-09-13T14:17:46.806568+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:50.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:50 smithi099 bash[32082]: cluster 2024-09-13T14:17:48.807108+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:50.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:50 smithi099 bash[22656]: cluster 2024-09-13T14:17:48.807108+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:50.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:50 smithi177 bash[24494]: cluster 2024-09-13T14:17:48.807108+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[22656]: audit 2024-09-13T14:17:50.305665+0000 mon.a (mon.0) 491 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "414b880c-6534-495f-a54c-f8d28ba81c73"}]: dispatch 2024-09-13T14:17:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[22656]: audit 2024-09-13T14:17:50.306220+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.177:0/2548253440' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "414b880c-6534-495f-a54c-f8d28ba81c73"}]: dispatch 2024-09-13T14:17:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[22656]: audit 2024-09-13T14:17:50.314875+0000 mon.a (mon.0) 492 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "414b880c-6534-495f-a54c-f8d28ba81c73"}]': finished 2024-09-13T14:17:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[22656]: cluster 2024-09-13T14:17:50.315065+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:17:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[22656]: audit 2024-09-13T14:17:50.315338+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:17:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[22656]: audit 2024-09-13T14:17:50.934883+0000 mon.b (mon.1) 15 : audit [DBG] from='client.? 172.21.15.177:0/1826662478' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[32082]: audit 2024-09-13T14:17:50.305665+0000 mon.a (mon.0) 491 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "414b880c-6534-495f-a54c-f8d28ba81c73"}]: dispatch 2024-09-13T14:17:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[32082]: audit 2024-09-13T14:17:50.306220+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.177:0/2548253440' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "414b880c-6534-495f-a54c-f8d28ba81c73"}]: dispatch 2024-09-13T14:17:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[32082]: audit 2024-09-13T14:17:50.314875+0000 mon.a (mon.0) 492 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "414b880c-6534-495f-a54c-f8d28ba81c73"}]': finished 2024-09-13T14:17:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[32082]: cluster 2024-09-13T14:17:50.315065+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:17:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[32082]: audit 2024-09-13T14:17:50.315338+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:17:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:51 smithi099 bash[32082]: audit 2024-09-13T14:17:50.934883+0000 mon.b (mon.1) 15 : audit [DBG] from='client.? 172.21.15.177:0/1826662478' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:51 smithi177 bash[24494]: audit 2024-09-13T14:17:50.305665+0000 mon.a (mon.0) 491 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "414b880c-6534-495f-a54c-f8d28ba81c73"}]: dispatch 2024-09-13T14:17:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:51 smithi177 bash[24494]: audit 2024-09-13T14:17:50.306220+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.177:0/2548253440' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "414b880c-6534-495f-a54c-f8d28ba81c73"}]: dispatch 2024-09-13T14:17:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:51 smithi177 bash[24494]: audit 2024-09-13T14:17:50.314875+0000 mon.a (mon.0) 492 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "414b880c-6534-495f-a54c-f8d28ba81c73"}]': finished 2024-09-13T14:17:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:51 smithi177 bash[24494]: cluster 2024-09-13T14:17:50.315065+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-13T14:17:51.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:51 smithi177 bash[24494]: audit 2024-09-13T14:17:50.315338+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:17:51.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:51 smithi177 bash[24494]: audit 2024-09-13T14:17:50.934883+0000 mon.b (mon.1) 15 : audit [DBG] from='client.? 172.21.15.177:0/1826662478' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:52.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:52 smithi099 bash[22656]: cluster 2024-09-13T14:17:50.807638+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:52 smithi099 bash[32082]: cluster 2024-09-13T14:17:50.807638+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:52.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:52 smithi177 bash[24494]: cluster 2024-09-13T14:17:50.807638+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:54 smithi099 bash[22656]: cluster 2024-09-13T14:17:52.808157+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:54 smithi099 bash[32082]: cluster 2024-09-13T14:17:52.808157+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:54 smithi177 bash[24494]: cluster 2024-09-13T14:17:52.808157+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:56.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:56 smithi099 bash[22656]: cluster 2024-09-13T14:17:54.808670+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:56.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:56 smithi099 bash[32082]: cluster 2024-09-13T14:17:54.808670+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:56.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:56 smithi177 bash[24494]: cluster 2024-09-13T14:17:54.808670+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:58.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:17:58 smithi177 bash[24494]: cluster 2024-09-13T14:17:56.809207+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:58.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:17:58 smithi099 bash[22656]: cluster 2024-09-13T14:17:56.809207+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:17:58.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:17:58 smithi099 bash[32082]: cluster 2024-09-13T14:17:56.809207+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:00.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:00 smithi177 bash[24494]: cluster 2024-09-13T14:17:58.809725+0000 mgr.y (mgr.14150) 252 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:00.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:00 smithi099 bash[22656]: cluster 2024-09-13T14:17:58.809725+0000 mgr.y (mgr.14150) 252 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:00.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:00 smithi099 bash[32082]: cluster 2024-09-13T14:17:58.809725+0000 mgr.y (mgr.14150) 252 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:02.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:02 smithi177 bash[24494]: cluster 2024-09-13T14:18:00.810249+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:02.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:02 smithi099 bash[32082]: cluster 2024-09-13T14:18:00.810249+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:02.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:02 smithi099 bash[22656]: cluster 2024-09-13T14:18:00.810249+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:03.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:03 smithi177 bash[24494]: audit 2024-09-13T14:18:02.678264+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:18:03.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:03 smithi177 bash[24494]: audit 2024-09-13T14:18:02.680119+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:03 smithi099 bash[22656]: audit 2024-09-13T14:18:02.678264+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:18:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:03 smithi099 bash[22656]: audit 2024-09-13T14:18:02.680119+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:03 smithi099 bash[32082]: audit 2024-09-13T14:18:02.678264+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:18:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:03 smithi099 bash[32082]: audit 2024-09-13T14:18:02.680119+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:04.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:04 smithi177 bash[24494]: cephadm 2024-09-13T14:18:02.681619+0000 mgr.y (mgr.14150) 254 : cephadm [INF] Deploying daemon osd.5 on smithi177 2024-09-13T14:18:04.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:04 smithi177 bash[24494]: cluster 2024-09-13T14:18:02.810702+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:04.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:04 smithi099 bash[22656]: cephadm 2024-09-13T14:18:02.681619+0000 mgr.y (mgr.14150) 254 : cephadm [INF] Deploying daemon osd.5 on smithi177 2024-09-13T14:18:04.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:04 smithi099 bash[22656]: cluster 2024-09-13T14:18:02.810702+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:04.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:04 smithi099 bash[32082]: cephadm 2024-09-13T14:18:02.681619+0000 mgr.y (mgr.14150) 254 : cephadm [INF] Deploying daemon osd.5 on smithi177 2024-09-13T14:18:04.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:04 smithi099 bash[32082]: cluster 2024-09-13T14:18:02.810702+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:06.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:06 smithi177 bash[24494]: cluster 2024-09-13T14:18:04.811225+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:06.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:06 smithi099 bash[22656]: cluster 2024-09-13T14:18:04.811225+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:06.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:06 smithi099 bash[32082]: cluster 2024-09-13T14:18:04.811225+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:07.101 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:06 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:18:07.102 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:18:06 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:18:07.102 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:18:06 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:18:07.352 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:18:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:18:07.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:18:07.353 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:18:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:18:07.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:07 smithi177 bash[24494]: audit 2024-09-13T14:18:07.242143+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:07.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:07 smithi177 bash[24494]: audit 2024-09-13T14:18:07.249962+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:07.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:07 smithi177 bash[24494]: audit 2024-09-13T14:18:07.258115+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:07 smithi099 bash[22656]: audit 2024-09-13T14:18:07.242143+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:07 smithi099 bash[22656]: audit 2024-09-13T14:18:07.249962+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:07 smithi099 bash[22656]: audit 2024-09-13T14:18:07.258115+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:07 smithi099 bash[32082]: audit 2024-09-13T14:18:07.242143+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:07 smithi099 bash[32082]: audit 2024-09-13T14:18:07.249962+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:07 smithi099 bash[32082]: audit 2024-09-13T14:18:07.258115+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:08.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:08 smithi177 bash[24494]: cluster 2024-09-13T14:18:06.811759+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:08 smithi099 bash[22656]: cluster 2024-09-13T14:18:06.811759+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:08.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:08 smithi099 bash[32082]: cluster 2024-09-13T14:18:06.811759+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:10.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:10 smithi177 bash[24494]: cluster 2024-09-13T14:18:08.812277+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:10.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:10 smithi099 bash[22656]: cluster 2024-09-13T14:18:08.812277+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:10.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:10 smithi099 bash[32082]: cluster 2024-09-13T14:18:08.812277+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:12 smithi177 bash[24494]: cluster 2024-09-13T14:18:10.812816+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:12 smithi177 bash[24494]: audit 2024-09-13T14:18:12.303407+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:12 smithi177 bash[24494]: audit 2024-09-13T14:18:12.312086+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:12.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:12 smithi177 bash[24494]: audit 2024-09-13T14:18:12.313953+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:12.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:12 smithi177 bash[24494]: audit 2024-09-13T14:18:12.315640+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:12.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:12 smithi177 bash[24494]: audit 2024-09-13T14:18:12.325263+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:12.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[22656]: cluster 2024-09-13T14:18:10.812816+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:12.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[22656]: audit 2024-09-13T14:18:12.303407+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[22656]: audit 2024-09-13T14:18:12.312086+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[22656]: audit 2024-09-13T14:18:12.313953+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[22656]: audit 2024-09-13T14:18:12.315640+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[22656]: audit 2024-09-13T14:18:12.325263+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:12.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[32082]: cluster 2024-09-13T14:18:10.812816+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:12.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[32082]: audit 2024-09-13T14:18:12.303407+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:12.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[32082]: audit 2024-09-13T14:18:12.312086+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:12.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[32082]: audit 2024-09-13T14:18:12.313953+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:12.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[32082]: audit 2024-09-13T14:18:12.315640+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:12.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:12 smithi099 bash[32082]: audit 2024-09-13T14:18:12.325263+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:14.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:14 smithi099 bash[22656]: cluster 2024-09-13T14:18:12.813351+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:14.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:14 smithi099 bash[22656]: audit 2024-09-13T14:18:13.950739+0000 mon.a (mon.0) 505 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:14.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:14 smithi099 bash[22656]: audit 2024-09-13T14:18:13.951350+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:14 smithi099 bash[32082]: cluster 2024-09-13T14:18:12.813351+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:14 smithi099 bash[32082]: audit 2024-09-13T14:18:13.950739+0000 mon.a (mon.0) 505 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:14 smithi099 bash[32082]: audit 2024-09-13T14:18:13.951350+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:14.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:14 smithi177 bash[24494]: cluster 2024-09-13T14:18:12.813351+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:14.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:14 smithi177 bash[24494]: audit 2024-09-13T14:18:13.950739+0000 mon.a (mon.0) 505 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:14.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:14 smithi177 bash[24494]: audit 2024-09-13T14:18:13.951350+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:15.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:15 smithi177 bash[24494]: audit 2024-09-13T14:18:14.372884+0000 mon.a (mon.0) 506 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:18:15.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:15 smithi177 bash[24494]: cluster 2024-09-13T14:18:14.372998+0000 mon.a (mon.0) 507 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:18:15.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:15 smithi177 bash[24494]: audit 2024-09-13T14:18:14.373628+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:15.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:15 smithi177 bash[24494]: audit 2024-09-13T14:18:14.375700+0000 mon.b (mon.1) 17 : audit [INF] from='osd.5 [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:18:15.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:15 smithi177 bash[24494]: audit 2024-09-13T14:18:14.381217+0000 mon.a (mon.0) 509 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:18:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:15 smithi099 bash[22656]: audit 2024-09-13T14:18:14.372884+0000 mon.a (mon.0) 506 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:18:15.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:15 smithi099 bash[22656]: cluster 2024-09-13T14:18:14.372998+0000 mon.a (mon.0) 507 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:18:15.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:15 smithi099 bash[22656]: audit 2024-09-13T14:18:14.373628+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:15.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:15 smithi099 bash[22656]: audit 2024-09-13T14:18:14.375700+0000 mon.b (mon.1) 17 : audit [INF] from='osd.5 [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:18:15.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:15 smithi099 bash[22656]: audit 2024-09-13T14:18:14.381217+0000 mon.a (mon.0) 509 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:18:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:15 smithi099 bash[32082]: audit 2024-09-13T14:18:14.372884+0000 mon.a (mon.0) 506 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:18:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:15 smithi099 bash[32082]: cluster 2024-09-13T14:18:14.372998+0000 mon.a (mon.0) 507 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:18:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:15 smithi099 bash[32082]: audit 2024-09-13T14:18:14.373628+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:15 smithi099 bash[32082]: audit 2024-09-13T14:18:14.375700+0000 mon.b (mon.1) 17 : audit [INF] from='osd.5 [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:18:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:15 smithi099 bash[32082]: audit 2024-09-13T14:18:14.381217+0000 mon.a (mon.0) 509 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:18:16.047 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 5 on host 'smithi177' 2024-09-13T14:18:16.694 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:16 smithi177 bash[24494]: cluster 2024-09-13T14:18:14.814001+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:16.694 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:16 smithi177 bash[24494]: audit 2024-09-13T14:18:15.391837+0000 mon.a (mon.0) 510 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:18:16.694 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:16 smithi177 bash[24494]: cluster 2024-09-13T14:18:15.392101+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:18:16.695 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:16 smithi177 bash[24494]: audit 2024-09-13T14:18:15.392503+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:16.695 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:16 smithi177 bash[24494]: audit 2024-09-13T14:18:15.396611+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:16.695 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:16 smithi177 bash[24494]: audit 2024-09-13T14:18:16.028592+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:16.695 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:16 smithi177 bash[24494]: audit 2024-09-13T14:18:16.036603+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:16.695 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:16 smithi177 bash[24494]: audit 2024-09-13T14:18:16.045357+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:16.696 DEBUG:teuthology.orchestra.run.smithi177:osd.5> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T14:18:16.699 INFO:tasks.cephadm:Deploying osd.6 on smithi177 with /dev/vg_nvme/lv_2... 2024-09-13T14:18:16.699 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T14:18:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[22656]: cluster 2024-09-13T14:18:14.814001+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[22656]: audit 2024-09-13T14:18:15.391837+0000 mon.a (mon.0) 510 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:18:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[22656]: cluster 2024-09-13T14:18:15.392101+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[22656]: audit 2024-09-13T14:18:15.392503+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[22656]: audit 2024-09-13T14:18:15.396611+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[22656]: audit 2024-09-13T14:18:16.028592+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[22656]: audit 2024-09-13T14:18:16.036603+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[22656]: audit 2024-09-13T14:18:16.045357+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[32082]: cluster 2024-09-13T14:18:14.814001+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[32082]: audit 2024-09-13T14:18:15.391837+0000 mon.a (mon.0) 510 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[32082]: cluster 2024-09-13T14:18:15.392101+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[32082]: audit 2024-09-13T14:18:15.392503+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[32082]: audit 2024-09-13T14:18:15.396611+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[32082]: audit 2024-09-13T14:18:16.028592+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:16.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[32082]: audit 2024-09-13T14:18:16.036603+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:16.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:16 smithi099 bash[32082]: audit 2024-09-13T14:18:16.045357+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:17 smithi099 bash[22656]: cluster 2024-09-13T14:18:14.948637+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:18:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:17 smithi099 bash[22656]: cluster 2024-09-13T14:18:14.948716+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:18:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:17 smithi099 bash[22656]: cluster 2024-09-13T14:18:16.392708+0000 mon.a (mon.0) 517 : cluster [INF] osd.5 [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587] boot 2024-09-13T14:18:17.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:17 smithi099 bash[22656]: cluster 2024-09-13T14:18:16.392856+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:18:17.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:17 smithi099 bash[22656]: audit 2024-09-13T14:18:16.393586+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:17 smithi099 bash[32082]: cluster 2024-09-13T14:18:14.948637+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:18:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:17 smithi099 bash[32082]: cluster 2024-09-13T14:18:14.948716+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:18:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:17 smithi099 bash[32082]: cluster 2024-09-13T14:18:16.392708+0000 mon.a (mon.0) 517 : cluster [INF] osd.5 [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587] boot 2024-09-13T14:18:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:17 smithi099 bash[32082]: cluster 2024-09-13T14:18:16.392856+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:18:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:17 smithi099 bash[32082]: audit 2024-09-13T14:18:16.393586+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:17.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:17 smithi177 bash[24494]: cluster 2024-09-13T14:18:14.948637+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:18:17.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:17 smithi177 bash[24494]: cluster 2024-09-13T14:18:14.948716+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:18:17.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:17 smithi177 bash[24494]: cluster 2024-09-13T14:18:16.392708+0000 mon.a (mon.0) 517 : cluster [INF] osd.5 [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587] boot 2024-09-13T14:18:17.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:17 smithi177 bash[24494]: cluster 2024-09-13T14:18:16.392856+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-13T14:18:17.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:17 smithi177 bash[24494]: audit 2024-09-13T14:18:16.393586+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:18 smithi099 bash[22656]: cluster 2024-09-13T14:18:16.814511+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:18 smithi099 bash[22656]: cluster 2024-09-13T14:18:17.394857+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:18:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:18 smithi099 bash[32082]: cluster 2024-09-13T14:18:16.814511+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:18 smithi099 bash[32082]: cluster 2024-09-13T14:18:17.394857+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:18:18.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:18 smithi177 bash[24494]: cluster 2024-09-13T14:18:16.814511+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:18.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:18 smithi177 bash[24494]: cluster 2024-09-13T14:18:17.394857+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:18:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:19 smithi099 bash[22656]: cluster 2024-09-13T14:18:18.405139+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:18:19.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:19 smithi099 bash[32082]: cluster 2024-09-13T14:18:18.405139+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:18:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:19 smithi177 bash[24494]: cluster 2024-09-13T14:18:18.405139+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:18:20.804 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:18:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:20 smithi099 bash[22656]: cluster 2024-09-13T14:18:18.815011+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:20.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:20 smithi099 bash[32082]: cluster 2024-09-13T14:18:18.815011+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:20.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:20 smithi177 bash[24494]: cluster 2024-09-13T14:18:18.815011+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:22.671 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:22 smithi177 bash[24494]: cluster 2024-09-13T14:18:20.815523+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:22.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:22 smithi099 bash[22656]: cluster 2024-09-13T14:18:20.815523+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:22.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:22 smithi099 bash[32082]: cluster 2024-09-13T14:18:20.815523+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:24.817 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:24 smithi177 bash[24494]: cluster 2024-09-13T14:18:22.816094+0000 mgr.y (mgr.14150) 265 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:24.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:24 smithi099 bash[22656]: cluster 2024-09-13T14:18:22.816094+0000 mgr.y (mgr.14150) 265 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:24.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:24 smithi099 bash[32082]: cluster 2024-09-13T14:18:22.816094+0000 mgr.y (mgr.14150) 265 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:26.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:26 smithi099 bash[22656]: cluster 2024-09-13T14:18:24.816664+0000 mgr.y (mgr.14150) 266 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:26.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:26 smithi099 bash[32082]: cluster 2024-09-13T14:18:24.816664+0000 mgr.y (mgr.14150) 266 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:26.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:26 smithi177 bash[24494]: cluster 2024-09-13T14:18:24.816664+0000 mgr.y (mgr.14150) 266 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[22656]: cephadm 2024-09-13T14:18:26.523108+0000 mgr.y (mgr.14150) 267 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:18:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[22656]: audit 2024-09-13T14:18:26.533365+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[22656]: audit 2024-09-13T14:18:26.543683+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[22656]: audit 2024-09-13T14:18:26.545645+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[22656]: audit 2024-09-13T14:18:26.547606+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[22656]: audit 2024-09-13T14:18:26.559123+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[22656]: audit 2024-09-13T14:18:26.561296+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[22656]: audit 2024-09-13T14:18:26.563078+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[22656]: audit 2024-09-13T14:18:26.572809+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[32082]: cephadm 2024-09-13T14:18:26.523108+0000 mgr.y (mgr.14150) 267 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[32082]: audit 2024-09-13T14:18:26.533365+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[32082]: audit 2024-09-13T14:18:26.543683+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[32082]: audit 2024-09-13T14:18:26.545645+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:27.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[32082]: audit 2024-09-13T14:18:26.547606+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:27.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[32082]: audit 2024-09-13T14:18:26.559123+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[32082]: audit 2024-09-13T14:18:26.561296+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:27.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[32082]: audit 2024-09-13T14:18:26.563078+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:27.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:27 smithi099 bash[32082]: audit 2024-09-13T14:18:26.572809+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:27 smithi177 bash[24494]: cephadm 2024-09-13T14:18:26.523108+0000 mgr.y (mgr.14150) 267 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:18:27.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:27 smithi177 bash[24494]: audit 2024-09-13T14:18:26.533365+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:27 smithi177 bash[24494]: audit 2024-09-13T14:18:26.543683+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:27 smithi177 bash[24494]: audit 2024-09-13T14:18:26.545645+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:27.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:27 smithi177 bash[24494]: audit 2024-09-13T14:18:26.547606+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:27.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:27 smithi177 bash[24494]: audit 2024-09-13T14:18:26.559123+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:27.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:27 smithi177 bash[24494]: audit 2024-09-13T14:18:26.561296+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:27.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:27 smithi177 bash[24494]: audit 2024-09-13T14:18:26.563078+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:27.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:27 smithi177 bash[24494]: audit 2024-09-13T14:18:26.572809+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:18:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:28 smithi099 bash[22656]: cephadm 2024-09-13T14:18:26.548870+0000 mgr.y (mgr.14150) 268 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 8632M 2024-09-13T14:18:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:28 smithi099 bash[22656]: cluster 2024-09-13T14:18:26.817239+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:28.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:28 smithi099 bash[32082]: cephadm 2024-09-13T14:18:26.548870+0000 mgr.y (mgr.14150) 268 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 8632M 2024-09-13T14:18:28.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:28 smithi099 bash[32082]: cluster 2024-09-13T14:18:26.817239+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:28.825 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-09-13T14:18:28.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:28 smithi177 bash[24494]: cephadm 2024-09-13T14:18:26.548870+0000 mgr.y (mgr.14150) 268 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 8632M 2024-09-13T14:18:28.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:28 smithi177 bash[24494]: cluster 2024-09-13T14:18:26.817239+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:28.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi177:vg_nvme/lv_2 2024-09-13T14:18:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:30 smithi099 bash[22656]: cluster 2024-09-13T14:18:28.817826+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:30 smithi099 bash[32082]: cluster 2024-09-13T14:18:28.817826+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:30.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:30 smithi177 bash[24494]: cluster 2024-09-13T14:18:28.817826+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:32.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:32 smithi099 bash[22656]: cluster 2024-09-13T14:18:30.818448+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:32.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:32 smithi099 bash[32082]: cluster 2024-09-13T14:18:30.818448+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:32.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:32 smithi177 bash[24494]: cluster 2024-09-13T14:18:30.818448+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:33.588 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:18:34.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:34 smithi177 bash[24494]: cluster 2024-09-13T14:18:32.819121+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:34.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:34 smithi099 bash[32082]: cluster 2024-09-13T14:18:32.819121+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:34.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:34 smithi099 bash[22656]: cluster 2024-09-13T14:18:32.819121+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:36.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:36 smithi099 bash[22656]: cluster 2024-09-13T14:18:34.819688+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:36.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:36 smithi099 bash[32082]: cluster 2024-09-13T14:18:34.819688+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:36.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:36 smithi177 bash[24494]: cluster 2024-09-13T14:18:34.819688+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:37 smithi099 bash[22656]: cluster 2024-09-13T14:18:36.820328+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:37 smithi099 bash[22656]: audit 2024-09-13T14:18:36.961317+0000 mgr.y (mgr.14150) 275 : audit [DBG] from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:37 smithi099 bash[22656]: audit 2024-09-13T14:18:36.967544+0000 mon.a (mon.0) 530 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:18:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:37 smithi099 bash[22656]: audit 2024-09-13T14:18:36.973470+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:18:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:37 smithi099 bash[22656]: audit 2024-09-13T14:18:36.975276+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:37.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:37 smithi099 bash[32082]: cluster 2024-09-13T14:18:36.820328+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:37.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:37 smithi099 bash[32082]: audit 2024-09-13T14:18:36.961317+0000 mgr.y (mgr.14150) 275 : audit [DBG] from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:37.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:37 smithi099 bash[32082]: audit 2024-09-13T14:18:36.967544+0000 mon.a (mon.0) 530 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:18:37.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:37 smithi099 bash[32082]: audit 2024-09-13T14:18:36.973470+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:18:37.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:37 smithi099 bash[32082]: audit 2024-09-13T14:18:36.975276+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:37.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:37 smithi177 bash[24494]: cluster 2024-09-13T14:18:36.820328+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:37.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:37 smithi177 bash[24494]: audit 2024-09-13T14:18:36.961317+0000 mgr.y (mgr.14150) 275 : audit [DBG] from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:37.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:37 smithi177 bash[24494]: audit 2024-09-13T14:18:36.967544+0000 mon.a (mon.0) 530 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:18:37.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:37 smithi177 bash[24494]: audit 2024-09-13T14:18:36.973470+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:18:37.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:37 smithi177 bash[24494]: audit 2024-09-13T14:18:36.975276+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:40.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:39 smithi099 bash[22656]: cluster 2024-09-13T14:18:38.820940+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:40.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:39 smithi099 bash[32082]: cluster 2024-09-13T14:18:38.820940+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:40.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:39 smithi177 bash[24494]: cluster 2024-09-13T14:18:38.820940+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:42.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:41 smithi177 bash[24494]: cluster 2024-09-13T14:18:40.821527+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:42.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:41 smithi099 bash[22656]: cluster 2024-09-13T14:18:40.821527+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:42.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:41 smithi099 bash[32082]: cluster 2024-09-13T14:18:40.821527+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:43 smithi177 bash[24494]: cluster 2024-09-13T14:18:42.822206+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:44.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:43 smithi099 bash[22656]: cluster 2024-09-13T14:18:42.822206+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:44.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:43 smithi099 bash[32082]: cluster 2024-09-13T14:18:42.822206+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:46.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[22656]: cluster 2024-09-13T14:18:44.822841+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:46.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[22656]: audit 2024-09-13T14:18:45.438867+0000 mon.a (mon.0) 533 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b"}]: dispatch 2024-09-13T14:18:46.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[22656]: audit 2024-09-13T14:18:45.439146+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.177:0/2212014319' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b"}]: dispatch 2024-09-13T14:18:46.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[22656]: audit 2024-09-13T14:18:45.447929+0000 mon.a (mon.0) 534 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b"}]': finished 2024-09-13T14:18:46.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[22656]: cluster 2024-09-13T14:18:45.448058+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:18:46.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[22656]: audit 2024-09-13T14:18:45.448337+0000 mon.a (mon.0) 536 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:46.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[32082]: cluster 2024-09-13T14:18:44.822841+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:46.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[32082]: audit 2024-09-13T14:18:45.438867+0000 mon.a (mon.0) 533 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b"}]: dispatch 2024-09-13T14:18:46.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[32082]: audit 2024-09-13T14:18:45.439146+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.177:0/2212014319' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b"}]: dispatch 2024-09-13T14:18:46.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[32082]: audit 2024-09-13T14:18:45.447929+0000 mon.a (mon.0) 534 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b"}]': finished 2024-09-13T14:18:46.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[32082]: cluster 2024-09-13T14:18:45.448058+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:18:46.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:45 smithi099 bash[32082]: audit 2024-09-13T14:18:45.448337+0000 mon.a (mon.0) 536 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:46.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:45 smithi177 bash[24494]: cluster 2024-09-13T14:18:44.822841+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:46.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:45 smithi177 bash[24494]: audit 2024-09-13T14:18:45.438867+0000 mon.a (mon.0) 533 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b"}]: dispatch 2024-09-13T14:18:46.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:45 smithi177 bash[24494]: audit 2024-09-13T14:18:45.439146+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.177:0/2212014319' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b"}]: dispatch 2024-09-13T14:18:46.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:45 smithi177 bash[24494]: audit 2024-09-13T14:18:45.447929+0000 mon.a (mon.0) 534 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b"}]': finished 2024-09-13T14:18:46.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:45 smithi177 bash[24494]: cluster 2024-09-13T14:18:45.448058+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-13T14:18:46.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:45 smithi177 bash[24494]: audit 2024-09-13T14:18:45.448337+0000 mon.a (mon.0) 536 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:47.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:46 smithi099 bash[22656]: audit 2024-09-13T14:18:46.091426+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.177:0/3754142939' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:18:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:46 smithi099 bash[32082]: audit 2024-09-13T14:18:46.091426+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.177:0/3754142939' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:18:47.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:46 smithi177 bash[24494]: audit 2024-09-13T14:18:46.091426+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.177:0/3754142939' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:18:48.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:47 smithi099 bash[22656]: cluster 2024-09-13T14:18:46.823421+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:48.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:47 smithi099 bash[32082]: cluster 2024-09-13T14:18:46.823421+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:48.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:47 smithi177 bash[24494]: cluster 2024-09-13T14:18:46.823421+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:50.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:49 smithi099 bash[22656]: cluster 2024-09-13T14:18:48.823928+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:50.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:49 smithi099 bash[32082]: cluster 2024-09-13T14:18:48.823928+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:50.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:49 smithi177 bash[24494]: cluster 2024-09-13T14:18:48.823928+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:52.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:51 smithi099 bash[22656]: cluster 2024-09-13T14:18:50.824435+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:52.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:51 smithi099 bash[32082]: cluster 2024-09-13T14:18:50.824435+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:52.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:51 smithi177 bash[24494]: cluster 2024-09-13T14:18:50.824435+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:54.187 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:53 smithi177 bash[24494]: cluster 2024-09-13T14:18:52.824982+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:54.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:53 smithi099 bash[32082]: cluster 2024-09-13T14:18:52.824982+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:54.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:53 smithi099 bash[22656]: cluster 2024-09-13T14:18:52.824982+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:56.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:55 smithi099 bash[32082]: cluster 2024-09-13T14:18:54.825522+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:56.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:55 smithi099 bash[22656]: cluster 2024-09-13T14:18:54.825522+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:56.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:55 smithi177 bash[24494]: cluster 2024-09-13T14:18:54.825522+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:57 smithi177 bash[24494]: cluster 2024-09-13T14:18:56.826057+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:58.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:57 smithi099 bash[32082]: cluster 2024-09-13T14:18:56.826057+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:58.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:57 smithi099 bash[22656]: cluster 2024-09-13T14:18:56.826057+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:18:59.291 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:58 smithi177 bash[24494]: audit 2024-09-13T14:18:58.055014+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:18:59.292 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:58 smithi177 bash[24494]: audit 2024-09-13T14:18:58.056684+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:59.292 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:58 smithi177 bash[24494]: cephadm 2024-09-13T14:18:58.058139+0000 mgr.y (mgr.14150) 286 : cephadm [INF] Deploying daemon osd.6 on smithi177 2024-09-13T14:18:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:58 smithi099 bash[22656]: audit 2024-09-13T14:18:58.055014+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:18:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:58 smithi099 bash[22656]: audit 2024-09-13T14:18:58.056684+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:58 smithi099 bash[22656]: cephadm 2024-09-13T14:18:58.058139+0000 mgr.y (mgr.14150) 286 : cephadm [INF] Deploying daemon osd.6 on smithi177 2024-09-13T14:18:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:58 smithi099 bash[32082]: audit 2024-09-13T14:18:58.055014+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:18:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:58 smithi099 bash[32082]: audit 2024-09-13T14:18:58.056684+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:58 smithi099 bash[32082]: cephadm 2024-09-13T14:18:58.058139+0000 mgr.y (mgr.14150) 286 : cephadm [INF] Deploying daemon osd.6 on smithi177 2024-09-13T14:19:00.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:18:59 smithi177 bash[24494]: cluster 2024-09-13T14:18:58.826607+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:00.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:18:59 smithi099 bash[22656]: cluster 2024-09-13T14:18:58.826607+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:00.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:18:59 smithi099 bash[32082]: cluster 2024-09-13T14:18:58.826607+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:02.269 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:01 smithi177 bash[24494]: cluster 2024-09-13T14:19:00.827130+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:02.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:01 smithi099 bash[22656]: cluster 2024-09-13T14:19:00.827130+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:02.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:01 smithi099 bash[32082]: cluster 2024-09-13T14:19:00.827130+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:02.599 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:02.599 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:02.600 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:19:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:02.600 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:19:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:02.600 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:19:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:02.600 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:19:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:02.600 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:19:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:02.601 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:19:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:03.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:02 smithi099 bash[22656]: audit 2024-09-13T14:19:02.633980+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:03.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:02 smithi099 bash[22656]: audit 2024-09-13T14:19:02.641606+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:03.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:02 smithi099 bash[22656]: audit 2024-09-13T14:19:02.652151+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:02 smithi099 bash[32082]: audit 2024-09-13T14:19:02.633980+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:02 smithi099 bash[32082]: audit 2024-09-13T14:19:02.641606+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:02 smithi099 bash[32082]: audit 2024-09-13T14:19:02.652151+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:03.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:02 smithi177 bash[24494]: audit 2024-09-13T14:19:02.633980+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:03.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:02 smithi177 bash[24494]: audit 2024-09-13T14:19:02.641606+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:03.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:02 smithi177 bash[24494]: audit 2024-09-13T14:19:02.652151+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:04.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:03 smithi177 bash[24494]: cluster 2024-09-13T14:19:02.827658+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:04.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:03 smithi099 bash[22656]: cluster 2024-09-13T14:19:02.827658+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:04.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:03 smithi099 bash[32082]: cluster 2024-09-13T14:19:02.827658+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:06.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:05 smithi177 bash[24494]: cluster 2024-09-13T14:19:04.828166+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:06.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:05 smithi099 bash[22656]: cluster 2024-09-13T14:19:04.828166+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:06.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:05 smithi099 bash[32082]: cluster 2024-09-13T14:19:04.828166+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:08.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:07 smithi099 bash[22656]: cluster 2024-09-13T14:19:06.828684+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:08.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:07 smithi099 bash[22656]: audit 2024-09-13T14:19:07.725021+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:08.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:07 smithi099 bash[22656]: audit 2024-09-13T14:19:07.734115+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:08.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:07 smithi099 bash[32082]: cluster 2024-09-13T14:19:06.828684+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:08.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:07 smithi099 bash[32082]: audit 2024-09-13T14:19:07.725021+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:08.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:07 smithi099 bash[32082]: audit 2024-09-13T14:19:07.734115+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:08.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:07 smithi177 bash[24494]: cluster 2024-09-13T14:19:06.828684+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:08.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:07 smithi177 bash[24494]: audit 2024-09-13T14:19:07.725021+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:08.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:07 smithi177 bash[24494]: audit 2024-09-13T14:19:07.734115+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:09.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:08 smithi099 bash[22656]: audit 2024-09-13T14:19:08.162518+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:08 smithi099 bash[22656]: audit 2024-09-13T14:19:08.164172+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:08 smithi099 bash[22656]: audit 2024-09-13T14:19:08.173834+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:09.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:08 smithi099 bash[32082]: audit 2024-09-13T14:19:08.162518+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:09.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:08 smithi099 bash[32082]: audit 2024-09-13T14:19:08.164172+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:09.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:08 smithi099 bash[32082]: audit 2024-09-13T14:19:08.173834+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:09.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:08 smithi177 bash[24494]: audit 2024-09-13T14:19:08.162518+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:09.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:08 smithi177 bash[24494]: audit 2024-09-13T14:19:08.164172+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:09.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:08 smithi177 bash[24494]: audit 2024-09-13T14:19:08.173834+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:10.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:10 smithi099 bash[22656]: cluster 2024-09-13T14:19:08.829339+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:10.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:10 smithi099 bash[22656]: audit 2024-09-13T14:19:09.398446+0000 mon.a (mon.0) 547 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:19:10.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:10 smithi099 bash[22656]: audit 2024-09-13T14:19:09.398474+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:19:10.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:10 smithi099 bash[32082]: cluster 2024-09-13T14:19:08.829339+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:10.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:10 smithi099 bash[32082]: audit 2024-09-13T14:19:09.398446+0000 mon.a (mon.0) 547 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:19:10.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:10 smithi099 bash[32082]: audit 2024-09-13T14:19:09.398474+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:19:10.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:10 smithi177 bash[24494]: cluster 2024-09-13T14:19:08.829339+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:10.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:10 smithi177 bash[24494]: audit 2024-09-13T14:19:09.398446+0000 mon.a (mon.0) 547 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:19:10.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:10 smithi177 bash[24494]: audit 2024-09-13T14:19:09.398474+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:19:11.425 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 6 on host 'smithi177' 2024-09-13T14:19:11.443 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:11 smithi177 bash[24494]: audit 2024-09-13T14:19:10.186832+0000 mon.a (mon.0) 548 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:19:11.443 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:11 smithi177 bash[24494]: cluster 2024-09-13T14:19:10.186957+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:19:11.443 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:11 smithi177 bash[24494]: audit 2024-09-13T14:19:10.187247+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:11.443 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:11 smithi177 bash[24494]: audit 2024-09-13T14:19:10.188826+0000 mon.b (mon.1) 21 : audit [INF] from='osd.6 [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:19:11.443 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:11 smithi177 bash[24494]: audit 2024-09-13T14:19:10.188988+0000 mon.a (mon.0) 551 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:19:11.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:11 smithi099 bash[22656]: audit 2024-09-13T14:19:10.186832+0000 mon.a (mon.0) 548 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:19:11.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:11 smithi099 bash[22656]: cluster 2024-09-13T14:19:10.186957+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:19:11.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:11 smithi099 bash[22656]: audit 2024-09-13T14:19:10.187247+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:11.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:11 smithi099 bash[22656]: audit 2024-09-13T14:19:10.188826+0000 mon.b (mon.1) 21 : audit [INF] from='osd.6 [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:19:11.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:11 smithi099 bash[22656]: audit 2024-09-13T14:19:10.188988+0000 mon.a (mon.0) 551 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:19:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:11 smithi099 bash[32082]: audit 2024-09-13T14:19:10.186832+0000 mon.a (mon.0) 548 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:19:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:11 smithi099 bash[32082]: cluster 2024-09-13T14:19:10.186957+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-13T14:19:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:11 smithi099 bash[32082]: audit 2024-09-13T14:19:10.187247+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:11 smithi099 bash[32082]: audit 2024-09-13T14:19:10.188826+0000 mon.b (mon.1) 21 : audit [INF] from='osd.6 [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:19:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:11 smithi099 bash[32082]: audit 2024-09-13T14:19:10.188988+0000 mon.a (mon.0) 551 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:19:12.072 DEBUG:teuthology.orchestra.run.smithi177:osd.6> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T14:19:12.075 INFO:tasks.cephadm:Deploying osd.7 on smithi177 with /dev/vg_nvme/lv_1... 2024-09-13T14:19:12.075 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T14:19:12.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:12 smithi177 bash[24494]: cluster 2024-09-13T14:19:10.829914+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:12.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:12 smithi177 bash[24494]: audit 2024-09-13T14:19:11.189334+0000 mon.a (mon.0) 552 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:19:12.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:12 smithi177 bash[24494]: cluster 2024-09-13T14:19:11.189536+0000 mon.a (mon.0) 553 : cluster [DBG] osdmap e42: 7 total, 6 up, 7 in 2024-09-13T14:19:12.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:12 smithi177 bash[24494]: audit 2024-09-13T14:19:11.190352+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:12.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:12 smithi177 bash[24494]: audit 2024-09-13T14:19:11.200055+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:12.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:12 smithi177 bash[24494]: audit 2024-09-13T14:19:11.405229+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:12.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:12 smithi177 bash[24494]: audit 2024-09-13T14:19:11.413273+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:12.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:12 smithi177 bash[24494]: audit 2024-09-13T14:19:11.422097+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[22656]: cluster 2024-09-13T14:19:10.829914+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[22656]: audit 2024-09-13T14:19:11.189334+0000 mon.a (mon.0) 552 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:19:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[22656]: cluster 2024-09-13T14:19:11.189536+0000 mon.a (mon.0) 553 : cluster [DBG] osdmap e42: 7 total, 6 up, 7 in 2024-09-13T14:19:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[22656]: audit 2024-09-13T14:19:11.190352+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[22656]: audit 2024-09-13T14:19:11.200055+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[22656]: audit 2024-09-13T14:19:11.405229+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[22656]: audit 2024-09-13T14:19:11.413273+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[22656]: audit 2024-09-13T14:19:11.422097+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[32082]: cluster 2024-09-13T14:19:10.829914+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[32082]: audit 2024-09-13T14:19:11.189334+0000 mon.a (mon.0) 552 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:19:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[32082]: cluster 2024-09-13T14:19:11.189536+0000 mon.a (mon.0) 553 : cluster [DBG] osdmap e42: 7 total, 6 up, 7 in 2024-09-13T14:19:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[32082]: audit 2024-09-13T14:19:11.190352+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[32082]: audit 2024-09-13T14:19:11.200055+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[32082]: audit 2024-09-13T14:19:11.405229+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[32082]: audit 2024-09-13T14:19:11.413273+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:12 smithi099 bash[32082]: audit 2024-09-13T14:19:11.422097+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:13.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[22656]: cluster 2024-09-13T14:19:10.431326+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:19:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[22656]: cluster 2024-09-13T14:19:10.431386+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:19:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[22656]: audit 2024-09-13T14:19:12.201857+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[22656]: cluster 2024-09-13T14:19:12.208905+0000 mon.a (mon.0) 560 : cluster [INF] osd.6 [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594] boot 2024-09-13T14:19:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[22656]: cluster 2024-09-13T14:19:12.208974+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:19:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[22656]: audit 2024-09-13T14:19:12.209487+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[22656]: cluster 2024-09-13T14:19:13.024092+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:19:13.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[32082]: cluster 2024-09-13T14:19:10.431326+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:19:13.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[32082]: cluster 2024-09-13T14:19:10.431386+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:19:13.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[32082]: audit 2024-09-13T14:19:12.201857+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:13.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[32082]: cluster 2024-09-13T14:19:12.208905+0000 mon.a (mon.0) 560 : cluster [INF] osd.6 [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594] boot 2024-09-13T14:19:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[32082]: cluster 2024-09-13T14:19:12.208974+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:19:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[32082]: audit 2024-09-13T14:19:12.209487+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:13 smithi099 bash[32082]: cluster 2024-09-13T14:19:13.024092+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:19:13.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:13 smithi177 bash[24494]: cluster 2024-09-13T14:19:10.431326+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:19:13.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:13 smithi177 bash[24494]: cluster 2024-09-13T14:19:10.431386+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:19:13.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:13 smithi177 bash[24494]: audit 2024-09-13T14:19:12.201857+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:13.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:13 smithi177 bash[24494]: cluster 2024-09-13T14:19:12.208905+0000 mon.a (mon.0) 560 : cluster [INF] osd.6 [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594] boot 2024-09-13T14:19:13.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:13 smithi177 bash[24494]: cluster 2024-09-13T14:19:12.208974+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-13T14:19:13.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:13 smithi177 bash[24494]: audit 2024-09-13T14:19:12.209487+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:13.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:13 smithi177 bash[24494]: cluster 2024-09-13T14:19:13.024092+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-13T14:19:14.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:14 smithi099 bash[22656]: cluster 2024-09-13T14:19:12.830484+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:14.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:14 smithi099 bash[22656]: cluster 2024-09-13T14:19:14.025867+0000 mon.a (mon.0) 564 : cluster [DBG] osdmap e45: 7 total, 7 up, 7 in 2024-09-13T14:19:14.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:14 smithi099 bash[32082]: cluster 2024-09-13T14:19:12.830484+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:14.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:14 smithi099 bash[32082]: cluster 2024-09-13T14:19:14.025867+0000 mon.a (mon.0) 564 : cluster [DBG] osdmap e45: 7 total, 7 up, 7 in 2024-09-13T14:19:14.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:14 smithi177 bash[24494]: cluster 2024-09-13T14:19:12.830484+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T14:19:14.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:14 smithi177 bash[24494]: cluster 2024-09-13T14:19:14.025867+0000 mon.a (mon.0) 564 : cluster [DBG] osdmap e45: 7 total, 7 up, 7 in 2024-09-13T14:19:16.215 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:19:16.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:16 smithi099 bash[22656]: cluster 2024-09-13T14:19:14.831015+0000 mgr.y (mgr.14150) 295 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:16.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:16 smithi099 bash[32082]: cluster 2024-09-13T14:19:14.831015+0000 mgr.y (mgr.14150) 295 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:16.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:16 smithi177 bash[24494]: cluster 2024-09-13T14:19:14.831015+0000 mgr.y (mgr.14150) 295 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:18.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:18 smithi177 bash[24494]: cluster 2024-09-13T14:19:16.831499+0000 mgr.y (mgr.14150) 296 : cluster [DBG] pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:18.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:18 smithi099 bash[22656]: cluster 2024-09-13T14:19:16.831499+0000 mgr.y (mgr.14150) 296 : cluster [DBG] pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:18.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:18 smithi099 bash[32082]: cluster 2024-09-13T14:19:16.831499+0000 mgr.y (mgr.14150) 296 : cluster [DBG] pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:20.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:20 smithi099 bash[22656]: cluster 2024-09-13T14:19:18.832019+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:20.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:20 smithi099 bash[32082]: cluster 2024-09-13T14:19:18.832019+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:20.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:20 smithi177 bash[24494]: cluster 2024-09-13T14:19:18.832019+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:21.414 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-09-13T14:19:21.454 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi177:vg_nvme/lv_1 2024-09-13T14:19:22.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:22 smithi099 bash[22656]: cluster 2024-09-13T14:19:20.832544+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:22.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:22 smithi099 bash[32082]: cluster 2024-09-13T14:19:20.832544+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:22.580 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:22 smithi177 bash[24494]: cluster 2024-09-13T14:19:20.832544+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:24.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:24 smithi099 bash[22656]: cluster 2024-09-13T14:19:22.833102+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:24.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:24 smithi099 bash[32082]: cluster 2024-09-13T14:19:22.833102+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:24.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:24 smithi177 bash[24494]: cluster 2024-09-13T14:19:22.833102+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: cephadm 2024-09-13T14:19:24.321939+0000 mgr.y (mgr.14150) 300 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:19:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: audit 2024-09-13T14:19:24.330735+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: audit 2024-09-13T14:19:24.339610+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: audit 2024-09-13T14:19:24.341754+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: audit 2024-09-13T14:19:24.343527+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: audit 2024-09-13T14:19:24.345123+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: cephadm 2024-09-13T14:19:24.346287+0000 mgr.y (mgr.14150) 301 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 5755M 2024-09-13T14:19:25.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: audit 2024-09-13T14:19:24.354813+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: audit 2024-09-13T14:19:24.356849+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:25.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: audit 2024-09-13T14:19:24.358505+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:25.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:25 smithi177 bash[24494]: audit 2024-09-13T14:19:24.368530+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: cephadm 2024-09-13T14:19:24.321939+0000 mgr.y (mgr.14150) 300 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:19:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: audit 2024-09-13T14:19:24.330735+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: audit 2024-09-13T14:19:24.339610+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: audit 2024-09-13T14:19:24.341754+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: audit 2024-09-13T14:19:24.343527+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: audit 2024-09-13T14:19:24.345123+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: cephadm 2024-09-13T14:19:24.346287+0000 mgr.y (mgr.14150) 301 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 5755M 2024-09-13T14:19:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: audit 2024-09-13T14:19:24.354813+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: audit 2024-09-13T14:19:24.356849+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: audit 2024-09-13T14:19:24.358505+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[22656]: audit 2024-09-13T14:19:24.368530+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: cephadm 2024-09-13T14:19:24.321939+0000 mgr.y (mgr.14150) 300 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: audit 2024-09-13T14:19:24.330735+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: audit 2024-09-13T14:19:24.339610+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: audit 2024-09-13T14:19:24.341754+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: audit 2024-09-13T14:19:24.343527+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: audit 2024-09-13T14:19:24.345123+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: cephadm 2024-09-13T14:19:24.346287+0000 mgr.y (mgr.14150) 301 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 5755M 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: audit 2024-09-13T14:19:24.354813+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: audit 2024-09-13T14:19:24.356849+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: audit 2024-09-13T14:19:24.358505+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:25 smithi099 bash[32082]: audit 2024-09-13T14:19:24.368530+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:26.801 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:19:26.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:26 smithi099 bash[22656]: cluster 2024-09-13T14:19:24.833608+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:26.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:26 smithi099 bash[32082]: cluster 2024-09-13T14:19:24.833608+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:26.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:26 smithi177 bash[24494]: cluster 2024-09-13T14:19:24.833608+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:28.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:28 smithi177 bash[24494]: cluster 2024-09-13T14:19:26.834185+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:28.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:28 smithi099 bash[22656]: cluster 2024-09-13T14:19:26.834185+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:28.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:28 smithi099 bash[32082]: cluster 2024-09-13T14:19:26.834185+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:30.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:30 smithi099 bash[22656]: cluster 2024-09-13T14:19:28.834688+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:30 smithi099 bash[22656]: audit 2024-09-13T14:19:30.119813+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:19:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:30 smithi099 bash[22656]: audit 2024-09-13T14:19:30.126005+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:19:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:30 smithi099 bash[22656]: audit 2024-09-13T14:19:30.127896+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:30 smithi099 bash[32082]: cluster 2024-09-13T14:19:28.834688+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:30 smithi099 bash[32082]: audit 2024-09-13T14:19:30.119813+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:19:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:30 smithi099 bash[32082]: audit 2024-09-13T14:19:30.126005+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:19:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:30 smithi099 bash[32082]: audit 2024-09-13T14:19:30.127896+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:30.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:30 smithi177 bash[24494]: cluster 2024-09-13T14:19:28.834688+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:30.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:30 smithi177 bash[24494]: audit 2024-09-13T14:19:30.119813+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:19:30.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:30 smithi177 bash[24494]: audit 2024-09-13T14:19:30.126005+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:19:30.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:30 smithi177 bash[24494]: audit 2024-09-13T14:19:30.127896+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:31.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:31 smithi099 bash[22656]: audit 2024-09-13T14:19:30.113446+0000 mgr.y (mgr.14150) 305 : audit [DBG] from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:31.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:31 smithi099 bash[32082]: audit 2024-09-13T14:19:30.113446+0000 mgr.y (mgr.14150) 305 : audit [DBG] from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:31.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:31 smithi177 bash[24494]: audit 2024-09-13T14:19:30.113446+0000 mgr.y (mgr.14150) 305 : audit [DBG] from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:32.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:32 smithi099 bash[22656]: cluster 2024-09-13T14:19:30.835202+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:32.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:32 smithi099 bash[32082]: cluster 2024-09-13T14:19:30.835202+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:32.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:32 smithi177 bash[24494]: cluster 2024-09-13T14:19:30.835202+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:34.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:34 smithi099 bash[22656]: cluster 2024-09-13T14:19:32.835746+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:34.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:34 smithi099 bash[32082]: cluster 2024-09-13T14:19:32.835746+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:34.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:34 smithi177 bash[24494]: cluster 2024-09-13T14:19:32.835746+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:36.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:36 smithi099 bash[22656]: cluster 2024-09-13T14:19:34.836296+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:36.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:36 smithi099 bash[32082]: cluster 2024-09-13T14:19:34.836296+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:36.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:36 smithi177 bash[24494]: cluster 2024-09-13T14:19:34.836296+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:38.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:38 smithi099 bash[22656]: cluster 2024-09-13T14:19:36.836825+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:38.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:38 smithi099 bash[32082]: cluster 2024-09-13T14:19:36.836825+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:38.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:38 smithi177 bash[24494]: cluster 2024-09-13T14:19:36.836825+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:39.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[22656]: audit 2024-09-13T14:19:38.685827+0000 mon.b (mon.1) 22 : audit [INF] from='client.? 172.21.15.177:0/2263468291' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad2c2977-fd6d-42f8-962e-907c2bf3204b"}]: dispatch 2024-09-13T14:19:39.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[22656]: audit 2024-09-13T14:19:38.686418+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad2c2977-fd6d-42f8-962e-907c2bf3204b"}]: dispatch 2024-09-13T14:19:39.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[22656]: audit 2024-09-13T14:19:38.695517+0000 mon.a (mon.0) 578 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad2c2977-fd6d-42f8-962e-907c2bf3204b"}]': finished 2024-09-13T14:19:39.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[22656]: cluster 2024-09-13T14:19:38.695646+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:19:39.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[22656]: audit 2024-09-13T14:19:38.695942+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:19:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[22656]: audit 2024-09-13T14:19:39.349878+0000 mon.b (mon.1) 23 : audit [DBG] from='client.? 172.21.15.177:0/2564365704' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:19:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[32082]: audit 2024-09-13T14:19:38.685827+0000 mon.b (mon.1) 22 : audit [INF] from='client.? 172.21.15.177:0/2263468291' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad2c2977-fd6d-42f8-962e-907c2bf3204b"}]: dispatch 2024-09-13T14:19:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[32082]: audit 2024-09-13T14:19:38.686418+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad2c2977-fd6d-42f8-962e-907c2bf3204b"}]: dispatch 2024-09-13T14:19:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[32082]: audit 2024-09-13T14:19:38.695517+0000 mon.a (mon.0) 578 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad2c2977-fd6d-42f8-962e-907c2bf3204b"}]': finished 2024-09-13T14:19:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[32082]: cluster 2024-09-13T14:19:38.695646+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:19:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[32082]: audit 2024-09-13T14:19:38.695942+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:19:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:39 smithi099 bash[32082]: audit 2024-09-13T14:19:39.349878+0000 mon.b (mon.1) 23 : audit [DBG] from='client.? 172.21.15.177:0/2564365704' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:19:39.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:39 smithi177 bash[24494]: audit 2024-09-13T14:19:38.685827+0000 mon.b (mon.1) 22 : audit [INF] from='client.? 172.21.15.177:0/2263468291' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad2c2977-fd6d-42f8-962e-907c2bf3204b"}]: dispatch 2024-09-13T14:19:39.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:39 smithi177 bash[24494]: audit 2024-09-13T14:19:38.686418+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad2c2977-fd6d-42f8-962e-907c2bf3204b"}]: dispatch 2024-09-13T14:19:39.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:39 smithi177 bash[24494]: audit 2024-09-13T14:19:38.695517+0000 mon.a (mon.0) 578 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad2c2977-fd6d-42f8-962e-907c2bf3204b"}]': finished 2024-09-13T14:19:39.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:39 smithi177 bash[24494]: cluster 2024-09-13T14:19:38.695646+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-13T14:19:39.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:39 smithi177 bash[24494]: audit 2024-09-13T14:19:38.695942+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:19:39.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:39 smithi177 bash[24494]: audit 2024-09-13T14:19:39.349878+0000 mon.b (mon.1) 23 : audit [DBG] from='client.? 172.21.15.177:0/2564365704' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:19:40.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:40 smithi099 bash[22656]: cluster 2024-09-13T14:19:38.837297+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:40.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:40 smithi099 bash[32082]: cluster 2024-09-13T14:19:38.837297+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:40 smithi177 bash[24494]: cluster 2024-09-13T14:19:38.837297+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:42.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:42 smithi099 bash[22656]: cluster 2024-09-13T14:19:40.837941+0000 mgr.y (mgr.14150) 311 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:42.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:42 smithi099 bash[32082]: cluster 2024-09-13T14:19:40.837941+0000 mgr.y (mgr.14150) 311 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:42.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:42 smithi177 bash[24494]: cluster 2024-09-13T14:19:40.837941+0000 mgr.y (mgr.14150) 311 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:44.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:44 smithi099 bash[22656]: cluster 2024-09-13T14:19:42.838543+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:44.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:44 smithi099 bash[32082]: cluster 2024-09-13T14:19:42.838543+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:44.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:44 smithi177 bash[24494]: cluster 2024-09-13T14:19:42.838543+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:46.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:46 smithi099 bash[22656]: cluster 2024-09-13T14:19:44.839206+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:46.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:46 smithi099 bash[32082]: cluster 2024-09-13T14:19:44.839206+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:46.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:46 smithi177 bash[24494]: cluster 2024-09-13T14:19:44.839206+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:48.697 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:48 smithi177 bash[24494]: cluster 2024-09-13T14:19:46.839766+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:48 smithi099 bash[22656]: cluster 2024-09-13T14:19:46.839766+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:48.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:48 smithi099 bash[32082]: cluster 2024-09-13T14:19:46.839766+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:50.574 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:50 smithi177 bash[24494]: cluster 2024-09-13T14:19:48.840307+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:50.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:50 smithi099 bash[22656]: cluster 2024-09-13T14:19:48.840307+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:50.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:50 smithi099 bash[32082]: cluster 2024-09-13T14:19:48.840307+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:51.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:51 smithi099 bash[22656]: audit 2024-09-13T14:19:51.223562+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:19:51.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:51 smithi099 bash[22656]: audit 2024-09-13T14:19:51.225373+0000 mon.a (mon.0) 582 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:51.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:51 smithi099 bash[32082]: audit 2024-09-13T14:19:51.223562+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:19:51.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:51 smithi099 bash[32082]: audit 2024-09-13T14:19:51.225373+0000 mon.a (mon.0) 582 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:51.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:51 smithi177 bash[24494]: audit 2024-09-13T14:19:51.223562+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:19:51.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:51 smithi177 bash[24494]: audit 2024-09-13T14:19:51.225373+0000 mon.a (mon.0) 582 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:52.736 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:52 smithi177 bash[24494]: cluster 2024-09-13T14:19:50.840826+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:52.736 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:52 smithi177 bash[24494]: cephadm 2024-09-13T14:19:51.226943+0000 mgr.y (mgr.14150) 317 : cephadm [INF] Deploying daemon osd.7 on smithi177 2024-09-13T14:19:52.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:52 smithi099 bash[22656]: cluster 2024-09-13T14:19:50.840826+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:52.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:52 smithi099 bash[22656]: cephadm 2024-09-13T14:19:51.226943+0000 mgr.y (mgr.14150) 317 : cephadm [INF] Deploying daemon osd.7 on smithi177 2024-09-13T14:19:52.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:52 smithi099 bash[32082]: cluster 2024-09-13T14:19:50.840826+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:52.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:52 smithi099 bash[32082]: cephadm 2024-09-13T14:19:51.226943+0000 mgr.y (mgr.14150) 317 : cephadm [INF] Deploying daemon osd.7 on smithi177 2024-09-13T14:19:54.763 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:54 smithi177 bash[24494]: cluster 2024-09-13T14:19:52.841388+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:54.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:54 smithi099 bash[22656]: cluster 2024-09-13T14:19:52.841388+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:54.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:54 smithi099 bash[32082]: cluster 2024-09-13T14:19:52.841388+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:55.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:55 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:55.858 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:19:55 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:55.858 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:19:55 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:55.858 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:19:55 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:55.858 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:19:55 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:56.358 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:19:55 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:56.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:55 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:56.358 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:19:55 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:56.358 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:19:55 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:56.359 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:19:55 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:19:56.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:56 smithi099 bash[22656]: cluster 2024-09-13T14:19:54.841975+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:56.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:56 smithi099 bash[22656]: audit 2024-09-13T14:19:55.997472+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:56.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:56 smithi099 bash[22656]: audit 2024-09-13T14:19:56.007015+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:56.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:56 smithi099 bash[22656]: audit 2024-09-13T14:19:56.015931+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:56.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:56 smithi099 bash[32082]: cluster 2024-09-13T14:19:54.841975+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:56.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:56 smithi099 bash[32082]: audit 2024-09-13T14:19:55.997472+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:56.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:56 smithi099 bash[32082]: audit 2024-09-13T14:19:56.007015+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:56.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:56 smithi099 bash[32082]: audit 2024-09-13T14:19:56.015931+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:56.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:56 smithi177 bash[24494]: cluster 2024-09-13T14:19:54.841975+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:56.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:56 smithi177 bash[24494]: audit 2024-09-13T14:19:55.997472+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:56.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:56 smithi177 bash[24494]: audit 2024-09-13T14:19:56.007015+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:56.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:56 smithi177 bash[24494]: audit 2024-09-13T14:19:56.015931+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:19:58.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:19:58 smithi177 bash[24494]: cluster 2024-09-13T14:19:56.842477+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:58.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:19:58 smithi099 bash[22656]: cluster 2024-09-13T14:19:56.842477+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:19:58.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:19:58 smithi099 bash[32082]: cluster 2024-09-13T14:19:56.842477+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:00.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:00 smithi099 bash[32082]: cluster 2024-09-13T14:19:58.842974+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:00.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:00 smithi099 bash[32082]: cluster 2024-09-13T14:20:00.000112+0000 mon.a (mon.0) 586 : cluster [INF] overall HEALTH_OK 2024-09-13T14:20:00.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:00 smithi099 bash[22656]: cluster 2024-09-13T14:19:58.842974+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:00.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:00 smithi099 bash[22656]: cluster 2024-09-13T14:20:00.000112+0000 mon.a (mon.0) 586 : cluster [INF] overall HEALTH_OK 2024-09-13T14:20:00.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:00 smithi177 bash[24494]: cluster 2024-09-13T14:19:58.842974+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:00.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:00 smithi177 bash[24494]: cluster 2024-09-13T14:20:00.000112+0000 mon.a (mon.0) 586 : cluster [INF] overall HEALTH_OK 2024-09-13T14:20:02.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[32082]: cluster 2024-09-13T14:20:00.843506+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:02.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[32082]: audit 2024-09-13T14:20:02.097108+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:02.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[32082]: audit 2024-09-13T14:20:02.106544+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:02.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[32082]: audit 2024-09-13T14:20:02.108513+0000 mon.a (mon.0) 589 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:02.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[32082]: audit 2024-09-13T14:20:02.110035+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:02.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[32082]: audit 2024-09-13T14:20:02.119761+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:02.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[22656]: cluster 2024-09-13T14:20:00.843506+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:02.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[22656]: audit 2024-09-13T14:20:02.097108+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:02.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[22656]: audit 2024-09-13T14:20:02.106544+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:02.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[22656]: audit 2024-09-13T14:20:02.108513+0000 mon.a (mon.0) 589 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:02.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[22656]: audit 2024-09-13T14:20:02.110035+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:02.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:02 smithi099 bash[22656]: audit 2024-09-13T14:20:02.119761+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:02.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:02 smithi177 bash[24494]: cluster 2024-09-13T14:20:00.843506+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:02.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:02 smithi177 bash[24494]: audit 2024-09-13T14:20:02.097108+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:02.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:02 smithi177 bash[24494]: audit 2024-09-13T14:20:02.106544+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:02.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:02 smithi177 bash[24494]: audit 2024-09-13T14:20:02.108513+0000 mon.a (mon.0) 589 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:02.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:02 smithi177 bash[24494]: audit 2024-09-13T14:20:02.110035+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:02.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:02 smithi177 bash[24494]: audit 2024-09-13T14:20:02.119761+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:03.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:03 smithi099 bash[22656]: audit 2024-09-13T14:20:02.868145+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:20:03.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:03 smithi099 bash[22656]: audit 2024-09-13T14:20:02.868576+0000 mon.a (mon.0) 592 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:20:03.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:03 smithi099 bash[32082]: audit 2024-09-13T14:20:02.868145+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:20:03.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:03 smithi099 bash[32082]: audit 2024-09-13T14:20:02.868576+0000 mon.a (mon.0) 592 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:20:03.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:03 smithi177 bash[24494]: audit 2024-09-13T14:20:02.868145+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:20:03.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:03 smithi177 bash[24494]: audit 2024-09-13T14:20:02.868576+0000 mon.a (mon.0) 592 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:20:04.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[22656]: cluster 2024-09-13T14:20:02.844055+0000 mgr.y (mgr.14150) 323 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:04.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[22656]: audit 2024-09-13T14:20:03.475156+0000 mon.a (mon.0) 593 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:20:04.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[22656]: cluster 2024-09-13T14:20:03.475365+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:20:04.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[22656]: audit 2024-09-13T14:20:03.475665+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:04.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[22656]: audit 2024-09-13T14:20:03.476773+0000 mon.b (mon.1) 25 : audit [INF] from='osd.7 [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:20:04.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[22656]: audit 2024-09-13T14:20:03.477403+0000 mon.a (mon.0) 596 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:20:04.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[32082]: cluster 2024-09-13T14:20:02.844055+0000 mgr.y (mgr.14150) 323 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:04.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[32082]: audit 2024-09-13T14:20:03.475156+0000 mon.a (mon.0) 593 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:20:04.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[32082]: cluster 2024-09-13T14:20:03.475365+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:20:04.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[32082]: audit 2024-09-13T14:20:03.475665+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:04.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[32082]: audit 2024-09-13T14:20:03.476773+0000 mon.b (mon.1) 25 : audit [INF] from='osd.7 [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:20:04.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:04 smithi099 bash[32082]: audit 2024-09-13T14:20:03.477403+0000 mon.a (mon.0) 596 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:20:04.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:04 smithi177 bash[24494]: cluster 2024-09-13T14:20:02.844055+0000 mgr.y (mgr.14150) 323 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:04 smithi177 bash[24494]: audit 2024-09-13T14:20:03.475156+0000 mon.a (mon.0) 593 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:20:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:04 smithi177 bash[24494]: cluster 2024-09-13T14:20:03.475365+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-13T14:20:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:04 smithi177 bash[24494]: audit 2024-09-13T14:20:03.475665+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:04 smithi177 bash[24494]: audit 2024-09-13T14:20:03.476773+0000 mon.b (mon.1) 25 : audit [INF] from='osd.7 [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:20:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:04 smithi177 bash[24494]: audit 2024-09-13T14:20:03.477403+0000 mon.a (mon.0) 596 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-09-13T14:20:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:05 smithi177 bash[24494]: audit 2024-09-13T14:20:04.479893+0000 mon.a (mon.0) 597 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:20:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:05 smithi177 bash[24494]: cluster 2024-09-13T14:20:04.480091+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e48: 8 total, 7 up, 8 in 2024-09-13T14:20:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:05 smithi177 bash[24494]: audit 2024-09-13T14:20:04.480860+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:05 smithi177 bash[24494]: audit 2024-09-13T14:20:04.492284+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:05 smithi177 bash[24494]: cluster 2024-09-13T14:20:05.480168+0000 mon.a (mon.0) 601 : cluster [INF] osd.7 [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946] boot 2024-09-13T14:20:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:05 smithi177 bash[24494]: cluster 2024-09-13T14:20:05.480274+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:20:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:05 smithi177 bash[24494]: audit 2024-09-13T14:20:05.480621+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:05.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[22656]: audit 2024-09-13T14:20:04.479893+0000 mon.a (mon.0) 597 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:20:05.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[22656]: cluster 2024-09-13T14:20:04.480091+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e48: 8 total, 7 up, 8 in 2024-09-13T14:20:05.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[22656]: audit 2024-09-13T14:20:04.480860+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:05.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[22656]: audit 2024-09-13T14:20:04.492284+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:05.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[22656]: cluster 2024-09-13T14:20:05.480168+0000 mon.a (mon.0) 601 : cluster [INF] osd.7 [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946] boot 2024-09-13T14:20:05.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[22656]: cluster 2024-09-13T14:20:05.480274+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:20:05.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[22656]: audit 2024-09-13T14:20:05.480621+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:05.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[32082]: audit 2024-09-13T14:20:04.479893+0000 mon.a (mon.0) 597 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-09-13T14:20:05.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[32082]: cluster 2024-09-13T14:20:04.480091+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e48: 8 total, 7 up, 8 in 2024-09-13T14:20:05.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[32082]: audit 2024-09-13T14:20:04.480860+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:05.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[32082]: audit 2024-09-13T14:20:04.492284+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:05.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[32082]: cluster 2024-09-13T14:20:05.480168+0000 mon.a (mon.0) 601 : cluster [INF] osd.7 [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946] boot 2024-09-13T14:20:05.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[32082]: cluster 2024-09-13T14:20:05.480274+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-13T14:20:05.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:05 smithi099 bash[32082]: audit 2024-09-13T14:20:05.480621+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:20:06.448 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 7 on host 'smithi177' 2024-09-13T14:20:06.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[22656]: cluster 2024-09-13T14:20:03.826278+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:20:06.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[22656]: cluster 2024-09-13T14:20:03.826384+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:20:06.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[22656]: cluster 2024-09-13T14:20:04.844598+0000 mgr.y (mgr.14150) 324 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:06.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[22656]: audit 2024-09-13T14:20:06.429193+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:06.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[22656]: audit 2024-09-13T14:20:06.437466+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:06.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[22656]: audit 2024-09-13T14:20:06.447509+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:06.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[22656]: cluster 2024-09-13T14:20:06.494625+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:20:06.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[32082]: cluster 2024-09-13T14:20:03.826278+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:20:06.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[32082]: cluster 2024-09-13T14:20:03.826384+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:20:06.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[32082]: cluster 2024-09-13T14:20:04.844598+0000 mgr.y (mgr.14150) 324 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:06.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[32082]: audit 2024-09-13T14:20:06.429193+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:06.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[32082]: audit 2024-09-13T14:20:06.437466+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:06.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[32082]: audit 2024-09-13T14:20:06.447509+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:06.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:06 smithi099 bash[32082]: cluster 2024-09-13T14:20:06.494625+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:20:06.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:06 smithi177 bash[24494]: cluster 2024-09-13T14:20:03.826278+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T14:20:06.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:06 smithi177 bash[24494]: cluster 2024-09-13T14:20:03.826384+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T14:20:06.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:06 smithi177 bash[24494]: cluster 2024-09-13T14:20:04.844598+0000 mgr.y (mgr.14150) 324 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:20:06.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:06 smithi177 bash[24494]: audit 2024-09-13T14:20:06.429193+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:06.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:06 smithi177 bash[24494]: audit 2024-09-13T14:20:06.437466+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:06.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:06 smithi177 bash[24494]: audit 2024-09-13T14:20:06.447509+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:06.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:06 smithi177 bash[24494]: cluster 2024-09-13T14:20:06.494625+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-13T14:20:07.127 DEBUG:teuthology.orchestra.run.smithi177:osd.7> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T14:20:07.130 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T14:20:07.130 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T14:20:08.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:08 smithi099 bash[22656]: cluster 2024-09-13T14:20:06.845190+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:08 smithi099 bash[22656]: cluster 2024-09-13T14:20:07.510522+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:20:08.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:08 smithi099 bash[32082]: cluster 2024-09-13T14:20:06.845190+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:08.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:08 smithi099 bash[32082]: cluster 2024-09-13T14:20:07.510522+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:20:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:08 smithi177 bash[24494]: cluster 2024-09-13T14:20:06.845190+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:08 smithi177 bash[24494]: cluster 2024-09-13T14:20:07.510522+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-13T14:20:10.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:10 smithi099 bash[32082]: cluster 2024-09-13T14:20:08.845844+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:10.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:10 smithi099 bash[22656]: cluster 2024-09-13T14:20:08.845844+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:10 smithi177 bash[24494]: cluster 2024-09-13T14:20:08.845844+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:11.893 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:12.784 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:12 smithi177 bash[24494]: cluster 2024-09-13T14:20:10.846534+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:12.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:12 smithi099 bash[32082]: cluster 2024-09-13T14:20:10.846534+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:12.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:12 smithi099 bash[22656]: cluster 2024-09-13T14:20:10.846534+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:14.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:14 smithi099 bash[32082]: cluster 2024-09-13T14:20:12.847224+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:14.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:14 smithi099 bash[22656]: cluster 2024-09-13T14:20:12.847224+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:14.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:14 smithi177 bash[24494]: cluster 2024-09-13T14:20:12.847224+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:15.251 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:20:15.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:15 smithi177 bash[24494]: audit 2024-09-13T14:20:15.255497+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.99:0/1155822434' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:20:15.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:15 smithi099 bash[22656]: audit 2024-09-13T14:20:15.255497+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.99:0/1155822434' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:20:15.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:15 smithi099 bash[32082]: audit 2024-09-13T14:20:15.255497+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.99:0/1155822434' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:20:16.053 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":51,"num_osds":8,"num_up_osds":8,"osd_up_since":1726237205,"num_in_osds":8,"osd_in_since":1726237178,"num_remapped_pgs":0} 2024-09-13T14:20:16.054 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:20:16.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: cluster 2024-09-13T14:20:14.847936+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: audit 2024-09-13T14:20:16.110579+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: audit 2024-09-13T14:20:16.122398+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: audit 2024-09-13T14:20:16.124907+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: audit 2024-09-13T14:20:16.126658+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: audit 2024-09-13T14:20:16.128296+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: audit 2024-09-13T14:20:16.129806+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: audit 2024-09-13T14:20:16.139241+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: audit 2024-09-13T14:20:16.141413+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: audit 2024-09-13T14:20:16.142929+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[22656]: audit 2024-09-13T14:20:16.155985+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: cluster 2024-09-13T14:20:14.847936+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: audit 2024-09-13T14:20:16.110579+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: audit 2024-09-13T14:20:16.122398+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: audit 2024-09-13T14:20:16.124907+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: audit 2024-09-13T14:20:16.126658+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: audit 2024-09-13T14:20:16.128296+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: audit 2024-09-13T14:20:16.129806+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: audit 2024-09-13T14:20:16.139241+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: audit 2024-09-13T14:20:16.141413+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: audit 2024-09-13T14:20:16.142929+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:16 smithi099 bash[32082]: audit 2024-09-13T14:20:16.155985+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: cluster 2024-09-13T14:20:14.847936+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: audit 2024-09-13T14:20:16.110579+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: audit 2024-09-13T14:20:16.122398+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: audit 2024-09-13T14:20:16.124907+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: audit 2024-09-13T14:20:16.126658+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: audit 2024-09-13T14:20:16.128296+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: audit 2024-09-13T14:20:16.129806+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:20:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: audit 2024-09-13T14:20:16.139241+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: audit 2024-09-13T14:20:16.141413+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: audit 2024-09-13T14:20:16.142929+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:16 smithi177 bash[24494]: audit 2024-09-13T14:20:16.155985+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:17.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:17 smithi099 bash[22656]: cephadm 2024-09-13T14:20:16.102020+0000 mgr.y (mgr.14150) 330 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:20:17.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:17 smithi099 bash[22656]: cephadm 2024-09-13T14:20:16.130961+0000 mgr.y (mgr.14150) 331 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 4316M 2024-09-13T14:20:17.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:17 smithi099 bash[32082]: cephadm 2024-09-13T14:20:16.102020+0000 mgr.y (mgr.14150) 330 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:20:17.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:17 smithi099 bash[32082]: cephadm 2024-09-13T14:20:16.130961+0000 mgr.y (mgr.14150) 331 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 4316M 2024-09-13T14:20:17.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:17 smithi177 bash[24494]: cephadm 2024-09-13T14:20:16.102020+0000 mgr.y (mgr.14150) 330 : cephadm [INF] Detected new or changed devices on smithi177 2024-09-13T14:20:17.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:17 smithi177 bash[24494]: cephadm 2024-09-13T14:20:16.130961+0000 mgr.y (mgr.14150) 331 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 4316M 2024-09-13T14:20:18.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:18 smithi099 bash[22656]: cluster 2024-09-13T14:20:16.848613+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:18.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:18 smithi099 bash[32082]: cluster 2024-09-13T14:20:16.848613+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:18.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:18 smithi177 bash[24494]: cluster 2024-09-13T14:20:16.848613+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:20.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:20 smithi099 bash[22656]: cluster 2024-09-13T14:20:18.849082+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:20.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:20 smithi099 bash[32082]: cluster 2024-09-13T14:20:18.849082+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:20.821 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:20.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:20 smithi177 bash[24494]: cluster 2024-09-13T14:20:18.849082+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:22.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:22 smithi099 bash[32082]: cluster 2024-09-13T14:20:20.849514+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:22.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:22 smithi099 bash[22656]: cluster 2024-09-13T14:20:20.849514+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:22.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:22 smithi177 bash[24494]: cluster 2024-09-13T14:20:20.849514+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:24.475 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:20:24.476 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":51,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:10:21.363918+0000","modified":"2024-09-13T14:20:07.497799+0000","last_up_change":"2024-09-13T14:20:05.471230+0000","last_in_change":"2024-09-13T14:19:38.687766+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:15:37.290226+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"0624feaa-339b-4468-be53-582a1f04352c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6803","nonce":500907575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6805","nonce":500907575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6809","nonce":500907575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6807","nonce":500907575}]},"public_addr":"172.21.15.99:6803/500907575","cluster_addr":"172.21.15.99:6805/500907575","heartbeat_back_addr":"172.21.15.99:6809/500907575","heartbeat_front_addr":"172.21.15.99:6807/500907575","state":["exists","up"]},{"osd":1,"uuid":"b7b63553-e49d-4636-b3e2-43b72556a1a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6811","nonce":4008940153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6813","nonce":4008940153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6817","nonce":4008940153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6815","nonce":4008940153}]},"public_addr":"172.21.15.99:6811/4008940153","cluster_addr":"172.21.15.99:6813/4008940153","heartbeat_back_addr":"172.21.15.99:6817/4008940153","heartbeat_front_addr":"172.21.15.99:6815/4008940153","state":["exists","up"]},{"osd":2,"uuid":"3b9eff6d-5f5c-4843-a1ff-815a902167ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6819","nonce":4220960661}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6821","nonce":4220960661}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6825","nonce":4220960661}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6823","nonce":4220960661}]},"public_addr":"172.21.15.99:6819/4220960661","cluster_addr":"172.21.15.99:6821/4220960661","heartbeat_back_addr":"172.21.15.99:6825/4220960661","heartbeat_front_addr":"172.21.15.99:6823/4220960661","state":["exists","up"]},{"osd":3,"uuid":"686c8d3c-9cab-4920-b02b-d243324f5808","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6827","nonce":1221827921}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6829","nonce":1221827921}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6833","nonce":1221827921}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6831","nonce":1221827921}]},"public_addr":"172.21.15.99:6827/1221827921","cluster_addr":"172.21.15.99:6829/1221827921","heartbeat_back_addr":"172.21.15.99:6833/1221827921","heartbeat_front_addr":"172.21.15.99:6831/1221827921","state":["exists","up"]},{"osd":4,"uuid":"2d56712b-a6fb-4089-9583-850beed92b37","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6801","nonce":2146549575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6803","nonce":2146549575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6807","nonce":2146549575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6805","nonce":2146549575}]},"public_addr":"172.21.15.177:6801/2146549575","cluster_addr":"172.21.15.177:6803/2146549575","heartbeat_back_addr":"172.21.15.177:6807/2146549575","heartbeat_front_addr":"172.21.15.177:6805/2146549575","state":["exists","up"]},{"osd":5,"uuid":"414b880c-6534-495f-a54c-f8d28ba81c73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6809","nonce":947070587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6811","nonce":947070587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6815","nonce":947070587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6813","nonce":947070587}]},"public_addr":"172.21.15.177:6809/947070587","cluster_addr":"172.21.15.177:6811/947070587","heartbeat_back_addr":"172.21.15.177:6815/947070587","heartbeat_front_addr":"172.21.15.177:6813/947070587","state":["exists","up"]},{"osd":6,"uuid":"ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6817","nonce":3817692594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6819","nonce":3817692594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6823","nonce":3817692594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6821","nonce":3817692594}]},"public_addr":"172.21.15.177:6817/3817692594","cluster_addr":"172.21.15.177:6819/3817692594","heartbeat_back_addr":"172.21.15.177:6823/3817692594","heartbeat_front_addr":"172.21.15.177:6821/3817692594","state":["exists","up"]},{"osd":7,"uuid":"ad2c2977-fd6d-42f8-962e-907c2bf3204b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6825","nonce":1984676946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6827","nonce":1984676946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6831","nonce":1984676946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6829","nonce":1984676946}]},"public_addr":"172.21.15.177:6825/1984676946","cluster_addr":"172.21.15.177:6827/1984676946","heartbeat_back_addr":"172.21.15.177:6831/1984676946","heartbeat_front_addr":"172.21.15.177:6829/1984676946","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:46.390607+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:38.398637+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:15:33.420625+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:30.964755+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:22.513614+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:14.948720+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:19:10.431389+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:20:03.826389+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6800/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/4129805635":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/1548831286":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/3192879133":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/881490667":"2024-09-14T14:10:48.295312+0000","172.21.15.99:6800/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:6801/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/888319381":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/795365709":"2024-09-14T14:10:48.295312+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:20:24.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:24 smithi099 bash[22656]: cluster 2024-09-13T14:20:22.850015+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:24.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:24 smithi099 bash[22656]: audit 2024-09-13T14:20:24.479881+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.99:0/499201447' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:20:24.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:24 smithi099 bash[32082]: cluster 2024-09-13T14:20:22.850015+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:24.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:24 smithi099 bash[32082]: audit 2024-09-13T14:20:24.479881+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.99:0/499201447' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:20:24.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:24 smithi177 bash[24494]: cluster 2024-09-13T14:20:22.850015+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:24.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:24 smithi177 bash[24494]: audit 2024-09-13T14:20:24.479881+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.99:0/499201447' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:20:25.365 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T14:15:37.290226+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T14:20:25.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T14:20:26.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:26 smithi099 bash[22656]: cluster 2024-09-13T14:20:24.850524+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:26.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:26 smithi099 bash[32082]: cluster 2024-09-13T14:20:24.850524+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:26.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:26 smithi177 bash[24494]: cluster 2024-09-13T14:20:24.850524+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:28.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:28 smithi099 bash[22656]: cluster 2024-09-13T14:20:26.851090+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:28.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:28 smithi099 bash[32082]: cluster 2024-09-13T14:20:26.851090+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:28.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:28 smithi177 bash[24494]: cluster 2024-09-13T14:20:26.851090+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:30.130 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:30.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:30 smithi099 bash[32082]: cluster 2024-09-13T14:20:28.851637+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:30.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:30 smithi099 bash[22656]: cluster 2024-09-13T14:20:28.851637+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:30.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:30 smithi177 bash[24494]: cluster 2024-09-13T14:20:28.851637+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:32.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:32 smithi099 bash[22656]: cluster 2024-09-13T14:20:30.852210+0000 mgr.y (mgr.14150) 339 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:32.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:32 smithi099 bash[32082]: cluster 2024-09-13T14:20:30.852210+0000 mgr.y (mgr.14150) 339 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:32.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:32 smithi177 bash[24494]: cluster 2024-09-13T14:20:30.852210+0000 mgr.y (mgr.14150) 339 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:33.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:33 smithi177 bash[24494]: cluster 2024-09-13T14:20:32.852719+0000 mgr.y (mgr.14150) 340 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:33.896 INFO:teuthology.orchestra.run.smithi099.stdout:pg_num: 1 2024-09-13T14:20:33.908 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:33 smithi099 bash[22656]: cluster 2024-09-13T14:20:32.852719+0000 mgr.y (mgr.14150) 340 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:33.908 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:33 smithi099 bash[32082]: cluster 2024-09-13T14:20:32.852719+0000 mgr.y (mgr.14150) 340 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:34.615 INFO:tasks.cephadm:Adding prometheus.a on smithi177 2024-09-13T14:20:34.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi177=a' 2024-09-13T14:20:34.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:34 smithi177 bash[24494]: audit 2024-09-13T14:20:33.901148+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.99:0/1924748188' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:20:35.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:34 smithi099 bash[32082]: audit 2024-09-13T14:20:33.901148+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.99:0/1924748188' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:20:35.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:34 smithi099 bash[22656]: audit 2024-09-13T14:20:33.901148+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.99:0/1924748188' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:20:35.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:35 smithi177 bash[24494]: cluster 2024-09-13T14:20:34.853256+0000 mgr.y (mgr.14150) 341 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:36.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:35 smithi099 bash[32082]: cluster 2024-09-13T14:20:34.853256+0000 mgr.y (mgr.14150) 341 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:36.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:35 smithi099 bash[22656]: cluster 2024-09-13T14:20:34.853256+0000 mgr.y (mgr.14150) 341 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:38.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:37 smithi099 bash[32082]: cluster 2024-09-13T14:20:36.853904+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:38.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:37 smithi099 bash[22656]: cluster 2024-09-13T14:20:36.853904+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:38.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:37 smithi177 bash[24494]: cluster 2024-09-13T14:20:36.853904+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:39.372 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:20:40.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:39 smithi099 bash[32082]: cluster 2024-09-13T14:20:38.854519+0000 mgr.y (mgr.14150) 343 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:40.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:39 smithi099 bash[22656]: cluster 2024-09-13T14:20:38.854519+0000 mgr.y (mgr.14150) 343 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:40.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:39 smithi177 bash[24494]: cluster 2024-09-13T14:20:38.854519+0000 mgr.y (mgr.14150) 343 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:42.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:41 smithi099 bash[32082]: cluster 2024-09-13T14:20:40.855098+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:42.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:41 smithi099 bash[22656]: cluster 2024-09-13T14:20:40.855098+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:42.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:41 smithi177 bash[24494]: cluster 2024-09-13T14:20:40.855098+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:42.732 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled prometheus update... 2024-09-13T14:20:43.414 DEBUG:teuthology.orchestra.run.smithi177:prometheus.a> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-13T14:20:43.417 INFO:tasks.cephadm:Adding node-exporter.a on smithi099 2024-09-13T14:20:43.417 INFO:tasks.cephadm:Adding node-exporter.b on smithi177 2024-09-13T14:20:43.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi099=a;smithi177=b' 2024-09-13T14:20:44.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[22656]: audit 2024-09-13T14:20:42.724617+0000 mgr.y (mgr.14150) 345 : audit [DBG] from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:44.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[22656]: cephadm 2024-09-13T14:20:42.727530+0000 mgr.y (mgr.14150) 346 : cephadm [INF] Saving service prometheus spec with placement smithi177=a;count:1 2024-09-13T14:20:44.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[22656]: audit 2024-09-13T14:20:42.736806+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:44.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[22656]: audit 2024-09-13T14:20:42.738521+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:44.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[22656]: cluster 2024-09-13T14:20:42.855713+0000 mgr.y (mgr.14150) 347 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:44.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[22656]: audit 2024-09-13T14:20:43.231384+0000 mon.a (mon.0) 623 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:44.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[22656]: audit 2024-09-13T14:20:43.232649+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:44.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[22656]: audit 2024-09-13T14:20:43.243087+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:44.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[22656]: cephadm 2024-09-13T14:20:43.263652+0000 mgr.y (mgr.14150) 348 : cephadm [INF] Deploying daemon prometheus.a on smithi177 2024-09-13T14:20:44.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[32082]: audit 2024-09-13T14:20:42.724617+0000 mgr.y (mgr.14150) 345 : audit [DBG] from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:44.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[32082]: cephadm 2024-09-13T14:20:42.727530+0000 mgr.y (mgr.14150) 346 : cephadm [INF] Saving service prometheus spec with placement smithi177=a;count:1 2024-09-13T14:20:44.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[32082]: audit 2024-09-13T14:20:42.736806+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:44.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[32082]: audit 2024-09-13T14:20:42.738521+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:44.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[32082]: cluster 2024-09-13T14:20:42.855713+0000 mgr.y (mgr.14150) 347 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:44.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[32082]: audit 2024-09-13T14:20:43.231384+0000 mon.a (mon.0) 623 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:44.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[32082]: audit 2024-09-13T14:20:43.232649+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:44.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[32082]: audit 2024-09-13T14:20:43.243087+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:44.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:43 smithi099 bash[32082]: cephadm 2024-09-13T14:20:43.263652+0000 mgr.y (mgr.14150) 348 : cephadm [INF] Deploying daemon prometheus.a on smithi177 2024-09-13T14:20:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:43 smithi177 bash[24494]: audit 2024-09-13T14:20:42.724617+0000 mgr.y (mgr.14150) 345 : audit [DBG] from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:43 smithi177 bash[24494]: cephadm 2024-09-13T14:20:42.727530+0000 mgr.y (mgr.14150) 346 : cephadm [INF] Saving service prometheus spec with placement smithi177=a;count:1 2024-09-13T14:20:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:43 smithi177 bash[24494]: audit 2024-09-13T14:20:42.736806+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:43 smithi177 bash[24494]: audit 2024-09-13T14:20:42.738521+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:43 smithi177 bash[24494]: cluster 2024-09-13T14:20:42.855713+0000 mgr.y (mgr.14150) 347 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:43 smithi177 bash[24494]: audit 2024-09-13T14:20:43.231384+0000 mon.a (mon.0) 623 : audit [DBG] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:43 smithi177 bash[24494]: audit 2024-09-13T14:20:43.232649+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:43 smithi177 bash[24494]: audit 2024-09-13T14:20:43.243087+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:43 smithi177 bash[24494]: cephadm 2024-09-13T14:20:43.263652+0000 mgr.y (mgr.14150) 348 : cephadm [INF] Deploying daemon prometheus.a on smithi177 2024-09-13T14:20:46.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:45 smithi099 bash[22656]: cluster 2024-09-13T14:20:44.856310+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:46.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:45 smithi099 bash[32082]: cluster 2024-09-13T14:20:44.856310+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:46.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:45 smithi177 bash[24494]: cluster 2024-09-13T14:20:44.856310+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:48.197 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:20:48.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:47 smithi099 bash[22656]: cluster 2024-09-13T14:20:46.856872+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:48.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:47 smithi099 bash[32082]: cluster 2024-09-13T14:20:46.856872+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:48.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:47 smithi177 bash[24494]: cluster 2024-09-13T14:20:46.856872+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:50.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:49 smithi099 bash[22656]: cluster 2024-09-13T14:20:48.857480+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:50.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:49 smithi099 bash[32082]: cluster 2024-09-13T14:20:48.857480+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:50.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:49 smithi177 bash[24494]: cluster 2024-09-13T14:20:48.857480+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:52.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:51 smithi177 bash[24494]: cluster 2024-09-13T14:20:50.858165+0000 mgr.y (mgr.14150) 352 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:52.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:51 smithi099 bash[22656]: cluster 2024-09-13T14:20:50.858165+0000 mgr.y (mgr.14150) 352 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:52.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:51 smithi099 bash[32082]: cluster 2024-09-13T14:20:50.858165+0000 mgr.y (mgr.14150) 352 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:54.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:53 smithi099 bash[22656]: cluster 2024-09-13T14:20:52.858758+0000 mgr.y (mgr.14150) 353 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:54.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:53 smithi099 bash[32082]: cluster 2024-09-13T14:20:52.858758+0000 mgr.y (mgr.14150) 353 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:54.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:53 smithi177 bash[24494]: cluster 2024-09-13T14:20:52.858758+0000 mgr.y (mgr.14150) 353 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:55.500 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled node-exporter update... 2024-09-13T14:20:56.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:56 smithi099 bash[22656]: cluster 2024-09-13T14:20:54.859337+0000 mgr.y (mgr.14150) 354 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:56.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:56 smithi099 bash[22656]: audit 2024-09-13T14:20:55.494850+0000 mgr.y (mgr.14150) 355 : audit [DBG] from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi099=a;smithi177=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:56.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:56 smithi099 bash[22656]: cephadm 2024-09-13T14:20:55.497938+0000 mgr.y (mgr.14150) 356 : cephadm [INF] Saving service node-exporter spec with placement smithi099=a;smithi177=b;count:2 2024-09-13T14:20:56.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:56 smithi099 bash[22656]: audit 2024-09-13T14:20:55.504930+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:56.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:56 smithi099 bash[32082]: cluster 2024-09-13T14:20:54.859337+0000 mgr.y (mgr.14150) 354 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:56.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:56 smithi099 bash[32082]: audit 2024-09-13T14:20:55.494850+0000 mgr.y (mgr.14150) 355 : audit [DBG] from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi099=a;smithi177=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:56.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:56 smithi099 bash[32082]: cephadm 2024-09-13T14:20:55.497938+0000 mgr.y (mgr.14150) 356 : cephadm [INF] Saving service node-exporter spec with placement smithi099=a;smithi177=b;count:2 2024-09-13T14:20:56.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:56 smithi099 bash[32082]: audit 2024-09-13T14:20:55.504930+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:56.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:56 smithi177 bash[24494]: cluster 2024-09-13T14:20:54.859337+0000 mgr.y (mgr.14150) 354 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:56.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:56 smithi177 bash[24494]: audit 2024-09-13T14:20:55.494850+0000 mgr.y (mgr.14150) 355 : audit [DBG] from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi099=a;smithi177=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:56.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:56 smithi177 bash[24494]: cephadm 2024-09-13T14:20:55.497938+0000 mgr.y (mgr.14150) 356 : cephadm [INF] Saving service node-exporter spec with placement smithi099=a;smithi177=b;count:2 2024-09-13T14:20:56.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:56 smithi177 bash[24494]: audit 2024-09-13T14:20:55.504930+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:20:56.892 DEBUG:teuthology.orchestra.run.smithi099:node-exporter.a> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-13T14:20:56.894 DEBUG:teuthology.orchestra.run.smithi177:node-exporter.b> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-13T14:20:56.898 INFO:tasks.cephadm:Adding alertmanager.a on smithi099 2024-09-13T14:20:56.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi099=a' 2024-09-13T14:20:58.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:20:58 smithi099 bash[22656]: cluster 2024-09-13T14:20:56.859927+0000 mgr.y (mgr.14150) 357 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:58.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:20:58 smithi099 bash[32082]: cluster 2024-09-13T14:20:56.859927+0000 mgr.y (mgr.14150) 357 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:20:58.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:20:58 smithi177 bash[24494]: cluster 2024-09-13T14:20:56.859927+0000 mgr.y (mgr.14150) 357 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:00.473 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.474 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.474 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.474 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.475 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.475 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:00 smithi099 bash[22656]: cluster 2024-09-13T14:20:58.860504+0000 mgr.y (mgr.14150) 358 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:00.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:00 smithi099 bash[32082]: cluster 2024-09-13T14:20:58.860504+0000 mgr.y (mgr.14150) 358 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:00.827 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.827 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.827 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.828 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.828 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.828 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.828 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.828 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.828 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: Started Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:00.829 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:00 smithi177 bash[24494]: cluster 2024-09-13T14:20:58.860504+0000 mgr.y (mgr.14150) 358 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:00.829 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:00.829 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:21:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:01.690 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:21:02.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[22656]: audit 2024-09-13T14:21:00.765461+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:21:02.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[22656]: audit 2024-09-13T14:21:00.774714+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:21:02.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[22656]: audit 2024-09-13T14:21:00.783269+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:21:02.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[22656]: audit 2024-09-13T14:21:00.791848+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:21:02.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[22656]: cluster 2024-09-13T14:21:00.861074+0000 mgr.y (mgr.14150) 359 : cluster [DBG] pgmap v332: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:02.069 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[22909]: ignoring --setuser ceph since I am not root 2024-09-13T14:21:02.070 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[22909]: ignoring --setgroup ceph since I am not root 2024-09-13T14:21:02.070 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[22909]: debug 2024-09-13T14:21:01.843+0000 7fc5f54b5640 1 -- 172.21.15.99:0/3159060961 <== mon.0 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561cd8772340 con 0x561cd877ec00 2024-09-13T14:21:02.070 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[22909]: debug 2024-09-13T14:21:01.947+0000 7fc5f7d0fec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:21:02.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[32082]: audit 2024-09-13T14:21:00.765461+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:21:02.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[32082]: audit 2024-09-13T14:21:00.774714+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:21:02.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[32082]: audit 2024-09-13T14:21:00.783269+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:21:02.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[32082]: audit 2024-09-13T14:21:00.791848+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:21:02.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:01 smithi099 bash[32082]: cluster 2024-09-13T14:21:00.861074+0000 mgr.y (mgr.14150) 359 : cluster [DBG] pgmap v332: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:02.107 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:01 smithi177 bash[25434]: ignoring --setuser ceph since I am not root 2024-09-13T14:21:02.107 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:01 smithi177 bash[25434]: ignoring --setgroup ceph since I am not root 2024-09-13T14:21:02.107 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:01 smithi177 bash[25434]: debug 2024-09-13T14:21:01.851+0000 7fd4b56fd640 1 -- 172.21.15.177:0/3402354700 <== mon.2 v2:172.21.15.99:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55589ef5a340 con 0x55589ef67000 2024-09-13T14:21:02.107 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:01 smithi177 bash[25434]: debug 2024-09-13T14:21:01.959+0000 7fd4b7f57ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:21:02.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:01 smithi177 bash[24494]: audit 2024-09-13T14:21:00.765461+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:21:02.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:01 smithi177 bash[24494]: audit 2024-09-13T14:21:00.774714+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:21:02.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:01 smithi177 bash[24494]: audit 2024-09-13T14:21:00.783269+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' 2024-09-13T14:21:02.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:01 smithi177 bash[24494]: audit 2024-09-13T14:21:00.791848+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:21:02.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:01 smithi177 bash[24494]: cluster 2024-09-13T14:21:00.861074+0000 mgr.y (mgr.14150) 359 : cluster [DBG] pgmap v332: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:02.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.292Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:21:02.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.292Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:21:02.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.292Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi177 (none))" 2024-09-13T14:21:02.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.292Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:21:02.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.292Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:21:02.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.294Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:21:02.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.295Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:21:02.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.296Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:21:02.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.296Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:21:02.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.297Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:21:02.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.297Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.67µs 2024-09-13T14:21:02.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.297Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:21:02.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.298Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-13T14:21:02.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.298Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.802µs wal_replay_duration=230.363µs wbl_replay_duration=88ns total_replay_duration=290.183µs 2024-09-13T14:21:02.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.298Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:21:02.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.298Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:21:02.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.298Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:21:02.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.316Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.970651ms db_storage=765ns remote_storage=3.484µs web_handler=374ns query_engine=641ns scrape=256.418µs scrape_sd=17.641µs notify=824ns notify_sd=1.28µs rules=17.427352ms tracing=6.302µs 2024-09-13T14:21:02.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.317Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:21:02.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[48613]: ts=2024-09-13T14:21:02.317Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:21:03.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:02 smithi099 bash[22656]: audit 2024-09-13T14:21:01.800653+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:21:03.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:02 smithi099 bash[22656]: cluster 2024-09-13T14:21:01.800770+0000 mon.a (mon.0) 632 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-13T14:21:03.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:02 smithi099 bash[32082]: audit 2024-09-13T14:21:01.800653+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:21:03.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:02 smithi099 bash[32082]: cluster 2024-09-13T14:21:01.800770+0000 mon.a (mon.0) 632 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-13T14:21:03.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[24494]: audit 2024-09-13T14:21:01.800653+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14150 172.21.15.99:0/4097418121' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:21:03.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:02 smithi177 bash[24494]: cluster 2024-09-13T14:21:01.800770+0000 mon.a (mon.0) 632 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-13T14:21:03.509 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:03 smithi177 bash[25434]: debug 2024-09-13T14:21:03.171+0000 7fd4b7f57ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:21:03.510 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:03 smithi177 bash[25434]: debug 2024-09-13T14:21:03.235+0000 7fd4b7f57ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:21:03.510 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:03 smithi177 bash[25434]: debug 2024-09-13T14:21:03.387+0000 7fd4b7f57ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:21:03.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:03 smithi099 bash[22909]: debug 2024-09-13T14:21:03.115+0000 7fc5f7d0fec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:21:03.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:03 smithi099 bash[22909]: debug 2024-09-13T14:21:03.179+0000 7fc5f7d0fec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:21:03.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:03 smithi099 bash[22909]: debug 2024-09-13T14:21:03.327+0000 7fc5f7d0fec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:21:04.162 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:04 smithi177 bash[25434]: debug 2024-09-13T14:21:04.027+0000 7fd4b7f57ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:21:04.162 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:04 smithi177 bash[25434]: debug 2024-09-13T14:21:04.095+0000 7fd4b7f57ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:21:04.232 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:03 smithi099 bash[22909]: debug 2024-09-13T14:21:03.943+0000 7fc5f7d0fec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:21:04.232 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:04 smithi099 bash[22909]: debug 2024-09-13T14:21:04.011+0000 7fc5f7d0fec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:21:04.233 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:04 smithi099 bash[22909]: debug 2024-09-13T14:21:04.079+0000 7fc5f7d0fec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:21:04.465 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:04 smithi177 bash[25434]: debug 2024-09-13T14:21:04.163+0000 7fd4b7f57ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:21:04.465 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:04 smithi177 bash[25434]: debug 2024-09-13T14:21:04.319+0000 7fd4b7f57ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:21:04.465 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:04 smithi177 bash[25434]: debug 2024-09-13T14:21:04.399+0000 7fd4b7f57ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:21:04.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:04 smithi099 bash[22909]: debug 2024-09-13T14:21:04.235+0000 7fc5f7d0fec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:21:04.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:04 smithi099 bash[22909]: debug 2024-09-13T14:21:04.307+0000 7fc5f7d0fec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:21:04.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:04 smithi099 bash[22909]: debug 2024-09-13T14:21:04.375+0000 7fc5f7d0fec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:21:04.760 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:04 smithi177 bash[25434]: debug 2024-09-13T14:21:04.467+0000 7fd4b7f57ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:21:04.760 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:04 smithi177 bash[25434]: debug 2024-09-13T14:21:04.695+0000 7fd4b7f57ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:21:04.902 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:04 smithi099 bash[22909]: debug 2024-09-13T14:21:04.599+0000 7fc5f7d0fec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:21:04.902 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:04 smithi099 bash[22909]: debug 2024-09-13T14:21:04.663+0000 7fc5f7d0fec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:21:04.902 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:04 smithi099 bash[22909]: debug 2024-09-13T14:21:04.827+0000 7fc5f7d0fec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:21:05.107 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:04 smithi177 bash[25434]: debug 2024-09-13T14:21:04.763+0000 7fd4b7f57ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:21:05.107 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:04 smithi177 bash[25434]: debug 2024-09-13T14:21:04.927+0000 7fd4b7f57ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:21:05.107 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:05 smithi177 bash[25434]: debug 2024-09-13T14:21:05.003+0000 7fd4b7f57ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:21:05.289 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:04 smithi099 bash[22909]: debug 2024-09-13T14:21:04.903+0000 7fc5f7d0fec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:21:05.289 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:05 smithi099 bash[22909]: debug 2024-09-13T14:21:05.131+0000 7fc5f7d0fec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:21:05.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:05 smithi099 bash[22909]: debug 2024-09-13T14:21:05.291+0000 7fc5f7d0fec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:21:05.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:05 smithi099 bash[22909]: debug 2024-09-13T14:21:05.367+0000 7fc5f7d0fec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:21:05.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:05 smithi099 bash[22909]: debug 2024-09-13T14:21:05.519+0000 7fc5f7d0fec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:21:05.607 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:05 smithi177 bash[25434]: debug 2024-09-13T14:21:05.235+0000 7fd4b7f57ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:21:05.607 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:05 smithi177 bash[25434]: debug 2024-09-13T14:21:05.395+0000 7fd4b7f57ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:21:05.607 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:05 smithi177 bash[25434]: debug 2024-09-13T14:21:05.471+0000 7fd4b7f57ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:21:06.107 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:05 smithi177 bash[25434]: debug 2024-09-13T14:21:05.627+0000 7fd4b7f57ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:21:06.555 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:06 smithi099 bash[22909]: debug 2024-09-13T14:21:06.291+0000 7fc5f7d0fec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:21:06.555 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:06 smithi099 bash[22909]: debug 2024-09-13T14:21:06.363+0000 7fc5f7d0fec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:21:06.670 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:06 smithi177 bash[25434]: debug 2024-09-13T14:21:06.403+0000 7fd4b7f57ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:21:06.670 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:06 smithi177 bash[25434]: debug 2024-09-13T14:21:06.475+0000 7fd4b7f57ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:21:06.818 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:06 smithi099 bash[22909]: debug 2024-09-13T14:21:06.555+0000 7fc5f7d0fec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:21:06.819 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:06 smithi099 bash[22909]: debug 2024-09-13T14:21:06.627+0000 7fc5f7d0fec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:21:07.057 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:06 smithi177 bash[25434]: debug 2024-09-13T14:21:06.671+0000 7fd4b7f57ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:21:07.057 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:06 smithi177 bash[25434]: debug 2024-09-13T14:21:06.743+0000 7fd4b7f57ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:21:07.249 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:06 smithi099 bash[22909]: debug 2024-09-13T14:21:06.939+0000 7fc5f7d0fec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:21:07.249 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22909]: debug 2024-09-13T14:21:07.015+0000 7fc5f7d0fec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:21:07.249 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22909]: [13/Sep/2024:14:21:07] ENGINE Bus STARTING 2024-09-13T14:21:07.249 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22909]: CherryPy Checker: 2024-09-13T14:21:07.250 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22909]: The Application mounted at '' has an empty config. 2024-09-13T14:21:07.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: cluster 2024-09-13T14:21:07.041259+0000 mon.a (mon.0) 633 : cluster [INF] Active manager daemon y restarted 2024-09-13T14:21:07.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: cluster 2024-09-13T14:21:07.041826+0000 mon.a (mon.0) 634 : cluster [INF] Activating manager daemon y 2024-09-13T14:21:07.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: cluster 2024-09-13T14:21:07.054160+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:21:07.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: cluster 2024-09-13T14:21:07.054849+0000 mon.a (mon.0) 636 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0132155s), standbys: x 2024-09-13T14:21:07.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.057488+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:21:07.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.057996+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:21:07.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.058469+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.059766+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.060280+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.060632+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.061027+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.061357+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.061740+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.062105+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.062431+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.062700+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.062986+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.063272+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:21:07.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.063452+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:21:07.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: audit 2024-09-13T14:21:07.064024+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:21:07.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[32082]: cluster 2024-09-13T14:21:07.088914+0000 mon.a (mon.0) 637 : cluster [INF] Manager daemon y is now available 2024-09-13T14:21:07.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: cluster 2024-09-13T14:21:07.041259+0000 mon.a (mon.0) 633 : cluster [INF] Active manager daemon y restarted 2024-09-13T14:21:07.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: cluster 2024-09-13T14:21:07.041826+0000 mon.a (mon.0) 634 : cluster [INF] Activating manager daemon y 2024-09-13T14:21:07.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: cluster 2024-09-13T14:21:07.054160+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:21:07.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: cluster 2024-09-13T14:21:07.054849+0000 mon.a (mon.0) 636 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0132155s), standbys: x 2024-09-13T14:21:07.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.057488+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:21:07.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.057996+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:21:07.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.058469+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:21:07.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.059766+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.060280+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.060632+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.061027+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.061357+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.061740+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.062105+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.062431+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.062700+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.062986+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.063272+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.063452+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:21:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: audit 2024-09-13T14:21:07.064024+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:21:07.254 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22656]: cluster 2024-09-13T14:21:07.088914+0000 mon.a (mon.0) 637 : cluster [INF] Manager daemon y is now available 2024-09-13T14:21:07.323 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[25434]: debug 2024-09-13T14:21:07.059+0000 7fd4b7f57ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:21:07.323 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[25434]: debug 2024-09-13T14:21:07.139+0000 7fd4b7f57ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:21:07.323 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[25434]: [13/Sep/2024:14:21:07] ENGINE Bus STARTING 2024-09-13T14:21:07.323 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[25434]: CherryPy Checker: 2024-09-13T14:21:07.323 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[25434]: The Application mounted at '' has an empty config. 2024-09-13T14:21:07.323 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[25434]: [13/Sep/2024:14:21:07] ENGINE Serving on http://:::9283 2024-09-13T14:21:07.323 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[25434]: [13/Sep/2024:14:21:07] ENGINE Bus STARTED 2024-09-13T14:21:07.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: cluster 2024-09-13T14:21:07.041259+0000 mon.a (mon.0) 633 : cluster [INF] Active manager daemon y restarted 2024-09-13T14:21:07.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: cluster 2024-09-13T14:21:07.041826+0000 mon.a (mon.0) 634 : cluster [INF] Activating manager daemon y 2024-09-13T14:21:07.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: cluster 2024-09-13T14:21:07.054160+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-13T14:21:07.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: cluster 2024-09-13T14:21:07.054849+0000 mon.a (mon.0) 636 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0132155s), standbys: x 2024-09-13T14:21:07.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.057488+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:21:07.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.057996+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:21:07.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.058469+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:21:07.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.059766+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:21:07.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.060280+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.060632+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.061027+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.061357+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.061740+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.062105+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.062431+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.062700+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.062986+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.063272+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.063452+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: audit 2024-09-13T14:21:07.064024+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:21:07.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:07 smithi177 bash[24494]: cluster 2024-09-13T14:21:07.088914+0000 mon.a (mon.0) 637 : cluster [INF] Manager daemon y is now available 2024-09-13T14:21:07.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22909]: [13/Sep/2024:14:21:07] ENGINE Serving on http://:::9283 2024-09-13T14:21:07.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:07 smithi099 bash[22909]: [13/Sep/2024:14:21:07] ENGINE Bus STARTED 2024-09-13T14:21:08.081 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled alertmanager update... 2024-09-13T14:21:08.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.108818+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:08.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.130495+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.133879+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:08.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.140510+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:21:08.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.140890+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:21:08.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.147438+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.147805+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: cluster 2024-09-13T14:21:07.164118+0000 mon.a (mon.0) 641 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: cluster 2024-09-13T14:21:07.164225+0000 mon.a (mon.0) 642 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.166981+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.167971+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.169589+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.170980+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.706996+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:07.711652+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: cluster 2024-09-13T14:21:08.063553+0000 mon.a (mon.0) 645 : cluster [DBG] mgrmap e18: y(active, since 1.0219s), standbys: x 2024-09-13T14:21:08.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:08 smithi177 bash[24494]: audit 2024-09-13T14:21:08.085511+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.108818+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.130495+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.133879+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.140510+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.140890+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.147438+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.147805+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: cluster 2024-09-13T14:21:07.164118+0000 mon.a (mon.0) 641 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: cluster 2024-09-13T14:21:07.164225+0000 mon.a (mon.0) 642 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.166981+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.167971+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.169589+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:21:08.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.170980+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.706996+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:07.711652+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: cluster 2024-09-13T14:21:08.063553+0000 mon.a (mon.0) 645 : cluster [DBG] mgrmap e18: y(active, since 1.0219s), standbys: x 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22656]: audit 2024-09-13T14:21:08.085511+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.108818+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.130495+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.133879+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.140510+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.140890+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.147438+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:21:08.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.147805+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:21:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: cluster 2024-09-13T14:21:07.164118+0000 mon.a (mon.0) 641 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:21:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: cluster 2024-09-13T14:21:07.164225+0000 mon.a (mon.0) 642 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:21:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.166981+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:21:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.167971+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:21:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.169589+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:21:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.170980+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.? 172.21.15.177:0/4254607027' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:21:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.706996+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:07.711652+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: cluster 2024-09-13T14:21:08.063553+0000 mon.a (mon.0) 645 : cluster [DBG] mgrmap e18: y(active, since 1.0219s), standbys: x 2024-09-13T14:21:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[32082]: audit 2024-09-13T14:21:08.085511+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:08.898 DEBUG:teuthology.orchestra.run.smithi099:alertmanager.a> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-13T14:21:08.900 INFO:tasks.cephadm:Adding grafana.a on smithi177 2024-09-13T14:21:08.900 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi177=a' 2024-09-13T14:21:09.097 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: debug 2024-09-13T14:21:08.843+0000 7fc59aa81640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:21:08] ENGINE Error in HTTPServer.serve 2024-09-13T14:21:09.097 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: Traceback (most recent call last): 2024-09-13T14:21:09.097 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:21:09.097 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: self._connections.run(self.expiration_interval) 2024-09-13T14:21:09.097 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:21:09.097 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: self._run(expiration_interval) 2024-09-13T14:21:09.097 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:21:09.097 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:21:09.097 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:21:09.097 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:21:09.098 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:21:09.098 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: s = self.context.wrap_socket( 2024-09-13T14:21:09.098 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:21:09.098 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: return self.sslsocket_class._create( 2024-09-13T14:21:09.098 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:21:09.098 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: self.do_handshake() 2024-09-13T14:21:09.098 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:21:09.098 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: self._sslobj.do_handshake() 2024-09-13T14:21:09.098 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:08 smithi099 bash[22909]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:21:09.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: cephadm 2024-09-13T14:21:08.076511+0000 mgr.y (mgr.24287) 2 : cephadm [INF] Saving service alertmanager spec with placement smithi099=a;count:1 2024-09-13T14:21:09.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: cluster 2024-09-13T14:21:08.079572+0000 mgr.y (mgr.24287) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:09.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: cephadm 2024-09-13T14:21:08.725700+0000 mgr.y (mgr.24287) 4 : cephadm [INF] [13/Sep/2024:14:21:08] ENGINE Bus STARTING 2024-09-13T14:21:09.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: cephadm 2024-09-13T14:21:08.846454+0000 mgr.y (mgr.24287) 5 : cephadm [INF] [13/Sep/2024:14:21:08] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:21:09.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: cephadm 2024-09-13T14:21:08.846587+0000 mgr.y (mgr.24287) 6 : cephadm [INF] [13/Sep/2024:14:21:08] ENGINE Bus STARTED 2024-09-13T14:21:09.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: cephadm 2024-09-13T14:21:08.848522+0000 mgr.y (mgr.24287) 7 : cephadm [ERR] [13/Sep/2024:14:21:08] ENGINE Error in HTTPServer.serve 2024-09-13T14:21:09.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: Traceback (most recent call last): 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: self._connections.run(self.expiration_interval) 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: self._run(expiration_interval) 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: s = self.context.wrap_socket( 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: return self.sslsocket_class._create( 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:21:09.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: self.do_handshake() 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: self._sslobj.do_handshake() 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[22656]: audit 2024-09-13T14:21:08.852381+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: cephadm 2024-09-13T14:21:08.076511+0000 mgr.y (mgr.24287) 2 : cephadm [INF] Saving service alertmanager spec with placement smithi099=a;count:1 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: cluster 2024-09-13T14:21:08.079572+0000 mgr.y (mgr.24287) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: cephadm 2024-09-13T14:21:08.725700+0000 mgr.y (mgr.24287) 4 : cephadm [INF] [13/Sep/2024:14:21:08] ENGINE Bus STARTING 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: cephadm 2024-09-13T14:21:08.846454+0000 mgr.y (mgr.24287) 5 : cephadm [INF] [13/Sep/2024:14:21:08] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: cephadm 2024-09-13T14:21:08.846587+0000 mgr.y (mgr.24287) 6 : cephadm [INF] [13/Sep/2024:14:21:08] ENGINE Bus STARTED 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: cephadm 2024-09-13T14:21:08.848522+0000 mgr.y (mgr.24287) 7 : cephadm [ERR] [13/Sep/2024:14:21:08] ENGINE Error in HTTPServer.serve 2024-09-13T14:21:09.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: Traceback (most recent call last): 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: self._connections.run(self.expiration_interval) 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: self._run(expiration_interval) 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: s = self.context.wrap_socket( 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: return self.sslsocket_class._create( 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:21:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: self.do_handshake() 2024-09-13T14:21:09.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:21:09.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: self._sslobj.do_handshake() 2024-09-13T14:21:09.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:21:09.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:09 smithi099 bash[32082]: audit 2024-09-13T14:21:08.852381+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:09.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: cephadm 2024-09-13T14:21:08.076511+0000 mgr.y (mgr.24287) 2 : cephadm [INF] Saving service alertmanager spec with placement smithi099=a;count:1 2024-09-13T14:21:09.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: cluster 2024-09-13T14:21:08.079572+0000 mgr.y (mgr.24287) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:09.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: cephadm 2024-09-13T14:21:08.725700+0000 mgr.y (mgr.24287) 4 : cephadm [INF] [13/Sep/2024:14:21:08] ENGINE Bus STARTING 2024-09-13T14:21:09.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: cephadm 2024-09-13T14:21:08.846454+0000 mgr.y (mgr.24287) 5 : cephadm [INF] [13/Sep/2024:14:21:08] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:21:09.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: cephadm 2024-09-13T14:21:08.846587+0000 mgr.y (mgr.24287) 6 : cephadm [INF] [13/Sep/2024:14:21:08] ENGINE Bus STARTED 2024-09-13T14:21:09.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: cephadm 2024-09-13T14:21:08.848522+0000 mgr.y (mgr.24287) 7 : cephadm [ERR] [13/Sep/2024:14:21:08] ENGINE Error in HTTPServer.serve 2024-09-13T14:21:09.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: Traceback (most recent call last): 2024-09-13T14:21:09.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:21:09.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: self._connections.run(self.expiration_interval) 2024-09-13T14:21:09.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:21:09.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: self._run(expiration_interval) 2024-09-13T14:21:09.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:21:09.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:21:09.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:21:09.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:21:09.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:21:09.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: s = self.context.wrap_socket( 2024-09-13T14:21:09.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:21:09.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: return self.sslsocket_class._create( 2024-09-13T14:21:09.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:21:09.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: self.do_handshake() 2024-09-13T14:21:09.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:21:09.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: self._sslobj.do_handshake() 2024-09-13T14:21:09.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:21:09.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:09 smithi177 bash[24494]: audit 2024-09-13T14:21:08.852381+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:10.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:10 smithi177 bash[24494]: cluster 2024-09-13T14:21:09.062032+0000 mgr.y (mgr.24287) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:10.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:10 smithi177 bash[24494]: cluster 2024-09-13T14:21:09.864541+0000 mon.a (mon.0) 648 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T14:21:10.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:10 smithi099 bash[22656]: cluster 2024-09-13T14:21:09.062032+0000 mgr.y (mgr.24287) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:10.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:10 smithi099 bash[22656]: cluster 2024-09-13T14:21:09.864541+0000 mon.a (mon.0) 648 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T14:21:10.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:10 smithi099 bash[32082]: cluster 2024-09-13T14:21:09.062032+0000 mgr.y (mgr.24287) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:10.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:10 smithi099 bash[32082]: cluster 2024-09-13T14:21:09.864541+0000 mon.a (mon.0) 648 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T14:21:12.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:12 smithi099 bash[22656]: cluster 2024-09-13T14:21:11.062564+0000 mgr.y (mgr.24287) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:12.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:12 smithi099 bash[22656]: cluster 2024-09-13T14:21:11.123743+0000 mon.a (mon.0) 649 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T14:21:12.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:12 smithi099 bash[32082]: cluster 2024-09-13T14:21:11.062564+0000 mgr.y (mgr.24287) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:12.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:12 smithi099 bash[32082]: cluster 2024-09-13T14:21:11.123743+0000 mon.a (mon.0) 649 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T14:21:12.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:12 smithi177 bash[24494]: cluster 2024-09-13T14:21:11.062564+0000 mgr.y (mgr.24287) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:12.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:12 smithi177 bash[24494]: cluster 2024-09-13T14:21:11.123743+0000 mon.a (mon.0) 649 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T14:21:13.428 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:21:13.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:12.781004+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:12.790735+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:12.865485+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:12.871218+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.474454+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.481277+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.483550+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.483928+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.485180+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.485502+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.487490+0000 mon.c (mon.2) 37 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.487811+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:13.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.488869+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:13.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.489163+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:13.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.492335+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:13.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.500076+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:13.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.509942+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:13.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.511350+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:13.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:13 smithi177 bash[24494]: audit 2024-09-13T14:21:13.512385+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:14.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:12.781004+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:12.790735+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:12.865485+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:12.871218+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.474454+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.481277+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.483550+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.483928+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.485180+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.485502+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.487490+0000 mon.c (mon.2) 37 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.487811+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.488869+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.489163+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.492335+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.500076+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.509942+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.511350+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[22656]: audit 2024-09-13T14:21:13.512385+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:12.781004+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:12.790735+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:12.865485+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:12.871218+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.474454+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.481277+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.483550+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.483928+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.485180+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.485502+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.487490+0000 mon.c (mon.2) 37 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.487811+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.488869+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.489163+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:21:14.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.492335+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.500076+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.509942+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:14.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.511350+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:14.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:13 smithi099 bash[32082]: audit 2024-09-13T14:21:13.512385+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:14.504 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.504 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.505 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.505 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.505 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.506 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.506 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.774 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.775 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.775 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.775 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.776 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.776 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:14.776 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:14 smithi099 systemd[1]: Started Ceph node-exporter.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:15.068 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[56349]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-09-13T14:21:15.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: cluster 2024-09-13T14:21:13.063045+0000 mgr.y (mgr.24287) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:15.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: cephadm 2024-09-13T14:21:13.493511+0000 mgr.y (mgr.24287) 11 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 4060M 2024-09-13T14:21:15.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: cephadm 2024-09-13T14:21:13.514075+0000 mgr.y (mgr.24287) 12 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:21:15.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: cephadm 2024-09-13T14:21:13.514310+0000 mgr.y (mgr.24287) 13 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:21:15.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: cephadm 2024-09-13T14:21:13.663993+0000 mgr.y (mgr.24287) 14 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:15.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: cephadm 2024-09-13T14:21:13.671623+0000 mgr.y (mgr.24287) 15 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:15.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: cephadm 2024-09-13T14:21:13.785603+0000 mgr.y (mgr.24287) 16 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: cephadm 2024-09-13T14:21:13.789698+0000 mgr.y (mgr.24287) 17 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: audit 2024-09-13T14:21:14.024536+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: audit 2024-09-13T14:21:14.031711+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: audit 2024-09-13T14:21:14.047249+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: audit 2024-09-13T14:21:14.057653+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: audit 2024-09-13T14:21:14.064050+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[32082]: audit 2024-09-13T14:21:14.779034+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: cluster 2024-09-13T14:21:13.063045+0000 mgr.y (mgr.24287) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: cephadm 2024-09-13T14:21:13.493511+0000 mgr.y (mgr.24287) 11 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 4060M 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: cephadm 2024-09-13T14:21:13.514075+0000 mgr.y (mgr.24287) 12 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:21:15.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: cephadm 2024-09-13T14:21:13.514310+0000 mgr.y (mgr.24287) 13 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: cephadm 2024-09-13T14:21:13.663993+0000 mgr.y (mgr.24287) 14 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: cephadm 2024-09-13T14:21:13.671623+0000 mgr.y (mgr.24287) 15 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: cephadm 2024-09-13T14:21:13.785603+0000 mgr.y (mgr.24287) 16 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: cephadm 2024-09-13T14:21:13.789698+0000 mgr.y (mgr.24287) 17 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: audit 2024-09-13T14:21:14.024536+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: audit 2024-09-13T14:21:14.031711+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: audit 2024-09-13T14:21:14.047249+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: audit 2024-09-13T14:21:14.057653+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: audit 2024-09-13T14:21:14.064050+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:14 smithi099 bash[22656]: audit 2024-09-13T14:21:14.779034+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: cluster 2024-09-13T14:21:13.063045+0000 mgr.y (mgr.24287) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: cephadm 2024-09-13T14:21:13.493511+0000 mgr.y (mgr.24287) 11 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 4060M 2024-09-13T14:21:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: cephadm 2024-09-13T14:21:13.514075+0000 mgr.y (mgr.24287) 12 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:21:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: cephadm 2024-09-13T14:21:13.514310+0000 mgr.y (mgr.24287) 13 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:21:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: cephadm 2024-09-13T14:21:13.663993+0000 mgr.y (mgr.24287) 14 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: cephadm 2024-09-13T14:21:13.671623+0000 mgr.y (mgr.24287) 15 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:21:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: cephadm 2024-09-13T14:21:13.785603+0000 mgr.y (mgr.24287) 16 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: cephadm 2024-09-13T14:21:13.789698+0000 mgr.y (mgr.24287) 17 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:21:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: audit 2024-09-13T14:21:14.024536+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: audit 2024-09-13T14:21:14.031711+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: audit 2024-09-13T14:21:14.047249+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: audit 2024-09-13T14:21:14.057653+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: audit 2024-09-13T14:21:14.064050+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:14 smithi177 bash[24494]: audit 2024-09-13T14:21:14.779034+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:15.491 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.491 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.491 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.491 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.492 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.492 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.492 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.492 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.492 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.493 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.493 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.493 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.493 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.493 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:21:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:21:15.493 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:15.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: v1.5.0: Pulling from prometheus/node-exporter 2024-09-13T14:21:15.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: 22b70bddd3ac: Pulling fs layer 2024-09-13T14:21:15.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: 5c12815fee55: Pulling fs layer 2024-09-13T14:21:15.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: c0e87333d380: Pulling fs layer 2024-09-13T14:21:15.783 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:15 smithi177 systemd[1]: Started Ceph node-exporter.b for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:15.783 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[49423]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-09-13T14:21:16.068 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: 5c12815fee55: Verifying Checksum 2024-09-13T14:21:16.069 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: 5c12815fee55: Download complete 2024-09-13T14:21:16.069 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: 22b70bddd3ac: Verifying Checksum 2024-09-13T14:21:16.069 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: 22b70bddd3ac: Download complete 2024-09-13T14:21:16.069 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: c0e87333d380: Verifying Checksum 2024-09-13T14:21:16.069 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: c0e87333d380: Download complete 2024-09-13T14:21:16.069 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[56349]: 22b70bddd3ac: Pull complete 2024-09-13T14:21:16.069 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:15 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:16.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: cephadm 2024-09-13T14:21:13.904675+0000 mgr.y (mgr.24287) 18 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:16.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: cephadm 2024-09-13T14:21:13.913284+0000 mgr.y (mgr.24287) 19 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:16.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: cephadm 2024-09-13T14:21:14.066702+0000 mgr.y (mgr.24287) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi099 2024-09-13T14:21:16.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: audit 2024-09-13T14:21:14.784475+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: audit 2024-09-13T14:21:14.798023+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: cephadm 2024-09-13T14:21:14.798870+0000 mgr.y (mgr.24287) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi177 2024-09-13T14:21:16.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: audit 2024-09-13T14:21:15.539204+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: audit 2024-09-13T14:21:15.551027+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: audit 2024-09-13T14:21:15.561736+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: audit 2024-09-13T14:21:15.570263+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[22656]: audit 2024-09-13T14:21:15.594569+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: cephadm 2024-09-13T14:21:13.904675+0000 mgr.y (mgr.24287) 18 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: cephadm 2024-09-13T14:21:13.913284+0000 mgr.y (mgr.24287) 19 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: cephadm 2024-09-13T14:21:14.066702+0000 mgr.y (mgr.24287) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi099 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: audit 2024-09-13T14:21:14.784475+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: audit 2024-09-13T14:21:14.798023+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: cephadm 2024-09-13T14:21:14.798870+0000 mgr.y (mgr.24287) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi177 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: audit 2024-09-13T14:21:15.539204+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: audit 2024-09-13T14:21:15.551027+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: audit 2024-09-13T14:21:15.561736+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: audit 2024-09-13T14:21:15.570263+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:15 smithi099 bash[32082]: audit 2024-09-13T14:21:15.594569+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: cephadm 2024-09-13T14:21:13.904675+0000 mgr.y (mgr.24287) 18 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: cephadm 2024-09-13T14:21:13.913284+0000 mgr.y (mgr.24287) 19 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:21:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: cephadm 2024-09-13T14:21:14.066702+0000 mgr.y (mgr.24287) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi099 2024-09-13T14:21:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: audit 2024-09-13T14:21:14.784475+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: audit 2024-09-13T14:21:14.798023+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: cephadm 2024-09-13T14:21:14.798870+0000 mgr.y (mgr.24287) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi177 2024-09-13T14:21:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: audit 2024-09-13T14:21:15.539204+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: audit 2024-09-13T14:21:15.551027+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: audit 2024-09-13T14:21:15.561736+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: audit 2024-09-13T14:21:15.570263+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:15 smithi177 bash[24494]: audit 2024-09-13T14:21:15.594569+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:17.068 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:16 smithi099 bash[56349]: 5c12815fee55: Pull complete 2024-09-13T14:21:17.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:16 smithi099 bash[32082]: cluster 2024-09-13T14:21:15.063667+0000 mgr.y (mgr.24287) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:21:17.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:16 smithi099 bash[32082]: cephadm 2024-09-13T14:21:15.620290+0000 mgr.y (mgr.24287) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi099 2024-09-13T14:21:17.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:16 smithi099 bash[22656]: cluster 2024-09-13T14:21:15.063667+0000 mgr.y (mgr.24287) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:21:17.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:16 smithi099 bash[22656]: cephadm 2024-09-13T14:21:15.620290+0000 mgr.y (mgr.24287) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi099 2024-09-13T14:21:17.107 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:16 smithi177 bash[49423]: v1.5.0: Pulling from prometheus/node-exporter 2024-09-13T14:21:17.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:16 smithi177 bash[24494]: cluster 2024-09-13T14:21:15.063667+0000 mgr.y (mgr.24287) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T14:21:17.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:16 smithi177 bash[24494]: cephadm 2024-09-13T14:21:15.620290+0000 mgr.y (mgr.24287) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi099 2024-09-13T14:21:17.379 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:17 smithi099 bash[56349]: c0e87333d380: Pull complete 2024-09-13T14:21:17.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:17 smithi177 bash[49423]: 22b70bddd3ac: Pulling fs layer 2024-09-13T14:21:17.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:17 smithi177 bash[49423]: 5c12815fee55: Pulling fs layer 2024-09-13T14:21:17.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:17 smithi177 bash[49423]: c0e87333d380: Pulling fs layer 2024-09-13T14:21:17.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:17 smithi177 bash[49423]: 5c12815fee55: Verifying Checksum 2024-09-13T14:21:17.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:17 smithi177 bash[49423]: 5c12815fee55: Download complete 2024-09-13T14:21:17.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:17 smithi177 bash[49423]: 22b70bddd3ac: Download complete 2024-09-13T14:21:17.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:17 smithi177 bash[49423]: c0e87333d380: Verifying Checksum 2024-09-13T14:21:17.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:17 smithi177 bash[49423]: c0e87333d380: Download complete 2024-09-13T14:21:17.533 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:17 smithi177 bash[49423]: 22b70bddd3ac: Pull complete 2024-09-13T14:21:17.636 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled grafana update... 2024-09-13T14:21:17.825 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:17 smithi099 bash[56349]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-09-13T14:21:17.825 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:17 smithi099 bash[56349]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T14:21:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:18 smithi099 bash[22656]: cluster 2024-09-13T14:21:17.064246+0000 mgr.y (mgr.24287) 24 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:21:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:18 smithi099 bash[22656]: audit 2024-09-13T14:21:17.133368+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:18 smithi099 bash[22656]: audit 2024-09-13T14:21:17.631198+0000 mgr.y (mgr.24287) 25 : audit [DBG] from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:18 smithi099 bash[22656]: cephadm 2024-09-13T14:21:17.634653+0000 mgr.y (mgr.24287) 26 : cephadm [INF] Saving service grafana spec with placement smithi177=a;count:1 2024-09-13T14:21:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:18 smithi099 bash[22656]: audit 2024-09-13T14:21:17.640924+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:18.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:18 smithi099 bash[32082]: cluster 2024-09-13T14:21:17.064246+0000 mgr.y (mgr.24287) 24 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:21:18.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:18 smithi099 bash[32082]: audit 2024-09-13T14:21:17.133368+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:18.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:18 smithi099 bash[32082]: audit 2024-09-13T14:21:17.631198+0000 mgr.y (mgr.24287) 25 : audit [DBG] from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:18.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:18 smithi099 bash[32082]: cephadm 2024-09-13T14:21:17.634653+0000 mgr.y (mgr.24287) 26 : cephadm [INF] Saving service grafana spec with placement smithi177=a;count:1 2024-09-13T14:21:18.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:18 smithi099 bash[32082]: audit 2024-09-13T14:21:17.640924+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:18.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:18 smithi177 bash[24494]: cluster 2024-09-13T14:21:17.064246+0000 mgr.y (mgr.24287) 24 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:21:18.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:18 smithi177 bash[24494]: audit 2024-09-13T14:21:17.133368+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:18.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:18 smithi177 bash[24494]: audit 2024-09-13T14:21:17.631198+0000 mgr.y (mgr.24287) 25 : audit [DBG] from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:21:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:18 smithi177 bash[24494]: cephadm 2024-09-13T14:21:17.634653+0000 mgr.y (mgr.24287) 26 : cephadm [INF] Saving service grafana spec with placement smithi177=a;count:1 2024-09-13T14:21:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:18 smithi177 bash[24494]: audit 2024-09-13T14:21:17.640924+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:19.022 DEBUG:teuthology.orchestra.run.smithi177:grafana.a> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@grafana.a.service 2024-09-13T14:21:19.024 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T14:21:19.024 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T14:21:19.361 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:18 smithi177 bash[49423]: 5c12815fee55: Pull complete 2024-09-13T14:21:19.746 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:19 smithi177 bash[49423]: c0e87333d380: Pull complete 2024-09-13T14:21:19.747 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:19 smithi177 bash[49423]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-09-13T14:21:19.747 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:19 smithi177 bash[49423]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T14:21:20.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[22656]: cluster 2024-09-13T14:21:19.064854+0000 mgr.y (mgr.24287) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:21:20.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[32082]: cluster 2024-09-13T14:21:19.064854+0000 mgr.y (mgr.24287) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:21:20.319 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.012Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T14:21:20.319 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.012Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T14:21:20.319 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.013Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:21:20.320 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.013Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:21:20.320 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.013Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:21:20.320 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.013Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:21:20.320 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:21:20.320 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:21:20.320 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:21:20.320 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:21:20.320 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:21:20.320 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:21:20.320 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:21:20.321 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:21:20.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:21:20.323 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:21:20.323 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:21:20.323 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:21:20.323 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:21:20.323 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:21:20.323 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:21:20.323 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:21:20.323 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:21:20.323 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:21:20.323 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:20 smithi099 bash[56349]: ts=2024-09-13T14:21:20.014Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:21:20.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:20 smithi177 bash[24494]: cluster 2024-09-13T14:21:19.064854+0000 mgr.y (mgr.24287) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T14:21:21.608 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.279Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T14:21:21.608 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.279Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T14:21:21.608 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.280Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:21:21.608 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.280Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:21:21.608 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.280Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.280Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:21:21.609 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:21:21.610 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:21:21.610 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:21:21.610 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:21:21.610 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:21:21.610 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:21:21.610 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:21:21.610 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:21:21.610 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:21:21.610 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:21:21.610 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:21:21.611 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:21:21.612 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:21:21 smithi177 bash[49423]: ts=2024-09-13T14:21:21.281Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:21:22.542 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:22 smithi099 bash[22656]: cluster 2024-09-13T14:21:21.065443+0000 mgr.y (mgr.24287) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:21:22.542 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:22 smithi099 bash[32082]: cluster 2024-09-13T14:21:21.065443+0000 mgr.y (mgr.24287) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:21:22.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:22 smithi177 bash[24494]: cluster 2024-09-13T14:21:21.065443+0000 mgr.y (mgr.24287) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:21:23.420 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:23 smithi099 bash[22656]: audit 2024-09-13T14:21:22.134912+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:23.420 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:23 smithi099 bash[32082]: audit 2024-09-13T14:21:22.134912+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:23.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:23 smithi177 bash[24494]: audit 2024-09-13T14:21:22.134912+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:23.788 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:24.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:24 smithi099 bash[22656]: cluster 2024-09-13T14:21:23.066105+0000 mgr.y (mgr.24287) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:21:24.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:24 smithi099 bash[32082]: cluster 2024-09-13T14:21:23.066105+0000 mgr.y (mgr.24287) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:21:24.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:24 smithi177 bash[24494]: cluster 2024-09-13T14:21:23.066105+0000 mgr.y (mgr.24287) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:21:25.857 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:25 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:21:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:21:26.297 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:26 smithi099 bash[22656]: cluster 2024-09-13T14:21:25.066687+0000 mgr.y (mgr.24287) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:21:26.297 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:26 smithi099 bash[32082]: cluster 2024-09-13T14:21:25.066687+0000 mgr.y (mgr.24287) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:21:26.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:26 smithi177 bash[24494]: cluster 2024-09-13T14:21:25.066687+0000 mgr.y (mgr.24287) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T14:21:28.210 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.211 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.211 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.211 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:28 smithi099 bash[22656]: cluster 2024-09-13T14:21:27.067275+0000 mgr.y (mgr.24287) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:28.212 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.212 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.213 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.213 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.213 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.213 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:28 smithi099 bash[32082]: cluster 2024-09-13T14:21:27.067275+0000 mgr.y (mgr.24287) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:28.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.569 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.569 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.570 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.570 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.571 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.571 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.571 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.571 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:28.571 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:28 smithi099 systemd[1]: Started Ceph alertmanager.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:21:28.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:28 smithi177 bash[24494]: cluster 2024-09-13T14:21:27.067275+0000 mgr.y (mgr.24287) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:29.369 INFO:teuthology.orchestra.run.smithi099.stdout:[client.0] 2024-09-13T14:21:29.369 INFO:teuthology.orchestra.run.smithi099.stdout: key = AQBpSuRml83dFRAAv615p5S0RqnlkppmBBAD9Q== 2024-09-13T14:21:29.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[22656]: audit 2024-09-13T14:21:28.494261+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[22656]: audit 2024-09-13T14:21:28.500884+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[22656]: audit 2024-09-13T14:21:28.505531+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[22656]: audit 2024-09-13T14:21:28.509700+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[22656]: audit 2024-09-13T14:21:28.539033+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:29.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[22656]: audit 2024-09-13T14:21:29.366593+0000 mon.a (mon.0) 682 : audit [INF] from='client.? 172.21.15.99:0/3484311730' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:21:29.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[22656]: audit 2024-09-13T14:21:29.374696+0000 mon.a (mon.0) 683 : audit [INF] from='client.? 172.21.15.99:0/3484311730' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:21:29.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[32082]: audit 2024-09-13T14:21:28.494261+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[32082]: audit 2024-09-13T14:21:28.500884+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[32082]: audit 2024-09-13T14:21:28.505531+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[32082]: audit 2024-09-13T14:21:28.509700+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[32082]: audit 2024-09-13T14:21:28.539033+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:29.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[32082]: audit 2024-09-13T14:21:29.366593+0000 mon.a (mon.0) 682 : audit [INF] from='client.? 172.21.15.99:0/3484311730' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:21:29.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:29 smithi099 bash[32082]: audit 2024-09-13T14:21:29.374696+0000 mon.a (mon.0) 683 : audit [INF] from='client.? 172.21.15.99:0/3484311730' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:21:29.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:29 smithi177 bash[24494]: audit 2024-09-13T14:21:28.494261+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:29 smithi177 bash[24494]: audit 2024-09-13T14:21:28.500884+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:29 smithi177 bash[24494]: audit 2024-09-13T14:21:28.505531+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:29 smithi177 bash[24494]: audit 2024-09-13T14:21:28.509700+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:29.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:29 smithi177 bash[24494]: audit 2024-09-13T14:21:28.539033+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:29.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:29 smithi177 bash[24494]: audit 2024-09-13T14:21:29.366593+0000 mon.a (mon.0) 682 : audit [INF] from='client.? 172.21.15.99:0/3484311730' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:21:29.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:29 smithi177 bash[24494]: audit 2024-09-13T14:21:29.374696+0000 mon.a (mon.0) 683 : audit [INF] from='client.? 172.21.15.99:0/3484311730' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:21:30.742 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:21:30.742 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T14:21:30.742 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T14:21:30.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:30 smithi099 bash[22656]: cluster 2024-09-13T14:21:29.067750+0000 mgr.y (mgr.24287) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:30.762 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:30 smithi099 bash[32082]: cluster 2024-09-13T14:21:29.067750+0000 mgr.y (mgr.24287) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:30.765 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T14:21:30.776 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:30 smithi177 bash[24494]: cluster 2024-09-13T14:21:29.067750+0000 mgr.y (mgr.24287) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:31.068 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:30 smithi099 bash[56811]: ts=2024-09-13T14:21:30.927Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T14:21:31.069 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:30 smithi099 bash[56811]: ts=2024-09-13T14:21:30.927Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T14:21:31.069 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:30 smithi099 bash[56811]: ts=2024-09-13T14:21:30.929Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.99 port=9094 2024-09-13T14:21:31.069 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:30 smithi099 bash[56811]: ts=2024-09-13T14:21:30.930Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:21:31.069 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:30 smithi099 bash[56811]: ts=2024-09-13T14:21:30.960Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:21:31.069 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:30 smithi099 bash[56811]: ts=2024-09-13T14:21:30.961Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:21:31.069 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:30 smithi099 bash[56811]: ts=2024-09-13T14:21:30.964Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:21:31.069 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:30 smithi099 bash[56811]: ts=2024-09-13T14:21:30.964Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:21:32.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:32 smithi099 bash[22656]: cluster 2024-09-13T14:21:31.068322+0000 mgr.y (mgr.24287) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:32.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:32 smithi099 bash[22656]: audit 2024-09-13T14:21:32.148055+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:32.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:32 smithi099 bash[32082]: cluster 2024-09-13T14:21:31.068322+0000 mgr.y (mgr.24287) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:32.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:32 smithi099 bash[32082]: audit 2024-09-13T14:21:32.148055+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:32.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:32 smithi177 bash[24494]: cluster 2024-09-13T14:21:31.068322+0000 mgr.y (mgr.24287) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:32.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:32 smithi177 bash[24494]: audit 2024-09-13T14:21:32.148055+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:33.318 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:32 smithi099 bash[56811]: ts=2024-09-13T14:21:32.930Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00050596s 2024-09-13T14:21:34.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:34 smithi099 bash[22656]: cluster 2024-09-13T14:21:33.068883+0000 mgr.y (mgr.24287) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:34.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:34 smithi099 bash[22656]: audit 2024-09-13T14:21:33.864081+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:34.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:34 smithi099 bash[22656]: audit 2024-09-13T14:21:33.878109+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:34.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:34 smithi099 bash[32082]: cluster 2024-09-13T14:21:33.068883+0000 mgr.y (mgr.24287) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:34.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:34 smithi099 bash[32082]: audit 2024-09-13T14:21:33.864081+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:34.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:34 smithi099 bash[32082]: audit 2024-09-13T14:21:33.878109+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:34.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:34 smithi177 bash[24494]: cluster 2024-09-13T14:21:33.068883+0000 mgr.y (mgr.24287) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:34.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:34 smithi177 bash[24494]: audit 2024-09-13T14:21:33.864081+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:34.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:34 smithi177 bash[24494]: audit 2024-09-13T14:21:33.878109+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:35.356 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:21:35.857 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:35 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:21:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:21:36.310 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:35 smithi099 bash[22909]: [13/Sep/2024:14:21:35] ENGINE Bus STOPPING 2024-09-13T14:21:36.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22909]: [13/Sep/2024:14:21:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:21:36.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22909]: [13/Sep/2024:14:21:36] ENGINE Bus STOPPED 2024-09-13T14:21:36.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22909]: [13/Sep/2024:14:21:36] ENGINE Bus STARTING 2024-09-13T14:21:36.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22909]: [13/Sep/2024:14:21:36] ENGINE Serving on http://:::9283 2024-09-13T14:21:36.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22909]: [13/Sep/2024:14:21:36] ENGINE Bus STARTED 2024-09-13T14:21:36.607 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:21:36 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:21:37.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22656]: cluster 2024-09-13T14:21:35.069479+0000 mgr.y (mgr.24287) 35 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:37.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22656]: audit 2024-09-13T14:21:35.796417+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22656]: audit 2024-09-13T14:21:35.802924+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22656]: audit 2024-09-13T14:21:35.804507+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:37.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22656]: audit 2024-09-13T14:21:35.805501+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:37.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22656]: audit 2024-09-13T14:21:35.810726+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22656]: audit 2024-09-13T14:21:35.888556+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22656]: audit 2024-09-13T14:21:35.895800+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22656]: audit 2024-09-13T14:21:35.901625+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:21:37.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[22656]: audit 2024-09-13T14:21:35.907843+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[32082]: cluster 2024-09-13T14:21:35.069479+0000 mgr.y (mgr.24287) 35 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:37.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[32082]: audit 2024-09-13T14:21:35.796417+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[32082]: audit 2024-09-13T14:21:35.802924+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[32082]: audit 2024-09-13T14:21:35.804507+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:37.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[32082]: audit 2024-09-13T14:21:35.805501+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:37.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[32082]: audit 2024-09-13T14:21:35.810726+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[32082]: audit 2024-09-13T14:21:35.888556+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[32082]: audit 2024-09-13T14:21:35.895800+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[32082]: audit 2024-09-13T14:21:35.901625+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:21:37.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:36 smithi099 bash[32082]: audit 2024-09-13T14:21:35.907843+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:36 smithi177 bash[24494]: cluster 2024-09-13T14:21:35.069479+0000 mgr.y (mgr.24287) 35 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:36 smithi177 bash[24494]: audit 2024-09-13T14:21:35.796417+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:36 smithi177 bash[24494]: audit 2024-09-13T14:21:35.802924+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:36 smithi177 bash[24494]: audit 2024-09-13T14:21:35.804507+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:36 smithi177 bash[24494]: audit 2024-09-13T14:21:35.805501+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:36 smithi177 bash[24494]: audit 2024-09-13T14:21:35.810726+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:36 smithi177 bash[24494]: audit 2024-09-13T14:21:35.888556+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:36 smithi177 bash[24494]: audit 2024-09-13T14:21:35.895800+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:36 smithi177 bash[24494]: audit 2024-09-13T14:21:35.901625+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:21:37.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:36 smithi177 bash[24494]: audit 2024-09-13T14:21:35.907843+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:21:38.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:37 smithi099 bash[22656]: audit 2024-09-13T14:21:35.902696+0000 mgr.y (mgr.24287) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:21:38.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:37 smithi099 bash[22656]: cephadm 2024-09-13T14:21:35.927153+0000 mgr.y (mgr.24287) 37 : cephadm [INF] Deploying daemon grafana.a on smithi177 2024-09-13T14:21:38.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:37 smithi099 bash[22656]: audit 2024-09-13T14:21:37.135176+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:38.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:37 smithi099 bash[32082]: audit 2024-09-13T14:21:35.902696+0000 mgr.y (mgr.24287) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:21:38.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:37 smithi099 bash[32082]: cephadm 2024-09-13T14:21:35.927153+0000 mgr.y (mgr.24287) 37 : cephadm [INF] Deploying daemon grafana.a on smithi177 2024-09-13T14:21:38.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:37 smithi099 bash[32082]: audit 2024-09-13T14:21:37.135176+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:38.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:37 smithi177 bash[24494]: audit 2024-09-13T14:21:35.902696+0000 mgr.y (mgr.24287) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:21:38.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:37 smithi177 bash[24494]: cephadm 2024-09-13T14:21:35.927153+0000 mgr.y (mgr.24287) 37 : cephadm [INF] Deploying daemon grafana.a on smithi177 2024-09-13T14:21:38.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:37 smithi177 bash[24494]: audit 2024-09-13T14:21:37.135176+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:39.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:38 smithi099 bash[22656]: cluster 2024-09-13T14:21:37.070053+0000 mgr.y (mgr.24287) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:39.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:38 smithi099 bash[32082]: cluster 2024-09-13T14:21:37.070053+0000 mgr.y (mgr.24287) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:39.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:38 smithi177 bash[24494]: cluster 2024-09-13T14:21:37.070053+0000 mgr.y (mgr.24287) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:40.323 INFO:teuthology.orchestra.run.smithi177.stdout:[client.1] 2024-09-13T14:21:40.323 INFO:teuthology.orchestra.run.smithi177.stdout: key = AQB0SuRmuaYhExAAaRDY1Ns6m9hbqdFur+zFSA== 2024-09-13T14:21:41.068 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:21:40 smithi099 bash[56811]: ts=2024-09-13T14:21:40.932Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002248328s 2024-09-13T14:21:41.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:40 smithi099 bash[22656]: cluster 2024-09-13T14:21:39.070657+0000 mgr.y (mgr.24287) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:41.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:40 smithi099 bash[22656]: audit 2024-09-13T14:21:40.318824+0000 mon.b (mon.1) 30 : audit [INF] from='client.? 172.21.15.177:0/1748380549' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:21:41.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:40 smithi099 bash[22656]: audit 2024-09-13T14:21:40.320657+0000 mon.a (mon.0) 693 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:21:41.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:40 smithi099 bash[22656]: audit 2024-09-13T14:21:40.328687+0000 mon.a (mon.0) 694 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:21:41.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:40 smithi099 bash[32082]: cluster 2024-09-13T14:21:39.070657+0000 mgr.y (mgr.24287) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:41.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:40 smithi099 bash[32082]: audit 2024-09-13T14:21:40.318824+0000 mon.b (mon.1) 30 : audit [INF] from='client.? 172.21.15.177:0/1748380549' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:21:41.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:40 smithi099 bash[32082]: audit 2024-09-13T14:21:40.320657+0000 mon.a (mon.0) 693 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:21:41.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:40 smithi099 bash[32082]: audit 2024-09-13T14:21:40.328687+0000 mon.a (mon.0) 694 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:21:41.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:40 smithi177 bash[24494]: cluster 2024-09-13T14:21:39.070657+0000 mgr.y (mgr.24287) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:41.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:40 smithi177 bash[24494]: audit 2024-09-13T14:21:40.318824+0000 mon.b (mon.1) 30 : audit [INF] from='client.? 172.21.15.177:0/1748380549' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:21:41.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:40 smithi177 bash[24494]: audit 2024-09-13T14:21:40.320657+0000 mon.a (mon.0) 693 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:21:41.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:40 smithi177 bash[24494]: audit 2024-09-13T14:21:40.328687+0000 mon.a (mon.0) 694 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:21:42.538 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-09-13T14:21:42.538 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-13T14:21:42.538 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-13T14:21:42.602 INFO:tasks.cephadm:Creating RBD pool 2024-09-13T14:21:42.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-13T14:21:43.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:42 smithi099 bash[22656]: cluster 2024-09-13T14:21:41.071216+0000 mgr.y (mgr.24287) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:43.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:42 smithi099 bash[32082]: cluster 2024-09-13T14:21:41.071216+0000 mgr.y (mgr.24287) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:43.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:42 smithi177 bash[24494]: cluster 2024-09-13T14:21:41.071216+0000 mgr.y (mgr.24287) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:45.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:44 smithi177 bash[24494]: cluster 2024-09-13T14:21:43.071783+0000 mgr.y (mgr.24287) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:45.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:44 smithi099 bash[22656]: cluster 2024-09-13T14:21:43.071783+0000 mgr.y (mgr.24287) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:45.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:44 smithi099 bash[32082]: cluster 2024-09-13T14:21:43.071783+0000 mgr.y (mgr.24287) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:45.857 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:45 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:21:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:21:47.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:46 smithi177 bash[24494]: cluster 2024-09-13T14:21:45.072396+0000 mgr.y (mgr.24287) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:47.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:46 smithi099 bash[22656]: cluster 2024-09-13T14:21:45.072396+0000 mgr.y (mgr.24287) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:47.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:46 smithi099 bash[32082]: cluster 2024-09-13T14:21:45.072396+0000 mgr.y (mgr.24287) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:47.385 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:49.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:48 smithi099 bash[22656]: cluster 2024-09-13T14:21:47.072973+0000 mgr.y (mgr.24287) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:49.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:48 smithi099 bash[32082]: cluster 2024-09-13T14:21:47.072973+0000 mgr.y (mgr.24287) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:49.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:48 smithi177 bash[24494]: cluster 2024-09-13T14:21:47.072973+0000 mgr.y (mgr.24287) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:51.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:50 smithi099 bash[22656]: cluster 2024-09-13T14:21:49.073593+0000 mgr.y (mgr.24287) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:51.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:50 smithi099 bash[32082]: cluster 2024-09-13T14:21:49.073593+0000 mgr.y (mgr.24287) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:51.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:50 smithi177 bash[24494]: cluster 2024-09-13T14:21:49.073593+0000 mgr.y (mgr.24287) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:51.849 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'rbd' created 2024-09-13T14:21:52.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:51 smithi099 bash[22656]: audit 2024-09-13T14:21:51.107715+0000 mon.a (mon.0) 695 : audit [INF] from='client.? 172.21.15.99:0/2066336375' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:21:52.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:51 smithi099 bash[32082]: audit 2024-09-13T14:21:51.107715+0000 mon.a (mon.0) 695 : audit [INF] from='client.? 172.21.15.99:0/2066336375' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:21:52.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:51 smithi177 bash[24494]: audit 2024-09-13T14:21:51.107715+0000 mon.a (mon.0) 695 : audit [INF] from='client.? 172.21.15.99:0/2066336375' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:21:52.667 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-13T14:21:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:52 smithi099 bash[22656]: cluster 2024-09-13T14:21:51.074171+0000 mgr.y (mgr.24287) 45 : cluster [DBG] pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:52 smithi099 bash[22656]: audit 2024-09-13T14:21:51.853652+0000 mon.a (mon.0) 696 : audit [INF] from='client.? 172.21.15.99:0/2066336375' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:21:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:52 smithi099 bash[22656]: cluster 2024-09-13T14:21:51.853821+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:21:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:52 smithi099 bash[22656]: audit 2024-09-13T14:21:52.136034+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:53.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:52 smithi099 bash[32082]: cluster 2024-09-13T14:21:51.074171+0000 mgr.y (mgr.24287) 45 : cluster [DBG] pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:53.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:52 smithi099 bash[32082]: audit 2024-09-13T14:21:51.853652+0000 mon.a (mon.0) 696 : audit [INF] from='client.? 172.21.15.99:0/2066336375' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:21:53.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:52 smithi099 bash[32082]: cluster 2024-09-13T14:21:51.853821+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:21:53.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:52 smithi099 bash[32082]: audit 2024-09-13T14:21:52.136034+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:52 smithi177 bash[24494]: cluster 2024-09-13T14:21:51.074171+0000 mgr.y (mgr.24287) 45 : cluster [DBG] pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:52 smithi177 bash[24494]: audit 2024-09-13T14:21:51.853652+0000 mon.a (mon.0) 696 : audit [INF] from='client.? 172.21.15.99:0/2066336375' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:21:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:52 smithi177 bash[24494]: cluster 2024-09-13T14:21:51.853821+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-13T14:21:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:52 smithi177 bash[24494]: audit 2024-09-13T14:21:52.136034+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:54.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:53 smithi177 bash[24494]: cluster 2024-09-13T14:21:52.852608+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:21:54.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:53 smithi099 bash[22656]: cluster 2024-09-13T14:21:52.852608+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:21:54.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:53 smithi099 bash[32082]: cluster 2024-09-13T14:21:52.852608+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:21:55.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:54 smithi177 bash[24494]: cluster 2024-09-13T14:21:53.074946+0000 mgr.y (mgr.24287) 46 : cluster [DBG] pgmap v28: 9 pgs: 1 creating+activating, 6 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:55.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:54 smithi177 bash[24494]: cluster 2024-09-13T14:21:53.857981+0000 mon.a (mon.0) 699 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:21:55.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:54 smithi177 bash[24494]: cluster 2024-09-13T14:21:53.876958+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:21:55.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:54 smithi099 bash[22656]: cluster 2024-09-13T14:21:53.074946+0000 mgr.y (mgr.24287) 46 : cluster [DBG] pgmap v28: 9 pgs: 1 creating+activating, 6 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:55.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:54 smithi099 bash[22656]: cluster 2024-09-13T14:21:53.857981+0000 mon.a (mon.0) 699 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:21:55.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:54 smithi099 bash[22656]: cluster 2024-09-13T14:21:53.876958+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:21:55.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:54 smithi099 bash[32082]: cluster 2024-09-13T14:21:53.074946+0000 mgr.y (mgr.24287) 46 : cluster [DBG] pgmap v28: 9 pgs: 1 creating+activating, 6 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:55.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:54 smithi099 bash[32082]: cluster 2024-09-13T14:21:53.857981+0000 mon.a (mon.0) 699 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:21:55.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:54 smithi099 bash[32082]: cluster 2024-09-13T14:21:53.876958+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:21:55.857 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:21:55 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:21:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:21:56.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:55 smithi099 bash[22656]: cluster 2024-09-13T14:21:55.075588+0000 mgr.y (mgr.24287) 47 : cluster [DBG] pgmap v30: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:56.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:55 smithi099 bash[32082]: cluster 2024-09-13T14:21:55.075588+0000 mgr.y (mgr.24287) 47 : cluster [DBG] pgmap v30: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:55 smithi177 bash[24494]: cluster 2024-09-13T14:21:55.075588+0000 mgr.y (mgr.24287) 47 : cluster [DBG] pgmap v30: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:57.448 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:58.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:21:58 smithi099 bash[22656]: cluster 2024-09-13T14:21:57.076225+0000 mgr.y (mgr.24287) 48 : cluster [DBG] pgmap v31: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:58.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:21:58 smithi099 bash[32082]: cluster 2024-09-13T14:21:57.076225+0000 mgr.y (mgr.24287) 48 : cluster [DBG] pgmap v31: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:21:58.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:21:58 smithi177 bash[24494]: cluster 2024-09-13T14:21:57.076225+0000 mgr.y (mgr.24287) 48 : cluster [DBG] pgmap v31: 9 pgs: 2 creating+activating, 7 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:00.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:00 smithi099 bash[22656]: cluster 2024-09-13T14:21:59.076974+0000 mgr.y (mgr.24287) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:00.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:00 smithi099 bash[32082]: cluster 2024-09-13T14:21:59.076974+0000 mgr.y (mgr.24287) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:00.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:00 smithi177 bash[24494]: cluster 2024-09-13T14:21:59.076974+0000 mgr.y (mgr.24287) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:01 smithi177 bash[24494]: audit 2024-09-13T14:22:00.833197+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.99:0/157201850' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:22:01.490 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:01 smithi177 bash[24494]: audit 2024-09-13T14:22:00.833534+0000 mon.a (mon.0) 701 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:22:01.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:01 smithi099 bash[22656]: audit 2024-09-13T14:22:00.833197+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.99:0/157201850' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:22:01.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:01 smithi099 bash[22656]: audit 2024-09-13T14:22:00.833534+0000 mon.a (mon.0) 701 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:22:01.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:01 smithi099 bash[32082]: audit 2024-09-13T14:22:00.833197+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.99:0/157201850' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:22:01.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:01 smithi099 bash[32082]: audit 2024-09-13T14:22:00.833534+0000 mon.a (mon.0) 701 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:22:02.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:02 smithi177 bash[24494]: cluster 2024-09-13T14:22:01.077584+0000 mgr.y (mgr.24287) 50 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:02.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:02 smithi177 bash[24494]: audit 2024-09-13T14:22:01.151597+0000 mon.a (mon.0) 702 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:22:02.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:02 smithi177 bash[24494]: cluster 2024-09-13T14:22:01.151716+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:22:02.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:02 smithi099 bash[22656]: cluster 2024-09-13T14:22:01.077584+0000 mgr.y (mgr.24287) 50 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:02.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:02 smithi099 bash[22656]: audit 2024-09-13T14:22:01.151597+0000 mon.a (mon.0) 702 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:22:02.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:02 smithi099 bash[22656]: cluster 2024-09-13T14:22:01.151716+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:22:02.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:02 smithi099 bash[32082]: cluster 2024-09-13T14:22:01.077584+0000 mgr.y (mgr.24287) 50 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:02.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:02 smithi099 bash[32082]: audit 2024-09-13T14:22:01.151597+0000 mon.a (mon.0) 702 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:22:02.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:02 smithi099 bash[32082]: cluster 2024-09-13T14:22:01.151716+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:22:03.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:03 smithi177 bash[24494]: cluster 2024-09-13T14:22:02.153834+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:22:03.431 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:03 smithi099 bash[22656]: cluster 2024-09-13T14:22:02.153834+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:22:03.432 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:03 smithi099 bash[32082]: cluster 2024-09-13T14:22:02.153834+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:22:04.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-13T14:22:04.161 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.162 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.162 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.162 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.163 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.163 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.163 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.163 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.164 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.164 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.164 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:04 smithi099 bash[22656]: cluster 2024-09-13T14:22:03.078415+0000 mgr.y (mgr.24287) 51 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:04 smithi099 bash[22656]: cluster 2024-09-13T14:22:03.156948+0000 mon.a (mon.0) 705 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:22:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:04 smithi099 bash[22656]: cluster 2024-09-13T14:22:03.157000+0000 mon.a (mon.0) 706 : cluster [INF] Cluster is now healthy 2024-09-13T14:22:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:04 smithi099 bash[22656]: cluster 2024-09-13T14:22:03.175128+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:22:04.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:04 smithi099 bash[32082]: cluster 2024-09-13T14:22:03.078415+0000 mgr.y (mgr.24287) 51 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:04.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:04 smithi099 bash[32082]: cluster 2024-09-13T14:22:03.156948+0000 mon.a (mon.0) 705 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:22:04.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:04 smithi099 bash[32082]: cluster 2024-09-13T14:22:03.157000+0000 mon.a (mon.0) 706 : cluster [INF] Cluster is now healthy 2024-09-13T14:22:04.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:04 smithi099 bash[32082]: cluster 2024-09-13T14:22:03.175128+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:22:04.544 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.545 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.545 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:04 smithi177 bash[24494]: cluster 2024-09-13T14:22:03.078415+0000 mgr.y (mgr.24287) 51 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:04.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:04 smithi177 bash[24494]: cluster 2024-09-13T14:22:03.156948+0000 mon.a (mon.0) 705 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:22:04.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:04 smithi177 bash[24494]: cluster 2024-09-13T14:22:03.157000+0000 mon.a (mon.0) 706 : cluster [INF] Cluster is now healthy 2024-09-13T14:22:04.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:04 smithi177 bash[24494]: cluster 2024-09-13T14:22:03.175128+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:22:04.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.546 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.546 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.546 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: Started Ceph grafana.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:04.547 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.547 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:04.547 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:22:04 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:22:05.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[22656]: audit 2024-09-13T14:22:04.430336+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[22656]: audit 2024-09-13T14:22:04.442435+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[22656]: audit 2024-09-13T14:22:04.451787+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[22656]: audit 2024-09-13T14:22:04.461973+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[22656]: audit 2024-09-13T14:22:04.518180+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:05.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[22656]: audit 2024-09-13T14:22:04.798288+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[22656]: audit 2024-09-13T14:22:04.804088+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[32082]: audit 2024-09-13T14:22:04.430336+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[32082]: audit 2024-09-13T14:22:04.442435+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[32082]: audit 2024-09-13T14:22:04.451787+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[32082]: audit 2024-09-13T14:22:04.461973+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[32082]: audit 2024-09-13T14:22:04.518180+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:05.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[32082]: audit 2024-09-13T14:22:04.798288+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:05 smithi099 bash[32082]: audit 2024-09-13T14:22:04.804088+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.857 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:22:05 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:22:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:22:05.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:05 smithi177 bash[24494]: audit 2024-09-13T14:22:04.430336+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:05 smithi177 bash[24494]: audit 2024-09-13T14:22:04.442435+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:05 smithi177 bash[24494]: audit 2024-09-13T14:22:04.451787+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:05 smithi177 bash[24494]: audit 2024-09-13T14:22:04.461973+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:05 smithi177 bash[24494]: audit 2024-09-13T14:22:04.518180+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:05.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:05 smithi177 bash[24494]: audit 2024-09-13T14:22:04.798288+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:05.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:05 smithi177 bash[24494]: audit 2024-09-13T14:22:04.804088+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:06.565 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.297762885Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T14:22:06.565 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.297961522Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T14:22:06.565 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.297983217Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:22:06.565 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.297991991Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:22:06.565 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.297999362Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:22:06.565 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.298006629Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.298013467Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.298020316Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.298027351Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.298034653Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.298041561Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.298047978Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.298057177Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=settings t=2024-09-13T14:22:06.29806556Z level=info msg="App mode production" 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=sqlstore t=2024-09-13T14:22:06.298135977Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=sqlstore t=2024-09-13T14:22:06.298166698Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.299241896Z level=info msg="Starting DB migrations" 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.299327771Z level=info msg="Executing migration" id="create migration_log table" 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.36725679Z level=info msg="Executing migration" id="create user table" 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.418009717Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-13T14:22:06.566 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.468455342Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-13T14:22:06.567 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.519141152Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-13T14:22:06.567 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.569575614Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-13T14:22:06.826 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[24494]: cluster 2024-09-13T14:22:05.078998+0000 mgr.y (mgr.24287) 52 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-13T14:22:06.826 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.62016673Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-13T14:22:06.826 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.712824293Z level=info msg="Executing migration" id="create user table v2" 2024-09-13T14:22:06.826 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.755400209Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-13T14:22:06.826 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.780602547Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-13T14:22:06.827 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.814291613Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-13T14:22:06.827 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.83124929Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-13T14:22:07.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:06 smithi099 bash[22656]: cluster 2024-09-13T14:22:05.078998+0000 mgr.y (mgr.24287) 52 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-13T14:22:07.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:06 smithi099 bash[32082]: cluster 2024-09-13T14:22:05.078998+0000 mgr.y (mgr.24287) 52 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-13T14:22:07.079 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.864932711Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-13T14:22:07.079 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.881786557Z level=info msg="Executing migration" id="Update user table charset" 2024-09-13T14:22:07.079 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.907016188Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-13T14:22:07.079 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.932331079Z level=info msg="Executing migration" id="Add missing user data" 2024-09-13T14:22:07.079 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.957523035Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-13T14:22:07.079 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:06 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:06.983132068Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-13T14:22:07.080 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.008384665Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-13T14:22:07.080 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.033450641Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-13T14:22:07.080 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.058840801Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-13T14:22:07.080 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.084190968Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-13T14:22:07.340 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.109480728Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-13T14:22:07.340 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.134693835Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-13T14:22:07.340 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.176696922Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-13T14:22:07.340 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.201901788Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-13T14:22:07.340 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.22716986Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-13T14:22:07.340 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.252425946Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-13T14:22:07.340 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.294451536Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-13T14:22:07.341 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.319692905Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-13T14:22:07.341 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.344998812Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-13T14:22:07.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.370253004Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-13T14:22:07.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.412168547Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-13T14:22:07.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.437398549Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-13T14:22:07.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.462695963Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-13T14:22:07.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.487978001Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-13T14:22:07.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.51337716Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-13T14:22:07.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.555239325Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-13T14:22:07.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.580587557Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-13T14:22:07.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.605803587Z level=info msg="Executing migration" id="create star table" 2024-09-13T14:22:07.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[24494]: audit 2024-09-13T14:22:07.143082+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:07.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[24494]: audit 2024-09-13T14:22:07.168025+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:07.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.631180346Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-13T14:22:07.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.656473477Z level=info msg="Executing migration" id="create org table v1" 2024-09-13T14:22:07.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.681639588Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-13T14:22:07.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.706975821Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-13T14:22:07.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.740662175Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-13T14:22:07.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.790957876Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-13T14:22:07.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.850073387Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-13T14:22:08.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:07 smithi099 bash[22656]: audit 2024-09-13T14:22:07.143082+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:08.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:07 smithi099 bash[22656]: audit 2024-09-13T14:22:07.168025+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:08.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:07 smithi099 bash[32082]: audit 2024-09-13T14:22:07.143082+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:08.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:07 smithi099 bash[32082]: audit 2024-09-13T14:22:07.168025+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:08.148 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.892064465Z level=info msg="Executing migration" id="Update org table charset" 2024-09-13T14:22:08.148 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.917362253Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-13T14:22:08.148 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.942656554Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-13T14:22:08.148 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.968018806Z level=info msg="Executing migration" id="create dashboard table" 2024-09-13T14:22:08.148 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:07 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:07.993302975Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-13T14:22:08.148 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.018517118Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-13T14:22:08.148 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.08538449Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-13T14:22:08.148 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.110882538Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-13T14:22:08.149 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.152795556Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-13T14:22:08.425 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.194730137Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-13T14:22:08.425 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.220073135Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-13T14:22:08.425 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.245333908Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-13T14:22:08.425 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.287202991Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-13T14:22:08.425 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.337707608Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-13T14:22:08.425 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.363010762Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-13T14:22:08.426 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.388294518Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-13T14:22:08.426 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.430140092Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-13T14:22:08.677 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.455526645Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-13T14:22:08.678 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.480784325Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-13T14:22:08.678 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.522760512Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-13T14:22:08.678 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.564662573Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-13T14:22:08.678 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.590024126Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-13T14:22:08.679 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.615207156Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-13T14:22:08.679 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.640500963Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-13T14:22:08.679 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.682642937Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-13T14:22:08.800 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:08.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[24494]: cluster 2024-09-13T14:22:07.079552+0000 mgr.y (mgr.24287) 53 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-13T14:22:08.939 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.708113299Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-13T14:22:08.939 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.73348755Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-13T14:22:08.939 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.758734331Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-13T14:22:08.939 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.784100971Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-13T14:22:08.939 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.809327771Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-13T14:22:08.939 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.851448667Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-13T14:22:08.940 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.893372036Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-13T14:22:08.940 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.918889047Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-13T14:22:08.940 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.944116192Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-13T14:22:09.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:08 smithi099 bash[22656]: cluster 2024-09-13T14:22:07.079552+0000 mgr.y (mgr.24287) 53 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-13T14:22:09.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:08 smithi099 bash[32082]: cluster 2024-09-13T14:22:07.079552+0000 mgr.y (mgr.24287) 53 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-09-13T14:22:09.217 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.969303877Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-13T14:22:09.217 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:08 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:08.994648915Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-13T14:22:09.217 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.020022173Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-13T14:22:09.217 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.045274861Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-13T14:22:09.217 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.070606723Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-13T14:22:09.217 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.112598819Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-13T14:22:09.217 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.137894039Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-13T14:22:09.217 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.179860825Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-13T14:22:09.218 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.221828311Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-13T14:22:09.469 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.247119645Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-13T14:22:09.469 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.288996183Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-13T14:22:09.469 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.314329382Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-13T14:22:09.469 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.356389489Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-13T14:22:09.469 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.381549543Z level=info msg="Executing migration" id="create data_source table" 2024-09-13T14:22:09.470 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.406937528Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-13T14:22:09.470 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.449005228Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-13T14:22:09.470 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.474114962Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-13T14:22:09.722 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.516171147Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-13T14:22:09.722 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.558457089Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-13T14:22:09.722 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.583779392Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-13T14:22:09.722 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.625704425Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-13T14:22:09.722 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.651032824Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-13T14:22:09.722 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.676319672Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-13T14:22:09.722 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.701526561Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-13T14:22:09.722 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.726857617Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-13T14:22:09.983 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.752080199Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-13T14:22:09.983 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.777399766Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-13T14:22:09.983 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.819525448Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-13T14:22:09.983 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.861449888Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-13T14:22:09.983 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.886741098Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-13T14:22:09.983 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.912163593Z level=info msg="Executing migration" id="Add uid column" 2024-09-13T14:22:09.983 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.937358025Z level=info msg="Executing migration" id="Update uid value" 2024-09-13T14:22:09.983 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.96260785Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-13T14:22:09.983 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:09 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:09.987880664Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-13T14:22:10.235 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.029936166Z level=info msg="Executing migration" id="create api_key table" 2024-09-13T14:22:10.235 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.055228884Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-13T14:22:10.235 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.080338307Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-13T14:22:10.236 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.105711609Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-13T14:22:10.236 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.147651309Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-13T14:22:10.236 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.173067844Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-13T14:22:10.236 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.21499421Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-13T14:22:10.236 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.240283805Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-13T14:22:10.487 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[24494]: cluster 2024-09-13T14:22:09.080017+0000 mgr.y (mgr.24287) 54 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 258 B/s wr, 0 op/s 2024-09-13T14:22:10.487 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[24494]: audit 2024-09-13T14:22:09.331433+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:10.487 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[24494]: audit 2024-09-13T14:22:09.339839+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:10.488 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.282213854Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-13T14:22:10.488 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.324204015Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-13T14:22:10.488 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.3998017Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-13T14:22:10.488 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.441962846Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-13T14:22:10.488 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.467268618Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-13T14:22:10.488 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.492452245Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-13T14:22:10.753 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.517634286Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-13T14:22:10.753 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.559716768Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-13T14:22:10.753 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.584995257Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-13T14:22:10.753 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.70060918Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-13T14:22:10.753 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.758148449Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-13T14:22:10.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:10 smithi099 bash[22656]: cluster 2024-09-13T14:22:09.080017+0000 mgr.y (mgr.24287) 54 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 258 B/s wr, 0 op/s 2024-09-13T14:22:10.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:10 smithi099 bash[22656]: audit 2024-09-13T14:22:09.331433+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:10.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:10 smithi099 bash[22656]: audit 2024-09-13T14:22:09.339839+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:10.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:10 smithi099 bash[32082]: cluster 2024-09-13T14:22:09.080017+0000 mgr.y (mgr.24287) 54 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 258 B/s wr, 0 op/s 2024-09-13T14:22:10.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:10 smithi099 bash[32082]: audit 2024-09-13T14:22:09.331433+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:10.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:10 smithi099 bash[32082]: audit 2024-09-13T14:22:09.339839+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:11.015 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.814569404Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-13T14:22:11.015 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.927859306Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-13T14:22:11.015 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.96998237Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-13T14:22:11.015 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:10 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:10.995382213Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-13T14:22:11.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.020753096Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-13T14:22:11.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.062755275Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-13T14:22:11.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.104987509Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-13T14:22:11.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.146885999Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-13T14:22:11.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.18909172Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-13T14:22:11.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.214387921Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-13T14:22:11.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.239692199Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-13T14:22:11.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.281655422Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-13T14:22:11.562 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.323668994Z level=info msg="Executing migration" id="create quota table v1" 2024-09-13T14:22:11.563 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.365775056Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-13T14:22:11.563 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.407593383Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-13T14:22:11.563 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.458366726Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-13T14:22:11.563 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.500358691Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-13T14:22:11.563 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.525629902Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-13T14:22:11.563 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.567665703Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-13T14:22:11.824 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.609816833Z level=info msg="Executing migration" id="create session table" 2024-09-13T14:22:11.824 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.635214957Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-13T14:22:11.824 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.660432769Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-13T14:22:11.824 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.685660753Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-13T14:22:11.824 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.711070263Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-13T14:22:11.824 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.736311931Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-13T14:22:11.824 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.76147901Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-13T14:22:11.824 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.786824128Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-13T14:22:11.825 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.828901969Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-13T14:22:12.084 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.854280337Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-13T14:22:12.084 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.92126257Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-13T14:22:12.085 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.954875675Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-13T14:22:12.085 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:11 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:11.988435052Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-13T14:22:12.085 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.038910602Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-13T14:22:12.085 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.064215176Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-13T14:22:12.085 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.089352497Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-13T14:22:12.337 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.131475323Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-13T14:22:12.337 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.156776146Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-13T14:22:12.337 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.182070584Z level=info msg="Executing migration" id="create alert table v1" 2024-09-13T14:22:12.337 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.207384138Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-13T14:22:12.337 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.232640423Z level=info msg="Executing migration" id="add index alert state" 2024-09-13T14:22:12.337 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.274591566Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-13T14:22:12.337 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.316634257Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-13T14:22:12.589 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.342149999Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-13T14:22:12.590 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.367490193Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-13T14:22:12.590 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.409410523Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-13T14:22:12.590 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.459826672Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-13T14:22:12.590 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.485283589Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-13T14:22:12.590 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.510444633Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-13T14:22:12.590 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.552370176Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-13T14:22:12.590 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.594458849Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-13T14:22:12.590 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[24494]: cluster 2024-09-13T14:22:11.080550+0000 mgr.y (mgr.24287) 55 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 229 B/s wr, 0 op/s 2024-09-13T14:22:12.590 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[24494]: audit 2024-09-13T14:22:11.395118+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:12.591 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[24494]: audit 2024-09-13T14:22:11.408872+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:12.591 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[24494]: audit 2024-09-13T14:22:11.413148+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:12.591 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[24494]: audit 2024-09-13T14:22:11.415193+0000 mon.c (mon.2) 52 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:12.591 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[24494]: audit 2024-09-13T14:22:11.426062+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:12.591 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[24494]: cephadm 2024-09-13T14:22:11.492641+0000 mgr.y (mgr.24287) 56 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:22:12.591 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[24494]: cephadm 2024-09-13T14:22:11.501090+0000 mgr.y (mgr.24287) 57 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi099 2024-09-13T14:22:12.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[22656]: cluster 2024-09-13T14:22:11.080550+0000 mgr.y (mgr.24287) 55 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 229 B/s wr, 0 op/s 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[22656]: audit 2024-09-13T14:22:11.395118+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[22656]: audit 2024-09-13T14:22:11.408872+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[22656]: audit 2024-09-13T14:22:11.413148+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[22656]: audit 2024-09-13T14:22:11.415193+0000 mon.c (mon.2) 52 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[22656]: audit 2024-09-13T14:22:11.426062+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[22656]: cephadm 2024-09-13T14:22:11.492641+0000 mgr.y (mgr.24287) 56 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[22656]: cephadm 2024-09-13T14:22:11.501090+0000 mgr.y (mgr.24287) 57 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi099 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[32082]: cluster 2024-09-13T14:22:11.080550+0000 mgr.y (mgr.24287) 55 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 229 B/s wr, 0 op/s 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[32082]: audit 2024-09-13T14:22:11.395118+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[32082]: audit 2024-09-13T14:22:11.408872+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:12.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[32082]: audit 2024-09-13T14:22:11.413148+0000 mon.c (mon.2) 51 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:12.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[32082]: audit 2024-09-13T14:22:11.415193+0000 mon.c (mon.2) 52 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:12.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[32082]: audit 2024-09-13T14:22:11.426062+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:12.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[32082]: cephadm 2024-09-13T14:22:11.492641+0000 mgr.y (mgr.24287) 56 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:22:12.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:12 smithi099 bash[32082]: cephadm 2024-09-13T14:22:11.501090+0000 mgr.y (mgr.24287) 57 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi099 2024-09-13T14:22:12.841 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.619811249Z level=info msg="Executing migration" id="Add column is_default" 2024-09-13T14:22:12.842 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.661761205Z level=info msg="Executing migration" id="Add column frequency" 2024-09-13T14:22:12.842 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.703765063Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-13T14:22:12.842 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.729074148Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-13T14:22:12.842 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.771113605Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-13T14:22:12.842 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.804762288Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-13T14:22:12.842 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.846705837Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-13T14:22:12.844 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:22:13.107 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.872054267Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-13T14:22:13.107 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.897373141Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-13T14:22:13.108 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.922618976Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-13T14:22:13.108 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.964593308Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-13T14:22:13.108 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:12 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:12.989813151Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-13T14:22:13.108 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.015168508Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-13T14:22:13.108 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.040406538Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-13T14:22:13.108 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.09078109Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-13T14:22:13.389 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.141143913Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-13T14:22:13.389 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.191561725Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-13T14:22:13.389 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.225409838Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-13T14:22:13.389 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.259043407Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-13T14:22:13.389 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.30936981Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-13T14:22:13.389 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.334669453Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-13T14:22:13.389 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.359996541Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-13T14:22:13.389 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.385224974Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-13T14:22:13.650 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[24494]: audit 2024-09-13T14:22:12.851165+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.99:0/4052838686' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:22:13.650 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[24494]: cluster 2024-09-13T14:22:13.066862+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:22:13.650 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.427392844Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-13T14:22:13.650 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.452600982Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-13T14:22:13.650 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.477894477Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-13T14:22:13.650 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.519921877Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-13T14:22:13.650 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.562151479Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-13T14:22:13.650 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.587423729Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-13T14:22:13.650 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.629359844Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-13T14:22:13.651 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.654685052Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-13T14:22:13.677 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:13 smithi099 bash[22656]: audit 2024-09-13T14:22:12.851165+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.99:0/4052838686' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:22:13.677 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:13 smithi099 bash[22656]: cluster 2024-09-13T14:22:13.066862+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:22:13.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:13 smithi099 bash[32082]: audit 2024-09-13T14:22:12.851165+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.99:0/4052838686' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:22:13.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:13 smithi099 bash[32082]: cluster 2024-09-13T14:22:13.066862+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:22:13.892 INFO:teuthology.orchestra.run.smithi099.stdout:[{"version":19,"timestamp":"2024-09-13T14:21:35.903638+0000","name":"","changes":[{"name":"mgr/mgr/dashboard/GRAFANA_API_SSL_VERIFY","new_value":"false"}]}] 2024-09-13T14:22:13.892 INFO:tasks.ceph_manager:config epoch is 19 2024-09-13T14:22:13.893 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:22:13.893 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:22:13.893 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:22:13.910 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.679971951Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-13T14:22:13.910 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.722013487Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-13T14:22:13.910 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.747353929Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-13T14:22:13.911 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.772675067Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-13T14:22:13.911 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.806225387Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-13T14:22:13.911 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.864842018Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-13T14:22:13.911 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.91538539Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-13T14:22:14.180 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.949105947Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-13T14:22:14.180 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:13 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:13.991043378Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-13T14:22:14.180 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.016320588Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-13T14:22:14.180 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.04149744Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-13T14:22:14.180 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.083519949Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-13T14:22:14.180 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.13413297Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-13T14:22:14.180 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.159416783Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-13T14:22:14.180 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.18473946Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-13T14:22:14.432 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[24494]: cluster 2024-09-13T14:22:13.081124+0000 mgr.y (mgr.24287) 58 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 103 B/s wr, 0 op/s 2024-09-13T14:22:14.432 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.209925265Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-13T14:22:14.432 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.235181313Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-13T14:22:14.432 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.27725257Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-13T14:22:14.432 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.319241137Z level=info msg="Executing migration" id="Move region to single row" 2024-09-13T14:22:14.432 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.344573Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-13T14:22:14.432 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.369733049Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-13T14:22:14.433 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.411811387Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-13T14:22:14.693 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.437082884Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-13T14:22:14.693 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.462330965Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-13T14:22:14.693 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.487568244Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-13T14:22:14.693 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.512765913Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-13T14:22:14.693 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.554771465Z level=info msg="Executing migration" id="create test_data table" 2024-09-13T14:22:14.693 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.580110828Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-13T14:22:14.693 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.605382371Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-13T14:22:14.694 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.647329006Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-13T14:22:14.694 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.672675871Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-13T14:22:14.694 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.697884504Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-13T14:22:14.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:14 smithi099 bash[22656]: cluster 2024-09-13T14:22:13.081124+0000 mgr.y (mgr.24287) 58 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 103 B/s wr, 0 op/s 2024-09-13T14:22:14.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:14 smithi099 bash[32082]: cluster 2024-09-13T14:22:13.081124+0000 mgr.y (mgr.24287) 58 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 103 B/s wr, 0 op/s 2024-09-13T14:22:14.954 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.714882206Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-13T14:22:14.954 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.740134323Z level=info msg="Executing migration" id="create team table" 2024-09-13T14:22:14.955 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.765411071Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-13T14:22:14.955 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.79075844Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-13T14:22:14.955 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.816114172Z level=info msg="Executing migration" id="create team member table" 2024-09-13T14:22:14.955 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.841319694Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-13T14:22:14.955 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.866496637Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-13T14:22:14.955 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.908506989Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-13T14:22:14.955 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.934057807Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-13T14:22:14.955 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:14 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:14.959258506Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-13T14:22:15.206 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.00123979Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-13T14:22:15.207 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.026493689Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-13T14:22:15.207 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.051867148Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-13T14:22:15.207 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.077094908Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-13T14:22:15.207 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.102366093Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-13T14:22:15.207 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.144291463Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-13T14:22:15.207 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.16962852Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-13T14:22:15.207 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.211574486Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-13T14:22:15.475 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:22:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:22:15.475 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.236993815Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-13T14:22:15.475 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.278999063Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-13T14:22:15.475 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.312654161Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-13T14:22:15.475 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.329403803Z level=info msg="Executing migration" id="create tag table" 2024-09-13T14:22:15.476 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.363255738Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-13T14:22:15.476 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.405333573Z level=info msg="Executing migration" id="create login attempt table" 2024-09-13T14:22:15.476 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.430742119Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-13T14:22:15.770 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.480125875Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-13T14:22:15.770 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.514466884Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-13T14:22:15.770 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.564896348Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-13T14:22:15.770 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.606860256Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-13T14:22:15.770 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.648962549Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-13T14:22:15.770 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.674161476Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-13T14:22:15.771 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.69948395Z level=info msg="Executing migration" id="create user auth table" 2024-09-13T14:22:15.771 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.774881373Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-13T14:22:16.022 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.800207063Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-13T14:22:16.022 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.825433429Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-13T14:22:16.022 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.876015259Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-13T14:22:16.023 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.918080019Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-13T14:22:16.023 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:15 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:15.943352354Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-13T14:22:16.023 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.00292841Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-13T14:22:16.023 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.027370628Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-13T14:22:16.291 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.052623711Z level=info msg="Executing migration" id="create server_lock table" 2024-09-13T14:22:16.291 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.077881498Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-13T14:22:16.291 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.119931028Z level=info msg="Executing migration" id="create user auth token table" 2024-09-13T14:22:16.292 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.145191584Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-13T14:22:16.292 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.18708834Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-13T14:22:16.292 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.229092897Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-13T14:22:16.292 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.254352527Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-13T14:22:16.292 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.296250701Z level=info msg="Executing migration" id="create cache_data table" 2024-09-13T14:22:16.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[24494]: cluster 2024-09-13T14:22:15.081692+0000 mgr.y (mgr.24287) 59 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-13T14:22:16.569 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.321457239Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-13T14:22:16.569 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.363531467Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-13T14:22:16.569 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.388779925Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-13T14:22:16.570 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.430974407Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-13T14:22:16.570 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.456103651Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-13T14:22:16.570 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.506436897Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-13T14:22:16.570 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.531871913Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-13T14:22:16.570 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.573707104Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-13T14:22:16.726 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:16 smithi099 bash[22656]: cluster 2024-09-13T14:22:15.081692+0000 mgr.y (mgr.24287) 59 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-13T14:22:16.726 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:16 smithi099 bash[32082]: cluster 2024-09-13T14:22:15.081692+0000 mgr.y (mgr.24287) 59 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-13T14:22:16.726 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:16 smithi099 systemd[1]: Stopping Ceph alertmanager.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.59900768Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.640991574Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.666291666Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.69152786Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.716736145Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.733653137Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.750888803Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.767839786Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.784792981Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.801618379Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-13T14:22:16.839 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.843658915Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-13T14:22:17.046 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:16 smithi099 bash[56811]: ts=2024-09-13T14:22:16.732Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:22:17.091 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.885567677Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-13T14:22:17.091 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.910945205Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-13T14:22:17.091 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.93626941Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-13T14:22:17.091 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.961453972Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-13T14:22:17.091 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:16 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:16.986743602Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-13T14:22:17.091 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.028748252Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-13T14:22:17.091 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.054003988Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-13T14:22:17.092 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.095869639Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-13T14:22:17.343 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.121277521Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-13T14:22:17.344 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.179943449Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-13T14:22:17.344 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.221931252Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-13T14:22:17.344 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.247251759Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-13T14:22:17.344 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.289198719Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-13T14:22:17.344 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.314563505Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-13T14:22:17.344 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.331487082Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-13T14:22:17.344 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.348467928Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-13T14:22:17.596 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.365372479Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-13T14:22:17.596 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.390639432Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-13T14:22:17.597 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.432530811Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-13T14:22:17.597 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.457773249Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-13T14:22:17.597 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.48306284Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-13T14:22:17.597 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.508330881Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-13T14:22:17.597 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.533473645Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-13T14:22:17.597 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.575553356Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-13T14:22:17.598 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.601138487Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-13T14:22:17.712 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:17 smithi099 bash[58362]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:22:17.713 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:17 smithi099 bash[58441]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:22:17.713 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:17 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:22:17.713 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:17 smithi099 systemd[1]: Stopped Ceph alertmanager.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:17.713 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:17 smithi099 systemd[1]: Started Ceph alertmanager.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:17.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.626335553Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-13T14:22:17.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.668321423Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-13T14:22:17.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.693591893Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-13T14:22:17.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.718942329Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-13T14:22:17.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.744268006Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-13T14:22:17.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.769545234Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-13T14:22:17.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.794693788Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-13T14:22:17.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.819910958Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-13T14:22:18.107 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.861956538Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-13T14:22:18.107 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.920897627Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-13T14:22:18.107 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:17 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:17.979740699Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-13T14:22:18.491 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:18 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:18.198625546Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-13T14:22:18.491 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:18 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:18.315041262Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-13T14:22:18.664 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:18.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:18 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:18.49038921Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-13T14:22:18.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:18 smithi177 bash[24494]: cluster 2024-09-13T14:22:17.082155+0000 mgr.y (mgr.24287) 60 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-13T14:22:18.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:18 smithi177 bash[24494]: audit 2024-09-13T14:22:17.648586+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:18.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:18 smithi177 bash[24494]: audit 2024-09-13T14:22:17.655021+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:18.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:18 smithi177 bash[24494]: cephadm 2024-09-13T14:22:17.656736+0000 mgr.y (mgr.24287) 61 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:22:18.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:18 smithi177 bash[24494]: cephadm 2024-09-13T14:22:17.665393+0000 mgr.y (mgr.24287) 62 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi177 2024-09-13T14:22:19.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:18 smithi099 bash[32082]: cluster 2024-09-13T14:22:17.082155+0000 mgr.y (mgr.24287) 60 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-13T14:22:19.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:18 smithi099 bash[32082]: audit 2024-09-13T14:22:17.648586+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:19.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:18 smithi099 bash[32082]: audit 2024-09-13T14:22:17.655021+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:19.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:18 smithi099 bash[32082]: cephadm 2024-09-13T14:22:17.656736+0000 mgr.y (mgr.24287) 61 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:22:19.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:18 smithi099 bash[32082]: cephadm 2024-09-13T14:22:17.665393+0000 mgr.y (mgr.24287) 62 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi177 2024-09-13T14:22:19.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:18 smithi099 bash[22656]: cluster 2024-09-13T14:22:17.082155+0000 mgr.y (mgr.24287) 60 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-09-13T14:22:19.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:18 smithi099 bash[22656]: audit 2024-09-13T14:22:17.648586+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:19.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:18 smithi099 bash[22656]: audit 2024-09-13T14:22:17.655021+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:19.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:18 smithi099 bash[22656]: cephadm 2024-09-13T14:22:17.656736+0000 mgr.y (mgr.24287) 61 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:22:19.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:18 smithi099 bash[22656]: cephadm 2024-09-13T14:22:17.665393+0000 mgr.y (mgr.24287) 62 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi177 2024-09-13T14:22:19.357 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:18 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:18.919577407Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-13T14:22:19.357 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:19 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:19.059014653Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-13T14:22:19.357 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:19 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:19.150608995Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-13T14:22:19.568 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:19 smithi099 bash[58469]: ts=2024-09-13T14:22:19.414Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T14:22:19.568 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:19 smithi099 bash[58469]: ts=2024-09-13T14:22:19.414Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T14:22:19.568 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:19 smithi099 bash[58469]: ts=2024-09-13T14:22:19.416Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.99 port=9094 2024-09-13T14:22:19.568 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:19 smithi099 bash[58469]: ts=2024-09-13T14:22:19.417Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:22:19.568 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:19 smithi099 bash[58469]: ts=2024-09-13T14:22:19.448Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:22:19.568 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:19 smithi099 bash[58469]: ts=2024-09-13T14:22:19.448Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:22:19.568 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:19 smithi099 bash[58469]: ts=2024-09-13T14:22:19.452Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:22:19.568 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:19 smithi099 bash[58469]: ts=2024-09-13T14:22:19.452Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:22:19.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:19 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:19.544550401Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-13T14:22:19.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:19 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:19.678351005Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-13T14:22:19.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:19 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:19.779238275Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-13T14:22:20.151 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:19 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:19.87352394Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-13T14:22:20.151 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:19 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:19.955627056Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-13T14:22:20.151 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.013935141Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-13T14:22:20.152 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.097518811Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-13T14:22:20.467 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.16460214Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-13T14:22:20.467 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.2065463Z level=info msg="Executing migration" id="create alert_image table" 2024-09-13T14:22:20.467 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.223599883Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-13T14:22:20.467 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.265546404Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-13T14:22:20.467 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.324554492Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-13T14:22:20.467 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.350221925Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-13T14:22:20.467 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.391848195Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T14:22:20.467 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.392073829Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T14:22:20.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[24494]: cluster 2024-09-13T14:22:19.082851+0000 mgr.y (mgr.24287) 63 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:20.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.471983058Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-13T14:22:20.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.55994633Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-13T14:22:20.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.652027879Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-13T14:22:20.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.652594094Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-13T14:22:20.859 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.653142372Z level=info msg="alerts found to migrate" alerts=0 2024-09-13T14:22:21.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:20 smithi099 bash[22656]: cluster 2024-09-13T14:22:19.082851+0000 mgr.y (mgr.24287) 63 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:21.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:20 smithi099 bash[32082]: cluster 2024-09-13T14:22:19.082851+0000 mgr.y (mgr.24287) 63 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:21.173 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:20 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:20.888638539Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-13T14:22:21.460 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:21 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:21.178582215Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-13T14:22:21.460 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:21 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:21.262824706Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-13T14:22:21.460 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:21 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:21.33833769Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-13T14:22:21.460 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:21 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:21.41698066Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-13T14:22:21.460 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:21 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:21.465282998Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-13T14:22:21.567 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:21 smithi099 bash[58469]: ts=2024-09-13T14:22:21.418Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000479698s 2024-09-13T14:22:21.792 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:21 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:21.524061862Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-13T14:22:22.072 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:21 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:21.797340371Z level=info msg="Executing migration" id="create data_keys table" 2024-09-13T14:22:22.072 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:21 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:21.971717129Z level=info msg="Executing migration" id="create secrets table" 2024-09-13T14:22:22.357 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:22 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:22.077224493Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-13T14:22:22.357 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:22 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:22.161294426Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-13T14:22:22.775 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:22 smithi177 bash[24494]: cluster 2024-09-13T14:22:21.083389+0000 mgr.y (mgr.24287) 64 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:22.776 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:22 smithi177 bash[24494]: audit 2024-09-13T14:22:22.143640+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:22.776 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:22 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:22.490887219Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-13T14:22:22.776 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:22 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:22.60425742Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-13T14:22:22.776 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:22 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:22.689674877Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-13T14:22:22.776 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:22 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:22.78062669Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-13T14:22:22.796 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:22:22.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:22 smithi099 bash[22656]: cluster 2024-09-13T14:22:21.083389+0000 mgr.y (mgr.24287) 64 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:22.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:22 smithi099 bash[22656]: audit 2024-09-13T14:22:22.143640+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:22.806 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:22 smithi099 bash[32082]: cluster 2024-09-13T14:22:21.083389+0000 mgr.y (mgr.24287) 64 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:22.807 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:22 smithi099 bash[32082]: audit 2024-09-13T14:22:22.143640+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:23.030 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:22 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:22.872715434Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-13T14:22:23.030 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:22 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:22.948779817Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-13T14:22:23.304 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.03497993Z level=info msg="Executing migration" id="create permission table" 2024-09-13T14:22:23.304 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.109822077Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-13T14:22:23.304 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.166130336Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-13T14:22:23.304 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.199978627Z level=info msg="Executing migration" id="create role table" 2024-09-13T14:22:23.304 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.250618537Z level=info msg="Executing migration" id="add column display_name" 2024-09-13T14:22:23.584 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.318183163Z level=info msg="Executing migration" id="add column group_name" 2024-09-13T14:22:23.585 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.385512458Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-13T14:22:23.585 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.485963192Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-13T14:22:23.601 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":20,"flags":0,"active_gid":24287,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2887456408},{"type":"v1","addr":"172.21.15.99:6801","nonce":2887456408}]},"active_addr":"172.21.15.99:6801/2887456408","active_change":"2024-09-13T14:21:07.041615+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24304,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.99:8443/","prometheus":"http://172.21.15.99:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":52,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":1638158155}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":2140252887}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":4242039269}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":316966644}]}]} 2024-09-13T14:22:23.606 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:22:23.606 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:22:23.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:22:23.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[24494]: audit 2024-09-13T14:22:22.797820+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.99:0/1610489782' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:22:23.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.589514077Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-13T14:22:23.858 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.782634963Z level=info msg="Executing migration" id="create team role table" 2024-09-13T14:22:24.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:23 smithi099 bash[22656]: audit 2024-09-13T14:22:22.797820+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.99:0/1610489782' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:22:24.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:23 smithi099 bash[32082]: audit 2024-09-13T14:22:22.797820+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.99:0/1610489782' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:22:24.342 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:23 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:23.970014389Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-13T14:22:24.342 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.217195128Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-13T14:22:24.607 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 systemd[1]: Stopping Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:22:24.607 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.347479384Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-13T14:22:24.607 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.423303195Z level=info msg="Executing migration" id="create user role table" 2024-09-13T14:22:24.608 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.507864611Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-13T14:22:24.608 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.549577106Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-13T14:22:24.608 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.57492863Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-13T14:22:24.900 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[24494]: cluster 2024-09-13T14:22:23.084086+0000 mgr.y (mgr.24287) 65 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:24.901 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.61714525Z level=info msg="Executing migration" id="create builtin role table" 2024-09-13T14:22:24.901 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.685026326Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-13T14:22:24.901 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.734290586Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-13T14:22:24.901 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.814102638Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-13T14:22:24.901 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.675Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:22:24.901 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.675Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:22:24.902 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.675Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:22:24.902 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.675Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:22:24.902 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.675Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:22:24.902 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.675Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:22:24.902 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.675Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:22:24.902 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.675Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:22:24.902 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.675Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:22:24.902 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.676Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:22:24.902 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.676Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:22:24.902 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[48613]: ts=2024-09-13T14:22:24.676Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:22:25.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:24 smithi099 bash[22656]: cluster 2024-09-13T14:22:23.084086+0000 mgr.y (mgr.24287) 65 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:25.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:24 smithi099 bash[32082]: cluster 2024-09-13T14:22:23.084086+0000 mgr.y (mgr.24287) 65 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:25.242 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:24 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:24.90594942Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-13T14:22:25.242 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:25 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:25.088498741Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-13T14:22:25.561 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:25 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:25.24369622Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-13T14:22:25.827 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:25 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:25.566877656Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-13T14:22:25.827 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:25 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:25.706020836Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-13T14:22:25.828 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:25 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:25.781839965Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-13T14:22:25.828 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:25 smithi177 bash[50930]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:22:25.828 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:25 smithi177 bash[50991]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:22:25.828 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:25 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:22:25.828 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:25 smithi177 systemd[1]: Stopped Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:25.828 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:25 smithi177 systemd[1]: Started Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:22:26.107 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:25 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:25.832919953Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-13T14:22:26.107 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:25 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:25.883750354Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-13T14:22:26.107 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:25 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:25.951784049Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-13T14:22:26.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:25 smithi099 bash[22909]: [13/Sep/2024:14:22:25] ENGINE Bus STOPPING 2024-09-13T14:22:26.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:25 smithi099 bash[22909]: [13/Sep/2024:14:22:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:22:26.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:25 smithi099 bash[22909]: [13/Sep/2024:14:22:25] ENGINE Bus STOPPED 2024-09-13T14:22:26.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:25 smithi099 bash[22909]: [13/Sep/2024:14:22:25] ENGINE Bus STARTING 2024-09-13T14:22:26.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22909]: [13/Sep/2024:14:22:26] ENGINE Serving on http://:::9283 2024-09-13T14:22:26.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22909]: [13/Sep/2024:14:22:26] ENGINE Bus STARTED 2024-09-13T14:22:26.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22909]: [13/Sep/2024:14:22:26] ENGINE Bus STOPPING 2024-09-13T14:22:26.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:26.185054399Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-13T14:22:26.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:26.397599164Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-13T14:22:26.827 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:26.512203799Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-13T14:22:27.068 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22909]: [13/Sep/2024:14:22:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:22:27.068 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22909]: [13/Sep/2024:14:22:26] ENGINE Bus STOPPED 2024-09-13T14:22:27.068 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22909]: [13/Sep/2024:14:22:26] ENGINE Bus STARTING 2024-09-13T14:22:27.068 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22909]: [13/Sep/2024:14:22:26] ENGINE Serving on http://:::9283 2024-09-13T14:22:27.068 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22909]: [13/Sep/2024:14:22:26] ENGINE Bus STARTED 2024-09-13T14:22:27.069 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22909]: [13/Sep/2024:14:22:26] ENGINE Bus STOPPING 2024-09-13T14:22:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: cluster 2024-09-13T14:22:25.084661+0000 mgr.y (mgr.24287) 66 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.835103+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.849536+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.857521+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:22:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.858197+0000 mgr.y (mgr.24287) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:22:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.859923+0000 mon.c (mon.2) 56 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi099.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:22:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.860676+0000 mgr.y (mgr.24287) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi099.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:22:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.868224+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.880527+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.890062+0000 mon.c (mon.2) 58 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.900412+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.916282+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.922850+0000 mon.c (mon.2) 60 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.934125+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[32082]: audit 2024-09-13T14:22:25.944088+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: cluster 2024-09-13T14:22:25.084661+0000 mgr.y (mgr.24287) 66 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.835103+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.849536+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.857521+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.858197+0000 mgr.y (mgr.24287) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.859923+0000 mon.c (mon.2) 56 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi099.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.860676+0000 mgr.y (mgr.24287) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi099.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.868224+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.880527+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.890062+0000 mon.c (mon.2) 58 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.900412+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.916282+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.922850+0000 mon.c (mon.2) 60 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.934125+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:26 smithi099 bash[22656]: audit 2024-09-13T14:22:25.944088+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:27.093 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:26.836188781Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-13T14:22:27.093 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:26.981151871Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: cluster 2024-09-13T14:22:25.084661+0000 mgr.y (mgr.24287) 66 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.835103+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.849536+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.857521+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.858197+0000 mgr.y (mgr.24287) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.859923+0000 mon.c (mon.2) 56 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi099.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.860676+0000 mgr.y (mgr.24287) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi099.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.868224+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.880527+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.890062+0000 mon.c (mon.2) 58 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.900412+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.916282+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:22:27.095 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.922850+0000 mon.c (mon.2) 60 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:22:27.095 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.934125+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:27.095 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:26 smithi177 bash[24494]: audit 2024-09-13T14:22:25.944088+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:27.357 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.098407854Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-13T14:22:27.357 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.207173999Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-13T14:22:27.357 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.299167777Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-13T14:22:27.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[22909]: [13/Sep/2024:14:22:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:22:27.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[22909]: [13/Sep/2024:14:22:27] ENGINE Bus STOPPED 2024-09-13T14:22:27.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[22909]: [13/Sep/2024:14:22:27] ENGINE Bus STARTING 2024-09-13T14:22:27.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[22909]: [13/Sep/2024:14:22:27] ENGINE Serving on http://:::9283 2024-09-13T14:22:27.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[22909]: [13/Sep/2024:14:22:27] ENGINE Bus STARTED 2024-09-13T14:22:27.646 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.366599717Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-13T14:22:27.646 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.467565521Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-13T14:22:27.646 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.567206959Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-13T14:22:27.646 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.609596028Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-13T14:22:27.916 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[24494]: audit 2024-09-13T14:22:25.881645+0000 mgr.y (mgr.24287) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:27.916 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[24494]: audit 2024-09-13T14:22:25.891131+0000 mgr.y (mgr.24287) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:22:27.916 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[24494]: audit 2024-09-13T14:22:25.917273+0000 mgr.y (mgr.24287) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:22:27.916 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[24494]: audit 2024-09-13T14:22:25.923843+0000 mgr.y (mgr.24287) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:22:27.916 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.651683611Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-13T14:22:27.916 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.727467981Z level=info msg="Executing migration" id="add correlation config column" 2024-09-13T14:22:27.917 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.752917235Z level=info msg="Executing migration" id="create entity_events table" 2024-09-13T14:22:27.917 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.778184671Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-13T14:22:27.917 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.820202927Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:22:27.917 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.820589946Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:22:27.917 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.870665749Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:22:27.917 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.871048605Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:22:27.917 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.920923273Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-13T14:22:27.917 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.673Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:22:27.917 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.673Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:22:27.917 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.673Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi177 (none))" 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.673Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.673Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.675Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.675Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.676Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.676Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.679Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.679Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.918µs 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.679Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.679Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.680Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.680Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.38µs wal_replay_duration=631.622µs wbl_replay_duration=103ns total_replay_duration=675.911µs 2024-09-13T14:22:27.918 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.681Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:22:27.919 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.681Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:22:27.919 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.681Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:22:27.919 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.698Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.670044ms db_storage=912ns remote_storage=1.814µs web_handler=557ns query_engine=668ns scrape=320.424µs scrape_sd=29.509µs notify=29.077µs notify_sd=10.137µs rules=16.958779ms tracing=5.554µs 2024-09-13T14:22:27.919 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.699Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:22:27.919 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[51021]: ts=2024-09-13T14:22:27.699Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:22:28.193 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:27 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:27.962940126Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-13T14:22:28.193 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.00903898Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:22:28.194 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.038589922Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:22:28.194 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.063911972Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-13T14:22:28.194 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.105854248Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T14:22:28.194 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.131189388Z level=info msg="Executing migration" id="Drop public config table" 2024-09-13T14:22:28.194 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.156489373Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-13T14:22:28.194 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.198442369Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-13T14:22:28.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[22656]: audit 2024-09-13T14:22:25.881645+0000 mgr.y (mgr.24287) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:28.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[22656]: audit 2024-09-13T14:22:25.891131+0000 mgr.y (mgr.24287) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:22:28.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[22656]: audit 2024-09-13T14:22:25.917273+0000 mgr.y (mgr.24287) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:22:28.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[22656]: audit 2024-09-13T14:22:25.923843+0000 mgr.y (mgr.24287) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:22:28.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[32082]: audit 2024-09-13T14:22:25.881645+0000 mgr.y (mgr.24287) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:22:28.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[32082]: audit 2024-09-13T14:22:25.891131+0000 mgr.y (mgr.24287) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:22:28.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[32082]: audit 2024-09-13T14:22:25.917273+0000 mgr.y (mgr.24287) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:22:28.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:27 smithi099 bash[32082]: audit 2024-09-13T14:22:25.923843+0000 mgr.y (mgr.24287) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:22:28.385 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:28.455 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.215276254Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T14:22:28.455 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.249207609Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-13T14:22:28.455 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.291066192Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-13T14:22:28.455 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.324675136Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-13T14:22:28.455 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.349946937Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-13T14:22:28.455 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.366838521Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-13T14:22:28.456 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.383786136Z level=info msg="Executing migration" id="add share column" 2024-09-13T14:22:28.456 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.434481556Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-13T14:22:28.456 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.459842939Z level=info msg="Executing migration" id="create file table" 2024-09-13T14:22:28.707 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.485198405Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-13T14:22:28.707 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.510457807Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-13T14:22:28.708 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.535705109Z level=info msg="Executing migration" id="create file_meta table" 2024-09-13T14:22:28.708 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.577536825Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-13T14:22:28.708 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.602905174Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-13T14:22:28.708 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.644750556Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-13T14:22:28.708 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.686890762Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-13T14:22:28.708 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.7121272Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-13T14:22:28.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[24494]: cluster 2024-09-13T14:22:27.085272+0000 mgr.y (mgr.24287) 73 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:28.960 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.737365963Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-13T14:22:28.960 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.754267306Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-13T14:22:28.960 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.787913455Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-13T14:22:28.960 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.805019354Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-13T14:22:28.960 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.846772955Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-13T14:22:28.960 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.888961179Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-13T14:22:28.960 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.914196251Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-13T14:22:28.960 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.939482969Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-13T14:22:28.960 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.964741891Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-13T14:22:29.211 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:28 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:28.989985214Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-13T14:22:29.212 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.056877437Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-13T14:22:29.212 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.082089838Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-13T14:22:29.212 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.107397423Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-13T14:22:29.212 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.149477565Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-13T14:22:29.212 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.191219395Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-13T14:22:29.212 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.216636077Z level=info msg="Executing migration" id="create folder table" 2024-09-13T14:22:29.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:28 smithi099 bash[22656]: cluster 2024-09-13T14:22:27.085272+0000 mgr.y (mgr.24287) 73 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:29.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:28 smithi099 bash[32082]: cluster 2024-09-13T14:22:27.085272+0000 mgr.y (mgr.24287) 73 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:29.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.241795299Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-13T14:22:29.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.283754574Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-13T14:22:29.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.325783027Z level=info msg="Executing migration" id="Update folder title length" 2024-09-13T14:22:29.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.351420495Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-13T14:22:29.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=migrator t=2024-09-13T14:22:29.376625075Z level=info msg="migrations completed" performed=485 skipped=0 duration=23.077317986s 2024-09-13T14:22:29.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=sqlstore t=2024-09-13T14:22:29.378015256Z level=info msg="Created default organization" 2024-09-13T14:22:29.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=plugin.loader t=2024-09-13T14:22:29.512002704Z level=info msg="Plugin registered" pluginID=input 2024-09-13T14:22:29.762 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:22:29 smithi099 bash[58469]: ts=2024-09-13T14:22:29.419Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002093971s 2024-09-13T14:22:29.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=plugin.loader t=2024-09-13T14:22:29.570785604Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T14:22:29.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=plugin.loader t=2024-09-13T14:22:29.570850377Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T14:22:29.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=secrets t=2024-09-13T14:22:29.570932041Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:22:29.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=query_data t=2024-09-13T14:22:29.575250943Z level=info msg="Query Service initialization" 2024-09-13T14:22:29.857 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:29 smithi177 bash[50319]: logger=live.push_http t=2024-09-13T14:22:29.581724604Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:22:31.018 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:30 smithi177 bash[24494]: cluster 2024-09-13T14:22:29.086025+0000 mgr.y (mgr.24287) 74 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:31.018 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:30 smithi177 bash[50319]: logger=infra.usagestats.collector t=2024-09-13T14:22:30.766829437Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:22:31.018 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:30 smithi177 bash[50319]: logger=provisioning.datasources t=2024-09-13T14:22:30.767225745Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-13T14:22:31.018 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:30 smithi177 bash[50319]: logger=provisioning.datasources t=2024-09-13T14:22:30.801521471Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:22:31.018 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:30 smithi177 bash[50319]: logger=provisioning.datasources t=2024-09-13T14:22:30.896838418Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-13T14:22:31.018 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:30 smithi177 bash[50319]: logger=provisioning.alerting t=2024-09-13T14:22:30.970755034Z level=info msg="starting to provision alerting" 2024-09-13T14:22:31.018 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:30 smithi177 bash[50319]: logger=provisioning.alerting t=2024-09-13T14:22:30.970810079Z level=info msg="finished to provision alerting" 2024-09-13T14:22:31.018 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:30 smithi177 bash[50319]: logger=grafanaStorageLogger t=2024-09-13T14:22:30.971046278Z level=info msg="storage starting" 2024-09-13T14:22:31.018 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:30 smithi177 bash[50319]: logger=ngalert.state.manager t=2024-09-13T14:22:30.971860421Z level=info msg="Warming state cache for startup" 2024-09-13T14:22:31.019 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:30 smithi177 bash[50319]: logger=http.server t=2024-09-13T14:22:30.975011953Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:22:31.019 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:31 smithi177 bash[50319]: logger=ngalert.state.manager t=2024-09-13T14:22:31.023132276Z level=info msg="State cache has been initialized" states=0 duration=51.270721ms 2024-09-13T14:22:31.019 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:31 smithi177 bash[50319]: logger=ticker t=2024-09-13T14:22:31.023224935Z level=info msg=starting first_tick=2024-09-13T14:22:40Z 2024-09-13T14:22:31.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:30 smithi099 bash[22656]: cluster 2024-09-13T14:22:29.086025+0000 mgr.y (mgr.24287) 74 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:31.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:30 smithi099 bash[32082]: cluster 2024-09-13T14:22:29.086025+0000 mgr.y (mgr.24287) 74 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:31.357 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:22:31 smithi177 bash[50319]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:22:31.023243956Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T14:22:32.277 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:22:32.278 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":59,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:10:21.363918+0000","modified":"2024-09-13T14:22:13.058616+0000","last_up_change":"2024-09-13T14:20:05.471230+0000","last_in_change":"2024-09-13T14:19:38.687766+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:15:37.290226+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:21:51.531836+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"0624feaa-339b-4468-be53-582a1f04352c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6803","nonce":500907575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6805","nonce":500907575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6809","nonce":500907575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6807","nonce":500907575}]},"public_addr":"172.21.15.99:6803/500907575","cluster_addr":"172.21.15.99:6805/500907575","heartbeat_back_addr":"172.21.15.99:6809/500907575","heartbeat_front_addr":"172.21.15.99:6807/500907575","state":["exists","up"]},{"osd":1,"uuid":"b7b63553-e49d-4636-b3e2-43b72556a1a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6811","nonce":4008940153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6813","nonce":4008940153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6817","nonce":4008940153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6815","nonce":4008940153}]},"public_addr":"172.21.15.99:6811/4008940153","cluster_addr":"172.21.15.99:6813/4008940153","heartbeat_back_addr":"172.21.15.99:6817/4008940153","heartbeat_front_addr":"172.21.15.99:6815/4008940153","state":["exists","up"]},{"osd":2,"uuid":"3b9eff6d-5f5c-4843-a1ff-815a902167ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6819","nonce":4220960661}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6821","nonce":4220960661}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6825","nonce":4220960661}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6823","nonce":4220960661}]},"public_addr":"172.21.15.99:6819/4220960661","cluster_addr":"172.21.15.99:6821/4220960661","heartbeat_back_addr":"172.21.15.99:6825/4220960661","heartbeat_front_addr":"172.21.15.99:6823/4220960661","state":["exists","up"]},{"osd":3,"uuid":"686c8d3c-9cab-4920-b02b-d243324f5808","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6827","nonce":1221827921}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6829","nonce":1221827921}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6833","nonce":1221827921}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6831","nonce":1221827921}]},"public_addr":"172.21.15.99:6827/1221827921","cluster_addr":"172.21.15.99:6829/1221827921","heartbeat_back_addr":"172.21.15.99:6833/1221827921","heartbeat_front_addr":"172.21.15.99:6831/1221827921","state":["exists","up"]},{"osd":4,"uuid":"2d56712b-a6fb-4089-9583-850beed92b37","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6801","nonce":2146549575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6803","nonce":2146549575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6807","nonce":2146549575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6805","nonce":2146549575}]},"public_addr":"172.21.15.177:6801/2146549575","cluster_addr":"172.21.15.177:6803/2146549575","heartbeat_back_addr":"172.21.15.177:6807/2146549575","heartbeat_front_addr":"172.21.15.177:6805/2146549575","state":["exists","up"]},{"osd":5,"uuid":"414b880c-6534-495f-a54c-f8d28ba81c73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6809","nonce":947070587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6811","nonce":947070587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6815","nonce":947070587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6813","nonce":947070587}]},"public_addr":"172.21.15.177:6809/947070587","cluster_addr":"172.21.15.177:6811/947070587","heartbeat_back_addr":"172.21.15.177:6815/947070587","heartbeat_front_addr":"172.21.15.177:6813/947070587","state":["exists","up"]},{"osd":6,"uuid":"ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6817","nonce":3817692594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6819","nonce":3817692594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6823","nonce":3817692594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6821","nonce":3817692594}]},"public_addr":"172.21.15.177:6817/3817692594","cluster_addr":"172.21.15.177:6819/3817692594","heartbeat_back_addr":"172.21.15.177:6823/3817692594","heartbeat_front_addr":"172.21.15.177:6821/3817692594","state":["exists","up"]},{"osd":7,"uuid":"ad2c2977-fd6d-42f8-962e-907c2bf3204b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6825","nonce":1984676946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6827","nonce":1984676946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6831","nonce":1984676946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6829","nonce":1984676946}]},"public_addr":"172.21.15.177:6825/1984676946","cluster_addr":"172.21.15.177:6827/1984676946","heartbeat_back_addr":"172.21.15.177:6831/1984676946","heartbeat_front_addr":"172.21.15.177:6829/1984676946","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:46.390607+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:38.398637+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:15:33.420625+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:30.964755+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:22.513614+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:14.948720+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:19:10.431389+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:20:03.826389+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6800/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/795365709":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/17074640":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6800/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/881490667":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/888319381":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/4129805635":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6800/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6801/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:6801/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/405712510":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/1548831286":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/3483428223":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/3192879133":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/1363241618":"2024-09-14T14:21:07.041302+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:22:32.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[22656]: cluster 2024-09-13T14:22:31.086539+0000 mgr.y (mgr.24287) 75 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:32.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[22656]: audit 2024-09-13T14:22:31.623817+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:32.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[22656]: audit 2024-09-13T14:22:31.629427+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:32.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[22656]: audit 2024-09-13T14:22:32.282777+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.99:0/3074961840' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:22:32.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[22656]: audit 2024-09-13T14:22:32.539996+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:32.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[22656]: audit 2024-09-13T14:22:32.555387+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:32.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[32082]: cluster 2024-09-13T14:22:31.086539+0000 mgr.y (mgr.24287) 75 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:32.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[32082]: audit 2024-09-13T14:22:31.623817+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:32.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[32082]: audit 2024-09-13T14:22:31.629427+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:32.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[32082]: audit 2024-09-13T14:22:32.282777+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.99:0/3074961840' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:22:32.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[32082]: audit 2024-09-13T14:22:32.539996+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:32.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:32 smithi099 bash[32082]: audit 2024-09-13T14:22:32.555387+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:33.107 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:22:33.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:22:33.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:32 smithi177 bash[24494]: cluster 2024-09-13T14:22:31.086539+0000 mgr.y (mgr.24287) 75 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:33.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:32 smithi177 bash[24494]: audit 2024-09-13T14:22:31.623817+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:33.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:32 smithi177 bash[24494]: audit 2024-09-13T14:22:31.629427+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:33.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:32 smithi177 bash[24494]: audit 2024-09-13T14:22:32.282777+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.99:0/3074961840' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:22:33.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:32 smithi177 bash[24494]: audit 2024-09-13T14:22:32.539996+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:33.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:32 smithi177 bash[24494]: audit 2024-09-13T14:22:32.555387+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:34.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[32082]: audit 2024-09-13T14:22:32.918843+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:34.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[32082]: audit 2024-09-13T14:22:32.920412+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:34.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[32082]: audit 2024-09-13T14:22:32.932175+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:34.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[32082]: audit 2024-09-13T14:22:32.981831+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:34.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[32082]: audit 2024-09-13T14:22:32.984685+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:34.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[32082]: audit 2024-09-13T14:22:32.985940+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:34.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[32082]: audit 2024-09-13T14:22:32.997275+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:34.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[22656]: audit 2024-09-13T14:22:32.918843+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:34.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[22656]: audit 2024-09-13T14:22:32.920412+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:34.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[22656]: audit 2024-09-13T14:22:32.932175+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:34.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[22656]: audit 2024-09-13T14:22:32.981831+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:34.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[22656]: audit 2024-09-13T14:22:32.984685+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:34.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[22656]: audit 2024-09-13T14:22:32.985940+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:34.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:33 smithi099 bash[22656]: audit 2024-09-13T14:22:32.997275+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:33 smithi177 bash[24494]: audit 2024-09-13T14:22:32.918843+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:33 smithi177 bash[24494]: audit 2024-09-13T14:22:32.920412+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:33 smithi177 bash[24494]: audit 2024-09-13T14:22:32.932175+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:33 smithi177 bash[24494]: audit 2024-09-13T14:22:32.981831+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:33 smithi177 bash[24494]: audit 2024-09-13T14:22:32.984685+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:34.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:33 smithi177 bash[24494]: audit 2024-09-13T14:22:32.985940+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:34.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:33 smithi177 bash[24494]: audit 2024-09-13T14:22:32.997275+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:22:35.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:34 smithi099 bash[32082]: cluster 2024-09-13T14:22:33.086991+0000 mgr.y (mgr.24287) 76 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:35.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:34 smithi099 bash[22656]: cluster 2024-09-13T14:22:33.086991+0000 mgr.y (mgr.24287) 76 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:35.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:34 smithi177 bash[24494]: cluster 2024-09-13T14:22:33.086991+0000 mgr.y (mgr.24287) 76 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:35.857 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:22:35 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:22:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:22:37.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:36 smithi099 bash[32082]: cluster 2024-09-13T14:22:35.087492+0000 mgr.y (mgr.24287) 77 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:37.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:36 smithi099 bash[22656]: cluster 2024-09-13T14:22:35.087492+0000 mgr.y (mgr.24287) 77 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:37.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:36 smithi177 bash[24494]: cluster 2024-09-13T14:22:35.087492+0000 mgr.y (mgr.24287) 77 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:37.887 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:37.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:37 smithi099 bash[22656]: audit 2024-09-13T14:22:37.143766+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:37.982 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:37 smithi099 bash[32082]: audit 2024-09-13T14:22:37.143766+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:38.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:37 smithi177 bash[24494]: audit 2024-09-13T14:22:37.143766+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:39.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:38 smithi099 bash[22656]: cluster 2024-09-13T14:22:37.088032+0000 mgr.y (mgr.24287) 78 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:39.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:38 smithi099 bash[32082]: cluster 2024-09-13T14:22:37.088032+0000 mgr.y (mgr.24287) 78 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:39.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:38 smithi177 bash[24494]: cluster 2024-09-13T14:22:37.088032+0000 mgr.y (mgr.24287) 78 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:41.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:40 smithi099 bash[22656]: cluster 2024-09-13T14:22:39.088720+0000 mgr.y (mgr.24287) 79 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:41.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:40 smithi099 bash[32082]: cluster 2024-09-13T14:22:39.088720+0000 mgr.y (mgr.24287) 79 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:41.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:40 smithi177 bash[24494]: cluster 2024-09-13T14:22:39.088720+0000 mgr.y (mgr.24287) 79 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:41.524 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:22:41.524 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":59,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:10:21.363918+0000","modified":"2024-09-13T14:22:13.058616+0000","last_up_change":"2024-09-13T14:20:05.471230+0000","last_in_change":"2024-09-13T14:19:38.687766+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:15:37.290226+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:21:51.531836+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"0624feaa-339b-4468-be53-582a1f04352c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6803","nonce":500907575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6805","nonce":500907575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6809","nonce":500907575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6807","nonce":500907575}]},"public_addr":"172.21.15.99:6803/500907575","cluster_addr":"172.21.15.99:6805/500907575","heartbeat_back_addr":"172.21.15.99:6809/500907575","heartbeat_front_addr":"172.21.15.99:6807/500907575","state":["exists","up"]},{"osd":1,"uuid":"b7b63553-e49d-4636-b3e2-43b72556a1a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6811","nonce":4008940153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6813","nonce":4008940153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6817","nonce":4008940153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6815","nonce":4008940153}]},"public_addr":"172.21.15.99:6811/4008940153","cluster_addr":"172.21.15.99:6813/4008940153","heartbeat_back_addr":"172.21.15.99:6817/4008940153","heartbeat_front_addr":"172.21.15.99:6815/4008940153","state":["exists","up"]},{"osd":2,"uuid":"3b9eff6d-5f5c-4843-a1ff-815a902167ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6819","nonce":4220960661}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6821","nonce":4220960661}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6825","nonce":4220960661}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6823","nonce":4220960661}]},"public_addr":"172.21.15.99:6819/4220960661","cluster_addr":"172.21.15.99:6821/4220960661","heartbeat_back_addr":"172.21.15.99:6825/4220960661","heartbeat_front_addr":"172.21.15.99:6823/4220960661","state":["exists","up"]},{"osd":3,"uuid":"686c8d3c-9cab-4920-b02b-d243324f5808","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6827","nonce":1221827921}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6829","nonce":1221827921}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6833","nonce":1221827921}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6831","nonce":1221827921}]},"public_addr":"172.21.15.99:6827/1221827921","cluster_addr":"172.21.15.99:6829/1221827921","heartbeat_back_addr":"172.21.15.99:6833/1221827921","heartbeat_front_addr":"172.21.15.99:6831/1221827921","state":["exists","up"]},{"osd":4,"uuid":"2d56712b-a6fb-4089-9583-850beed92b37","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6801","nonce":2146549575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6803","nonce":2146549575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6807","nonce":2146549575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6805","nonce":2146549575}]},"public_addr":"172.21.15.177:6801/2146549575","cluster_addr":"172.21.15.177:6803/2146549575","heartbeat_back_addr":"172.21.15.177:6807/2146549575","heartbeat_front_addr":"172.21.15.177:6805/2146549575","state":["exists","up"]},{"osd":5,"uuid":"414b880c-6534-495f-a54c-f8d28ba81c73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6809","nonce":947070587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6811","nonce":947070587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6815","nonce":947070587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6813","nonce":947070587}]},"public_addr":"172.21.15.177:6809/947070587","cluster_addr":"172.21.15.177:6811/947070587","heartbeat_back_addr":"172.21.15.177:6815/947070587","heartbeat_front_addr":"172.21.15.177:6813/947070587","state":["exists","up"]},{"osd":6,"uuid":"ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6817","nonce":3817692594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6819","nonce":3817692594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6823","nonce":3817692594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6821","nonce":3817692594}]},"public_addr":"172.21.15.177:6817/3817692594","cluster_addr":"172.21.15.177:6819/3817692594","heartbeat_back_addr":"172.21.15.177:6823/3817692594","heartbeat_front_addr":"172.21.15.177:6821/3817692594","state":["exists","up"]},{"osd":7,"uuid":"ad2c2977-fd6d-42f8-962e-907c2bf3204b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6825","nonce":1984676946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6827","nonce":1984676946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6831","nonce":1984676946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6829","nonce":1984676946}]},"public_addr":"172.21.15.177:6825/1984676946","cluster_addr":"172.21.15.177:6827/1984676946","heartbeat_back_addr":"172.21.15.177:6831/1984676946","heartbeat_front_addr":"172.21.15.177:6829/1984676946","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:46.390607+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:38.398637+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:15:33.420625+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:30.964755+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:22.513614+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:14.948720+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:19:10.431389+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:20:03.826389+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6801/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6800/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/795365709":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/17074640":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6800/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/881490667":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/888319381":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/4129805635":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6800/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6801/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:6801/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/405712510":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/1548831286":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/3483428223":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/3192879133":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/1363241618":"2024-09-14T14:21:07.041302+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:22:41.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:41 smithi099 bash[32082]: audit 2024-09-13T14:22:41.529089+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.99:0/4237554536' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:22:41.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:41 smithi099 bash[22656]: audit 2024-09-13T14:22:41.529089+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.99:0/4237554536' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:22:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:41 smithi177 bash[24494]: audit 2024-09-13T14:22:41.529089+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.99:0/4237554536' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:22:42.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:22:42.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:22:42.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:22:42.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:22:42.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:22:42.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:22:42.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:22:42.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:22:42.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:42 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:22:42] "GET /metrics HTTP/1.1" 200 203884 "" "Prometheus/2.43.0" 2024-09-13T14:22:43.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:42 smithi099 bash[22656]: cluster 2024-09-13T14:22:41.089270+0000 mgr.y (mgr.24287) 80 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:43.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:42 smithi099 bash[32082]: cluster 2024-09-13T14:22:41.089270+0000 mgr.y (mgr.24287) 80 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:43.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:42 smithi177 bash[24494]: cluster 2024-09-13T14:22:41.089270+0000 mgr.y (mgr.24287) 80 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:45.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:44 smithi099 bash[22656]: cluster 2024-09-13T14:22:43.089919+0000 mgr.y (mgr.24287) 81 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:45.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:44 smithi099 bash[32082]: cluster 2024-09-13T14:22:43.089919+0000 mgr.y (mgr.24287) 81 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:45.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:44 smithi177 bash[24494]: cluster 2024-09-13T14:22:43.089919+0000 mgr.y (mgr.24287) 81 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:45.857 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:22:45 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:22:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:22:47.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:46 smithi099 bash[22656]: cluster 2024-09-13T14:22:45.090434+0000 mgr.y (mgr.24287) 82 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:47.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:46 smithi099 bash[32082]: cluster 2024-09-13T14:22:45.090434+0000 mgr.y (mgr.24287) 82 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:47.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:46 smithi177 bash[24494]: cluster 2024-09-13T14:22:45.090434+0000 mgr.y (mgr.24287) 82 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:47.163 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:47.164 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:47.164 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:47.164 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:47.165 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:47.166 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:47.166 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:47.168 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:49.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:48 smithi099 bash[22656]: cluster 2024-09-13T14:22:47.091010+0000 mgr.y (mgr.24287) 83 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:49.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:48 smithi099 bash[32082]: cluster 2024-09-13T14:22:47.091010+0000 mgr.y (mgr.24287) 83 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:49.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:48 smithi177 bash[24494]: cluster 2024-09-13T14:22:47.091010+0000 mgr.y (mgr.24287) 83 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:51.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:50 smithi099 bash[22656]: cluster 2024-09-13T14:22:49.091696+0000 mgr.y (mgr.24287) 84 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:51.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:50 smithi099 bash[32082]: cluster 2024-09-13T14:22:49.091696+0000 mgr.y (mgr.24287) 84 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:51.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:50 smithi177 bash[24494]: cluster 2024-09-13T14:22:49.091696+0000 mgr.y (mgr.24287) 84 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:52.471 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:22:52 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:22:52] "GET /metrics HTTP/1.1" 200 203884 "" "Prometheus/2.43.0" 2024-09-13T14:22:52.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:52 smithi099 bash[22656]: cluster 2024-09-13T14:22:51.092292+0000 mgr.y (mgr.24287) 85 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:52.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:52 smithi099 bash[22656]: audit 2024-09-13T14:22:52.144043+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:52.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:52 smithi099 bash[32082]: cluster 2024-09-13T14:22:51.092292+0000 mgr.y (mgr.24287) 85 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:52.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:52 smithi099 bash[32082]: audit 2024-09-13T14:22:52.144043+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:52 smithi177 bash[24494]: cluster 2024-09-13T14:22:51.092292+0000 mgr.y (mgr.24287) 85 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:52 smithi177 bash[24494]: audit 2024-09-13T14:22:52.144043+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:54.972 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:54 smithi099 bash[22656]: cluster 2024-09-13T14:22:53.092927+0000 mgr.y (mgr.24287) 86 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:54.973 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:54 smithi099 bash[32082]: cluster 2024-09-13T14:22:53.092927+0000 mgr.y (mgr.24287) 86 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:55.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:54 smithi177 bash[24494]: cluster 2024-09-13T14:22:53.092927+0000 mgr.y (mgr.24287) 86 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:55.857 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:22:55 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:22:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:22:56.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:56 smithi099 bash[22656]: cluster 2024-09-13T14:22:55.093355+0000 mgr.y (mgr.24287) 87 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:56.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:56 smithi099 bash[32082]: cluster 2024-09-13T14:22:55.093355+0000 mgr.y (mgr.24287) 87 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:57.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:56 smithi177 bash[24494]: cluster 2024-09-13T14:22:55.093355+0000 mgr.y (mgr.24287) 87 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:59.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:22:58 smithi099 bash[22656]: cluster 2024-09-13T14:22:57.093864+0000 mgr.y (mgr.24287) 88 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:59.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:22:58 smithi099 bash[32082]: cluster 2024-09-13T14:22:57.093864+0000 mgr.y (mgr.24287) 88 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:22:59.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:22:58 smithi177 bash[24494]: cluster 2024-09-13T14:22:57.093864+0000 mgr.y (mgr.24287) 88 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:00.938 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:00 smithi099 bash[22656]: cluster 2024-09-13T14:22:59.094532+0000 mgr.y (mgr.24287) 89 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:00.938 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:00 smithi099 bash[32082]: cluster 2024-09-13T14:22:59.094532+0000 mgr.y (mgr.24287) 89 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:01.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:00 smithi177 bash[24494]: cluster 2024-09-13T14:22:59.094532+0000 mgr.y (mgr.24287) 89 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:02.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:23:02 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:02] "GET /metrics HTTP/1.1" 200 203854 "" "Prometheus/2.43.0" 2024-09-13T14:23:02.929 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:02 smithi099 bash[22656]: cluster 2024-09-13T14:23:01.094872+0000 mgr.y (mgr.24287) 90 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:02.930 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:02 smithi099 bash[32082]: cluster 2024-09-13T14:23:01.094872+0000 mgr.y (mgr.24287) 90 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:03.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:02 smithi177 bash[24494]: cluster 2024-09-13T14:23:01.094872+0000 mgr.y (mgr.24287) 90 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:04.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:04 smithi099 bash[32082]: cluster 2024-09-13T14:23:03.095413+0000 mgr.y (mgr.24287) 91 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:04.809 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:04 smithi099 bash[22656]: cluster 2024-09-13T14:23:03.095413+0000 mgr.y (mgr.24287) 91 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:04.809 INFO:teuthology.orchestra.run.smithi099.stdout:34359738479 2024-09-13T14:23:04.810 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:23:05.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:04 smithi177 bash[24494]: cluster 2024-09-13T14:23:03.095413+0000 mgr.y (mgr.24287) 91 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:05.857 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:23:05 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:23:06.562 INFO:teuthology.orchestra.run.smithi099.stdout:184683593776 2024-09-13T14:23:06.563 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:23:06.766 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:06 smithi099 bash[22656]: cluster 2024-09-13T14:23:05.095952+0000 mgr.y (mgr.24287) 92 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:06.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:06 smithi099 bash[32082]: cluster 2024-09-13T14:23:05.095952+0000 mgr.y (mgr.24287) 92 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:07.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:06 smithi177 bash[24494]: cluster 2024-09-13T14:23:05.095952+0000 mgr.y (mgr.24287) 92 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:08.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:07 smithi099 bash[22656]: audit 2024-09-13T14:23:07.151612+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:08.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:07 smithi099 bash[32082]: audit 2024-09-13T14:23:07.151612+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:08.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:07 smithi177 bash[24494]: audit 2024-09-13T14:23:07.151612+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:08.419 INFO:teuthology.orchestra.run.smithi099.stdout:55834574950 2024-09-13T14:23:08.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:23:08.506 INFO:teuthology.orchestra.run.smithi099.stdout:111669149777 2024-09-13T14:23:08.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:23:08.756 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:08 smithi099 bash[22656]: cluster 2024-09-13T14:23:07.096493+0000 mgr.y (mgr.24287) 93 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:08.756 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:08 smithi099 bash[32082]: cluster 2024-09-13T14:23:07.096493+0000 mgr.y (mgr.24287) 93 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:09.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:08 smithi177 bash[24494]: cluster 2024-09-13T14:23:07.096493+0000 mgr.y (mgr.24287) 93 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:10.065 INFO:teuthology.orchestra.run.smithi099.stdout:210453397542 2024-09-13T14:23:10.066 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:23:10.195 INFO:teuthology.orchestra.run.smithi099.stdout:77309411420 2024-09-13T14:23:10.196 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:23:10.839 INFO:teuthology.orchestra.run.smithi099.stdout:158913790012 2024-09-13T14:23:10.840 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:23:11.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:10 smithi099 bash[22656]: cluster 2024-09-13T14:23:09.096923+0000 mgr.y (mgr.24287) 94 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:11.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:10 smithi099 bash[32082]: cluster 2024-09-13T14:23:09.096923+0000 mgr.y (mgr.24287) 94 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:11.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:10 smithi177 bash[24494]: cluster 2024-09-13T14:23:09.096923+0000 mgr.y (mgr.24287) 94 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:11.177 INFO:teuthology.orchestra.run.smithi099.stdout:133143986247 2024-09-13T14:23:11.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:23:12.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:23:12 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:12] "GET /metrics HTTP/1.1" 200 203854 "" "Prometheus/2.43.0" 2024-09-13T14:23:13.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:12 smithi099 bash[22656]: cluster 2024-09-13T14:23:11.097427+0000 mgr.y (mgr.24287) 95 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:13.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:12 smithi099 bash[32082]: cluster 2024-09-13T14:23:11.097427+0000 mgr.y (mgr.24287) 95 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:13.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:12 smithi177 bash[24494]: cluster 2024-09-13T14:23:11.097427+0000 mgr.y (mgr.24287) 95 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:13.592 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:13.593 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:13.593 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:13.593 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:14.607 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:14.608 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:14.608 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:15.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:14 smithi099 bash[22656]: cluster 2024-09-13T14:23:13.097813+0000 mgr.y (mgr.24287) 96 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:15.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:14 smithi099 bash[32082]: cluster 2024-09-13T14:23:13.097813+0000 mgr.y (mgr.24287) 96 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:14 smithi177 bash[24494]: cluster 2024-09-13T14:23:13.097813+0000 mgr.y (mgr.24287) 96 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:15.629 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:15.857 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:23:15 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:23:17.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:16 smithi099 bash[22656]: cluster 2024-09-13T14:23:15.098303+0000 mgr.y (mgr.24287) 97 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:17.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:16 smithi099 bash[32082]: cluster 2024-09-13T14:23:15.098303+0000 mgr.y (mgr.24287) 97 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:17.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:16 smithi177 bash[24494]: cluster 2024-09-13T14:23:15.098303+0000 mgr.y (mgr.24287) 97 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:18.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:18 smithi099 bash[22656]: cluster 2024-09-13T14:23:17.098804+0000 mgr.y (mgr.24287) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:18.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:18 smithi099 bash[32082]: cluster 2024-09-13T14:23:17.098804+0000 mgr.y (mgr.24287) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:19.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:18 smithi177 bash[24494]: cluster 2024-09-13T14:23:17.098804+0000 mgr.y (mgr.24287) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:21.029 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:20 smithi099 bash[22656]: cluster 2024-09-13T14:23:19.099413+0000 mgr.y (mgr.24287) 99 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:21.030 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:20 smithi099 bash[32082]: cluster 2024-09-13T14:23:19.099413+0000 mgr.y (mgr.24287) 99 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:21.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:20 smithi177 bash[24494]: cluster 2024-09-13T14:23:19.099413+0000 mgr.y (mgr.24287) 99 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:22.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:23:22 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:22] "GET /metrics HTTP/1.1" 200 203854 "" "Prometheus/2.43.0" 2024-09-13T14:23:23.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:22 smithi099 bash[22656]: cluster 2024-09-13T14:23:21.099814+0000 mgr.y (mgr.24287) 100 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:23.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:22 smithi099 bash[22656]: audit 2024-09-13T14:23:22.151801+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:23.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:22 smithi099 bash[32082]: cluster 2024-09-13T14:23:21.099814+0000 mgr.y (mgr.24287) 100 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:23.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:22 smithi099 bash[32082]: audit 2024-09-13T14:23:22.151801+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:23.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:22 smithi177 bash[24494]: cluster 2024-09-13T14:23:21.099814+0000 mgr.y (mgr.24287) 100 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:23.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:22 smithi177 bash[24494]: audit 2024-09-13T14:23:22.151801+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:24.877 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:24 smithi099 bash[22656]: cluster 2024-09-13T14:23:23.100354+0000 mgr.y (mgr.24287) 101 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:24 smithi177 bash[24494]: cluster 2024-09-13T14:23:23.100354+0000 mgr.y (mgr.24287) 101 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:25.227 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:24 smithi099 bash[32082]: cluster 2024-09-13T14:23:23.100354+0000 mgr.y (mgr.24287) 101 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:25.856 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:23:25 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:23:27.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:26 smithi099 bash[22656]: cluster 2024-09-13T14:23:25.100923+0000 mgr.y (mgr.24287) 102 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:27.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:26 smithi099 bash[32082]: cluster 2024-09-13T14:23:25.100923+0000 mgr.y (mgr.24287) 102 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:27.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:26 smithi177 bash[24494]: cluster 2024-09-13T14:23:25.100923+0000 mgr.y (mgr.24287) 102 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:28.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:28 smithi099 bash[22656]: cluster 2024-09-13T14:23:27.101454+0000 mgr.y (mgr.24287) 103 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:28.909 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:28 smithi099 bash[32082]: cluster 2024-09-13T14:23:27.101454+0000 mgr.y (mgr.24287) 103 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:29.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:28 smithi177 bash[24494]: cluster 2024-09-13T14:23:27.101454+0000 mgr.y (mgr.24287) 103 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:29.231 INFO:teuthology.orchestra.run.smithi099.stdout:34359738484 2024-09-13T14:23:30.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:29 smithi099 bash[22656]: audit 2024-09-13T14:23:29.238085+0000 mon.a (mon.0) 735 : audit [DBG] from='client.? 172.21.15.99:0/3296774840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:23:30.033 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:29 smithi099 bash[32082]: audit 2024-09-13T14:23:29.238085+0000 mon.a (mon.0) 735 : audit [DBG] from='client.? 172.21.15.99:0/3296774840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:23:30.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:29 smithi177 bash[24494]: audit 2024-09-13T14:23:29.238085+0000 mon.a (mon.0) 735 : audit [DBG] from='client.? 172.21.15.99:0/3296774840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:23:30.334 INFO:teuthology.orchestra.run.smithi099.stdout:184683593782 2024-09-13T14:23:31.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:30 smithi099 bash[32082]: cluster 2024-09-13T14:23:29.101821+0000 mgr.y (mgr.24287) 104 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:31.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:30 smithi099 bash[32082]: audit 2024-09-13T14:23:30.341493+0000 mon.a (mon.0) 736 : audit [DBG] from='client.? 172.21.15.99:0/2265821159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:23:31.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:30 smithi099 bash[22656]: cluster 2024-09-13T14:23:29.101821+0000 mgr.y (mgr.24287) 104 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:31.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:30 smithi099 bash[22656]: audit 2024-09-13T14:23:30.341493+0000 mon.a (mon.0) 736 : audit [DBG] from='client.? 172.21.15.99:0/2265821159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:23:31.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:30 smithi177 bash[24494]: cluster 2024-09-13T14:23:29.101821+0000 mgr.y (mgr.24287) 104 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:31.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:30 smithi177 bash[24494]: audit 2024-09-13T14:23:30.341493+0000 mon.a (mon.0) 736 : audit [DBG] from='client.? 172.21.15.99:0/2265821159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:23:31.532 INFO:teuthology.orchestra.run.smithi099.stdout:55834574955 2024-09-13T14:23:31.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:31 smithi099 bash[22656]: audit 2024-09-13T14:23:31.539444+0000 mon.a (mon.0) 737 : audit [DBG] from='client.? 172.21.15.99:0/1854858644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:23:31.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:31 smithi099 bash[32082]: audit 2024-09-13T14:23:31.539444+0000 mon.a (mon.0) 737 : audit [DBG] from='client.? 172.21.15.99:0/1854858644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:23:32.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:31 smithi177 bash[24494]: audit 2024-09-13T14:23:31.539444+0000 mon.a (mon.0) 737 : audit [DBG] from='client.? 172.21.15.99:0/1854858644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:23:32.412 INFO:teuthology.orchestra.run.smithi099.stdout:210453397547 2024-09-13T14:23:32.424 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:23:32 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:32] "GET /metrics HTTP/1.1" 200 203872 "" "Prometheus/2.43.0" 2024-09-13T14:23:32.772 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738479 got 34359738484 for osd.0 2024-09-13T14:23:32.772 DEBUG:teuthology.parallel:result is None 2024-09-13T14:23:33.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:32 smithi099 bash[32082]: cluster 2024-09-13T14:23:31.102314+0000 mgr.y (mgr.24287) 105 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:33.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:32 smithi099 bash[32082]: audit 2024-09-13T14:23:32.419161+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.99:0/165006058' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:23:33.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:32 smithi099 bash[22656]: cluster 2024-09-13T14:23:31.102314+0000 mgr.y (mgr.24287) 105 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:33.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:32 smithi099 bash[22656]: audit 2024-09-13T14:23:32.419161+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.99:0/165006058' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:23:33.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:32 smithi177 bash[24494]: cluster 2024-09-13T14:23:31.102314+0000 mgr.y (mgr.24287) 105 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:33.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:32 smithi177 bash[24494]: audit 2024-09-13T14:23:32.419161+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.99:0/165006058' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:23:33.833 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593776 got 184683593782 for osd.6 2024-09-13T14:23:33.834 DEBUG:teuthology.parallel:result is None 2024-09-13T14:23:34.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:33 smithi099 bash[22656]: audit 2024-09-13T14:23:33.053444+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:34.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:33 smithi099 bash[22656]: audit 2024-09-13T14:23:33.518850+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:34.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:33 smithi099 bash[22656]: audit 2024-09-13T14:23:33.520528+0000 mon.c (mon.2) 76 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:34.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:33 smithi099 bash[22656]: audit 2024-09-13T14:23:33.528391+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:23:34.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:33 smithi099 bash[32082]: audit 2024-09-13T14:23:33.053444+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:34.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:33 smithi099 bash[32082]: audit 2024-09-13T14:23:33.518850+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:34.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:33 smithi099 bash[32082]: audit 2024-09-13T14:23:33.520528+0000 mon.c (mon.2) 76 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:34.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:33 smithi099 bash[32082]: audit 2024-09-13T14:23:33.528391+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:23:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:33 smithi177 bash[24494]: audit 2024-09-13T14:23:33.053444+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:33 smithi177 bash[24494]: audit 2024-09-13T14:23:33.518850+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:33 smithi177 bash[24494]: audit 2024-09-13T14:23:33.520528+0000 mon.c (mon.2) 76 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:23:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:33 smithi177 bash[24494]: audit 2024-09-13T14:23:33.528391+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:23:34.290 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574950 got 55834574955 for osd.1 2024-09-13T14:23:34.290 DEBUG:teuthology.parallel:result is None 2024-09-13T14:23:34.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:34 smithi099 bash[22656]: cluster 2024-09-13T14:23:33.102755+0000 mgr.y (mgr.24287) 106 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:34.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:34 smithi099 bash[32082]: cluster 2024-09-13T14:23:33.102755+0000 mgr.y (mgr.24287) 106 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:35.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:34 smithi177 bash[24494]: cluster 2024-09-13T14:23:33.102755+0000 mgr.y (mgr.24287) 106 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:35.536 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397542 got 210453397547 for osd.7 2024-09-13T14:23:35.536 DEBUG:teuthology.parallel:result is None 2024-09-13T14:23:35.772 INFO:teuthology.orchestra.run.smithi099.stdout:111669149783 2024-09-13T14:23:35.789 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:23:35 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:23:35.912 INFO:teuthology.orchestra.run.smithi099.stdout:77309411426 2024-09-13T14:23:35.933 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:35 smithi099 bash[22656]: audit 2024-09-13T14:23:35.779276+0000 mon.a (mon.0) 739 : audit [DBG] from='client.? 172.21.15.99:0/425825887' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:23:35.933 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:35 smithi099 bash[32082]: audit 2024-09-13T14:23:35.779276+0000 mon.a (mon.0) 739 : audit [DBG] from='client.? 172.21.15.99:0/425825887' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:23:36.008 INFO:teuthology.orchestra.run.smithi099.stdout:133143986252 2024-09-13T14:23:36.020 INFO:teuthology.orchestra.run.smithi099.stdout:158913790017 2024-09-13T14:23:36.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:35 smithi177 bash[24494]: audit 2024-09-13T14:23:35.779276+0000 mon.a (mon.0) 739 : audit [DBG] from='client.? 172.21.15.99:0/425825887' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:23:37.019 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:36 smithi099 bash[32082]: cluster 2024-09-13T14:23:35.103282+0000 mgr.y (mgr.24287) 107 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:37.019 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:36 smithi099 bash[32082]: audit 2024-09-13T14:23:35.919336+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.99:0/451960925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:23:37.020 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:36 smithi099 bash[32082]: audit 2024-09-13T14:23:36.015137+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.99:0/520999606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:23:37.020 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:36 smithi099 bash[32082]: audit 2024-09-13T14:23:36.027876+0000 mon.a (mon.0) 741 : audit [DBG] from='client.? 172.21.15.99:0/685235193' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:23:37.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:36 smithi099 bash[22656]: cluster 2024-09-13T14:23:35.103282+0000 mgr.y (mgr.24287) 107 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:37.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:36 smithi099 bash[22656]: audit 2024-09-13T14:23:35.919336+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.99:0/451960925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:23:37.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:36 smithi099 bash[22656]: audit 2024-09-13T14:23:36.015137+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.99:0/520999606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:23:37.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:36 smithi099 bash[22656]: audit 2024-09-13T14:23:36.027876+0000 mon.a (mon.0) 741 : audit [DBG] from='client.? 172.21.15.99:0/685235193' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:23:37.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:36 smithi177 bash[24494]: cluster 2024-09-13T14:23:35.103282+0000 mgr.y (mgr.24287) 107 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:37.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:36 smithi177 bash[24494]: audit 2024-09-13T14:23:35.919336+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.99:0/451960925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:23:37.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:36 smithi177 bash[24494]: audit 2024-09-13T14:23:36.015137+0000 mon.a (mon.0) 740 : audit [DBG] from='client.? 172.21.15.99:0/520999606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:23:37.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:36 smithi177 bash[24494]: audit 2024-09-13T14:23:36.027876+0000 mon.a (mon.0) 741 : audit [DBG] from='client.? 172.21.15.99:0/685235193' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:23:37.343 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149777 got 111669149783 for osd.3 2024-09-13T14:23:37.343 DEBUG:teuthology.parallel:result is None 2024-09-13T14:23:38.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:37 smithi099 bash[22656]: audit 2024-09-13T14:23:37.152076+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:38.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:37 smithi099 bash[32082]: audit 2024-09-13T14:23:37.152076+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:38.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:37 smithi177 bash[24494]: audit 2024-09-13T14:23:37.152076+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:38.403 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411420 got 77309411426 for osd.2 2024-09-13T14:23:38.404 DEBUG:teuthology.parallel:result is None 2024-09-13T14:23:38.416 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986247 got 133143986252 for osd.4 2024-09-13T14:23:38.416 DEBUG:teuthology.parallel:result is None 2024-09-13T14:23:38.429 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790012 got 158913790017 for osd.5 2024-09-13T14:23:38.429 DEBUG:teuthology.parallel:result is None 2024-09-13T14:23:38.429 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:23:38.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:23:39.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:38 smithi099 bash[22656]: cluster 2024-09-13T14:23:37.103777+0000 mgr.y (mgr.24287) 108 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:39.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:38 smithi099 bash[32082]: cluster 2024-09-13T14:23:37.103777+0000 mgr.y (mgr.24287) 108 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:39.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:38 smithi177 bash[24494]: cluster 2024-09-13T14:23:37.103777+0000 mgr.y (mgr.24287) 108 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:41.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:40 smithi099 bash[22656]: cluster 2024-09-13T14:23:39.104414+0000 mgr.y (mgr.24287) 109 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:41.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:40 smithi099 bash[32082]: cluster 2024-09-13T14:23:39.104414+0000 mgr.y (mgr.24287) 109 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:41.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:40 smithi177 bash[24494]: cluster 2024-09-13T14:23:39.104414+0000 mgr.y (mgr.24287) 109 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:42.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:23:42 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:42] "GET /metrics HTTP/1.1" 200 203870 "" "Prometheus/2.43.0" 2024-09-13T14:23:43.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:42 smithi099 bash[22656]: cluster 2024-09-13T14:23:41.104954+0000 mgr.y (mgr.24287) 110 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:43.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:42 smithi099 bash[32082]: cluster 2024-09-13T14:23:41.104954+0000 mgr.y (mgr.24287) 110 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:42 smithi177 bash[24494]: cluster 2024-09-13T14:23:41.104954+0000 mgr.y (mgr.24287) 110 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:43.247 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:45.009 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:44 smithi099 bash[22656]: cluster 2024-09-13T14:23:43.105575+0000 mgr.y (mgr.24287) 111 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:45.010 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:44 smithi099 bash[32082]: cluster 2024-09-13T14:23:43.105575+0000 mgr.y (mgr.24287) 111 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:45.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:44 smithi177 bash[24494]: cluster 2024-09-13T14:23:43.105575+0000 mgr.y (mgr.24287) 111 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:45.856 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:23:45 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:23:46.975 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:23:46.975 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:23:46.985 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:46 smithi099 bash[22656]: cluster 2024-09-13T14:23:45.106132+0000 mgr.y (mgr.24287) 112 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:46.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:46 smithi099 bash[32082]: cluster 2024-09-13T14:23:45.106132+0000 mgr.y (mgr.24287) 112 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:47.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:46 smithi177 bash[24494]: cluster 2024-09-13T14:23:45.106132+0000 mgr.y (mgr.24287) 112 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:47.854 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":89,"stamp":"2024-09-13T14:23:45.105860+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":115,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385156,"kb_used_data":6312,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576060,"statfs":{"total":767960285184,"available":765517885440,"internally_reserved":0,"allocated":6463488,"data_stored":3738605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003336"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.531004+0000","last_change":"2024-09-13T14:22:03.178246+0000","last_active":"2024-09-13T14:22:13.531004+0000","last_peered":"2024-09-13T14:22:13.531004+0000","last_clean":"2024-09-13T14:22:13.531004+0000","last_became_active":"2024-09-13T14:21:52.861399+0000","last_became_peered":"2024-09-13T14:21:52.861399+0000","last_unstale":"2024-09-13T14:22:13.531004+0000","last_undegraded":"2024-09-13T14:22:13.531004+0000","last_fullsized":"2024-09-13T14:22:13.531004+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:51.432081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.513123+0000","last_change":"2024-09-13T14:22:03.172028+0000","last_active":"2024-09-13T14:22:13.513123+0000","last_peered":"2024-09-13T14:22:13.513123+0000","last_clean":"2024-09-13T14:22:13.513123+0000","last_became_active":"2024-09-13T14:21:53.095058+0000","last_became_peered":"2024-09-13T14:21:53.095058+0000","last_unstale":"2024-09-13T14:22:13.513123+0000","last_undegraded":"2024-09-13T14:22:13.513123+0000","last_fullsized":"2024-09-13T14:22:13.513123+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:44.515568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080647399999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.495140+0000","last_change":"2024-09-13T14:22:03.357990+0000","last_active":"2024-09-13T14:22:13.495140+0000","last_peered":"2024-09-13T14:22:13.495140+0000","last_clean":"2024-09-13T14:22:13.495140+0000","last_became_active":"2024-09-13T14:21:53.093638+0000","last_became_peered":"2024-09-13T14:21:53.093638+0000","last_unstale":"2024-09-13T14:22:13.495140+0000","last_undegraded":"2024-09-13T14:22:13.495140+0000","last_fullsized":"2024-09-13T14:22:13.495140+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:24.868073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.513101+0000","last_change":"2024-09-13T14:22:03.171905+0000","last_active":"2024-09-13T14:22:13.513101+0000","last_peered":"2024-09-13T14:22:13.513101+0000","last_clean":"2024-09-13T14:22:13.513101+0000","last_became_active":"2024-09-13T14:21:52.884118+0000","last_became_peered":"2024-09-13T14:21:52.884118+0000","last_unstale":"2024-09-13T14:22:13.513101+0000","last_undegraded":"2024-09-13T14:22:13.513101+0000","last_fullsized":"2024-09-13T14:22:13.513101+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:26:17.407017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.692258+0000","last_change":"2024-09-13T14:22:03.177356+0000","last_active":"2024-09-13T14:22:13.692258+0000","last_peered":"2024-09-13T14:22:13.692258+0000","last_clean":"2024-09-13T14:22:13.692258+0000","last_became_active":"2024-09-13T14:21:52.861658+0000","last_became_peered":"2024-09-13T14:21:52.861658+0000","last_unstale":"2024-09-13T14:22:13.692258+0000","last_undegraded":"2024-09-13T14:22:13.692258+0000","last_fullsized":"2024-09-13T14:22:13.692258+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:10.200310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.064427+0000","last_change":"2024-09-13T14:22:03.172771+0000","last_active":"2024-09-13T14:22:13.064427+0000","last_peered":"2024-09-13T14:22:13.064427+0000","last_clean":"2024-09-13T14:22:13.064427+0000","last_became_active":"2024-09-13T14:21:52.884004+0000","last_became_peered":"2024-09-13T14:21:52.884004+0000","last_unstale":"2024-09-13T14:22:13.064427+0000","last_undegraded":"2024-09-13T14:22:13.064427+0000","last_fullsized":"2024-09-13T14:22:13.064427+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:20:21.593867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.495137+0000","last_change":"2024-09-13T14:22:03.357964+0000","last_active":"2024-09-13T14:22:13.495137+0000","last_peered":"2024-09-13T14:22:13.495137+0000","last_clean":"2024-09-13T14:22:13.495137+0000","last_became_active":"2024-09-13T14:21:52.883199+0000","last_became_peered":"2024-09-13T14:21:52.883199+0000","last_unstale":"2024-09-13T14:22:13.495137+0000","last_undegraded":"2024-09-13T14:22:13.495137+0000","last_fullsized":"2024-09-13T14:22:13.495137+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:09.955395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.692347+0000","last_change":"2024-09-13T14:22:03.176719+0000","last_active":"2024-09-13T14:22:13.692347+0000","last_peered":"2024-09-13T14:22:13.692347+0000","last_clean":"2024-09-13T14:22:13.692347+0000","last_became_active":"2024-09-13T14:21:52.862358+0000","last_became_peered":"2024-09-13T14:21:52.862358+0000","last_unstale":"2024-09-13T14:22:13.692347+0000","last_undegraded":"2024-09-13T14:22:13.692347+0000","last_fullsized":"2024-09-13T14:22:13.692347+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:22:54.637897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":49,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:22:03.357270+0000","last_change":"2024-09-13T14:20:07.527898+0000","last_active":"2024-09-13T14:22:03.357270+0000","last_peered":"2024-09-13T14:22:03.357270+0000","last_clean":"2024-09-13T14:22:03.357270+0000","last_became_active":"2024-09-13T14:20:07.515370+0000","last_became_peered":"2024-09-13T14:20:07.515370+0000","last_unstale":"2024-09-13T14:22:03.357270+0000","last_undegraded":"2024-09-13T14:22:03.357270+0000","last_fullsized":"2024-09-13T14:22:03.357270+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:45.300816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397550,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.84599999999999997,"15min":0.83999999999999997},"min":{"1min":0.56399999999999995,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.1859999999999999,"5min":1.268,"15min":1.268},"last":0.871},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.88,"15min":0.88800000000000001},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.22,"5min":1.22,"15min":1.22},"last":1.0409999999999999}]},{"osd":1,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.88900000000000001,"15min":0.89600000000000002},"min":{"1min":0.56000000000000005,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.224,"5min":1.278,"15min":1.278},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.89300000000000002,"15min":0.90200000000000002},"min":{"1min":0.57399999999999995,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.1060000000000001,"5min":1.234,"15min":1.234},"last":1.1319999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.96299999999999997,"15min":0.97199999999999998},"min":{"1min":0.61299999999999999,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.1779999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.98899999999999999,"15min":0.98499999999999999},"min":{"1min":0.64800000000000002,"5min":0.497,"15min":0.497},"max":{"1min":1.2350000000000001,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":1.109}]},{"osd":3,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.94499999999999995,"15min":0.94799999999999995},"min":{"1min":0.60399999999999998,"5min":0.44,"15min":0.44},"max":{"1min":1.1859999999999999,"5min":1.375,"15min":1.375},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.95099999999999996,"15min":0.94799999999999995},"min":{"1min":0.67600000000000005,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.173,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.93000000000000005}]},{"osd":4,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.88700000000000001,"15min":0.89300000000000002},"min":{"1min":0.41599999999999998,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.337,"5min":1.337,"15min":1.337},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.96899999999999997,"15min":0.97499999999999998},"min":{"1min":0.44500000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.2949999999999999,"5min":1.325,"15min":1.325},"last":0.68000000000000005}]},{"osd":5,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.93500000000000005,"15min":0.92000000000000004},"min":{"1min":0.38200000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2749999999999999,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"min":{"1min":0.47099999999999997,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.3100000000000001,"5min":1.401,"15min":1.401},"last":1.089}]},{"osd":6,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0149999999999999,"15min":1.004},"min":{"1min":0.55100000000000005,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.365,"5min":1.401,"15min":1.401},"last":0.89500000000000002},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.0660000000000001,"15min":1.0680000000000001},"min":{"1min":0.49299999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.419,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":0.97799999999999998}]}]},{"osd":6,"up_from":43,"seq":184683593785,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":835845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.871,"15min":0.86499999999999999},"min":{"1min":0.66500000000000004,"5min":0.379,"15min":0.379},"max":{"1min":1.238,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.83699999999999997,"15min":0.82699999999999996},"min":{"1min":0.56999999999999995,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.0600000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.85699999999999998,"15min":0.83199999999999996},"min":{"1min":0.60699999999999998,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.141,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.88,"5min":0.86599999999999999,"15min":0.85999999999999999},"min":{"1min":0.69499999999999995,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.05,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.2689999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.88,"15min":0.875},"min":{"1min":0.64100000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.1479999999999999,"5min":1.419,"15min":1.419},"last":1.0029999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.90500000000000003,"15min":0.879},"min":{"1min":0.67600000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":1.1459999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.88700000000000001,"15min":0.86399999999999999},"min":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.169,"5min":1.659,"15min":1.659},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.91000000000000003,"15min":0.89500000000000002},"min":{"1min":0.67000000000000004,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.111,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":0.77400000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.81599999999999995,"15min":0.77100000000000002},"min":{"1min":0.51700000000000002,"5min":0.313,"15min":0.313},"max":{"1min":1.258,"5min":1.258,"15min":1.258},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.82999999999999996,"15min":0.77000000000000002},"min":{"1min":0.61499999999999999,"5min":0.317,"15min":0.317},"max":{"1min":1.2070000000000001,"5min":1.423,"15min":1.423},"last":1.105}]},{"osd":5,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.89000000000000001,"15min":0.84499999999999997},"min":{"1min":0.66200000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":1.242},{"interface":"front","average":{"1min":0.995,"5min":0.91000000000000003,"15min":0.84699999999999998},"min":{"1min":0.64300000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.2230000000000001,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":1.2210000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:23:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.98299999999999998,"15min":0.98299999999999998},"min":{"1min":0.63,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.1930000000000001,"5min":1.353,"15min":1.353},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.0289999999999999,"15min":1.0129999999999999},"min":{"1min":0.81999999999999995,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.464,"5min":1.464,"15min":1.464},"last":1.3080000000000001}]}]},{"osd":5,"up_from":37,"seq":158913790019,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":245792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.82699999999999996,"15min":0.81100000000000005},"min":{"1min":0.54900000000000004,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.1719999999999999,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.874,"15min":0.82799999999999996},"min":{"1min":0.621,"5min":0.505,"15min":0.505},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.89100000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.86399999999999999,"15min":0.82699999999999996},"min":{"1min":0.66000000000000003,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.163,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":1.042},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.875,"15min":0.81899999999999995},"min":{"1min":0.68300000000000005,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.2190000000000001,"5min":2.0459999999999998,"15min":2.0459999999999998},"last":0.93200000000000005}]},{"osd":2,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.85999999999999999,"15min":0.80800000000000005},"min":{"1min":0.66400000000000003,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.2290000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.88900000000000001,"15min":0.85899999999999999},"min":{"1min":0.67000000000000004,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.141,"5min":1.27,"15min":1.27},"last":1.0880000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":0.92400000000000004,"15min":0.86399999999999999},"min":{"1min":0.77500000000000002,"5min":0.497,"15min":0.497},"max":{"1min":1.2410000000000001,"5min":1.573,"15min":1.573},"last":1.02},{"interface":"front","average":{"1min":1.018,"5min":0.94699999999999995,"15min":0.871},"min":{"1min":0.81200000000000006,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.2290000000000001,"5min":2.246,"15min":2.246},"last":0.91200000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.77500000000000002,"15min":0.67100000000000004},"min":{"1min":0.63700000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.86699999999999999,"15min":0.73599999999999999},"min":{"1min":0.61599999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.3080000000000001,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.95099999999999996}]},{"osd":6,"last update":"Fri Sep 13 14:23:20 2024","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":0.877,"15min":0.82799999999999996},"min":{"1min":0.77600000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.3799999999999999,"5min":1.488,"15min":1.488},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.03,"5min":0.95399999999999996,"15min":0.92500000000000004},"min":{"1min":0.68899999999999995,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.4399999999999999,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.069}]},{"osd":7,"last update":"Fri Sep 13 14:23:07 2024","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.0049999999999999,"15min":1.006},"min":{"1min":0.83899999999999997,"5min":0.376,"15min":0.376},"max":{"1min":1.8080000000000001,"5min":1.8080000000000001,"15min":1.8080000000000001},"last":0.85199999999999998},{"interface":"front","average":{"1min":1.077,"5min":0.96699999999999997,"15min":0.96099999999999997},"min":{"1min":0.69399999999999995,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.4850000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.67200000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986254,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297924,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447228,"statfs":{"total":95995035648,"available":95689961472,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:23:35 2024","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.86299999999999999,"15min":0.81100000000000005},"min":{"1min":0.52800000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2030000000000001,"5min":1.224,"15min":1.224},"last":0.996},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.85999999999999999,"15min":0.84399999999999997},"min":{"1min":0.65800000000000003,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.143,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.93500000000000005}]},{"osd":1,"last update":"Fri Sep 13 14:23:35 2024","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.879,"15min":0.80600000000000005},"min":{"1min":0.67100000000000004,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.1200000000000001,"5min":1.282,"15min":1.282},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.85999999999999999,"15min":0.83599999999999997},"min":{"1min":0.68300000000000005,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.97499999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:23:35 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.88700000000000001,"15min":0.878},"min":{"1min":0.65000000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.1060000000000001,"5min":1.3200000000000001,"15min":1.3939999999999999},"last":0.879},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.90000000000000002,"15min":0.83399999999999996},"min":{"1min":0.67700000000000005,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.129,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.80500000000000005}]},{"osd":3,"last update":"Fri Sep 13 14:23:35 2024","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.90600000000000003,"15min":0.86199999999999999},"min":{"1min":0.72999999999999998,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.1879999999999999,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.93300000000000005,"15min":0.85999999999999999},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.131,"5min":1.224,"15min":1.224},"last":0.84399999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.83799999999999997,"15min":0.76400000000000001},"min":{"1min":0.501,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.1020000000000001,"5min":1.238,"15min":1.238},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.85399999999999998,"15min":0.748},"min":{"1min":0.47599999999999998,"5min":0.37,"15min":0.37},"max":{"1min":1.238,"5min":1.298,"15min":1.298},"last":1.0289999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:23:21 2024","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.85399999999999998,"15min":0.80000000000000004},"min":{"1min":0.44500000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.222,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.89900000000000002,"15min":0.84299999999999997},"min":{"1min":0.57799999999999996,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.252,"5min":1.431,"15min":1.431},"last":0.76700000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.036,"15min":1.0640000000000001},"min":{"1min":0.53100000000000003,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.2849999999999999,"5min":1.343,"15min":1.343},"last":1.1910000000000001},{"interface":"front","average":{"1min":1.032,"5min":1.101,"15min":1.123},"min":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.2070000000000001,"5min":1.504,"15min":1.504},"last":0.95799999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149785,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:22:48 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.65700000000000003,"15min":0.65300000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.1919999999999999,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.746,"5min":0.72499999999999998,"15min":0.68999999999999995},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.878}]},{"osd":1,"last update":"Fri Sep 13 14:22:48 2024","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":0.67800000000000005,"15min":0.66800000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.95699999999999996,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.62},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.77000000000000002,"15min":0.73999999999999999},"min":{"1min":0.373,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.206,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.67400000000000004}]},{"osd":2,"last update":"Fri Sep 13 14:22:48 2024","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.82399999999999995,"15min":0.76400000000000001},"min":{"1min":0.41499999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.2709999999999999,"5min":1.296,"15min":1.296},"last":0.999},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.78000000000000003,"15min":0.75},"min":{"1min":0.35399999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.2310000000000001,"5min":1.2310000000000001,"15min":1.2310000000000001},"last":0.745}]},{"osd":4,"last update":"Fri Sep 13 14:23:32 2024","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.90400000000000003,"15min":0.877},"min":{"1min":0.57999999999999996,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2110000000000001,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":1.173},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.95999999999999996,"15min":0.90600000000000003},"min":{"1min":0.58499999999999996,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.1950000000000001,"5min":1.395,"15min":1.395},"last":1.081}]},{"osd":5,"last update":"Fri Sep 13 14:23:27 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.98499999999999999,"15min":0.96099999999999997},"min":{"1min":0.60099999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.258,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.92700000000000005,"15min":0.90400000000000003},"min":{"1min":0.56200000000000006,"5min":0.434,"15min":0.434},"max":{"1min":1.2569999999999999,"5min":1.7569999999999999,"15min":1.7569999999999999},"last":1.2030000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:23:21 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.0369999999999999,"15min":1.0609999999999999},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.2310000000000001,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":1.3160000000000001},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.004,"15min":1.0189999999999999},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.25,"5min":1.6439999999999999,"15min":1.6439999999999999},"last":1.2669999999999999}]},{"osd":7,"last update":"Fri Sep 13 14:23:13 2024","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.0269999999999999,"15min":1.0449999999999999},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":1.141},{"interface":"front","average":{"1min":1.002,"5min":1.163,"15min":1.2110000000000001},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.3,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":1.232}]}]},{"osd":2,"up_from":18,"seq":77309411428,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:22:48 2024","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.65900000000000003,"15min":0.61899999999999999},"min":{"1min":0.42199999999999999,"5min":0.34799999999999998,"15min":0.33300000000000002},"max":{"1min":0.85699999999999998,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.73799999999999999,"15min":0.63100000000000001},"min":{"1min":0.36099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.046,"5min":1.131,"15min":1.131},"last":0.79600000000000004}]},{"osd":1,"last update":"Fri Sep 13 14:22:48 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.67500000000000004,"15min":0.61399999999999999},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.85999999999999999,"5min":1.014,"15min":1.014},"last":0.754},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.77800000000000002,"15min":0.67100000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.62}]},{"osd":3,"last update":"Fri Sep 13 14:23:42 2024","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.83199999999999996,"15min":0.80800000000000005},"min":{"1min":0.44700000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.2,"5min":1.381,"15min":1.381},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.77200000000000002,"15min":0.72599999999999998},"min":{"1min":0.39900000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.03,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.91400000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:23:30 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.96999999999999997,"15min":0.90000000000000002},"min":{"1min":0.624,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.3009999999999999,"5min":1.881,"15min":1.881},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.97699999999999998,"15min":0.93100000000000005},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":1.012}]},{"osd":5,"last update":"Fri Sep 13 14:23:30 2024","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.0009999999999999,"15min":0.99299999999999999},"min":{"1min":0.67200000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.081},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.98699999999999999,"15min":0.93799999999999994},"min":{"1min":0.56899999999999995,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.363,"5min":1.446,"15min":1.446},"last":1.0640000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:23:17 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.95199999999999996,"15min":0.96999999999999997},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.149,"5min":1.915,"15min":1.915},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"min":{"1min":0.623,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.254,"5min":1.599,"15min":1.599},"last":1.2350000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:23:17 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.115,"15min":1.1319999999999999},"min":{"1min":0.75,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.375,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.115,"15min":1.127},"min":{"1min":0.77500000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.341,"5min":1.528,"15min":1.528},"last":1.2949999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574958,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":245792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:22:55 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.69199999999999995,"15min":0.57799999999999996},"min":{"1min":0.45700000000000002,"5min":0.42399999999999999,"15min":0.33200000000000002},"max":{"1min":0.94099999999999995,"5min":1.153,"15min":1.153},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.70099999999999996,"15min":0.58599999999999997},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.30199999999999999},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.64700000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.71899999999999997,"15min":0.64000000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.76100000000000001,"15min":0.67800000000000005},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.374},"max":{"1min":1.012,"5min":1.194,"15min":1.194},"last":0.81399999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":0.78600000000000003,"15min":0.75},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.96299999999999997,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.83299999999999996,"15min":0.78400000000000003},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.355,"5min":1.355,"15min":1.355},"last":0.98999999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:23:30 2024","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.97999999999999998,"15min":0.92400000000000004},"min":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.1759999999999999,"5min":1.647,"15min":1.647},"last":1.0429999999999999},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.92300000000000004,"15min":0.872},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2210000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":0.93400000000000005}]},{"osd":5,"last update":"Fri Sep 13 14:23:21 2024","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.96799999999999997,"15min":0.94199999999999995},"min":{"1min":0.79800000000000004,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.1459999999999999,"5min":1.9339999999999999,"15min":1.9339999999999999},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0109999999999999,"15min":0.95099999999999996},"min":{"1min":0.874,"5min":0.69199999999999995,"15min":0.69199999999999995},"max":{"1min":1.403,"5min":1.7929999999999999,"15min":1.7929999999999999},"last":0.86499999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:23:21 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.05,"15min":1.0489999999999999},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"max":{"1min":1.244,"5min":1.458,"15min":1.458},"last":1.103},{"interface":"front","average":{"1min":1,"5min":1.0049999999999999,"15min":0.98199999999999998},"min":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"max":{"1min":1.2829999999999999,"5min":1.716,"15min":1.716},"last":0.747}]},{"osd":7,"last update":"Fri Sep 13 14:23:11 2024","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0569999999999999,"15min":1.0580000000000001},"min":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"max":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.96099999999999997},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.1479999999999999,"15min":1.1539999999999999},"min":{"1min":0.85699999999999998,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.5329999999999999,"5min":1.8919999999999999,"15min":1.8919999999999999},"last":1.155}]}]},{"osd":0,"up_from":8,"seq":34359738488,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:22:53 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.63900000000000001,"15min":0.57699999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.30499999999999999},"max":{"1min":1.323,"5min":1.323,"15min":1.323},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.77400000000000002,"15min":0.625},"min":{"1min":0.41899999999999998,"5min":0.41799999999999998,"15min":0.26700000000000002},"max":{"1min":1.448,"5min":1.448,"15min":1.448},"last":0.93100000000000005}]},{"osd":2,"last update":"Fri Sep 13 14:22:53 2024","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.78200000000000003,"15min":0.65600000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.35799999999999998},"max":{"1min":1.5329999999999999,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":1.0289999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.71199999999999997,"15min":0.61799999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.0009999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.69699999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:22:48 2024","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.86199999999999999,"15min":0.78200000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.075},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.78600000000000003,"15min":0.73999999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.147,"5min":1.147,"15min":1.147},"last":0.80600000000000005}]},{"osd":4,"last update":"Fri Sep 13 14:23:31 2024","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.90200000000000002,"15min":0.84199999999999997},"min":{"1min":0.67500000000000004,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.931,"5min":1.931,"15min":1.931},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.98099999999999998,"15min":0.92800000000000005},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.177,"5min":1.431,"15min":1.431},"last":1.1180000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:23:26 2024","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.93899999999999995,"15min":0.91700000000000004},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":1},{"interface":"front","average":{"1min":0.95599999999999996,"5min":1.0089999999999999,"15min":1.012},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.3660000000000001,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":0.83899999999999997}]},{"osd":6,"last update":"Fri Sep 13 14:23:16 2024","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.92800000000000005,"15min":0.93000000000000005},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.145,"5min":1.286,"15min":1.286},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.071,"5min":1.0109999999999999,"15min":1.002},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":2.2040000000000002,"5min":2.2040000000000002,"15min":2.2040000000000002},"last":0.88700000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:23:16 2024","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.0620000000000001,"15min":1.0529999999999999},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":2.1200000000000001,"5min":2.1200000000000001,"15min":2.1200000000000001},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.117,"15min":1.123},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.3340000000000001,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":1.163}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:23:47.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:23:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:48 smithi177 bash[24494]: audit 2024-09-13T14:23:46.973968+0000 mgr.y (mgr.24287) 113 : audit [DBG] from='client.14556 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:23:49.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:48 smithi177 bash[24494]: cluster 2024-09-13T14:23:47.106470+0000 mgr.y (mgr.24287) 114 : cluster [DBG] pgmap v90: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:49.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:48 smithi099 bash[22656]: audit 2024-09-13T14:23:46.973968+0000 mgr.y (mgr.24287) 113 : audit [DBG] from='client.14556 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:23:49.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:48 smithi099 bash[22656]: cluster 2024-09-13T14:23:47.106470+0000 mgr.y (mgr.24287) 114 : cluster [DBG] pgmap v90: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:49.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:48 smithi099 bash[32082]: audit 2024-09-13T14:23:46.973968+0000 mgr.y (mgr.24287) 113 : audit [DBG] from='client.14556 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:23:49.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:48 smithi099 bash[32082]: cluster 2024-09-13T14:23:47.106470+0000 mgr.y (mgr.24287) 114 : cluster [DBG] pgmap v90: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:51.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:50 smithi177 bash[24494]: cluster 2024-09-13T14:23:49.107089+0000 mgr.y (mgr.24287) 115 : cluster [DBG] pgmap v91: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:51.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:50 smithi099 bash[22656]: cluster 2024-09-13T14:23:49.107089+0000 mgr.y (mgr.24287) 115 : cluster [DBG] pgmap v91: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:51.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:50 smithi099 bash[32082]: cluster 2024-09-13T14:23:49.107089+0000 mgr.y (mgr.24287) 115 : cluster [DBG] pgmap v91: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:52.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:23:52 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:52] "GET /metrics HTTP/1.1" 200 203870 "" "Prometheus/2.43.0" 2024-09-13T14:23:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:23:53.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:52 smithi099 bash[22656]: cluster 2024-09-13T14:23:51.107594+0000 mgr.y (mgr.24287) 116 : cluster [DBG] pgmap v92: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:53.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:52 smithi099 bash[22656]: audit 2024-09-13T14:23:52.152433+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:53.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:52 smithi099 bash[32082]: cluster 2024-09-13T14:23:51.107594+0000 mgr.y (mgr.24287) 116 : cluster [DBG] pgmap v92: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:53.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:52 smithi099 bash[32082]: audit 2024-09-13T14:23:52.152433+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:53.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:52 smithi177 bash[24494]: cluster 2024-09-13T14:23:51.107594+0000 mgr.y (mgr.24287) 116 : cluster [DBG] pgmap v92: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:53.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:52 smithi177 bash[24494]: audit 2024-09-13T14:23:52.152433+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:55.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:54 smithi177 bash[24494]: cluster 2024-09-13T14:23:53.108214+0000 mgr.y (mgr.24287) 117 : cluster [DBG] pgmap v93: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:55.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:54 smithi099 bash[22656]: cluster 2024-09-13T14:23:53.108214+0000 mgr.y (mgr.24287) 117 : cluster [DBG] pgmap v93: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:55.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:54 smithi099 bash[32082]: cluster 2024-09-13T14:23:53.108214+0000 mgr.y (mgr.24287) 117 : cluster [DBG] pgmap v93: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:55.856 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:23:55 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:23:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:23:56.477 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:23:56.478 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:23:57.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:56 smithi099 bash[22656]: cluster 2024-09-13T14:23:55.108761+0000 mgr.y (mgr.24287) 118 : cluster [DBG] pgmap v94: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:57.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:56 smithi099 bash[32082]: cluster 2024-09-13T14:23:55.108761+0000 mgr.y (mgr.24287) 118 : cluster [DBG] pgmap v94: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:57.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:56 smithi177 bash[24494]: cluster 2024-09-13T14:23:55.108761+0000 mgr.y (mgr.24287) 118 : cluster [DBG] pgmap v94: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:57.261 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":94,"stamp":"2024-09-13T14:23:55.108503+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":115,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385156,"kb_used_data":6312,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576060,"statfs":{"total":767960285184,"available":765517885440,"internally_reserved":0,"allocated":6463488,"data_stored":3738605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003265"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.531004+0000","last_change":"2024-09-13T14:22:03.178246+0000","last_active":"2024-09-13T14:22:13.531004+0000","last_peered":"2024-09-13T14:22:13.531004+0000","last_clean":"2024-09-13T14:22:13.531004+0000","last_became_active":"2024-09-13T14:21:52.861399+0000","last_became_peered":"2024-09-13T14:21:52.861399+0000","last_unstale":"2024-09-13T14:22:13.531004+0000","last_undegraded":"2024-09-13T14:22:13.531004+0000","last_fullsized":"2024-09-13T14:22:13.531004+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:51.432081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.513123+0000","last_change":"2024-09-13T14:22:03.172028+0000","last_active":"2024-09-13T14:22:13.513123+0000","last_peered":"2024-09-13T14:22:13.513123+0000","last_clean":"2024-09-13T14:22:13.513123+0000","last_became_active":"2024-09-13T14:21:53.095058+0000","last_became_peered":"2024-09-13T14:21:53.095058+0000","last_unstale":"2024-09-13T14:22:13.513123+0000","last_undegraded":"2024-09-13T14:22:13.513123+0000","last_fullsized":"2024-09-13T14:22:13.513123+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:44.515568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080647399999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.495140+0000","last_change":"2024-09-13T14:22:03.357990+0000","last_active":"2024-09-13T14:22:13.495140+0000","last_peered":"2024-09-13T14:22:13.495140+0000","last_clean":"2024-09-13T14:22:13.495140+0000","last_became_active":"2024-09-13T14:21:53.093638+0000","last_became_peered":"2024-09-13T14:21:53.093638+0000","last_unstale":"2024-09-13T14:22:13.495140+0000","last_undegraded":"2024-09-13T14:22:13.495140+0000","last_fullsized":"2024-09-13T14:22:13.495140+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:24.868073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.513101+0000","last_change":"2024-09-13T14:22:03.171905+0000","last_active":"2024-09-13T14:22:13.513101+0000","last_peered":"2024-09-13T14:22:13.513101+0000","last_clean":"2024-09-13T14:22:13.513101+0000","last_became_active":"2024-09-13T14:21:52.884118+0000","last_became_peered":"2024-09-13T14:21:52.884118+0000","last_unstale":"2024-09-13T14:22:13.513101+0000","last_undegraded":"2024-09-13T14:22:13.513101+0000","last_fullsized":"2024-09-13T14:22:13.513101+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:26:17.407017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.692258+0000","last_change":"2024-09-13T14:22:03.177356+0000","last_active":"2024-09-13T14:22:13.692258+0000","last_peered":"2024-09-13T14:22:13.692258+0000","last_clean":"2024-09-13T14:22:13.692258+0000","last_became_active":"2024-09-13T14:21:52.861658+0000","last_became_peered":"2024-09-13T14:21:52.861658+0000","last_unstale":"2024-09-13T14:22:13.692258+0000","last_undegraded":"2024-09-13T14:22:13.692258+0000","last_fullsized":"2024-09-13T14:22:13.692258+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:10.200310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.064427+0000","last_change":"2024-09-13T14:22:03.172771+0000","last_active":"2024-09-13T14:22:13.064427+0000","last_peered":"2024-09-13T14:22:13.064427+0000","last_clean":"2024-09-13T14:22:13.064427+0000","last_became_active":"2024-09-13T14:21:52.884004+0000","last_became_peered":"2024-09-13T14:21:52.884004+0000","last_unstale":"2024-09-13T14:22:13.064427+0000","last_undegraded":"2024-09-13T14:22:13.064427+0000","last_fullsized":"2024-09-13T14:22:13.064427+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:20:21.593867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.495137+0000","last_change":"2024-09-13T14:22:03.357964+0000","last_active":"2024-09-13T14:22:13.495137+0000","last_peered":"2024-09-13T14:22:13.495137+0000","last_clean":"2024-09-13T14:22:13.495137+0000","last_became_active":"2024-09-13T14:21:52.883199+0000","last_became_peered":"2024-09-13T14:21:52.883199+0000","last_unstale":"2024-09-13T14:22:13.495137+0000","last_undegraded":"2024-09-13T14:22:13.495137+0000","last_fullsized":"2024-09-13T14:22:13.495137+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:09.955395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T14:22:13.692347+0000","last_change":"2024-09-13T14:22:03.176719+0000","last_active":"2024-09-13T14:22:13.692347+0000","last_peered":"2024-09-13T14:22:13.692347+0000","last_clean":"2024-09-13T14:22:13.692347+0000","last_became_active":"2024-09-13T14:21:52.862358+0000","last_became_peered":"2024-09-13T14:21:52.862358+0000","last_unstale":"2024-09-13T14:22:13.692347+0000","last_undegraded":"2024-09-13T14:22:13.692347+0000","last_fullsized":"2024-09-13T14:22:13.692347+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:22:54.637897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":49,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T14:22:03.357270+0000","last_change":"2024-09-13T14:20:07.527898+0000","last_active":"2024-09-13T14:22:03.357270+0000","last_peered":"2024-09-13T14:22:03.357270+0000","last_clean":"2024-09-13T14:22:03.357270+0000","last_became_active":"2024-09-13T14:20:07.515370+0000","last_became_peered":"2024-09-13T14:20:07.515370+0000","last_unstale":"2024-09-13T14:22:03.357270+0000","last_undegraded":"2024-09-13T14:22:03.357270+0000","last_fullsized":"2024-09-13T14:22:03.357270+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:45.300816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397552,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.84599999999999997,"15min":0.83999999999999997},"min":{"1min":0.56399999999999995,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.1859999999999999,"5min":1.268,"15min":1.268},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.88,"15min":0.88800000000000001},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.22,"5min":1.22,"15min":1.22},"last":0.79700000000000004}]},{"osd":1,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.88900000000000001,"15min":0.89600000000000002},"min":{"1min":0.56000000000000005,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.224,"5min":1.278,"15min":1.278},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.89300000000000002,"15min":0.90200000000000002},"min":{"1min":0.57399999999999995,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.1060000000000001,"5min":1.234,"15min":1.234},"last":1.0669999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.96299999999999997,"15min":0.97199999999999998},"min":{"1min":0.61299999999999999,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.1779999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.98899999999999999,"15min":0.98499999999999999},"min":{"1min":0.64800000000000002,"5min":0.497,"15min":0.497},"max":{"1min":1.2350000000000001,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.91100000000000003}]},{"osd":3,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.94499999999999995,"15min":0.94799999999999995},"min":{"1min":0.60399999999999998,"5min":0.44,"15min":0.44},"max":{"1min":1.1859999999999999,"5min":1.375,"15min":1.375},"last":1.0920000000000001},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.95099999999999996,"15min":0.94799999999999995},"min":{"1min":0.67600000000000005,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.173,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.0069999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.88700000000000001,"15min":0.89300000000000002},"min":{"1min":0.41599999999999998,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.337,"5min":1.337,"15min":1.337},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.96899999999999997,"15min":0.97499999999999998},"min":{"1min":0.44500000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.2949999999999999,"5min":1.325,"15min":1.325},"last":1.036}]},{"osd":5,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.93500000000000005,"15min":0.92000000000000004},"min":{"1min":0.38200000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2749999999999999,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"min":{"1min":0.47099999999999997,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.3100000000000001,"5min":1.401,"15min":1.401},"last":0.96799999999999997}]},{"osd":6,"last update":"Fri Sep 13 14:23:14 2024","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0149999999999999,"15min":1.004},"min":{"1min":0.55100000000000005,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.365,"5min":1.401,"15min":1.401},"last":1.1140000000000001},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.0660000000000001,"15min":1.0680000000000001},"min":{"1min":0.49299999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.419,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":0.94699999999999995}]}]},{"osd":6,"up_from":43,"seq":184683593787,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":835845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.871,"15min":0.86499999999999999},"min":{"1min":0.66500000000000004,"5min":0.379,"15min":0.379},"max":{"1min":1.238,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.83699999999999997,"15min":0.82699999999999996},"min":{"1min":0.56999999999999995,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.95199999999999996}]},{"osd":1,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.85699999999999998,"15min":0.83199999999999996},"min":{"1min":0.60699999999999998,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.141,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.88,"5min":0.86599999999999999,"15min":0.85999999999999999},"min":{"1min":0.69499999999999995,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.05,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.139}]},{"osd":2,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.88,"15min":0.875},"min":{"1min":0.64100000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.1479999999999999,"5min":1.419,"15min":1.419},"last":0.90900000000000003},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.90500000000000003,"15min":0.879},"min":{"1min":0.67600000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":0.76700000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.88700000000000001,"15min":0.86399999999999999},"min":{"1min":0.52600000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.169,"5min":1.659,"15min":1.659},"last":0.98499999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.91000000000000003,"15min":0.89500000000000002},"min":{"1min":0.67000000000000004,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.111,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":1.0529999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.81599999999999995,"15min":0.77100000000000002},"min":{"1min":0.51700000000000002,"5min":0.313,"15min":0.313},"max":{"1min":1.258,"5min":1.258,"15min":1.258},"last":0.876},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.82999999999999996,"15min":0.77000000000000002},"min":{"1min":0.61499999999999999,"5min":0.317,"15min":0.317},"max":{"1min":1.2070000000000001,"5min":1.423,"15min":1.423},"last":1.1080000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:23:22 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.89000000000000001,"15min":0.84499999999999997},"min":{"1min":0.66200000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":1.085},{"interface":"front","average":{"1min":0.995,"5min":0.91000000000000003,"15min":0.84699999999999998},"min":{"1min":0.64300000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.2230000000000001,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":1.1639999999999999}]},{"osd":7,"last update":"Fri Sep 13 14:23:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.98299999999999998,"15min":0.98299999999999998},"min":{"1min":0.63,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.1930000000000001,"5min":1.353,"15min":1.353},"last":1.214},{"interface":"front","average":{"1min":1.093,"5min":1.0289999999999999,"15min":1.0129999999999999},"min":{"1min":0.81999999999999995,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.464,"5min":1.464,"15min":1.464},"last":1.024}]}]},{"osd":5,"up_from":37,"seq":158913790021,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":245792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.82699999999999996,"15min":0.81100000000000005},"min":{"1min":0.54900000000000004,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.1719999999999999,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.874,"15min":0.82799999999999996},"min":{"1min":0.621,"5min":0.505,"15min":0.505},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":1.0700000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.86399999999999999,"15min":0.82699999999999996},"min":{"1min":0.66000000000000003,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.163,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":0.997},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.875,"15min":0.81899999999999995},"min":{"1min":0.68300000000000005,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.2190000000000001,"5min":2.0459999999999998,"15min":2.0459999999999998},"last":0.94999999999999996}]},{"osd":2,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.85999999999999999,"15min":0.80800000000000005},"min":{"1min":0.66400000000000003,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.2290000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.88900000000000001,"15min":0.85899999999999999},"min":{"1min":0.67000000000000004,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.141,"5min":1.27,"15min":1.27},"last":0.79700000000000004}]},{"osd":3,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":0.92400000000000004,"15min":0.86399999999999999},"min":{"1min":0.77500000000000002,"5min":0.497,"15min":0.497},"max":{"1min":1.2410000000000001,"5min":1.573,"15min":1.573},"last":1.1000000000000001},{"interface":"front","average":{"1min":1.018,"5min":0.94699999999999995,"15min":0.871},"min":{"1min":0.81200000000000006,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.2290000000000001,"5min":2.246,"15min":2.246},"last":0.872}]},{"osd":4,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.77500000000000002,"15min":0.67100000000000004},"min":{"1min":0.63700000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.84099999999999997},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.86699999999999999,"15min":0.73599999999999999},"min":{"1min":0.61599999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.3080000000000001,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":1.0429999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:23:20 2024","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":0.877,"15min":0.82799999999999996},"min":{"1min":0.77600000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.3799999999999999,"5min":1.488,"15min":1.488},"last":1.1799999999999999},{"interface":"front","average":{"1min":1.03,"5min":0.95399999999999996,"15min":0.92500000000000004},"min":{"1min":0.68899999999999995,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.4399999999999999,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.161}]},{"osd":7,"last update":"Fri Sep 13 14:23:07 2024","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.0049999999999999,"15min":1.006},"min":{"1min":0.83899999999999997,"5min":0.376,"15min":0.376},"max":{"1min":1.8080000000000001,"5min":1.8080000000000001,"15min":1.8080000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.077,"5min":0.96699999999999997,"15min":0.96099999999999997},"min":{"1min":0.69399999999999995,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.4850000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":1.121}]}]},{"osd":4,"up_from":31,"seq":133143986256,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297924,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447228,"statfs":{"total":95995035648,"available":95689961472,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:23:35 2024","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.86299999999999999,"15min":0.81100000000000005},"min":{"1min":0.52800000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2030000000000001,"5min":1.224,"15min":1.224},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.85999999999999999,"15min":0.84399999999999997},"min":{"1min":0.65800000000000003,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.143,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.95399999999999996}]},{"osd":1,"last update":"Fri Sep 13 14:23:35 2024","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.879,"15min":0.80600000000000005},"min":{"1min":0.67100000000000004,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.1200000000000001,"5min":1.282,"15min":1.282},"last":1.034},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.85999999999999999,"15min":0.83599999999999997},"min":{"1min":0.68300000000000005,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.872}]},{"osd":2,"last update":"Fri Sep 13 14:23:35 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.88700000000000001,"15min":0.878},"min":{"1min":0.65000000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.1060000000000001,"5min":1.3200000000000001,"15min":1.3939999999999999},"last":1},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.90000000000000002,"15min":0.83399999999999996},"min":{"1min":0.67700000000000005,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.129,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.82799999999999996}]},{"osd":3,"last update":"Fri Sep 13 14:23:35 2024","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.90600000000000003,"15min":0.86199999999999999},"min":{"1min":0.72999999999999998,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.1879999999999999,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":1.1859999999999999},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.93300000000000005,"15min":0.85999999999999999},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.131,"5min":1.224,"15min":1.224},"last":0.91600000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:23:24 2024","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.83799999999999997,"15min":0.76400000000000001},"min":{"1min":0.501,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.1020000000000001,"5min":1.238,"15min":1.238},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.85399999999999998,"15min":0.748},"min":{"1min":0.47599999999999998,"5min":0.37,"15min":0.37},"max":{"1min":1.238,"5min":1.298,"15min":1.298},"last":1.0680000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:23:21 2024","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.85399999999999998,"15min":0.80000000000000004},"min":{"1min":0.44500000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.222,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":1.095},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.89900000000000002,"15min":0.84299999999999997},"min":{"1min":0.57799999999999996,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.252,"5min":1.431,"15min":1.431},"last":1.1120000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.036,"15min":1.0640000000000001},"min":{"1min":0.53100000000000003,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.2849999999999999,"5min":1.343,"15min":1.343},"last":1.1379999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.101,"15min":1.123},"min":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.2070000000000001,"5min":1.504,"15min":1.504},"last":1.1599999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149787,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:23:51 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.66400000000000003,"15min":0.65400000000000003},"min":{"1min":0.40200000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.93100000000000005,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.70799999999999996,"15min":0.68700000000000006},"min":{"1min":0.32200000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.90300000000000002,"5min":1.3,"15min":1.3},"last":0.64400000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:23:51 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.66200000000000003,"15min":0.66500000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.84999999999999998,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.76400000000000001,"15min":0.74099999999999999},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.91400000000000003,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.88700000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:23:51 2024","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.82699999999999996,"15min":0.76800000000000002},"min":{"1min":0.41699999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.131,"5min":1.296,"15min":1.296},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.77000000000000002,"15min":0.747},"min":{"1min":0.33900000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.94799999999999995,"5min":1.2310000000000001,"15min":1.2310000000000001},"last":0.78900000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:23:32 2024","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.90400000000000003,"15min":0.877},"min":{"1min":0.57999999999999996,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2110000000000001,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.95999999999999996,"15min":0.90600000000000003},"min":{"1min":0.58499999999999996,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.1950000000000001,"5min":1.395,"15min":1.395},"last":1.4950000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:23:27 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.98499999999999999,"15min":0.96099999999999997},"min":{"1min":0.60099999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.258,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":1.3580000000000001},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.92700000000000005,"15min":0.90400000000000003},"min":{"1min":0.56200000000000006,"5min":0.434,"15min":0.434},"max":{"1min":1.2569999999999999,"5min":1.7569999999999999,"15min":1.7569999999999999},"last":0.96199999999999997}]},{"osd":6,"last update":"Fri Sep 13 14:23:21 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.0369999999999999,"15min":1.0609999999999999},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.2310000000000001,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":1.409},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.004,"15min":1.0189999999999999},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.25,"5min":1.6439999999999999,"15min":1.6439999999999999},"last":0.93500000000000005}]},{"osd":7,"last update":"Fri Sep 13 14:23:13 2024","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.0269999999999999,"15min":1.0449999999999999},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":1.0209999999999999},{"interface":"front","average":{"1min":1.002,"5min":1.163,"15min":1.2110000000000001},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.3,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":1.4510000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411430,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:23:49 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.65800000000000003,"15min":0.62},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33300000000000002},"max":{"1min":0.85599999999999998,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.72699999999999998,"15min":0.63600000000000001},"min":{"1min":0.36299999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.92200000000000004,"5min":1.131,"15min":1.131},"last":0.69199999999999995}]},{"osd":1,"last update":"Fri Sep 13 14:23:49 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.66200000000000003,"15min":0.61499999999999999},"min":{"1min":0.35799999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.83099999999999996,"5min":1.014,"15min":1.014},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.76900000000000002,"15min":0.67900000000000005},"min":{"1min":0.41199999999999998,"5min":0.373,"15min":0.373},"max":{"1min":1.002,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.88600000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:23:42 2024","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.83199999999999996,"15min":0.80800000000000005},"min":{"1min":0.44700000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.2,"5min":1.381,"15min":1.381},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.77200000000000002,"15min":0.72599999999999998},"min":{"1min":0.39900000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.03,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.60699999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:23:30 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.96999999999999997,"15min":0.90000000000000002},"min":{"1min":0.624,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.3009999999999999,"5min":1.881,"15min":1.881},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.97699999999999998,"15min":0.93100000000000005},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":1.069}]},{"osd":5,"last update":"Fri Sep 13 14:23:30 2024","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.0009999999999999,"15min":0.99299999999999999},"min":{"1min":0.67200000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.1120000000000001},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.98699999999999999,"15min":0.93799999999999994},"min":{"1min":0.56899999999999995,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.363,"5min":1.446,"15min":1.446},"last":0.96799999999999997}]},{"osd":6,"last update":"Fri Sep 13 14:23:17 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.95199999999999996,"15min":0.96999999999999997},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.149,"5min":1.915,"15min":1.915},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"min":{"1min":0.623,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.254,"5min":1.599,"15min":1.599},"last":1.042}]},{"osd":7,"last update":"Fri Sep 13 14:23:17 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.115,"15min":1.1319999999999999},"min":{"1min":0.75,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.375,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.2070000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.115,"15min":1.127},"min":{"1min":0.77500000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.341,"5min":1.528,"15min":1.528},"last":0.92100000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574960,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":245792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:22:55 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.69199999999999995,"15min":0.57799999999999996},"min":{"1min":0.45700000000000002,"5min":0.42399999999999999,"15min":0.33200000000000002},"max":{"1min":0.94099999999999995,"5min":1.153,"15min":1.153},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.70099999999999996,"15min":0.58599999999999997},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.30199999999999999},"max":{"1min":1.3260000000000001,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.82599999999999996}]},{"osd":2,"last update":"Fri Sep 13 14:23:49 2024","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.72399999999999998,"15min":0.64800000000000002},"min":{"1min":0.39300000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0209999999999999,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.997},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.75700000000000001,"15min":0.68500000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.93999999999999995,"5min":1.194,"15min":1.194},"last":0.93899999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:23:49 2024","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.77500000000000002,"15min":0.748},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.92300000000000004,"5min":1.1850000000000001,"15min":1.2110000000000001},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.82899999999999996,"15min":0.78700000000000003},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.1120000000000001,"5min":1.355,"15min":1.355},"last":1.1120000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:23:30 2024","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.97999999999999998,"15min":0.92400000000000004},"min":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.1759999999999999,"5min":1.647,"15min":1.647},"last":1.1990000000000001},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.92300000000000004,"15min":0.872},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2210000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":1.0429999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:23:21 2024","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.96799999999999997,"15min":0.94199999999999995},"min":{"1min":0.79800000000000004,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.1459999999999999,"5min":1.9339999999999999,"15min":1.9339999999999999},"last":1.234},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0109999999999999,"15min":0.95099999999999996},"min":{"1min":0.874,"5min":0.69199999999999995,"15min":0.69199999999999995},"max":{"1min":1.403,"5min":1.7929999999999999,"15min":1.7929999999999999},"last":1.4319999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:23:21 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.05,"15min":1.0489999999999999},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"max":{"1min":1.244,"5min":1.458,"15min":1.458},"last":1.385},{"interface":"front","average":{"1min":1,"5min":1.0049999999999999,"15min":0.98199999999999998},"min":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"max":{"1min":1.2829999999999999,"5min":1.716,"15min":1.716},"last":1.1819999999999999}]},{"osd":7,"last update":"Fri Sep 13 14:23:11 2024","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0569999999999999,"15min":1.0580000000000001},"min":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"max":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":1.516},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.1479999999999999,"15min":1.1539999999999999},"min":{"1min":0.85699999999999998,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.5329999999999999,"5min":1.8919999999999999,"15min":1.8919999999999999},"last":1.4650000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738490,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:22:53 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.63900000000000001,"15min":0.57699999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.30499999999999999},"max":{"1min":1.323,"5min":1.323,"15min":1.323},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.77400000000000002,"15min":0.625},"min":{"1min":0.41899999999999998,"5min":0.41799999999999998,"15min":0.26700000000000002},"max":{"1min":1.448,"5min":1.448,"15min":1.448},"last":0.84099999999999997}]},{"osd":2,"last update":"Fri Sep 13 14:22:53 2024","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.78200000000000003,"15min":0.65600000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.35799999999999998},"max":{"1min":1.5329999999999999,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.71199999999999997,"15min":0.61799999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.0009999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.88500000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:23:51 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.85399999999999998,"15min":0.78500000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.77100000000000002,"15min":0.73799999999999999},"min":{"1min":0.39000000000000001,"5min":0.376,"15min":0.376},"max":{"1min":1.0029999999999999,"5min":1.147,"15min":1.147},"last":0.80500000000000005}]},{"osd":4,"last update":"Fri Sep 13 14:23:31 2024","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.90200000000000002,"15min":0.84199999999999997},"min":{"1min":0.67500000000000004,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.931,"5min":1.931,"15min":1.931},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.98099999999999998,"15min":0.92800000000000005},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.177,"5min":1.431,"15min":1.431},"last":0.98499999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:23:26 2024","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.93899999999999995,"15min":0.91700000000000004},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.95599999999999996,"5min":1.0089999999999999,"15min":1.012},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.3660000000000001,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":1.0289999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:23:16 2024","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.92800000000000005,"15min":0.93000000000000005},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.145,"5min":1.286,"15min":1.286},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.071,"5min":1.0109999999999999,"15min":1.002},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":2.2040000000000002,"5min":2.2040000000000002,"15min":2.2040000000000002},"last":1.0549999999999999}]},{"osd":7,"last update":"Fri Sep 13 14:23:16 2024","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.0620000000000001,"15min":1.0529999999999999},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":2.1200000000000001,"5min":2.1200000000000001,"15min":2.1200000000000001},"last":1.1299999999999999},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.117,"15min":1.123},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.3340000000000001,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":1.0740000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:23:57.263 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:23:57.263 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:23:57.263 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:23:57.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:23:58.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:57 smithi177 bash[24494]: audit 2024-09-13T14:23:56.476631+0000 mgr.y (mgr.24287) 119 : audit [DBG] from='client.24464 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:23:58.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:57 smithi099 bash[22656]: audit 2024-09-13T14:23:56.476631+0000 mgr.y (mgr.24287) 119 : audit [DBG] from='client.24464 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:23:58.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:57 smithi099 bash[32082]: audit 2024-09-13T14:23:56.476631+0000 mgr.y (mgr.24287) 119 : audit [DBG] from='client.24464 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:23:59.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:23:58 smithi177 bash[24494]: cluster 2024-09-13T14:23:57.109371+0000 mgr.y (mgr.24287) 120 : cluster [DBG] pgmap v95: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:59.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:23:58 smithi099 bash[22656]: cluster 2024-09-13T14:23:57.109371+0000 mgr.y (mgr.24287) 120 : cluster [DBG] pgmap v95: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:23:59.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:23:58 smithi099 bash[32082]: cluster 2024-09-13T14:23:57.109371+0000 mgr.y (mgr.24287) 120 : cluster [DBG] pgmap v95: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:01.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:00 smithi099 bash[22656]: cluster 2024-09-13T14:23:59.110069+0000 mgr.y (mgr.24287) 121 : cluster [DBG] pgmap v96: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:01.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:00 smithi099 bash[32082]: cluster 2024-09-13T14:23:59.110069+0000 mgr.y (mgr.24287) 121 : cluster [DBG] pgmap v96: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:01.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:00 smithi177 bash[24494]: cluster 2024-09-13T14:23:59.110069+0000 mgr.y (mgr.24287) 121 : cluster [DBG] pgmap v96: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:02.041 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:02.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:24:02 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:02] "GET /metrics HTTP/1.1" 200 203877 "" "Prometheus/2.43.0" 2024-09-13T14:24:03.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:02 smithi099 bash[22656]: cluster 2024-09-13T14:24:01.110563+0000 mgr.y (mgr.24287) 122 : cluster [DBG] pgmap v97: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:03.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:02 smithi099 bash[32082]: cluster 2024-09-13T14:24:01.110563+0000 mgr.y (mgr.24287) 122 : cluster [DBG] pgmap v97: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:03.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:02 smithi177 bash[24494]: cluster 2024-09-13T14:24:01.110563+0000 mgr.y (mgr.24287) 122 : cluster [DBG] pgmap v97: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:05.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:04 smithi099 bash[22656]: cluster 2024-09-13T14:24:03.111208+0000 mgr.y (mgr.24287) 123 : cluster [DBG] pgmap v98: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:04 smithi099 bash[32082]: cluster 2024-09-13T14:24:03.111208+0000 mgr.y (mgr.24287) 123 : cluster [DBG] pgmap v98: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:05.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:04 smithi177 bash[24494]: cluster 2024-09-13T14:24:03.111208+0000 mgr.y (mgr.24287) 123 : cluster [DBG] pgmap v98: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:05.856 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:24:05 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:24:05.876 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:24:05.877 INFO:teuthology.orchestra.run.smithi099.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:24:06.738 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:24:06.738 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T14:24:06.738 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T14:24:06.750 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi099.front.sepia.ceph.com 2024-09-13T14:24:06.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-09-13T14:24:07.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:06 smithi099 bash[22656]: cluster 2024-09-13T14:24:05.111785+0000 mgr.y (mgr.24287) 124 : cluster [DBG] pgmap v99: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:07.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:06 smithi099 bash[22656]: audit 2024-09-13T14:24:05.883910+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.99:0/749635718' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:24:07.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:06 smithi099 bash[32082]: cluster 2024-09-13T14:24:05.111785+0000 mgr.y (mgr.24287) 124 : cluster [DBG] pgmap v99: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:07.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:06 smithi099 bash[32082]: audit 2024-09-13T14:24:05.883910+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.99:0/749635718' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:24:07.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:06 smithi177 bash[24494]: cluster 2024-09-13T14:24:05.111785+0000 mgr.y (mgr.24287) 124 : cluster [DBG] pgmap v99: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:07.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:06 smithi177 bash[24494]: audit 2024-09-13T14:24:05.883910+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.99:0/749635718' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:24:08.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:07 smithi099 bash[22656]: audit 2024-09-13T14:24:07.152804+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:08.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:07 smithi099 bash[32082]: audit 2024-09-13T14:24:07.152804+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:08.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:07 smithi177 bash[24494]: audit 2024-09-13T14:24:07.152804+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:09.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:08 smithi099 bash[22656]: cluster 2024-09-13T14:24:07.112411+0000 mgr.y (mgr.24287) 125 : cluster [DBG] pgmap v100: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:09.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:08 smithi099 bash[32082]: cluster 2024-09-13T14:24:07.112411+0000 mgr.y (mgr.24287) 125 : cluster [DBG] pgmap v100: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:09.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:08 smithi177 bash[24494]: cluster 2024-09-13T14:24:07.112411+0000 mgr.y (mgr.24287) 125 : cluster [DBG] pgmap v100: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:10.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:09 smithi099 bash[22656]: cluster 2024-09-13T14:24:09.113176+0000 mgr.y (mgr.24287) 126 : cluster [DBG] pgmap v101: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:10.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:09 smithi099 bash[32082]: cluster 2024-09-13T14:24:09.113176+0000 mgr.y (mgr.24287) 126 : cluster [DBG] pgmap v101: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:10.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:09 smithi177 bash[24494]: cluster 2024-09-13T14:24:09.113176+0000 mgr.y (mgr.24287) 126 : cluster [DBG] pgmap v101: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:11.530 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:12.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:12 smithi099 bash[22656]: cluster 2024-09-13T14:24:11.113828+0000 mgr.y (mgr.24287) 127 : cluster [DBG] pgmap v102: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:12.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:24:12 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:12] "GET /metrics HTTP/1.1" 200 203890 "" "Prometheus/2.43.0" 2024-09-13T14:24:12.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:12 smithi099 bash[32082]: cluster 2024-09-13T14:24:11.113828+0000 mgr.y (mgr.24287) 127 : cluster [DBG] pgmap v102: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:12 smithi177 bash[24494]: cluster 2024-09-13T14:24:11.113828+0000 mgr.y (mgr.24287) 127 : cluster [DBG] pgmap v102: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:14.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:14 smithi099 bash[22656]: cluster 2024-09-13T14:24:13.114460+0000 mgr.y (mgr.24287) 128 : cluster [DBG] pgmap v103: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:14.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:14 smithi099 bash[32082]: cluster 2024-09-13T14:24:13.114460+0000 mgr.y (mgr.24287) 128 : cluster [DBG] pgmap v103: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:14.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:14 smithi177 bash[24494]: cluster 2024-09-13T14:24:13.114460+0000 mgr.y (mgr.24287) 128 : cluster [DBG] pgmap v103: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:15.856 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:24:15 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:24:16.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:16 smithi099 bash[22656]: cluster 2024-09-13T14:24:15.114793+0000 mgr.y (mgr.24287) 129 : cluster [DBG] pgmap v104: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:16.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:16 smithi099 bash[22656]: audit 2024-09-13T14:24:15.355339+0000 mgr.y (mgr.24287) 130 : audit [DBG] from='client.14574 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:16.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:16 smithi099 bash[22656]: audit 2024-09-13T14:24:15.356222+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:24:16.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:16 smithi099 bash[22656]: audit 2024-09-13T14:24:15.356807+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:24:16.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:16 smithi099 bash[32082]: cluster 2024-09-13T14:24:15.114793+0000 mgr.y (mgr.24287) 129 : cluster [DBG] pgmap v104: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:16.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:16 smithi099 bash[32082]: audit 2024-09-13T14:24:15.355339+0000 mgr.y (mgr.24287) 130 : audit [DBG] from='client.14574 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:16.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:16 smithi099 bash[32082]: audit 2024-09-13T14:24:15.356222+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:24:16.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:16 smithi099 bash[32082]: audit 2024-09-13T14:24:15.356807+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:24:16.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:16 smithi177 bash[24494]: cluster 2024-09-13T14:24:15.114793+0000 mgr.y (mgr.24287) 129 : cluster [DBG] pgmap v104: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:16.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:16 smithi177 bash[24494]: audit 2024-09-13T14:24:15.355339+0000 mgr.y (mgr.24287) 130 : audit [DBG] from='client.14574 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:16.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:16 smithi177 bash[24494]: audit 2024-09-13T14:24:15.356222+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:24:16.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:16 smithi177 bash[24494]: audit 2024-09-13T14:24:15.356807+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:24:17.483 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:17 smithi099 bash[32082]: audit 2024-09-13T14:24:16.193782+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:24:17.484 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:17 smithi099 bash[32082]: cluster 2024-09-13T14:24:16.193994+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:24:17.484 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:17 smithi099 bash[32082]: audit 2024-09-13T14:24:16.200540+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:17.484 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:17 smithi099 bash[32082]: audit 2024-09-13T14:24:16.201267+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:17.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:17 smithi099 bash[22656]: audit 2024-09-13T14:24:16.193782+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:24:17.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:17 smithi099 bash[22656]: cluster 2024-09-13T14:24:16.193994+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:24:17.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:17 smithi099 bash[22656]: audit 2024-09-13T14:24:16.200540+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:17.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:17 smithi099 bash[22656]: audit 2024-09-13T14:24:16.201267+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:17.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:17 smithi099 bash[22656]: debug 2024-09-13T14:24:17.236+0000 7fea4a400640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:24:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:17 smithi177 bash[24494]: audit 2024-09-13T14:24:16.193782+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:24:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:17 smithi177 bash[24494]: cluster 2024-09-13T14:24:16.193994+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:24:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:17 smithi177 bash[24494]: audit 2024-09-13T14:24:16.200540+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:17 smithi177 bash[24494]: audit 2024-09-13T14:24:16.201267+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:18.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: cluster 2024-09-13T14:24:17.115380+0000 mgr.y (mgr.24287) 131 : cluster [DBG] pgmap v106: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:18.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: cluster 2024-09-13T14:24:17.188088+0000 mon.a (mon.0) 746 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:24:18.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.208189+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:24:18.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: cluster 2024-09-13T14:24:17.208614+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:24:18.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.234881+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:18.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.236554+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:18.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: cluster 2024-09-13T14:24:17.237933+0000 mon.a (mon.0) 750 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:24:18.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: cluster 2024-09-13T14:24:17.237969+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:24:18.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: cluster 2024-09-13T14:24:17.254090+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:24:18.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.254221+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:24:18.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: cluster 2024-09-13T14:24:17.254328+0000 mon.a (mon.0) 754 : cluster [DBG] fsmap foo:0 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: cephadm 2024-09-13T14:24:17.262796+0000 mgr.y (mgr.24287) 132 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.270877+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.273728+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.276873+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.279113+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.286160+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.289965+0000 mon.c (mon.2) 88 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.290340+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.294703+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: audit 2024-09-13T14:24:17.295890+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[22656]: cephadm 2024-09-13T14:24:17.297079+0000 mgr.y (mgr.24287) 133 : cephadm [INF] Deploying daemon mds.foo.smithi099.yyrtld on smithi099 2024-09-13T14:24:18.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: cluster 2024-09-13T14:24:17.115380+0000 mgr.y (mgr.24287) 131 : cluster [DBG] pgmap v106: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: cluster 2024-09-13T14:24:17.188088+0000 mon.a (mon.0) 746 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.208189+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: cluster 2024-09-13T14:24:17.208614+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.234881+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.236554+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: cluster 2024-09-13T14:24:17.237933+0000 mon.a (mon.0) 750 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: cluster 2024-09-13T14:24:17.237969+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: cluster 2024-09-13T14:24:17.254090+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.254221+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: cluster 2024-09-13T14:24:17.254328+0000 mon.a (mon.0) 754 : cluster [DBG] fsmap foo:0 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: cephadm 2024-09-13T14:24:17.262796+0000 mgr.y (mgr.24287) 132 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.270877+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:18.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.273728+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:18.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.276873+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:18.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.279113+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:18.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.286160+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:18.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.289965+0000 mon.c (mon.2) 88 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:18.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.290340+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:18.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.294703+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:24:18.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: audit 2024-09-13T14:24:17.295890+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:18.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:18 smithi099 bash[32082]: cephadm 2024-09-13T14:24:17.297079+0000 mgr.y (mgr.24287) 133 : cephadm [INF] Deploying daemon mds.foo.smithi099.yyrtld on smithi099 2024-09-13T14:24:18.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: cluster 2024-09-13T14:24:17.115380+0000 mgr.y (mgr.24287) 131 : cluster [DBG] pgmap v106: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:18.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: cluster 2024-09-13T14:24:17.188088+0000 mon.a (mon.0) 746 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:24:18.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.208189+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:24:18.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: cluster 2024-09-13T14:24:17.208614+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:24:18.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.234881+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.236554+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: cluster 2024-09-13T14:24:17.237933+0000 mon.a (mon.0) 750 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: cluster 2024-09-13T14:24:17.237969+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: cluster 2024-09-13T14:24:17.254090+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.254221+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: cluster 2024-09-13T14:24:17.254328+0000 mon.a (mon.0) 754 : cluster [DBG] fsmap foo:0 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: cephadm 2024-09-13T14:24:17.262796+0000 mgr.y (mgr.24287) 132 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.270877+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.273728+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.276873+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.279113+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:18.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.286160+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:18.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.289965+0000 mon.c (mon.2) 88 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:18.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.290340+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:18.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.294703+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:24:18.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: audit 2024-09-13T14:24:17.295890+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:18.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:18 smithi177 bash[24494]: cephadm 2024-09-13T14:24:17.297079+0000 mgr.y (mgr.24287) 133 : cephadm [INF] Deploying daemon mds.foo.smithi099.yyrtld on smithi099 2024-09-13T14:24:18.979 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T14:24:19.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:19 smithi099 bash[22656]: cluster 2024-09-13T14:24:18.253800+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:24:19.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:19 smithi099 bash[32082]: cluster 2024-09-13T14:24:18.253800+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:24:19.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:19 smithi177 bash[24494]: cluster 2024-09-13T14:24:18.253800+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:24:20.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:20 smithi099 bash[22656]: cluster 2024-09-13T14:24:19.115974+0000 mgr.y (mgr.24287) 134 : cluster [DBG] pgmap v110: 73 pgs: 7 creating+peering, 66 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:20.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:20 smithi099 bash[22656]: cluster 2024-09-13T14:24:19.246254+0000 mon.a (mon.0) 760 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:24:20.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:20 smithi099 bash[22656]: cluster 2024-09-13T14:24:19.269325+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:24:20.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:20 smithi099 bash[32082]: cluster 2024-09-13T14:24:19.115974+0000 mgr.y (mgr.24287) 134 : cluster [DBG] pgmap v110: 73 pgs: 7 creating+peering, 66 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:20.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:20 smithi099 bash[32082]: cluster 2024-09-13T14:24:19.246254+0000 mon.a (mon.0) 760 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:24:20.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:20 smithi099 bash[32082]: cluster 2024-09-13T14:24:19.269325+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:24:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:20 smithi177 bash[24494]: cluster 2024-09-13T14:24:19.115974+0000 mgr.y (mgr.24287) 134 : cluster [DBG] pgmap v110: 73 pgs: 7 creating+peering, 66 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:20 smithi177 bash[24494]: cluster 2024-09-13T14:24:19.246254+0000 mon.a (mon.0) 760 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:24:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:20 smithi177 bash[24494]: cluster 2024-09-13T14:24:19.269325+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:24:20.855 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:20.856 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:20.856 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:20.857 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:20.857 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:20.857 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:20.857 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:20.857 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:20.858 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:20.858 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:20.858 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:21.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:21.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:21.318 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:21.318 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:21.319 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:21.319 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:21.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:20 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:22.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:22] "GET /metrics HTTP/1.1" 200 203890 "" "Prometheus/2.43.0" 2024-09-13T14:24:22.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[22656]: audit 2024-09-13T14:24:21.009804+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:22.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[22656]: audit 2024-09-13T14:24:21.018249+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:22.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[22656]: audit 2024-09-13T14:24:21.023793+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:22.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[22656]: audit 2024-09-13T14:24:21.024675+0000 mon.c (mon.2) 90 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:22.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[22656]: audit 2024-09-13T14:24:21.024982+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:22.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[22656]: audit 2024-09-13T14:24:21.030378+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:24:22.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[22656]: audit 2024-09-13T14:24:21.031219+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:22.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[22656]: cephadm 2024-09-13T14:24:21.032162+0000 mgr.y (mgr.24287) 135 : cephadm [INF] Deploying daemon mds.foo.smithi177.outtkv on smithi177 2024-09-13T14:24:22.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[22656]: cluster 2024-09-13T14:24:21.116613+0000 mgr.y (mgr.24287) 136 : cluster [DBG] pgmap v112: 73 pgs: 7 creating+peering, 66 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:22.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[32082]: audit 2024-09-13T14:24:21.009804+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:22.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[32082]: audit 2024-09-13T14:24:21.018249+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:22.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[32082]: audit 2024-09-13T14:24:21.023793+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:22.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[32082]: audit 2024-09-13T14:24:21.024675+0000 mon.c (mon.2) 90 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:22.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[32082]: audit 2024-09-13T14:24:21.024982+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:22.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[32082]: audit 2024-09-13T14:24:21.030378+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:24:22.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[32082]: audit 2024-09-13T14:24:21.031219+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:22.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[32082]: cephadm 2024-09-13T14:24:21.032162+0000 mgr.y (mgr.24287) 135 : cephadm [INF] Deploying daemon mds.foo.smithi177.outtkv on smithi177 2024-09-13T14:24:22.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:22 smithi099 bash[32082]: cluster 2024-09-13T14:24:21.116613+0000 mgr.y (mgr.24287) 136 : cluster [DBG] pgmap v112: 73 pgs: 7 creating+peering, 66 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:22.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:22 smithi177 bash[24494]: audit 2024-09-13T14:24:21.009804+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:22.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:22 smithi177 bash[24494]: audit 2024-09-13T14:24:21.018249+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:22.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:22 smithi177 bash[24494]: audit 2024-09-13T14:24:21.023793+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:22.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:22 smithi177 bash[24494]: audit 2024-09-13T14:24:21.024675+0000 mon.c (mon.2) 90 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:22.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:22 smithi177 bash[24494]: audit 2024-09-13T14:24:21.024982+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:24:22.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:22 smithi177 bash[24494]: audit 2024-09-13T14:24:21.030378+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:24:22.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:22 smithi177 bash[24494]: audit 2024-09-13T14:24:21.031219+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:22.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:22 smithi177 bash[24494]: cephadm 2024-09-13T14:24:21.032162+0000 mgr.y (mgr.24287) 135 : cephadm [INF] Deploying daemon mds.foo.smithi177.outtkv on smithi177 2024-09-13T14:24:22.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:22 smithi177 bash[24494]: cluster 2024-09-13T14:24:21.116613+0000 mgr.y (mgr.24287) 136 : cluster [DBG] pgmap v112: 73 pgs: 7 creating+peering, 66 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:23.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 bash[24494]: audit 2024-09-13T14:24:22.160527+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:23.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 bash[24494]: audit 2024-09-13T14:24:22.162803+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:23.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 bash[24494]: cluster 2024-09-13T14:24:23.086302+0000 mon.a (mon.0) 768 : cluster [INF] daemon mds.foo.smithi099.yyrtld assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:24:23.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 bash[24494]: cluster 2024-09-13T14:24:23.086537+0000 mon.a (mon.0) 769 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:24:23.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 bash[24494]: cluster 2024-09-13T14:24:23.086591+0000 mon.a (mon.0) 770 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:24:23.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 bash[24494]: cluster 2024-09-13T14:24:23.086633+0000 mon.a (mon.0) 771 : cluster [INF] Cluster is now healthy 2024-09-13T14:24:23.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 bash[24494]: cluster 2024-09-13T14:24:23.093632+0000 mon.a (mon.0) 772 : cluster [DBG] mds.? [v2:172.21.15.99:6834/3867979721,v1:172.21.15.99:6835/3867979721] up:boot 2024-09-13T14:24:23.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 bash[24494]: cluster 2024-09-13T14:24:23.093690+0000 mon.a (mon.0) 773 : cluster [DBG] fsmap foo:1 {0=foo.smithi099.yyrtld=up:creating} 2024-09-13T14:24:23.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 bash[24494]: audit 2024-09-13T14:24:23.095763+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:24:23.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 bash[24494]: cluster 2024-09-13T14:24:23.123160+0000 mon.a (mon.0) 774 : cluster [INF] daemon mds.foo.smithi099.yyrtld is now active in filesystem foo as rank 0 2024-09-13T14:24:23.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[22656]: audit 2024-09-13T14:24:22.160527+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:23.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[22656]: audit 2024-09-13T14:24:22.162803+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:23.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[22656]: cluster 2024-09-13T14:24:23.086302+0000 mon.a (mon.0) 768 : cluster [INF] daemon mds.foo.smithi099.yyrtld assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:24:23.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[22656]: cluster 2024-09-13T14:24:23.086537+0000 mon.a (mon.0) 769 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:24:23.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[22656]: cluster 2024-09-13T14:24:23.086591+0000 mon.a (mon.0) 770 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:24:23.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[22656]: cluster 2024-09-13T14:24:23.086633+0000 mon.a (mon.0) 771 : cluster [INF] Cluster is now healthy 2024-09-13T14:24:23.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[22656]: cluster 2024-09-13T14:24:23.093632+0000 mon.a (mon.0) 772 : cluster [DBG] mds.? [v2:172.21.15.99:6834/3867979721,v1:172.21.15.99:6835/3867979721] up:boot 2024-09-13T14:24:23.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[22656]: cluster 2024-09-13T14:24:23.093690+0000 mon.a (mon.0) 773 : cluster [DBG] fsmap foo:1 {0=foo.smithi099.yyrtld=up:creating} 2024-09-13T14:24:23.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[22656]: audit 2024-09-13T14:24:23.095763+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:24:23.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[22656]: cluster 2024-09-13T14:24:23.123160+0000 mon.a (mon.0) 774 : cluster [INF] daemon mds.foo.smithi099.yyrtld is now active in filesystem foo as rank 0 2024-09-13T14:24:23.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[32082]: audit 2024-09-13T14:24:22.160527+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[32082]: audit 2024-09-13T14:24:22.162803+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[32082]: cluster 2024-09-13T14:24:23.086302+0000 mon.a (mon.0) 768 : cluster [INF] daemon mds.foo.smithi099.yyrtld assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:24:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[32082]: cluster 2024-09-13T14:24:23.086537+0000 mon.a (mon.0) 769 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:24:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[32082]: cluster 2024-09-13T14:24:23.086591+0000 mon.a (mon.0) 770 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:24:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[32082]: cluster 2024-09-13T14:24:23.086633+0000 mon.a (mon.0) 771 : cluster [INF] Cluster is now healthy 2024-09-13T14:24:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[32082]: cluster 2024-09-13T14:24:23.093632+0000 mon.a (mon.0) 772 : cluster [DBG] mds.? [v2:172.21.15.99:6834/3867979721,v1:172.21.15.99:6835/3867979721] up:boot 2024-09-13T14:24:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[32082]: cluster 2024-09-13T14:24:23.093690+0000 mon.a (mon.0) 773 : cluster [DBG] fsmap foo:1 {0=foo.smithi099.yyrtld=up:creating} 2024-09-13T14:24:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[32082]: audit 2024-09-13T14:24:23.095763+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:24:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:23 smithi099 bash[32082]: cluster 2024-09-13T14:24:23.123160+0000 mon.a (mon.0) 774 : cluster [INF] daemon mds.foo.smithi099.yyrtld is now active in filesystem foo as rank 0 2024-09-13T14:24:23.763 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:23.763 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:23.764 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:23.764 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:23.764 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:23.765 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:23.765 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:23.765 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:23.765 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:24.024 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:24.024 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:24.025 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:24.025 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:24.025 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:24.026 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:24.026 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:24.026 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:24.026 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:24:23 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:24:24.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:24 smithi177 bash[24494]: cluster 2024-09-13T14:24:23.117261+0000 mgr.y (mgr.24287) 137 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:24.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:24 smithi177 bash[24494]: audit 2024-09-13T14:24:23.930927+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:24 smithi177 bash[24494]: audit 2024-09-13T14:24:23.947493+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:24 smithi177 bash[24494]: audit 2024-09-13T14:24:23.963209+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:24 smithi177 bash[24494]: audit 2024-09-13T14:24:23.978642+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:24 smithi177 bash[24494]: audit 2024-09-13T14:24:23.991840+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.290 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:24 smithi177 bash[24494]: audit 2024-09-13T14:24:24.048561+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:24.290 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:24 smithi177 bash[24494]: cluster 2024-09-13T14:24:24.105269+0000 mon.a (mon.0) 780 : cluster [DBG] mds.? [v2:172.21.15.99:6834/3867979721,v1:172.21.15.99:6835/3867979721] up:active 2024-09-13T14:24:24.290 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:24 smithi177 bash[24494]: cluster 2024-09-13T14:24:24.105346+0000 mon.a (mon.0) 781 : cluster [DBG] fsmap foo:1 {0=foo.smithi099.yyrtld=up:active} 2024-09-13T14:24:24.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[22656]: cluster 2024-09-13T14:24:23.117261+0000 mgr.y (mgr.24287) 137 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[22656]: audit 2024-09-13T14:24:23.930927+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[22656]: audit 2024-09-13T14:24:23.947493+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[22656]: audit 2024-09-13T14:24:23.963209+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[22656]: audit 2024-09-13T14:24:23.978642+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[22656]: audit 2024-09-13T14:24:23.991840+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[22656]: audit 2024-09-13T14:24:24.048561+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[22656]: cluster 2024-09-13T14:24:24.105269+0000 mon.a (mon.0) 780 : cluster [DBG] mds.? [v2:172.21.15.99:6834/3867979721,v1:172.21.15.99:6835/3867979721] up:active 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[22656]: cluster 2024-09-13T14:24:24.105346+0000 mon.a (mon.0) 781 : cluster [DBG] fsmap foo:1 {0=foo.smithi099.yyrtld=up:active} 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[32082]: cluster 2024-09-13T14:24:23.117261+0000 mgr.y (mgr.24287) 137 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[32082]: audit 2024-09-13T14:24:23.930927+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[32082]: audit 2024-09-13T14:24:23.947493+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[32082]: audit 2024-09-13T14:24:23.963209+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[32082]: audit 2024-09-13T14:24:23.978642+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[32082]: audit 2024-09-13T14:24:23.991840+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:24.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[32082]: audit 2024-09-13T14:24:24.048561+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:24.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[32082]: cluster 2024-09-13T14:24:24.105269+0000 mon.a (mon.0) 780 : cluster [DBG] mds.? [v2:172.21.15.99:6834/3867979721,v1:172.21.15.99:6835/3867979721] up:active 2024-09-13T14:24:24.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:24 smithi099 bash[32082]: cluster 2024-09-13T14:24:24.105346+0000 mon.a (mon.0) 781 : cluster [DBG] fsmap foo:1 {0=foo.smithi099.yyrtld=up:active} 2024-09-13T14:24:24.765 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:25.606 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:24:25 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:24:26.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:26 smithi099 bash[22656]: cluster 2024-09-13T14:24:25.117832+0000 mgr.y (mgr.24287) 138 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:26.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:26 smithi099 bash[32082]: cluster 2024-09-13T14:24:25.117832+0000 mgr.y (mgr.24287) 138 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:26.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:26 smithi177 bash[24494]: cluster 2024-09-13T14:24:25.117832+0000 mgr.y (mgr.24287) 138 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:27.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[32082]: cluster 2024-09-13T14:24:26.173865+0000 mon.a (mon.0) 782 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:boot 2024-09-13T14:24:27.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[32082]: cluster 2024-09-13T14:24:26.173972+0000 mon.a (mon.0) 783 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi099.yyrtld in favor of higher affinity standby. 2024-09-13T14:24:27.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[32082]: cluster 2024-09-13T14:24:26.174005+0000 mon.a (mon.0) 784 : cluster [WRN] Replacing daemon mds.foo.smithi099.yyrtld as rank 0 with standby daemon mds.foo.smithi177.outtkv 2024-09-13T14:24:27.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[32082]: cluster 2024-09-13T14:24:26.174371+0000 mon.a (mon.0) 785 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T14:24:27.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[32082]: cluster 2024-09-13T14:24:26.174388+0000 mon.a (mon.0) 786 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T14:24:27.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[32082]: cluster 2024-09-13T14:24:26.174756+0000 mon.a (mon.0) 787 : cluster [DBG] fsmap foo:1 {0=foo.smithi099.yyrtld=up:active} 1 up:standby 2024-09-13T14:24:27.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[32082]: audit 2024-09-13T14:24:26.177602+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.outtkv"}]: dispatch 2024-09-13T14:24:27.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[32082]: cluster 2024-09-13T14:24:26.187060+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:24:27.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[32082]: cluster 2024-09-13T14:24:26.187182+0000 mon.a (mon.0) 789 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi177.outtkv=up:replay} 2024-09-13T14:24:27.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[22656]: cluster 2024-09-13T14:24:26.173865+0000 mon.a (mon.0) 782 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:boot 2024-09-13T14:24:27.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[22656]: cluster 2024-09-13T14:24:26.173972+0000 mon.a (mon.0) 783 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi099.yyrtld in favor of higher affinity standby. 2024-09-13T14:24:27.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[22656]: cluster 2024-09-13T14:24:26.174005+0000 mon.a (mon.0) 784 : cluster [WRN] Replacing daemon mds.foo.smithi099.yyrtld as rank 0 with standby daemon mds.foo.smithi177.outtkv 2024-09-13T14:24:27.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[22656]: cluster 2024-09-13T14:24:26.174371+0000 mon.a (mon.0) 785 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T14:24:27.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[22656]: cluster 2024-09-13T14:24:26.174388+0000 mon.a (mon.0) 786 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T14:24:27.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[22656]: cluster 2024-09-13T14:24:26.174756+0000 mon.a (mon.0) 787 : cluster [DBG] fsmap foo:1 {0=foo.smithi099.yyrtld=up:active} 1 up:standby 2024-09-13T14:24:27.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[22656]: audit 2024-09-13T14:24:26.177602+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.outtkv"}]: dispatch 2024-09-13T14:24:27.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[22656]: cluster 2024-09-13T14:24:26.187060+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:24:27.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:27 smithi099 bash[22656]: cluster 2024-09-13T14:24:26.187182+0000 mon.a (mon.0) 789 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi177.outtkv=up:replay} 2024-09-13T14:24:27.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:27 smithi177 bash[24494]: cluster 2024-09-13T14:24:26.173865+0000 mon.a (mon.0) 782 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:boot 2024-09-13T14:24:27.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:27 smithi177 bash[24494]: cluster 2024-09-13T14:24:26.173972+0000 mon.a (mon.0) 783 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi099.yyrtld in favor of higher affinity standby. 2024-09-13T14:24:27.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:27 smithi177 bash[24494]: cluster 2024-09-13T14:24:26.174005+0000 mon.a (mon.0) 784 : cluster [WRN] Replacing daemon mds.foo.smithi099.yyrtld as rank 0 with standby daemon mds.foo.smithi177.outtkv 2024-09-13T14:24:27.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:27 smithi177 bash[24494]: cluster 2024-09-13T14:24:26.174371+0000 mon.a (mon.0) 785 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T14:24:27.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:27 smithi177 bash[24494]: cluster 2024-09-13T14:24:26.174388+0000 mon.a (mon.0) 786 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T14:24:27.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:27 smithi177 bash[24494]: cluster 2024-09-13T14:24:26.174756+0000 mon.a (mon.0) 787 : cluster [DBG] fsmap foo:1 {0=foo.smithi099.yyrtld=up:active} 1 up:standby 2024-09-13T14:24:27.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:27 smithi177 bash[24494]: audit 2024-09-13T14:24:26.177602+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.outtkv"}]: dispatch 2024-09-13T14:24:27.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:27 smithi177 bash[24494]: cluster 2024-09-13T14:24:26.187060+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:24:27.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:27 smithi177 bash[24494]: cluster 2024-09-13T14:24:26.187182+0000 mon.a (mon.0) 789 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi177.outtkv=up:replay} 2024-09-13T14:24:28.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[32082]: cluster 2024-09-13T14:24:27.118255+0000 mgr.y (mgr.24287) 139 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:28.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[32082]: cluster 2024-09-13T14:24:27.178236+0000 mon.a (mon.0) 790 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T14:24:28.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[32082]: cluster 2024-09-13T14:24:27.186600+0000 mon.a (mon.0) 791 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:reconnect 2024-09-13T14:24:28.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[32082]: cluster 2024-09-13T14:24:27.186653+0000 mon.a (mon.0) 792 : cluster [DBG] mds.? [v2:172.21.15.99:6834/2117449908,v1:172.21.15.99:6835/2117449908] up:boot 2024-09-13T14:24:28.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[32082]: cluster 2024-09-13T14:24:27.186713+0000 mon.a (mon.0) 793 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi177.outtkv=up:reconnect} 1 up:standby 2024-09-13T14:24:28.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[32082]: audit 2024-09-13T14:24:27.187116+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:24:28.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[32082]: audit 2024-09-13T14:24:27.262157+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:28.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[22656]: cluster 2024-09-13T14:24:27.118255+0000 mgr.y (mgr.24287) 139 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:28.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[22656]: cluster 2024-09-13T14:24:27.178236+0000 mon.a (mon.0) 790 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T14:24:28.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[22656]: cluster 2024-09-13T14:24:27.186600+0000 mon.a (mon.0) 791 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:reconnect 2024-09-13T14:24:28.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[22656]: cluster 2024-09-13T14:24:27.186653+0000 mon.a (mon.0) 792 : cluster [DBG] mds.? [v2:172.21.15.99:6834/2117449908,v1:172.21.15.99:6835/2117449908] up:boot 2024-09-13T14:24:28.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[22656]: cluster 2024-09-13T14:24:27.186713+0000 mon.a (mon.0) 793 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi177.outtkv=up:reconnect} 1 up:standby 2024-09-13T14:24:28.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[22656]: audit 2024-09-13T14:24:27.187116+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:24:28.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:28 smithi099 bash[22656]: audit 2024-09-13T14:24:27.262157+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:28.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:28 smithi177 bash[24494]: cluster 2024-09-13T14:24:27.118255+0000 mgr.y (mgr.24287) 139 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:28.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:28 smithi177 bash[24494]: cluster 2024-09-13T14:24:27.178236+0000 mon.a (mon.0) 790 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T14:24:28.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:28 smithi177 bash[24494]: cluster 2024-09-13T14:24:27.186600+0000 mon.a (mon.0) 791 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:reconnect 2024-09-13T14:24:28.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:28 smithi177 bash[24494]: cluster 2024-09-13T14:24:27.186653+0000 mon.a (mon.0) 792 : cluster [DBG] mds.? [v2:172.21.15.99:6834/2117449908,v1:172.21.15.99:6835/2117449908] up:boot 2024-09-13T14:24:28.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:28 smithi177 bash[24494]: cluster 2024-09-13T14:24:27.186713+0000 mon.a (mon.0) 793 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi177.outtkv=up:reconnect} 1 up:standby 2024-09-13T14:24:28.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:28 smithi177 bash[24494]: audit 2024-09-13T14:24:27.187116+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:24:28.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:28 smithi177 bash[24494]: audit 2024-09-13T14:24:27.262157+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:29.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:29 smithi099 bash[22656]: cluster 2024-09-13T14:24:28.194338+0000 mon.a (mon.0) 795 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:rejoin 2024-09-13T14:24:29.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:29 smithi099 bash[22656]: cluster 2024-09-13T14:24:28.194531+0000 mon.a (mon.0) 796 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi177.outtkv=up:rejoin} 1 up:standby 2024-09-13T14:24:29.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:29 smithi099 bash[22656]: cluster 2024-09-13T14:24:28.195087+0000 mon.a (mon.0) 797 : cluster [INF] daemon mds.foo.smithi177.outtkv is now active in filesystem foo as rank 0 2024-09-13T14:24:29.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:29 smithi099 bash[32082]: cluster 2024-09-13T14:24:28.194338+0000 mon.a (mon.0) 795 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:rejoin 2024-09-13T14:24:29.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:29 smithi099 bash[32082]: cluster 2024-09-13T14:24:28.194531+0000 mon.a (mon.0) 796 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi177.outtkv=up:rejoin} 1 up:standby 2024-09-13T14:24:29.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:29 smithi099 bash[32082]: cluster 2024-09-13T14:24:28.195087+0000 mon.a (mon.0) 797 : cluster [INF] daemon mds.foo.smithi177.outtkv is now active in filesystem foo as rank 0 2024-09-13T14:24:29.539 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T14:24:29.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:29 smithi177 bash[24494]: cluster 2024-09-13T14:24:28.194338+0000 mon.a (mon.0) 795 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:rejoin 2024-09-13T14:24:29.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:29 smithi177 bash[24494]: cluster 2024-09-13T14:24:28.194531+0000 mon.a (mon.0) 796 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi177.outtkv=up:rejoin} 1 up:standby 2024-09-13T14:24:29.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:29 smithi177 bash[24494]: cluster 2024-09-13T14:24:28.195087+0000 mon.a (mon.0) 797 : cluster [INF] daemon mds.foo.smithi177.outtkv is now active in filesystem foo as rank 0 2024-09-13T14:24:30.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:30 smithi099 bash[22656]: cluster 2024-09-13T14:24:29.119324+0000 mgr.y (mgr.24287) 140 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T14:24:30.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:30 smithi099 bash[22656]: cluster 2024-09-13T14:24:29.188593+0000 mon.a (mon.0) 798 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T14:24:30.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:30 smithi099 bash[22656]: cluster 2024-09-13T14:24:29.188653+0000 mon.a (mon.0) 799 : cluster [INF] Cluster is now healthy 2024-09-13T14:24:30.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:30 smithi099 bash[22656]: cluster 2024-09-13T14:24:29.206599+0000 mon.a (mon.0) 800 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:active 2024-09-13T14:24:30.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:30 smithi099 bash[22656]: cluster 2024-09-13T14:24:29.206742+0000 mon.a (mon.0) 801 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:24:30.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:30 smithi099 bash[32082]: cluster 2024-09-13T14:24:29.119324+0000 mgr.y (mgr.24287) 140 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T14:24:30.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:30 smithi099 bash[32082]: cluster 2024-09-13T14:24:29.188593+0000 mon.a (mon.0) 798 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T14:24:30.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:30 smithi099 bash[32082]: cluster 2024-09-13T14:24:29.188653+0000 mon.a (mon.0) 799 : cluster [INF] Cluster is now healthy 2024-09-13T14:24:30.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:30 smithi099 bash[32082]: cluster 2024-09-13T14:24:29.206599+0000 mon.a (mon.0) 800 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:active 2024-09-13T14:24:30.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:30 smithi099 bash[32082]: cluster 2024-09-13T14:24:29.206742+0000 mon.a (mon.0) 801 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:24:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:30 smithi177 bash[24494]: cluster 2024-09-13T14:24:29.119324+0000 mgr.y (mgr.24287) 140 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T14:24:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:30 smithi177 bash[24494]: cluster 2024-09-13T14:24:29.188593+0000 mon.a (mon.0) 798 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T14:24:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:30 smithi177 bash[24494]: cluster 2024-09-13T14:24:29.188653+0000 mon.a (mon.0) 799 : cluster [INF] Cluster is now healthy 2024-09-13T14:24:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:30 smithi177 bash[24494]: cluster 2024-09-13T14:24:29.206599+0000 mon.a (mon.0) 800 : cluster [DBG] mds.? [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] up:active 2024-09-13T14:24:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:30 smithi177 bash[24494]: cluster 2024-09-13T14:24:29.206742+0000 mon.a (mon.0) 801 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:24:32.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.658614+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.665050+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.702809+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.708498+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.709972+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:32.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.710660+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:32.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.714928+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.750998+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:32.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.752571+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:32.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.753258+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:32.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[22656]: audit 2024-09-13T14:24:30.758148+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.658614+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.665050+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.702809+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.708498+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.709972+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:32.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.710660+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:32.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.714928+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.750998+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:32.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.752571+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:32.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.753258+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:32.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:31 smithi099 bash[32082]: audit 2024-09-13T14:24:30.758148+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.658614+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.665050+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.702809+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.708498+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.709972+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:32.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.710660+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:32.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.714928+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.750998+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:32.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.752571+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:32.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.753258+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:32.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:31 smithi177 bash[24494]: audit 2024-09-13T14:24:30.758148+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:32.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:24:32 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:32] "GET /metrics HTTP/1.1" 200 208762 "" "Prometheus/2.43.0" 2024-09-13T14:24:33.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:32 smithi099 bash[22656]: cluster 2024-09-13T14:24:31.120025+0000 mgr.y (mgr.24287) 141 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T14:24:33.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:32 smithi099 bash[32082]: cluster 2024-09-13T14:24:31.120025+0000 mgr.y (mgr.24287) 141 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T14:24:33.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:32 smithi177 bash[24494]: cluster 2024-09-13T14:24:31.120025+0000 mgr.y (mgr.24287) 141 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T14:24:34.228 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:35.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:34 smithi099 bash[22656]: cluster 2024-09-13T14:24:33.121079+0000 mgr.y (mgr.24287) 142 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-09-13T14:24:35.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:34 smithi099 bash[32082]: cluster 2024-09-13T14:24:33.121079+0000 mgr.y (mgr.24287) 142 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-09-13T14:24:35.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:34 smithi177 bash[24494]: cluster 2024-09-13T14:24:33.121079+0000 mgr.y (mgr.24287) 142 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-09-13T14:24:35.856 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:24:35 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:24:36.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:36 smithi099 bash[22656]: cluster 2024-09-13T14:24:35.121778+0000 mgr.y (mgr.24287) 143 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-09-13T14:24:36.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:36 smithi099 bash[32082]: cluster 2024-09-13T14:24:35.121778+0000 mgr.y (mgr.24287) 143 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-09-13T14:24:37.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:36 smithi177 bash[24494]: cluster 2024-09-13T14:24:35.121778+0000 mgr.y (mgr.24287) 143 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-09-13T14:24:38.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:38 smithi099 bash[22656]: cluster 2024-09-13T14:24:37.122533+0000 mgr.y (mgr.24287) 144 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T14:24:38.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:38 smithi099 bash[22656]: audit 2024-09-13T14:24:37.162108+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:38.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:38 smithi099 bash[22656]: audit 2024-09-13T14:24:37.165926+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:38.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:38 smithi099 bash[22656]: audit 2024-09-13T14:24:37.280343+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:38.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:38 smithi099 bash[32082]: cluster 2024-09-13T14:24:37.122533+0000 mgr.y (mgr.24287) 144 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T14:24:38.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:38 smithi099 bash[32082]: audit 2024-09-13T14:24:37.162108+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:38.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:38 smithi099 bash[32082]: audit 2024-09-13T14:24:37.165926+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:38.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:38 smithi099 bash[32082]: audit 2024-09-13T14:24:37.280343+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:38.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:38 smithi177 bash[24494]: cluster 2024-09-13T14:24:37.122533+0000 mgr.y (mgr.24287) 144 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T14:24:38.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:38 smithi177 bash[24494]: audit 2024-09-13T14:24:37.162108+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:38.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:38 smithi177 bash[24494]: audit 2024-09-13T14:24:37.165926+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:38.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:38 smithi177 bash[24494]: audit 2024-09-13T14:24:37.280343+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:24:38.705 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-13T14:24:38.715 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:24:38.715 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:24:38.715 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:24:40.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:40 smithi099 bash[22656]: cluster 2024-09-13T14:24:39.123612+0000 mgr.y (mgr.24287) 145 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.1 KiB/s wr, 7 op/s 2024-09-13T14:24:40.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:40 smithi099 bash[32082]: cluster 2024-09-13T14:24:39.123612+0000 mgr.y (mgr.24287) 145 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.1 KiB/s wr, 7 op/s 2024-09-13T14:24:40.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:40 smithi177 bash[24494]: cluster 2024-09-13T14:24:39.123612+0000 mgr.y (mgr.24287) 145 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.1 KiB/s wr, 7 op/s 2024-09-13T14:24:42.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:42 smithi099 bash[22656]: cluster 2024-09-13T14:24:41.124251+0000 mgr.y (mgr.24287) 146 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T14:24:42.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:24:42 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:42] "GET /metrics HTTP/1.1" 200 240005 "" "Prometheus/2.43.0" 2024-09-13T14:24:42.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:42 smithi099 bash[32082]: cluster 2024-09-13T14:24:41.124251+0000 mgr.y (mgr.24287) 146 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T14:24:42.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:42 smithi177 bash[24494]: cluster 2024-09-13T14:24:41.124251+0000 mgr.y (mgr.24287) 146 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T14:24:43.492 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:44.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:44 smithi099 bash[22656]: cluster 2024-09-13T14:24:43.125373+0000 mgr.y (mgr.24287) 147 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T14:24:44.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:44 smithi099 bash[32082]: cluster 2024-09-13T14:24:43.125373+0000 mgr.y (mgr.24287) 147 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T14:24:44.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:44 smithi177 bash[24494]: cluster 2024-09-13T14:24:43.125373+0000 mgr.y (mgr.24287) 147 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T14:24:45.856 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:24:45 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:24:46.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:46 smithi099 bash[22656]: cluster 2024-09-13T14:24:45.126041+0000 mgr.y (mgr.24287) 148 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:46.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:46 smithi099 bash[32082]: cluster 2024-09-13T14:24:45.126041+0000 mgr.y (mgr.24287) 148 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:46.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:46 smithi177 bash[24494]: cluster 2024-09-13T14:24:45.126041+0000 mgr.y (mgr.24287) 148 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:47.552 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:24:48.366 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":20,"flags":0,"active_gid":24287,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2887456408},{"type":"v1","addr":"172.21.15.99:6801","nonce":2887456408}]},"active_addr":"172.21.15.99:6801/2887456408","active_change":"2024-09-13T14:21:07.041615+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24304,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.99:8443/","prometheus":"http://172.21.15.99:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":52,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":1638158155}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":2140252887}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":4242039269}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":316966644}]}]} 2024-09-13T14:24:48.371 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:24:48.371 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:24:48.371 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:24:48.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:48 smithi099 bash[32082]: cluster 2024-09-13T14:24:47.126672+0000 mgr.y (mgr.24287) 149 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:48.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:48 smithi099 bash[32082]: audit 2024-09-13T14:24:47.551741+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.99:0/1128216433' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:24:48.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:48 smithi099 bash[22656]: cluster 2024-09-13T14:24:47.126672+0000 mgr.y (mgr.24287) 149 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:48.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:48 smithi099 bash[22656]: audit 2024-09-13T14:24:47.551741+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.99:0/1128216433' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:24:48.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:48 smithi177 bash[24494]: cluster 2024-09-13T14:24:47.126672+0000 mgr.y (mgr.24287) 149 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:48.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:48 smithi177 bash[24494]: audit 2024-09-13T14:24:47.551741+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.99:0/1128216433' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:24:50.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:50 smithi099 bash[22656]: cluster 2024-09-13T14:24:49.127846+0000 mgr.y (mgr.24287) 150 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:50.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:50 smithi099 bash[32082]: cluster 2024-09-13T14:24:49.127846+0000 mgr.y (mgr.24287) 150 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:50.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:50 smithi177 bash[24494]: cluster 2024-09-13T14:24:49.127846+0000 mgr.y (mgr.24287) 150 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:52.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:24:52 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:52] "GET /metrics HTTP/1.1" 200 240005 "" "Prometheus/2.43.0" 2024-09-13T14:24:52.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:52 smithi099 bash[22656]: cluster 2024-09-13T14:24:51.128332+0000 mgr.y (mgr.24287) 151 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:52.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:52 smithi099 bash[22656]: audit 2024-09-13T14:24:52.153989+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:52.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:52 smithi099 bash[32082]: cluster 2024-09-13T14:24:51.128332+0000 mgr.y (mgr.24287) 151 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:52.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:52 smithi099 bash[32082]: audit 2024-09-13T14:24:52.153989+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:52.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:52 smithi177 bash[24494]: cluster 2024-09-13T14:24:51.128332+0000 mgr.y (mgr.24287) 151 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:52.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:52 smithi177 bash[24494]: audit 2024-09-13T14:24:52.153989+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:53.160 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:24:54.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:54 smithi099 bash[22656]: cluster 2024-09-13T14:24:53.129419+0000 mgr.y (mgr.24287) 152 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:54.456 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:54 smithi099 bash[32082]: cluster 2024-09-13T14:24:53.129419+0000 mgr.y (mgr.24287) 152 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:54.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:54 smithi177 bash[24494]: cluster 2024-09-13T14:24:53.129419+0000 mgr.y (mgr.24287) 152 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:55.856 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:24:55 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:24:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:24:56.464 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:56 smithi099 bash[32082]: cluster 2024-09-13T14:24:55.130154+0000 mgr.y (mgr.24287) 153 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:56.465 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:56 smithi099 bash[22656]: cluster 2024-09-13T14:24:55.130154+0000 mgr.y (mgr.24287) 153 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:56.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:56 smithi177 bash[24494]: cluster 2024-09-13T14:24:55.130154+0000 mgr.y (mgr.24287) 153 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:56.792 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:24:56.792 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":65,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:10:21.363918+0000","modified":"2024-09-13T14:24:26.174069+0000","last_up_change":"2024-09-13T14:20:05.471230+0000","last_in_change":"2024-09-13T14:19:38.687766+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:15:37.290226+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:21:51.531836+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:24:15.786407+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:24:16.627611+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"0624feaa-339b-4468-be53-582a1f04352c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6803","nonce":500907575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6805","nonce":500907575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6809","nonce":500907575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6807","nonce":500907575}]},"public_addr":"172.21.15.99:6803/500907575","cluster_addr":"172.21.15.99:6805/500907575","heartbeat_back_addr":"172.21.15.99:6809/500907575","heartbeat_front_addr":"172.21.15.99:6807/500907575","state":["exists","up"]},{"osd":1,"uuid":"b7b63553-e49d-4636-b3e2-43b72556a1a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6811","nonce":4008940153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6813","nonce":4008940153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6817","nonce":4008940153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6815","nonce":4008940153}]},"public_addr":"172.21.15.99:6811/4008940153","cluster_addr":"172.21.15.99:6813/4008940153","heartbeat_back_addr":"172.21.15.99:6817/4008940153","heartbeat_front_addr":"172.21.15.99:6815/4008940153","state":["exists","up"]},{"osd":2,"uuid":"3b9eff6d-5f5c-4843-a1ff-815a902167ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6819","nonce":4220960661}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6821","nonce":4220960661}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6825","nonce":4220960661}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6823","nonce":4220960661}]},"public_addr":"172.21.15.99:6819/4220960661","cluster_addr":"172.21.15.99:6821/4220960661","heartbeat_back_addr":"172.21.15.99:6825/4220960661","heartbeat_front_addr":"172.21.15.99:6823/4220960661","state":["exists","up"]},{"osd":3,"uuid":"686c8d3c-9cab-4920-b02b-d243324f5808","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6827","nonce":1221827921}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6829","nonce":1221827921}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6833","nonce":1221827921}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6831","nonce":1221827921}]},"public_addr":"172.21.15.99:6827/1221827921","cluster_addr":"172.21.15.99:6829/1221827921","heartbeat_back_addr":"172.21.15.99:6833/1221827921","heartbeat_front_addr":"172.21.15.99:6831/1221827921","state":["exists","up"]},{"osd":4,"uuid":"2d56712b-a6fb-4089-9583-850beed92b37","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6801","nonce":2146549575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6803","nonce":2146549575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6807","nonce":2146549575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6805","nonce":2146549575}]},"public_addr":"172.21.15.177:6801/2146549575","cluster_addr":"172.21.15.177:6803/2146549575","heartbeat_back_addr":"172.21.15.177:6807/2146549575","heartbeat_front_addr":"172.21.15.177:6805/2146549575","state":["exists","up"]},{"osd":5,"uuid":"414b880c-6534-495f-a54c-f8d28ba81c73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6809","nonce":947070587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6811","nonce":947070587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6815","nonce":947070587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6813","nonce":947070587}]},"public_addr":"172.21.15.177:6809/947070587","cluster_addr":"172.21.15.177:6811/947070587","heartbeat_back_addr":"172.21.15.177:6815/947070587","heartbeat_front_addr":"172.21.15.177:6813/947070587","state":["exists","up"]},{"osd":6,"uuid":"ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6817","nonce":3817692594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6819","nonce":3817692594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6823","nonce":3817692594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6821","nonce":3817692594}]},"public_addr":"172.21.15.177:6817/3817692594","cluster_addr":"172.21.15.177:6819/3817692594","heartbeat_back_addr":"172.21.15.177:6823/3817692594","heartbeat_front_addr":"172.21.15.177:6821/3817692594","state":["exists","up"]},{"osd":7,"uuid":"ad2c2977-fd6d-42f8-962e-907c2bf3204b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6825","nonce":1984676946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6827","nonce":1984676946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6831","nonce":1984676946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6829","nonce":1984676946}]},"public_addr":"172.21.15.177:6825/1984676946","cluster_addr":"172.21.15.177:6827/1984676946","heartbeat_back_addr":"172.21.15.177:6831/1984676946","heartbeat_front_addr":"172.21.15.177:6829/1984676946","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:46.390607+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:38.398637+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:15:33.420625+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:30.964755+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:22.513614+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:14.948720+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:19:10.431389+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:20:03.826389+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6835/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.99:6801/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6800/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/795365709":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/17074640":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6800/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/881490667":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/888319381":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/4129805635":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6834/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.99:6800/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6801/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:6801/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/405712510":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/1548831286":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/3483428223":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/3192879133":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/1363241618":"2024-09-14T14:21:07.041302+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:24:57.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:57 smithi099 bash[22656]: audit 2024-09-13T14:24:56.797792+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.99:0/96263241' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:24:57.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:57 smithi099 bash[32082]: audit 2024-09-13T14:24:56.797792+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.99:0/96263241' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:24:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:57 smithi177 bash[24494]: audit 2024-09-13T14:24:56.797792+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.99:0/96263241' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:24:57.650 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:24:57.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:24:58.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:24:58 smithi099 bash[22656]: cluster 2024-09-13T14:24:57.130846+0000 mgr.y (mgr.24287) 154 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:58.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:24:58 smithi099 bash[32082]: cluster 2024-09-13T14:24:57.130846+0000 mgr.y (mgr.24287) 154 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:24:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:24:58 smithi177 bash[24494]: cluster 2024-09-13T14:24:57.130846+0000 mgr.y (mgr.24287) 154 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:00.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:00 smithi099 bash[22656]: cluster 2024-09-13T14:24:59.132024+0000 mgr.y (mgr.24287) 155 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:00.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:00 smithi099 bash[32082]: cluster 2024-09-13T14:24:59.132024+0000 mgr.y (mgr.24287) 155 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:00.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:00 smithi177 bash[24494]: cluster 2024-09-13T14:24:59.132024+0000 mgr.y (mgr.24287) 155 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:02.444 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:02.540 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:02 smithi099 bash[22656]: cluster 2024-09-13T14:25:01.132728+0000 mgr.y (mgr.24287) 156 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:02.540 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:25:02 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:02] "GET /metrics HTTP/1.1" 200 240010 "" "Prometheus/2.43.0" 2024-09-13T14:25:02.541 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:02 smithi099 bash[32082]: cluster 2024-09-13T14:25:01.132728+0000 mgr.y (mgr.24287) 156 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:02.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:02 smithi177 bash[24494]: cluster 2024-09-13T14:25:01.132728+0000 mgr.y (mgr.24287) 156 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:04.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:04 smithi099 bash[22656]: cluster 2024-09-13T14:25:03.133858+0000 mgr.y (mgr.24287) 157 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:04.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:04 smithi099 bash[32082]: cluster 2024-09-13T14:25:03.133858+0000 mgr.y (mgr.24287) 157 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:04.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:04 smithi177 bash[24494]: cluster 2024-09-13T14:25:03.133858+0000 mgr.y (mgr.24287) 157 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:05.855 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:25:05 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:25:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:25:06.180 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":65,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:10:21.363918+0000","modified":"2024-09-13T14:24:26.174069+0000","last_up_change":"2024-09-13T14:20:05.471230+0000","last_in_change":"2024-09-13T14:19:38.687766+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:15:37.290226+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:21:51.531836+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:24:15.786407+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:24:16.627611+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"0624feaa-339b-4468-be53-582a1f04352c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6803","nonce":500907575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6805","nonce":500907575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6809","nonce":500907575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":500907575},{"type":"v1","addr":"172.21.15.99:6807","nonce":500907575}]},"public_addr":"172.21.15.99:6803/500907575","cluster_addr":"172.21.15.99:6805/500907575","heartbeat_back_addr":"172.21.15.99:6809/500907575","heartbeat_front_addr":"172.21.15.99:6807/500907575","state":["exists","up"]},{"osd":1,"uuid":"b7b63553-e49d-4636-b3e2-43b72556a1a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6811","nonce":4008940153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6813","nonce":4008940153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6817","nonce":4008940153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":4008940153},{"type":"v1","addr":"172.21.15.99:6815","nonce":4008940153}]},"public_addr":"172.21.15.99:6811/4008940153","cluster_addr":"172.21.15.99:6813/4008940153","heartbeat_back_addr":"172.21.15.99:6817/4008940153","heartbeat_front_addr":"172.21.15.99:6815/4008940153","state":["exists","up"]},{"osd":2,"uuid":"3b9eff6d-5f5c-4843-a1ff-815a902167ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6819","nonce":4220960661}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6821","nonce":4220960661}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6825","nonce":4220960661}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":4220960661},{"type":"v1","addr":"172.21.15.99:6823","nonce":4220960661}]},"public_addr":"172.21.15.99:6819/4220960661","cluster_addr":"172.21.15.99:6821/4220960661","heartbeat_back_addr":"172.21.15.99:6825/4220960661","heartbeat_front_addr":"172.21.15.99:6823/4220960661","state":["exists","up"]},{"osd":3,"uuid":"686c8d3c-9cab-4920-b02b-d243324f5808","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6827","nonce":1221827921}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6829","nonce":1221827921}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6833","nonce":1221827921}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":1221827921},{"type":"v1","addr":"172.21.15.99:6831","nonce":1221827921}]},"public_addr":"172.21.15.99:6827/1221827921","cluster_addr":"172.21.15.99:6829/1221827921","heartbeat_back_addr":"172.21.15.99:6833/1221827921","heartbeat_front_addr":"172.21.15.99:6831/1221827921","state":["exists","up"]},{"osd":4,"uuid":"2d56712b-a6fb-4089-9583-850beed92b37","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6801","nonce":2146549575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6803","nonce":2146549575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6807","nonce":2146549575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2146549575},{"type":"v1","addr":"172.21.15.177:6805","nonce":2146549575}]},"public_addr":"172.21.15.177:6801/2146549575","cluster_addr":"172.21.15.177:6803/2146549575","heartbeat_back_addr":"172.21.15.177:6807/2146549575","heartbeat_front_addr":"172.21.15.177:6805/2146549575","state":["exists","up"]},{"osd":5,"uuid":"414b880c-6534-495f-a54c-f8d28ba81c73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6809","nonce":947070587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6811","nonce":947070587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6815","nonce":947070587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6813","nonce":947070587}]},"public_addr":"172.21.15.177:6809/947070587","cluster_addr":"172.21.15.177:6811/947070587","heartbeat_back_addr":"172.21.15.177:6815/947070587","heartbeat_front_addr":"172.21.15.177:6813/947070587","state":["exists","up"]},{"osd":6,"uuid":"ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6817","nonce":3817692594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6819","nonce":3817692594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6823","nonce":3817692594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6821","nonce":3817692594}]},"public_addr":"172.21.15.177:6817/3817692594","cluster_addr":"172.21.15.177:6819/3817692594","heartbeat_back_addr":"172.21.15.177:6823/3817692594","heartbeat_front_addr":"172.21.15.177:6821/3817692594","state":["exists","up"]},{"osd":7,"uuid":"ad2c2977-fd6d-42f8-962e-907c2bf3204b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6825","nonce":1984676946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6827","nonce":1984676946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6831","nonce":1984676946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6829","nonce":1984676946}]},"public_addr":"172.21.15.177:6825/1984676946","cluster_addr":"172.21.15.177:6827/1984676946","heartbeat_back_addr":"172.21.15.177:6831/1984676946","heartbeat_front_addr":"172.21.15.177:6829/1984676946","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:46.390607+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:38.398637+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:15:33.420625+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:30.964755+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:22.513614+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:14.948720+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:19:10.431389+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:20:03.826389+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:6835/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.99:6801/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6800/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/795365709":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/17074640":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6800/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/881490667":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/888319381":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/4129805635":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6834/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.99:6800/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6801/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:6801/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/405712510":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/1548831286":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/3483428223":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/3192879133":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/1363241618":"2024-09-14T14:21:07.041302+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:25:06.490 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:06 smithi099 bash[22656]: cluster 2024-09-13T14:25:05.134583+0000 mgr.y (mgr.24287) 158 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:06.490 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:06 smithi099 bash[22656]: audit 2024-09-13T14:25:06.185745+0000 mon.a (mon.0) 811 : audit [DBG] from='client.? 172.21.15.99:0/1816985029' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:25:06.491 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:06 smithi099 bash[32082]: cluster 2024-09-13T14:25:05.134583+0000 mgr.y (mgr.24287) 158 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:06.491 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:06 smithi099 bash[32082]: audit 2024-09-13T14:25:06.185745+0000 mon.a (mon.0) 811 : audit [DBG] from='client.? 172.21.15.99:0/1816985029' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:25:06.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:06 smithi177 bash[24494]: cluster 2024-09-13T14:25:05.134583+0000 mgr.y (mgr.24287) 158 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:06 smithi177 bash[24494]: audit 2024-09-13T14:25:06.185745+0000 mon.a (mon.0) 811 : audit [DBG] from='client.? 172.21.15.99:0/1816985029' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:25:07.006 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:25:07.006 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:25:07.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:25:07.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:25:07.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:25:07.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:25:07.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:25:07.009 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:25:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[32082]: audit 2024-09-13T14:25:07.102743+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[32082]: audit 2024-09-13T14:25:07.103100+0000 mon.c (mon.2) 106 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[32082]: audit 2024-09-13T14:25:07.103482+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[32082]: audit 2024-09-13T14:25:07.103613+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[32082]: audit 2024-09-13T14:25:07.104500+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[32082]: audit 2024-09-13T14:25:07.104938+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[32082]: audit 2024-09-13T14:25:07.154225+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[22656]: audit 2024-09-13T14:25:07.102743+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[22656]: audit 2024-09-13T14:25:07.103100+0000 mon.c (mon.2) 106 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[22656]: audit 2024-09-13T14:25:07.103482+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[22656]: audit 2024-09-13T14:25:07.103613+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[22656]: audit 2024-09-13T14:25:07.104500+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[22656]: audit 2024-09-13T14:25:07.104938+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:07 smithi099 bash[22656]: audit 2024-09-13T14:25:07.154225+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:07 smithi177 bash[24494]: audit 2024-09-13T14:25:07.102743+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:07 smithi177 bash[24494]: audit 2024-09-13T14:25:07.103100+0000 mon.c (mon.2) 106 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:07 smithi177 bash[24494]: audit 2024-09-13T14:25:07.103482+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:07 smithi177 bash[24494]: audit 2024-09-13T14:25:07.103613+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:07 smithi177 bash[24494]: audit 2024-09-13T14:25:07.104500+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:07 smithi177 bash[24494]: audit 2024-09-13T14:25:07.104938+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-13T14:25:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:07 smithi177 bash[24494]: audit 2024-09-13T14:25:07.154225+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:08.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[22656]: cluster 2024-09-13T14:25:07.135090+0000 mgr.y (mgr.24287) 159 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:08.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[22656]: audit 2024-09-13T14:25:07.236424+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-13T14:25:08.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[22656]: audit 2024-09-13T14:25:07.236774+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-13T14:25:08.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[22656]: audit 2024-09-13T14:25:07.236929+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-13T14:25:08.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[22656]: cluster 2024-09-13T14:25:07.237121+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:25:08.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[22656]: cluster 2024-09-13T14:25:08.241788+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:25:08.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[32082]: cluster 2024-09-13T14:25:07.135090+0000 mgr.y (mgr.24287) 159 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:08.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[32082]: audit 2024-09-13T14:25:07.236424+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-13T14:25:08.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[32082]: audit 2024-09-13T14:25:07.236774+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-13T14:25:08.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[32082]: audit 2024-09-13T14:25:07.236929+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-13T14:25:08.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[32082]: cluster 2024-09-13T14:25:07.237121+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:25:08.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:08 smithi099 bash[32082]: cluster 2024-09-13T14:25:08.241788+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:25:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:08 smithi177 bash[24494]: cluster 2024-09-13T14:25:07.135090+0000 mgr.y (mgr.24287) 159 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:08 smithi177 bash[24494]: audit 2024-09-13T14:25:07.236424+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-13T14:25:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:08 smithi177 bash[24494]: audit 2024-09-13T14:25:07.236774+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-13T14:25:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:08 smithi177 bash[24494]: audit 2024-09-13T14:25:07.236929+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-13T14:25:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:08 smithi177 bash[24494]: cluster 2024-09-13T14:25:07.237121+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:25:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:08 smithi177 bash[24494]: cluster 2024-09-13T14:25:08.241788+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:25:10.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:10 smithi099 bash[22656]: cluster 2024-09-13T14:25:09.135659+0000 mgr.y (mgr.24287) 160 : cluster [DBG] pgmap v139: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:10 smithi099 bash[32082]: cluster 2024-09-13T14:25:09.135659+0000 mgr.y (mgr.24287) 160 : cluster [DBG] pgmap v139: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:10.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:10 smithi177 bash[24494]: cluster 2024-09-13T14:25:09.135659+0000 mgr.y (mgr.24287) 160 : cluster [DBG] pgmap v139: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:11.924 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:11.925 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:11.925 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:11.926 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:11.926 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:11.926 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:11.931 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:11.931 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:12.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:12 smithi099 bash[22656]: cluster 2024-09-13T14:25:11.136268+0000 mgr.y (mgr.24287) 161 : cluster [DBG] pgmap v140: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:12.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:25:12 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:12] "GET /metrics HTTP/1.1" 200 239967 "" "Prometheus/2.43.0" 2024-09-13T14:25:12.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:12 smithi099 bash[32082]: cluster 2024-09-13T14:25:11.136268+0000 mgr.y (mgr.24287) 161 : cluster [DBG] pgmap v140: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:12.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:12 smithi177 bash[24494]: cluster 2024-09-13T14:25:11.136268+0000 mgr.y (mgr.24287) 161 : cluster [DBG] pgmap v140: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:14.360 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:14 smithi099 bash[22656]: cluster 2024-09-13T14:25:13.137280+0000 mgr.y (mgr.24287) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:14.361 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:14 smithi099 bash[32082]: cluster 2024-09-13T14:25:13.137280+0000 mgr.y (mgr.24287) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:14.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:14 smithi177 bash[24494]: cluster 2024-09-13T14:25:13.137280+0000 mgr.y (mgr.24287) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:15.855 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:25:15 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:25:16.529 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:16 smithi099 bash[22656]: cluster 2024-09-13T14:25:15.137805+0000 mgr.y (mgr.24287) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:16.529 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:16 smithi099 bash[32082]: cluster 2024-09-13T14:25:15.137805+0000 mgr.y (mgr.24287) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:16.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:16 smithi177 bash[24494]: cluster 2024-09-13T14:25:15.137805+0000 mgr.y (mgr.24287) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:18.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:18 smithi099 bash[22656]: cluster 2024-09-13T14:25:17.138385+0000 mgr.y (mgr.24287) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:18.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:18 smithi099 bash[32082]: cluster 2024-09-13T14:25:17.138385+0000 mgr.y (mgr.24287) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:18.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:18 smithi177 bash[24494]: cluster 2024-09-13T14:25:17.138385+0000 mgr.y (mgr.24287) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:20.538 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:20 smithi099 bash[22656]: cluster 2024-09-13T14:25:19.139479+0000 mgr.y (mgr.24287) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:20.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:20 smithi099 bash[32082]: cluster 2024-09-13T14:25:19.139479+0000 mgr.y (mgr.24287) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:20.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:20 smithi177 bash[24494]: cluster 2024-09-13T14:25:19.139479+0000 mgr.y (mgr.24287) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:22.376 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:25:22 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:22] "GET /metrics HTTP/1.1" 200 239967 "" "Prometheus/2.43.0" 2024-09-13T14:25:22.376 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:22 smithi099 bash[22656]: cluster 2024-09-13T14:25:21.140048+0000 mgr.y (mgr.24287) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:22.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:22 smithi177 bash[24494]: cluster 2024-09-13T14:25:21.140048+0000 mgr.y (mgr.24287) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:22.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:22 smithi177 bash[24494]: audit 2024-09-13T14:25:22.154642+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:22.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:22 smithi099 bash[22656]: audit 2024-09-13T14:25:22.154642+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:22.796 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:22 smithi099 bash[32082]: cluster 2024-09-13T14:25:21.140048+0000 mgr.y (mgr.24287) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:22.796 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:22 smithi099 bash[32082]: audit 2024-09-13T14:25:22.154642+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:24.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:24 smithi099 bash[22656]: cluster 2024-09-13T14:25:23.141104+0000 mgr.y (mgr.24287) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:24.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:24 smithi099 bash[32082]: cluster 2024-09-13T14:25:23.141104+0000 mgr.y (mgr.24287) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:24.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:24 smithi177 bash[24494]: cluster 2024-09-13T14:25:23.141104+0000 mgr.y (mgr.24287) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:25.855 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:25:25 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:25:26.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:26 smithi099 bash[22656]: cluster 2024-09-13T14:25:25.141847+0000 mgr.y (mgr.24287) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:26.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:26 smithi099 bash[32082]: cluster 2024-09-13T14:25:25.141847+0000 mgr.y (mgr.24287) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:26.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:26 smithi177 bash[24494]: cluster 2024-09-13T14:25:25.141847+0000 mgr.y (mgr.24287) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:27.842 INFO:teuthology.orchestra.run.smithi099.stdout:55834574979 2024-09-13T14:25:27.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:25:28.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:28 smithi099 bash[22656]: cluster 2024-09-13T14:25:27.142504+0000 mgr.y (mgr.24287) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:28.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:28 smithi099 bash[32082]: cluster 2024-09-13T14:25:27.142504+0000 mgr.y (mgr.24287) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:28.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:28 smithi177 bash[24494]: cluster 2024-09-13T14:25:27.142504+0000 mgr.y (mgr.24287) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:30.503 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:30 smithi099 bash[22656]: cluster 2024-09-13T14:25:29.143596+0000 mgr.y (mgr.24287) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:30.504 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:30 smithi099 bash[32082]: cluster 2024-09-13T14:25:29.143596+0000 mgr.y (mgr.24287) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:30.542 INFO:teuthology.orchestra.run.smithi099.stdout:158913790040 2024-09-13T14:25:30.543 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:25:30.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:30 smithi177 bash[24494]: cluster 2024-09-13T14:25:29.143596+0000 mgr.y (mgr.24287) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:31.393 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:31 smithi099 bash[22656]: audit 2024-09-13T14:25:30.799780+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:31.393 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:31 smithi099 bash[32082]: audit 2024-09-13T14:25:30.799780+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:31.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:31 smithi177 bash[24494]: audit 2024-09-13T14:25:30.799780+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:32.159 INFO:teuthology.orchestra.run.smithi099.stdout:184683593807 2024-09-13T14:25:32.159 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:25:32.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:32 smithi099 bash[22656]: cluster 2024-09-13T14:25:31.144400+0000 mgr.y (mgr.24287) 171 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:32.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:32 smithi099 bash[22656]: audit 2024-09-13T14:25:31.295407+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:32.392 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:25:32 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:32] "GET /metrics HTTP/1.1" 200 239992 "" "Prometheus/2.43.0" 2024-09-13T14:25:32.416 INFO:teuthology.orchestra.run.smithi099.stdout:77309411450 2024-09-13T14:25:32.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:25:32.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:32 smithi099 bash[22656]: audit 2024-09-13T14:25:31.302918+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:32.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:32 smithi099 bash[22656]: audit 2024-09-13T14:25:31.313184+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:25:32.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:32 smithi177 bash[24494]: cluster 2024-09-13T14:25:31.144400+0000 mgr.y (mgr.24287) 171 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:32.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:32 smithi177 bash[24494]: audit 2024-09-13T14:25:31.295407+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:32.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:32 smithi177 bash[24494]: audit 2024-09-13T14:25:31.302918+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:32.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:32 smithi177 bash[24494]: audit 2024-09-13T14:25:31.313184+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:25:32.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:32 smithi099 bash[32082]: cluster 2024-09-13T14:25:31.144400+0000 mgr.y (mgr.24287) 171 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:32.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:32 smithi099 bash[32082]: audit 2024-09-13T14:25:31.295407+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:32.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:32 smithi099 bash[32082]: audit 2024-09-13T14:25:31.302918+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:32.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:32 smithi099 bash[32082]: audit 2024-09-13T14:25:31.313184+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:25:33.845 INFO:teuthology.orchestra.run.smithi099.stdout:34359738510 2024-09-13T14:25:33.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:25:34.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:34 smithi099 bash[22656]: cluster 2024-09-13T14:25:33.145468+0000 mgr.y (mgr.24287) 172 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:34.447 INFO:teuthology.orchestra.run.smithi099.stdout:111669149807 2024-09-13T14:25:34.448 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:25:34.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:34 smithi177 bash[24494]: cluster 2024-09-13T14:25:33.145468+0000 mgr.y (mgr.24287) 172 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:34.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:34 smithi099 bash[32082]: cluster 2024-09-13T14:25:33.145468+0000 mgr.y (mgr.24287) 172 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:34.647 INFO:teuthology.orchestra.run.smithi099.stdout:133143986276 2024-09-13T14:25:34.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:25:34.837 INFO:teuthology.orchestra.run.smithi099.stdout:210453397572 2024-09-13T14:25:34.838 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:25:35.855 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:25:35 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:25:36.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:36 smithi099 bash[22656]: cluster 2024-09-13T14:25:35.146194+0000 mgr.y (mgr.24287) 173 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:36.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:36 smithi099 bash[32082]: cluster 2024-09-13T14:25:35.146194+0000 mgr.y (mgr.24287) 173 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:36.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:36 smithi177 bash[24494]: cluster 2024-09-13T14:25:35.146194+0000 mgr.y (mgr.24287) 173 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:36.994 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:36.995 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:36.995 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:37.002 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:37.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:37 smithi099 bash[22656]: audit 2024-09-13T14:25:37.155114+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:37.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:37 smithi099 bash[32082]: audit 2024-09-13T14:25:37.155114+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:37.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:37 smithi177 bash[24494]: audit 2024-09-13T14:25:37.155114+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:38.010 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:38.473 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:38 smithi099 bash[22656]: cluster 2024-09-13T14:25:37.146876+0000 mgr.y (mgr.24287) 174 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:38.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:38 smithi099 bash[32082]: cluster 2024-09-13T14:25:37.146876+0000 mgr.y (mgr.24287) 174 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:38.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:38 smithi177 bash[24494]: cluster 2024-09-13T14:25:37.146876+0000 mgr.y (mgr.24287) 174 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:39.031 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:39.032 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:39.032 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:25:40.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:40 smithi099 bash[22656]: cluster 2024-09-13T14:25:39.147976+0000 mgr.y (mgr.24287) 175 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:40.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:40 smithi099 bash[32082]: cluster 2024-09-13T14:25:39.147976+0000 mgr.y (mgr.24287) 175 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:40.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:40 smithi177 bash[24494]: cluster 2024-09-13T14:25:39.147976+0000 mgr.y (mgr.24287) 175 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:42.432 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:42 smithi099 bash[22656]: cluster 2024-09-13T14:25:41.148613+0000 mgr.y (mgr.24287) 176 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:42.433 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:25:42 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:42] "GET /metrics HTTP/1.1" 200 239982 "" "Prometheus/2.43.0" 2024-09-13T14:25:42.433 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:42 smithi099 bash[32082]: cluster 2024-09-13T14:25:41.148613+0000 mgr.y (mgr.24287) 176 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:42.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:42 smithi177 bash[24494]: cluster 2024-09-13T14:25:41.148613+0000 mgr.y (mgr.24287) 176 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:44.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:44 smithi099 bash[32082]: cluster 2024-09-13T14:25:43.149608+0000 mgr.y (mgr.24287) 177 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:44.579 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:44 smithi099 bash[22656]: cluster 2024-09-13T14:25:43.149608+0000 mgr.y (mgr.24287) 177 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:44.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:44 smithi177 bash[24494]: cluster 2024-09-13T14:25:43.149608+0000 mgr.y (mgr.24287) 177 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:45.855 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:25:45 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:25:46.443 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:46 smithi099 bash[22656]: cluster 2024-09-13T14:25:45.150223+0000 mgr.y (mgr.24287) 178 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:46.443 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:46 smithi099 bash[32082]: cluster 2024-09-13T14:25:45.150223+0000 mgr.y (mgr.24287) 178 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:46.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:46 smithi177 bash[24494]: cluster 2024-09-13T14:25:45.150223+0000 mgr.y (mgr.24287) 178 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:48.393 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:48 smithi099 bash[22656]: cluster 2024-09-13T14:25:47.150665+0000 mgr.y (mgr.24287) 179 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:48.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:48 smithi177 bash[24494]: cluster 2024-09-13T14:25:47.150665+0000 mgr.y (mgr.24287) 179 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:48.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:48 smithi099 bash[32082]: cluster 2024-09-13T14:25:47.150665+0000 mgr.y (mgr.24287) 179 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:49.300 INFO:teuthology.orchestra.run.smithi099.stdout:184683593810 2024-09-13T14:25:49.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:49 smithi177 bash[24494]: audit 2024-09-13T14:25:49.306950+0000 mon.a (mon.0) 821 : audit [DBG] from='client.? 172.21.15.99:0/815439801' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:25:49.788 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:49 smithi099 bash[32082]: audit 2024-09-13T14:25:49.306950+0000 mon.a (mon.0) 821 : audit [DBG] from='client.? 172.21.15.99:0/815439801' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:25:49.789 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:49 smithi099 bash[22656]: audit 2024-09-13T14:25:49.306950+0000 mon.a (mon.0) 821 : audit [DBG] from='client.? 172.21.15.99:0/815439801' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:25:50.598 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:50 smithi099 bash[32082]: cluster 2024-09-13T14:25:49.151183+0000 mgr.y (mgr.24287) 180 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:50.599 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:50 smithi099 bash[22656]: cluster 2024-09-13T14:25:49.151183+0000 mgr.y (mgr.24287) 180 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:50.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:50 smithi177 bash[24494]: cluster 2024-09-13T14:25:49.151183+0000 mgr.y (mgr.24287) 180 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:52.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:25:52 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:52] "GET /metrics HTTP/1.1" 200 239982 "" "Prometheus/2.43.0" 2024-09-13T14:25:52.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:52 smithi177 bash[24494]: cluster 2024-09-13T14:25:51.151710+0000 mgr.y (mgr.24287) 181 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:52.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:52 smithi177 bash[24494]: audit 2024-09-13T14:25:52.155558+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:52.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:52 smithi099 bash[22656]: cluster 2024-09-13T14:25:51.151710+0000 mgr.y (mgr.24287) 181 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:52.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:52 smithi099 bash[22656]: audit 2024-09-13T14:25:52.155558+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:52.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:52 smithi099 bash[32082]: cluster 2024-09-13T14:25:51.151710+0000 mgr.y (mgr.24287) 181 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:52.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:52 smithi099 bash[32082]: audit 2024-09-13T14:25:52.155558+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:54.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:54 smithi177 bash[24494]: cluster 2024-09-13T14:25:53.152934+0000 mgr.y (mgr.24287) 182 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:54.672 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:54 smithi099 bash[22656]: cluster 2024-09-13T14:25:53.152934+0000 mgr.y (mgr.24287) 182 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:54.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:54 smithi099 bash[32082]: cluster 2024-09-13T14:25:53.152934+0000 mgr.y (mgr.24287) 182 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:54.699 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593807 got 184683593810 for osd.6 2024-09-13T14:25:54.700 DEBUG:teuthology.parallel:result is None 2024-09-13T14:25:55.000 INFO:teuthology.orchestra.run.smithi099.stdout:55834574985 2024-09-13T14:25:55.214 INFO:teuthology.orchestra.run.smithi099.stdout:158913790046 2024-09-13T14:25:55.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:55 smithi099 bash[22656]: audit 2024-09-13T14:25:55.007527+0000 mon.a (mon.0) 822 : audit [DBG] from='client.? 172.21.15.99:0/3322204226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:25:55.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:55 smithi099 bash[22656]: audit 2024-09-13T14:25:55.221012+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.99:0/3545897262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:25:55.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:55 smithi099 bash[32082]: audit 2024-09-13T14:25:55.007527+0000 mon.a (mon.0) 822 : audit [DBG] from='client.? 172.21.15.99:0/3322204226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:25:55.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:55 smithi099 bash[32082]: audit 2024-09-13T14:25:55.221012+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.99:0/3545897262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:25:55.717 INFO:teuthology.orchestra.run.smithi099.stdout:111669149812 2024-09-13T14:25:55.855 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:25:55 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:25:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:25:55.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:55 smithi177 bash[24494]: audit 2024-09-13T14:25:55.007527+0000 mon.a (mon.0) 822 : audit [DBG] from='client.? 172.21.15.99:0/3322204226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:25:55.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:55 smithi177 bash[24494]: audit 2024-09-13T14:25:55.221012+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.99:0/3545897262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:25:55.906 INFO:teuthology.orchestra.run.smithi099.stdout:77309411455 2024-09-13T14:25:56.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:56 smithi099 bash[22656]: cluster 2024-09-13T14:25:55.153546+0000 mgr.y (mgr.24287) 183 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:56.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:56 smithi099 bash[22656]: audit 2024-09-13T14:25:55.724315+0000 mon.a (mon.0) 823 : audit [DBG] from='client.? 172.21.15.99:0/3011321677' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:25:56.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:56 smithi099 bash[22656]: audit 2024-09-13T14:25:55.913584+0000 mon.a (mon.0) 824 : audit [DBG] from='client.? 172.21.15.99:0/1891724692' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:25:56.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:56 smithi099 bash[32082]: cluster 2024-09-13T14:25:55.153546+0000 mgr.y (mgr.24287) 183 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:56.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:56 smithi099 bash[32082]: audit 2024-09-13T14:25:55.724315+0000 mon.a (mon.0) 823 : audit [DBG] from='client.? 172.21.15.99:0/3011321677' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:25:56.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:56 smithi099 bash[32082]: audit 2024-09-13T14:25:55.913584+0000 mon.a (mon.0) 824 : audit [DBG] from='client.? 172.21.15.99:0/1891724692' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:25:56.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:56 smithi177 bash[24494]: cluster 2024-09-13T14:25:55.153546+0000 mgr.y (mgr.24287) 183 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:56.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:56 smithi177 bash[24494]: audit 2024-09-13T14:25:55.724315+0000 mon.a (mon.0) 823 : audit [DBG] from='client.? 172.21.15.99:0/3011321677' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:25:56.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:56 smithi177 bash[24494]: audit 2024-09-13T14:25:55.913584+0000 mon.a (mon.0) 824 : audit [DBG] from='client.? 172.21.15.99:0/1891724692' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:25:57.220 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790040 got 158913790046 for osd.5 2024-09-13T14:25:57.220 DEBUG:teuthology.parallel:result is None 2024-09-13T14:25:57.673 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574979 got 55834574985 for osd.1 2024-09-13T14:25:57.673 DEBUG:teuthology.parallel:result is None 2024-09-13T14:25:58.553 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:58 smithi099 bash[32082]: cluster 2024-09-13T14:25:57.154161+0000 mgr.y (mgr.24287) 184 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:58.553 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:58 smithi099 bash[22656]: cluster 2024-09-13T14:25:57.154161+0000 mgr.y (mgr.24287) 184 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:58.554 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149807 got 111669149812 for osd.3 2024-09-13T14:25:58.555 DEBUG:teuthology.parallel:result is None 2024-09-13T14:25:58.829 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411450 got 77309411455 for osd.2 2024-09-13T14:25:58.829 DEBUG:teuthology.parallel:result is None 2024-09-13T14:25:58.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:58 smithi177 bash[24494]: cluster 2024-09-13T14:25:57.154161+0000 mgr.y (mgr.24287) 184 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:25:59.054 INFO:teuthology.orchestra.run.smithi099.stdout:133143986281 2024-09-13T14:25:59.091 INFO:teuthology.orchestra.run.smithi099.stdout:34359738515 2024-09-13T14:25:59.117 INFO:teuthology.orchestra.run.smithi099.stdout:210453397577 2024-09-13T14:25:59.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:59 smithi099 bash[32082]: audit 2024-09-13T14:25:59.060829+0000 mon.a (mon.0) 825 : audit [DBG] from='client.? 172.21.15.99:0/2782834334' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:25:59.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:59 smithi099 bash[32082]: audit 2024-09-13T14:25:59.097868+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.99:0/247716068' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:25:59.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:25:59 smithi099 bash[32082]: audit 2024-09-13T14:25:59.124490+0000 mon.a (mon.0) 826 : audit [DBG] from='client.? 172.21.15.99:0/2668187985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:25:59.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:59 smithi099 bash[22656]: audit 2024-09-13T14:25:59.060829+0000 mon.a (mon.0) 825 : audit [DBG] from='client.? 172.21.15.99:0/2782834334' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:25:59.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:59 smithi099 bash[22656]: audit 2024-09-13T14:25:59.097868+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.99:0/247716068' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:25:59.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:25:59 smithi099 bash[22656]: audit 2024-09-13T14:25:59.124490+0000 mon.a (mon.0) 826 : audit [DBG] from='client.? 172.21.15.99:0/2668187985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:25:59.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:59 smithi177 bash[24494]: audit 2024-09-13T14:25:59.060829+0000 mon.a (mon.0) 825 : audit [DBG] from='client.? 172.21.15.99:0/2782834334' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:25:59.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:59 smithi177 bash[24494]: audit 2024-09-13T14:25:59.097868+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.99:0/247716068' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:25:59.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:25:59 smithi177 bash[24494]: audit 2024-09-13T14:25:59.124490+0000 mon.a (mon.0) 826 : audit [DBG] from='client.? 172.21.15.99:0/2668187985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:26:00.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:00 smithi099 bash[32082]: cluster 2024-09-13T14:25:59.154692+0000 mgr.y (mgr.24287) 185 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:00.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:00 smithi099 bash[22656]: cluster 2024-09-13T14:25:59.154692+0000 mgr.y (mgr.24287) 185 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:00.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:00 smithi177 bash[24494]: cluster 2024-09-13T14:25:59.154692+0000 mgr.y (mgr.24287) 185 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:01.088 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986276 got 133143986281 for osd.4 2024-09-13T14:26:01.088 DEBUG:teuthology.parallel:result is None 2024-09-13T14:26:01.105 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397572 got 210453397577 for osd.7 2024-09-13T14:26:01.105 DEBUG:teuthology.parallel:result is None 2024-09-13T14:26:01.157 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738510 got 34359738515 for osd.0 2024-09-13T14:26:01.158 DEBUG:teuthology.parallel:result is None 2024-09-13T14:26:01.158 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:26:01.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:26:02.377 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:02 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:02] "GET /metrics HTTP/1.1" 200 239954 "" "Prometheus/2.43.0" 2024-09-13T14:26:02.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:02 smithi099 bash[22656]: cluster 2024-09-13T14:26:01.155161+0000 mgr.y (mgr.24287) 186 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:02.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:02 smithi099 bash[32082]: cluster 2024-09-13T14:26:01.155161+0000 mgr.y (mgr.24287) 186 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:02.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:02 smithi177 bash[24494]: cluster 2024-09-13T14:26:01.155161+0000 mgr.y (mgr.24287) 186 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:04.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:04 smithi099 bash[22656]: cluster 2024-09-13T14:26:03.156170+0000 mgr.y (mgr.24287) 187 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:04.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:04 smithi099 bash[32082]: cluster 2024-09-13T14:26:03.156170+0000 mgr.y (mgr.24287) 187 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:04.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:04 smithi177 bash[24494]: cluster 2024-09-13T14:26:03.156170+0000 mgr.y (mgr.24287) 187 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:05.855 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:26:05 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:26:05.950 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:06.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:06 smithi099 bash[22656]: cluster 2024-09-13T14:26:05.156766+0000 mgr.y (mgr.24287) 188 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:06.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:06 smithi099 bash[32082]: cluster 2024-09-13T14:26:05.156766+0000 mgr.y (mgr.24287) 188 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:06.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:06 smithi177 bash[24494]: cluster 2024-09-13T14:26:05.156766+0000 mgr.y (mgr.24287) 188 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:07.776 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:07 smithi099 bash[22656]: audit 2024-09-13T14:26:07.155773+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:07.777 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:07 smithi099 bash[32082]: audit 2024-09-13T14:26:07.155773+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:07.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:07 smithi177 bash[24494]: audit 2024-09-13T14:26:07.155773+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:08.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:08 smithi099 bash[22656]: cluster 2024-09-13T14:26:07.157382+0000 mgr.y (mgr.24287) 189 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:08.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:08 smithi099 bash[32082]: cluster 2024-09-13T14:26:07.157382+0000 mgr.y (mgr.24287) 189 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:08 smithi177 bash[24494]: cluster 2024-09-13T14:26:07.157382+0000 mgr.y (mgr.24287) 189 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:09.889 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:26:09.890 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:26:10.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:10 smithi099 bash[22656]: cluster 2024-09-13T14:26:09.158501+0000 mgr.y (mgr.24287) 190 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:10.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:10 smithi099 bash[22656]: audit 2024-09-13T14:26:09.874182+0000 mgr.y (mgr.24287) 191 : audit [DBG] from='client.14718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:26:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:10 smithi099 bash[32082]: cluster 2024-09-13T14:26:09.158501+0000 mgr.y (mgr.24287) 190 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:10 smithi099 bash[32082]: audit 2024-09-13T14:26:09.874182+0000 mgr.y (mgr.24287) 191 : audit [DBG] from='client.14718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:26:10.701 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":169,"stamp":"2024-09-13T14:26:09.157700+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":173,"num_read_kb":280,"num_write":160,"num_write_kb":1643,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127,"ondisk_log_size":127,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386020,"kb_used_data":7176,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575196,"statfs":{"total":767960285184,"available":765517000704,"internally_reserved":0,"allocated":7348224,"data_stored":4207019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003872"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.243044+0000","last_change":"2024-09-13T14:24:18.584431+0000","last_active":"2024-09-13T14:25:07.243044+0000","last_peered":"2024-09-13T14:25:07.243044+0000","last_clean":"2024-09-13T14:25:07.243044+0000","last_became_active":"2024-09-13T14:24:18.584135+0000","last_became_peered":"2024-09-13T14:24:18.584135+0000","last_unstale":"2024-09-13T14:25:07.243044+0000","last_undegraded":"2024-09-13T14:25:07.243044+0000","last_fullsized":"2024-09-13T14:25:07.243044+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:43.909934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331230+0000","last_change":"2024-09-13T14:24:17.242167+0000","last_active":"2024-09-13T14:25:07.331230+0000","last_peered":"2024-09-13T14:25:07.331230+0000","last_clean":"2024-09-13T14:25:07.331230+0000","last_became_active":"2024-09-13T14:24:17.241529+0000","last_became_peered":"2024-09-13T14:24:17.241529+0000","last_unstale":"2024-09-13T14:25:07.331230+0000","last_undegraded":"2024-09-13T14:25:07.331230+0000","last_fullsized":"2024-09-13T14:25:07.331230+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:46:10.269246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251546+0000","last_change":"2024-09-13T14:24:17.417723+0000","last_active":"2024-09-13T14:25:07.251546+0000","last_peered":"2024-09-13T14:25:07.251546+0000","last_clean":"2024-09-13T14:25:07.251546+0000","last_became_active":"2024-09-13T14:24:17.416614+0000","last_became_peered":"2024-09-13T14:24:17.416614+0000","last_unstale":"2024-09-13T14:25:07.251546+0000","last_undegraded":"2024-09-13T14:25:07.251546+0000","last_fullsized":"2024-09-13T14:25:07.251546+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:19.028275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238242+0000","last_change":"2024-09-13T14:24:17.233597+0000","last_active":"2024-09-13T14:25:07.238242+0000","last_peered":"2024-09-13T14:25:07.238242+0000","last_clean":"2024-09-13T14:25:07.238242+0000","last_became_active":"2024-09-13T14:24:17.231997+0000","last_became_peered":"2024-09-13T14:24:17.231997+0000","last_unstale":"2024-09-13T14:25:07.238242+0000","last_undegraded":"2024-09-13T14:25:07.238242+0000","last_fullsized":"2024-09-13T14:25:07.238242+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:37:44.732444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238865+0000","last_change":"2024-09-13T14:24:17.417310+0000","last_active":"2024-09-13T14:25:07.238865+0000","last_peered":"2024-09-13T14:25:07.238865+0000","last_clean":"2024-09-13T14:25:07.238865+0000","last_became_active":"2024-09-13T14:24:17.416984+0000","last_became_peered":"2024-09-13T14:24:17.416984+0000","last_unstale":"2024-09-13T14:25:07.238865+0000","last_undegraded":"2024-09-13T14:25:07.238865+0000","last_fullsized":"2024-09-13T14:25:07.238865+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:22.686718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331004+0000","last_change":"2024-09-13T14:24:17.241349+0000","last_active":"2024-09-13T14:25:07.331004+0000","last_peered":"2024-09-13T14:25:07.331004+0000","last_clean":"2024-09-13T14:25:07.331004+0000","last_became_active":"2024-09-13T14:24:17.240753+0000","last_became_peered":"2024-09-13T14:24:17.240753+0000","last_unstale":"2024-09-13T14:25:07.331004+0000","last_undegraded":"2024-09-13T14:25:07.331004+0000","last_fullsized":"2024-09-13T14:25:07.331004+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:34:11.369561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242757+0000","last_change":"2024-09-13T14:24:18.583287+0000","last_active":"2024-09-13T14:25:07.242757+0000","last_peered":"2024-09-13T14:25:07.242757+0000","last_clean":"2024-09-13T14:25:07.242757+0000","last_became_active":"2024-09-13T14:24:18.582766+0000","last_became_peered":"2024-09-13T14:24:18.582766+0000","last_unstale":"2024-09-13T14:25:07.242757+0000","last_undegraded":"2024-09-13T14:25:07.242757+0000","last_fullsized":"2024-09-13T14:25:07.242757+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:40.011693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.245764+0000","last_change":"2024-09-13T14:24:17.231522+0000","last_active":"2024-09-13T14:25:07.245764+0000","last_peered":"2024-09-13T14:25:07.245764+0000","last_clean":"2024-09-13T14:25:07.245764+0000","last_became_active":"2024-09-13T14:24:17.230661+0000","last_became_peered":"2024-09-13T14:24:17.230661+0000","last_unstale":"2024-09-13T14:25:07.245764+0000","last_undegraded":"2024-09-13T14:25:07.245764+0000","last_fullsized":"2024-09-13T14:25:07.245764+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:36.292481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331690+0000","last_change":"2024-09-13T14:24:17.245589+0000","last_active":"2024-09-13T14:25:07.331690+0000","last_peered":"2024-09-13T14:25:07.331690+0000","last_clean":"2024-09-13T14:25:07.331690+0000","last_became_active":"2024-09-13T14:24:17.244773+0000","last_became_peered":"2024-09-13T14:24:17.244773+0000","last_unstale":"2024-09-13T14:25:07.331690+0000","last_undegraded":"2024-09-13T14:25:07.331690+0000","last_fullsized":"2024-09-13T14:25:07.331690+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:39.537514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.250718+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:25:07.250718+0000","last_peered":"2024-09-13T14:25:07.250718+0000","last_clean":"2024-09-13T14:25:07.250718+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:25:07.250718+0000","last_undegraded":"2024-09-13T14:25:07.250718+0000","last_fullsized":"2024-09-13T14:25:07.250718+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331643+0000","last_change":"2024-09-13T14:24:17.243027+0000","last_active":"2024-09-13T14:25:07.331643+0000","last_peered":"2024-09-13T14:25:07.331643+0000","last_clean":"2024-09-13T14:25:07.331643+0000","last_became_active":"2024-09-13T14:24:17.242540+0000","last_became_peered":"2024-09-13T14:24:17.242540+0000","last_unstale":"2024-09-13T14:25:07.331643+0000","last_undegraded":"2024-09-13T14:25:07.331643+0000","last_fullsized":"2024-09-13T14:25:07.331643+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:23:48.038754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238753+0000","last_change":"2024-09-13T14:24:17.271552+0000","last_active":"2024-09-13T14:25:07.238753+0000","last_peered":"2024-09-13T14:25:07.238753+0000","last_clean":"2024-09-13T14:25:07.238753+0000","last_became_active":"2024-09-13T14:24:17.271089+0000","last_became_peered":"2024-09-13T14:24:17.271089+0000","last_unstale":"2024-09-13T14:25:07.238753+0000","last_undegraded":"2024-09-13T14:25:07.238753+0000","last_fullsized":"2024-09-13T14:25:07.238753+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:24:46.046603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331540+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:25:07.331540+0000","last_peered":"2024-09-13T14:25:07.331540+0000","last_clean":"2024-09-13T14:25:07.331540+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:25:07.331540+0000","last_undegraded":"2024-09-13T14:25:07.331540+0000","last_fullsized":"2024-09-13T14:25:07.331540+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.244063+0000","last_change":"2024-09-13T14:24:17.414918+0000","last_active":"2024-09-13T14:25:07.244063+0000","last_peered":"2024-09-13T14:25:07.244063+0000","last_clean":"2024-09-13T14:25:07.244063+0000","last_became_active":"2024-09-13T14:24:17.414020+0000","last_became_peered":"2024-09-13T14:24:17.414020+0000","last_unstale":"2024-09-13T14:25:07.244063+0000","last_undegraded":"2024-09-13T14:25:07.244063+0000","last_fullsized":"2024-09-13T14:25:07.244063+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:51.567074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251335+0000","last_change":"2024-09-13T14:24:17.227167+0000","last_active":"2024-09-13T14:25:07.251335+0000","last_peered":"2024-09-13T14:25:07.251335+0000","last_clean":"2024-09-13T14:25:07.251335+0000","last_became_active":"2024-09-13T14:24:17.226643+0000","last_became_peered":"2024-09-13T14:24:17.226643+0000","last_unstale":"2024-09-13T14:25:07.251335+0000","last_undegraded":"2024-09-13T14:25:07.251335+0000","last_fullsized":"2024-09-13T14:25:07.251335+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:43:23.833520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238384+0000","last_change":"2024-09-13T14:24:17.271572+0000","last_active":"2024-09-13T14:25:07.238384+0000","last_peered":"2024-09-13T14:25:07.238384+0000","last_clean":"2024-09-13T14:25:07.238384+0000","last_became_active":"2024-09-13T14:24:17.271212+0000","last_became_peered":"2024-09-13T14:24:17.271212+0000","last_unstale":"2024-09-13T14:25:07.238384+0000","last_undegraded":"2024-09-13T14:25:07.238384+0000","last_fullsized":"2024-09-13T14:25:07.238384+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:23.750381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241503+0000","last_change":"2024-09-13T14:24:17.246522+0000","last_active":"2024-09-13T14:25:07.241503+0000","last_peered":"2024-09-13T14:25:07.241503+0000","last_clean":"2024-09-13T14:25:07.241503+0000","last_became_active":"2024-09-13T14:24:17.246005+0000","last_became_peered":"2024-09-13T14:24:17.246005+0000","last_unstale":"2024-09-13T14:25:07.241503+0000","last_undegraded":"2024-09-13T14:25:07.241503+0000","last_fullsized":"2024-09-13T14:25:07.241503+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:52.502894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239518+0000","last_change":"2024-09-13T14:24:17.260664+0000","last_active":"2024-09-13T14:25:07.239518+0000","last_peered":"2024-09-13T14:25:07.239518+0000","last_clean":"2024-09-13T14:25:07.239518+0000","last_became_active":"2024-09-13T14:24:17.260374+0000","last_became_peered":"2024-09-13T14:24:17.260374+0000","last_unstale":"2024-09-13T14:25:07.239518+0000","last_undegraded":"2024-09-13T14:25:07.239518+0000","last_fullsized":"2024-09-13T14:25:07.239518+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:31:01.451956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.244318+0000","last_change":"2024-09-13T14:24:17.229223+0000","last_active":"2024-09-13T14:25:07.244318+0000","last_peered":"2024-09-13T14:25:07.244318+0000","last_clean":"2024-09-13T14:25:07.244318+0000","last_became_active":"2024-09-13T14:24:17.224775+0000","last_became_peered":"2024-09-13T14:24:17.224775+0000","last_unstale":"2024-09-13T14:25:07.244318+0000","last_undegraded":"2024-09-13T14:25:07.244318+0000","last_fullsized":"2024-09-13T14:25:07.244318+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:05.931104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239877+0000","last_change":"2024-09-13T14:24:17.270348+0000","last_active":"2024-09-13T14:25:07.239877+0000","last_peered":"2024-09-13T14:25:07.239877+0000","last_clean":"2024-09-13T14:25:07.239877+0000","last_became_active":"2024-09-13T14:24:17.269821+0000","last_became_peered":"2024-09-13T14:24:17.269821+0000","last_unstale":"2024-09-13T14:25:07.239877+0000","last_undegraded":"2024-09-13T14:25:07.239877+0000","last_fullsized":"2024-09-13T14:25:07.239877+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:28.259239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239368+0000","last_change":"2024-09-13T14:24:17.237862+0000","last_active":"2024-09-13T14:25:07.239368+0000","last_peered":"2024-09-13T14:25:07.239368+0000","last_clean":"2024-09-13T14:25:07.239368+0000","last_became_active":"2024-09-13T14:24:17.233347+0000","last_became_peered":"2024-09-13T14:24:17.233347+0000","last_unstale":"2024-09-13T14:25:07.239368+0000","last_undegraded":"2024-09-13T14:25:07.239368+0000","last_fullsized":"2024-09-13T14:25:07.239368+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:22:33.207677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.240195+0000","last_change":"2024-09-13T14:24:17.271150+0000","last_active":"2024-09-13T14:25:07.240195+0000","last_peered":"2024-09-13T14:25:07.240195+0000","last_clean":"2024-09-13T14:25:07.240195+0000","last_became_active":"2024-09-13T14:24:17.270643+0000","last_became_peered":"2024-09-13T14:24:17.270643+0000","last_unstale":"2024-09-13T14:25:07.240195+0000","last_undegraded":"2024-09-13T14:25:07.240195+0000","last_fullsized":"2024-09-13T14:25:07.240195+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:37:21.003725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242323+0000","last_change":"2024-09-13T14:24:17.251584+0000","last_active":"2024-09-13T14:25:07.242323+0000","last_peered":"2024-09-13T14:25:07.242323+0000","last_clean":"2024-09-13T14:25:07.242323+0000","last_became_active":"2024-09-13T14:24:17.249275+0000","last_became_peered":"2024-09-13T14:24:17.249275+0000","last_unstale":"2024-09-13T14:25:07.242323+0000","last_undegraded":"2024-09-13T14:25:07.242323+0000","last_fullsized":"2024-09-13T14:25:07.242323+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:45.889488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238182+0000","last_change":"2024-09-13T14:24:17.278060+0000","last_active":"2024-09-13T14:25:07.238182+0000","last_peered":"2024-09-13T14:25:07.238182+0000","last_clean":"2024-09-13T14:25:07.238182+0000","last_became_active":"2024-09-13T14:24:17.277722+0000","last_became_peered":"2024-09-13T14:24:17.277722+0000","last_unstale":"2024-09-13T14:25:07.238182+0000","last_undegraded":"2024-09-13T14:25:07.238182+0000","last_fullsized":"2024-09-13T14:25:07.238182+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:15.030086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":57,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.330919+0000","last_change":"2024-09-13T14:20:07.527898+0000","last_active":"2024-09-13T14:25:07.330919+0000","last_peered":"2024-09-13T14:25:07.330919+0000","last_clean":"2024-09-13T14:25:07.330919+0000","last_became_active":"2024-09-13T14:20:07.515370+0000","last_became_peered":"2024-09-13T14:20:07.515370+0000","last_unstale":"2024-09-13T14:25:07.330919+0000","last_undegraded":"2024-09-13T14:25:07.330919+0000","last_fullsized":"2024-09-13T14:25:07.330919+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:45.300816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":28,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.250979+0000","last_change":"2024-09-13T14:22:03.176719+0000","last_active":"2024-09-13T14:25:07.250979+0000","last_peered":"2024-09-13T14:25:07.250979+0000","last_clean":"2024-09-13T14:25:07.250979+0000","last_became_active":"2024-09-13T14:21:52.862358+0000","last_became_peered":"2024-09-13T14:21:52.862358+0000","last_unstale":"2024-09-13T14:25:07.250979+0000","last_undegraded":"2024-09-13T14:25:07.250979+0000","last_fullsized":"2024-09-13T14:25:07.250979+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:22:54.637897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241553+0000","last_change":"2024-09-13T14:24:17.225596+0000","last_active":"2024-09-13T14:25:07.241553+0000","last_peered":"2024-09-13T14:25:07.241553+0000","last_clean":"2024-09-13T14:25:07.241553+0000","last_became_active":"2024-09-13T14:24:17.224861+0000","last_became_peered":"2024-09-13T14:24:17.224861+0000","last_unstale":"2024-09-13T14:25:07.241553+0000","last_undegraded":"2024-09-13T14:25:07.241553+0000","last_fullsized":"2024-09-13T14:25:07.241553+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:23.004735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.244671+0000","last_change":"2024-09-13T14:24:17.415855+0000","last_active":"2024-09-13T14:25:07.244671+0000","last_peered":"2024-09-13T14:25:07.244671+0000","last_clean":"2024-09-13T14:25:07.244671+0000","last_became_active":"2024-09-13T14:24:17.413815+0000","last_became_peered":"2024-09-13T14:24:17.413815+0000","last_unstale":"2024-09-13T14:25:07.244671+0000","last_undegraded":"2024-09-13T14:25:07.244671+0000","last_fullsized":"2024-09-13T14:25:07.244671+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:25:00.118559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.330858+0000","last_change":"2024-09-13T14:22:03.357964+0000","last_active":"2024-09-13T14:25:07.330858+0000","last_peered":"2024-09-13T14:25:07.330858+0000","last_clean":"2024-09-13T14:25:07.330858+0000","last_became_active":"2024-09-13T14:21:52.883199+0000","last_became_peered":"2024-09-13T14:21:52.883199+0000","last_unstale":"2024-09-13T14:25:07.330858+0000","last_undegraded":"2024-09-13T14:25:07.330858+0000","last_fullsized":"2024-09-13T14:25:07.330858+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:09.955395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239060+0000","last_change":"2024-09-13T14:24:17.240639+0000","last_active":"2024-09-13T14:25:07.239060+0000","last_peered":"2024-09-13T14:25:07.239060+0000","last_clean":"2024-09-13T14:25:07.239060+0000","last_became_active":"2024-09-13T14:24:17.239244+0000","last_became_peered":"2024-09-13T14:24:17.239244+0000","last_unstale":"2024-09-13T14:25:07.239060+0000","last_undegraded":"2024-09-13T14:25:07.239060+0000","last_fullsized":"2024-09-13T14:25:07.239060+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:50:52.952146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238809+0000","last_change":"2024-09-13T14:24:17.270423+0000","last_active":"2024-09-13T14:25:07.238809+0000","last_peered":"2024-09-13T14:25:07.238809+0000","last_clean":"2024-09-13T14:25:07.238809+0000","last_became_active":"2024-09-13T14:24:17.269949+0000","last_became_peered":"2024-09-13T14:24:17.269949+0000","last_unstale":"2024-09-13T14:25:07.238809+0000","last_undegraded":"2024-09-13T14:25:07.238809+0000","last_fullsized":"2024-09-13T14:25:07.238809+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:47:38.534443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241070+0000","last_change":"2024-09-13T14:22:03.172771+0000","last_active":"2024-09-13T14:25:07.241070+0000","last_peered":"2024-09-13T14:25:07.241070+0000","last_clean":"2024-09-13T14:25:07.241070+0000","last_became_active":"2024-09-13T14:21:52.884004+0000","last_became_peered":"2024-09-13T14:21:52.884004+0000","last_unstale":"2024-09-13T14:25:07.241070+0000","last_undegraded":"2024-09-13T14:25:07.241070+0000","last_fullsized":"2024-09-13T14:25:07.241070+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:20:21.593867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239701+0000","last_change":"2024-09-13T14:24:17.237493+0000","last_active":"2024-09-13T14:25:07.239701+0000","last_peered":"2024-09-13T14:25:07.239701+0000","last_clean":"2024-09-13T14:25:07.239701+0000","last_became_active":"2024-09-13T14:24:17.236513+0000","last_became_peered":"2024-09-13T14:24:17.236513+0000","last_unstale":"2024-09-13T14:25:07.239701+0000","last_undegraded":"2024-09-13T14:25:07.239701+0000","last_fullsized":"2024-09-13T14:25:07.239701+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:54:30.116051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:25:08.721247+0000","last_change":"2024-09-13T14:25:08.721247+0000","last_active":"2024-09-13T14:25:08.721247+0000","last_peered":"2024-09-13T14:25:08.721247+0000","last_clean":"2024-09-13T14:25:08.721247+0000","last_became_active":"2024-09-13T14:25:08.720536+0000","last_became_peered":"2024-09-13T14:25:08.720536+0000","last_unstale":"2024-09-13T14:25:08.721247+0000","last_undegraded":"2024-09-13T14:25:08.721247+0000","last_fullsized":"2024-09-13T14:25:08.721247+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:06:19.551355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":29,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251041+0000","last_change":"2024-09-13T14:22:03.177356+0000","last_active":"2024-09-13T14:25:07.251041+0000","last_peered":"2024-09-13T14:25:07.251041+0000","last_clean":"2024-09-13T14:25:07.251041+0000","last_became_active":"2024-09-13T14:21:52.861658+0000","last_became_peered":"2024-09-13T14:21:52.861658+0000","last_unstale":"2024-09-13T14:25:07.251041+0000","last_undegraded":"2024-09-13T14:25:07.251041+0000","last_fullsized":"2024-09-13T14:25:07.251041+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:10.200310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242107+0000","last_change":"2024-09-13T14:24:17.226365+0000","last_active":"2024-09-13T14:25:07.242107+0000","last_peered":"2024-09-13T14:25:07.242107+0000","last_clean":"2024-09-13T14:25:07.242107+0000","last_became_active":"2024-09-13T14:24:17.224828+0000","last_became_peered":"2024-09-13T14:24:17.224828+0000","last_unstale":"2024-09-13T14:25:07.242107+0000","last_undegraded":"2024-09-13T14:25:07.242107+0000","last_fullsized":"2024-09-13T14:25:07.242107+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:55:23.147922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.240475+0000","last_change":"2024-09-13T14:24:17.275739+0000","last_active":"2024-09-13T14:25:07.240475+0000","last_peered":"2024-09-13T14:25:07.240475+0000","last_clean":"2024-09-13T14:25:07.240475+0000","last_became_active":"2024-09-13T14:24:17.275258+0000","last_became_peered":"2024-09-13T14:24:17.275258+0000","last_unstale":"2024-09-13T14:25:07.240475+0000","last_undegraded":"2024-09-13T14:25:07.240475+0000","last_fullsized":"2024-09-13T14:25:07.240475+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:14:59.287041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251767+0000","last_change":"2024-09-13T14:24:17.227028+0000","last_active":"2024-09-13T14:25:07.251767+0000","last_peered":"2024-09-13T14:25:07.251767+0000","last_clean":"2024-09-13T14:25:07.251767+0000","last_became_active":"2024-09-13T14:24:17.226003+0000","last_became_peered":"2024-09-13T14:24:17.226003+0000","last_unstale":"2024-09-13T14:25:07.251767+0000","last_undegraded":"2024-09-13T14:25:07.251767+0000","last_fullsized":"2024-09-13T14:25:07.251767+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:35.841414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.240629+0000","last_change":"2024-09-13T14:22:03.171905+0000","last_active":"2024-09-13T14:25:07.240629+0000","last_peered":"2024-09-13T14:25:07.240629+0000","last_clean":"2024-09-13T14:25:07.240629+0000","last_became_active":"2024-09-13T14:21:52.884118+0000","last_became_peered":"2024-09-13T14:21:52.884118+0000","last_unstale":"2024-09-13T14:25:07.240629+0000","last_undegraded":"2024-09-13T14:25:07.240629+0000","last_fullsized":"2024-09-13T14:25:07.240629+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:26:17.407017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.237109+0000","last_change":"2024-09-13T14:24:17.277988+0000","last_active":"2024-09-13T14:25:07.237109+0000","last_peered":"2024-09-13T14:25:07.237109+0000","last_clean":"2024-09-13T14:25:07.237109+0000","last_became_active":"2024-09-13T14:24:17.277640+0000","last_became_peered":"2024-09-13T14:24:17.277640+0000","last_unstale":"2024-09-13T14:25:07.237109+0000","last_undegraded":"2024-09-13T14:25:07.237109+0000","last_fullsized":"2024-09-13T14:25:07.237109+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:50.707882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.240554+0000","last_change":"2024-09-13T14:24:17.237357+0000","last_active":"2024-09-13T14:25:07.240554+0000","last_peered":"2024-09-13T14:25:07.240554+0000","last_clean":"2024-09-13T14:25:07.240554+0000","last_became_active":"2024-09-13T14:24:17.235978+0000","last_became_peered":"2024-09-13T14:24:17.235978+0000","last_unstale":"2024-09-13T14:25:07.240554+0000","last_undegraded":"2024-09-13T14:25:07.240554+0000","last_fullsized":"2024-09-13T14:25:07.240554+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:36.835080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331606+0000","last_change":"2024-09-13T14:22:03.357990+0000","last_active":"2024-09-13T14:25:07.331606+0000","last_peered":"2024-09-13T14:25:07.331606+0000","last_clean":"2024-09-13T14:25:07.331606+0000","last_became_active":"2024-09-13T14:21:53.093638+0000","last_became_peered":"2024-09-13T14:21:53.093638+0000","last_unstale":"2024-09-13T14:25:07.331606+0000","last_undegraded":"2024-09-13T14:25:07.331606+0000","last_fullsized":"2024-09-13T14:25:07.331606+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:24.868073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238448+0000","last_change":"2024-09-13T14:24:17.413269+0000","last_active":"2024-09-13T14:25:07.238448+0000","last_peered":"2024-09-13T14:25:07.238448+0000","last_clean":"2024-09-13T14:25:07.238448+0000","last_became_active":"2024-09-13T14:24:17.412907+0000","last_became_peered":"2024-09-13T14:24:17.412907+0000","last_unstale":"2024-09-13T14:25:07.238448+0000","last_undegraded":"2024-09-13T14:25:07.238448+0000","last_fullsized":"2024-09-13T14:25:07.238448+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:05.270670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242898+0000","last_change":"2024-09-13T14:24:17.255947+0000","last_active":"2024-09-13T14:25:07.242898+0000","last_peered":"2024-09-13T14:25:07.242898+0000","last_clean":"2024-09-13T14:25:07.242898+0000","last_became_active":"2024-09-13T14:24:17.247909+0000","last_became_peered":"2024-09-13T14:24:17.247909+0000","last_unstale":"2024-09-13T14:25:07.242898+0000","last_undegraded":"2024-09-13T14:25:07.242898+0000","last_fullsized":"2024-09-13T14:25:07.242898+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:10.451346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.237952+0000","last_change":"2024-09-13T14:22:03.172028+0000","last_active":"2024-09-13T14:25:07.237952+0000","last_peered":"2024-09-13T14:25:07.237952+0000","last_clean":"2024-09-13T14:25:07.237952+0000","last_became_active":"2024-09-13T14:21:53.095058+0000","last_became_peered":"2024-09-13T14:21:53.095058+0000","last_unstale":"2024-09-13T14:25:07.237952+0000","last_undegraded":"2024-09-13T14:25:07.237952+0000","last_fullsized":"2024-09-13T14:25:07.237952+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:44.515568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080647399999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242247+0000","last_change":"2024-09-13T14:24:18.951583+0000","last_active":"2024-09-13T14:25:07.242247+0000","last_peered":"2024-09-13T14:25:07.242247+0000","last_clean":"2024-09-13T14:25:07.242247+0000","last_became_active":"2024-09-13T14:24:18.950822+0000","last_became_peered":"2024-09-13T14:24:18.950822+0000","last_unstale":"2024-09-13T14:25:07.242247+0000","last_undegraded":"2024-09-13T14:25:07.242247+0000","last_fullsized":"2024-09-13T14:25:07.242247+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:53:04.214093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":16,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238940+0000","last_change":"2024-09-13T14:24:17.231972+0000","last_active":"2024-09-13T14:25:07.238940+0000","last_peered":"2024-09-13T14:25:07.238940+0000","last_clean":"2024-09-13T14:25:07.238940+0000","last_became_active":"2024-09-13T14:24:17.231185+0000","last_became_peered":"2024-09-13T14:24:17.231185+0000","last_unstale":"2024-09-13T14:25:07.238940+0000","last_undegraded":"2024-09-13T14:25:07.238940+0000","last_fullsized":"2024-09-13T14:25:07.238940+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:16.680078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.245309+0000","last_change":"2024-09-13T14:22:03.178246+0000","last_active":"2024-09-13T14:25:07.245309+0000","last_peered":"2024-09-13T14:25:07.245309+0000","last_clean":"2024-09-13T14:25:07.245309+0000","last_became_active":"2024-09-13T14:21:52.861399+0000","last_became_peered":"2024-09-13T14:21:52.861399+0000","last_unstale":"2024-09-13T14:25:07.245309+0000","last_undegraded":"2024-09-13T14:25:07.245309+0000","last_fullsized":"2024-09-13T14:25:07.245309+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:51.432081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239217+0000","last_change":"2024-09-13T14:24:17.260443+0000","last_active":"2024-09-13T14:25:07.239217+0000","last_peered":"2024-09-13T14:25:07.239217+0000","last_clean":"2024-09-13T14:25:07.239217+0000","last_became_active":"2024-09-13T14:24:17.259887+0000","last_became_peered":"2024-09-13T14:24:17.259887+0000","last_unstale":"2024-09-13T14:25:07.239217+0000","last_undegraded":"2024-09-13T14:25:07.239217+0000","last_fullsized":"2024-09-13T14:25:07.239217+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:53.307078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:25:09.211515+0000","last_change":"2024-09-13T14:25:09.211515+0000","last_active":"2024-09-13T14:25:09.211515+0000","last_peered":"2024-09-13T14:25:09.211515+0000","last_clean":"2024-09-13T14:25:09.211515+0000","last_became_active":"2024-09-13T14:25:09.210780+0000","last_became_peered":"2024-09-13T14:25:09.210780+0000","last_unstale":"2024-09-13T14:25:09.211515+0000","last_undegraded":"2024-09-13T14:25:09.211515+0000","last_fullsized":"2024-09-13T14:25:09.211515+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:01.297911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"65'7","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239763+0000","last_change":"2024-09-13T14:24:17.235747+0000","last_active":"2024-09-13T14:25:07.239763+0000","last_peered":"2024-09-13T14:25:07.239763+0000","last_clean":"2024-09-13T14:25:07.239763+0000","last_became_active":"2024-09-13T14:24:17.231904+0000","last_became_peered":"2024-09-13T14:24:17.231904+0000","last_unstale":"2024-09-13T14:25:07.239763+0000","last_undegraded":"2024-09-13T14:25:07.239763+0000","last_fullsized":"2024-09-13T14:25:07.239763+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:34.362299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.237190+0000","last_change":"2024-09-13T14:24:17.276111+0000","last_active":"2024-09-13T14:25:07.237190+0000","last_peered":"2024-09-13T14:25:07.237190+0000","last_clean":"2024-09-13T14:25:07.237190+0000","last_became_active":"2024-09-13T14:24:17.275576+0000","last_became_peered":"2024-09-13T14:24:17.275576+0000","last_unstale":"2024-09-13T14:25:07.237190+0000","last_undegraded":"2024-09-13T14:25:07.237190+0000","last_fullsized":"2024-09-13T14:25:07.237190+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:45.859807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331241+0000","last_change":"2024-09-13T14:24:17.244991+0000","last_active":"2024-09-13T14:25:07.331241+0000","last_peered":"2024-09-13T14:25:07.331241+0000","last_clean":"2024-09-13T14:25:07.331241+0000","last_became_active":"2024-09-13T14:24:17.244107+0000","last_became_peered":"2024-09-13T14:24:17.244107+0000","last_unstale":"2024-09-13T14:25:07.331241+0000","last_undegraded":"2024-09-13T14:25:07.331241+0000","last_fullsized":"2024-09-13T14:25:07.331241+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:11:14.995141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.250824+0000","last_change":"2024-09-13T14:24:17.230464+0000","last_active":"2024-09-13T14:25:07.250824+0000","last_peered":"2024-09-13T14:25:07.250824+0000","last_clean":"2024-09-13T14:25:07.250824+0000","last_became_active":"2024-09-13T14:24:17.229956+0000","last_became_peered":"2024-09-13T14:24:17.229956+0000","last_unstale":"2024-09-13T14:25:07.250824+0000","last_undegraded":"2024-09-13T14:25:07.250824+0000","last_fullsized":"2024-09-13T14:25:07.250824+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:35:37.489846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241649+0000","last_change":"2024-09-13T14:24:18.613468+0000","last_active":"2024-09-13T14:25:07.241649+0000","last_peered":"2024-09-13T14:25:07.241649+0000","last_clean":"2024-09-13T14:25:07.241649+0000","last_became_active":"2024-09-13T14:24:18.612550+0000","last_became_peered":"2024-09-13T14:24:18.612550+0000","last_unstale":"2024-09-13T14:25:07.241649+0000","last_undegraded":"2024-09-13T14:25:07.241649+0000","last_fullsized":"2024-09-13T14:25:07.241649+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:42.550479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239751+0000","last_change":"2024-09-13T14:24:17.236823+0000","last_active":"2024-09-13T14:25:07.239751+0000","last_peered":"2024-09-13T14:25:07.239751+0000","last_clean":"2024-09-13T14:25:07.239751+0000","last_became_active":"2024-09-13T14:24:17.236174+0000","last_became_peered":"2024-09-13T14:24:17.236174+0000","last_unstale":"2024-09-13T14:25:07.239751+0000","last_undegraded":"2024-09-13T14:25:07.239751+0000","last_fullsized":"2024-09-13T14:25:07.239751+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:53.380050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:25:09.212576+0000","last_change":"2024-09-13T14:25:09.212576+0000","last_active":"2024-09-13T14:25:09.212576+0000","last_peered":"2024-09-13T14:25:09.212576+0000","last_clean":"2024-09-13T14:25:09.212576+0000","last_became_active":"2024-09-13T14:25:09.211610+0000","last_became_peered":"2024-09-13T14:25:09.211610+0000","last_unstale":"2024-09-13T14:25:09.212576+0000","last_undegraded":"2024-09-13T14:25:09.212576+0000","last_fullsized":"2024-09-13T14:25:09.212576+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:45.964193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.245089+0000","last_change":"2024-09-13T14:24:17.268884+0000","last_active":"2024-09-13T14:25:07.245089+0000","last_peered":"2024-09-13T14:25:07.245089+0000","last_clean":"2024-09-13T14:25:07.245089+0000","last_became_active":"2024-09-13T14:24:17.268250+0000","last_became_peered":"2024-09-13T14:24:17.268250+0000","last_unstale":"2024-09-13T14:25:07.245089+0000","last_undegraded":"2024-09-13T14:25:07.245089+0000","last_fullsized":"2024-09-13T14:25:07.245089+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:15:32.373469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242259+0000","last_change":"2024-09-13T14:24:17.250386+0000","last_active":"2024-09-13T14:25:07.242259+0000","last_peered":"2024-09-13T14:25:07.242259+0000","last_clean":"2024-09-13T14:25:07.242259+0000","last_became_active":"2024-09-13T14:24:17.248972+0000","last_became_peered":"2024-09-13T14:24:17.248972+0000","last_unstale":"2024-09-13T14:25:07.242259+0000","last_undegraded":"2024-09-13T14:25:07.242259+0000","last_fullsized":"2024-09-13T14:25:07.242259+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:32.448624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238740+0000","last_change":"2024-09-13T14:24:17.413237+0000","last_active":"2024-09-13T14:25:07.238740+0000","last_peered":"2024-09-13T14:25:07.238740+0000","last_clean":"2024-09-13T14:25:07.238740+0000","last_became_active":"2024-09-13T14:24:17.412906+0000","last_became_peered":"2024-09-13T14:24:17.412906+0000","last_unstale":"2024-09-13T14:25:07.238740+0000","last_undegraded":"2024-09-13T14:25:07.238740+0000","last_fullsized":"2024-09-13T14:25:07.238740+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:15.744172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.236952+0000","last_change":"2024-09-13T14:24:17.235517+0000","last_active":"2024-09-13T14:25:07.236952+0000","last_peered":"2024-09-13T14:25:07.236952+0000","last_clean":"2024-09-13T14:25:07.236952+0000","last_became_active":"2024-09-13T14:24:17.234813+0000","last_became_peered":"2024-09-13T14:24:17.234813+0000","last_unstale":"2024-09-13T14:25:07.236952+0000","last_undegraded":"2024-09-13T14:25:07.236952+0000","last_fullsized":"2024-09-13T14:25:07.236952+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:03.127643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242252+0000","last_change":"2024-09-13T14:24:18.584092+0000","last_active":"2024-09-13T14:25:07.242252+0000","last_peered":"2024-09-13T14:25:07.242252+0000","last_clean":"2024-09-13T14:25:07.242252+0000","last_became_active":"2024-09-13T14:24:18.583264+0000","last_became_peered":"2024-09-13T14:24:18.583264+0000","last_unstale":"2024-09-13T14:25:07.242252+0000","last_undegraded":"2024-09-13T14:25:07.242252+0000","last_fullsized":"2024-09-13T14:25:07.242252+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:05.370920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239850+0000","last_change":"2024-09-13T14:24:17.232228+0000","last_active":"2024-09-13T14:25:07.239850+0000","last_peered":"2024-09-13T14:25:07.239850+0000","last_clean":"2024-09-13T14:25:07.239850+0000","last_became_active":"2024-09-13T14:24:17.231457+0000","last_became_peered":"2024-09-13T14:24:17.231457+0000","last_unstale":"2024-09-13T14:25:07.239850+0000","last_undegraded":"2024-09-13T14:25:07.239850+0000","last_fullsized":"2024-09-13T14:25:07.239850+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:12.673393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241630+0000","last_change":"2024-09-13T14:24:17.278478+0000","last_active":"2024-09-13T14:25:07.241630+0000","last_peered":"2024-09-13T14:25:07.241630+0000","last_clean":"2024-09-13T14:25:07.241630+0000","last_became_active":"2024-09-13T14:24:17.278108+0000","last_became_peered":"2024-09-13T14:24:17.278108+0000","last_unstale":"2024-09-13T14:25:07.241630+0000","last_undegraded":"2024-09-13T14:25:07.241630+0000","last_fullsized":"2024-09-13T14:25:07.241630+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:53.787023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238572+0000","last_change":"2024-09-13T14:24:17.233488+0000","last_active":"2024-09-13T14:25:07.238572+0000","last_peered":"2024-09-13T14:25:07.238572+0000","last_clean":"2024-09-13T14:25:07.238572+0000","last_became_active":"2024-09-13T14:24:17.232403+0000","last_became_peered":"2024-09-13T14:24:17.232403+0000","last_unstale":"2024-09-13T14:25:07.238572+0000","last_undegraded":"2024-09-13T14:25:07.238572+0000","last_fullsized":"2024-09-13T14:25:07.238572+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:06.054933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239767+0000","last_change":"2024-09-13T14:24:17.273086+0000","last_active":"2024-09-13T14:25:07.239767+0000","last_peered":"2024-09-13T14:25:07.239767+0000","last_clean":"2024-09-13T14:25:07.239767+0000","last_became_active":"2024-09-13T14:24:17.272459+0000","last_became_peered":"2024-09-13T14:24:17.272459+0000","last_unstale":"2024-09-13T14:25:07.239767+0000","last_undegraded":"2024-09-13T14:25:07.239767+0000","last_fullsized":"2024-09-13T14:25:07.239767+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:34:24.756921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251225+0000","last_change":"2024-09-13T14:24:17.233611+0000","last_active":"2024-09-13T14:25:07.251225+0000","last_peered":"2024-09-13T14:25:07.251225+0000","last_clean":"2024-09-13T14:25:07.251225+0000","last_became_active":"2024-09-13T14:24:17.233075+0000","last_became_peered":"2024-09-13T14:24:17.233075+0000","last_unstale":"2024-09-13T14:25:07.251225+0000","last_undegraded":"2024-09-13T14:25:07.251225+0000","last_fullsized":"2024-09-13T14:25:07.251225+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:22.672021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239071+0000","last_change":"2024-09-13T14:24:17.263201+0000","last_active":"2024-09-13T14:25:07.239071+0000","last_peered":"2024-09-13T14:25:07.239071+0000","last_clean":"2024-09-13T14:25:07.239071+0000","last_became_active":"2024-09-13T14:24:17.262746+0000","last_became_peered":"2024-09-13T14:24:17.262746+0000","last_unstale":"2024-09-13T14:25:07.239071+0000","last_undegraded":"2024-09-13T14:25:07.239071+0000","last_fullsized":"2024-09-13T14:25:07.239071+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:57.696774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251656+0000","last_change":"2024-09-13T14:24:17.240313+0000","last_active":"2024-09-13T14:25:07.251656+0000","last_peered":"2024-09-13T14:25:07.251656+0000","last_clean":"2024-09-13T14:25:07.251656+0000","last_became_active":"2024-09-13T14:24:17.239555+0000","last_became_peered":"2024-09-13T14:24:17.239555+0000","last_unstale":"2024-09-13T14:25:07.251656+0000","last_undegraded":"2024-09-13T14:25:07.251656+0000","last_fullsized":"2024-09-13T14:25:07.251656+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:44.983674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241865+0000","last_change":"2024-09-13T14:24:18.951307+0000","last_active":"2024-09-13T14:25:07.241865+0000","last_peered":"2024-09-13T14:25:07.241865+0000","last_clean":"2024-09-13T14:25:07.241865+0000","last_became_active":"2024-09-13T14:24:18.950577+0000","last_became_peered":"2024-09-13T14:24:18.950577+0000","last_unstale":"2024-09-13T14:25:07.241865+0000","last_undegraded":"2024-09-13T14:25:07.241865+0000","last_fullsized":"2024-09-13T14:25:07.241865+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:30.356259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242594+0000","last_change":"2024-09-13T14:24:17.225321+0000","last_active":"2024-09-13T14:25:07.242594+0000","last_peered":"2024-09-13T14:25:07.242594+0000","last_clean":"2024-09-13T14:25:07.242594+0000","last_became_active":"2024-09-13T14:24:17.224552+0000","last_became_peered":"2024-09-13T14:24:17.224552+0000","last_unstale":"2024-09-13T14:25:07.242594+0000","last_undegraded":"2024-09-13T14:25:07.242594+0000","last_fullsized":"2024-09-13T14:25:07.242594+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:20:14.242431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238724+0000","last_change":"2024-09-13T14:24:17.271339+0000","last_active":"2024-09-13T14:25:07.238724+0000","last_peered":"2024-09-13T14:25:07.238724+0000","last_clean":"2024-09-13T14:25:07.238724+0000","last_became_active":"2024-09-13T14:24:17.270640+0000","last_became_peered":"2024-09-13T14:24:17.270640+0000","last_unstale":"2024-09-13T14:25:07.238724+0000","last_undegraded":"2024-09-13T14:25:07.238724+0000","last_fullsized":"2024-09-13T14:25:07.238724+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:42:09.267241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239669+0000","last_change":"2024-09-13T14:24:17.240983+0000","last_active":"2024-09-13T14:25:07.239669+0000","last_peered":"2024-09-13T14:25:07.239669+0000","last_clean":"2024-09-13T14:25:07.239669+0000","last_became_active":"2024-09-13T14:24:17.239412+0000","last_became_peered":"2024-09-13T14:24:17.239412+0000","last_unstale":"2024-09-13T14:25:07.239669+0000","last_undegraded":"2024-09-13T14:25:07.239669+0000","last_fullsized":"2024-09-13T14:25:07.239669+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:50:49.775014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":47,"num_read_kb":49,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397580,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298632,"kb_used_data":1276,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446520,"statfs":{"total":95995035648,"available":95689236480,"internally_reserved":0,"allocated":1306624,"data_stored":895899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:15 2024","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.83099999999999996,"15min":0.83499999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.2130000000000001,"5min":1.268,"15min":1.268},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.878,"15min":0.88700000000000001},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":0.97999999999999998}]},{"osd":1,"last update":"Fri Sep 13 14:25:15 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.85799999999999998,"15min":0.88600000000000001},"min":{"1min":0.63200000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.0580000000000001,"5min":1.278,"15min":1.278},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.90900000000000003,"15min":0.90700000000000003},"min":{"1min":0.56799999999999995,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.123,"5min":1.234,"15min":1.234},"last":0.83099999999999996}]},{"osd":2,"last update":"Fri Sep 13 14:25:15 2024","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.94299999999999995,"15min":0.96499999999999997},"min":{"1min":0.55200000000000005,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.5189999999999999,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.875},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.002,"15min":0.98999999999999999},"min":{"1min":0.74399999999999999,"5min":0.497,"15min":0.497},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.86199999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:25:15 2024","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.95899999999999996,"15min":0.95299999999999996},"min":{"1min":0.64800000000000002,"5min":0.44,"15min":0.44},"max":{"1min":1.3120000000000001,"5min":1.375,"15min":1.375},"last":0.751},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.93500000000000005,"15min":0.94299999999999995},"min":{"1min":0.64700000000000002,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.282,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.78300000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:25:15 2024","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.879,"15min":0.89100000000000001},"min":{"1min":0.58599999999999997,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.1259999999999999,"5min":1.337,"15min":1.337},"last":0.626},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.97799999999999998,"15min":0.97799999999999998},"min":{"1min":0.433,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.56699999999999995}]},{"osd":5,"last update":"Fri Sep 13 14:25:15 2024","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.93600000000000005,"15min":0.92000000000000004},"min":{"1min":0.48399999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.244,"5min":1.341,"15min":1.341},"last":1.01},{"interface":"front","average":{"1min":1.0309999999999999,"5min":0.995,"15min":0.98799999999999999},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.3540000000000001,"5min":1.401,"15min":1.401},"last":0.73099999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:25:15 2024","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.0369999999999999,"15min":1.0109999999999999},"min":{"1min":0.70799999999999996,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.2809999999999999,"5min":1.401,"15min":1.401},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.0669999999999999,"15min":1.0680000000000001},"min":{"1min":0.68600000000000005,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.3720000000000001,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":0.81200000000000006}]}]},{"osd":6,"up_from":43,"seq":184683593815,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1260,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1290240,"data_stored":895142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.88600000000000001,"15min":0.87},"min":{"1min":0.52400000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.298,"5min":1.298,"15min":1.298},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.85699999999999998,"15min":0.83299999999999996},"min":{"1min":0.64000000000000001,"5min":0.51100000000000001,"15min":0.49299999999999999},"max":{"1min":1.232,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.97199999999999998}]},{"osd":1,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.88700000000000001,"15min":0.84199999999999997},"min":{"1min":0.61599999999999999,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.169,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":1.054},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.90500000000000003,"15min":0.873},"min":{"1min":0.627,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.82399999999999995}]},{"osd":2,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.91200000000000003,"15min":0.88600000000000001},"min":{"1min":0.59399999999999997,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.3580000000000001,"5min":1.3759999999999999,"15min":1.419},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.94099999999999995,"15min":0.89100000000000001},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.55400000000000005},"max":{"1min":1.228,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":1.228}]},{"osd":3,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.91600000000000004,"15min":0.874},"min":{"1min":0.55100000000000005,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.264,"5min":1.659,"15min":1.659},"last":0.93300000000000005},{"interface":"front","average":{"1min":1.004,"5min":0.94799999999999995,"15min":0.90800000000000003},"min":{"1min":0.81699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.252,"5min":1.3420000000000001,"15min":1.5169999999999999},"last":1.0249999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.86399999999999999,"15min":0.78700000000000003},"min":{"1min":0.64700000000000002,"5min":0.313,"15min":0.313},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":1.091},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.91300000000000003,"15min":0.79700000000000004},"min":{"1min":0.63700000000000001,"5min":0.377,"15min":0.317},"max":{"1min":1.28,"5min":1.423,"15min":1.423},"last":1.153}]},{"osd":5,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":0.96299999999999997,"15min":0.86899999999999999},"min":{"1min":0.70799999999999996,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.3300000000000001,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":1.1819999999999999},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.996,"15min":0.876},"min":{"1min":0.72599999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.3069999999999999,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":1.2030000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:25:16 2024","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.028,"15min":0.998},"min":{"1min":0.71999999999999997,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.3700000000000001,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":1.121},{"interface":"front","average":{"1min":1.157,"5min":1.091,"15min":1.0329999999999999},"min":{"1min":0.96099999999999997,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.4199999999999999,"5min":1.464,"15min":1.464},"last":1.278}]}]},{"osd":5,"up_from":37,"seq":158913790049,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":692224,"data_stored":304107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.81899999999999995,"15min":0.81299999999999994},"min":{"1min":0.58399999999999996,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.222,"5min":1.323,"15min":1.323},"last":1.234},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.91500000000000004,"15min":0.84799999999999998},"min":{"1min":0.52800000000000002,"5min":0.505,"15min":0.505},"max":{"1min":1.204,"5min":1.45,"15min":1.45},"last":1.0209999999999999}]},{"osd":1,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.876,"15min":0.83999999999999997},"min":{"1min":0.57099999999999995,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.91800000000000004,"15min":0.83599999999999997},"min":{"1min":0.60899999999999999,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.161,"5min":2.0459999999999998,"15min":2.0459999999999998},"last":0.78100000000000003}]},{"osd":2,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.90400000000000003,"15min":0.82899999999999996},"min":{"1min":0.67400000000000004,"5min":0.502,"15min":0.48399999999999999},"max":{"1min":1.236,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.92200000000000004,"15min":0.875},"min":{"1min":0.76800000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.2290000000000001,"5min":1.3400000000000001,"15min":1.3400000000000001},"last":1.1830000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.95999999999999996,"15min":0.88500000000000001},"min":{"1min":0.745,"5min":0.497,"15min":0.497},"max":{"1min":1.1679999999999999,"5min":1.573,"15min":1.573},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.995,"15min":0.89300000000000002},"min":{"1min":0.70799999999999996,"5min":0.52400000000000002,"15min":0.51400000000000001},"max":{"1min":1.288,"5min":2.246,"15min":2.246},"last":1.294}]},{"osd":4,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.878,"15min":0.71499999999999997},"min":{"1min":0.621,"5min":0.39200000000000002,"15min":0.29399999999999998},"max":{"1min":1.3169999999999999,"5min":1.359,"15min":1.359},"last":1.2110000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.95499999999999996,"15min":0.77900000000000003},"min":{"1min":0.70399999999999996,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":1.3029999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":1.0620000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":0.95999999999999996,"15min":0.85599999999999998},"min":{"1min":0.68400000000000005,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.272,"5min":1.488,"15min":1.488},"last":1.151},{"interface":"front","average":{"1min":1.048,"5min":1.0149999999999999,"15min":0.94499999999999995},"min":{"1min":0.81399999999999995,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.2609999999999999,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.125}]},{"osd":7,"last update":"Fri Sep 13 14:25:10 2024","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.036,"15min":1.016},"min":{"1min":0.81699999999999995,"5min":0.376,"15min":0.376},"max":{"1min":1.2569999999999999,"5min":1.8080000000000001,"15min":1.8080000000000001},"last":1.258},{"interface":"front","average":{"1min":1.052,"5min":1.016,"15min":0.97799999999999998},"min":{"1min":0.68400000000000005,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.1950000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":1.341}]}]},{"osd":4,"up_from":31,"seq":133143986284,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":305461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.88600000000000001,"15min":0.82499999999999996},"min":{"1min":0.61799999999999999,"5min":0.52800000000000002,"15min":0.52300000000000002},"max":{"1min":1.2050000000000001,"5min":1.224,"15min":1.224},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.92300000000000004,"15min":0.85799999999999998},"min":{"1min":0.54900000000000004,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.2410000000000001,"5min":1.53,"15min":1.53},"last":0.75}]},{"osd":1,"last update":"Fri Sep 13 14:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.90500000000000003,"15min":0.82299999999999995},"min":{"1min":0.63900000000000001,"5min":0.60299999999999998,"15min":0.56499999999999995},"max":{"1min":1.1579999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":1.5960000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.90200000000000002,"15min":0.85199999999999998},"min":{"1min":0.73599999999999999,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.202,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":1.28}]},{"osd":2,"last update":"Fri Sep 13 14:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.92300000000000004,"15min":0.88700000000000001},"min":{"1min":0.72099999999999997,"5min":0.56699999999999995,"15min":0.52300000000000002},"max":{"1min":1.24,"5min":1.3200000000000001,"15min":1.3939999999999999},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.91500000000000004,"15min":0.84799999999999998},"min":{"1min":0.68999999999999995,"5min":0.59399999999999997,"15min":0.56799999999999995},"max":{"1min":1.1279999999999999,"5min":1.728,"15min":1.728},"last":1.1850000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.96999999999999997,"15min":0.88700000000000001},"min":{"1min":0.60499999999999998,"5min":0.59299999999999997,"15min":0.57199999999999995},"max":{"1min":1.264,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":1.1120000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.97299999999999998,"15min":0.88100000000000001},"min":{"1min":0.627,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":1.238}]},{"osd":5,"last update":"Fri Sep 13 14:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.88,"15min":0.78500000000000003},"min":{"1min":0.67700000000000005,"5min":0.44,"15min":0.38800000000000001},"max":{"1min":1.135,"5min":1.216,"15min":1.238},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.002,"5min":0.96799999999999997,"15min":0.79100000000000004},"min":{"1min":0.66500000000000004,"5min":0.46600000000000003,"15min":0.37},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.85799999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.92000000000000004,"15min":0.82199999999999995},"min":{"1min":0.621,"5min":0.44500000000000001,"15min":0.38200000000000001},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96199999999999997,"15min":0.86399999999999999},"min":{"1min":0.65000000000000002,"5min":0.44600000000000001,"15min":0.33900000000000002},"max":{"1min":1.2370000000000001,"5min":1.431,"15min":1.431},"last":0.89500000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:25:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.03,"15min":1.0620000000000001},"min":{"1min":0.82099999999999995,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.3089999999999999,"5min":1.411,"15min":1.411},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1040000000000001,"15min":1.1240000000000001},"min":{"1min":0.80300000000000005,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.371,"5min":1.504,"15min":1.504},"last":0.92500000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149815,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298028,"kb_used_data":672,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447124,"statfs":{"total":95995035648,"available":95689854976,"internally_reserved":0,"allocated":688128,"data_stored":303853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:54 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.64100000000000001,"15min":0.65400000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.171,"5min":1.1919999999999999,"15min":1.4730000000000001},"last":0.312},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.72599999999999998,"15min":0.69899999999999995},"min":{"1min":0.47699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.093,"5min":1.3,"15min":1.3},"last":0.44900000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:25:54 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.67300000000000004,"15min":0.67000000000000004},"min":{"1min":0.40600000000000003,"5min":0.316,"15min":0.316},"max":{"1min":1.0209999999999999,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.77600000000000002,"15min":0.751},"min":{"1min":0.46500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.1259999999999999,"5min":1.206,"15min":1.2070000000000001},"last":0.39200000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:25:54 2024","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.83499999999999996,"15min":0.78600000000000003},"min":{"1min":0.44500000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.754,"5min":0.76400000000000001,"15min":0.751},"min":{"1min":0.51500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.0660000000000001,"5min":1.2310000000000001,"15min":1.2310000000000001},"last":0.32800000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:25:43 2024","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.91200000000000003,"15min":0.88600000000000001},"min":{"1min":0.55700000000000005,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.98099999999999998,"15min":0.92700000000000005},"min":{"1min":0.65500000000000003,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.4259999999999999,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.83199999999999996}]},{"osd":5,"last update":"Fri Sep 13 14:25:30 2024","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.0309999999999999,"15min":0.98199999999999998},"min":{"1min":0.81899999999999995,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.5349999999999999,"5min":1.5349999999999999,"15min":1.5349999999999999},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.94499999999999995,"15min":0.91500000000000004},"min":{"1min":0.623,"5min":0.434,"15min":0.434},"max":{"1min":1.452,"5min":1.452,"15min":1.7569999999999999},"last":0.624}]},{"osd":6,"last update":"Fri Sep 13 14:25:27 2024","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.0640000000000001,"15min":1.0700000000000001},"min":{"1min":0.79700000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.486,"5min":1.514,"15min":1.514},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.034,"5min":1.008,"15min":1.02},"min":{"1min":0.63700000000000001,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.474,"5min":1.6439999999999999,"15min":1.6439999999999999},"last":0.63800000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:25:17 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.0209999999999999,"15min":1.0429999999999999},"min":{"1min":0.72699999999999998,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.1399999999999999,"15min":1.2030000000000001},"min":{"1min":0.81999999999999995,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.587,"5min":2.2349999999999999,"15min":2.2349999999999999},"last":0.878}]}]},{"osd":2,"up_from":18,"seq":77309411458,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298028,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447124,"statfs":{"total":95995035648,"available":95689854976,"internally_reserved":0,"allocated":692224,"data_stored":304367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:50 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65900000000000003,"15min":0.63},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.0429999999999999,"5min":1.228,"15min":1.3069999999999999},"last":0.621},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.71099999999999997,"15min":0.65500000000000003},"min":{"1min":0.378,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.29,"5min":1.29,"15min":1.29},"last":0.84299999999999997}]},{"osd":1,"last update":"Fri Sep 13 14:25:50 2024","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.65500000000000003,"15min":0.627},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.92300000000000004,"5min":0.97699999999999998,"15min":1.014},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.77300000000000002,"15min":0.70499999999999996},"min":{"1min":0.39700000000000002,"5min":0.373,"15min":0.373},"max":{"1min":1.1970000000000001,"5min":1.375,"15min":1.375},"last":0.749}]},{"osd":3,"last update":"Fri Sep 13 14:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.81399999999999995,"15min":0.80800000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.034,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":0.92300000000000004},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.75600000000000001,"15min":0.73599999999999999},"min":{"1min":0.42099999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":0.72199999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:25:36 2024","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.98699999999999999,"15min":0.91900000000000004},"min":{"1min":0.67400000000000004,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.754,"5min":1.754,"15min":1.881},"last":1.014},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.98099999999999998,"15min":0.94199999999999995},"min":{"1min":0.745,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.2629999999999999,"5min":1.498,"15min":1.498},"last":1.121}]},{"osd":5,"last update":"Fri Sep 13 14:25:36 2024","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.004,"15min":0.995},"min":{"1min":0.55900000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.23,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0009999999999999,"15min":0.94899999999999995},"min":{"1min":0.67300000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.327,"5min":1.363,"15min":1.446},"last":1.161}]},{"osd":6,"last update":"Fri Sep 13 14:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.93700000000000006,"15min":0.96499999999999997},"min":{"1min":0.79600000000000004,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":1.915},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.034,"5min":1.0429999999999999,"15min":1.0409999999999999},"min":{"1min":0.67200000000000004,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.6659999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.0680000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:25:22 2024","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.093,"15min":1.125},"min":{"1min":0.76600000000000001,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.2450000000000001,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":1.2450000000000001},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.1040000000000001,"15min":1.1240000000000001},"min":{"1min":0.751,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.5740000000000001,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":1.222}]}]},{"osd":1,"up_from":13,"seq":55834574988,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298020,"kb_used_data":668,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447132,"statfs":{"total":95995035648,"available":95689863168,"internally_reserved":0,"allocated":684032,"data_stored":303467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:26:03 2024","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.67300000000000004,"15min":0.60899999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.33200000000000002},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70699999999999996,"15min":0.625},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.4330000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.88500000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:25:52 2024","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.71499999999999997,"15min":0.66200000000000003},"min":{"1min":0.35099999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.016,"5min":1.0209999999999999,"15min":1.1359999999999999},"last":0.627},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.77300000000000002,"15min":0.70799999999999996},"min":{"1min":0.45500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.1859999999999999,"5min":1.194,"15min":1.194},"last":0.81000000000000005}]},{"osd":3,"last update":"Fri Sep 13 14:25:52 2024","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.75600000000000001,"15min":0.751},"min":{"1min":0.41099999999999998,"5min":0.31,"15min":0.31},"max":{"1min":1.0429999999999999,"5min":1.071,"15min":1.2110000000000001},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.86199999999999999,"15min":0.80500000000000005},"min":{"1min":0.49399999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.2170000000000001,"5min":1.355,"15min":1.355},"last":0.94499999999999995}]},{"osd":4,"last update":"Fri Sep 13 14:25:38 2024","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0089999999999999,"15min":0.94399999999999995},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.4890000000000001,"5min":1.647,"15min":1.647},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.002,"5min":0.95599999999999996,"15min":0.88700000000000001},"min":{"1min":0.52300000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.3,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":1.0009999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.98899999999999999,"15min":0.94899999999999995},"min":{"1min":0.73399999999999999,"5min":0.72199999999999998,"15min":0.64500000000000002},"max":{"1min":1.3420000000000001,"5min":1.9339999999999999,"15min":1.9339999999999999},"last":0.78700000000000003},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.024,"15min":0.96099999999999997},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.347,"5min":1.7929999999999999,"15min":1.7929999999999999},"last":0.97299999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.05,"15min":1.0489999999999999},"min":{"1min":0.63600000000000001,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.3280000000000001,"5min":1.458,"15min":1.458},"last":1.173},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.018,"15min":0.98599999999999999},"min":{"1min":0.68600000000000005,"5min":0.624,"15min":0.624},"max":{"1min":1.5549999999999999,"5min":1.716,"15min":1.716},"last":1.0720000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:25:15 2024","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.048,"15min":1.0549999999999999},"min":{"1min":0.63900000000000001,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.4430000000000001,"5min":1.516,"15min":1.516},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.147,"15min":1.153},"min":{"1min":0.82599999999999996,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":1.3899999999999999,"5min":1.8919999999999999,"15min":1.8919999999999999},"last":1.111}]}]},{"osd":0,"up_from":8,"seq":34359738518,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298612,"kb_used_data":1256,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446540,"statfs":{"total":95995035648,"available":95689256960,"internally_reserved":0,"allocated":1286144,"data_stored":894723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65900000000000003,"15min":0.60099999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.30499999999999999},"max":{"1min":1.087,"5min":1.323,"15min":1.323},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.77500000000000002,"15min":0.67300000000000004},"min":{"1min":0.441,"5min":0.307,"15min":0.26700000000000002},"max":{"1min":1.2070000000000001,"5min":1.448,"15min":1.448},"last":0.77800000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.76300000000000001,"15min":0.68799999999999994},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.35799999999999998},"max":{"1min":1.0860000000000001,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.746,"5min":0.69199999999999995,"15min":0.64400000000000002},"min":{"1min":0.35899999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":0.82999999999999996}]},{"osd":3,"last update":"Fri Sep 13 14:25:54 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.85599999999999998,"15min":0.80400000000000005},"min":{"1min":0.52800000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":0.96399999999999997},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.78400000000000003,"15min":0.751},"min":{"1min":0.42099999999999999,"5min":0.376,"15min":0.376},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.996}]},{"osd":4,"last update":"Fri Sep 13 14:25:32 2024","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.92400000000000004,"15min":0.85699999999999998},"min":{"1min":0.76500000000000001,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.1759999999999999,"5min":1.931,"15min":1.931},"last":1.0249999999999999},{"interface":"front","average":{"1min":1.079,"5min":1.012,"15min":0.94799999999999995},"min":{"1min":0.83599999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.3640000000000001,"5min":1.431,"15min":1.431},"last":1.0580000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:25:30 2024","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.97999999999999998,"15min":0.92900000000000005},"min":{"1min":0.85099999999999998,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.4770000000000001,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":1.1799999999999999},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.038,"15min":1.0209999999999999},"min":{"1min":0.76500000000000001,"5min":0.62,"15min":0.62},"max":{"1min":1.4450000000000001,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":1.115}]},{"osd":6,"last update":"Fri Sep 13 14:25:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":0.95999999999999996,"15min":0.93999999999999995},"min":{"1min":0.82799999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.3109999999999999,"5min":1.45,"15min":1.45},"last":0.93400000000000005},{"interface":"front","average":{"1min":1.095,"5min":1.032,"15min":1.0089999999999999},"min":{"1min":0.83699999999999997,"5min":0.623,"15min":0.623},"max":{"1min":1.405,"5min":2.2040000000000002,"15min":2.2040000000000002},"last":1.091}]},{"osd":7,"last update":"Fri Sep 13 14:25:19 2024","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.0880000000000001,"15min":1.0620000000000001},"min":{"1min":0.877,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.355,"5min":2.1200000000000001,"15min":2.1200000000000001},"last":1.131},{"interface":"front","average":{"1min":1.2,"5min":1.125,"15min":1.1259999999999999},"min":{"1min":0.81299999999999994,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.522,"5min":1.657,"15min":1.657},"last":1.155}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:26:10.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:26:10.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:10 smithi177 bash[24494]: cluster 2024-09-13T14:26:09.158501+0000 mgr.y (mgr.24287) 190 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:10.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:10 smithi177 bash[24494]: audit 2024-09-13T14:26:09.874182+0000 mgr.y (mgr.24287) 191 : audit [DBG] from='client.14718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:26:12.403 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:12 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:12] "GET /metrics HTTP/1.1" 200 239954 "" "Prometheus/2.43.0" 2024-09-13T14:26:12.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:12 smithi099 bash[22656]: cluster 2024-09-13T14:26:11.159136+0000 mgr.y (mgr.24287) 192 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:12.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:12 smithi099 bash[32082]: cluster 2024-09-13T14:26:11.159136+0000 mgr.y (mgr.24287) 192 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:12.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:12 smithi177 bash[24494]: cluster 2024-09-13T14:26:11.159136+0000 mgr.y (mgr.24287) 192 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:14.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:14 smithi099 bash[22656]: cluster 2024-09-13T14:26:13.160169+0000 mgr.y (mgr.24287) 193 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:14.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:14 smithi099 bash[32082]: cluster 2024-09-13T14:26:13.160169+0000 mgr.y (mgr.24287) 193 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:14.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:14 smithi177 bash[24494]: cluster 2024-09-13T14:26:13.160169+0000 mgr.y (mgr.24287) 193 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:15.490 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:15.855 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:26:15 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:26:16.803 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:16 smithi099 bash[22656]: cluster 2024-09-13T14:26:15.160784+0000 mgr.y (mgr.24287) 194 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:16.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:16 smithi099 bash[32082]: cluster 2024-09-13T14:26:15.160784+0000 mgr.y (mgr.24287) 194 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:16.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:16 smithi177 bash[24494]: cluster 2024-09-13T14:26:15.160784+0000 mgr.y (mgr.24287) 194 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:18.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:18 smithi099 bash[32082]: cluster 2024-09-13T14:26:17.161411+0000 mgr.y (mgr.24287) 195 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:18.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:18 smithi099 bash[22656]: cluster 2024-09-13T14:26:17.161411+0000 mgr.y (mgr.24287) 195 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:18.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:18 smithi177 bash[24494]: cluster 2024-09-13T14:26:17.161411+0000 mgr.y (mgr.24287) 195 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:19.183 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:26:19.184 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:26:19.991 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":174,"stamp":"2024-09-13T14:26:19.161709+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":173,"num_read_kb":280,"num_write":160,"num_write_kb":1643,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127,"ondisk_log_size":127,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386020,"kb_used_data":7176,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575196,"statfs":{"total":767960285184,"available":765517000704,"internally_reserved":0,"allocated":7348224,"data_stored":4207019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004695"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.243044+0000","last_change":"2024-09-13T14:24:18.584431+0000","last_active":"2024-09-13T14:25:07.243044+0000","last_peered":"2024-09-13T14:25:07.243044+0000","last_clean":"2024-09-13T14:25:07.243044+0000","last_became_active":"2024-09-13T14:24:18.584135+0000","last_became_peered":"2024-09-13T14:24:18.584135+0000","last_unstale":"2024-09-13T14:25:07.243044+0000","last_undegraded":"2024-09-13T14:25:07.243044+0000","last_fullsized":"2024-09-13T14:25:07.243044+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:43.909934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331230+0000","last_change":"2024-09-13T14:24:17.242167+0000","last_active":"2024-09-13T14:25:07.331230+0000","last_peered":"2024-09-13T14:25:07.331230+0000","last_clean":"2024-09-13T14:25:07.331230+0000","last_became_active":"2024-09-13T14:24:17.241529+0000","last_became_peered":"2024-09-13T14:24:17.241529+0000","last_unstale":"2024-09-13T14:25:07.331230+0000","last_undegraded":"2024-09-13T14:25:07.331230+0000","last_fullsized":"2024-09-13T14:25:07.331230+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:46:10.269246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251546+0000","last_change":"2024-09-13T14:24:17.417723+0000","last_active":"2024-09-13T14:25:07.251546+0000","last_peered":"2024-09-13T14:25:07.251546+0000","last_clean":"2024-09-13T14:25:07.251546+0000","last_became_active":"2024-09-13T14:24:17.416614+0000","last_became_peered":"2024-09-13T14:24:17.416614+0000","last_unstale":"2024-09-13T14:25:07.251546+0000","last_undegraded":"2024-09-13T14:25:07.251546+0000","last_fullsized":"2024-09-13T14:25:07.251546+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:19.028275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238242+0000","last_change":"2024-09-13T14:24:17.233597+0000","last_active":"2024-09-13T14:25:07.238242+0000","last_peered":"2024-09-13T14:25:07.238242+0000","last_clean":"2024-09-13T14:25:07.238242+0000","last_became_active":"2024-09-13T14:24:17.231997+0000","last_became_peered":"2024-09-13T14:24:17.231997+0000","last_unstale":"2024-09-13T14:25:07.238242+0000","last_undegraded":"2024-09-13T14:25:07.238242+0000","last_fullsized":"2024-09-13T14:25:07.238242+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:37:44.732444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238865+0000","last_change":"2024-09-13T14:24:17.417310+0000","last_active":"2024-09-13T14:25:07.238865+0000","last_peered":"2024-09-13T14:25:07.238865+0000","last_clean":"2024-09-13T14:25:07.238865+0000","last_became_active":"2024-09-13T14:24:17.416984+0000","last_became_peered":"2024-09-13T14:24:17.416984+0000","last_unstale":"2024-09-13T14:25:07.238865+0000","last_undegraded":"2024-09-13T14:25:07.238865+0000","last_fullsized":"2024-09-13T14:25:07.238865+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:22.686718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331004+0000","last_change":"2024-09-13T14:24:17.241349+0000","last_active":"2024-09-13T14:25:07.331004+0000","last_peered":"2024-09-13T14:25:07.331004+0000","last_clean":"2024-09-13T14:25:07.331004+0000","last_became_active":"2024-09-13T14:24:17.240753+0000","last_became_peered":"2024-09-13T14:24:17.240753+0000","last_unstale":"2024-09-13T14:25:07.331004+0000","last_undegraded":"2024-09-13T14:25:07.331004+0000","last_fullsized":"2024-09-13T14:25:07.331004+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:34:11.369561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242757+0000","last_change":"2024-09-13T14:24:18.583287+0000","last_active":"2024-09-13T14:25:07.242757+0000","last_peered":"2024-09-13T14:25:07.242757+0000","last_clean":"2024-09-13T14:25:07.242757+0000","last_became_active":"2024-09-13T14:24:18.582766+0000","last_became_peered":"2024-09-13T14:24:18.582766+0000","last_unstale":"2024-09-13T14:25:07.242757+0000","last_undegraded":"2024-09-13T14:25:07.242757+0000","last_fullsized":"2024-09-13T14:25:07.242757+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:40.011693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.245764+0000","last_change":"2024-09-13T14:24:17.231522+0000","last_active":"2024-09-13T14:25:07.245764+0000","last_peered":"2024-09-13T14:25:07.245764+0000","last_clean":"2024-09-13T14:25:07.245764+0000","last_became_active":"2024-09-13T14:24:17.230661+0000","last_became_peered":"2024-09-13T14:24:17.230661+0000","last_unstale":"2024-09-13T14:25:07.245764+0000","last_undegraded":"2024-09-13T14:25:07.245764+0000","last_fullsized":"2024-09-13T14:25:07.245764+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:36.292481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331690+0000","last_change":"2024-09-13T14:24:17.245589+0000","last_active":"2024-09-13T14:25:07.331690+0000","last_peered":"2024-09-13T14:25:07.331690+0000","last_clean":"2024-09-13T14:25:07.331690+0000","last_became_active":"2024-09-13T14:24:17.244773+0000","last_became_peered":"2024-09-13T14:24:17.244773+0000","last_unstale":"2024-09-13T14:25:07.331690+0000","last_undegraded":"2024-09-13T14:25:07.331690+0000","last_fullsized":"2024-09-13T14:25:07.331690+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:17:39.537514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.250718+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:25:07.250718+0000","last_peered":"2024-09-13T14:25:07.250718+0000","last_clean":"2024-09-13T14:25:07.250718+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:25:07.250718+0000","last_undegraded":"2024-09-13T14:25:07.250718+0000","last_fullsized":"2024-09-13T14:25:07.250718+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331643+0000","last_change":"2024-09-13T14:24:17.243027+0000","last_active":"2024-09-13T14:25:07.331643+0000","last_peered":"2024-09-13T14:25:07.331643+0000","last_clean":"2024-09-13T14:25:07.331643+0000","last_became_active":"2024-09-13T14:24:17.242540+0000","last_became_peered":"2024-09-13T14:24:17.242540+0000","last_unstale":"2024-09-13T14:25:07.331643+0000","last_undegraded":"2024-09-13T14:25:07.331643+0000","last_fullsized":"2024-09-13T14:25:07.331643+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:23:48.038754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238753+0000","last_change":"2024-09-13T14:24:17.271552+0000","last_active":"2024-09-13T14:25:07.238753+0000","last_peered":"2024-09-13T14:25:07.238753+0000","last_clean":"2024-09-13T14:25:07.238753+0000","last_became_active":"2024-09-13T14:24:17.271089+0000","last_became_peered":"2024-09-13T14:24:17.271089+0000","last_unstale":"2024-09-13T14:25:07.238753+0000","last_undegraded":"2024-09-13T14:25:07.238753+0000","last_fullsized":"2024-09-13T14:25:07.238753+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:24:46.046603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331540+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:25:07.331540+0000","last_peered":"2024-09-13T14:25:07.331540+0000","last_clean":"2024-09-13T14:25:07.331540+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:25:07.331540+0000","last_undegraded":"2024-09-13T14:25:07.331540+0000","last_fullsized":"2024-09-13T14:25:07.331540+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.244063+0000","last_change":"2024-09-13T14:24:17.414918+0000","last_active":"2024-09-13T14:25:07.244063+0000","last_peered":"2024-09-13T14:25:07.244063+0000","last_clean":"2024-09-13T14:25:07.244063+0000","last_became_active":"2024-09-13T14:24:17.414020+0000","last_became_peered":"2024-09-13T14:24:17.414020+0000","last_unstale":"2024-09-13T14:25:07.244063+0000","last_undegraded":"2024-09-13T14:25:07.244063+0000","last_fullsized":"2024-09-13T14:25:07.244063+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:51.567074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251335+0000","last_change":"2024-09-13T14:24:17.227167+0000","last_active":"2024-09-13T14:25:07.251335+0000","last_peered":"2024-09-13T14:25:07.251335+0000","last_clean":"2024-09-13T14:25:07.251335+0000","last_became_active":"2024-09-13T14:24:17.226643+0000","last_became_peered":"2024-09-13T14:24:17.226643+0000","last_unstale":"2024-09-13T14:25:07.251335+0000","last_undegraded":"2024-09-13T14:25:07.251335+0000","last_fullsized":"2024-09-13T14:25:07.251335+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:43:23.833520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238384+0000","last_change":"2024-09-13T14:24:17.271572+0000","last_active":"2024-09-13T14:25:07.238384+0000","last_peered":"2024-09-13T14:25:07.238384+0000","last_clean":"2024-09-13T14:25:07.238384+0000","last_became_active":"2024-09-13T14:24:17.271212+0000","last_became_peered":"2024-09-13T14:24:17.271212+0000","last_unstale":"2024-09-13T14:25:07.238384+0000","last_undegraded":"2024-09-13T14:25:07.238384+0000","last_fullsized":"2024-09-13T14:25:07.238384+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:23.750381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241503+0000","last_change":"2024-09-13T14:24:17.246522+0000","last_active":"2024-09-13T14:25:07.241503+0000","last_peered":"2024-09-13T14:25:07.241503+0000","last_clean":"2024-09-13T14:25:07.241503+0000","last_became_active":"2024-09-13T14:24:17.246005+0000","last_became_peered":"2024-09-13T14:24:17.246005+0000","last_unstale":"2024-09-13T14:25:07.241503+0000","last_undegraded":"2024-09-13T14:25:07.241503+0000","last_fullsized":"2024-09-13T14:25:07.241503+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:52.502894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239518+0000","last_change":"2024-09-13T14:24:17.260664+0000","last_active":"2024-09-13T14:25:07.239518+0000","last_peered":"2024-09-13T14:25:07.239518+0000","last_clean":"2024-09-13T14:25:07.239518+0000","last_became_active":"2024-09-13T14:24:17.260374+0000","last_became_peered":"2024-09-13T14:24:17.260374+0000","last_unstale":"2024-09-13T14:25:07.239518+0000","last_undegraded":"2024-09-13T14:25:07.239518+0000","last_fullsized":"2024-09-13T14:25:07.239518+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:31:01.451956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.244318+0000","last_change":"2024-09-13T14:24:17.229223+0000","last_active":"2024-09-13T14:25:07.244318+0000","last_peered":"2024-09-13T14:25:07.244318+0000","last_clean":"2024-09-13T14:25:07.244318+0000","last_became_active":"2024-09-13T14:24:17.224775+0000","last_became_peered":"2024-09-13T14:24:17.224775+0000","last_unstale":"2024-09-13T14:25:07.244318+0000","last_undegraded":"2024-09-13T14:25:07.244318+0000","last_fullsized":"2024-09-13T14:25:07.244318+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:05.931104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239877+0000","last_change":"2024-09-13T14:24:17.270348+0000","last_active":"2024-09-13T14:25:07.239877+0000","last_peered":"2024-09-13T14:25:07.239877+0000","last_clean":"2024-09-13T14:25:07.239877+0000","last_became_active":"2024-09-13T14:24:17.269821+0000","last_became_peered":"2024-09-13T14:24:17.269821+0000","last_unstale":"2024-09-13T14:25:07.239877+0000","last_undegraded":"2024-09-13T14:25:07.239877+0000","last_fullsized":"2024-09-13T14:25:07.239877+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:28.259239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239368+0000","last_change":"2024-09-13T14:24:17.237862+0000","last_active":"2024-09-13T14:25:07.239368+0000","last_peered":"2024-09-13T14:25:07.239368+0000","last_clean":"2024-09-13T14:25:07.239368+0000","last_became_active":"2024-09-13T14:24:17.233347+0000","last_became_peered":"2024-09-13T14:24:17.233347+0000","last_unstale":"2024-09-13T14:25:07.239368+0000","last_undegraded":"2024-09-13T14:25:07.239368+0000","last_fullsized":"2024-09-13T14:25:07.239368+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:22:33.207677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.240195+0000","last_change":"2024-09-13T14:24:17.271150+0000","last_active":"2024-09-13T14:25:07.240195+0000","last_peered":"2024-09-13T14:25:07.240195+0000","last_clean":"2024-09-13T14:25:07.240195+0000","last_became_active":"2024-09-13T14:24:17.270643+0000","last_became_peered":"2024-09-13T14:24:17.270643+0000","last_unstale":"2024-09-13T14:25:07.240195+0000","last_undegraded":"2024-09-13T14:25:07.240195+0000","last_fullsized":"2024-09-13T14:25:07.240195+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:37:21.003725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242323+0000","last_change":"2024-09-13T14:24:17.251584+0000","last_active":"2024-09-13T14:25:07.242323+0000","last_peered":"2024-09-13T14:25:07.242323+0000","last_clean":"2024-09-13T14:25:07.242323+0000","last_became_active":"2024-09-13T14:24:17.249275+0000","last_became_peered":"2024-09-13T14:24:17.249275+0000","last_unstale":"2024-09-13T14:25:07.242323+0000","last_undegraded":"2024-09-13T14:25:07.242323+0000","last_fullsized":"2024-09-13T14:25:07.242323+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:45.889488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238182+0000","last_change":"2024-09-13T14:24:17.278060+0000","last_active":"2024-09-13T14:25:07.238182+0000","last_peered":"2024-09-13T14:25:07.238182+0000","last_clean":"2024-09-13T14:25:07.238182+0000","last_became_active":"2024-09-13T14:24:17.277722+0000","last_became_peered":"2024-09-13T14:24:17.277722+0000","last_unstale":"2024-09-13T14:25:07.238182+0000","last_undegraded":"2024-09-13T14:25:07.238182+0000","last_fullsized":"2024-09-13T14:25:07.238182+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:59:15.030086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":57,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.330919+0000","last_change":"2024-09-13T14:20:07.527898+0000","last_active":"2024-09-13T14:25:07.330919+0000","last_peered":"2024-09-13T14:25:07.330919+0000","last_clean":"2024-09-13T14:25:07.330919+0000","last_became_active":"2024-09-13T14:20:07.515370+0000","last_became_peered":"2024-09-13T14:20:07.515370+0000","last_unstale":"2024-09-13T14:25:07.330919+0000","last_undegraded":"2024-09-13T14:25:07.330919+0000","last_fullsized":"2024-09-13T14:25:07.330919+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:45.300816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":28,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.250979+0000","last_change":"2024-09-13T14:22:03.176719+0000","last_active":"2024-09-13T14:25:07.250979+0000","last_peered":"2024-09-13T14:25:07.250979+0000","last_clean":"2024-09-13T14:25:07.250979+0000","last_became_active":"2024-09-13T14:21:52.862358+0000","last_became_peered":"2024-09-13T14:21:52.862358+0000","last_unstale":"2024-09-13T14:25:07.250979+0000","last_undegraded":"2024-09-13T14:25:07.250979+0000","last_fullsized":"2024-09-13T14:25:07.250979+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:22:54.637897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241553+0000","last_change":"2024-09-13T14:24:17.225596+0000","last_active":"2024-09-13T14:25:07.241553+0000","last_peered":"2024-09-13T14:25:07.241553+0000","last_clean":"2024-09-13T14:25:07.241553+0000","last_became_active":"2024-09-13T14:24:17.224861+0000","last_became_peered":"2024-09-13T14:24:17.224861+0000","last_unstale":"2024-09-13T14:25:07.241553+0000","last_undegraded":"2024-09-13T14:25:07.241553+0000","last_fullsized":"2024-09-13T14:25:07.241553+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:23.004735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.244671+0000","last_change":"2024-09-13T14:24:17.415855+0000","last_active":"2024-09-13T14:25:07.244671+0000","last_peered":"2024-09-13T14:25:07.244671+0000","last_clean":"2024-09-13T14:25:07.244671+0000","last_became_active":"2024-09-13T14:24:17.413815+0000","last_became_peered":"2024-09-13T14:24:17.413815+0000","last_unstale":"2024-09-13T14:25:07.244671+0000","last_undegraded":"2024-09-13T14:25:07.244671+0000","last_fullsized":"2024-09-13T14:25:07.244671+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:25:00.118559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.330858+0000","last_change":"2024-09-13T14:22:03.357964+0000","last_active":"2024-09-13T14:25:07.330858+0000","last_peered":"2024-09-13T14:25:07.330858+0000","last_clean":"2024-09-13T14:25:07.330858+0000","last_became_active":"2024-09-13T14:21:52.883199+0000","last_became_peered":"2024-09-13T14:21:52.883199+0000","last_unstale":"2024-09-13T14:25:07.330858+0000","last_undegraded":"2024-09-13T14:25:07.330858+0000","last_fullsized":"2024-09-13T14:25:07.330858+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:15:09.955395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239060+0000","last_change":"2024-09-13T14:24:17.240639+0000","last_active":"2024-09-13T14:25:07.239060+0000","last_peered":"2024-09-13T14:25:07.239060+0000","last_clean":"2024-09-13T14:25:07.239060+0000","last_became_active":"2024-09-13T14:24:17.239244+0000","last_became_peered":"2024-09-13T14:24:17.239244+0000","last_unstale":"2024-09-13T14:25:07.239060+0000","last_undegraded":"2024-09-13T14:25:07.239060+0000","last_fullsized":"2024-09-13T14:25:07.239060+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:50:52.952146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238809+0000","last_change":"2024-09-13T14:24:17.270423+0000","last_active":"2024-09-13T14:25:07.238809+0000","last_peered":"2024-09-13T14:25:07.238809+0000","last_clean":"2024-09-13T14:25:07.238809+0000","last_became_active":"2024-09-13T14:24:17.269949+0000","last_became_peered":"2024-09-13T14:24:17.269949+0000","last_unstale":"2024-09-13T14:25:07.238809+0000","last_undegraded":"2024-09-13T14:25:07.238809+0000","last_fullsized":"2024-09-13T14:25:07.238809+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:47:38.534443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241070+0000","last_change":"2024-09-13T14:22:03.172771+0000","last_active":"2024-09-13T14:25:07.241070+0000","last_peered":"2024-09-13T14:25:07.241070+0000","last_clean":"2024-09-13T14:25:07.241070+0000","last_became_active":"2024-09-13T14:21:52.884004+0000","last_became_peered":"2024-09-13T14:21:52.884004+0000","last_unstale":"2024-09-13T14:25:07.241070+0000","last_undegraded":"2024-09-13T14:25:07.241070+0000","last_fullsized":"2024-09-13T14:25:07.241070+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:20:21.593867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239701+0000","last_change":"2024-09-13T14:24:17.237493+0000","last_active":"2024-09-13T14:25:07.239701+0000","last_peered":"2024-09-13T14:25:07.239701+0000","last_clean":"2024-09-13T14:25:07.239701+0000","last_became_active":"2024-09-13T14:24:17.236513+0000","last_became_peered":"2024-09-13T14:24:17.236513+0000","last_unstale":"2024-09-13T14:25:07.239701+0000","last_undegraded":"2024-09-13T14:25:07.239701+0000","last_fullsized":"2024-09-13T14:25:07.239701+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:54:30.116051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:25:08.721247+0000","last_change":"2024-09-13T14:25:08.721247+0000","last_active":"2024-09-13T14:25:08.721247+0000","last_peered":"2024-09-13T14:25:08.721247+0000","last_clean":"2024-09-13T14:25:08.721247+0000","last_became_active":"2024-09-13T14:25:08.720536+0000","last_became_peered":"2024-09-13T14:25:08.720536+0000","last_unstale":"2024-09-13T14:25:08.721247+0000","last_undegraded":"2024-09-13T14:25:08.721247+0000","last_fullsized":"2024-09-13T14:25:08.721247+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:06:19.551355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":29,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251041+0000","last_change":"2024-09-13T14:22:03.177356+0000","last_active":"2024-09-13T14:25:07.251041+0000","last_peered":"2024-09-13T14:25:07.251041+0000","last_clean":"2024-09-13T14:25:07.251041+0000","last_became_active":"2024-09-13T14:21:52.861658+0000","last_became_peered":"2024-09-13T14:21:52.861658+0000","last_unstale":"2024-09-13T14:25:07.251041+0000","last_undegraded":"2024-09-13T14:25:07.251041+0000","last_fullsized":"2024-09-13T14:25:07.251041+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:10.200310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242107+0000","last_change":"2024-09-13T14:24:17.226365+0000","last_active":"2024-09-13T14:25:07.242107+0000","last_peered":"2024-09-13T14:25:07.242107+0000","last_clean":"2024-09-13T14:25:07.242107+0000","last_became_active":"2024-09-13T14:24:17.224828+0000","last_became_peered":"2024-09-13T14:24:17.224828+0000","last_unstale":"2024-09-13T14:25:07.242107+0000","last_undegraded":"2024-09-13T14:25:07.242107+0000","last_fullsized":"2024-09-13T14:25:07.242107+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:55:23.147922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.240475+0000","last_change":"2024-09-13T14:24:17.275739+0000","last_active":"2024-09-13T14:25:07.240475+0000","last_peered":"2024-09-13T14:25:07.240475+0000","last_clean":"2024-09-13T14:25:07.240475+0000","last_became_active":"2024-09-13T14:24:17.275258+0000","last_became_peered":"2024-09-13T14:24:17.275258+0000","last_unstale":"2024-09-13T14:25:07.240475+0000","last_undegraded":"2024-09-13T14:25:07.240475+0000","last_fullsized":"2024-09-13T14:25:07.240475+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:14:59.287041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251767+0000","last_change":"2024-09-13T14:24:17.227028+0000","last_active":"2024-09-13T14:25:07.251767+0000","last_peered":"2024-09-13T14:25:07.251767+0000","last_clean":"2024-09-13T14:25:07.251767+0000","last_became_active":"2024-09-13T14:24:17.226003+0000","last_became_peered":"2024-09-13T14:24:17.226003+0000","last_unstale":"2024-09-13T14:25:07.251767+0000","last_undegraded":"2024-09-13T14:25:07.251767+0000","last_fullsized":"2024-09-13T14:25:07.251767+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:35.841414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.240629+0000","last_change":"2024-09-13T14:22:03.171905+0000","last_active":"2024-09-13T14:25:07.240629+0000","last_peered":"2024-09-13T14:25:07.240629+0000","last_clean":"2024-09-13T14:25:07.240629+0000","last_became_active":"2024-09-13T14:21:52.884118+0000","last_became_peered":"2024-09-13T14:21:52.884118+0000","last_unstale":"2024-09-13T14:25:07.240629+0000","last_undegraded":"2024-09-13T14:25:07.240629+0000","last_fullsized":"2024-09-13T14:25:07.240629+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:26:17.407017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.237109+0000","last_change":"2024-09-13T14:24:17.277988+0000","last_active":"2024-09-13T14:25:07.237109+0000","last_peered":"2024-09-13T14:25:07.237109+0000","last_clean":"2024-09-13T14:25:07.237109+0000","last_became_active":"2024-09-13T14:24:17.277640+0000","last_became_peered":"2024-09-13T14:24:17.277640+0000","last_unstale":"2024-09-13T14:25:07.237109+0000","last_undegraded":"2024-09-13T14:25:07.237109+0000","last_fullsized":"2024-09-13T14:25:07.237109+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:50.707882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.240554+0000","last_change":"2024-09-13T14:24:17.237357+0000","last_active":"2024-09-13T14:25:07.240554+0000","last_peered":"2024-09-13T14:25:07.240554+0000","last_clean":"2024-09-13T14:25:07.240554+0000","last_became_active":"2024-09-13T14:24:17.235978+0000","last_became_peered":"2024-09-13T14:24:17.235978+0000","last_unstale":"2024-09-13T14:25:07.240554+0000","last_undegraded":"2024-09-13T14:25:07.240554+0000","last_fullsized":"2024-09-13T14:25:07.240554+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:36.835080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331606+0000","last_change":"2024-09-13T14:22:03.357990+0000","last_active":"2024-09-13T14:25:07.331606+0000","last_peered":"2024-09-13T14:25:07.331606+0000","last_clean":"2024-09-13T14:25:07.331606+0000","last_became_active":"2024-09-13T14:21:53.093638+0000","last_became_peered":"2024-09-13T14:21:53.093638+0000","last_unstale":"2024-09-13T14:25:07.331606+0000","last_undegraded":"2024-09-13T14:25:07.331606+0000","last_fullsized":"2024-09-13T14:25:07.331606+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:15:24.868073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238448+0000","last_change":"2024-09-13T14:24:17.413269+0000","last_active":"2024-09-13T14:25:07.238448+0000","last_peered":"2024-09-13T14:25:07.238448+0000","last_clean":"2024-09-13T14:25:07.238448+0000","last_became_active":"2024-09-13T14:24:17.412907+0000","last_became_peered":"2024-09-13T14:24:17.412907+0000","last_unstale":"2024-09-13T14:25:07.238448+0000","last_undegraded":"2024-09-13T14:25:07.238448+0000","last_fullsized":"2024-09-13T14:25:07.238448+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:20:05.270670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242898+0000","last_change":"2024-09-13T14:24:17.255947+0000","last_active":"2024-09-13T14:25:07.242898+0000","last_peered":"2024-09-13T14:25:07.242898+0000","last_clean":"2024-09-13T14:25:07.242898+0000","last_became_active":"2024-09-13T14:24:17.247909+0000","last_became_peered":"2024-09-13T14:24:17.247909+0000","last_unstale":"2024-09-13T14:25:07.242898+0000","last_undegraded":"2024-09-13T14:25:07.242898+0000","last_fullsized":"2024-09-13T14:25:07.242898+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:10.451346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.237952+0000","last_change":"2024-09-13T14:22:03.172028+0000","last_active":"2024-09-13T14:25:07.237952+0000","last_peered":"2024-09-13T14:25:07.237952+0000","last_clean":"2024-09-13T14:25:07.237952+0000","last_became_active":"2024-09-13T14:21:53.095058+0000","last_became_peered":"2024-09-13T14:21:53.095058+0000","last_unstale":"2024-09-13T14:25:07.237952+0000","last_undegraded":"2024-09-13T14:25:07.237952+0000","last_fullsized":"2024-09-13T14:25:07.237952+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:44.515568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080647399999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242247+0000","last_change":"2024-09-13T14:24:18.951583+0000","last_active":"2024-09-13T14:25:07.242247+0000","last_peered":"2024-09-13T14:25:07.242247+0000","last_clean":"2024-09-13T14:25:07.242247+0000","last_became_active":"2024-09-13T14:24:18.950822+0000","last_became_peered":"2024-09-13T14:24:18.950822+0000","last_unstale":"2024-09-13T14:25:07.242247+0000","last_undegraded":"2024-09-13T14:25:07.242247+0000","last_fullsized":"2024-09-13T14:25:07.242247+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:53:04.214093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":16,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238940+0000","last_change":"2024-09-13T14:24:17.231972+0000","last_active":"2024-09-13T14:25:07.238940+0000","last_peered":"2024-09-13T14:25:07.238940+0000","last_clean":"2024-09-13T14:25:07.238940+0000","last_became_active":"2024-09-13T14:24:17.231185+0000","last_became_peered":"2024-09-13T14:24:17.231185+0000","last_unstale":"2024-09-13T14:25:07.238940+0000","last_undegraded":"2024-09-13T14:25:07.238940+0000","last_fullsized":"2024-09-13T14:25:07.238940+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:16.680078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.245309+0000","last_change":"2024-09-13T14:22:03.178246+0000","last_active":"2024-09-13T14:25:07.245309+0000","last_peered":"2024-09-13T14:25:07.245309+0000","last_clean":"2024-09-13T14:25:07.245309+0000","last_became_active":"2024-09-13T14:21:52.861399+0000","last_became_peered":"2024-09-13T14:21:52.861399+0000","last_unstale":"2024-09-13T14:25:07.245309+0000","last_undegraded":"2024-09-13T14:25:07.245309+0000","last_fullsized":"2024-09-13T14:25:07.245309+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:51.432081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239217+0000","last_change":"2024-09-13T14:24:17.260443+0000","last_active":"2024-09-13T14:25:07.239217+0000","last_peered":"2024-09-13T14:25:07.239217+0000","last_clean":"2024-09-13T14:25:07.239217+0000","last_became_active":"2024-09-13T14:24:17.259887+0000","last_became_peered":"2024-09-13T14:24:17.259887+0000","last_unstale":"2024-09-13T14:25:07.239217+0000","last_undegraded":"2024-09-13T14:25:07.239217+0000","last_fullsized":"2024-09-13T14:25:07.239217+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:53.307078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:25:09.211515+0000","last_change":"2024-09-13T14:25:09.211515+0000","last_active":"2024-09-13T14:25:09.211515+0000","last_peered":"2024-09-13T14:25:09.211515+0000","last_clean":"2024-09-13T14:25:09.211515+0000","last_became_active":"2024-09-13T14:25:09.210780+0000","last_became_peered":"2024-09-13T14:25:09.210780+0000","last_unstale":"2024-09-13T14:25:09.211515+0000","last_undegraded":"2024-09-13T14:25:09.211515+0000","last_fullsized":"2024-09-13T14:25:09.211515+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:01.297911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"65'7","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239763+0000","last_change":"2024-09-13T14:24:17.235747+0000","last_active":"2024-09-13T14:25:07.239763+0000","last_peered":"2024-09-13T14:25:07.239763+0000","last_clean":"2024-09-13T14:25:07.239763+0000","last_became_active":"2024-09-13T14:24:17.231904+0000","last_became_peered":"2024-09-13T14:24:17.231904+0000","last_unstale":"2024-09-13T14:25:07.239763+0000","last_undegraded":"2024-09-13T14:25:07.239763+0000","last_fullsized":"2024-09-13T14:25:07.239763+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:34.362299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.237190+0000","last_change":"2024-09-13T14:24:17.276111+0000","last_active":"2024-09-13T14:25:07.237190+0000","last_peered":"2024-09-13T14:25:07.237190+0000","last_clean":"2024-09-13T14:25:07.237190+0000","last_became_active":"2024-09-13T14:24:17.275576+0000","last_became_peered":"2024-09-13T14:24:17.275576+0000","last_unstale":"2024-09-13T14:25:07.237190+0000","last_undegraded":"2024-09-13T14:25:07.237190+0000","last_fullsized":"2024-09-13T14:25:07.237190+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:45.859807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.331241+0000","last_change":"2024-09-13T14:24:17.244991+0000","last_active":"2024-09-13T14:25:07.331241+0000","last_peered":"2024-09-13T14:25:07.331241+0000","last_clean":"2024-09-13T14:25:07.331241+0000","last_became_active":"2024-09-13T14:24:17.244107+0000","last_became_peered":"2024-09-13T14:24:17.244107+0000","last_unstale":"2024-09-13T14:25:07.331241+0000","last_undegraded":"2024-09-13T14:25:07.331241+0000","last_fullsized":"2024-09-13T14:25:07.331241+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:11:14.995141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.250824+0000","last_change":"2024-09-13T14:24:17.230464+0000","last_active":"2024-09-13T14:25:07.250824+0000","last_peered":"2024-09-13T14:25:07.250824+0000","last_clean":"2024-09-13T14:25:07.250824+0000","last_became_active":"2024-09-13T14:24:17.229956+0000","last_became_peered":"2024-09-13T14:24:17.229956+0000","last_unstale":"2024-09-13T14:25:07.250824+0000","last_undegraded":"2024-09-13T14:25:07.250824+0000","last_fullsized":"2024-09-13T14:25:07.250824+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:35:37.489846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241649+0000","last_change":"2024-09-13T14:24:18.613468+0000","last_active":"2024-09-13T14:25:07.241649+0000","last_peered":"2024-09-13T14:25:07.241649+0000","last_clean":"2024-09-13T14:25:07.241649+0000","last_became_active":"2024-09-13T14:24:18.612550+0000","last_became_peered":"2024-09-13T14:24:18.612550+0000","last_unstale":"2024-09-13T14:25:07.241649+0000","last_undegraded":"2024-09-13T14:25:07.241649+0000","last_fullsized":"2024-09-13T14:25:07.241649+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:36:42.550479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239751+0000","last_change":"2024-09-13T14:24:17.236823+0000","last_active":"2024-09-13T14:25:07.239751+0000","last_peered":"2024-09-13T14:25:07.239751+0000","last_clean":"2024-09-13T14:25:07.239751+0000","last_became_active":"2024-09-13T14:24:17.236174+0000","last_became_peered":"2024-09-13T14:24:17.236174+0000","last_unstale":"2024-09-13T14:25:07.239751+0000","last_undegraded":"2024-09-13T14:25:07.239751+0000","last_fullsized":"2024-09-13T14:25:07.239751+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:53.380050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":24,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-13T14:25:09.212576+0000","last_change":"2024-09-13T14:25:09.212576+0000","last_active":"2024-09-13T14:25:09.212576+0000","last_peered":"2024-09-13T14:25:09.212576+0000","last_clean":"2024-09-13T14:25:09.212576+0000","last_became_active":"2024-09-13T14:25:09.211610+0000","last_became_peered":"2024-09-13T14:25:09.211610+0000","last_unstale":"2024-09-13T14:25:09.212576+0000","last_undegraded":"2024-09-13T14:25:09.212576+0000","last_fullsized":"2024-09-13T14:25:09.212576+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:45.964193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.245089+0000","last_change":"2024-09-13T14:24:17.268884+0000","last_active":"2024-09-13T14:25:07.245089+0000","last_peered":"2024-09-13T14:25:07.245089+0000","last_clean":"2024-09-13T14:25:07.245089+0000","last_became_active":"2024-09-13T14:24:17.268250+0000","last_became_peered":"2024-09-13T14:24:17.268250+0000","last_unstale":"2024-09-13T14:25:07.245089+0000","last_undegraded":"2024-09-13T14:25:07.245089+0000","last_fullsized":"2024-09-13T14:25:07.245089+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:15:32.373469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242259+0000","last_change":"2024-09-13T14:24:17.250386+0000","last_active":"2024-09-13T14:25:07.242259+0000","last_peered":"2024-09-13T14:25:07.242259+0000","last_clean":"2024-09-13T14:25:07.242259+0000","last_became_active":"2024-09-13T14:24:17.248972+0000","last_became_peered":"2024-09-13T14:24:17.248972+0000","last_unstale":"2024-09-13T14:25:07.242259+0000","last_undegraded":"2024-09-13T14:25:07.242259+0000","last_fullsized":"2024-09-13T14:25:07.242259+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:32.448624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238740+0000","last_change":"2024-09-13T14:24:17.413237+0000","last_active":"2024-09-13T14:25:07.238740+0000","last_peered":"2024-09-13T14:25:07.238740+0000","last_clean":"2024-09-13T14:25:07.238740+0000","last_became_active":"2024-09-13T14:24:17.412906+0000","last_became_peered":"2024-09-13T14:24:17.412906+0000","last_unstale":"2024-09-13T14:25:07.238740+0000","last_undegraded":"2024-09-13T14:25:07.238740+0000","last_fullsized":"2024-09-13T14:25:07.238740+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:15.744172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.236952+0000","last_change":"2024-09-13T14:24:17.235517+0000","last_active":"2024-09-13T14:25:07.236952+0000","last_peered":"2024-09-13T14:25:07.236952+0000","last_clean":"2024-09-13T14:25:07.236952+0000","last_became_active":"2024-09-13T14:24:17.234813+0000","last_became_peered":"2024-09-13T14:24:17.234813+0000","last_unstale":"2024-09-13T14:25:07.236952+0000","last_undegraded":"2024-09-13T14:25:07.236952+0000","last_fullsized":"2024-09-13T14:25:07.236952+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:03.127643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242252+0000","last_change":"2024-09-13T14:24:18.584092+0000","last_active":"2024-09-13T14:25:07.242252+0000","last_peered":"2024-09-13T14:25:07.242252+0000","last_clean":"2024-09-13T14:25:07.242252+0000","last_became_active":"2024-09-13T14:24:18.583264+0000","last_became_peered":"2024-09-13T14:24:18.583264+0000","last_unstale":"2024-09-13T14:25:07.242252+0000","last_undegraded":"2024-09-13T14:25:07.242252+0000","last_fullsized":"2024-09-13T14:25:07.242252+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:05.370920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239850+0000","last_change":"2024-09-13T14:24:17.232228+0000","last_active":"2024-09-13T14:25:07.239850+0000","last_peered":"2024-09-13T14:25:07.239850+0000","last_clean":"2024-09-13T14:25:07.239850+0000","last_became_active":"2024-09-13T14:24:17.231457+0000","last_became_peered":"2024-09-13T14:24:17.231457+0000","last_unstale":"2024-09-13T14:25:07.239850+0000","last_undegraded":"2024-09-13T14:25:07.239850+0000","last_fullsized":"2024-09-13T14:25:07.239850+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:12.673393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241630+0000","last_change":"2024-09-13T14:24:17.278478+0000","last_active":"2024-09-13T14:25:07.241630+0000","last_peered":"2024-09-13T14:25:07.241630+0000","last_clean":"2024-09-13T14:25:07.241630+0000","last_became_active":"2024-09-13T14:24:17.278108+0000","last_became_peered":"2024-09-13T14:24:17.278108+0000","last_unstale":"2024-09-13T14:25:07.241630+0000","last_undegraded":"2024-09-13T14:25:07.241630+0000","last_fullsized":"2024-09-13T14:25:07.241630+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:53.787023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238572+0000","last_change":"2024-09-13T14:24:17.233488+0000","last_active":"2024-09-13T14:25:07.238572+0000","last_peered":"2024-09-13T14:25:07.238572+0000","last_clean":"2024-09-13T14:25:07.238572+0000","last_became_active":"2024-09-13T14:24:17.232403+0000","last_became_peered":"2024-09-13T14:24:17.232403+0000","last_unstale":"2024-09-13T14:25:07.238572+0000","last_undegraded":"2024-09-13T14:25:07.238572+0000","last_fullsized":"2024-09-13T14:25:07.238572+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:06.054933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239767+0000","last_change":"2024-09-13T14:24:17.273086+0000","last_active":"2024-09-13T14:25:07.239767+0000","last_peered":"2024-09-13T14:25:07.239767+0000","last_clean":"2024-09-13T14:25:07.239767+0000","last_became_active":"2024-09-13T14:24:17.272459+0000","last_became_peered":"2024-09-13T14:24:17.272459+0000","last_unstale":"2024-09-13T14:25:07.239767+0000","last_undegraded":"2024-09-13T14:25:07.239767+0000","last_fullsized":"2024-09-13T14:25:07.239767+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:34:24.756921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251225+0000","last_change":"2024-09-13T14:24:17.233611+0000","last_active":"2024-09-13T14:25:07.251225+0000","last_peered":"2024-09-13T14:25:07.251225+0000","last_clean":"2024-09-13T14:25:07.251225+0000","last_became_active":"2024-09-13T14:24:17.233075+0000","last_became_peered":"2024-09-13T14:24:17.233075+0000","last_unstale":"2024-09-13T14:25:07.251225+0000","last_undegraded":"2024-09-13T14:25:07.251225+0000","last_fullsized":"2024-09-13T14:25:07.251225+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:22.672021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239071+0000","last_change":"2024-09-13T14:24:17.263201+0000","last_active":"2024-09-13T14:25:07.239071+0000","last_peered":"2024-09-13T14:25:07.239071+0000","last_clean":"2024-09-13T14:25:07.239071+0000","last_became_active":"2024-09-13T14:24:17.262746+0000","last_became_peered":"2024-09-13T14:24:17.262746+0000","last_unstale":"2024-09-13T14:25:07.239071+0000","last_undegraded":"2024-09-13T14:25:07.239071+0000","last_fullsized":"2024-09-13T14:25:07.239071+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:57.696774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.251656+0000","last_change":"2024-09-13T14:24:17.240313+0000","last_active":"2024-09-13T14:25:07.251656+0000","last_peered":"2024-09-13T14:25:07.251656+0000","last_clean":"2024-09-13T14:25:07.251656+0000","last_became_active":"2024-09-13T14:24:17.239555+0000","last_became_peered":"2024-09-13T14:24:17.239555+0000","last_unstale":"2024-09-13T14:25:07.251656+0000","last_undegraded":"2024-09-13T14:25:07.251656+0000","last_fullsized":"2024-09-13T14:25:07.251656+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:32:44.983674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.241865+0000","last_change":"2024-09-13T14:24:18.951307+0000","last_active":"2024-09-13T14:25:07.241865+0000","last_peered":"2024-09-13T14:25:07.241865+0000","last_clean":"2024-09-13T14:25:07.241865+0000","last_became_active":"2024-09-13T14:24:18.950577+0000","last_became_peered":"2024-09-13T14:24:18.950577+0000","last_unstale":"2024-09-13T14:25:07.241865+0000","last_undegraded":"2024-09-13T14:25:07.241865+0000","last_fullsized":"2024-09-13T14:25:07.241865+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":63,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:30.356259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":17,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.242594+0000","last_change":"2024-09-13T14:24:17.225321+0000","last_active":"2024-09-13T14:25:07.242594+0000","last_peered":"2024-09-13T14:25:07.242594+0000","last_clean":"2024-09-13T14:25:07.242594+0000","last_became_active":"2024-09-13T14:24:17.224552+0000","last_became_peered":"2024-09-13T14:24:17.224552+0000","last_unstale":"2024-09-13T14:25:07.242594+0000","last_undegraded":"2024-09-13T14:25:07.242594+0000","last_fullsized":"2024-09-13T14:25:07.242594+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:20:14.242431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.238724+0000","last_change":"2024-09-13T14:24:17.271339+0000","last_active":"2024-09-13T14:25:07.238724+0000","last_peered":"2024-09-13T14:25:07.238724+0000","last_clean":"2024-09-13T14:25:07.238724+0000","last_became_active":"2024-09-13T14:24:17.270640+0000","last_became_peered":"2024-09-13T14:24:17.270640+0000","last_unstale":"2024-09-13T14:25:07.238724+0000","last_undegraded":"2024-09-13T14:25:07.238724+0000","last_fullsized":"2024-09-13T14:25:07.238724+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:42:09.267241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:25:07.239669+0000","last_change":"2024-09-13T14:24:17.240983+0000","last_active":"2024-09-13T14:25:07.239669+0000","last_peered":"2024-09-13T14:25:07.239669+0000","last_clean":"2024-09-13T14:25:07.239669+0000","last_became_active":"2024-09-13T14:24:17.239412+0000","last_became_peered":"2024-09-13T14:24:17.239412+0000","last_unstale":"2024-09-13T14:25:07.239669+0000","last_undegraded":"2024-09-13T14:25:07.239669+0000","last_fullsized":"2024-09-13T14:25:07.239669+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:50:49.775014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":47,"num_read_kb":49,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397582,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298632,"kb_used_data":1276,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446520,"statfs":{"total":95995035648,"available":95689236480,"internally_reserved":0,"allocated":1306624,"data_stored":895899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:26:16 2024","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.83299999999999996,"15min":0.83599999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.23,"5min":1.23,"15min":1.268},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.873,"15min":0.88500000000000001},"min":{"1min":0.55400000000000005,"5min":0.505,"15min":0.505},"max":{"1min":1.145,"5min":1.349,"15min":1.349},"last":0.90300000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:26:16 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.86199999999999999,"15min":0.88700000000000001},"min":{"1min":0.60399999999999998,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.278},"last":1.1220000000000001},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.91200000000000003,"15min":0.90800000000000003},"min":{"1min":0.56899999999999995,"5min":0.50700000000000001,"15min":0.48099999999999998},"max":{"1min":1.1279999999999999,"5min":1.234,"15min":1.234},"last":0.71499999999999997}]},{"osd":2,"last update":"Fri Sep 13 14:26:16 2024","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.93500000000000005,"15min":0.96299999999999997},"min":{"1min":0.58699999999999997,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.2789999999999999,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0089999999999999,"15min":0.99199999999999999},"min":{"1min":0.63100000000000001,"5min":0.53300000000000003,"15min":0.497},"max":{"1min":1.3440000000000001,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.82699999999999996}]},{"osd":3,"last update":"Fri Sep 13 14:26:16 2024","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.94699999999999995,"15min":0.94899999999999995},"min":{"1min":0.52700000000000002,"5min":0.44,"15min":0.44},"max":{"1min":1.3069999999999999,"5min":1.375,"15min":1.375},"last":1.0860000000000001},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.94399999999999995,"15min":0.94599999999999995},"min":{"1min":0.54300000000000004,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.2210000000000001,"5min":1.282,"15min":1.3420000000000001},"last":1.0629999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:26:16 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.86799999999999999,"15min":0.88700000000000001},"min":{"1min":0.60199999999999998,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.226,"5min":1.337,"15min":1.337},"last":1.226},{"interface":"front","average":{"1min":0.873,"5min":0.95699999999999996,"15min":0.97099999999999997},"min":{"1min":0.48999999999999999,"5min":0.35399999999999998,"15min":0.34399999999999997},"max":{"1min":1.258,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.78800000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:26:16 2024","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.95199999999999996,"15min":0.92600000000000005},"min":{"1min":0.44900000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2909999999999999,"5min":1.341,"15min":1.341},"last":0.93300000000000005},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.99299999999999999,"15min":0.98699999999999999},"min":{"1min":0.63700000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.319,"5min":1.401,"15min":1.401},"last":1.0149999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:26:16 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.0509999999999999,"15min":1.016},"min":{"1min":0.80400000000000005,"5min":0.55100000000000005,"15min":0.38800000000000001},"max":{"1min":1.2350000000000001,"5min":1.401,"15min":1.401},"last":1.151},{"interface":"front","average":{"1min":1.03,"5min":1.0589999999999999,"15min":1.0649999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3819999999999999,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":0.877}]}]},{"osd":6,"up_from":43,"seq":184683593817,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1260,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1290240,"data_stored":895142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.88600000000000001,"15min":0.87},"min":{"1min":0.52400000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.298,"5min":1.298,"15min":1.298},"last":1.012},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.85699999999999998,"15min":0.83299999999999996},"min":{"1min":0.64000000000000001,"5min":0.51100000000000001,"15min":0.49299999999999999},"max":{"1min":1.232,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.876}]},{"osd":1,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.88700000000000001,"15min":0.84199999999999997},"min":{"1min":0.61599999999999999,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.169,"5min":1.3260000000000001,"15min":1.3260000000000001},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.90500000000000003,"15min":0.873},"min":{"1min":0.627,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.65800000000000003}]},{"osd":2,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.91200000000000003,"15min":0.88600000000000001},"min":{"1min":0.59399999999999997,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.3580000000000001,"5min":1.3759999999999999,"15min":1.419},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.94099999999999995,"15min":0.89100000000000001},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.55400000000000005},"max":{"1min":1.228,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":0.94699999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.91600000000000004,"15min":0.874},"min":{"1min":0.55100000000000005,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.264,"5min":1.659,"15min":1.659},"last":0.98899999999999999},{"interface":"front","average":{"1min":1.004,"5min":0.94799999999999995,"15min":0.90800000000000003},"min":{"1min":0.81699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.252,"5min":1.3420000000000001,"15min":1.5169999999999999},"last":0.92500000000000004}]},{"osd":4,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.86399999999999999,"15min":0.78700000000000003},"min":{"1min":0.64700000000000002,"5min":0.313,"15min":0.313},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.91300000000000003,"15min":0.79700000000000004},"min":{"1min":0.63700000000000001,"5min":0.377,"15min":0.317},"max":{"1min":1.28,"5min":1.423,"15min":1.423},"last":0.69399999999999995}]},{"osd":5,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":0.96299999999999997,"15min":0.86899999999999999},"min":{"1min":0.70799999999999996,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.3300000000000001,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.996,"15min":0.876},"min":{"1min":0.72599999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.3069999999999999,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":0.90100000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:26:16 2024","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0409999999999999,"15min":1.002},"min":{"1min":0.70199999999999996,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.2529999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.167,"5min":1.1240000000000001,"15min":1.044},"min":{"1min":0.92700000000000005,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.657,"5min":1.657,"15min":1.657},"last":1.0409999999999999}]}]},{"osd":5,"up_from":37,"seq":158913790051,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":692224,"data_stored":304107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.81899999999999995,"15min":0.81299999999999994},"min":{"1min":0.58399999999999996,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.222,"5min":1.323,"15min":1.323},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.91500000000000004,"15min":0.84799999999999998},"min":{"1min":0.52800000000000002,"5min":0.505,"15min":0.505},"max":{"1min":1.204,"5min":1.45,"15min":1.45},"last":0.69699999999999995}]},{"osd":1,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.876,"15min":0.83999999999999997},"min":{"1min":0.57099999999999995,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.91800000000000004,"15min":0.83599999999999997},"min":{"1min":0.60899999999999999,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.161,"5min":2.0459999999999998,"15min":2.0459999999999998},"last":1.016}]},{"osd":2,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.90400000000000003,"15min":0.82899999999999996},"min":{"1min":0.67400000000000004,"5min":0.502,"15min":0.48399999999999999},"max":{"1min":1.236,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.92200000000000004,"15min":0.875},"min":{"1min":0.76800000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.2290000000000001,"5min":1.3400000000000001,"15min":1.3400000000000001},"last":0.81799999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.95999999999999996,"15min":0.88500000000000001},"min":{"1min":0.745,"5min":0.497,"15min":0.497},"max":{"1min":1.1679999999999999,"5min":1.573,"15min":1.573},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.995,"15min":0.89300000000000002},"min":{"1min":0.70799999999999996,"5min":0.52400000000000002,"15min":0.51400000000000001},"max":{"1min":1.288,"5min":2.246,"15min":2.246},"last":0.97899999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.878,"15min":0.71499999999999997},"min":{"1min":0.621,"5min":0.39200000000000002,"15min":0.29399999999999998},"max":{"1min":1.3169999999999999,"5min":1.359,"15min":1.359},"last":1.089},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.95499999999999996,"15min":0.77900000000000003},"min":{"1min":0.70399999999999996,"5min":0.38200000000000001,"15min":0.32700000000000001},"max":{"1min":1.3029999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.93100000000000005}]},{"osd":6,"last update":"Fri Sep 13 14:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":0.95999999999999996,"15min":0.85599999999999998},"min":{"1min":0.68400000000000005,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.272,"5min":1.488,"15min":1.488},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.048,"5min":1.0149999999999999,"15min":0.94499999999999995},"min":{"1min":0.81399999999999995,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.2609999999999999,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.286}]},{"osd":7,"last update":"Fri Sep 13 14:26:11 2024","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.0569999999999999,"15min":1.0229999999999999},"min":{"1min":0.85499999999999998,"5min":0.376,"15min":0.376},"max":{"1min":1.339,"5min":1.8080000000000001,"15min":1.8080000000000001},"last":1.345},{"interface":"front","average":{"1min":1.133,"5min":1.0509999999999999,"15min":0.98899999999999999},"min":{"1min":0.79200000000000004,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.3859999999999999,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":1.0580000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986286,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":305461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.88600000000000001,"15min":0.82499999999999996},"min":{"1min":0.61799999999999999,"5min":0.52800000000000002,"15min":0.52300000000000002},"max":{"1min":1.2050000000000001,"5min":1.224,"15min":1.224},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.92300000000000004,"15min":0.85799999999999998},"min":{"1min":0.54900000000000004,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.2410000000000001,"5min":1.53,"15min":1.53},"last":1.0469999999999999}]},{"osd":1,"last update":"Fri Sep 13 14:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.90500000000000003,"15min":0.82299999999999995},"min":{"1min":0.63900000000000001,"5min":0.60299999999999998,"15min":0.56499999999999995},"max":{"1min":1.1579999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.87},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.90200000000000002,"15min":0.85199999999999998},"min":{"1min":0.73599999999999999,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.202,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":0.90500000000000003}]},{"osd":2,"last update":"Fri Sep 13 14:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.92300000000000004,"15min":0.88700000000000001},"min":{"1min":0.72099999999999997,"5min":0.56699999999999995,"15min":0.52300000000000002},"max":{"1min":1.24,"5min":1.3200000000000001,"15min":1.3939999999999999},"last":1.2150000000000001},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.91500000000000004,"15min":0.84799999999999998},"min":{"1min":0.68999999999999995,"5min":0.59399999999999997,"15min":0.56799999999999995},"max":{"1min":1.1279999999999999,"5min":1.728,"15min":1.728},"last":0.82699999999999996}]},{"osd":3,"last update":"Fri Sep 13 14:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.96999999999999997,"15min":0.88700000000000001},"min":{"1min":0.60499999999999998,"5min":0.59299999999999997,"15min":0.57199999999999995},"max":{"1min":1.264,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.97299999999999998,"15min":0.88100000000000001},"min":{"1min":0.627,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":1.083}]},{"osd":5,"last update":"Fri Sep 13 14:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.88,"15min":0.78500000000000003},"min":{"1min":0.67700000000000005,"5min":0.44,"15min":0.38800000000000001},"max":{"1min":1.135,"5min":1.216,"15min":1.238},"last":0.75600000000000001},{"interface":"front","average":{"1min":1.002,"5min":0.96799999999999997,"15min":0.79100000000000004},"min":{"1min":0.66500000000000004,"5min":0.46600000000000003,"15min":0.37},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":1.121}]},{"osd":6,"last update":"Fri Sep 13 14:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.92000000000000004,"15min":0.82199999999999995},"min":{"1min":0.621,"5min":0.44500000000000001,"15min":0.38200000000000001},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96199999999999997,"15min":0.86399999999999999},"min":{"1min":0.65000000000000002,"5min":0.44600000000000001,"15min":0.33900000000000002},"max":{"1min":1.2370000000000001,"5min":1.431,"15min":1.431},"last":1.151}]},{"osd":7,"last update":"Fri Sep 13 14:25:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.03,"15min":1.0620000000000001},"min":{"1min":0.82099999999999995,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.3089999999999999,"5min":1.411,"15min":1.411},"last":1.2370000000000001},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1040000000000001,"15min":1.1240000000000001},"min":{"1min":0.80300000000000005,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.371,"5min":1.504,"15min":1.504},"last":1.1799999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149817,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298028,"kb_used_data":672,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447124,"statfs":{"total":95995035648,"available":95689854976,"internally_reserved":0,"allocated":688128,"data_stored":303853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:54 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.64100000000000001,"15min":0.65400000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.171,"5min":1.1919999999999999,"15min":1.4730000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.72599999999999998,"15min":0.69899999999999995},"min":{"1min":0.47699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.093,"5min":1.3,"15min":1.3},"last":0.80400000000000005}]},{"osd":1,"last update":"Fri Sep 13 14:25:54 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.67300000000000004,"15min":0.67000000000000004},"min":{"1min":0.40600000000000003,"5min":0.316,"15min":0.316},"max":{"1min":1.0209999999999999,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.77600000000000002,"15min":0.751},"min":{"1min":0.46500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.1259999999999999,"5min":1.206,"15min":1.2070000000000001},"last":0.66800000000000004}]},{"osd":2,"last update":"Fri Sep 13 14:25:54 2024","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.83499999999999996,"15min":0.78600000000000003},"min":{"1min":0.44500000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.754,"5min":0.76400000000000001,"15min":0.751},"min":{"1min":0.51500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.0660000000000001,"5min":1.2310000000000001,"15min":1.2310000000000001},"last":0.96099999999999997}]},{"osd":4,"last update":"Fri Sep 13 14:25:43 2024","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.91200000000000003,"15min":0.88600000000000001},"min":{"1min":0.55700000000000005,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.98099999999999998,"15min":0.92700000000000005},"min":{"1min":0.65500000000000003,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.4259999999999999,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.83999999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:25:30 2024","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.0309999999999999,"15min":0.98199999999999998},"min":{"1min":0.81899999999999995,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.5349999999999999,"5min":1.5349999999999999,"15min":1.5349999999999999},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.94499999999999995,"15min":0.91500000000000004},"min":{"1min":0.623,"5min":0.434,"15min":0.434},"max":{"1min":1.452,"5min":1.452,"15min":1.7569999999999999},"last":0.98699999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:25:27 2024","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.0640000000000001,"15min":1.0700000000000001},"min":{"1min":0.79700000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.486,"5min":1.514,"15min":1.514},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.034,"5min":1.008,"15min":1.02},"min":{"1min":0.63700000000000001,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.474,"5min":1.6439999999999999,"15min":1.6439999999999999},"last":0.90900000000000003}]},{"osd":7,"last update":"Fri Sep 13 14:26:17 2024","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.0029999999999999,"15min":1.0369999999999999},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.28,"5min":1.389,"15min":1.389},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.111,"15min":1.1930000000000001},"min":{"1min":0.72399999999999998,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.5509999999999999,"5min":1.587,"15min":2.2349999999999999},"last":1.0720000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411460,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298028,"kb_used_data":676,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447124,"statfs":{"total":95995035648,"available":95689854976,"internally_reserved":0,"allocated":692224,"data_stored":304367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:25:50 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65900000000000003,"15min":0.63},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.0429999999999999,"5min":1.228,"15min":1.3069999999999999},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.71099999999999997,"15min":0.65500000000000003},"min":{"1min":0.378,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.29,"5min":1.29,"15min":1.29},"last":0.64600000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:25:50 2024","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.65500000000000003,"15min":0.627},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.92300000000000004,"5min":0.97699999999999998,"15min":1.014},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.77300000000000002,"15min":0.70499999999999996},"min":{"1min":0.39700000000000002,"5min":0.373,"15min":0.373},"max":{"1min":1.1970000000000001,"5min":1.375,"15min":1.375},"last":1.0069999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.81399999999999995,"15min":0.80800000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.034,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.75600000000000001,"15min":0.73599999999999999},"min":{"1min":0.42099999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.0469999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:25:36 2024","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.98699999999999999,"15min":0.91900000000000004},"min":{"1min":0.67400000000000004,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.754,"5min":1.754,"15min":1.881},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.98099999999999998,"15min":0.94199999999999995},"min":{"1min":0.745,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.2629999999999999,"5min":1.498,"15min":1.498},"last":1.1299999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:25:36 2024","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.004,"15min":0.995},"min":{"1min":0.55900000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.23,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.22},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0009999999999999,"15min":0.94899999999999995},"min":{"1min":0.67300000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.327,"5min":1.363,"15min":1.446},"last":0.91200000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.93700000000000006,"15min":0.96499999999999997},"min":{"1min":0.79600000000000004,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":1.915},"last":1.0840000000000001},{"interface":"front","average":{"1min":1.034,"5min":1.0429999999999999,"15min":1.0409999999999999},"min":{"1min":0.67200000000000004,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.6659999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":0.98299999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:25:22 2024","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.093,"15min":1.125},"min":{"1min":0.76600000000000001,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.2450000000000001,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":1.29},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.1040000000000001,"15min":1.1240000000000001},"min":{"1min":0.751,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.5740000000000001,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":1.1659999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574990,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298020,"kb_used_data":668,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447132,"statfs":{"total":95995035648,"available":95689863168,"internally_reserved":0,"allocated":684032,"data_stored":303467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:26:03 2024","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.67300000000000004,"15min":0.60899999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.33200000000000002},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70699999999999996,"15min":0.625},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.4330000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.44700000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:25:52 2024","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.71499999999999997,"15min":0.66200000000000003},"min":{"1min":0.35099999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.016,"5min":1.0209999999999999,"15min":1.1359999999999999},"last":0.314},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.77300000000000002,"15min":0.70799999999999996},"min":{"1min":0.45500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.1859999999999999,"5min":1.194,"15min":1.194},"last":0.46899999999999997}]},{"osd":3,"last update":"Fri Sep 13 14:25:52 2024","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.75600000000000001,"15min":0.751},"min":{"1min":0.41099999999999998,"5min":0.31,"15min":0.31},"max":{"1min":1.0429999999999999,"5min":1.071,"15min":1.2110000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.86199999999999999,"15min":0.80500000000000005},"min":{"1min":0.49399999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.2170000000000001,"5min":1.355,"15min":1.355},"last":0.41799999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:25:38 2024","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0089999999999999,"15min":0.94399999999999995},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.4890000000000001,"5min":1.647,"15min":1.647},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.002,"5min":0.95599999999999996,"15min":0.88700000000000001},"min":{"1min":0.52300000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.3,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":0.75600000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.98899999999999999,"15min":0.94899999999999995},"min":{"1min":0.73399999999999999,"5min":0.72199999999999998,"15min":0.64500000000000002},"max":{"1min":1.3420000000000001,"5min":1.9339999999999999,"15min":1.9339999999999999},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.024,"15min":0.96099999999999997},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.347,"5min":1.7929999999999999,"15min":1.7929999999999999},"last":0.88600000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:25:28 2024","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.05,"15min":1.0489999999999999},"min":{"1min":0.63600000000000001,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.3280000000000001,"5min":1.458,"15min":1.458},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.018,"15min":0.98599999999999999},"min":{"1min":0.68600000000000005,"5min":0.624,"15min":0.624},"max":{"1min":1.5549999999999999,"5min":1.716,"15min":1.716},"last":0.63500000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:26:16 2024","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.0449999999999999,"15min":1.054},"min":{"1min":0.72099999999999997,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.3720000000000001,"5min":1.516,"15min":1.516},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.1339999999999999,"15min":1.149},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.4199999999999999,"5min":1.5329999999999999,"15min":1.8919999999999999},"last":0.61099999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738520,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298612,"kb_used_data":1256,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446540,"statfs":{"total":95995035648,"available":95689256960,"internally_reserved":0,"allocated":1286144,"data_stored":894723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65900000000000003,"15min":0.60099999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.30499999999999999},"max":{"1min":1.087,"5min":1.323,"15min":1.323},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.77500000000000002,"15min":0.67300000000000004},"min":{"1min":0.441,"5min":0.307,"15min":0.26700000000000002},"max":{"1min":1.2070000000000001,"5min":1.448,"15min":1.448},"last":0.89900000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.76300000000000001,"15min":0.68799999999999994},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.35799999999999998},"max":{"1min":1.0860000000000001,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.624},{"interface":"front","average":{"1min":0.746,"5min":0.69199999999999995,"15min":0.64400000000000002},"min":{"1min":0.35899999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.032}]},{"osd":3,"last update":"Fri Sep 13 14:25:54 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.85599999999999998,"15min":0.80400000000000005},"min":{"1min":0.52800000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":0.746},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.78400000000000003,"15min":0.751},"min":{"1min":0.42099999999999999,"5min":0.376,"15min":0.376},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.79100000000000004}]},{"osd":4,"last update":"Fri Sep 13 14:25:32 2024","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.92400000000000004,"15min":0.85699999999999998},"min":{"1min":0.76500000000000001,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.1759999999999999,"5min":1.931,"15min":1.931},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.079,"5min":1.012,"15min":0.94799999999999995},"min":{"1min":0.83599999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.3640000000000001,"5min":1.431,"15min":1.431},"last":1.0620000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:25:30 2024","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.97999999999999998,"15min":0.92900000000000005},"min":{"1min":0.85099999999999998,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.4770000000000001,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":1.153},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.038,"15min":1.0209999999999999},"min":{"1min":0.76500000000000001,"5min":0.62,"15min":0.62},"max":{"1min":1.4450000000000001,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":0.997}]},{"osd":6,"last update":"Fri Sep 13 14:25:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":0.95999999999999996,"15min":0.93999999999999995},"min":{"1min":0.82799999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.3109999999999999,"5min":1.45,"15min":1.45},"last":0.95199999999999996},{"interface":"front","average":{"1min":1.095,"5min":1.032,"15min":1.0089999999999999},"min":{"1min":0.83699999999999997,"5min":0.623,"15min":0.623},"max":{"1min":1.405,"5min":2.2040000000000002,"15min":2.2040000000000002},"last":1.24}]},{"osd":7,"last update":"Fri Sep 13 14:25:19 2024","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.0880000000000001,"15min":1.0620000000000001},"min":{"1min":0.877,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.355,"5min":2.1200000000000001,"15min":2.1200000000000001},"last":1.1990000000000001},{"interface":"front","average":{"1min":1.2,"5min":1.125,"15min":1.1259999999999999},"min":{"1min":0.81299999999999994,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.522,"5min":1.657,"15min":1.657},"last":1.1259999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:26:19.996 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:26:19.996 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:26:19.996 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:26:19.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:26:20.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:20 smithi099 bash[22656]: cluster 2024-09-13T14:26:19.162481+0000 mgr.y (mgr.24287) 196 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:20.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:20 smithi099 bash[22656]: audit 2024-09-13T14:26:19.179820+0000 mgr.y (mgr.24287) 197 : audit [DBG] from='client.24626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:26:20.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:20 smithi099 bash[32082]: cluster 2024-09-13T14:26:19.162481+0000 mgr.y (mgr.24287) 196 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:20.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:20 smithi099 bash[32082]: audit 2024-09-13T14:26:19.179820+0000 mgr.y (mgr.24287) 197 : audit [DBG] from='client.24626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:26:20.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:20 smithi177 bash[24494]: cluster 2024-09-13T14:26:19.162481+0000 mgr.y (mgr.24287) 196 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:20.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:20 smithi177 bash[24494]: audit 2024-09-13T14:26:19.179820+0000 mgr.y (mgr.24287) 197 : audit [DBG] from='client.24626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:26:22.423 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:22 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:22] "GET /metrics HTTP/1.1" 200 239954 "" "Prometheus/2.43.0" 2024-09-13T14:26:22.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:22 smithi099 bash[22656]: cluster 2024-09-13T14:26:21.163110+0000 mgr.y (mgr.24287) 198 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:22.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:22 smithi099 bash[22656]: audit 2024-09-13T14:26:22.156116+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:22.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:22 smithi099 bash[32082]: cluster 2024-09-13T14:26:21.163110+0000 mgr.y (mgr.24287) 198 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:22.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:22 smithi099 bash[32082]: audit 2024-09-13T14:26:22.156116+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:22.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:22 smithi177 bash[24494]: cluster 2024-09-13T14:26:21.163110+0000 mgr.y (mgr.24287) 198 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:22.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:22 smithi177 bash[24494]: audit 2024-09-13T14:26:22.156116+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:24.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:24 smithi099 bash[22656]: cluster 2024-09-13T14:26:23.164163+0000 mgr.y (mgr.24287) 199 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:24.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:24 smithi099 bash[32082]: cluster 2024-09-13T14:26:23.164163+0000 mgr.y (mgr.24287) 199 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:24.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:24 smithi177 bash[24494]: cluster 2024-09-13T14:26:23.164163+0000 mgr.y (mgr.24287) 199 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:24.784 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:25.855 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:26:25 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:26:26.506 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:26 smithi099 bash[32082]: cluster 2024-09-13T14:26:25.164844+0000 mgr.y (mgr.24287) 200 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:26.506 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:26 smithi099 bash[22656]: cluster 2024-09-13T14:26:25.164844+0000 mgr.y (mgr.24287) 200 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:26.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:26 smithi177 bash[24494]: cluster 2024-09-13T14:26:25.164844+0000 mgr.y (mgr.24287) 200 : cluster [DBG] pgmap v177: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:28.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:28 smithi099 bash[22656]: cluster 2024-09-13T14:26:27.165467+0000 mgr.y (mgr.24287) 201 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:28.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:28 smithi099 bash[32082]: cluster 2024-09-13T14:26:27.165467+0000 mgr.y (mgr.24287) 201 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:28.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:28 smithi177 bash[24494]: cluster 2024-09-13T14:26:27.165467+0000 mgr.y (mgr.24287) 201 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:28.669 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:26:28.669 INFO:teuthology.orchestra.run.smithi099.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:26:29.542 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:29 smithi099 bash[22656]: audit 2024-09-13T14:26:28.675648+0000 mon.a (mon.0) 827 : audit [DBG] from='client.? 172.21.15.99:0/4195329680' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:26:29.542 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:29 smithi099 bash[32082]: audit 2024-09-13T14:26:28.675648+0000 mon.a (mon.0) 827 : audit [DBG] from='client.? 172.21.15.99:0/4195329680' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:26:29.543 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:26:29.543 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:26:29.562 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-09-13T14:26:29.562 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T14:26:29.570 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T14:26:29.571 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:26:29.571 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-13T14:26:29.576 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T14:26:29.577 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T14:26:29.577 DEBUG:teuthology.orchestra.run.smithi099:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:26:29.577 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:26:29.578 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T14:26:29.578 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi099.front.sepia.ceph.com 2024-09-13T14:26:29.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-09-13T14:26:29.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:26:29.584 INFO:teuthology.orchestra.run.smithi099.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T14:26:29.584 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:26:29.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:29 smithi177 bash[24494]: audit 2024-09-13T14:26:28.675648+0000 mon.a (mon.0) 827 : audit [DBG] from='client.? 172.21.15.99:0/4195329680' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:26:29.632 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T14:26:29.632 DEBUG:teuthology.orchestra.run.smithi099:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T14:26:29.683 INFO:tasks.workunit:timeout=3h 2024-09-13T14:26:29.683 INFO:tasks.workunit:cleanup=True 2024-09-13T14:26:29.684 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:26:29.737 INFO:tasks.workunit.client.0.smithi099.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T14:26:30.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:30 smithi099 bash[22656]: cluster 2024-09-13T14:26:29.166575+0000 mgr.y (mgr.24287) 202 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:30.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:30 smithi099 bash[32082]: cluster 2024-09-13T14:26:29.166575+0000 mgr.y (mgr.24287) 202 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:30.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:30 smithi177 bash[24494]: cluster 2024-09-13T14:26:29.166575+0000 mgr.y (mgr.24287) 202 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:32.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:32] "GET /metrics HTTP/1.1" 200 239959 "" "Prometheus/2.43.0" 2024-09-13T14:26:32.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[22656]: cluster 2024-09-13T14:26:31.167121+0000 mgr.y (mgr.24287) 203 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:32.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[22656]: audit 2024-09-13T14:26:31.358542+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:32.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[22656]: audit 2024-09-13T14:26:31.831226+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:32.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[22656]: audit 2024-09-13T14:26:31.832632+0000 mon.c (mon.2) 121 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:32.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[22656]: audit 2024-09-13T14:26:31.839577+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:32.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[32082]: cluster 2024-09-13T14:26:31.167121+0000 mgr.y (mgr.24287) 203 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:32.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[32082]: audit 2024-09-13T14:26:31.358542+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:32.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[32082]: audit 2024-09-13T14:26:31.831226+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:32.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[32082]: audit 2024-09-13T14:26:31.832632+0000 mon.c (mon.2) 121 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:32.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:32 smithi099 bash[32082]: audit 2024-09-13T14:26:31.839577+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:32.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:32 smithi177 bash[24494]: cluster 2024-09-13T14:26:31.167121+0000 mgr.y (mgr.24287) 203 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:32.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:32 smithi177 bash[24494]: audit 2024-09-13T14:26:31.358542+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:32.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:32 smithi177 bash[24494]: audit 2024-09-13T14:26:31.831226+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:32.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:32 smithi177 bash[24494]: audit 2024-09-13T14:26:31.832632+0000 mon.c (mon.2) 121 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:32.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:32 smithi177 bash[24494]: audit 2024-09-13T14:26:31.839577+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:34.358 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:34.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:34 smithi099 bash[22656]: cluster 2024-09-13T14:26:33.168126+0000 mgr.y (mgr.24287) 204 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:34.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:34 smithi099 bash[32082]: cluster 2024-09-13T14:26:33.168126+0000 mgr.y (mgr.24287) 204 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:34.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:34 smithi177 bash[24494]: cluster 2024-09-13T14:26:33.168126+0000 mgr.y (mgr.24287) 204 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:35.000 INFO:tasks.workunit.client.0.smithi099.stderr:Updating files: 99% (11602/11629) Updating files: 100% (11629/11629) Updating files: 100% (11629/11629), done. 2024-09-13T14:26:35.018 DEBUG:teuthology.orchestra.run.smithi099:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T14:26:35.064 INFO:tasks.workunit.client.0.smithi099.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T14:26:35.067 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:26:35.067 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T14:26:35.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:26:35 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:26:36.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:36 smithi099 bash[22656]: cluster 2024-09-13T14:26:35.168575+0000 mgr.y (mgr.24287) 205 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:36.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:36 smithi099 bash[32082]: cluster 2024-09-13T14:26:35.168575+0000 mgr.y (mgr.24287) 205 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:36.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:36 smithi177 bash[24494]: cluster 2024-09-13T14:26:35.168575+0000 mgr.y (mgr.24287) 205 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:37.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:37 smithi099 bash[22656]: audit 2024-09-13T14:26:37.156432+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:37.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:37 smithi099 bash[32082]: audit 2024-09-13T14:26:37.156432+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:37.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:37 smithi177 bash[24494]: audit 2024-09-13T14:26:37.156432+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:38.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:38 smithi099 bash[22656]: cluster 2024-09-13T14:26:37.175861+0000 mgr.y (mgr.24287) 206 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:38.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:38 smithi099 bash[32082]: cluster 2024-09-13T14:26:37.175861+0000 mgr.y (mgr.24287) 206 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:38.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:38 smithi177 bash[24494]: cluster 2024-09-13T14:26:37.175861+0000 mgr.y (mgr.24287) 206 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:40.022 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T14:26:40.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:40 smithi099 bash[22656]: cluster 2024-09-13T14:26:39.176888+0000 mgr.y (mgr.24287) 207 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:40.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:40 smithi099 bash[32082]: cluster 2024-09-13T14:26:39.176888+0000 mgr.y (mgr.24287) 207 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:40.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:40 smithi177 bash[24494]: cluster 2024-09-13T14:26:39.176888+0000 mgr.y (mgr.24287) 207 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:40.722 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T14:26:40.789 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T14:26:40.791 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:26:40.791 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T14:26:40.913 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T14:26:40.917 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T14:26:40.918 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T14:26:40.964 INFO:tasks.workunit:Running workunits matching rbd/test_librbd.sh on client.0... 2024-09-13T14:26:40.965 INFO:tasks.workunit:Running workunit rbd/test_librbd.sh... 2024-09-13T14:26:40.966 DEBUG:teuthology.orchestra.run.smithi099:workunit test rbd/test_librbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh 2024-09-13T14:26:41.159 INFO:tasks.workunit.client.0.smithi099.stderr:warning: failed to enable lockdep 2024-09-13T14:26:41.160 INFO:tasks.workunit.client.0.smithi099.stdout:seed 69110 2024-09-13T14:26:41.160 INFO:tasks.workunit.client.0.smithi099.stdout:[==========] Running 406 tests from 15 test suites. 2024-09-13T14:26:41.160 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] Global test environment set-up. 2024-09-13T14:26:41.160 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 16 tests from TestJournalReplay 2024-09-13T14:26:42.010 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:41 smithi099 bash[22909]: [13/Sep/2024:14:26:41] ENGINE Bus STOPPING 2024-09-13T14:26:42.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:42] "GET /metrics HTTP/1.1" 200 239935 "" "Prometheus/2.43.0" 2024-09-13T14:26:42.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22909]: [13/Sep/2024:14:26:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:26:42.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22909]: [13/Sep/2024:14:26:42] ENGINE Bus STOPPED 2024-09-13T14:26:42.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22909]: [13/Sep/2024:14:26:42] ENGINE Bus STARTING 2024-09-13T14:26:42.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22909]: [13/Sep/2024:14:26:42] ENGINE Serving on http://:::9283 2024-09-13T14:26:42.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22909]: [13/Sep/2024:14:26:42] ENGINE Bus STARTED 2024-09-13T14:26:42.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T14:26:42.693 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.AioDiscardEvent 2024-09-13T14:26:43.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22656]: cluster 2024-09-13T14:26:41.177322+0000 mgr.y (mgr.24287) 208 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:43.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22656]: cluster 2024-09-13T14:26:41.684735+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:26:43.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22656]: audit 2024-09-13T14:26:41.687460+0000 mon.a (mon.0) 830 : audit [INF] from='client.? 172.21.15.99:0/3236449265' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:43.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22656]: audit 2024-09-13T14:26:41.702687+0000 mon.a (mon.0) 831 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T14:26:43.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[22656]: audit 2024-09-13T14:26:41.713027+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:43.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[32082]: cluster 2024-09-13T14:26:41.177322+0000 mgr.y (mgr.24287) 208 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:43.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[32082]: cluster 2024-09-13T14:26:41.684735+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:26:43.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[32082]: audit 2024-09-13T14:26:41.687460+0000 mon.a (mon.0) 830 : audit [INF] from='client.? 172.21.15.99:0/3236449265' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:43.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[32082]: audit 2024-09-13T14:26:41.702687+0000 mon.a (mon.0) 831 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T14:26:43.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:42 smithi099 bash[32082]: audit 2024-09-13T14:26:41.713027+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:43.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:42 smithi177 bash[24494]: cluster 2024-09-13T14:26:41.177322+0000 mgr.y (mgr.24287) 208 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:43.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:42 smithi177 bash[24494]: cluster 2024-09-13T14:26:41.684735+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:26:43.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:42 smithi177 bash[24494]: audit 2024-09-13T14:26:41.687460+0000 mon.a (mon.0) 830 : audit [INF] from='client.? 172.21.15.99:0/3236449265' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:43.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:42 smithi177 bash[24494]: audit 2024-09-13T14:26:41.702687+0000 mon.a (mon.0) 831 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T14:26:43.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:42 smithi177 bash[24494]: audit 2024-09-13T14:26:41.713027+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:44.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:43 smithi099 bash[22656]: audit 2024-09-13T14:26:42.698076+0000 mon.a (mon.0) 832 : audit [INF] from='client.? 172.21.15.99:0/3236449265' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:44.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:43 smithi099 bash[22656]: cluster 2024-09-13T14:26:42.698957+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:26:44.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:43 smithi099 bash[32082]: audit 2024-09-13T14:26:42.698076+0000 mon.a (mon.0) 832 : audit [INF] from='client.? 172.21.15.99:0/3236449265' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:44.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:43 smithi099 bash[32082]: cluster 2024-09-13T14:26:42.698957+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:26:44.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:43 smithi177 bash[24494]: audit 2024-09-13T14:26:42.698076+0000 mon.a (mon.0) 832 : audit [INF] from='client.? 172.21.15.99:0/3236449265' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:44.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:43 smithi177 bash[24494]: cluster 2024-09-13T14:26:42.698957+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:26:44.744 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.744 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.AioDiscardEvent (2051 ms) 2024-09-13T14:26:44.744 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.AioWriteEvent 2024-09-13T14:26:44.766 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.766 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.AioWriteEvent (22 ms) 2024-09-13T14:26:44.766 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.AioFlushEvent 2024-09-13T14:26:44.788 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.788 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.AioFlushEvent (22 ms) 2024-09-13T14:26:44.788 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.SnapCreate 2024-09-13T14:26:44.810 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.810 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.SnapCreate (22 ms) 2024-09-13T14:26:44.810 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.SnapProtect 2024-09-13T14:26:44.832 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.832 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.SnapProtect (22 ms) 2024-09-13T14:26:44.832 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.SnapUnprotect 2024-09-13T14:26:44.852 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.853 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.SnapUnprotect (20 ms) 2024-09-13T14:26:44.853 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.SnapRename 2024-09-13T14:26:44.873 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.873 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.SnapRename (21 ms) 2024-09-13T14:26:44.873 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.SnapRollback 2024-09-13T14:26:44.893 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.893 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.SnapRollback (20 ms) 2024-09-13T14:26:44.893 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.SnapRemove 2024-09-13T14:26:44.913 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.913 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.SnapRemove (20 ms) 2024-09-13T14:26:44.913 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.Rename 2024-09-13T14:26:44.934 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.934 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.Rename (21 ms) 2024-09-13T14:26:44.934 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.Resize 2024-09-13T14:26:44.957 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.957 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.Resize (23 ms) 2024-09-13T14:26:44.958 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.Flatten 2024-09-13T14:26:44.977 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.977 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.Flatten (20 ms) 2024-09-13T14:26:44.977 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.UpdateFeatures 2024-09-13T14:26:44.998 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:44.999 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.UpdateFeatures (22 ms) 2024-09-13T14:26:44.999 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.MetadataSet 2024-09-13T14:26:45.018 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:45.018 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.MetadataSet (19 ms) 2024-09-13T14:26:45.018 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.MetadataRemove 2024-09-13T14:26:45.039 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:45.039 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.MetadataRemove (21 ms) 2024-09-13T14:26:45.039 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalReplay.ObjectPosition 2024-09-13T14:26:45.059 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:45.060 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalReplay.ObjectPosition (21 ms) 2024-09-13T14:26:45.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:44 smithi099 bash[22656]: cluster 2024-09-13T14:26:43.178457+0000 mgr.y (mgr.24287) 209 : cluster [DBG] pgmap v188: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:45.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:44 smithi099 bash[22656]: cluster 2024-09-13T14:26:43.700358+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:26:45.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:44 smithi099 bash[32082]: cluster 2024-09-13T14:26:43.178457+0000 mgr.y (mgr.24287) 209 : cluster [DBG] pgmap v188: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:45.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:44 smithi099 bash[32082]: cluster 2024-09-13T14:26:43.700358+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:26:45.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:44 smithi177 bash[24494]: cluster 2024-09-13T14:26:43.178457+0000 mgr.y (mgr.24287) 209 : cluster [DBG] pgmap v188: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:45.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:44 smithi177 bash[24494]: cluster 2024-09-13T14:26:43.700358+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:26:45.721 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:26:45 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:26:45.729 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 16 tests from TestJournalReplay (2367 ms total) 2024-09-13T14:26:45.730 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:26:45.730 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 3 tests from TestJournalEntries 2024-09-13T14:26:46.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:45 smithi099 bash[22656]: cluster 2024-09-13T14:26:44.720083+0000 mon.a (mon.0) 835 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:26:46.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:45 smithi099 bash[32082]: cluster 2024-09-13T14:26:44.720083+0000 mon.a (mon.0) 835 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:26:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:45 smithi177 bash[24494]: cluster 2024-09-13T14:26:44.720083+0000 mon.a (mon.0) 835 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:26:46.484 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:47.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:46 smithi099 bash[22656]: cluster 2024-09-13T14:26:45.179104+0000 mgr.y (mgr.24287) 210 : cluster [DBG] pgmap v191: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:47.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:46 smithi099 bash[22656]: cluster 2024-09-13T14:26:45.731595+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:26:47.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:46 smithi099 bash[32082]: cluster 2024-09-13T14:26:45.179104+0000 mgr.y (mgr.24287) 210 : cluster [DBG] pgmap v191: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:47.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:46 smithi099 bash[32082]: cluster 2024-09-13T14:26:45.731595+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:26:47.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:46 smithi177 bash[24494]: cluster 2024-09-13T14:26:45.179104+0000 mgr.y (mgr.24287) 210 : cluster [DBG] pgmap v191: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:47.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:46 smithi177 bash[24494]: cluster 2024-09-13T14:26:45.731595+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:26:47.741 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalEntries.AioWrite 2024-09-13T14:26:48.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[22656]: cluster 2024-09-13T14:26:46.737723+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:26:48.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[22656]: audit 2024-09-13T14:26:46.739440+0000 mon.a (mon.0) 838 : audit [INF] from='client.? 172.21.15.99:0/625424632' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:48.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[22656]: audit 2024-09-13T14:26:46.997109+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[22656]: audit 2024-09-13T14:26:47.002253+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[22656]: audit 2024-09-13T14:26:47.356147+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[22656]: audit 2024-09-13T14:26:47.370739+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[22656]: audit 2024-09-13T14:26:47.374212+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[22656]: audit 2024-09-13T14:26:47.376094+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[22656]: audit 2024-09-13T14:26:47.389211+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[32082]: cluster 2024-09-13T14:26:46.737723+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:26:48.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[32082]: audit 2024-09-13T14:26:46.739440+0000 mon.a (mon.0) 838 : audit [INF] from='client.? 172.21.15.99:0/625424632' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:48.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[32082]: audit 2024-09-13T14:26:46.997109+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[32082]: audit 2024-09-13T14:26:47.002253+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[32082]: audit 2024-09-13T14:26:47.356147+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[32082]: audit 2024-09-13T14:26:47.370739+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[32082]: audit 2024-09-13T14:26:47.374212+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:48.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[32082]: audit 2024-09-13T14:26:47.376094+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:48.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:47 smithi099 bash[32082]: audit 2024-09-13T14:26:47.389211+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:47 smithi177 bash[24494]: cluster 2024-09-13T14:26:46.737723+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:26:48.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:47 smithi177 bash[24494]: audit 2024-09-13T14:26:46.739440+0000 mon.a (mon.0) 838 : audit [INF] from='client.? 172.21.15.99:0/625424632' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:48.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:47 smithi177 bash[24494]: audit 2024-09-13T14:26:46.997109+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:47 smithi177 bash[24494]: audit 2024-09-13T14:26:47.002253+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:47 smithi177 bash[24494]: audit 2024-09-13T14:26:47.356147+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:47 smithi177 bash[24494]: audit 2024-09-13T14:26:47.370739+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:47 smithi177 bash[24494]: audit 2024-09-13T14:26:47.374212+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:48.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:47 smithi177 bash[24494]: audit 2024-09-13T14:26:47.376094+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:48.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:47 smithi177 bash[24494]: audit 2024-09-13T14:26:47.389211+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:26:48.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:48 smithi099 bash[22656]: cluster 2024-09-13T14:26:47.179782+0000 mgr.y (mgr.24287) 211 : cluster [DBG] pgmap v194: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:48.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:48 smithi099 bash[22656]: audit 2024-09-13T14:26:47.744648+0000 mon.a (mon.0) 844 : audit [INF] from='client.? 172.21.15.99:0/625424632' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:48.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:48 smithi099 bash[22656]: cluster 2024-09-13T14:26:47.745065+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:26:48.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:48 smithi099 bash[22656]: cluster 2024-09-13T14:26:48.000708+0000 mon.a (mon.0) 846 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:48.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:48 smithi099 bash[32082]: cluster 2024-09-13T14:26:47.179782+0000 mgr.y (mgr.24287) 211 : cluster [DBG] pgmap v194: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:48.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:48 smithi099 bash[32082]: audit 2024-09-13T14:26:47.744648+0000 mon.a (mon.0) 844 : audit [INF] from='client.? 172.21.15.99:0/625424632' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:48.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:48 smithi099 bash[32082]: cluster 2024-09-13T14:26:47.745065+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:26:48.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:48 smithi099 bash[32082]: cluster 2024-09-13T14:26:48.000708+0000 mon.a (mon.0) 846 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:49.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:48 smithi177 bash[24494]: cluster 2024-09-13T14:26:47.179782+0000 mgr.y (mgr.24287) 211 : cluster [DBG] pgmap v194: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:48 smithi177 bash[24494]: audit 2024-09-13T14:26:47.744648+0000 mon.a (mon.0) 844 : audit [INF] from='client.? 172.21.15.99:0/625424632' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:48 smithi177 bash[24494]: cluster 2024-09-13T14:26:47.745065+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:26:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:48 smithi177 bash[24494]: cluster 2024-09-13T14:26:48.000708+0000 mon.a (mon.0) 846 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:49.807 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:49.807 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalEntries.AioWrite (2066 ms) 2024-09-13T14:26:49.807 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalEntries.AioDiscard 2024-09-13T14:26:49.828 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:49.828 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalEntries.AioDiscard (21 ms) 2024-09-13T14:26:49.829 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestJournalEntries.AioFlush 2024-09-13T14:26:49.852 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:49.852 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestJournalEntries.AioFlush (23 ms) 2024-09-13T14:26:50.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:49 smithi099 bash[22656]: cluster 2024-09-13T14:26:48.761205+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:26:50.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:49 smithi099 bash[32082]: cluster 2024-09-13T14:26:48.761205+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:26:50.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:49 smithi177 bash[24494]: cluster 2024-09-13T14:26:48.761205+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:26:50.777 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 3 tests from TestJournalEntries (2110 ms total) 2024-09-13T14:26:50.777 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:26:50.777 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 2 tests from TestTrash 2024-09-13T14:26:51.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:50 smithi099 bash[22656]: cluster 2024-09-13T14:26:49.180282+0000 mgr.y (mgr.24287) 212 : cluster [DBG] pgmap v197: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:51.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:50 smithi099 bash[22656]: cluster 2024-09-13T14:26:49.763610+0000 mon.a (mon.0) 848 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:26:51.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:50 smithi099 bash[22656]: cluster 2024-09-13T14:26:49.763664+0000 mon.a (mon.0) 849 : cluster [INF] Cluster is now healthy 2024-09-13T14:26:51.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:50 smithi099 bash[22656]: cluster 2024-09-13T14:26:49.781211+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:26:51.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:50 smithi099 bash[32082]: cluster 2024-09-13T14:26:49.180282+0000 mgr.y (mgr.24287) 212 : cluster [DBG] pgmap v197: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:51.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:50 smithi099 bash[32082]: cluster 2024-09-13T14:26:49.763610+0000 mon.a (mon.0) 848 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:26:51.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:50 smithi099 bash[32082]: cluster 2024-09-13T14:26:49.763664+0000 mon.a (mon.0) 849 : cluster [INF] Cluster is now healthy 2024-09-13T14:26:51.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:50 smithi099 bash[32082]: cluster 2024-09-13T14:26:49.781211+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:26:51.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:50 smithi177 bash[24494]: cluster 2024-09-13T14:26:49.180282+0000 mgr.y (mgr.24287) 212 : cluster [DBG] pgmap v197: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:51.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:50 smithi177 bash[24494]: cluster 2024-09-13T14:26:49.763610+0000 mon.a (mon.0) 848 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:26:51.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:50 smithi177 bash[24494]: cluster 2024-09-13T14:26:49.763664+0000 mon.a (mon.0) 849 : cluster [INF] Cluster is now healthy 2024-09-13T14:26:51.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:50 smithi177 bash[24494]: cluster 2024-09-13T14:26:49.781211+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:26:52.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:51 smithi099 bash[22656]: cluster 2024-09-13T14:26:50.780674+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:26:52.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:51 smithi099 bash[32082]: cluster 2024-09-13T14:26:50.780674+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:26:52.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:51 smithi177 bash[24494]: cluster 2024-09-13T14:26:50.780674+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:26:52.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T14:26:52.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:52] "GET /metrics HTTP/1.1" 200 239935 "" "Prometheus/2.43.0" 2024-09-13T14:26:52.818 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestTrash.UserRemovingSource 2024-09-13T14:26:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[22656]: cluster 2024-09-13T14:26:51.180774+0000 mgr.y (mgr.24287) 213 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[22656]: cluster 2024-09-13T14:26:51.805318+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:26:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[22656]: audit 2024-09-13T14:26:51.807635+0000 mon.c (mon.2) 126 : audit [INF] from='client.? 172.21.15.99:0/2264872553' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[22656]: audit 2024-09-13T14:26:51.811622+0000 mon.a (mon.0) 853 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[22656]: audit 2024-09-13T14:26:52.156796+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:53.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[32082]: cluster 2024-09-13T14:26:51.180774+0000 mgr.y (mgr.24287) 213 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:53.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[32082]: cluster 2024-09-13T14:26:51.805318+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:26:53.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[32082]: audit 2024-09-13T14:26:51.807635+0000 mon.c (mon.2) 126 : audit [INF] from='client.? 172.21.15.99:0/2264872553' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:53.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[32082]: audit 2024-09-13T14:26:51.811622+0000 mon.a (mon.0) 853 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:53.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:52 smithi099 bash[32082]: audit 2024-09-13T14:26:52.156796+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:53.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:52 smithi177 bash[24494]: cluster 2024-09-13T14:26:51.180774+0000 mgr.y (mgr.24287) 213 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:53.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:52 smithi177 bash[24494]: cluster 2024-09-13T14:26:51.805318+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:26:53.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:52 smithi177 bash[24494]: audit 2024-09-13T14:26:51.807635+0000 mon.c (mon.2) 126 : audit [INF] from='client.? 172.21.15.99:0/2264872553' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:53.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:52 smithi177 bash[24494]: audit 2024-09-13T14:26:51.811622+0000 mon.a (mon.0) 853 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:53.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:52 smithi177 bash[24494]: audit 2024-09-13T14:26:52.156796+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:54.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:53 smithi177 bash[24494]: audit 2024-09-13T14:26:52.806324+0000 mon.a (mon.0) 854 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:54.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:53 smithi177 bash[24494]: cluster 2024-09-13T14:26:52.807749+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:26:54.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:53 smithi099 bash[22656]: audit 2024-09-13T14:26:52.806324+0000 mon.a (mon.0) 854 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:54.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:53 smithi099 bash[22656]: cluster 2024-09-13T14:26:52.807749+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:26:54.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:53 smithi099 bash[32082]: audit 2024-09-13T14:26:52.806324+0000 mon.a (mon.0) 854 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:54.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:53 smithi099 bash[32082]: cluster 2024-09-13T14:26:52.807749+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:26:54.996 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestTrash.UserRemovingSource (2178 ms) 2024-09-13T14:26:54.996 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestTrash.RestoreMirroringSource 2024-09-13T14:26:55.084 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestTrash.RestoreMirroringSource (88 ms) 2024-09-13T14:26:55.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:54 smithi177 bash[24494]: cluster 2024-09-13T14:26:53.181981+0000 mgr.y (mgr.24287) 214 : cluster [DBG] pgmap v203: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:55.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:54 smithi177 bash[24494]: cluster 2024-09-13T14:26:53.828720+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:26:55.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:54 smithi099 bash[22656]: cluster 2024-09-13T14:26:53.181981+0000 mgr.y (mgr.24287) 214 : cluster [DBG] pgmap v203: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:55.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:54 smithi099 bash[22656]: cluster 2024-09-13T14:26:53.828720+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:26:55.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:54 smithi099 bash[32082]: cluster 2024-09-13T14:26:53.181981+0000 mgr.y (mgr.24287) 214 : cluster [DBG] pgmap v203: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:55.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:54 smithi099 bash[32082]: cluster 2024-09-13T14:26:53.828720+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:26:55.851 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:26:55 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:26:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:26:55.865 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 2 tests from TestTrash (2266 ms total) 2024-09-13T14:26:55.865 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:26:55.865 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 1 test from TestOperations 2024-09-13T14:26:56.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:55 smithi177 bash[24494]: cluster 2024-09-13T14:26:54.852053+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:26:56.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:55 smithi099 bash[22656]: cluster 2024-09-13T14:26:54.852053+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:26:56.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:55 smithi099 bash[32082]: cluster 2024-09-13T14:26:54.852053+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:26:56.930 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:57.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:56 smithi177 bash[24494]: cluster 2024-09-13T14:26:55.182682+0000 mgr.y (mgr.24287) 215 : cluster [DBG] pgmap v206: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:57.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:56 smithi177 bash[24494]: cluster 2024-09-13T14:26:55.863703+0000 mon.a (mon.0) 858 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:26:57.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:56 smithi099 bash[22656]: cluster 2024-09-13T14:26:55.182682+0000 mgr.y (mgr.24287) 215 : cluster [DBG] pgmap v206: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:57.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:56 smithi099 bash[22656]: cluster 2024-09-13T14:26:55.863703+0000 mon.a (mon.0) 858 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:26:57.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:56 smithi099 bash[32082]: cluster 2024-09-13T14:26:55.182682+0000 mgr.y (mgr.24287) 215 : cluster [DBG] pgmap v206: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:57.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:56 smithi099 bash[32082]: cluster 2024-09-13T14:26:55.863703+0000 mon.a (mon.0) 858 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:26:57.882 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestOperations.DisableJournalingCorrupt 2024-09-13T14:26:58.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:57 smithi099 bash[22656]: cluster 2024-09-13T14:26:56.875884+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-09-13T14:26:58.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:57 smithi099 bash[22656]: audit 2024-09-13T14:26:56.878219+0000 mon.a (mon.0) 860 : audit [INF] from='client.? 172.21.15.99:0/1831023089' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:58.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:57 smithi099 bash[32082]: cluster 2024-09-13T14:26:56.875884+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-09-13T14:26:58.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:57 smithi099 bash[32082]: audit 2024-09-13T14:26:56.878219+0000 mon.a (mon.0) 860 : audit [INF] from='client.? 172.21.15.99:0/1831023089' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:58.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:57 smithi177 bash[24494]: cluster 2024-09-13T14:26:56.875884+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-09-13T14:26:58.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:57 smithi177 bash[24494]: audit 2024-09-13T14:26:56.878219+0000 mon.a (mon.0) 860 : audit [INF] from='client.? 172.21.15.99:0/1831023089' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:59.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:58 smithi099 bash[22656]: cluster 2024-09-13T14:26:57.183394+0000 mgr.y (mgr.24287) 216 : cluster [DBG] pgmap v209: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:59.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:58 smithi099 bash[22656]: cluster 2024-09-13T14:26:57.869382+0000 mon.a (mon.0) 861 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:59.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:58 smithi099 bash[22656]: audit 2024-09-13T14:26:57.886486+0000 mon.a (mon.0) 862 : audit [INF] from='client.? 172.21.15.99:0/1831023089' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:59.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:58 smithi099 bash[22656]: cluster 2024-09-13T14:26:57.886948+0000 mon.a (mon.0) 863 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:26:59.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:58 smithi099 bash[32082]: cluster 2024-09-13T14:26:57.183394+0000 mgr.y (mgr.24287) 216 : cluster [DBG] pgmap v209: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:59.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:58 smithi099 bash[32082]: cluster 2024-09-13T14:26:57.869382+0000 mon.a (mon.0) 861 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:59.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:58 smithi099 bash[32082]: audit 2024-09-13T14:26:57.886486+0000 mon.a (mon.0) 862 : audit [INF] from='client.? 172.21.15.99:0/1831023089' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:59.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:58 smithi099 bash[32082]: cluster 2024-09-13T14:26:57.886948+0000 mon.a (mon.0) 863 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:26:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:58 smithi177 bash[24494]: cluster 2024-09-13T14:26:57.183394+0000 mgr.y (mgr.24287) 216 : cluster [DBG] pgmap v209: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:26:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:58 smithi177 bash[24494]: cluster 2024-09-13T14:26:57.869382+0000 mon.a (mon.0) 861 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:58 smithi177 bash[24494]: audit 2024-09-13T14:26:57.886486+0000 mon.a (mon.0) 862 : audit [INF] from='client.? 172.21.15.99:0/1831023089' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:58 smithi177 bash[24494]: cluster 2024-09-13T14:26:57.886948+0000 mon.a (mon.0) 863 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:26:59.935 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:26:59.935 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestOperations.DisableJournalingCorrupt (2053 ms) 2024-09-13T14:27:00.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:26:59 smithi099 bash[22656]: cluster 2024-09-13T14:26:58.901280+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:27:00.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:26:59 smithi099 bash[32082]: cluster 2024-09-13T14:26:58.901280+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:27:00.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:26:59 smithi177 bash[24494]: cluster 2024-09-13T14:26:58.901280+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:27:00.915 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 1 test from TestOperations (2053 ms total) 2024-09-13T14:27:00.915 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:27:00.915 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 4 tests from TestObjectMap 2024-09-13T14:27:01.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:00 smithi099 bash[22656]: cluster 2024-09-13T14:26:59.184046+0000 mgr.y (mgr.24287) 217 : cluster [DBG] pgmap v212: 105 pgs: 11 creating+activating, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:01.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:00 smithi099 bash[22656]: cluster 2024-09-13T14:26:59.892348+0000 mon.a (mon.0) 865 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:27:01.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:00 smithi099 bash[22656]: cluster 2024-09-13T14:26:59.892392+0000 mon.a (mon.0) 866 : cluster [INF] Cluster is now healthy 2024-09-13T14:27:01.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:00 smithi099 bash[22656]: cluster 2024-09-13T14:26:59.909024+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:27:01.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:00 smithi099 bash[32082]: cluster 2024-09-13T14:26:59.184046+0000 mgr.y (mgr.24287) 217 : cluster [DBG] pgmap v212: 105 pgs: 11 creating+activating, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:01.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:00 smithi099 bash[32082]: cluster 2024-09-13T14:26:59.892348+0000 mon.a (mon.0) 865 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:27:01.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:00 smithi099 bash[32082]: cluster 2024-09-13T14:26:59.892392+0000 mon.a (mon.0) 866 : cluster [INF] Cluster is now healthy 2024-09-13T14:27:01.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:00 smithi099 bash[32082]: cluster 2024-09-13T14:26:59.909024+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:27:01.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:00 smithi177 bash[24494]: cluster 2024-09-13T14:26:59.184046+0000 mgr.y (mgr.24287) 217 : cluster [DBG] pgmap v212: 105 pgs: 11 creating+activating, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:01.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:00 smithi177 bash[24494]: cluster 2024-09-13T14:26:59.892348+0000 mon.a (mon.0) 865 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:27:01.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:00 smithi177 bash[24494]: cluster 2024-09-13T14:26:59.892392+0000 mon.a (mon.0) 866 : cluster [INF] Cluster is now healthy 2024-09-13T14:27:01.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:00 smithi177 bash[24494]: cluster 2024-09-13T14:26:59.909024+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:27:01.591 INFO:teuthology.orchestra.run.smithi099.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:02.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:27:02 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:02] "GET /metrics HTTP/1.1" 200 242639 "" "Prometheus/2.43.0" 2024-09-13T14:27:02.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[22656]: cluster 2024-09-13T14:27:00.910275+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:27:02.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[22656]: cluster 2024-09-13T14:27:01.184580+0000 mgr.y (mgr.24287) 218 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:02.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[22656]: audit 2024-09-13T14:27:01.588577+0000 mgr.y (mgr.24287) 219 : audit [DBG] from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:02.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[22656]: cephadm 2024-09-13T14:27:01.589184+0000 mgr.y (mgr.24287) 220 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:02.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[22656]: audit 2024-09-13T14:27:01.598013+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:02.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[22656]: audit 2024-09-13T14:27:01.599410+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:02.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[22656]: audit 2024-09-13T14:27:01.601614+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:02.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[22656]: audit 2024-09-13T14:27:01.602544+0000 mon.c (mon.2) 130 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:02.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[22656]: audit 2024-09-13T14:27:01.609371+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:02.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[22656]: cephadm 2024-09-13T14:27:01.645098+0000 mgr.y (mgr.24287) 221 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[32082]: cluster 2024-09-13T14:27:00.910275+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:27:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[32082]: cluster 2024-09-13T14:27:01.184580+0000 mgr.y (mgr.24287) 218 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[32082]: audit 2024-09-13T14:27:01.588577+0000 mgr.y (mgr.24287) 219 : audit [DBG] from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[32082]: cephadm 2024-09-13T14:27:01.589184+0000 mgr.y (mgr.24287) 220 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[32082]: audit 2024-09-13T14:27:01.598013+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[32082]: audit 2024-09-13T14:27:01.599410+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[32082]: audit 2024-09-13T14:27:01.601614+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[32082]: audit 2024-09-13T14:27:01.602544+0000 mon.c (mon.2) 130 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[32082]: audit 2024-09-13T14:27:01.609371+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:01 smithi099 bash[32082]: cephadm 2024-09-13T14:27:01.645098+0000 mgr.y (mgr.24287) 221 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:02.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:01 smithi177 bash[24494]: cluster 2024-09-13T14:27:00.910275+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:27:02.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:01 smithi177 bash[24494]: cluster 2024-09-13T14:27:01.184580+0000 mgr.y (mgr.24287) 218 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:02.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:01 smithi177 bash[24494]: audit 2024-09-13T14:27:01.588577+0000 mgr.y (mgr.24287) 219 : audit [DBG] from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:02.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:01 smithi177 bash[24494]: cephadm 2024-09-13T14:27:01.589184+0000 mgr.y (mgr.24287) 220 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:02.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:01 smithi177 bash[24494]: audit 2024-09-13T14:27:01.598013+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:02.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:01 smithi177 bash[24494]: audit 2024-09-13T14:27:01.599410+0000 mon.c (mon.2) 128 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:02.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:01 smithi177 bash[24494]: audit 2024-09-13T14:27:01.601614+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:02.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:01 smithi177 bash[24494]: audit 2024-09-13T14:27:01.602544+0000 mon.c (mon.2) 130 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:02.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:01 smithi177 bash[24494]: audit 2024-09-13T14:27:01.609371+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:02.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:01 smithi177 bash[24494]: cephadm 2024-09-13T14:27:01.645098+0000 mgr.y (mgr.24287) 221 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:02.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:27:02.940 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenCorrupt 2024-09-13T14:27:03.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:02 smithi099 bash[32082]: cluster 2024-09-13T14:27:01.933173+0000 mon.a (mon.0) 871 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:27:03.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:02 smithi099 bash[32082]: audit 2024-09-13T14:27:01.936390+0000 mon.a (mon.0) 872 : audit [INF] from='client.? 172.21.15.99:0/4175870544' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:27:03.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:02 smithi099 bash[22656]: cluster 2024-09-13T14:27:01.933173+0000 mon.a (mon.0) 871 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:27:03.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:02 smithi099 bash[22656]: audit 2024-09-13T14:27:01.936390+0000 mon.a (mon.0) 872 : audit [INF] from='client.? 172.21.15.99:0/4175870544' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:27:03.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:02 smithi177 bash[24494]: cluster 2024-09-13T14:27:01.933173+0000 mon.a (mon.0) 871 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:27:03.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:02 smithi177 bash[24494]: audit 2024-09-13T14:27:01.936390+0000 mon.a (mon.0) 872 : audit [INF] from='client.? 172.21.15.99:0/4175870544' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:27:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:03 smithi099 bash[22656]: audit 2024-09-13T14:27:02.935133+0000 mon.a (mon.0) 873 : audit [INF] from='client.? 172.21.15.99:0/4175870544' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:27:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:03 smithi099 bash[22656]: cluster 2024-09-13T14:27:02.935348+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:27:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:03 smithi099 bash[22656]: cluster 2024-09-13T14:27:03.185850+0000 mgr.y (mgr.24287) 222 : cluster [DBG] pgmap v218: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:03 smithi099 bash[22656]: cluster 2024-09-13T14:27:03.938706+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:27:04.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:03 smithi099 bash[32082]: audit 2024-09-13T14:27:02.935133+0000 mon.a (mon.0) 873 : audit [INF] from='client.? 172.21.15.99:0/4175870544' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:27:04.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:03 smithi099 bash[32082]: cluster 2024-09-13T14:27:02.935348+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:27:04.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:03 smithi099 bash[32082]: cluster 2024-09-13T14:27:03.185850+0000 mgr.y (mgr.24287) 222 : cluster [DBG] pgmap v218: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:04.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:03 smithi099 bash[32082]: cluster 2024-09-13T14:27:03.938706+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:27:04.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:03 smithi177 bash[24494]: audit 2024-09-13T14:27:02.935133+0000 mon.a (mon.0) 873 : audit [INF] from='client.? 172.21.15.99:0/4175870544' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:27:04.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:03 smithi177 bash[24494]: cluster 2024-09-13T14:27:02.935348+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:27:04.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:03 smithi177 bash[24494]: cluster 2024-09-13T14:27:03.185850+0000 mgr.y (mgr.24287) 222 : cluster [DBG] pgmap v218: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:04.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:03 smithi177 bash[24494]: cluster 2024-09-13T14:27:03.938706+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:27:05.017 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenCorrupt (2078 ms) 2024-09-13T14:27:05.018 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenTooSmall 2024-09-13T14:27:05.071 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenTooSmall (53 ms) 2024-09-13T14:27:05.071 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestObjectMap.InvalidateFlagOnDisk 2024-09-13T14:27:05.136 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestObjectMap.InvalidateFlagOnDisk (65 ms) 2024-09-13T14:27:05.136 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall 2024-09-13T14:27:05.187 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall (51 ms) 2024-09-13T14:27:05.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:27:05 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:05.971 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 4 tests from TestObjectMap (2247 ms total) 2024-09-13T14:27:05.971 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:27:05.971 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 2 tests from TestMirroringWatcher 2024-09-13T14:27:06.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:05 smithi099 bash[22656]: cluster 2024-09-13T14:27:04.954105+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-09-13T14:27:06.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:05 smithi099 bash[22656]: cluster 2024-09-13T14:27:05.186427+0000 mgr.y (mgr.24287) 223 : cluster [DBG] pgmap v221: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:05 smithi099 bash[32082]: cluster 2024-09-13T14:27:04.954105+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-09-13T14:27:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:05 smithi099 bash[32082]: cluster 2024-09-13T14:27:05.186427+0000 mgr.y (mgr.24287) 223 : cluster [DBG] pgmap v221: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:06.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:05 smithi177 bash[24494]: cluster 2024-09-13T14:27:04.954105+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-09-13T14:27:06.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:05 smithi177 bash[24494]: cluster 2024-09-13T14:27:05.186427+0000 mgr.y (mgr.24287) 223 : cluster [DBG] pgmap v221: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:07.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:06 smithi099 bash[22656]: cluster 2024-09-13T14:27:05.972327+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-09-13T14:27:07.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:06 smithi099 bash[32082]: cluster 2024-09-13T14:27:05.972327+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-09-13T14:27:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:06 smithi177 bash[24494]: cluster 2024-09-13T14:27:05.972327+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-09-13T14:27:07.459 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:07.981 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroringWatcher.ModeUpdated 2024-09-13T14:27:08.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:08 smithi099 bash[22656]: cluster 2024-09-13T14:27:06.971651+0000 mon.a (mon.0) 878 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-09-13T14:27:08.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:08 smithi099 bash[22656]: audit 2024-09-13T14:27:06.977180+0000 mon.a (mon.0) 879 : audit [INF] from='client.? 172.21.15.99:0/817353130' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:27:08.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:08 smithi099 bash[22656]: audit 2024-09-13T14:27:07.157123+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:08.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:08 smithi099 bash[22656]: cluster 2024-09-13T14:27:07.187192+0000 mgr.y (mgr.24287) 224 : cluster [DBG] pgmap v224: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:08.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:08 smithi099 bash[32082]: cluster 2024-09-13T14:27:06.971651+0000 mon.a (mon.0) 878 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-09-13T14:27:08.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:08 smithi099 bash[32082]: audit 2024-09-13T14:27:06.977180+0000 mon.a (mon.0) 879 : audit [INF] from='client.? 172.21.15.99:0/817353130' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:27:08.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:08 smithi099 bash[32082]: audit 2024-09-13T14:27:07.157123+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:08.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:08 smithi099 bash[32082]: cluster 2024-09-13T14:27:07.187192+0000 mgr.y (mgr.24287) 224 : cluster [DBG] pgmap v224: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:08.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:08 smithi177 bash[24494]: cluster 2024-09-13T14:27:06.971651+0000 mon.a (mon.0) 878 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-09-13T14:27:08.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:08 smithi177 bash[24494]: audit 2024-09-13T14:27:06.977180+0000 mon.a (mon.0) 879 : audit [INF] from='client.? 172.21.15.99:0/817353130' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:27:08.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:08 smithi177 bash[24494]: audit 2024-09-13T14:27:07.157123+0000 mon.c (mon.2) 131 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:08.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:08 smithi177 bash[24494]: cluster 2024-09-13T14:27:07.187192+0000 mgr.y (mgr.24287) 224 : cluster [DBG] pgmap v224: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:09.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:09 smithi099 bash[32082]: audit 2024-09-13T14:27:07.982736+0000 mon.a (mon.0) 880 : audit [INF] from='client.? 172.21.15.99:0/817353130' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:27:09.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:09 smithi099 bash[32082]: cluster 2024-09-13T14:27:07.984710+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:27:09.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:09 smithi099 bash[32082]: cluster 2024-09-13T14:27:07.985547+0000 mon.a (mon.0) 882 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:27:09.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:09 smithi099 bash[32082]: cluster 2024-09-13T14:27:08.980922+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:27:09.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:09 smithi099 bash[22656]: audit 2024-09-13T14:27:07.982736+0000 mon.a (mon.0) 880 : audit [INF] from='client.? 172.21.15.99:0/817353130' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:27:09.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:09 smithi099 bash[22656]: cluster 2024-09-13T14:27:07.984710+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:27:09.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:09 smithi099 bash[22656]: cluster 2024-09-13T14:27:07.985547+0000 mon.a (mon.0) 882 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:27:09.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:09 smithi099 bash[22656]: cluster 2024-09-13T14:27:08.980922+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:27:09.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:09 smithi177 bash[24494]: audit 2024-09-13T14:27:07.982736+0000 mon.a (mon.0) 880 : audit [INF] from='client.? 172.21.15.99:0/817353130' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:27:09.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:09 smithi177 bash[24494]: cluster 2024-09-13T14:27:07.984710+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:27:09.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:09 smithi177 bash[24494]: cluster 2024-09-13T14:27:07.985547+0000 mon.a (mon.0) 882 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:27:09.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:09 smithi177 bash[24494]: cluster 2024-09-13T14:27:08.980922+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:27:10.032 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroringWatcher.ModeUpdated (2051 ms) 2024-09-13T14:27:10.032 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroringWatcher.ImageStatusUpdated 2024-09-13T14:27:10.067 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroringWatcher.ImageStatusUpdated (35 ms) 2024-09-13T14:27:10.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:10 smithi099 bash[22656]: cluster 2024-09-13T14:27:09.187926+0000 mgr.y (mgr.24287) 225 : cluster [DBG] pgmap v227: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:10.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:10 smithi099 bash[22656]: cluster 2024-09-13T14:27:09.977158+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:27:10.284 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:10 smithi099 bash[32082]: cluster 2024-09-13T14:27:09.187926+0000 mgr.y (mgr.24287) 225 : cluster [DBG] pgmap v227: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:10.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:10 smithi099 bash[32082]: cluster 2024-09-13T14:27:09.977158+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:27:10.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:10 smithi177 bash[24494]: cluster 2024-09-13T14:27:09.187926+0000 mgr.y (mgr.24287) 225 : cluster [DBG] pgmap v227: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:10.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:10 smithi177 bash[24494]: cluster 2024-09-13T14:27:09.977158+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:27:11.028 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 2 tests from TestMirroringWatcher (2086 ms total) 2024-09-13T14:27:11.028 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:27:11.028 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 57 tests from TestMigration 2024-09-13T14:27:11.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:11 smithi099 bash[22656]: cluster 2024-09-13T14:27:10.002968+0000 mon.a (mon.0) 885 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:27:11.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:11 smithi099 bash[22656]: cluster 2024-09-13T14:27:10.003060+0000 mon.a (mon.0) 886 : cluster [INF] Cluster is now healthy 2024-09-13T14:27:11.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:11 smithi099 bash[32082]: cluster 2024-09-13T14:27:10.002968+0000 mon.a (mon.0) 885 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:27:11.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:11 smithi099 bash[32082]: cluster 2024-09-13T14:27:10.003060+0000 mon.a (mon.0) 886 : cluster [INF] Cluster is now healthy 2024-09-13T14:27:11.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:11 smithi177 bash[24494]: cluster 2024-09-13T14:27:10.002968+0000 mon.a (mon.0) 885 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:27:11.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:11 smithi177 bash[24494]: cluster 2024-09-13T14:27:10.003060+0000 mon.a (mon.0) 886 : cluster [INF] Cluster is now healthy 2024-09-13T14:27:12.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:27:12 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:12] "GET /metrics HTTP/1.1" 200 242865 "" "Prometheus/2.43.0" 2024-09-13T14:27:12.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:12 smithi099 bash[22656]: cluster 2024-09-13T14:27:11.030813+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:27:12.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:12 smithi099 bash[22656]: cluster 2024-09-13T14:27:11.188457+0000 mgr.y (mgr.24287) 226 : cluster [DBG] pgmap v230: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:12.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:12 smithi099 bash[32082]: cluster 2024-09-13T14:27:11.030813+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:27:12.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:12 smithi099 bash[32082]: cluster 2024-09-13T14:27:11.188457+0000 mgr.y (mgr.24287) 226 : cluster [DBG] pgmap v230: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:12.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:12 smithi177 bash[24494]: cluster 2024-09-13T14:27:11.030813+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:27:12.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:12 smithi177 bash[24494]: cluster 2024-09-13T14:27:11.188457+0000 mgr.y (mgr.24287) 226 : cluster [DBG] pgmap v230: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:13.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:13 smithi099 bash[32082]: cluster 2024-09-13T14:27:12.048848+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-09-13T14:27:13.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:13 smithi099 bash[32082]: audit 2024-09-13T14:27:12.052662+0000 mon.a (mon.0) 889 : audit [INF] from='client.? 172.21.15.99:0/2600942069' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:27:13.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:13 smithi099 bash[22656]: cluster 2024-09-13T14:27:12.048848+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-09-13T14:27:13.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:13 smithi099 bash[22656]: audit 2024-09-13T14:27:12.052662+0000 mon.a (mon.0) 889 : audit [INF] from='client.? 172.21.15.99:0/2600942069' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:27:13.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:13 smithi177 bash[24494]: cluster 2024-09-13T14:27:12.048848+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-09-13T14:27:13.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:13 smithi177 bash[24494]: audit 2024-09-13T14:27:12.052662+0000 mon.a (mon.0) 889 : audit [INF] from='client.? 172.21.15.99:0/2600942069' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:27:13.743 INFO:teuthology.orchestra.run.smithi099.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:alertmanager.a smithi099 *:9093,9094 running (4m) 26s ago 5m 13.3M - 0.25.0 c8568f914cd2 980bbf8d5947 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:grafana.a smithi177 *:3000 running (5m) 26s ago 5m 79.2M - 9.4.7 954c08fa6188 c1e2ef99714e 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi099.yyrtld smithi099 running (2m) 26s ago 2m 11.9M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 7bce2f3fd8d5 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi177.outtkv smithi177 running (2m) 26s ago 2m 13.2M - 17.2.7-1627-gbbdadfa3 a994b00deb3a b976db2e3d1b 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.x smithi177 *:8443 running (14m) 26s ago 14m 389M - 17.2.7-1627-gbbdadfa3 a994b00deb3a beb82bee3321 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.y smithi099 *:9283,8765 running (16m) 26s ago 16m 443M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 57a5525c7ca6 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:mon.a smithi099 running (16m) 26s ago 16m 50.7M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 0fdf0437940e 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:mon.b smithi177 running (14m) 26s ago 14m 39.6M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 660521332db7 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:mon.c smithi099 running (14m) 26s ago 14m 40.6M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a cd755a87885d 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.a smithi099 *:9100 running (5m) 26s ago 5m 8252k - 1.5.0 0da6a335fe13 01fa1bf71193 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.b smithi177 *:9100 running (5m) 26s ago 5m 8523k - 1.5.0 0da6a335fe13 467caacb90c1 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0 smithi099 running (13m) 26s ago 13m 45.3M 4060M 17.2.7-1627-gbbdadfa3 a994b00deb3a 3334f7b9d02c 2024-09-13T14:27:13.744 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1 smithi099 running (12m) 26s ago 12m 47.2M 4060M 17.2.7-1627-gbbdadfa3 a994b00deb3a 324b38b19317 2024-09-13T14:27:13.745 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2 smithi099 running (11m) 26s ago 11m 44.1M 4060M 17.2.7-1627-gbbdadfa3 a994b00deb3a e6fb2db24160 2024-09-13T14:27:13.745 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3 smithi099 running (10m) 26s ago 10m 42.8M 4060M 17.2.7-1627-gbbdadfa3 a994b00deb3a 1fad69868d22 2024-09-13T14:27:13.745 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4 smithi177 running (9m) 26s ago 9m 44.1M 4060M 17.2.7-1627-gbbdadfa3 a994b00deb3a f54f8f87d384 2024-09-13T14:27:13.745 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5 smithi177 running (9m) 26s ago 9m 41.4M 4060M 17.2.7-1627-gbbdadfa3 a994b00deb3a f285f4fee6d3 2024-09-13T14:27:13.745 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6 smithi177 running (8m) 26s ago 8m 41.4M 4060M 17.2.7-1627-gbbdadfa3 a994b00deb3a 2318776c78d4 2024-09-13T14:27:13.745 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7 smithi177 running (7m) 26s ago 7m 43.0M 4060M 17.2.7-1627-gbbdadfa3 a994b00deb3a e2d33a0b16ca 2024-09-13T14:27:13.745 INFO:teuthology.orchestra.run.smithi099.stdout:prometheus.a smithi177 *:9095 running (4m) 26s ago 6m 50.4M - 2.43.0 a07b618ecd1d 79e2c750302e 2024-09-13T14:27:14.088 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.Empty 2024-09-13T14:27:14.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:14 smithi099 bash[22656]: audit 2024-09-13T14:27:13.055554+0000 mon.a (mon.0) 890 : audit [INF] from='client.? 172.21.15.99:0/2600942069' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:27:14.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:14 smithi099 bash[22656]: cluster 2024-09-13T14:27:13.056714+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:27:14.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:14 smithi099 bash[22656]: cluster 2024-09-13T14:27:13.189534+0000 mgr.y (mgr.24287) 227 : cluster [DBG] pgmap v233: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:14.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:14 smithi099 bash[22656]: audit 2024-09-13T14:27:13.731922+0000 mgr.y (mgr.24287) 228 : audit [DBG] from='client.24746 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:14.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:14 smithi099 bash[32082]: audit 2024-09-13T14:27:13.055554+0000 mon.a (mon.0) 890 : audit [INF] from='client.? 172.21.15.99:0/2600942069' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:27:14.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:14 smithi099 bash[32082]: cluster 2024-09-13T14:27:13.056714+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:27:14.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:14 smithi099 bash[32082]: cluster 2024-09-13T14:27:13.189534+0000 mgr.y (mgr.24287) 227 : cluster [DBG] pgmap v233: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:14.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:14 smithi099 bash[32082]: audit 2024-09-13T14:27:13.731922+0000 mgr.y (mgr.24287) 228 : audit [DBG] from='client.24746 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:14.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:14 smithi177 bash[24494]: audit 2024-09-13T14:27:13.055554+0000 mon.a (mon.0) 890 : audit [INF] from='client.? 172.21.15.99:0/2600942069' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:27:14.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:14 smithi177 bash[24494]: cluster 2024-09-13T14:27:13.056714+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:27:14.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:14 smithi177 bash[24494]: cluster 2024-09-13T14:27:13.189534+0000 mgr.y (mgr.24287) 227 : cluster [DBG] pgmap v233: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:14.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:14 smithi177 bash[24494]: audit 2024-09-13T14:27:13.731922+0000 mgr.y (mgr.24287) 228 : audit [DBG] from='client.24746 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:15.055 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for minority of mons to upgrade' 2024-09-13T14:27:15.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:15 smithi099 bash[22656]: cluster 2024-09-13T14:27:14.091540+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:27:15.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:15 smithi099 bash[32082]: cluster 2024-09-13T14:27:14.091540+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:27:15.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:15 smithi177 bash[24494]: cluster 2024-09-13T14:27:14.091540+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:27:15.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:27:15 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:16.174 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:16.194 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:16.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:16 smithi177 bash[24494]: cluster 2024-09-13T14:27:15.085334+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:27:16.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:16 smithi177 bash[24494]: cluster 2024-09-13T14:27:15.190044+0000 mgr.y (mgr.24287) 229 : cluster [DBG] pgmap v236: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:16.524 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:16.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:16 smithi099 bash[22656]: cluster 2024-09-13T14:27:15.085334+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:27:16.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:16 smithi099 bash[22656]: cluster 2024-09-13T14:27:15.190044+0000 mgr.y (mgr.24287) 229 : cluster [DBG] pgmap v236: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:16.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:16 smithi099 bash[32082]: cluster 2024-09-13T14:27:15.085334+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:27:16.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:16 smithi099 bash[32082]: cluster 2024-09-13T14:27:15.190044+0000 mgr.y (mgr.24287) 229 : cluster [DBG] pgmap v236: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:16.716 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:16.912 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.Empty (2824 ms) 2024-09-13T14:27:16.912 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.OtherName 2024-09-13T14:27:16.980 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:17.005 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:17.320 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:17.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:17 smithi177 bash[24494]: cluster 2024-09-13T14:27:16.089581+0000 mon.a (mon.0) 894 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:27:17.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:17 smithi177 bash[24494]: cluster 2024-09-13T14:27:16.111813+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:27:17.494 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:17.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:17 smithi099 bash[32082]: cluster 2024-09-13T14:27:16.089581+0000 mon.a (mon.0) 894 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:27:17.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:17 smithi099 bash[32082]: cluster 2024-09-13T14:27:16.111813+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:27:17.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:17 smithi099 bash[22656]: cluster 2024-09-13T14:27:16.089581+0000 mon.a (mon.0) 894 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:27:17.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:17 smithi099 bash[22656]: cluster 2024-09-13T14:27:16.111813+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:27:17.687 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.OtherName (775 ms) 2024-09-13T14:27:17.687 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.OtherPool 2024-09-13T14:27:17.763 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:17.791 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:18.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:18 smithi099 bash[22656]: cluster 2024-09-13T14:27:17.190712+0000 mgr.y (mgr.24287) 230 : cluster [DBG] pgmap v238: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:18.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:18 smithi099 bash[32082]: cluster 2024-09-13T14:27:17.190712+0000 mgr.y (mgr.24287) 230 : cluster [DBG] pgmap v238: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:18.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:18 smithi177 bash[24494]: cluster 2024-09-13T14:27:17.190712+0000 mgr.y (mgr.24287) 230 : cluster [DBG] pgmap v238: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:27:19.381 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:19.521 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:19.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:19 smithi099 bash[32082]: cluster 2024-09-13T14:27:18.128516+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:27:19.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:19 smithi099 bash[22656]: cluster 2024-09-13T14:27:18.128516+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:27:19.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:19 smithi177 bash[24494]: cluster 2024-09-13T14:27:18.128516+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:27:19.682 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.OtherPool (1996 ms) 2024-09-13T14:27:19.683 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.OtherNamespace 2024-09-13T14:27:19.749 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:19.777 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:19.857 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:20.025 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:20.127 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:20.261 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.OtherNamespace (578 ms) 2024-09-13T14:27:20.261 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.DataPool 2024-09-13T14:27:20.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:20 smithi099 bash[22656]: cluster 2024-09-13T14:27:19.130586+0000 mon.a (mon.0) 897 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:27:20.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:20 smithi099 bash[22656]: cluster 2024-09-13T14:27:19.191380+0000 mgr.y (mgr.24287) 231 : cluster [DBG] pgmap v241: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 346 KiB/s rd, 20 KiB/s wr, 439 op/s 2024-09-13T14:27:20.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:20 smithi099 bash[32082]: cluster 2024-09-13T14:27:19.130586+0000 mon.a (mon.0) 897 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:27:20.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:20 smithi099 bash[32082]: cluster 2024-09-13T14:27:19.191380+0000 mgr.y (mgr.24287) 231 : cluster [DBG] pgmap v241: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 346 KiB/s rd, 20 KiB/s wr, 439 op/s 2024-09-13T14:27:20.325 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:20.345 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:20.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:20 smithi177 bash[24494]: cluster 2024-09-13T14:27:19.130586+0000 mon.a (mon.0) 897 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:27:20.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:20 smithi177 bash[24494]: cluster 2024-09-13T14:27:19.191380+0000 mgr.y (mgr.24287) 231 : cluster [DBG] pgmap v241: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 346 KiB/s rd, 20 KiB/s wr, 439 op/s 2024-09-13T14:27:20.702 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:20.897 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:21.098 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.DataPool (837 ms) 2024-09-13T14:27:21.098 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.AbortAfterPrepare 2024-09-13T14:27:21.160 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:21.183 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:21.496 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:27:21.722 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.AbortAfterPrepare (624 ms) 2024-09-13T14:27:21.722 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.AbortAfterFailedPrepare 2024-09-13T14:27:21.802 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:21.828 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:22.061 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.AbortAfterFailedPrepare (339 ms) 2024-09-13T14:27:22.061 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.AbortAfterExecute 2024-09-13T14:27:22.131 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:22.157 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:22.471 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:22.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:27:22 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:22] "GET /metrics HTTP/1.1" 200 242865 "" "Prometheus/2.43.0" 2024-09-13T14:27:22.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:22 smithi099 bash[32082]: cluster 2024-09-13T14:27:21.192106+0000 mgr.y (mgr.24287) 232 : cluster [DBG] pgmap v242: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 237 KiB/s rd, 14 KiB/s wr, 301 op/s 2024-09-13T14:27:22.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:22 smithi099 bash[32082]: audit 2024-09-13T14:27:22.165623+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:22.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:22 smithi099 bash[32082]: audit 2024-09-13T14:27:22.169688+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:22.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:22 smithi099 bash[22656]: cluster 2024-09-13T14:27:21.192106+0000 mgr.y (mgr.24287) 232 : cluster [DBG] pgmap v242: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 237 KiB/s rd, 14 KiB/s wr, 301 op/s 2024-09-13T14:27:22.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:22 smithi099 bash[22656]: audit 2024-09-13T14:27:22.165623+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:22.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:22 smithi099 bash[22656]: audit 2024-09-13T14:27:22.169688+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:22.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:22 smithi177 bash[24494]: cluster 2024-09-13T14:27:21.192106+0000 mgr.y (mgr.24287) 232 : cluster [DBG] pgmap v242: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 237 KiB/s rd, 14 KiB/s wr, 301 op/s 2024-09-13T14:27:22.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:22 smithi177 bash[24494]: audit 2024-09-13T14:27:22.165623+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:22.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:22 smithi177 bash[24494]: audit 2024-09-13T14:27:22.169688+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:22.636 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:27:22.814 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.AbortAfterExecute (753 ms) 2024-09-13T14:27:22.814 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.OtherPoolAbortAfterExecute 2024-09-13T14:27:22.861 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:22.878 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:23.160 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:23.323 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:27:23.582 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.OtherPoolAbortAfterExecute (768 ms) 2024-09-13T14:27:23.582 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.OtherNamespaceAbortAfterExecute 2024-09-13T14:27:23.637 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:23.667 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:23.905 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:23.994 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:27:24.168 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.OtherNamespaceAbortAfterExecute (587 ms) 2024-09-13T14:27:24.169 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.MirroringSamePool 2024-09-13T14:27:24.214 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:24.232 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:27:24.244 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.MirroringSamePool (75 ms) 2024-09-13T14:27:24.244 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.MirroringAbort 2024-09-13T14:27:24.293 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:24.312 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:27:24.324 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.MirroringAbort (81 ms) 2024-09-13T14:27:24.325 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.MirroringOtherPoolDisabled 2024-09-13T14:27:24.368 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:24.385 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:27:24.395 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.MirroringOtherPoolDisabled (71 ms) 2024-09-13T14:27:24.395 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.MirroringOtherPoolEnabled 2024-09-13T14:27:24.445 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:24.465 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:27:24.477 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.MirroringOtherPoolEnabled (81 ms) 2024-09-13T14:27:24.477 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.MirroringPool 2024-09-13T14:27:24.526 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:24.547 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:27:24.559 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.MirroringPool (83 ms) 2024-09-13T14:27:24.559 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.Group 2024-09-13T14:27:24.611 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:24.647 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:24.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:24 smithi099 bash[22656]: cluster 2024-09-13T14:27:23.193198+0000 mgr.y (mgr.24287) 233 : cluster [DBG] pgmap v243: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 697 KiB/s rd, 40 KiB/s wr, 882 op/s 2024-09-13T14:27:24.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:24 smithi099 bash[22656]: cluster 2024-09-13T14:27:23.546752+0000 mon.a (mon.0) 899 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:27:24.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:24 smithi099 bash[32082]: cluster 2024-09-13T14:27:23.193198+0000 mgr.y (mgr.24287) 233 : cluster [DBG] pgmap v243: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 697 KiB/s rd, 40 KiB/s wr, 882 op/s 2024-09-13T14:27:24.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:24 smithi099 bash[32082]: cluster 2024-09-13T14:27:23.546752+0000 mon.a (mon.0) 899 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:27:24.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:24 smithi177 bash[24494]: cluster 2024-09-13T14:27:23.193198+0000 mgr.y (mgr.24287) 233 : cluster [DBG] pgmap v243: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 697 KiB/s rd, 40 KiB/s wr, 882 op/s 2024-09-13T14:27:24.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:24 smithi177 bash[24494]: cluster 2024-09-13T14:27:23.546752+0000 mon.a (mon.0) 899 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:27:24.866 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:24.963 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:25.096 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.Group (536 ms) 2024-09-13T14:27:25.096 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.GroupAbort 2024-09-13T14:27:25.152 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:25.191 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:25.426 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:27:25.612 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.GroupAbort (516 ms) 2024-09-13T14:27:25.612 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.NoSnaps 2024-09-13T14:27:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:25.680 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:25.772 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:27:25.829 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:25.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:27:25 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:26.153 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:26.342 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:26.519 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.NoSnaps (907 ms) 2024-09-13T14:27:26.519 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.NoSnapsOtherPool 2024-09-13T14:27:26.563 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:26.581 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:26.658 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:27:26.730 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:26.737 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:27:26.808 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:26.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:26 smithi099 bash[22656]: cluster 2024-09-13T14:27:25.193620+0000 mgr.y (mgr.24287) 234 : cluster [DBG] pgmap v245: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 497 KiB/s rd, 28 KiB/s wr, 629 op/s 2024-09-13T14:27:26.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:26 smithi099 bash[32082]: cluster 2024-09-13T14:27:25.193620+0000 mgr.y (mgr.24287) 234 : cluster [DBG] pgmap v245: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 497 KiB/s rd, 28 KiB/s wr, 629 op/s 2024-09-13T14:27:26.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:26 smithi177 bash[24494]: cluster 2024-09-13T14:27:25.193620+0000 mgr.y (mgr.24287) 234 : cluster [DBG] pgmap v245: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 497 KiB/s rd, 28 KiB/s wr, 629 op/s 2024-09-13T14:27:27.023 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:27.154 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:27.298 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.NoSnapsOtherPool (779 ms) 2024-09-13T14:27:27.298 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.NoSnapsDataPool 2024-09-13T14:27:27.350 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:27.375 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:27.448 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:27:27.523 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:27.798 INFO:teuthology.orchestra.run.smithi099.stdout:wait for minority of mons to upgrade 2024-09-13T14:27:27.813 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:27.977 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:28.142 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.NoSnapsDataPool (844 ms) 2024-09-13T14:27:28.142 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.NoSnapsShrinkAfterPrepare 2024-09-13T14:27:28.195 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:28.213 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:28.279 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:27:28.329 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:28.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:28 smithi099 bash[22656]: cluster 2024-09-13T14:27:27.193991+0000 mgr.y (mgr.24287) 235 : cluster [DBG] pgmap v246: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 25 KiB/s wr, 551 op/s 2024-09-13T14:27:28.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:28 smithi099 bash[22656]: audit 2024-09-13T14:27:27.305035+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:28.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:28 smithi099 bash[32082]: cluster 2024-09-13T14:27:27.193991+0000 mgr.y (mgr.24287) 235 : cluster [DBG] pgmap v246: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 25 KiB/s wr, 551 op/s 2024-09-13T14:27:28.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:28 smithi099 bash[32082]: audit 2024-09-13T14:27:27.305035+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:28.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:28 smithi177 bash[24494]: cluster 2024-09-13T14:27:27.193991+0000 mgr.y (mgr.24287) 235 : cluster [DBG] pgmap v246: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 25 KiB/s wr, 551 op/s 2024-09-13T14:27:28.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:28 smithi177 bash[24494]: audit 2024-09-13T14:27:27.305035+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:28.622 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 41943040 2024-09-13T14:27:28.640 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:28.826 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:28.985 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.NoSnapsShrinkAfterPrepare (844 ms) 2024-09-13T14:27:28.985 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroBeforePrepare 2024-09-13T14:27:29.032 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:29.048 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:29.128 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:27:29.191 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 0 2024-09-13T14:27:29.220 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:29.433 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:29.570 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:29.656 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroBeforePrepare (670 ms) 2024-09-13T14:27:29.656 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroAfterPrepare 2024-09-13T14:27:29.707 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:29.732 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:29.809 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:27:29.881 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:30.189 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 0 2024-09-13T14:27:30.214 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:30.347 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:30.451 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroAfterPrepare (795 ms) 2024-09-13T14:27:30.451 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.NoSnapsExpandAfterPrepare 2024-09-13T14:27:30.513 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:30.539 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:30.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:30 smithi099 bash[22656]: cluster 2024-09-13T14:27:29.194846+0000 mgr.y (mgr.24287) 236 : cluster [DBG] pgmap v247: 137 pgs: 137 active+clean; 97 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 18 MiB/s wr, 1.11k op/s 2024-09-13T14:27:30.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:30 smithi099 bash[32082]: cluster 2024-09-13T14:27:29.194846+0000 mgr.y (mgr.24287) 236 : cluster [DBG] pgmap v247: 137 pgs: 137 active+clean; 97 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 18 MiB/s wr, 1.11k op/s 2024-09-13T14:27:30.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:30 smithi177 bash[24494]: cluster 2024-09-13T14:27:29.194846+0000 mgr.y (mgr.24287) 236 : cluster [DBG] pgmap v247: 137 pgs: 137 active+clean; 97 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 18 MiB/s wr, 1.11k op/s 2024-09-13T14:27:30.630 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:27:30.702 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:31.008 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:27:31.024 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:31.196 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:31.401 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.NoSnapsExpandAfterPrepare (950 ms) 2024-09-13T14:27:31.401 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.NoSnapsSnapAfterPrepare 2024-09-13T14:27:31.448 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:31.463 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:31.528 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:27:31.588 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:31.833 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: after_prepare_snap 2024-09-13T14:27:32.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:32 smithi099 bash[22656]: cluster 2024-09-13T14:27:31.195248+0000 mgr.y (mgr.24287) 237 : cluster [DBG] pgmap v248: 137 pgs: 137 active+clean; 97 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 18 MiB/s wr, 1.11k op/s 2024-09-13T14:27:32.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:27:32 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:32] "GET /metrics HTTP/1.1" 200 245647 "" "Prometheus/2.43.0" 2024-09-13T14:27:32.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:32 smithi099 bash[32082]: cluster 2024-09-13T14:27:31.195248+0000 mgr.y (mgr.24287) 237 : cluster [DBG] pgmap v248: 137 pgs: 137 active+clean; 97 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 18 MiB/s wr, 1.11k op/s 2024-09-13T14:27:32.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:32 smithi177 bash[24494]: cluster 2024-09-13T14:27:31.195248+0000 mgr.y (mgr.24287) 237 : cluster [DBG] pgmap v248: 137 pgs: 137 active+clean; 97 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 18 MiB/s wr, 1.11k op/s 2024-09-13T14:27:32.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:27:33.344 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 41943040 2024-09-13T14:27:33.371 INFO:tasks.workunit.client.0.smithi099.stdout:write: * 0~1000 2024-09-13T14:27:33.378 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:33.526 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:33.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:33 smithi177 bash[24494]: cluster 2024-09-13T14:27:32.327697+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:27:33.727 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.NoSnapsSnapAfterPrepare (2326 ms) 2024-09-13T14:27:33.727 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.Snaps 2024-09-13T14:27:33.776 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:33.795 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:33.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:33 smithi099 bash[22656]: cluster 2024-09-13T14:27:32.327697+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:27:33.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:33 smithi099 bash[32082]: cluster 2024-09-13T14:27:32.327697+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-09-13T14:27:33.865 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:27:34.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:34 smithi177 bash[24494]: cluster 2024-09-13T14:27:33.196091+0000 mgr.y (mgr.24287) 238 : cluster [DBG] pgmap v250: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s rd, 38 MiB/s wr, 1.22k op/s 2024-09-13T14:27:34.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:34 smithi177 bash[24494]: cluster 2024-09-13T14:27:33.331669+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-09-13T14:27:34.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:34 smithi099 bash[22656]: cluster 2024-09-13T14:27:33.196091+0000 mgr.y (mgr.24287) 238 : cluster [DBG] pgmap v250: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s rd, 38 MiB/s wr, 1.22k op/s 2024-09-13T14:27:34.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:34 smithi099 bash[22656]: cluster 2024-09-13T14:27:33.331669+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-09-13T14:27:34.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:34 smithi099 bash[32082]: cluster 2024-09-13T14:27:33.196091+0000 mgr.y (mgr.24287) 238 : cluster [DBG] pgmap v250: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s rd, 38 MiB/s wr, 1.22k op/s 2024-09-13T14:27:34.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:34 smithi099 bash[32082]: cluster 2024-09-13T14:27:33.331669+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-09-13T14:27:35.367 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 8388609~8388609 2024-09-13T14:27:35.390 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:27:35.390 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:27:35.391 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:27:35.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:35 smithi177 bash[24494]: cluster 2024-09-13T14:27:34.344215+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-09-13T14:27:35.604 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:27:35 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:35.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:35 smithi099 bash[22656]: cluster 2024-09-13T14:27:34.344215+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-09-13T14:27:35.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:35 smithi099 bash[32082]: cluster 2024-09-13T14:27:34.344215+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-09-13T14:27:36.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:36 smithi099 bash[22656]: cluster 2024-09-13T14:27:35.196773+0000 mgr.y (mgr.24287) 239 : cluster [DBG] pgmap v253: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 32 MiB/s wr, 853 op/s 2024-09-13T14:27:36.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:36 smithi099 bash[22656]: cluster 2024-09-13T14:27:35.345449+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:27:36.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:36 smithi099 bash[32082]: cluster 2024-09-13T14:27:35.196773+0000 mgr.y (mgr.24287) 239 : cluster [DBG] pgmap v253: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 32 MiB/s wr, 853 op/s 2024-09-13T14:27:36.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:36 smithi099 bash[32082]: cluster 2024-09-13T14:27:35.345449+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:27:36.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:36 smithi177 bash[24494]: cluster 2024-09-13T14:27:35.196773+0000 mgr.y (mgr.24287) 239 : cluster [DBG] pgmap v253: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 32 MiB/s wr, 853 op/s 2024-09-13T14:27:36.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:36 smithi177 bash[24494]: cluster 2024-09-13T14:27:35.345449+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:27:37.396 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 8388609~1000 2024-09-13T14:27:37.411 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16777228~1000 2024-09-13T14:27:37.412 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:27:37.428 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:37.431 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 83886079~8388609 2024-09-13T14:27:37.462 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:27:37.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:37 smithi099 bash[22656]: cluster 2024-09-13T14:27:36.361177+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:27:37.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:37 smithi099 bash[22656]: audit 2024-09-13T14:27:37.157772+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:37.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:37 smithi099 bash[32082]: cluster 2024-09-13T14:27:36.361177+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:27:37.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:37 smithi099 bash[32082]: audit 2024-09-13T14:27:37.157772+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:37.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:37 smithi177 bash[24494]: cluster 2024-09-13T14:27:36.361177+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:27:37.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:37 smithi177 bash[24494]: audit 2024-09-13T14:27:37.157772+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:38.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:38 smithi099 bash[22656]: cluster 2024-09-13T14:27:37.197398+0000 mgr.y (mgr.24287) 240 : cluster [DBG] pgmap v256: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:27:38.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:38 smithi099 bash[22656]: cluster 2024-09-13T14:27:37.385240+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:27:38.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:38 smithi099 bash[32082]: cluster 2024-09-13T14:27:37.197398+0000 mgr.y (mgr.24287) 240 : cluster [DBG] pgmap v256: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:27:38.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:38 smithi099 bash[32082]: cluster 2024-09-13T14:27:37.385240+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:27:38.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:38 smithi177 bash[24494]: cluster 2024-09-13T14:27:37.197398+0000 mgr.y (mgr.24287) 240 : cluster [DBG] pgmap v256: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:27:38.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:38 smithi177 bash[24494]: cluster 2024-09-13T14:27:37.385240+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:27:39.416 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:39.470 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83886079~1 2024-09-13T14:27:39.491 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:39.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:39 smithi099 bash[22656]: cluster 2024-09-13T14:27:38.384390+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:27:39.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:39 smithi099 bash[32082]: cluster 2024-09-13T14:27:38.384390+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:27:39.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:39 smithi177 bash[24494]: cluster 2024-09-13T14:27:38.384390+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:27:40.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:40 smithi099 bash[22656]: cluster 2024-09-13T14:27:39.198139+0000 mgr.y (mgr.24287) 241 : cluster [DBG] pgmap v259: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 353 op/s 2024-09-13T14:27:40.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:40 smithi099 bash[22656]: cluster 2024-09-13T14:27:39.407383+0000 mon.a (mon.0) 908 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:27:40.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:40 smithi099 bash[32082]: cluster 2024-09-13T14:27:39.198139+0000 mgr.y (mgr.24287) 241 : cluster [DBG] pgmap v259: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 353 op/s 2024-09-13T14:27:40.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:40 smithi099 bash[32082]: cluster 2024-09-13T14:27:39.407383+0000 mon.a (mon.0) 908 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:27:40.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:40 smithi177 bash[24494]: cluster 2024-09-13T14:27:39.198139+0000 mgr.y (mgr.24287) 241 : cluster [DBG] pgmap v259: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 353 op/s 2024-09-13T14:27:40.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:40 smithi177 bash[24494]: cluster 2024-09-13T14:27:39.407383+0000 mon.a (mon.0) 908 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:27:41.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:41 smithi099 bash[22656]: cluster 2024-09-13T14:27:40.416918+0000 mon.a (mon.0) 909 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:27:41.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:41 smithi099 bash[32082]: cluster 2024-09-13T14:27:40.416918+0000 mon.a (mon.0) 909 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:27:41.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:41 smithi177 bash[24494]: cluster 2024-09-13T14:27:40.416918+0000 mon.a (mon.0) 909 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:27:42.414 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:27:42 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:42] "GET /metrics HTTP/1.1" 200 246248 "" "Prometheus/2.43.0" 2024-09-13T14:27:42.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:42 smithi099 bash[32082]: cluster 2024-09-13T14:27:41.198860+0000 mgr.y (mgr.24287) 242 : cluster [DBG] pgmap v262: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 353 op/s 2024-09-13T14:27:42.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:42 smithi099 bash[32082]: cluster 2024-09-13T14:27:41.424870+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:27:42.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:42 smithi099 bash[22656]: cluster 2024-09-13T14:27:41.198860+0000 mgr.y (mgr.24287) 242 : cluster [DBG] pgmap v262: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 353 op/s 2024-09-13T14:27:42.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:42 smithi099 bash[22656]: cluster 2024-09-13T14:27:41.424870+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:27:42.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:42 smithi177 bash[24494]: cluster 2024-09-13T14:27:41.198860+0000 mgr.y (mgr.24287) 242 : cluster [DBG] pgmap v262: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 353 op/s 2024-09-13T14:27:42.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:42 smithi177 bash[24494]: cluster 2024-09-13T14:27:41.424870+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:27:43.180 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:43.365 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:43.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:43 smithi099 bash[22656]: cluster 2024-09-13T14:27:42.437442+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:27:43.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:43 smithi099 bash[32082]: cluster 2024-09-13T14:27:42.437442+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:27:43.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:43 smithi177 bash[24494]: cluster 2024-09-13T14:27:42.437442+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:27:44.766 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:44 smithi177 bash[24494]: cluster 2024-09-13T14:27:43.199743+0000 mgr.y (mgr.24287) 243 : cluster [DBG] pgmap v265: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 224 op/s 2024-09-13T14:27:44.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:44 smithi099 bash[22656]: cluster 2024-09-13T14:27:43.199743+0000 mgr.y (mgr.24287) 243 : cluster [DBG] pgmap v265: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 224 op/s 2024-09-13T14:27:44.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:44 smithi099 bash[32082]: cluster 2024-09-13T14:27:43.199743+0000 mgr.y (mgr.24287) 243 : cluster [DBG] pgmap v265: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 224 op/s 2024-09-13T14:27:45.459 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:27:45 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:45.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[22656]: cluster 2024-09-13T14:27:44.441584+0000 mon.a (mon.0) 912 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:27:45.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[22656]: audit 2024-09-13T14:27:44.553566+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:45.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[22656]: cephadm 2024-09-13T14:27:44.556133+0000 mgr.y (mgr.24287) 244 : cephadm [INF] Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:27:45.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[22656]: cephadm 2024-09-13T14:27:44.556219+0000 mgr.y (mgr.24287) 245 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:27:45.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[22656]: audit 2024-09-13T14:27:44.557365+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:45.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[22656]: audit 2024-09-13T14:27:44.565735+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:45.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[22656]: cephadm 2024-09-13T14:27:44.566878+0000 mgr.y (mgr.24287) 246 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:27:45.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[22656]: cephadm 2024-09-13T14:27:44.801669+0000 mgr.y (mgr.24287) 247 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi177 2024-09-13T14:27:45.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[32082]: cluster 2024-09-13T14:27:44.441584+0000 mon.a (mon.0) 912 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:27:45.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[32082]: audit 2024-09-13T14:27:44.553566+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:45.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[32082]: cephadm 2024-09-13T14:27:44.556133+0000 mgr.y (mgr.24287) 244 : cephadm [INF] Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:27:45.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[32082]: cephadm 2024-09-13T14:27:44.556219+0000 mgr.y (mgr.24287) 245 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:27:45.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[32082]: audit 2024-09-13T14:27:44.557365+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:45.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[32082]: audit 2024-09-13T14:27:44.565735+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:45.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[32082]: cephadm 2024-09-13T14:27:44.566878+0000 mgr.y (mgr.24287) 246 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:27:45.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:45 smithi099 bash[32082]: cephadm 2024-09-13T14:27:44.801669+0000 mgr.y (mgr.24287) 247 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi177 2024-09-13T14:27:45.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:45 smithi177 bash[24494]: cluster 2024-09-13T14:27:44.441584+0000 mon.a (mon.0) 912 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:27:45.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:45 smithi177 bash[24494]: audit 2024-09-13T14:27:44.553566+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:45.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:45 smithi177 bash[24494]: cephadm 2024-09-13T14:27:44.556133+0000 mgr.y (mgr.24287) 244 : cephadm [INF] Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:27:45.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:45 smithi177 bash[24494]: cephadm 2024-09-13T14:27:44.556219+0000 mgr.y (mgr.24287) 245 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:27:45.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:45 smithi177 bash[24494]: audit 2024-09-13T14:27:44.557365+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:45.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:45 smithi177 bash[24494]: audit 2024-09-13T14:27:44.565735+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:27:45.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:45 smithi177 bash[24494]: cephadm 2024-09-13T14:27:44.566878+0000 mgr.y (mgr.24287) 246 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:27:45.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:45 smithi177 bash[24494]: cephadm 2024-09-13T14:27:44.801669+0000 mgr.y (mgr.24287) 247 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi177 2024-09-13T14:27:46.600 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:27:46.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:46 smithi099 bash[22656]: cluster 2024-09-13T14:27:45.200422+0000 mgr.y (mgr.24287) 248 : cluster [DBG] pgmap v267: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 187 op/s 2024-09-13T14:27:46.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:46 smithi099 bash[22656]: cluster 2024-09-13T14:27:45.460204+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:27:46.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:46 smithi099 bash[22656]: audit 2024-09-13T14:27:46.285422+0000 mon.a (mon.0) 916 : audit [DBG] from='client.? 172.21.15.99:0/3667931371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:46.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:46 smithi099 bash[32082]: cluster 2024-09-13T14:27:45.200422+0000 mgr.y (mgr.24287) 248 : cluster [DBG] pgmap v267: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 187 op/s 2024-09-13T14:27:46.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:46 smithi099 bash[32082]: cluster 2024-09-13T14:27:45.460204+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:27:46.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:46 smithi099 bash[32082]: audit 2024-09-13T14:27:46.285422+0000 mon.a (mon.0) 916 : audit [DBG] from='client.? 172.21.15.99:0/3667931371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:46.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:46 smithi177 bash[24494]: cluster 2024-09-13T14:27:45.200422+0000 mgr.y (mgr.24287) 248 : cluster [DBG] pgmap v267: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 187 op/s 2024-09-13T14:27:46.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:46 smithi177 bash[24494]: cluster 2024-09-13T14:27:45.460204+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:27:46.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:46 smithi177 bash[24494]: audit 2024-09-13T14:27:46.285422+0000 mon.a (mon.0) 916 : audit [DBG] from='client.? 172.21.15.99:0/3667931371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:46.920 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.Snaps (13193 ms) 2024-09-13T14:27:46.920 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.SnapsOtherPool 2024-09-13T14:27:46.974 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:46.996 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:47.086 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:27:47.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:47 smithi099 bash[32082]: cluster 2024-09-13T14:27:46.483416+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:27:47.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:47 smithi099 bash[32082]: audit 2024-09-13T14:27:46.591761+0000 mgr.y (mgr.24287) 249 : audit [DBG] from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:47.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:47 smithi099 bash[32082]: audit 2024-09-13T14:27:46.896540+0000 mgr.y (mgr.24287) 250 : audit [DBG] from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:47.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:47 smithi099 bash[22656]: cluster 2024-09-13T14:27:46.483416+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:27:47.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:47 smithi099 bash[22656]: audit 2024-09-13T14:27:46.591761+0000 mgr.y (mgr.24287) 249 : audit [DBG] from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:47.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:47 smithi099 bash[22656]: audit 2024-09-13T14:27:46.896540+0000 mgr.y (mgr.24287) 250 : audit [DBG] from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:47.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:47 smithi177 bash[24494]: cluster 2024-09-13T14:27:46.483416+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:27:47.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:47 smithi177 bash[24494]: audit 2024-09-13T14:27:46.591761+0000 mgr.y (mgr.24287) 249 : audit [DBG] from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:47.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:47 smithi177 bash[24494]: audit 2024-09-13T14:27:46.896540+0000 mgr.y (mgr.24287) 250 : audit [DBG] from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:48.526 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 8388609~8388609 2024-09-13T14:27:48.575 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:27:48.575 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:27:48.576 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:27:48.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:48 smithi099 bash[22656]: cluster 2024-09-13T14:27:47.201143+0000 mgr.y (mgr.24287) 251 : cluster [DBG] pgmap v270: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.9 KiB/s wr, 188 op/s 2024-09-13T14:27:48.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:48 smithi099 bash[22656]: cluster 2024-09-13T14:27:47.492550+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:27:48.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:48 smithi099 bash[32082]: cluster 2024-09-13T14:27:47.201143+0000 mgr.y (mgr.24287) 251 : cluster [DBG] pgmap v270: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.9 KiB/s wr, 188 op/s 2024-09-13T14:27:48.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:48 smithi099 bash[32082]: cluster 2024-09-13T14:27:47.492550+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:27:48.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:48 smithi177 bash[24494]: cluster 2024-09-13T14:27:47.201143+0000 mgr.y (mgr.24287) 251 : cluster [DBG] pgmap v270: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.9 KiB/s wr, 188 op/s 2024-09-13T14:27:48.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:48 smithi177 bash[24494]: cluster 2024-09-13T14:27:47.492550+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:27:49.548 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:27:49.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:49 smithi099 bash[22656]: cluster 2024-09-13T14:27:48.506992+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:27:49.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:49 smithi099 bash[22656]: audit 2024-09-13T14:27:49.247573+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.99:0/2771349689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:49.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:49 smithi099 bash[32082]: cluster 2024-09-13T14:27:48.506992+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:27:49.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:49 smithi099 bash[32082]: audit 2024-09-13T14:27:49.247573+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.99:0/2771349689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:49.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:49 smithi177 bash[24494]: cluster 2024-09-13T14:27:48.506992+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:27:49.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:49 smithi177 bash[24494]: audit 2024-09-13T14:27:49.247573+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.99:0/2771349689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:50.550 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 8388609~1000 2024-09-13T14:27:50.566 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16777228~1000 2024-09-13T14:27:50.566 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:27:50.586 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 83886079~8388609 2024-09-13T14:27:50.634 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:27:50.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:50 smithi099 bash[22656]: cluster 2024-09-13T14:27:49.201847+0000 mgr.y (mgr.24287) 252 : cluster [DBG] pgmap v273: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T14:27:50.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:50 smithi099 bash[22656]: cluster 2024-09-13T14:27:49.518462+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:27:50.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:50 smithi099 bash[22656]: audit 2024-09-13T14:27:49.543519+0000 mgr.y (mgr.24287) 253 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:50.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:50 smithi099 bash[22656]: audit 2024-09-13T14:27:49.826824+0000 mgr.y (mgr.24287) 254 : audit [DBG] from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:50.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:50 smithi099 bash[32082]: cluster 2024-09-13T14:27:49.201847+0000 mgr.y (mgr.24287) 252 : cluster [DBG] pgmap v273: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T14:27:50.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:50 smithi099 bash[32082]: cluster 2024-09-13T14:27:49.518462+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:27:50.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:50 smithi099 bash[32082]: audit 2024-09-13T14:27:49.543519+0000 mgr.y (mgr.24287) 253 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:50.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:50 smithi099 bash[32082]: audit 2024-09-13T14:27:49.826824+0000 mgr.y (mgr.24287) 254 : audit [DBG] from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:50.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:50 smithi177 bash[24494]: cluster 2024-09-13T14:27:49.201847+0000 mgr.y (mgr.24287) 252 : cluster [DBG] pgmap v273: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T14:27:50.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:50 smithi177 bash[24494]: cluster 2024-09-13T14:27:49.518462+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:27:50.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:50 smithi177 bash[24494]: audit 2024-09-13T14:27:49.543519+0000 mgr.y (mgr.24287) 253 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:50.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:50 smithi177 bash[24494]: audit 2024-09-13T14:27:49.826824+0000 mgr.y (mgr.24287) 254 : audit [DBG] from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:51.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:51 smithi099 bash[22656]: cluster 2024-09-13T14:27:50.526135+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-09-13T14:27:51.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:51 smithi099 bash[32082]: cluster 2024-09-13T14:27:50.526135+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-09-13T14:27:51.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:51 smithi177 bash[24494]: cluster 2024-09-13T14:27:50.526135+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-09-13T14:27:52.467 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:27:52.522 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:27:52 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:52] "GET /metrics HTTP/1.1" 200 246248 "" "Prometheus/2.43.0" 2024-09-13T14:27:52.561 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:27:52.600 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83886079~1 2024-09-13T14:27:52.617 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:27:52.641 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:27:52.711 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:27:52.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:52 smithi099 bash[22656]: cluster 2024-09-13T14:27:51.202562+0000 mgr.y (mgr.24287) 255 : cluster [DBG] pgmap v276: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T14:27:52.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:52 smithi099 bash[22656]: cluster 2024-09-13T14:27:51.535370+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-09-13T14:27:52.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:52 smithi099 bash[22656]: audit 2024-09-13T14:27:52.158143+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:52.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:52 smithi099 bash[22656]: audit 2024-09-13T14:27:52.174426+0000 mon.a (mon.0) 924 : audit [DBG] from='client.? 172.21.15.99:0/2869394997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:52.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:52 smithi099 bash[32082]: cluster 2024-09-13T14:27:51.202562+0000 mgr.y (mgr.24287) 255 : cluster [DBG] pgmap v276: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T14:27:52.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:52 smithi099 bash[32082]: cluster 2024-09-13T14:27:51.535370+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-09-13T14:27:52.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:52 smithi099 bash[32082]: audit 2024-09-13T14:27:52.158143+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:52.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:52 smithi099 bash[32082]: audit 2024-09-13T14:27:52.174426+0000 mon.a (mon.0) 924 : audit [DBG] from='client.? 172.21.15.99:0/2869394997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:52.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:52 smithi177 bash[24494]: cluster 2024-09-13T14:27:51.202562+0000 mgr.y (mgr.24287) 255 : cluster [DBG] pgmap v276: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T14:27:52.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:52 smithi177 bash[24494]: cluster 2024-09-13T14:27:51.535370+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-09-13T14:27:52.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:52 smithi177 bash[24494]: audit 2024-09-13T14:27:52.158143+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:52.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:52 smithi177 bash[24494]: audit 2024-09-13T14:27:52.174426+0000 mon.a (mon.0) 924 : audit [DBG] from='client.? 172.21.15.99:0/2869394997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:53.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:53 smithi099 bash[22656]: audit 2024-09-13T14:27:52.461408+0000 mgr.y (mgr.24287) 256 : audit [DBG] from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:53.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:53 smithi099 bash[22656]: cluster 2024-09-13T14:27:52.546718+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-09-13T14:27:53.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:53 smithi099 bash[22656]: audit 2024-09-13T14:27:52.788467+0000 mgr.y (mgr.24287) 257 : audit [DBG] from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:53.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:53 smithi099 bash[32082]: audit 2024-09-13T14:27:52.461408+0000 mgr.y (mgr.24287) 256 : audit [DBG] from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:53.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:53 smithi099 bash[32082]: cluster 2024-09-13T14:27:52.546718+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-09-13T14:27:53.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:53 smithi099 bash[32082]: audit 2024-09-13T14:27:52.788467+0000 mgr.y (mgr.24287) 257 : audit [DBG] from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:53.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:53 smithi177 bash[24494]: audit 2024-09-13T14:27:52.461408+0000 mgr.y (mgr.24287) 256 : audit [DBG] from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:53.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:53 smithi177 bash[24494]: cluster 2024-09-13T14:27:52.546718+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-09-13T14:27:53.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:53 smithi177 bash[24494]: audit 2024-09-13T14:27:52.788467+0000 mgr.y (mgr.24287) 257 : audit [DBG] from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:54.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:54 smithi099 bash[32082]: cluster 2024-09-13T14:27:53.204204+0000 mgr.y (mgr.24287) 258 : cluster [DBG] pgmap v279: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 155 KiB/s rd, 16 MiB/s wr, 228 op/s 2024-09-13T14:27:54.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:54 smithi099 bash[32082]: cluster 2024-09-13T14:27:53.560862+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:27:54.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:54 smithi099 bash[22656]: cluster 2024-09-13T14:27:53.204204+0000 mgr.y (mgr.24287) 258 : cluster [DBG] pgmap v279: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 155 KiB/s rd, 16 MiB/s wr, 228 op/s 2024-09-13T14:27:54.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:54 smithi099 bash[22656]: cluster 2024-09-13T14:27:53.560862+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:27:54.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:54 smithi177 bash[24494]: cluster 2024-09-13T14:27:53.204204+0000 mgr.y (mgr.24287) 258 : cluster [DBG] pgmap v279: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 155 KiB/s rd, 16 MiB/s wr, 228 op/s 2024-09-13T14:27:54.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:54 smithi177 bash[24494]: cluster 2024-09-13T14:27:53.560862+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:27:55.434 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:27:55.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:55 smithi099 bash[22656]: cluster 2024-09-13T14:27:54.563314+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:27:55.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:55 smithi099 bash[22656]: audit 2024-09-13T14:27:55.139473+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.99:0/4162673234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:55.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:55 smithi099 bash[32082]: cluster 2024-09-13T14:27:54.563314+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:27:55.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:55 smithi099 bash[32082]: audit 2024-09-13T14:27:55.139473+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.99:0/4162673234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:55.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:27:55 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:27:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:27:55.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:55 smithi177 bash[24494]: cluster 2024-09-13T14:27:54.563314+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:27:55.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:55 smithi177 bash[24494]: audit 2024-09-13T14:27:55.139473+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.99:0/4162673234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:56.354 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:27:56.684 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:27:56.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:56 smithi177 bash[24494]: cluster 2024-09-13T14:27:55.204778+0000 mgr.y (mgr.24287) 259 : cluster [DBG] pgmap v282: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 155 KiB/s rd, 16 MiB/s wr, 228 op/s 2024-09-13T14:27:56.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:56 smithi177 bash[24494]: audit 2024-09-13T14:27:55.429114+0000 mgr.y (mgr.24287) 260 : audit [DBG] from='client.24812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:56.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:56 smithi177 bash[24494]: cluster 2024-09-13T14:27:55.578384+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-09-13T14:27:56.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:56 smithi177 bash[24494]: audit 2024-09-13T14:27:55.726860+0000 mgr.y (mgr.24287) 261 : audit [DBG] from='client.14916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:57.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:56 smithi099 bash[22656]: cluster 2024-09-13T14:27:55.204778+0000 mgr.y (mgr.24287) 259 : cluster [DBG] pgmap v282: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 155 KiB/s rd, 16 MiB/s wr, 228 op/s 2024-09-13T14:27:57.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:56 smithi099 bash[22656]: audit 2024-09-13T14:27:55.429114+0000 mgr.y (mgr.24287) 260 : audit [DBG] from='client.24812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:57.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:56 smithi099 bash[22656]: cluster 2024-09-13T14:27:55.578384+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-09-13T14:27:57.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:56 smithi099 bash[22656]: audit 2024-09-13T14:27:55.726860+0000 mgr.y (mgr.24287) 261 : audit [DBG] from='client.14916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:57.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:56 smithi099 bash[32082]: cluster 2024-09-13T14:27:55.204778+0000 mgr.y (mgr.24287) 259 : cluster [DBG] pgmap v282: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 155 KiB/s rd, 16 MiB/s wr, 228 op/s 2024-09-13T14:27:57.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:56 smithi099 bash[32082]: audit 2024-09-13T14:27:55.429114+0000 mgr.y (mgr.24287) 260 : audit [DBG] from='client.24812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:57.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:56 smithi099 bash[32082]: cluster 2024-09-13T14:27:55.578384+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-09-13T14:27:57.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:56 smithi099 bash[32082]: audit 2024-09-13T14:27:55.726860+0000 mgr.y (mgr.24287) 261 : audit [DBG] from='client.14916 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:58.368 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:27:58.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:58 smithi177 bash[24494]: cluster 2024-09-13T14:27:57.205485+0000 mgr.y (mgr.24287) 262 : cluster [DBG] pgmap v284: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 133 KiB/s rd, 14 MiB/s wr, 196 op/s 2024-09-13T14:27:58.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:58 smithi177 bash[24494]: cluster 2024-09-13T14:27:57.591118+0000 mon.a (mon.0) 929 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-09-13T14:27:58.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:58 smithi177 bash[24494]: audit 2024-09-13T14:27:58.074216+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.99:0/1462911135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:58.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:58 smithi177 bash[24494]: cluster 2024-09-13T14:27:58.551329+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-09-13T14:27:59.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:58 smithi099 bash[22656]: cluster 2024-09-13T14:27:57.205485+0000 mgr.y (mgr.24287) 262 : cluster [DBG] pgmap v284: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 133 KiB/s rd, 14 MiB/s wr, 196 op/s 2024-09-13T14:27:59.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:58 smithi099 bash[22656]: cluster 2024-09-13T14:27:57.591118+0000 mon.a (mon.0) 929 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-09-13T14:27:59.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:58 smithi099 bash[22656]: audit 2024-09-13T14:27:58.074216+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.99:0/1462911135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:59.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:58 smithi099 bash[22656]: cluster 2024-09-13T14:27:58.551329+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-09-13T14:27:59.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:58 smithi099 bash[32082]: cluster 2024-09-13T14:27:57.205485+0000 mgr.y (mgr.24287) 262 : cluster [DBG] pgmap v284: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 133 KiB/s rd, 14 MiB/s wr, 196 op/s 2024-09-13T14:27:59.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:58 smithi099 bash[32082]: cluster 2024-09-13T14:27:57.591118+0000 mon.a (mon.0) 929 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-09-13T14:27:59.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:58 smithi099 bash[32082]: audit 2024-09-13T14:27:58.074216+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.99:0/1462911135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:27:59.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:58 smithi099 bash[32082]: cluster 2024-09-13T14:27:58.551329+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-09-13T14:27:59.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:59 smithi177 bash[24494]: audit 2024-09-13T14:27:58.362875+0000 mgr.y (mgr.24287) 263 : audit [DBG] from='client.14928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:59.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:27:59 smithi177 bash[24494]: audit 2024-09-13T14:27:58.667428+0000 mgr.y (mgr.24287) 264 : audit [DBG] from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:00.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:59 smithi099 bash[22656]: audit 2024-09-13T14:27:58.362875+0000 mgr.y (mgr.24287) 263 : audit [DBG] from='client.14928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:00.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:27:59 smithi099 bash[22656]: audit 2024-09-13T14:27:58.667428+0000 mgr.y (mgr.24287) 264 : audit [DBG] from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:00.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:59 smithi099 bash[32082]: audit 2024-09-13T14:27:58.362875+0000 mgr.y (mgr.24287) 263 : audit [DBG] from='client.14928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:00.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:27:59 smithi099 bash[32082]: audit 2024-09-13T14:27:58.667428+0000 mgr.y (mgr.24287) 264 : audit [DBG] from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:00.199 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.SnapsOtherPool (13279 ms) 2024-09-13T14:28:00.199 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.SnapsDataPool 2024-09-13T14:28:00.284 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:28:00.314 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:28:00.430 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:28:00.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:00 smithi177 bash[24494]: cluster 2024-09-13T14:27:59.206257+0000 mgr.y (mgr.24287) 265 : cluster [DBG] pgmap v287: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s rd, 8.6 MiB/s wr, 319 op/s 2024-09-13T14:28:00.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:00 smithi177 bash[24494]: cluster 2024-09-13T14:27:59.605668+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-09-13T14:28:01.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:00 smithi099 bash[22656]: cluster 2024-09-13T14:27:59.206257+0000 mgr.y (mgr.24287) 265 : cluster [DBG] pgmap v287: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s rd, 8.6 MiB/s wr, 319 op/s 2024-09-13T14:28:01.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:00 smithi099 bash[22656]: cluster 2024-09-13T14:27:59.605668+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-09-13T14:28:01.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:00 smithi099 bash[32082]: cluster 2024-09-13T14:27:59.206257+0000 mgr.y (mgr.24287) 265 : cluster [DBG] pgmap v287: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s rd, 8.6 MiB/s wr, 319 op/s 2024-09-13T14:28:01.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:00 smithi099 bash[32082]: cluster 2024-09-13T14:27:59.605668+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-09-13T14:28:01.321 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:01.633 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 8388609~8388609 2024-09-13T14:28:01.662 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:28:01.663 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:28:01.663 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:28:02.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:01 smithi099 bash[22656]: cluster 2024-09-13T14:28:00.606169+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-09-13T14:28:02.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:01 smithi099 bash[22656]: audit 2024-09-13T14:28:01.015525+0000 mon.a (mon.0) 933 : audit [DBG] from='client.? 172.21.15.99:0/4200405802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:02.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:01 smithi099 bash[32082]: cluster 2024-09-13T14:28:00.606169+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-09-13T14:28:02.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:01 smithi099 bash[32082]: audit 2024-09-13T14:28:01.015525+0000 mon.a (mon.0) 933 : audit [DBG] from='client.? 172.21.15.99:0/4200405802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:02.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:01 smithi177 bash[24494]: cluster 2024-09-13T14:28:00.606169+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-09-13T14:28:02.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:01 smithi177 bash[24494]: audit 2024-09-13T14:28:01.015525+0000 mon.a (mon.0) 933 : audit [DBG] from='client.? 172.21.15.99:0/4200405802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:02.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:28:02 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:02] "GET /metrics HTTP/1.1" 200 246284 "" "Prometheus/2.43.0" 2024-09-13T14:28:03.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:02 smithi099 bash[22656]: cluster 2024-09-13T14:28:01.206967+0000 mgr.y (mgr.24287) 266 : cluster [DBG] pgmap v290: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 371 op/s 2024-09-13T14:28:03.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:02 smithi099 bash[22656]: audit 2024-09-13T14:28:01.315502+0000 mgr.y (mgr.24287) 267 : audit [DBG] from='client.24848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:03.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:02 smithi099 bash[22656]: audit 2024-09-13T14:28:01.606215+0000 mgr.y (mgr.24287) 268 : audit [DBG] from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:03.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:02 smithi099 bash[22656]: cluster 2024-09-13T14:28:01.622234+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:28:03.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:02 smithi099 bash[32082]: cluster 2024-09-13T14:28:01.206967+0000 mgr.y (mgr.24287) 266 : cluster [DBG] pgmap v290: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 371 op/s 2024-09-13T14:28:03.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:02 smithi099 bash[32082]: audit 2024-09-13T14:28:01.315502+0000 mgr.y (mgr.24287) 267 : audit [DBG] from='client.24848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:03.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:02 smithi099 bash[32082]: audit 2024-09-13T14:28:01.606215+0000 mgr.y (mgr.24287) 268 : audit [DBG] from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:03.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:02 smithi099 bash[32082]: cluster 2024-09-13T14:28:01.622234+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:28:03.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:02 smithi177 bash[24494]: cluster 2024-09-13T14:28:01.206967+0000 mgr.y (mgr.24287) 266 : cluster [DBG] pgmap v290: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 371 op/s 2024-09-13T14:28:03.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:02 smithi177 bash[24494]: audit 2024-09-13T14:28:01.315502+0000 mgr.y (mgr.24287) 267 : audit [DBG] from='client.24848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:03.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:02 smithi177 bash[24494]: audit 2024-09-13T14:28:01.606215+0000 mgr.y (mgr.24287) 268 : audit [DBG] from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:03.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:02 smithi177 bash[24494]: cluster 2024-09-13T14:28:01.622234+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:28:03.661 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 8388609~1000 2024-09-13T14:28:03.682 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16777228~1000 2024-09-13T14:28:03.683 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:28:03.702 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 83886079~8388609 2024-09-13T14:28:03.737 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:28:04.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:03 smithi099 bash[22656]: cluster 2024-09-13T14:28:02.634771+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:28:04.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:03 smithi099 bash[32082]: cluster 2024-09-13T14:28:02.634771+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:28:04.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:03 smithi177 bash[24494]: cluster 2024-09-13T14:28:02.634771+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:28:04.278 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:05.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:04 smithi099 bash[22656]: cluster 2024-09-13T14:28:03.208499+0000 mgr.y (mgr.24287) 269 : cluster [DBG] pgmap v293: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-09-13T14:28:05.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:04 smithi099 bash[22656]: cluster 2024-09-13T14:28:03.645871+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:28:05.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:04 smithi099 bash[22656]: audit 2024-09-13T14:28:03.977474+0000 mon.a (mon.0) 937 : audit [DBG] from='client.? 172.21.15.99:0/2275771812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:05.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:04 smithi099 bash[32082]: cluster 2024-09-13T14:28:03.208499+0000 mgr.y (mgr.24287) 269 : cluster [DBG] pgmap v293: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-09-13T14:28:05.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:04 smithi099 bash[32082]: cluster 2024-09-13T14:28:03.645871+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:28:05.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:04 smithi099 bash[32082]: audit 2024-09-13T14:28:03.977474+0000 mon.a (mon.0) 937 : audit [DBG] from='client.? 172.21.15.99:0/2275771812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:05.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:04 smithi177 bash[24494]: cluster 2024-09-13T14:28:03.208499+0000 mgr.y (mgr.24287) 269 : cluster [DBG] pgmap v293: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-09-13T14:28:05.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:04 smithi177 bash[24494]: cluster 2024-09-13T14:28:03.645871+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:28:05.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:04 smithi177 bash[24494]: audit 2024-09-13T14:28:03.977474+0000 mon.a (mon.0) 937 : audit [DBG] from='client.? 172.21.15.99:0/2275771812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:05.651 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:05 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:28:05.678 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:28:05.740 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83886079~1 2024-09-13T14:28:05.765 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:28:06.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:05 smithi099 bash[22656]: audit 2024-09-13T14:28:04.272672+0000 mgr.y (mgr.24287) 270 : audit [DBG] from='client.24866 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:05 smithi099 bash[22656]: audit 2024-09-13T14:28:04.564114+0000 mgr.y (mgr.24287) 271 : audit [DBG] from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:05 smithi099 bash[22656]: cluster 2024-09-13T14:28:04.645055+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:28:06.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:05 smithi099 bash[32082]: audit 2024-09-13T14:28:04.272672+0000 mgr.y (mgr.24287) 270 : audit [DBG] from='client.24866 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:05 smithi099 bash[32082]: audit 2024-09-13T14:28:04.564114+0000 mgr.y (mgr.24287) 271 : audit [DBG] from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:05 smithi099 bash[32082]: cluster 2024-09-13T14:28:04.645055+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:28:06.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:05 smithi177 bash[24494]: audit 2024-09-13T14:28:04.272672+0000 mgr.y (mgr.24287) 270 : audit [DBG] from='client.24866 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:05 smithi177 bash[24494]: audit 2024-09-13T14:28:04.564114+0000 mgr.y (mgr.24287) 271 : audit [DBG] from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:05 smithi177 bash[24494]: cluster 2024-09-13T14:28:04.645055+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:28:07.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:06 smithi099 bash[32082]: cluster 2024-09-13T14:28:05.209233+0000 mgr.y (mgr.24287) 272 : cluster [DBG] pgmap v296: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-09-13T14:28:07.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:06 smithi099 bash[32082]: cluster 2024-09-13T14:28:05.664659+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:28:07.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:06 smithi099 bash[22656]: cluster 2024-09-13T14:28:05.209233+0000 mgr.y (mgr.24287) 272 : cluster [DBG] pgmap v296: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-09-13T14:28:07.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:06 smithi099 bash[22656]: cluster 2024-09-13T14:28:05.664659+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:28:07.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:06 smithi177 bash[24494]: cluster 2024-09-13T14:28:05.209233+0000 mgr.y (mgr.24287) 272 : cluster [DBG] pgmap v296: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-09-13T14:28:07.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:06 smithi177 bash[24494]: cluster 2024-09-13T14:28:05.664659+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:28:07.200 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:08.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:07 smithi099 bash[22656]: cluster 2024-09-13T14:28:06.669327+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:28:08.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:07 smithi099 bash[22656]: audit 2024-09-13T14:28:06.909736+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.99:0/3434490099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:08.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:07 smithi099 bash[22656]: audit 2024-09-13T14:28:07.158493+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:08.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:07 smithi099 bash[32082]: cluster 2024-09-13T14:28:06.669327+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:28:08.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:07 smithi099 bash[32082]: audit 2024-09-13T14:28:06.909736+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.99:0/3434490099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:08.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:07 smithi099 bash[32082]: audit 2024-09-13T14:28:07.158493+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:08.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:07 smithi177 bash[24494]: cluster 2024-09-13T14:28:06.669327+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:28:08.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:07 smithi177 bash[24494]: audit 2024-09-13T14:28:06.909736+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.99:0/3434490099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:08.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:07 smithi177 bash[24494]: audit 2024-09-13T14:28:07.158493+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:09.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:08 smithi099 bash[32082]: audit 2024-09-13T14:28:07.194218+0000 mgr.y (mgr.24287) 273 : audit [DBG] from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:09.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:08 smithi099 bash[32082]: cluster 2024-09-13T14:28:07.209735+0000 mgr.y (mgr.24287) 274 : cluster [DBG] pgmap v299: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:09.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:08 smithi099 bash[32082]: audit 2024-09-13T14:28:07.483152+0000 mgr.y (mgr.24287) 275 : audit [DBG] from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:09.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:08 smithi099 bash[32082]: cluster 2024-09-13T14:28:07.683690+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:28:09.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:08 smithi099 bash[32082]: cluster 2024-09-13T14:28:08.555033+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:28:09.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:08 smithi099 bash[22656]: audit 2024-09-13T14:28:07.194218+0000 mgr.y (mgr.24287) 273 : audit [DBG] from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:09.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:08 smithi099 bash[22656]: cluster 2024-09-13T14:28:07.209735+0000 mgr.y (mgr.24287) 274 : cluster [DBG] pgmap v299: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:09.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:08 smithi099 bash[22656]: audit 2024-09-13T14:28:07.483152+0000 mgr.y (mgr.24287) 275 : audit [DBG] from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:09.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:08 smithi099 bash[22656]: cluster 2024-09-13T14:28:07.683690+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:28:09.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:08 smithi099 bash[22656]: cluster 2024-09-13T14:28:08.555033+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:28:09.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:08 smithi177 bash[24494]: audit 2024-09-13T14:28:07.194218+0000 mgr.y (mgr.24287) 273 : audit [DBG] from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:09.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:08 smithi177 bash[24494]: cluster 2024-09-13T14:28:07.209735+0000 mgr.y (mgr.24287) 274 : cluster [DBG] pgmap v299: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:09.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:08 smithi177 bash[24494]: audit 2024-09-13T14:28:07.483152+0000 mgr.y (mgr.24287) 275 : audit [DBG] from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:09.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:08 smithi177 bash[24494]: cluster 2024-09-13T14:28:07.683690+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:28:09.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:08 smithi177 bash[24494]: cluster 2024-09-13T14:28:08.555033+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:28:09.498 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:28:09.735 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:28:10.169 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:11.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:10 smithi099 bash[22656]: cluster 2024-09-13T14:28:09.210470+0000 mgr.y (mgr.24287) 276 : cluster [DBG] pgmap v302: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T14:28:11.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:10 smithi099 bash[22656]: audit 2024-09-13T14:28:09.874185+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.99:0/3896955035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:11.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:10 smithi099 bash[32082]: cluster 2024-09-13T14:28:09.210470+0000 mgr.y (mgr.24287) 276 : cluster [DBG] pgmap v302: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T14:28:11.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:10 smithi099 bash[32082]: audit 2024-09-13T14:28:09.874185+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.99:0/3896955035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:11.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:10 smithi177 bash[24494]: cluster 2024-09-13T14:28:09.210470+0000 mgr.y (mgr.24287) 276 : cluster [DBG] pgmap v302: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T14:28:11.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:10 smithi177 bash[24494]: audit 2024-09-13T14:28:09.874185+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.99:0/3896955035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:12.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:11 smithi099 bash[22656]: audit 2024-09-13T14:28:10.163440+0000 mgr.y (mgr.24287) 277 : audit [DBG] from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:12.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:11 smithi099 bash[22656]: audit 2024-09-13T14:28:10.452500+0000 mgr.y (mgr.24287) 278 : audit [DBG] from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:12.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:11 smithi099 bash[22656]: cluster 2024-09-13T14:28:10.696704+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:28:12.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:11 smithi099 bash[32082]: audit 2024-09-13T14:28:10.163440+0000 mgr.y (mgr.24287) 277 : audit [DBG] from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:12.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:11 smithi099 bash[32082]: audit 2024-09-13T14:28:10.452500+0000 mgr.y (mgr.24287) 278 : audit [DBG] from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:12.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:11 smithi099 bash[32082]: cluster 2024-09-13T14:28:10.696704+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:28:12.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:11 smithi177 bash[24494]: audit 2024-09-13T14:28:10.163440+0000 mgr.y (mgr.24287) 277 : audit [DBG] from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:12.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:11 smithi177 bash[24494]: audit 2024-09-13T14:28:10.452500+0000 mgr.y (mgr.24287) 278 : audit [DBG] from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:12.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:11 smithi177 bash[24494]: cluster 2024-09-13T14:28:10.696704+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:28:12.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:28:12 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:12] "GET /metrics HTTP/1.1" 200 246354 "" "Prometheus/2.43.0" 2024-09-13T14:28:13.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:12 smithi099 bash[32082]: cluster 2024-09-13T14:28:11.211208+0000 mgr.y (mgr.24287) 279 : cluster [DBG] pgmap v304: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 105 KiB/s rd, 3.5 MiB/s wr, 149 op/s 2024-09-13T14:28:13.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:12 smithi099 bash[32082]: cluster 2024-09-13T14:28:11.713737+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:28:13.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:12 smithi099 bash[22656]: cluster 2024-09-13T14:28:11.211208+0000 mgr.y (mgr.24287) 279 : cluster [DBG] pgmap v304: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 105 KiB/s rd, 3.5 MiB/s wr, 149 op/s 2024-09-13T14:28:13.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:12 smithi099 bash[22656]: cluster 2024-09-13T14:28:11.713737+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:28:13.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:12 smithi177 bash[24494]: cluster 2024-09-13T14:28:11.211208+0000 mgr.y (mgr.24287) 279 : cluster [DBG] pgmap v304: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 105 KiB/s rd, 3.5 MiB/s wr, 149 op/s 2024-09-13T14:28:13.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:12 smithi177 bash[24494]: cluster 2024-09-13T14:28:11.713737+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:28:13.131 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.SnapsDataPool (12933 ms) 2024-09-13T14:28:13.131 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.SnapsShrinkAfterPrepare 2024-09-13T14:28:13.188 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:28:13.193 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:13.202 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:28:13.278 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:28:14.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:13 smithi099 bash[22656]: cluster 2024-09-13T14:28:12.718353+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:28:14.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:13 smithi099 bash[22656]: audit 2024-09-13T14:28:12.833771+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.99:0/2788397678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:14.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:13 smithi099 bash[32082]: cluster 2024-09-13T14:28:12.718353+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:28:14.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:13 smithi099 bash[32082]: audit 2024-09-13T14:28:12.833771+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.99:0/2788397678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:14.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:13 smithi177 bash[24494]: cluster 2024-09-13T14:28:12.718353+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:28:14.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:13 smithi177 bash[24494]: audit 2024-09-13T14:28:12.833771+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.99:0/2788397678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:14.756 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 8388609~8388609 2024-09-13T14:28:14.793 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:28:14.794 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:28:14.794 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:28:15.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:14 smithi099 bash[22656]: audit 2024-09-13T14:28:13.187301+0000 mgr.y (mgr.24287) 280 : audit [DBG] from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:15.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:14 smithi099 bash[22656]: cluster 2024-09-13T14:28:13.212288+0000 mgr.y (mgr.24287) 281 : cluster [DBG] pgmap v307: 137 pgs: 137 active+clean; 393 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 60 MiB/s rd, 5.2 MiB/s wr, 374 op/s 2024-09-13T14:28:15.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:14 smithi099 bash[22656]: audit 2024-09-13T14:28:13.481626+0000 mgr.y (mgr.24287) 282 : audit [DBG] from='client.24926 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:15.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:14 smithi099 bash[22656]: cluster 2024-09-13T14:28:13.737838+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:28:15.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:14 smithi099 bash[32082]: audit 2024-09-13T14:28:13.187301+0000 mgr.y (mgr.24287) 280 : audit [DBG] from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:15.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:14 smithi099 bash[32082]: cluster 2024-09-13T14:28:13.212288+0000 mgr.y (mgr.24287) 281 : cluster [DBG] pgmap v307: 137 pgs: 137 active+clean; 393 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 60 MiB/s rd, 5.2 MiB/s wr, 374 op/s 2024-09-13T14:28:15.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:14 smithi099 bash[32082]: audit 2024-09-13T14:28:13.481626+0000 mgr.y (mgr.24287) 282 : audit [DBG] from='client.24926 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:15.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:14 smithi099 bash[32082]: cluster 2024-09-13T14:28:13.737838+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:28:15.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:14 smithi177 bash[24494]: audit 2024-09-13T14:28:13.187301+0000 mgr.y (mgr.24287) 280 : audit [DBG] from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:15.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:14 smithi177 bash[24494]: cluster 2024-09-13T14:28:13.212288+0000 mgr.y (mgr.24287) 281 : cluster [DBG] pgmap v307: 137 pgs: 137 active+clean; 393 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 60 MiB/s rd, 5.2 MiB/s wr, 374 op/s 2024-09-13T14:28:15.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:14 smithi177 bash[24494]: audit 2024-09-13T14:28:13.481626+0000 mgr.y (mgr.24287) 282 : audit [DBG] from='client.24926 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:15.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:14 smithi177 bash[24494]: cluster 2024-09-13T14:28:13.737838+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:28:15.744 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:15 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:28:16.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:15 smithi099 bash[22656]: cluster 2024-09-13T14:28:14.738548+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:28:16.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:15 smithi099 bash[32082]: cluster 2024-09-13T14:28:14.738548+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:28:16.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:15 smithi177 bash[24494]: cluster 2024-09-13T14:28:14.738548+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:28:16.126 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:16.789 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 8388609~1000 2024-09-13T14:28:16.804 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16777228~1000 2024-09-13T14:28:16.804 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:28:16.819 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 83886079~8388609 2024-09-13T14:28:16.874 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:28:17.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:16 smithi099 bash[22656]: cluster 2024-09-13T14:28:15.213006+0000 mgr.y (mgr.24287) 283 : cluster [DBG] pgmap v310: 137 pgs: 137 active+clean; 393 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 435 op/s 2024-09-13T14:28:17.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:16 smithi099 bash[22656]: cluster 2024-09-13T14:28:15.759912+0000 mon.a (mon.0) 948 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:28:17.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:16 smithi099 bash[22656]: audit 2024-09-13T14:28:15.836021+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.99:0/542891446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:17.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:16 smithi099 bash[32082]: cluster 2024-09-13T14:28:15.213006+0000 mgr.y (mgr.24287) 283 : cluster [DBG] pgmap v310: 137 pgs: 137 active+clean; 393 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 435 op/s 2024-09-13T14:28:17.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:16 smithi099 bash[32082]: cluster 2024-09-13T14:28:15.759912+0000 mon.a (mon.0) 948 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:28:17.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:16 smithi099 bash[32082]: audit 2024-09-13T14:28:15.836021+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.99:0/542891446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:17.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:16 smithi177 bash[24494]: cluster 2024-09-13T14:28:15.213006+0000 mgr.y (mgr.24287) 283 : cluster [DBG] pgmap v310: 137 pgs: 137 active+clean; 393 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 435 op/s 2024-09-13T14:28:17.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:16 smithi177 bash[24494]: cluster 2024-09-13T14:28:15.759912+0000 mon.a (mon.0) 948 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:28:17.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:16 smithi177 bash[24494]: audit 2024-09-13T14:28:15.836021+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.99:0/542891446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:18.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:17 smithi099 bash[32082]: audit 2024-09-13T14:28:16.120826+0000 mgr.y (mgr.24287) 284 : audit [DBG] from='client.24938 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:18.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:17 smithi099 bash[32082]: audit 2024-09-13T14:28:16.409819+0000 mgr.y (mgr.24287) 285 : audit [DBG] from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:18.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:17 smithi099 bash[32082]: cluster 2024-09-13T14:28:16.772838+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:28:18.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:17 smithi099 bash[22656]: audit 2024-09-13T14:28:16.120826+0000 mgr.y (mgr.24287) 284 : audit [DBG] from='client.24938 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:18.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:17 smithi099 bash[22656]: audit 2024-09-13T14:28:16.409819+0000 mgr.y (mgr.24287) 285 : audit [DBG] from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:18.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:17 smithi099 bash[22656]: cluster 2024-09-13T14:28:16.772838+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:28:18.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:17 smithi177 bash[24494]: audit 2024-09-13T14:28:16.120826+0000 mgr.y (mgr.24287) 284 : audit [DBG] from='client.24938 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:18.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:17 smithi177 bash[24494]: audit 2024-09-13T14:28:16.409819+0000 mgr.y (mgr.24287) 285 : audit [DBG] from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:18.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:17 smithi177 bash[24494]: cluster 2024-09-13T14:28:16.772838+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:28:18.562 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:28:18.596 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83886079~1 2024-09-13T14:28:18.607 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:28:19.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:18 smithi099 bash[32082]: cluster 2024-09-13T14:28:17.213747+0000 mgr.y (mgr.24287) 286 : cluster [DBG] pgmap v313: 137 pgs: 137 active+clean; 393 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:19.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:18 smithi099 bash[32082]: cluster 2024-09-13T14:28:17.780253+0000 mon.a (mon.0) 950 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:28:19.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:18 smithi099 bash[32082]: cluster 2024-09-13T14:28:18.555176+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:28:19.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:18 smithi099 bash[22656]: cluster 2024-09-13T14:28:17.213747+0000 mgr.y (mgr.24287) 286 : cluster [DBG] pgmap v313: 137 pgs: 137 active+clean; 393 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:19.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:18 smithi099 bash[22656]: cluster 2024-09-13T14:28:17.780253+0000 mon.a (mon.0) 950 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:28:19.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:18 smithi099 bash[22656]: cluster 2024-09-13T14:28:18.555176+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:28:19.079 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:19.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:18 smithi177 bash[24494]: cluster 2024-09-13T14:28:17.213747+0000 mgr.y (mgr.24287) 286 : cluster [DBG] pgmap v313: 137 pgs: 137 active+clean; 393 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:19.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:18 smithi177 bash[24494]: cluster 2024-09-13T14:28:17.780253+0000 mon.a (mon.0) 950 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:28:19.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:18 smithi177 bash[24494]: cluster 2024-09-13T14:28:18.555176+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:28:20.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:19 smithi099 bash[22656]: audit 2024-09-13T14:28:18.781738+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.99:0/1621257919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:20.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:19 smithi099 bash[22656]: cluster 2024-09-13T14:28:19.560513+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:28:20.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:19 smithi099 bash[32082]: audit 2024-09-13T14:28:18.781738+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.99:0/1621257919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:20.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:19 smithi099 bash[32082]: cluster 2024-09-13T14:28:19.560513+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:28:20.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:19 smithi177 bash[24494]: audit 2024-09-13T14:28:18.781738+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.99:0/1621257919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:20.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:19 smithi177 bash[24494]: cluster 2024-09-13T14:28:19.560513+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:28:21.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:20 smithi099 bash[22656]: audit 2024-09-13T14:28:19.073741+0000 mgr.y (mgr.24287) 287 : audit [DBG] from='client.15054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:21.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:20 smithi099 bash[22656]: cluster 2024-09-13T14:28:19.214245+0000 mgr.y (mgr.24287) 288 : cluster [DBG] pgmap v316: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-09-13T14:28:21.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:20 smithi099 bash[22656]: audit 2024-09-13T14:28:19.360621+0000 mgr.y (mgr.24287) 289 : audit [DBG] from='client.15060 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:21.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:20 smithi099 bash[22656]: cluster 2024-09-13T14:28:20.563455+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:28:21.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:20 smithi099 bash[32082]: audit 2024-09-13T14:28:19.073741+0000 mgr.y (mgr.24287) 287 : audit [DBG] from='client.15054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:21.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:20 smithi099 bash[32082]: cluster 2024-09-13T14:28:19.214245+0000 mgr.y (mgr.24287) 288 : cluster [DBG] pgmap v316: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-09-13T14:28:21.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:20 smithi099 bash[32082]: audit 2024-09-13T14:28:19.360621+0000 mgr.y (mgr.24287) 289 : audit [DBG] from='client.15060 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:21.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:20 smithi099 bash[32082]: cluster 2024-09-13T14:28:20.563455+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:28:21.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:20 smithi177 bash[24494]: audit 2024-09-13T14:28:19.073741+0000 mgr.y (mgr.24287) 287 : audit [DBG] from='client.15054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:21.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:20 smithi177 bash[24494]: cluster 2024-09-13T14:28:19.214245+0000 mgr.y (mgr.24287) 288 : cluster [DBG] pgmap v316: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-09-13T14:28:21.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:20 smithi177 bash[24494]: audit 2024-09-13T14:28:19.360621+0000 mgr.y (mgr.24287) 289 : audit [DBG] from='client.15060 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:21.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:20 smithi177 bash[24494]: cluster 2024-09-13T14:28:20.563455+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:28:22.050 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:22.195 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 41943040 2024-09-13T14:28:22.224 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:28:22.378 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:28:22.554 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:28:22 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:22] "GET /metrics HTTP/1.1" 200 246354 "" "Prometheus/2.43.0" 2024-09-13T14:28:22.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:22 smithi099 bash[22656]: cluster 2024-09-13T14:28:21.214953+0000 mgr.y (mgr.24287) 290 : cluster [DBG] pgmap v319: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-09-13T14:28:22.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:22 smithi099 bash[22656]: cluster 2024-09-13T14:28:21.565727+0000 mon.a (mon.0) 954 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:28:22.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:22 smithi099 bash[22656]: audit 2024-09-13T14:28:21.718139+0000 mon.a (mon.0) 955 : audit [DBG] from='client.? 172.21.15.99:0/3965551262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:22.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:22 smithi099 bash[22656]: audit 2024-09-13T14:28:22.158409+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:22.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:22 smithi099 bash[32082]: cluster 2024-09-13T14:28:21.214953+0000 mgr.y (mgr.24287) 290 : cluster [DBG] pgmap v319: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-09-13T14:28:22.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:22 smithi099 bash[32082]: cluster 2024-09-13T14:28:21.565727+0000 mon.a (mon.0) 954 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:28:22.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:22 smithi099 bash[32082]: audit 2024-09-13T14:28:21.718139+0000 mon.a (mon.0) 955 : audit [DBG] from='client.? 172.21.15.99:0/3965551262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:22.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:22 smithi099 bash[32082]: audit 2024-09-13T14:28:22.158409+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:22.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:22 smithi177 bash[24494]: cluster 2024-09-13T14:28:21.214953+0000 mgr.y (mgr.24287) 290 : cluster [DBG] pgmap v319: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 101 KiB/s rd, 12 MiB/s wr, 152 op/s 2024-09-13T14:28:22.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:22 smithi177 bash[24494]: cluster 2024-09-13T14:28:21.565727+0000 mon.a (mon.0) 954 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:28:22.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:22 smithi177 bash[24494]: audit 2024-09-13T14:28:21.718139+0000 mon.a (mon.0) 955 : audit [DBG] from='client.? 172.21.15.99:0/3965551262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:22.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:22 smithi177 bash[24494]: audit 2024-09-13T14:28:22.158409+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:24.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:23 smithi099 bash[22656]: audit 2024-09-13T14:28:22.046363+0000 mgr.y (mgr.24287) 291 : audit [DBG] from='client.24974 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:24.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:23 smithi099 bash[22656]: audit 2024-09-13T14:28:22.372185+0000 mgr.y (mgr.24287) 292 : audit [DBG] from='client.24980 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:24.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:23 smithi099 bash[22656]: cluster 2024-09-13T14:28:22.796168+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:28:24.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:23 smithi099 bash[32082]: audit 2024-09-13T14:28:22.046363+0000 mgr.y (mgr.24287) 291 : audit [DBG] from='client.24974 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:24.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:23 smithi099 bash[32082]: audit 2024-09-13T14:28:22.372185+0000 mgr.y (mgr.24287) 292 : audit [DBG] from='client.24980 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:24.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:23 smithi099 bash[32082]: cluster 2024-09-13T14:28:22.796168+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:28:24.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:23 smithi177 bash[24494]: audit 2024-09-13T14:28:22.046363+0000 mgr.y (mgr.24287) 291 : audit [DBG] from='client.24974 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:24.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:23 smithi177 bash[24494]: audit 2024-09-13T14:28:22.372185+0000 mgr.y (mgr.24287) 292 : audit [DBG] from='client.24980 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:24.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:23 smithi177 bash[24494]: cluster 2024-09-13T14:28:22.796168+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:28:25.040 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:25.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:24 smithi099 bash[22656]: cluster 2024-09-13T14:28:23.216512+0000 mgr.y (mgr.24287) 293 : cluster [DBG] pgmap v322: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 520 op/s 2024-09-13T14:28:25.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:24 smithi099 bash[22656]: cluster 2024-09-13T14:28:23.807924+0000 mon.a (mon.0) 957 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:28:25.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:24 smithi099 bash[22656]: audit 2024-09-13T14:28:24.718023+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.99:0/2387978794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:25.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:24 smithi099 bash[32082]: cluster 2024-09-13T14:28:23.216512+0000 mgr.y (mgr.24287) 293 : cluster [DBG] pgmap v322: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 520 op/s 2024-09-13T14:28:25.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:24 smithi099 bash[32082]: cluster 2024-09-13T14:28:23.807924+0000 mon.a (mon.0) 957 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:28:25.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:24 smithi099 bash[32082]: audit 2024-09-13T14:28:24.718023+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.99:0/2387978794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:25.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:24 smithi177 bash[24494]: cluster 2024-09-13T14:28:23.216512+0000 mgr.y (mgr.24287) 293 : cluster [DBG] pgmap v322: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 520 op/s 2024-09-13T14:28:25.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:24 smithi177 bash[24494]: cluster 2024-09-13T14:28:23.807924+0000 mon.a (mon.0) 957 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:28:25.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:24 smithi177 bash[24494]: audit 2024-09-13T14:28:24.718023+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.99:0/2387978794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:25.192 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.SnapsShrinkAfterPrepare (12060 ms) 2024-09-13T14:28:25.192 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroBeforePrepare 2024-09-13T14:28:25.229 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:28:25.245 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:28:25.323 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:28:25.812 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:25 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:28:26.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:25 smithi099 bash[22656]: cluster 2024-09-13T14:28:24.813601+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:28:26.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:25 smithi099 bash[32082]: cluster 2024-09-13T14:28:24.813601+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:28:26.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:25 smithi177 bash[24494]: cluster 2024-09-13T14:28:24.813601+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:28:26.854 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 8388609~8388609 2024-09-13T14:28:26.888 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:28:26.888 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:28:26.888 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:28:27.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:26 smithi099 bash[22656]: audit 2024-09-13T14:28:25.028726+0000 mgr.y (mgr.24287) 294 : audit [DBG] from='client.15090 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:27.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:26 smithi099 bash[22656]: cluster 2024-09-13T14:28:25.216981+0000 mgr.y (mgr.24287) 295 : cluster [DBG] pgmap v325: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 520 op/s 2024-09-13T14:28:27.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:26 smithi099 bash[22656]: audit 2024-09-13T14:28:25.342905+0000 mgr.y (mgr.24287) 296 : audit [DBG] from='client.24998 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:27.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:26 smithi099 bash[22656]: cluster 2024-09-13T14:28:25.827984+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:28:27.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:26 smithi099 bash[32082]: audit 2024-09-13T14:28:25.028726+0000 mgr.y (mgr.24287) 294 : audit [DBG] from='client.15090 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:27.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:26 smithi099 bash[32082]: cluster 2024-09-13T14:28:25.216981+0000 mgr.y (mgr.24287) 295 : cluster [DBG] pgmap v325: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 520 op/s 2024-09-13T14:28:27.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:26 smithi099 bash[32082]: audit 2024-09-13T14:28:25.342905+0000 mgr.y (mgr.24287) 296 : audit [DBG] from='client.24998 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:27.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:26 smithi099 bash[32082]: cluster 2024-09-13T14:28:25.827984+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:28:27.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:26 smithi177 bash[24494]: audit 2024-09-13T14:28:25.028726+0000 mgr.y (mgr.24287) 294 : audit [DBG] from='client.15090 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:27.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:26 smithi177 bash[24494]: cluster 2024-09-13T14:28:25.216981+0000 mgr.y (mgr.24287) 295 : cluster [DBG] pgmap v325: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 520 op/s 2024-09-13T14:28:27.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:26 smithi177 bash[24494]: audit 2024-09-13T14:28:25.342905+0000 mgr.y (mgr.24287) 296 : audit [DBG] from='client.24998 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:27.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:26 smithi177 bash[24494]: cluster 2024-09-13T14:28:25.827984+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:28:27.977 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:28.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:27 smithi177 bash[24494]: cluster 2024-09-13T14:28:26.839517+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:28:28.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:27 smithi177 bash[24494]: audit 2024-09-13T14:28:27.688057+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.99:0/919741572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:28.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:27 smithi099 bash[32082]: cluster 2024-09-13T14:28:26.839517+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:28:28.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:27 smithi099 bash[32082]: audit 2024-09-13T14:28:27.688057+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.99:0/919741572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:28.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:27 smithi099 bash[22656]: cluster 2024-09-13T14:28:26.839517+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:28:28.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:27 smithi099 bash[22656]: audit 2024-09-13T14:28:27.688057+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.99:0/919741572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:28.577 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 8388609~1000 2024-09-13T14:28:28.596 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16777228~1000 2024-09-13T14:28:28.596 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:28:28.619 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 83886079~8388609 2024-09-13T14:28:28.667 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:28:29.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:28 smithi177 bash[24494]: cluster 2024-09-13T14:28:27.217701+0000 mgr.y (mgr.24287) 297 : cluster [DBG] pgmap v328: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:29.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:28 smithi177 bash[24494]: cluster 2024-09-13T14:28:27.853789+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:28:29.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:28 smithi177 bash[24494]: cluster 2024-09-13T14:28:28.564277+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:28:29.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:28 smithi099 bash[22656]: cluster 2024-09-13T14:28:27.217701+0000 mgr.y (mgr.24287) 297 : cluster [DBG] pgmap v328: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:29.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:28 smithi099 bash[22656]: cluster 2024-09-13T14:28:27.853789+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:28:29.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:28 smithi099 bash[22656]: cluster 2024-09-13T14:28:28.564277+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:28:29.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:28 smithi099 bash[32082]: cluster 2024-09-13T14:28:27.217701+0000 mgr.y (mgr.24287) 297 : cluster [DBG] pgmap v328: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:29.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:28 smithi099 bash[32082]: cluster 2024-09-13T14:28:27.853789+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:28:29.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:28 smithi099 bash[32082]: cluster 2024-09-13T14:28:28.564277+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:28:30.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:29 smithi177 bash[24494]: audit 2024-09-13T14:28:27.972413+0000 mgr.y (mgr.24287) 298 : audit [DBG] from='client.25010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:30.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:29 smithi177 bash[24494]: audit 2024-09-13T14:28:28.260189+0000 mgr.y (mgr.24287) 299 : audit [DBG] from='client.25016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:30.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:29 smithi177 bash[24494]: cluster 2024-09-13T14:28:29.566318+0000 mon.a (mon.0) 963 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:28:30.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:29 smithi099 bash[22656]: audit 2024-09-13T14:28:27.972413+0000 mgr.y (mgr.24287) 298 : audit [DBG] from='client.25010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:30.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:29 smithi099 bash[22656]: audit 2024-09-13T14:28:28.260189+0000 mgr.y (mgr.24287) 299 : audit [DBG] from='client.25016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:30.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:29 smithi099 bash[22656]: cluster 2024-09-13T14:28:29.566318+0000 mon.a (mon.0) 963 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:28:30.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:29 smithi099 bash[32082]: audit 2024-09-13T14:28:27.972413+0000 mgr.y (mgr.24287) 298 : audit [DBG] from='client.25010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:30.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:29 smithi099 bash[32082]: audit 2024-09-13T14:28:28.260189+0000 mgr.y (mgr.24287) 299 : audit [DBG] from='client.25016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:30.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:29 smithi099 bash[32082]: cluster 2024-09-13T14:28:29.566318+0000 mon.a (mon.0) 963 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:28:30.574 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:28:30.625 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83886079~1 2024-09-13T14:28:30.644 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 0 2024-09-13T14:28:30.677 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:28:30.915 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:31.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:30 smithi177 bash[24494]: cluster 2024-09-13T14:28:29.218595+0000 mgr.y (mgr.24287) 300 : cluster [DBG] pgmap v331: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T14:28:31.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:30 smithi177 bash[24494]: cluster 2024-09-13T14:28:30.566747+0000 mon.a (mon.0) 964 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:28:31.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:30 smithi177 bash[24494]: audit 2024-09-13T14:28:30.615346+0000 mon.a (mon.0) 965 : audit [DBG] from='client.? 172.21.15.99:0/2369293667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:31.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:30 smithi099 bash[22656]: cluster 2024-09-13T14:28:29.218595+0000 mgr.y (mgr.24287) 300 : cluster [DBG] pgmap v331: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T14:28:31.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:30 smithi099 bash[22656]: cluster 2024-09-13T14:28:30.566747+0000 mon.a (mon.0) 964 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:28:31.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:30 smithi099 bash[22656]: audit 2024-09-13T14:28:30.615346+0000 mon.a (mon.0) 965 : audit [DBG] from='client.? 172.21.15.99:0/2369293667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:31.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:30 smithi099 bash[32082]: cluster 2024-09-13T14:28:29.218595+0000 mgr.y (mgr.24287) 300 : cluster [DBG] pgmap v331: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T14:28:31.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:30 smithi099 bash[32082]: cluster 2024-09-13T14:28:30.566747+0000 mon.a (mon.0) 964 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:28:31.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:30 smithi099 bash[32082]: audit 2024-09-13T14:28:30.615346+0000 mon.a (mon.0) 965 : audit [DBG] from='client.? 172.21.15.99:0/2369293667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:32.115 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:31 smithi099 bash[22656]: audit 2024-09-13T14:28:30.909806+0000 mgr.y (mgr.24287) 301 : audit [DBG] from='client.25028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:32.115 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:31 smithi099 bash[22656]: cluster 2024-09-13T14:28:31.569622+0000 mon.a (mon.0) 966 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:28:32.115 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:31 smithi099 bash[32082]: audit 2024-09-13T14:28:30.909806+0000 mgr.y (mgr.24287) 301 : audit [DBG] from='client.25028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:32.115 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:31 smithi099 bash[32082]: cluster 2024-09-13T14:28:31.569622+0000 mon.a (mon.0) 966 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:28:32.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:31 smithi177 bash[24494]: audit 2024-09-13T14:28:30.909806+0000 mgr.y (mgr.24287) 301 : audit [DBG] from='client.25028 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:32.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:31 smithi177 bash[24494]: cluster 2024-09-13T14:28:31.569622+0000 mon.a (mon.0) 966 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:28:32.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:28:32 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:32] "GET /metrics HTTP/1.1" 200 246406 "" "Prometheus/2.43.0" 2024-09-13T14:28:33.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:32 smithi099 bash[32082]: audit 2024-09-13T14:28:31.195937+0000 mgr.y (mgr.24287) 302 : audit [DBG] from='client.15132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:33.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:32 smithi099 bash[32082]: cluster 2024-09-13T14:28:31.219234+0000 mgr.y (mgr.24287) 303 : cluster [DBG] pgmap v334: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T14:28:33.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:32 smithi099 bash[32082]: cluster 2024-09-13T14:28:32.572863+0000 mon.a (mon.0) 967 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:28:33.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:32 smithi099 bash[22656]: audit 2024-09-13T14:28:31.195937+0000 mgr.y (mgr.24287) 302 : audit [DBG] from='client.15132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:33.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:32 smithi099 bash[22656]: cluster 2024-09-13T14:28:31.219234+0000 mgr.y (mgr.24287) 303 : cluster [DBG] pgmap v334: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T14:28:33.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:32 smithi099 bash[22656]: cluster 2024-09-13T14:28:32.572863+0000 mon.a (mon.0) 967 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:28:33.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:32 smithi177 bash[24494]: audit 2024-09-13T14:28:31.195937+0000 mgr.y (mgr.24287) 302 : audit [DBG] from='client.15132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:33.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:32 smithi177 bash[24494]: cluster 2024-09-13T14:28:31.219234+0000 mgr.y (mgr.24287) 303 : cluster [DBG] pgmap v334: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T14:28:33.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:32 smithi177 bash[24494]: cluster 2024-09-13T14:28:32.572863+0000 mon.a (mon.0) 967 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:28:33.859 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:34.207 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:28:34.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:33 smithi099 bash[22656]: audit 2024-09-13T14:28:33.547480+0000 mon.a (mon.0) 968 : audit [DBG] from='client.? 172.21.15.99:0/2430890412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:34.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:33 smithi099 bash[22656]: cluster 2024-09-13T14:28:33.562363+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:28:34.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:33 smithi099 bash[32082]: audit 2024-09-13T14:28:33.547480+0000 mon.a (mon.0) 968 : audit [DBG] from='client.? 172.21.15.99:0/2430890412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:34.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:33 smithi099 bash[32082]: cluster 2024-09-13T14:28:33.562363+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:28:34.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:33 smithi177 bash[24494]: audit 2024-09-13T14:28:33.547480+0000 mon.a (mon.0) 968 : audit [DBG] from='client.? 172.21.15.99:0/2430890412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:34.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:33 smithi177 bash[24494]: cluster 2024-09-13T14:28:33.562363+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:28:34.537 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:28:35.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:34 smithi099 bash[22656]: cluster 2024-09-13T14:28:33.220469+0000 mgr.y (mgr.24287) 304 : cluster [DBG] pgmap v337: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 184 op/s 2024-09-13T14:28:35.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:34 smithi099 bash[22656]: audit 2024-09-13T14:28:33.852262+0000 mgr.y (mgr.24287) 305 : audit [DBG] from='client.25046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:35.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:34 smithi099 bash[32082]: cluster 2024-09-13T14:28:33.220469+0000 mgr.y (mgr.24287) 304 : cluster [DBG] pgmap v337: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 184 op/s 2024-09-13T14:28:35.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:34 smithi099 bash[32082]: audit 2024-09-13T14:28:33.852262+0000 mgr.y (mgr.24287) 305 : audit [DBG] from='client.25046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:35.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:34 smithi177 bash[24494]: cluster 2024-09-13T14:28:33.220469+0000 mgr.y (mgr.24287) 304 : cluster [DBG] pgmap v337: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 184 op/s 2024-09-13T14:28:35.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:34 smithi177 bash[24494]: audit 2024-09-13T14:28:33.852262+0000 mgr.y (mgr.24287) 305 : audit [DBG] from='client.25046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:35.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:35 smithi177 bash[25434]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:28:36.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:35 smithi099 bash[22656]: audit 2024-09-13T14:28:34.184933+0000 mgr.y (mgr.24287) 306 : audit [DBG] from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:36.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:35 smithi099 bash[22656]: cluster 2024-09-13T14:28:34.893108+0000 mon.a (mon.0) 970 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:28:36.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:35 smithi099 bash[32082]: audit 2024-09-13T14:28:34.184933+0000 mgr.y (mgr.24287) 306 : audit [DBG] from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:36.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:35 smithi099 bash[32082]: cluster 2024-09-13T14:28:34.893108+0000 mon.a (mon.0) 970 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:28:36.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:35 smithi177 bash[24494]: audit 2024-09-13T14:28:34.184933+0000 mgr.y (mgr.24287) 306 : audit [DBG] from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:36.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:35 smithi177 bash[24494]: cluster 2024-09-13T14:28:34.893108+0000 mon.a (mon.0) 970 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:28:36.827 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:37.264 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.SnapsShrinkToZeroBeforePrepare (12073 ms) 2024-09-13T14:28:37.265 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroAfterPrepare 2024-09-13T14:28:37.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:36 smithi099 bash[22656]: cluster 2024-09-13T14:28:35.221295+0000 mgr.y (mgr.24287) 307 : cluster [DBG] pgmap v340: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 184 op/s 2024-09-13T14:28:37.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:36 smithi099 bash[22656]: cluster 2024-09-13T14:28:35.903780+0000 mon.a (mon.0) 971 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:28:37.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:36 smithi099 bash[22656]: audit 2024-09-13T14:28:36.535250+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.99:0/2208158378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:37.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:36 smithi099 bash[32082]: cluster 2024-09-13T14:28:35.221295+0000 mgr.y (mgr.24287) 307 : cluster [DBG] pgmap v340: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 184 op/s 2024-09-13T14:28:37.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:36 smithi099 bash[32082]: cluster 2024-09-13T14:28:35.903780+0000 mon.a (mon.0) 971 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:28:37.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:36 smithi099 bash[32082]: audit 2024-09-13T14:28:36.535250+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.99:0/2208158378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:37.339 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:28:37.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:36 smithi177 bash[24494]: cluster 2024-09-13T14:28:35.221295+0000 mgr.y (mgr.24287) 307 : cluster [DBG] pgmap v340: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 184 op/s 2024-09-13T14:28:37.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:36 smithi177 bash[24494]: cluster 2024-09-13T14:28:35.903780+0000 mon.a (mon.0) 971 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:28:37.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:36 smithi177 bash[24494]: audit 2024-09-13T14:28:36.535250+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.99:0/2208158378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:37.366 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:28:37.470 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:28:38.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:37 smithi099 bash[22656]: audit 2024-09-13T14:28:36.821991+0000 mgr.y (mgr.24287) 308 : audit [DBG] from='client.15162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:38.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:37 smithi099 bash[22656]: cluster 2024-09-13T14:28:36.901957+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:28:38.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:37 smithi099 bash[22656]: audit 2024-09-13T14:28:37.158794+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:38.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:37 smithi099 bash[32082]: audit 2024-09-13T14:28:36.821991+0000 mgr.y (mgr.24287) 308 : audit [DBG] from='client.15162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:38.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:37 smithi099 bash[32082]: cluster 2024-09-13T14:28:36.901957+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:28:38.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:37 smithi099 bash[32082]: audit 2024-09-13T14:28:37.158794+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:38.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:37 smithi177 bash[24494]: audit 2024-09-13T14:28:36.821991+0000 mgr.y (mgr.24287) 308 : audit [DBG] from='client.15162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:38.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:37 smithi177 bash[24494]: cluster 2024-09-13T14:28:36.901957+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:28:38.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:37 smithi177 bash[24494]: audit 2024-09-13T14:28:37.158794+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:38.954 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 8388609~8388609 2024-09-13T14:28:38.982 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:28:38.982 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:28:38.983 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:28:39.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:38 smithi099 bash[22656]: audit 2024-09-13T14:28:37.131384+0000 mgr.y (mgr.24287) 309 : audit [DBG] from='client.15168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:39.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:38 smithi099 bash[22656]: cluster 2024-09-13T14:28:37.222003+0000 mgr.y (mgr.24287) 310 : cluster [DBG] pgmap v343: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:39.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:38 smithi099 bash[22656]: cluster 2024-09-13T14:28:37.926639+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:28:39.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:38 smithi099 bash[32082]: audit 2024-09-13T14:28:37.131384+0000 mgr.y (mgr.24287) 309 : audit [DBG] from='client.15168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:39.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:38 smithi099 bash[32082]: cluster 2024-09-13T14:28:37.222003+0000 mgr.y (mgr.24287) 310 : cluster [DBG] pgmap v343: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:39.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:38 smithi099 bash[32082]: cluster 2024-09-13T14:28:37.926639+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:28:39.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:38 smithi177 bash[24494]: audit 2024-09-13T14:28:37.131384+0000 mgr.y (mgr.24287) 309 : audit [DBG] from='client.15168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:39.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:38 smithi177 bash[24494]: cluster 2024-09-13T14:28:37.222003+0000 mgr.y (mgr.24287) 310 : cluster [DBG] pgmap v343: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:28:39.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:38 smithi177 bash[24494]: cluster 2024-09-13T14:28:37.926639+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:28:39.779 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:40.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:39 smithi099 bash[22656]: cluster 2024-09-13T14:28:38.930462+0000 mon.a (mon.0) 974 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:28:40.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:39 smithi099 bash[22656]: cluster 2024-09-13T14:28:39.222554+0000 mgr.y (mgr.24287) 311 : cluster [DBG] pgmap v346: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 470 op/s 2024-09-13T14:28:40.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:39 smithi099 bash[22656]: audit 2024-09-13T14:28:39.485430+0000 mon.a (mon.0) 975 : audit [DBG] from='client.? 172.21.15.99:0/985188741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:40.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:39 smithi099 bash[22656]: audit 2024-09-13T14:28:39.773239+0000 mgr.y (mgr.24287) 312 : audit [DBG] from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:40.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:39 smithi099 bash[32082]: cluster 2024-09-13T14:28:38.930462+0000 mon.a (mon.0) 974 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:28:40.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:39 smithi099 bash[32082]: cluster 2024-09-13T14:28:39.222554+0000 mgr.y (mgr.24287) 311 : cluster [DBG] pgmap v346: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 470 op/s 2024-09-13T14:28:40.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:39 smithi099 bash[32082]: audit 2024-09-13T14:28:39.485430+0000 mon.a (mon.0) 975 : audit [DBG] from='client.? 172.21.15.99:0/985188741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:40.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:39 smithi099 bash[32082]: audit 2024-09-13T14:28:39.773239+0000 mgr.y (mgr.24287) 312 : audit [DBG] from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:40.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:39 smithi177 bash[24494]: cluster 2024-09-13T14:28:38.930462+0000 mon.a (mon.0) 974 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:28:40.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:39 smithi177 bash[24494]: cluster 2024-09-13T14:28:39.222554+0000 mgr.y (mgr.24287) 311 : cluster [DBG] pgmap v346: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 470 op/s 2024-09-13T14:28:40.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:39 smithi177 bash[24494]: audit 2024-09-13T14:28:39.485430+0000 mon.a (mon.0) 975 : audit [DBG] from='client.? 172.21.15.99:0/985188741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:40.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:39 smithi177 bash[24494]: audit 2024-09-13T14:28:39.773239+0000 mgr.y (mgr.24287) 312 : audit [DBG] from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:40.969 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 8388609~1000 2024-09-13T14:28:40.985 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16777228~1000 2024-09-13T14:28:40.985 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:28:41.005 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 83886079~8388609 2024-09-13T14:28:41.057 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:28:41.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:40 smithi177 bash[24494]: cluster 2024-09-13T14:28:39.946126+0000 mon.a (mon.0) 976 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:28:41.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:40 smithi177 bash[24494]: audit 2024-09-13T14:28:40.070438+0000 mgr.y (mgr.24287) 313 : audit [DBG] from='client.25088 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:41.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:40 smithi099 bash[22656]: cluster 2024-09-13T14:28:39.946126+0000 mon.a (mon.0) 976 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:28:41.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:40 smithi099 bash[22656]: audit 2024-09-13T14:28:40.070438+0000 mgr.y (mgr.24287) 313 : audit [DBG] from='client.25088 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:41.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:40 smithi099 bash[32082]: cluster 2024-09-13T14:28:39.946126+0000 mon.a (mon.0) 976 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:28:41.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:40 smithi099 bash[32082]: audit 2024-09-13T14:28:40.070438+0000 mgr.y (mgr.24287) 313 : audit [DBG] from='client.25088 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:42.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:42] "GET /metrics HTTP/1.1" 200 246400 "" "Prometheus/2.43.0" 2024-09-13T14:28:42.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:41 smithi099 bash[22656]: cluster 2024-09-13T14:28:40.958180+0000 mon.a (mon.0) 977 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:28:42.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:41 smithi099 bash[22656]: cluster 2024-09-13T14:28:41.223224+0000 mgr.y (mgr.24287) 314 : cluster [DBG] pgmap v349: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 470 op/s 2024-09-13T14:28:42.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:41 smithi099 bash[32082]: cluster 2024-09-13T14:28:40.958180+0000 mon.a (mon.0) 977 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:28:42.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:41 smithi099 bash[32082]: cluster 2024-09-13T14:28:41.223224+0000 mgr.y (mgr.24287) 314 : cluster [DBG] pgmap v349: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 470 op/s 2024-09-13T14:28:42.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:41 smithi177 bash[24494]: cluster 2024-09-13T14:28:40.958180+0000 mon.a (mon.0) 977 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:28:42.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:41 smithi177 bash[24494]: cluster 2024-09-13T14:28:41.223224+0000 mgr.y (mgr.24287) 314 : cluster [DBG] pgmap v349: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 470 op/s 2024-09-13T14:28:42.723 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:42.987 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:28:43.028 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83886079~1 2024-09-13T14:28:43.039 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:28:43.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22656]: cluster 2024-09-13T14:28:41.967044+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:28:43.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22656]: cephadm 2024-09-13T14:28:42.112994+0000 mgr.y (mgr.24287) 315 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T14:28:43.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22656]: audit 2024-09-13T14:28:42.123335+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:43.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22656]: audit 2024-09-13T14:28:42.125316+0000 mon.c (mon.2) 149 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:28:43.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22656]: audit 2024-09-13T14:28:42.126104+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:28:43.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22656]: audit 2024-09-13T14:28:42.128023+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:28:43.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22656]: audit 2024-09-13T14:28:42.129395+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:43.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22656]: cephadm 2024-09-13T14:28:42.131001+0000 mgr.y (mgr.24287) 316 : cephadm [INF] Deploying daemon mgr.x on smithi177 2024-09-13T14:28:43.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22656]: audit 2024-09-13T14:28:42.422163+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.99:0/4016819992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:43.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[22656]: audit 2024-09-13T14:28:42.717162+0000 mgr.y (mgr.24287) 317 : audit [DBG] from='client.15198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[32082]: cluster 2024-09-13T14:28:41.967044+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:28:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[32082]: cephadm 2024-09-13T14:28:42.112994+0000 mgr.y (mgr.24287) 315 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T14:28:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[32082]: audit 2024-09-13T14:28:42.123335+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[32082]: audit 2024-09-13T14:28:42.125316+0000 mon.c (mon.2) 149 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:28:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[32082]: audit 2024-09-13T14:28:42.126104+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:28:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[32082]: audit 2024-09-13T14:28:42.128023+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:28:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[32082]: audit 2024-09-13T14:28:42.129395+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[32082]: cephadm 2024-09-13T14:28:42.131001+0000 mgr.y (mgr.24287) 316 : cephadm [INF] Deploying daemon mgr.x on smithi177 2024-09-13T14:28:43.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[32082]: audit 2024-09-13T14:28:42.422163+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.99:0/4016819992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:43.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:42 smithi099 bash[32082]: audit 2024-09-13T14:28:42.717162+0000 mgr.y (mgr.24287) 317 : audit [DBG] from='client.15198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:43.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:42 smithi177 bash[24494]: cluster 2024-09-13T14:28:41.967044+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:28:43.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:42 smithi177 bash[24494]: cephadm 2024-09-13T14:28:42.112994+0000 mgr.y (mgr.24287) 315 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T14:28:43.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:42 smithi177 bash[24494]: audit 2024-09-13T14:28:42.123335+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:43.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:42 smithi177 bash[24494]: audit 2024-09-13T14:28:42.125316+0000 mon.c (mon.2) 149 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:28:43.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:42 smithi177 bash[24494]: audit 2024-09-13T14:28:42.126104+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:28:43.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:42 smithi177 bash[24494]: audit 2024-09-13T14:28:42.128023+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:28:43.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:42 smithi177 bash[24494]: audit 2024-09-13T14:28:42.129395+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:43.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:42 smithi177 bash[24494]: cephadm 2024-09-13T14:28:42.131001+0000 mgr.y (mgr.24287) 316 : cephadm [INF] Deploying daemon mgr.x on smithi177 2024-09-13T14:28:43.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:42 smithi177 bash[24494]: audit 2024-09-13T14:28:42.422163+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.99:0/4016819992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:43.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:42 smithi177 bash[24494]: audit 2024-09-13T14:28:42.717162+0000 mgr.y (mgr.24287) 317 : audit [DBG] from='client.15198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:44.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:43 smithi099 bash[22656]: cluster 2024-09-13T14:28:42.968965+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:28:44.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:43 smithi099 bash[22656]: audit 2024-09-13T14:28:43.022427+0000 mgr.y (mgr.24287) 318 : audit [DBG] from='client.25106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:44.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:43 smithi099 bash[22656]: cluster 2024-09-13T14:28:43.224870+0000 mgr.y (mgr.24287) 319 : cluster [DBG] pgmap v352: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 199 op/s 2024-09-13T14:28:44.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:43 smithi099 bash[22656]: cluster 2024-09-13T14:28:43.570819+0000 mon.a (mon.0) 983 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:28:44.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:43 smithi099 bash[32082]: cluster 2024-09-13T14:28:42.968965+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:28:44.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:43 smithi099 bash[32082]: audit 2024-09-13T14:28:43.022427+0000 mgr.y (mgr.24287) 318 : audit [DBG] from='client.25106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:44.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:43 smithi099 bash[32082]: cluster 2024-09-13T14:28:43.224870+0000 mgr.y (mgr.24287) 319 : cluster [DBG] pgmap v352: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 199 op/s 2024-09-13T14:28:44.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:43 smithi099 bash[32082]: cluster 2024-09-13T14:28:43.570819+0000 mon.a (mon.0) 983 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:28:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:43 smithi177 bash[24494]: cluster 2024-09-13T14:28:42.968965+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:28:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:43 smithi177 bash[24494]: audit 2024-09-13T14:28:43.022427+0000 mgr.y (mgr.24287) 318 : audit [DBG] from='client.25106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:43 smithi177 bash[24494]: cluster 2024-09-13T14:28:43.224870+0000 mgr.y (mgr.24287) 319 : cluster [DBG] pgmap v352: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 199 op/s 2024-09-13T14:28:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:43 smithi177 bash[24494]: cluster 2024-09-13T14:28:43.570819+0000 mon.a (mon.0) 983 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:28:44.852 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:44 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:44.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:44 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:44.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:44 smithi177 systemd[1]: Stopping Ceph mgr.x for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:28:44.853 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:28:44 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:44.854 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:28:44 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:44.854 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:28:44 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:44.854 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:28:44 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:44.855 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:28:44 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:44.855 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:28:44 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:44.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:28:44 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:45.679 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:45.839 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:45 smithi177 bash[24494]: cluster 2024-09-13T14:28:44.573491+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:28:45.840 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:45 smithi177 bash[24494]: audit 2024-09-13T14:28:45.371837+0000 mon.a (mon.0) 985 : audit [DBG] from='client.? 172.21.15.99:0/2860232901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:45.840 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:45 smithi177 bash[52767]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:28:45.840 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:28:45.840 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:45 smithi177 bash[52827]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:28:45.840 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T14:28:45.840 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: Stopped Ceph mgr.x for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:28:46.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:45 smithi099 bash[22656]: cluster 2024-09-13T14:28:44.573491+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:28:46.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:45 smithi099 bash[22656]: audit 2024-09-13T14:28:45.371837+0000 mon.a (mon.0) 985 : audit [DBG] from='client.? 172.21.15.99:0/2860232901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:46.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:45 smithi099 bash[32082]: cluster 2024-09-13T14:28:44.573491+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:28:46.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:45 smithi099 bash[32082]: audit 2024-09-13T14:28:45.371837+0000 mon.a (mon.0) 985 : audit [DBG] from='client.? 172.21.15.99:0/2860232901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:46.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:46.102 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:46.103 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:46.103 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:46.103 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:46.104 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:46.104 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:46.104 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:46.104 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:28:46.105 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:45 smithi177 systemd[1]: Started Ceph mgr.x for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:28:46.254 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 0 2024-09-13T14:28:46.328 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:28:46.510 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:28:46.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:46 smithi177 bash[24494]: cluster 2024-09-13T14:28:45.225536+0000 mgr.y (mgr.24287) 320 : cluster [DBG] pgmap v355: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 199 op/s 2024-09-13T14:28:46.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:46 smithi177 bash[24494]: cluster 2024-09-13T14:28:45.584007+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-09-13T14:28:46.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:46 smithi177 bash[24494]: audit 2024-09-13T14:28:45.673323+0000 mgr.y (mgr.24287) 321 : audit [DBG] from='client.15216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:46.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:46 smithi177 bash[24494]: audit 2024-09-13T14:28:46.007969+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:46.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:46 smithi177 bash[24494]: audit 2024-09-13T14:28:46.016878+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:46.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:46 smithi177 bash[24494]: audit 2024-09-13T14:28:46.018742+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:47.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[32082]: cluster 2024-09-13T14:28:45.225536+0000 mgr.y (mgr.24287) 320 : cluster [DBG] pgmap v355: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 199 op/s 2024-09-13T14:28:47.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[32082]: cluster 2024-09-13T14:28:45.584007+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-09-13T14:28:47.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[32082]: audit 2024-09-13T14:28:45.673323+0000 mgr.y (mgr.24287) 321 : audit [DBG] from='client.15216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:47.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[32082]: audit 2024-09-13T14:28:46.007969+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:47.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[32082]: audit 2024-09-13T14:28:46.016878+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:47.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[32082]: audit 2024-09-13T14:28:46.018742+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:47.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[22656]: cluster 2024-09-13T14:28:45.225536+0000 mgr.y (mgr.24287) 320 : cluster [DBG] pgmap v355: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 199 op/s 2024-09-13T14:28:47.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[22656]: cluster 2024-09-13T14:28:45.584007+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-09-13T14:28:47.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[22656]: audit 2024-09-13T14:28:45.673323+0000 mgr.y (mgr.24287) 321 : audit [DBG] from='client.15216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:47.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[22656]: audit 2024-09-13T14:28:46.007969+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:47.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[22656]: audit 2024-09-13T14:28:46.016878+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:47.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:46 smithi099 bash[22656]: audit 2024-09-13T14:28:46.018742+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:47.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:47 smithi177 bash[24494]: audit 2024-09-13T14:28:45.985707+0000 mgr.y (mgr.24287) 322 : audit [DBG] from='client.15222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:47.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:47 smithi177 bash[52886]: debug 2024-09-13T14:28:47.511+0000 7ff3890bb640 1 -- 172.21.15.177:0/145217062 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55af20e670e0 con 0x55af20e68800 2024-09-13T14:28:48.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:47 smithi099 bash[22656]: audit 2024-09-13T14:28:45.985707+0000 mgr.y (mgr.24287) 322 : audit [DBG] from='client.15222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:48.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:47 smithi099 bash[32082]: audit 2024-09-13T14:28:45.985707+0000 mgr.y (mgr.24287) 322 : audit [DBG] from='client.15222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:48.651 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:48.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:48 smithi177 bash[24494]: cluster 2024-09-13T14:28:47.226279+0000 mgr.y (mgr.24287) 323 : cluster [DBG] pgmap v357: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 131 KiB/s rd, 7.5 MiB/s wr, 187 op/s 2024-09-13T14:28:48.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:48 smithi177 bash[24494]: cluster 2024-09-13T14:28:47.593183+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-09-13T14:28:48.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:48 smithi177 bash[24494]: audit 2024-09-13T14:28:48.342497+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.99:0/916645719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:49.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:48 smithi099 bash[22656]: cluster 2024-09-13T14:28:47.226279+0000 mgr.y (mgr.24287) 323 : cluster [DBG] pgmap v357: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 131 KiB/s rd, 7.5 MiB/s wr, 187 op/s 2024-09-13T14:28:49.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:48 smithi099 bash[22656]: cluster 2024-09-13T14:28:47.593183+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-09-13T14:28:49.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:48 smithi099 bash[22656]: audit 2024-09-13T14:28:48.342497+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.99:0/916645719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:49.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:48 smithi099 bash[32082]: cluster 2024-09-13T14:28:47.226279+0000 mgr.y (mgr.24287) 323 : cluster [DBG] pgmap v357: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 131 KiB/s rd, 7.5 MiB/s wr, 187 op/s 2024-09-13T14:28:49.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:48 smithi099 bash[32082]: cluster 2024-09-13T14:28:47.593183+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-09-13T14:28:49.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:48 smithi099 bash[32082]: audit 2024-09-13T14:28:48.342497+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.99:0/916645719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:49.852 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:49 smithi177 bash[24494]: cluster 2024-09-13T14:28:48.606806+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-09-13T14:28:49.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:49 smithi177 bash[24494]: audit 2024-09-13T14:28:48.646670+0000 mgr.y (mgr.24287) 324 : audit [DBG] from='client.15240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:49.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:49 smithi177 bash[24494]: audit 2024-09-13T14:28:48.924628+0000 mgr.y (mgr.24287) 325 : audit [DBG] from='client.15246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:50.065 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.SnapsShrinkToZeroAfterPrepare (12800 ms) 2024-09-13T14:28:50.065 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare 2024-09-13T14:28:50.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:49 smithi099 bash[22656]: cluster 2024-09-13T14:28:48.606806+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-09-13T14:28:50.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:49 smithi099 bash[22656]: audit 2024-09-13T14:28:48.646670+0000 mgr.y (mgr.24287) 324 : audit [DBG] from='client.15240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:50.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:49 smithi099 bash[22656]: audit 2024-09-13T14:28:48.924628+0000 mgr.y (mgr.24287) 325 : audit [DBG] from='client.15246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:50.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:49 smithi099 bash[32082]: cluster 2024-09-13T14:28:48.606806+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-09-13T14:28:50.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:49 smithi099 bash[32082]: audit 2024-09-13T14:28:48.646670+0000 mgr.y (mgr.24287) 324 : audit [DBG] from='client.15240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:50.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:49 smithi099 bash[32082]: audit 2024-09-13T14:28:48.924628+0000 mgr.y (mgr.24287) 325 : audit [DBG] from='client.15246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:50.143 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:28:50.171 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:28:50.283 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:28:50.852 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:50 smithi177 bash[24494]: cluster 2024-09-13T14:28:49.227082+0000 mgr.y (mgr.24287) 326 : cluster [DBG] pgmap v360: 137 pgs: 137 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s rd, 5.2 MiB/s wr, 382 op/s 2024-09-13T14:28:50.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:50 smithi177 bash[24494]: cluster 2024-09-13T14:28:49.612771+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-09-13T14:28:51.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:50 smithi099 bash[22656]: cluster 2024-09-13T14:28:49.227082+0000 mgr.y (mgr.24287) 326 : cluster [DBG] pgmap v360: 137 pgs: 137 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s rd, 5.2 MiB/s wr, 382 op/s 2024-09-13T14:28:51.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:50 smithi099 bash[22656]: cluster 2024-09-13T14:28:49.612771+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-09-13T14:28:51.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:50 smithi099 bash[32082]: cluster 2024-09-13T14:28:49.227082+0000 mgr.y (mgr.24287) 326 : cluster [DBG] pgmap v360: 137 pgs: 137 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s rd, 5.2 MiB/s wr, 382 op/s 2024-09-13T14:28:51.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:50 smithi099 bash[32082]: cluster 2024-09-13T14:28:49.612771+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-09-13T14:28:51.578 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:51.641 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 8388609~8388609 2024-09-13T14:28:51.659 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:28:51.659 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:28:51.659 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:28:52.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:51 smithi099 bash[22656]: cluster 2024-09-13T14:28:50.621299+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-09-13T14:28:52.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:51 smithi099 bash[22656]: audit 2024-09-13T14:28:51.291821+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.99:0/3509669932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:52.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:51 smithi099 bash[22656]: audit 2024-09-13T14:28:51.416885+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:52.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:51 smithi099 bash[22656]: audit 2024-09-13T14:28:51.427158+0000 mon.a (mon.0) 994 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:52.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:51 smithi099 bash[32082]: cluster 2024-09-13T14:28:50.621299+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-09-13T14:28:52.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:51 smithi099 bash[32082]: audit 2024-09-13T14:28:51.291821+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.99:0/3509669932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:52.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:51 smithi099 bash[32082]: audit 2024-09-13T14:28:51.416885+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:52.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:51 smithi099 bash[32082]: audit 2024-09-13T14:28:51.427158+0000 mon.a (mon.0) 994 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:52.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:51 smithi177 bash[24494]: cluster 2024-09-13T14:28:50.621299+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-09-13T14:28:52.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:51 smithi177 bash[24494]: audit 2024-09-13T14:28:51.291821+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.99:0/3509669932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:52.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:51 smithi177 bash[24494]: audit 2024-09-13T14:28:51.416885+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:52.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:51 smithi177 bash[24494]: audit 2024-09-13T14:28:51.427158+0000 mon.a (mon.0) 994 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:52.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:52] "GET /metrics HTTP/1.1" 200 246400 "" "Prometheus/2.43.0" 2024-09-13T14:28:53.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[32082]: cluster 2024-09-13T14:28:51.227528+0000 mgr.y (mgr.24287) 327 : cluster [DBG] pgmap v363: 137 pgs: 137 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 6.0 MiB/s wr, 445 op/s 2024-09-13T14:28:53.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[32082]: audit 2024-09-13T14:28:51.574367+0000 mgr.y (mgr.24287) 328 : audit [DBG] from='client.15258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:53.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[32082]: cluster 2024-09-13T14:28:51.632195+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-09-13T14:28:53.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[32082]: audit 2024-09-13T14:28:51.861779+0000 mgr.y (mgr.24287) 329 : audit [DBG] from='client.15264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:53.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[32082]: audit 2024-09-13T14:28:52.159412+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[22656]: cluster 2024-09-13T14:28:51.227528+0000 mgr.y (mgr.24287) 327 : cluster [DBG] pgmap v363: 137 pgs: 137 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 6.0 MiB/s wr, 445 op/s 2024-09-13T14:28:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[22656]: audit 2024-09-13T14:28:51.574367+0000 mgr.y (mgr.24287) 328 : audit [DBG] from='client.15258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:53.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[22656]: cluster 2024-09-13T14:28:51.632195+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-09-13T14:28:53.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[22656]: audit 2024-09-13T14:28:51.861779+0000 mgr.y (mgr.24287) 329 : audit [DBG] from='client.15264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:53.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:52 smithi099 bash[22656]: audit 2024-09-13T14:28:52.159412+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:53.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:52 smithi177 bash[24494]: cluster 2024-09-13T14:28:51.227528+0000 mgr.y (mgr.24287) 327 : cluster [DBG] pgmap v363: 137 pgs: 137 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 6.0 MiB/s wr, 445 op/s 2024-09-13T14:28:53.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:52 smithi177 bash[24494]: audit 2024-09-13T14:28:51.574367+0000 mgr.y (mgr.24287) 328 : audit [DBG] from='client.15258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:53.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:52 smithi177 bash[24494]: cluster 2024-09-13T14:28:51.632195+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-09-13T14:28:53.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:52 smithi177 bash[24494]: audit 2024-09-13T14:28:51.861779+0000 mgr.y (mgr.24287) 329 : audit [DBG] from='client.15264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:53.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:52 smithi177 bash[24494]: audit 2024-09-13T14:28:52.159412+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:53.587 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 8388609~1000 2024-09-13T14:28:53.602 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16777228~1000 2024-09-13T14:28:53.602 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:28:53.624 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 83886079~8388609 2024-09-13T14:28:53.674 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:28:53.906 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[52886]: [13/Sep/2024:14:28:53] ENGINE Bus STARTING 2024-09-13T14:28:53.906 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[52886]: CherryPy Checker: 2024-09-13T14:28:53.906 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[52886]: The Application mounted at '' has an empty config. 2024-09-13T14:28:53.906 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[24494]: cluster 2024-09-13T14:28:52.645529+0000 mon.a (mon.0) 996 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-09-13T14:28:53.906 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[24494]: audit 2024-09-13T14:28:52.778310+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:53.906 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[24494]: audit 2024-09-13T14:28:52.790889+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:53.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[24494]: audit 2024-09-13T14:28:53.400251+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:53.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[24494]: audit 2024-09-13T14:28:53.412607+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:53.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[24494]: cluster 2024-09-13T14:28:53.573933+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-09-13T14:28:54.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[22656]: cluster 2024-09-13T14:28:52.645529+0000 mon.a (mon.0) 996 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-09-13T14:28:54.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[22656]: audit 2024-09-13T14:28:52.778310+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:54.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[22656]: audit 2024-09-13T14:28:52.790889+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:54.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[22656]: audit 2024-09-13T14:28:53.400251+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:54.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[22656]: audit 2024-09-13T14:28:53.412607+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:54.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[22656]: cluster 2024-09-13T14:28:53.573933+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-09-13T14:28:54.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[32082]: cluster 2024-09-13T14:28:52.645529+0000 mon.a (mon.0) 996 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-09-13T14:28:54.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[32082]: audit 2024-09-13T14:28:52.778310+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:54.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[32082]: audit 2024-09-13T14:28:52.790889+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:54.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[32082]: audit 2024-09-13T14:28:53.400251+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:54.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[32082]: audit 2024-09-13T14:28:53.412607+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:28:54.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:53 smithi099 bash[32082]: cluster 2024-09-13T14:28:53.573933+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-09-13T14:28:54.352 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[52886]: [13/Sep/2024:14:28:53] ENGINE Serving on http://:::9283 2024-09-13T14:28:54.353 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:53 smithi177 bash[52886]: [13/Sep/2024:14:28:53] ENGINE Bus STARTED 2024-09-13T14:28:54.499 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:55.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[22656]: cluster 2024-09-13T14:28:53.228992+0000 mgr.y (mgr.24287) 330 : cluster [DBG] pgmap v366: 137 pgs: 137 active+clean; 593 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 142 op/s 2024-09-13T14:28:55.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[22656]: cluster 2024-09-13T14:28:53.808079+0000 mon.a (mon.0) 1002 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:28:55.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[22656]: cluster 2024-09-13T14:28:53.808310+0000 mon.a (mon.0) 1003 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:28:55.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[22656]: audit 2024-09-13T14:28:53.811087+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:28:55.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[22656]: audit 2024-09-13T14:28:53.811744+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[22656]: audit 2024-09-13T14:28:53.813560+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[22656]: audit 2024-09-13T14:28:53.815094+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[22656]: audit 2024-09-13T14:28:54.203481+0000 mon.a (mon.0) 1004 : audit [DBG] from='client.? 172.21.15.99:0/138515560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[22656]: cluster 2024-09-13T14:28:54.647334+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[32082]: cluster 2024-09-13T14:28:53.228992+0000 mgr.y (mgr.24287) 330 : cluster [DBG] pgmap v366: 137 pgs: 137 active+clean; 593 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 142 op/s 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[32082]: cluster 2024-09-13T14:28:53.808079+0000 mon.a (mon.0) 1002 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[32082]: cluster 2024-09-13T14:28:53.808310+0000 mon.a (mon.0) 1003 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[32082]: audit 2024-09-13T14:28:53.811087+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[32082]: audit 2024-09-13T14:28:53.811744+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[32082]: audit 2024-09-13T14:28:53.813560+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:28:55.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[32082]: audit 2024-09-13T14:28:53.815094+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:28:55.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[32082]: audit 2024-09-13T14:28:54.203481+0000 mon.a (mon.0) 1004 : audit [DBG] from='client.? 172.21.15.99:0/138515560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:55.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:54 smithi099 bash[32082]: cluster 2024-09-13T14:28:54.647334+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-09-13T14:28:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:54 smithi177 bash[24494]: cluster 2024-09-13T14:28:53.228992+0000 mgr.y (mgr.24287) 330 : cluster [DBG] pgmap v366: 137 pgs: 137 active+clean; 593 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 142 op/s 2024-09-13T14:28:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:54 smithi177 bash[24494]: cluster 2024-09-13T14:28:53.808079+0000 mon.a (mon.0) 1002 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:28:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:54 smithi177 bash[24494]: cluster 2024-09-13T14:28:53.808310+0000 mon.a (mon.0) 1003 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:28:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:54 smithi177 bash[24494]: audit 2024-09-13T14:28:53.811087+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:28:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:54 smithi177 bash[24494]: audit 2024-09-13T14:28:53.811744+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:28:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:54 smithi177 bash[24494]: audit 2024-09-13T14:28:53.813560+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:28:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:54 smithi177 bash[24494]: audit 2024-09-13T14:28:53.815094+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:28:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:54 smithi177 bash[24494]: audit 2024-09-13T14:28:54.203481+0000 mon.a (mon.0) 1004 : audit [DBG] from='client.? 172.21.15.99:0/138515560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:54 smithi177 bash[24494]: cluster 2024-09-13T14:28:54.647334+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-09-13T14:28:55.668 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:28:55.715 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83886079~1 2024-09-13T14:28:55.737 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:28:55.801 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:28:55 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:28:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:28:56.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:55 smithi099 bash[22656]: audit 2024-09-13T14:28:54.494439+0000 mgr.y (mgr.24287) 331 : audit [DBG] from='client.25172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:56.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:55 smithi099 bash[22656]: audit 2024-09-13T14:28:54.780730+0000 mgr.y (mgr.24287) 332 : audit [DBG] from='client.25178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:56.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:55 smithi099 bash[22656]: cluster 2024-09-13T14:28:54.806226+0000 mon.a (mon.0) 1006 : cluster [DBG] mgrmap e21: y(active, since 7m), standbys: x 2024-09-13T14:28:56.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:55 smithi099 bash[22656]: cluster 2024-09-13T14:28:55.654544+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-09-13T14:28:56.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:55 smithi099 bash[32082]: audit 2024-09-13T14:28:54.494439+0000 mgr.y (mgr.24287) 331 : audit [DBG] from='client.25172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:56.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:55 smithi099 bash[32082]: audit 2024-09-13T14:28:54.780730+0000 mgr.y (mgr.24287) 332 : audit [DBG] from='client.25178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:56.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:55 smithi099 bash[32082]: cluster 2024-09-13T14:28:54.806226+0000 mon.a (mon.0) 1006 : cluster [DBG] mgrmap e21: y(active, since 7m), standbys: x 2024-09-13T14:28:56.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:55 smithi099 bash[32082]: cluster 2024-09-13T14:28:55.654544+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-09-13T14:28:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:55 smithi177 bash[24494]: audit 2024-09-13T14:28:54.494439+0000 mgr.y (mgr.24287) 331 : audit [DBG] from='client.25172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:55 smithi177 bash[24494]: audit 2024-09-13T14:28:54.780730+0000 mgr.y (mgr.24287) 332 : audit [DBG] from='client.25178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:55 smithi177 bash[24494]: cluster 2024-09-13T14:28:54.806226+0000 mon.a (mon.0) 1006 : cluster [DBG] mgrmap e21: y(active, since 7m), standbys: x 2024-09-13T14:28:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:55 smithi177 bash[24494]: cluster 2024-09-13T14:28:55.654544+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-09-13T14:28:57.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:56 smithi099 bash[22656]: cluster 2024-09-13T14:28:55.229713+0000 mgr.y (mgr.24287) 333 : cluster [DBG] pgmap v369: 137 pgs: 137 active+clean; 593 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 142 op/s 2024-09-13T14:28:57.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:56 smithi099 bash[32082]: cluster 2024-09-13T14:28:55.229713+0000 mgr.y (mgr.24287) 333 : cluster [DBG] pgmap v369: 137 pgs: 137 active+clean; 593 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 142 op/s 2024-09-13T14:28:57.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:56 smithi177 bash[24494]: cluster 2024-09-13T14:28:55.229713+0000 mgr.y (mgr.24287) 333 : cluster [DBG] pgmap v369: 137 pgs: 137 active+clean; 593 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 142 op/s 2024-09-13T14:28:57.444 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:28:58.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:57 smithi177 bash[24494]: cluster 2024-09-13T14:28:56.814316+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-09-13T14:28:58.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:57 smithi177 bash[24494]: audit 2024-09-13T14:28:57.137831+0000 mon.a (mon.0) 1009 : audit [DBG] from='client.? 172.21.15.99:0/2685781026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:58.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:57 smithi099 bash[22656]: cluster 2024-09-13T14:28:56.814316+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-09-13T14:28:58.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:57 smithi099 bash[22656]: audit 2024-09-13T14:28:57.137831+0000 mon.a (mon.0) 1009 : audit [DBG] from='client.? 172.21.15.99:0/2685781026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:58.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:57 smithi099 bash[32082]: cluster 2024-09-13T14:28:56.814316+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-09-13T14:28:58.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:57 smithi099 bash[32082]: audit 2024-09-13T14:28:57.137831+0000 mon.a (mon.0) 1009 : audit [DBG] from='client.? 172.21.15.99:0/2685781026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:28:59.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:58 smithi177 bash[24494]: cluster 2024-09-13T14:28:57.230194+0000 mgr.y (mgr.24287) 334 : cluster [DBG] pgmap v372: 137 pgs: 137 active+clean; 593 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:28:59.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:58 smithi177 bash[24494]: audit 2024-09-13T14:28:57.437657+0000 mgr.y (mgr.24287) 335 : audit [DBG] from='client.25190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:59.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:58 smithi177 bash[24494]: audit 2024-09-13T14:28:57.723952+0000 mgr.y (mgr.24287) 336 : audit [DBG] from='client.25196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:59.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:58 smithi177 bash[24494]: cluster 2024-09-13T14:28:57.835909+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-09-13T14:28:59.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:28:58 smithi177 bash[24494]: cluster 2024-09-13T14:28:58.575306+0000 mon.a (mon.0) 1011 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-09-13T14:28:59.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:58 smithi099 bash[22656]: cluster 2024-09-13T14:28:57.230194+0000 mgr.y (mgr.24287) 334 : cluster [DBG] pgmap v372: 137 pgs: 137 active+clean; 593 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:28:59.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:58 smithi099 bash[22656]: audit 2024-09-13T14:28:57.437657+0000 mgr.y (mgr.24287) 335 : audit [DBG] from='client.25190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:59.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:58 smithi099 bash[22656]: audit 2024-09-13T14:28:57.723952+0000 mgr.y (mgr.24287) 336 : audit [DBG] from='client.25196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:59.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:58 smithi099 bash[22656]: cluster 2024-09-13T14:28:57.835909+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-09-13T14:28:59.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:28:58 smithi099 bash[22656]: cluster 2024-09-13T14:28:58.575306+0000 mon.a (mon.0) 1011 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-09-13T14:28:59.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:58 smithi099 bash[32082]: cluster 2024-09-13T14:28:57.230194+0000 mgr.y (mgr.24287) 334 : cluster [DBG] pgmap v372: 137 pgs: 137 active+clean; 593 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:28:59.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:58 smithi099 bash[32082]: audit 2024-09-13T14:28:57.437657+0000 mgr.y (mgr.24287) 335 : audit [DBG] from='client.25190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:59.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:58 smithi099 bash[32082]: audit 2024-09-13T14:28:57.723952+0000 mgr.y (mgr.24287) 336 : audit [DBG] from='client.25196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:59.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:58 smithi099 bash[32082]: cluster 2024-09-13T14:28:57.835909+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-09-13T14:28:59.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:28:58 smithi099 bash[32082]: cluster 2024-09-13T14:28:58.575306+0000 mon.a (mon.0) 1011 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-09-13T14:28:59.478 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:28:59.506 INFO:tasks.workunit.client.0.smithi099.stdout:write: * 83886080~1000 2024-09-13T14:28:59.569 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:28:59.842 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:29:00.377 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:01.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:00 smithi177 bash[24494]: cluster 2024-09-13T14:28:59.230856+0000 mgr.y (mgr.24287) 337 : cluster [DBG] pgmap v375: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T14:29:01.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:00 smithi177 bash[24494]: audit 2024-09-13T14:29:00.084978+0000 mon.c (mon.2) 156 : audit [DBG] from='client.? 172.21.15.99:0/3139022685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:01.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:00 smithi099 bash[22656]: cluster 2024-09-13T14:28:59.230856+0000 mgr.y (mgr.24287) 337 : cluster [DBG] pgmap v375: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T14:29:01.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:00 smithi099 bash[22656]: audit 2024-09-13T14:29:00.084978+0000 mon.c (mon.2) 156 : audit [DBG] from='client.? 172.21.15.99:0/3139022685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:01.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:00 smithi099 bash[32082]: cluster 2024-09-13T14:28:59.230856+0000 mgr.y (mgr.24287) 337 : cluster [DBG] pgmap v375: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T14:29:01.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:00 smithi099 bash[32082]: audit 2024-09-13T14:29:00.084978+0000 mon.c (mon.2) 156 : audit [DBG] from='client.? 172.21.15.99:0/3139022685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:02.115 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:01 smithi099 bash[22656]: audit 2024-09-13T14:29:00.371882+0000 mgr.y (mgr.24287) 338 : audit [DBG] from='client.15312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:02.115 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:01 smithi099 bash[22656]: audit 2024-09-13T14:29:00.662570+0000 mgr.y (mgr.24287) 339 : audit [DBG] from='client.15318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:02.115 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:01 smithi099 bash[22656]: cluster 2024-09-13T14:29:00.844455+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-09-13T14:29:02.115 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:01 smithi099 bash[32082]: audit 2024-09-13T14:29:00.371882+0000 mgr.y (mgr.24287) 338 : audit [DBG] from='client.15312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:02.116 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:01 smithi099 bash[32082]: audit 2024-09-13T14:29:00.662570+0000 mgr.y (mgr.24287) 339 : audit [DBG] from='client.15318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:02.116 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:01 smithi099 bash[32082]: cluster 2024-09-13T14:29:00.844455+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-09-13T14:29:02.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:01 smithi177 bash[24494]: audit 2024-09-13T14:29:00.371882+0000 mgr.y (mgr.24287) 338 : audit [DBG] from='client.15312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:02.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:01 smithi177 bash[24494]: audit 2024-09-13T14:29:00.662570+0000 mgr.y (mgr.24287) 339 : audit [DBG] from='client.15318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:02.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:01 smithi177 bash[24494]: cluster 2024-09-13T14:29:00.844455+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-09-13T14:29:02.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:02 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:02] "GET /metrics HTTP/1.1" 200 246442 "" "Prometheus/2.43.0" 2024-09-13T14:29:03.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:02 smithi177 bash[24494]: cluster 2024-09-13T14:29:01.231608+0000 mgr.y (mgr.24287) 340 : cluster [DBG] pgmap v377: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 108 KiB/s rd, 3.6 MiB/s wr, 153 op/s 2024-09-13T14:29:03.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:02 smithi177 bash[24494]: cluster 2024-09-13T14:29:01.872539+0000 mon.a (mon.0) 1013 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-09-13T14:29:03.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:02 smithi099 bash[32082]: cluster 2024-09-13T14:29:01.231608+0000 mgr.y (mgr.24287) 340 : cluster [DBG] pgmap v377: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 108 KiB/s rd, 3.6 MiB/s wr, 153 op/s 2024-09-13T14:29:03.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:02 smithi099 bash[32082]: cluster 2024-09-13T14:29:01.872539+0000 mon.a (mon.0) 1013 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-09-13T14:29:03.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:02 smithi099 bash[22656]: cluster 2024-09-13T14:29:01.231608+0000 mgr.y (mgr.24287) 340 : cluster [DBG] pgmap v377: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 108 KiB/s rd, 3.6 MiB/s wr, 153 op/s 2024-09-13T14:29:03.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:02 smithi099 bash[22656]: cluster 2024-09-13T14:29:01.872539+0000 mon.a (mon.0) 1013 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-09-13T14:29:03.346 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare (13282 ms) 2024-09-13T14:29:03.346 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare2 2024-09-13T14:29:03.387 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:03.401 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:03.423 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 41943040~10 2024-09-13T14:29:03.439 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:29:04.031 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 41943040 2024-09-13T14:29:04.061 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:29:04.188 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:03 smithi099 bash[32082]: cluster 2024-09-13T14:29:02.872565+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-09-13T14:29:04.188 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:03 smithi099 bash[32082]: audit 2024-09-13T14:29:03.051547+0000 mon.c (mon.2) 157 : audit [DBG] from='client.? 172.21.15.99:0/2461620979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:04.188 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:03 smithi099 bash[32082]: audit 2024-09-13T14:29:03.868956+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:29:04.188 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22909]: debug 2024-09-13T14:29:04.021+0000 7fc5f2cb0640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T14:29:04.188 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22909]: ignoring --setuser ceph since I am not root 2024-09-13T14:29:04.189 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22909]: ignoring --setgroup ceph since I am not root 2024-09-13T14:29:04.189 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:03 smithi099 bash[22656]: cluster 2024-09-13T14:29:02.872565+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-09-13T14:29:04.189 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:03 smithi099 bash[22656]: audit 2024-09-13T14:29:03.051547+0000 mon.c (mon.2) 157 : audit [DBG] from='client.? 172.21.15.99:0/2461620979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:04.189 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:03 smithi099 bash[22656]: audit 2024-09-13T14:29:03.868956+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:29:04.352 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[52886]: [13/Sep/2024:14:29:04] ENGINE Bus STOPPING 2024-09-13T14:29:04.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:03 smithi177 bash[24494]: cluster 2024-09-13T14:29:02.872565+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-09-13T14:29:04.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:03 smithi177 bash[24494]: audit 2024-09-13T14:29:03.051547+0000 mon.c (mon.2) 157 : audit [DBG] from='client.? 172.21.15.99:0/2461620979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:04.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:03 smithi177 bash[24494]: audit 2024-09-13T14:29:03.868956+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:29:04.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22909]: debug 2024-09-13T14:29:04.193+0000 7f0ff413aec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:29:04.852 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[52886]: [13/Sep/2024:14:29:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:29:04.852 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[52886]: [13/Sep/2024:14:29:04] ENGINE Bus STOPPED 2024-09-13T14:29:05.150 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: cluster 2024-09-13T14:29:03.233229+0000 mgr.y (mgr.24287) 341 : cluster [DBG] pgmap v380: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 5.2 MiB/s wr, 397 op/s 2024-09-13T14:29:05.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:03.381392+0000 mgr.y (mgr.24287) 342 : audit [DBG] from='client.15330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:05.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:03.670928+0000 mgr.y (mgr.24287) 343 : audit [DBG] from='client.15336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:05.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:03.882653+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:29:05.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:03.886139+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:05.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: cluster 2024-09-13T14:29:03.894848+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-09-13T14:29:05.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:03.899240+0000 mon.c (mon.2) 159 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:05.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:03.910710+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:29:05.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:03.984297+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:05.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:03.997232+0000 mon.c (mon.2) 161 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:03.997943+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: cluster 2024-09-13T14:29:03.998865+0000 mon.a (mon.0) 1020 : cluster [INF] Activating manager daemon x 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: cluster 2024-09-13T14:29:04.016430+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.020351+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: cluster 2024-09-13T14:29:04.020471+0000 mon.a (mon.0) 1023 : cluster [DBG] mgrmap e22: x(active, starting, since 0.021604s) 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.023778+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.024284+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.024766+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.025472+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.outtkv"}]: dispatch 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.026148+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.026833+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.027493+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:29:05.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.028347+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:29:05.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.029102+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:29:05.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.030053+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:29:05.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.030734+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:29:05.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.031334+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:29:05.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.031808+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:29:05.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.032281+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:29:05.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.033550+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:29:05.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.034097+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:29:05.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: audit 2024-09-13T14:29:04.034823+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:29:05.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:04 smithi177 bash[24494]: cluster 2024-09-13T14:29:04.556143+0000 mon.a (mon.0) 1024 : cluster [INF] Manager daemon x is now available 2024-09-13T14:29:05.292 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:05.310 INFO:tasks.workunit.client.0.smithi099.stdout:write: Y 41943040~5 2024-09-13T14:29:05.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: cluster 2024-09-13T14:29:03.233229+0000 mgr.y (mgr.24287) 341 : cluster [DBG] pgmap v380: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 5.2 MiB/s wr, 397 op/s 2024-09-13T14:29:05.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:03.381392+0000 mgr.y (mgr.24287) 342 : audit [DBG] from='client.15330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:05.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:03.670928+0000 mgr.y (mgr.24287) 343 : audit [DBG] from='client.15336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:03.882653+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:29:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:03.886139+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: cluster 2024-09-13T14:29:03.894848+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-09-13T14:29:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:03.899240+0000 mon.c (mon.2) 159 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:03.910710+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:29:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:03.984297+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:03.997232+0000 mon.c (mon.2) 161 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:29:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:03.997943+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:29:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: cluster 2024-09-13T14:29:03.998865+0000 mon.a (mon.0) 1020 : cluster [INF] Activating manager daemon x 2024-09-13T14:29:05.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: cluster 2024-09-13T14:29:04.016430+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-09-13T14:29:05.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.020351+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:29:05.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: cluster 2024-09-13T14:29:04.020471+0000 mon.a (mon.0) 1023 : cluster [DBG] mgrmap e22: x(active, starting, since 0.021604s) 2024-09-13T14:29:05.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.023778+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:29:05.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.024284+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:29:05.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.024766+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:29:05.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.025472+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.outtkv"}]: dispatch 2024-09-13T14:29:05.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.026148+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:29:05.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: cluster 2024-09-13T14:29:03.233229+0000 mgr.y (mgr.24287) 341 : cluster [DBG] pgmap v380: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 5.2 MiB/s wr, 397 op/s 2024-09-13T14:29:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:03.381392+0000 mgr.y (mgr.24287) 342 : audit [DBG] from='client.15330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:03.670928+0000 mgr.y (mgr.24287) 343 : audit [DBG] from='client.15336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:03.882653+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:29:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:03.886139+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: cluster 2024-09-13T14:29:03.894848+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-09-13T14:29:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:03.899240+0000 mon.c (mon.2) 159 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:03.910710+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.24287 ' entity='mgr.y' 2024-09-13T14:29:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:03.984297+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:03.997232+0000 mon.c (mon.2) 161 : audit [INF] from='mgr.24287 172.21.15.99:0/532006890' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:29:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:03.997943+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:29:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: cluster 2024-09-13T14:29:03.998865+0000 mon.a (mon.0) 1020 : cluster [INF] Activating manager daemon x 2024-09-13T14:29:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: cluster 2024-09-13T14:29:04.016430+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-09-13T14:29:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.020351+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:29:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: cluster 2024-09-13T14:29:04.020471+0000 mon.a (mon.0) 1023 : cluster [DBG] mgrmap e22: x(active, starting, since 0.021604s) 2024-09-13T14:29:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.023778+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:29:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.024284+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:29:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.024766+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:29:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.025472+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.outtkv"}]: dispatch 2024-09-13T14:29:05.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.026148+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:29:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.026833+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:29:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.027493+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:29:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.028347+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:29:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.029102+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:29:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.030053+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:29:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.030734+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:29:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.031334+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:29:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.031808+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:29:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.032281+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:29:05.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.033550+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:29:05.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.034097+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:29:05.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: audit 2024-09-13T14:29:04.034823+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:29:05.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[32082]: cluster 2024-09-13T14:29:04.556143+0000 mon.a (mon.0) 1024 : cluster [INF] Manager daemon x is now available 2024-09-13T14:29:05.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.026833+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:29:05.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.027493+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:29:05.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.028347+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:29:05.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.029102+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:29:05.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.030053+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:29:05.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.030734+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:29:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.031334+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:29:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.031808+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:29:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.032281+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:29:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.033550+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:29:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.034097+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:29:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: audit 2024-09-13T14:29:04.034823+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:29:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:04 smithi099 bash[22656]: cluster 2024-09-13T14:29:04.556143+0000 mon.a (mon.0) 1024 : cluster [INF] Manager daemon x is now available 2024-09-13T14:29:05.451 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: [13/Sep/2024:14:29:05] ENGINE Bus STARTING 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: debug 2024-09-13T14:29:05.210+0000 7ff2f398e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: debug 2024-09-13T14:29:05.210+0000 7ff2f398e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: debug 2024-09-13T14:29:05.210+0000 7ff2f398e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: debug 2024-09-13T14:29:05.210+0000 7ff2f398e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: debug 2024-09-13T14:29:05.210+0000 7ff2f398e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: debug 2024-09-13T14:29:05.214+0000 7ff2f198a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: debug 2024-09-13T14:29:05.214+0000 7ff2f198a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: debug 2024-09-13T14:29:05.214+0000 7ff2f198a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: debug 2024-09-13T14:29:05.214+0000 7ff2f198a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: debug 2024-09-13T14:29:05.214+0000 7ff2f198a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: [13/Sep/2024:14:29:05] ENGINE Serving on http://:::9283 2024-09-13T14:29:05.452 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: [13/Sep/2024:14:29:05] ENGINE Bus STARTED 2024-09-13T14:29:05.453 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:05 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:05] "GET /metrics HTTP/1.1" 200 38564 "" "Prometheus/2.43.0" 2024-09-13T14:29:05.489 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:29:05.622 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:29:05.632 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:05 smithi099 bash[22909]: debug 2024-09-13T14:29:05.481+0000 7f0ff413aec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:29:05.633 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:05 smithi099 bash[22909]: debug 2024-09-13T14:29:05.553+0000 7f0ff413aec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:29:06.001 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:05 smithi099 bash[22909]: debug 2024-09-13T14:29:05.721+0000 7f0ff413aec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:29:06.252 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare2 (2905 ms) 2024-09-13T14:29:06.252 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.SnapsSnapAfterPrepare 2024-09-13T14:29:06.304 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: cluster 2024-09-13T14:29:05.011685+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-09-13T14:29:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: cluster 2024-09-13T14:29:05.029604+0000 mon.a (mon.0) 1026 : cluster [DBG] mgrmap e23: x(active, since 1.03073s) 2024-09-13T14:29:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.077271+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.088102+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: cephadm 2024-09-13T14:29:05.092089+0000 mgr.x (mgr.24637) 2 : cephadm [INF] Checking for cert/key for grafana.a 2024-09-13T14:29:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: cephadm 2024-09-13T14:29:05.092189+0000 mgr.x (mgr.24637) 3 : cephadm [INF] Migrating grafana.a cert to cert store 2024-09-13T14:29:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.096817+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: cephadm 2024-09-13T14:29:05.100225+0000 mgr.x (mgr.24637) 4 : cephadm [INF] Migrating grafana.a key to cert store 2024-09-13T14:29:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.104357+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.112013+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.122300+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.129906+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:06.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.148442+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:29:06.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.148622+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:06.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.149515+0000 mgr.x (mgr.24637) 5 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:29:06.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.168001+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:29:06.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.170030+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:29:06.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.180796+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:29:06.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[32082]: audit 2024-09-13T14:29:05.182884+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:29:06.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: cluster 2024-09-13T14:29:05.011685+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-09-13T14:29:06.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: cluster 2024-09-13T14:29:05.029604+0000 mon.a (mon.0) 1026 : cluster [DBG] mgrmap e23: x(active, since 1.03073s) 2024-09-13T14:29:06.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.077271+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.088102+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: cephadm 2024-09-13T14:29:05.092089+0000 mgr.x (mgr.24637) 2 : cephadm [INF] Checking for cert/key for grafana.a 2024-09-13T14:29:06.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: cephadm 2024-09-13T14:29:05.092189+0000 mgr.x (mgr.24637) 3 : cephadm [INF] Migrating grafana.a cert to cert store 2024-09-13T14:29:06.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.096817+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: cephadm 2024-09-13T14:29:05.100225+0000 mgr.x (mgr.24637) 4 : cephadm [INF] Migrating grafana.a key to cert store 2024-09-13T14:29:06.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.104357+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.112013+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.122300+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.129906+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:06.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.148442+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:29:06.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.148622+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:06.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.149515+0000 mgr.x (mgr.24637) 5 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:29:06.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.168001+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:29:06.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.170030+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:29:06.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.180796+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:29:06.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22656]: audit 2024-09-13T14:29:05.182884+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:29:06.323 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:29:06.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: cluster 2024-09-13T14:29:05.011685+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-09-13T14:29:06.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: cluster 2024-09-13T14:29:05.029604+0000 mon.a (mon.0) 1026 : cluster [DBG] mgrmap e23: x(active, since 1.03073s) 2024-09-13T14:29:06.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.077271+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.088102+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: cephadm 2024-09-13T14:29:05.092089+0000 mgr.x (mgr.24637) 2 : cephadm [INF] Checking for cert/key for grafana.a 2024-09-13T14:29:06.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: cephadm 2024-09-13T14:29:05.092189+0000 mgr.x (mgr.24637) 3 : cephadm [INF] Migrating grafana.a cert to cert store 2024-09-13T14:29:06.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.096817+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: cephadm 2024-09-13T14:29:05.100225+0000 mgr.x (mgr.24637) 4 : cephadm [INF] Migrating grafana.a key to cert store 2024-09-13T14:29:06.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.104357+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.112013+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.122300+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:06.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.129906+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:29:06.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.148442+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:29:06.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.148622+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:06.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.149515+0000 mgr.x (mgr.24637) 5 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:29:06.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.168001+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:29:06.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.170030+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:29:06.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.180796+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:29:06.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[24494]: audit 2024-09-13T14:29:05.182884+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:29:06.412 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:29:06.432 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:06.722 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22909]: debug 2024-09-13T14:29:06.421+0000 7f0ff413aec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:29:06.722 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22909]: debug 2024-09-13T14:29:06.497+0000 7f0ff413aec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:29:06.722 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22909]: debug 2024-09-13T14:29:06.569+0000 7f0ff413aec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:29:06.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: debug 2024-09-13T14:29:06.374+0000 7ff2d7eda640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:29:06] ENGINE Error in HTTPServer.serve 2024-09-13T14:29:06.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: Traceback (most recent call last): 2024-09-13T14:29:06.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:29:06.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: self._connections.run(self.expiration_interval) 2024-09-13T14:29:06.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:29:06.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: self._run(expiration_interval) 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: s = self.context.wrap_socket( 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: return self.sslsocket_class._create( 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: self.do_handshake() 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:29:06.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: self._sslobj.do_handshake() 2024-09-13T14:29:06.855 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:06 smithi177 bash[52886]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:29:07.021 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22909]: debug 2024-09-13T14:29:06.725+0000 7f0ff413aec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:29:07.021 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22909]: debug 2024-09-13T14:29:06.805+0000 7f0ff413aec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:29:07.021 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:06 smithi099 bash[22909]: debug 2024-09-13T14:29:06.869+0000 7f0ff413aec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:29:07.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22909]: debug 2024-09-13T14:29:07.105+0000 7f0ff413aec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:29:07.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22909]: debug 2024-09-13T14:29:07.181+0000 7f0ff413aec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: cephadm 2024-09-13T14:29:05.748356+0000 mgr.x (mgr.24637) 6 : cephadm [INF] Deploying cephadm binary to smithi177 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: cephadm 2024-09-13T14:29:05.770454+0000 mgr.x (mgr.24637) 7 : cephadm [INF] Deploying cephadm binary to smithi099 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: cluster 2024-09-13T14:29:06.033671+0000 mgr.x (mgr.24637) 8 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: audit 2024-09-13T14:29:06.035599+0000 mon.c (mon.2) 162 : audit [DBG] from='client.? 172.21.15.99:0/2269410006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: cephadm 2024-09-13T14:29:06.056206+0000 mgr.x (mgr.24637) 9 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Bus STARTING 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: cluster 2024-09-13T14:29:06.124043+0000 mon.a (mon.0) 1035 : cluster [DBG] mgrmap e24: x(active, since 2s) 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: cluster 2024-09-13T14:29:06.128528+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: cephadm 2024-09-13T14:29:06.176668+0000 mgr.x (mgr.24637) 10 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Serving on http://172.21.15.177:8765 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: cephadm 2024-09-13T14:29:06.378043+0000 mgr.x (mgr.24637) 11 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Serving on https://172.21.15.177:7150 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: cephadm 2024-09-13T14:29:06.378232+0000 mgr.x (mgr.24637) 12 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Bus STARTED 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: cephadm 2024-09-13T14:29:06.380038+0000 mgr.x (mgr.24637) 13 : cephadm [ERR] [13/Sep/2024:14:29:06] ENGINE Error in HTTPServer.serve 2024-09-13T14:29:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: Traceback (most recent call last): 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: self._connections.run(self.expiration_interval) 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: self._run(expiration_interval) 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: s = self.context.wrap_socket( 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: return self.sslsocket_class._create( 2024-09-13T14:29:07.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:29:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: self.do_handshake() 2024-09-13T14:29:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:29:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: self._sslobj.do_handshake() 2024-09-13T14:29:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:29:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22656]: audit 2024-09-13T14:29:06.410557+0000 mgr.x (mgr.24637) 14 : audit [DBG] from='client.15381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:07.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: cephadm 2024-09-13T14:29:05.748356+0000 mgr.x (mgr.24637) 6 : cephadm [INF] Deploying cephadm binary to smithi177 2024-09-13T14:29:07.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: cephadm 2024-09-13T14:29:05.770454+0000 mgr.x (mgr.24637) 7 : cephadm [INF] Deploying cephadm binary to smithi099 2024-09-13T14:29:07.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: cluster 2024-09-13T14:29:06.033671+0000 mgr.x (mgr.24637) 8 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:29:07.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: audit 2024-09-13T14:29:06.035599+0000 mon.c (mon.2) 162 : audit [DBG] from='client.? 172.21.15.99:0/2269410006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: cephadm 2024-09-13T14:29:06.056206+0000 mgr.x (mgr.24637) 9 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Bus STARTING 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: cluster 2024-09-13T14:29:06.124043+0000 mon.a (mon.0) 1035 : cluster [DBG] mgrmap e24: x(active, since 2s) 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: cluster 2024-09-13T14:29:06.128528+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: cephadm 2024-09-13T14:29:06.176668+0000 mgr.x (mgr.24637) 10 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Serving on http://172.21.15.177:8765 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: cephadm 2024-09-13T14:29:06.378043+0000 mgr.x (mgr.24637) 11 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Serving on https://172.21.15.177:7150 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: cephadm 2024-09-13T14:29:06.378232+0000 mgr.x (mgr.24637) 12 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Bus STARTED 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: cephadm 2024-09-13T14:29:06.380038+0000 mgr.x (mgr.24637) 13 : cephadm [ERR] [13/Sep/2024:14:29:06] ENGINE Error in HTTPServer.serve 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: Traceback (most recent call last): 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: self._connections.run(self.expiration_interval) 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: self._run(expiration_interval) 2024-09-13T14:29:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:29:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:29:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:29:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:29:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:29:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: s = self.context.wrap_socket( 2024-09-13T14:29:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:29:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: return self.sslsocket_class._create( 2024-09-13T14:29:07.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:29:07.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: self.do_handshake() 2024-09-13T14:29:07.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:29:07.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: self._sslobj.do_handshake() 2024-09-13T14:29:07.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:29:07.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[32082]: audit 2024-09-13T14:29:06.410557+0000 mgr.x (mgr.24637) 14 : audit [DBG] from='client.15381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:07.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: cephadm 2024-09-13T14:29:05.748356+0000 mgr.x (mgr.24637) 6 : cephadm [INF] Deploying cephadm binary to smithi177 2024-09-13T14:29:07.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: cephadm 2024-09-13T14:29:05.770454+0000 mgr.x (mgr.24637) 7 : cephadm [INF] Deploying cephadm binary to smithi099 2024-09-13T14:29:07.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: cluster 2024-09-13T14:29:06.033671+0000 mgr.x (mgr.24637) 8 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:29:07.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: audit 2024-09-13T14:29:06.035599+0000 mon.c (mon.2) 162 : audit [DBG] from='client.? 172.21.15.99:0/2269410006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:07.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: cephadm 2024-09-13T14:29:06.056206+0000 mgr.x (mgr.24637) 9 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Bus STARTING 2024-09-13T14:29:07.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: cluster 2024-09-13T14:29:06.124043+0000 mon.a (mon.0) 1035 : cluster [DBG] mgrmap e24: x(active, since 2s) 2024-09-13T14:29:07.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: cluster 2024-09-13T14:29:06.128528+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-09-13T14:29:07.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: cephadm 2024-09-13T14:29:06.176668+0000 mgr.x (mgr.24637) 10 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Serving on http://172.21.15.177:8765 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: cephadm 2024-09-13T14:29:06.378043+0000 mgr.x (mgr.24637) 11 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Serving on https://172.21.15.177:7150 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: cephadm 2024-09-13T14:29:06.378232+0000 mgr.x (mgr.24637) 12 : cephadm [INF] [13/Sep/2024:14:29:06] ENGINE Bus STARTED 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: cephadm 2024-09-13T14:29:06.380038+0000 mgr.x (mgr.24637) 13 : cephadm [ERR] [13/Sep/2024:14:29:06] ENGINE Error in HTTPServer.serve 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: Traceback (most recent call last): 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: self._connections.run(self.expiration_interval) 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: self._run(expiration_interval) 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:29:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: s = self.context.wrap_socket( 2024-09-13T14:29:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:29:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: return self.sslsocket_class._create( 2024-09-13T14:29:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:29:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: self.do_handshake() 2024-09-13T14:29:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:29:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: self._sslobj.do_handshake() 2024-09-13T14:29:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:29:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:07 smithi177 bash[24494]: audit 2024-09-13T14:29:06.410557+0000 mgr.x (mgr.24637) 14 : audit [DBG] from='client.15381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:07.654 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22909]: debug 2024-09-13T14:29:07.349+0000 7f0ff413aec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:29:07.654 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22909]: debug 2024-09-13T14:29:07.425+0000 7f0ff413aec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:29:08.043 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22909]: debug 2024-09-13T14:29:07.657+0000 7f0ff413aec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:29:08.044 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22909]: debug 2024-09-13T14:29:07.817+0000 7f0ff413aec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:29:08.044 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:07 smithi099 bash[22909]: debug 2024-09-13T14:29:07.893+0000 7f0ff413aec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:29:08.164 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 8388609~8388609 2024-09-13T14:29:08.193 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:29:08.193 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:29:08.196 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:29:08.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:08 smithi099 bash[22909]: debug 2024-09-13T14:29:08.049+0000 7f0ff413aec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:29:08.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:08 smithi099 bash[22656]: audit 2024-09-13T14:29:06.735267+0000 mgr.x (mgr.24637) 15 : audit [DBG] from='client.25280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:08.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:08 smithi099 bash[22656]: cluster 2024-09-13T14:29:07.133689+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-09-13T14:29:08.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:08 smithi099 bash[32082]: audit 2024-09-13T14:29:06.735267+0000 mgr.x (mgr.24637) 15 : audit [DBG] from='client.25280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:08.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:08 smithi099 bash[32082]: cluster 2024-09-13T14:29:07.133689+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-09-13T14:29:08.602 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:08 smithi177 bash[24494]: audit 2024-09-13T14:29:06.735267+0000 mgr.x (mgr.24637) 15 : audit [DBG] from='client.25280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:08.602 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:08 smithi177 bash[24494]: cluster 2024-09-13T14:29:07.133689+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-09-13T14:29:09.133 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:08 smithi099 bash[22909]: debug 2024-09-13T14:29:08.877+0000 7f0ff413aec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:29:09.133 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:08 smithi099 bash[22909]: debug 2024-09-13T14:29:08.953+0000 7f0ff413aec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:29:09.405 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:09.549 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22909]: debug 2024-09-13T14:29:09.157+0000 7f0ff413aec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:29:09.550 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22909]: debug 2024-09-13T14:29:09.233+0000 7f0ff413aec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:29:09.550 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[32082]: cluster 2024-09-13T14:29:08.034113+0000 mgr.x (mgr.24637) 16 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:29:09.550 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[32082]: cluster 2024-09-13T14:29:08.151977+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-09-13T14:29:09.550 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[32082]: cluster 2024-09-13T14:29:08.152800+0000 mon.a (mon.0) 1039 : cluster [DBG] mgrmap e25: x(active, since 4s) 2024-09-13T14:29:09.550 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[32082]: cluster 2024-09-13T14:29:08.575218+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-09-13T14:29:09.550 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[32082]: audit 2024-09-13T14:29:09.093684+0000 mon.a (mon.0) 1041 : audit [DBG] from='client.? 172.21.15.99:0/2614320511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:09.550 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22656]: cluster 2024-09-13T14:29:08.034113+0000 mgr.x (mgr.24637) 16 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:29:09.550 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22656]: cluster 2024-09-13T14:29:08.151977+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-09-13T14:29:09.551 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22656]: cluster 2024-09-13T14:29:08.152800+0000 mon.a (mon.0) 1039 : cluster [DBG] mgrmap e25: x(active, since 4s) 2024-09-13T14:29:09.551 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22656]: cluster 2024-09-13T14:29:08.575218+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-09-13T14:29:09.551 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22656]: audit 2024-09-13T14:29:09.093684+0000 mon.a (mon.0) 1041 : audit [DBG] from='client.? 172.21.15.99:0/2614320511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:09.594 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 8388609~1000 2024-09-13T14:29:09.602 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:09 smithi177 bash[24494]: cluster 2024-09-13T14:29:08.034113+0000 mgr.x (mgr.24637) 16 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:29:09.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:09 smithi177 bash[24494]: cluster 2024-09-13T14:29:08.151977+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-09-13T14:29:09.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:09 smithi177 bash[24494]: cluster 2024-09-13T14:29:08.152800+0000 mon.a (mon.0) 1039 : cluster [DBG] mgrmap e25: x(active, since 4s) 2024-09-13T14:29:09.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:09 smithi177 bash[24494]: cluster 2024-09-13T14:29:08.575218+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-09-13T14:29:09.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:09 smithi177 bash[24494]: audit 2024-09-13T14:29:09.093684+0000 mon.a (mon.0) 1041 : audit [DBG] from='client.? 172.21.15.99:0/2614320511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:09.606 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16777228~1000 2024-09-13T14:29:09.606 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:29:09.620 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 83886079~8388609 2024-09-13T14:29:09.660 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:29:09.817 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22909]: debug 2024-09-13T14:29:09.553+0000 7f0ff413aec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:29:09.817 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22909]: debug 2024-09-13T14:29:09.641+0000 7f0ff413aec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:29:09.817 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22909]: [13/Sep/2024:14:29:09] ENGINE Bus STARTING 2024-09-13T14:29:09.818 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22909]: CherryPy Checker: 2024-09-13T14:29:09.818 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22909]: The Application mounted at '' has an empty config. 2024-09-13T14:29:09.818 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22909]: [13/Sep/2024:14:29:09] ENGINE Serving on http://:::9283 2024-09-13T14:29:09.818 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:09 smithi099 bash[22909]: [13/Sep/2024:14:29:09] ENGINE Bus STARTED 2024-09-13T14:29:10.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[22656]: audit 2024-09-13T14:29:09.402934+0000 mgr.x (mgr.24637) 17 : audit [DBG] from='client.15405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:10.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[22656]: cluster 2024-09-13T14:29:09.587257+0000 mon.a (mon.0) 1042 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-09-13T14:29:10.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[22656]: cluster 2024-09-13T14:29:09.666412+0000 mon.a (mon.0) 1043 : cluster [DBG] Standby manager daemon y started 2024-09-13T14:29:10.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[22656]: audit 2024-09-13T14:29:09.670418+0000 mon.a (mon.0) 1044 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:29:10.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[22656]: audit 2024-09-13T14:29:09.676806+0000 mon.a (mon.0) 1045 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:29:10.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[22656]: audit 2024-09-13T14:29:09.677655+0000 mon.a (mon.0) 1046 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:29:10.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[22656]: audit 2024-09-13T14:29:09.677948+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:29:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[32082]: audit 2024-09-13T14:29:09.402934+0000 mgr.x (mgr.24637) 17 : audit [DBG] from='client.15405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[32082]: cluster 2024-09-13T14:29:09.587257+0000 mon.a (mon.0) 1042 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-09-13T14:29:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[32082]: cluster 2024-09-13T14:29:09.666412+0000 mon.a (mon.0) 1043 : cluster [DBG] Standby manager daemon y started 2024-09-13T14:29:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[32082]: audit 2024-09-13T14:29:09.670418+0000 mon.a (mon.0) 1044 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:29:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[32082]: audit 2024-09-13T14:29:09.676806+0000 mon.a (mon.0) 1045 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:29:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[32082]: audit 2024-09-13T14:29:09.677655+0000 mon.a (mon.0) 1046 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:29:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:10 smithi099 bash[32082]: audit 2024-09-13T14:29:09.677948+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:29:10.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:10 smithi177 bash[24494]: audit 2024-09-13T14:29:09.402934+0000 mgr.x (mgr.24637) 17 : audit [DBG] from='client.15405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:10.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:10 smithi177 bash[24494]: cluster 2024-09-13T14:29:09.587257+0000 mon.a (mon.0) 1042 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-09-13T14:29:10.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:10 smithi177 bash[24494]: cluster 2024-09-13T14:29:09.666412+0000 mon.a (mon.0) 1043 : cluster [DBG] Standby manager daemon y started 2024-09-13T14:29:10.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:10 smithi177 bash[24494]: audit 2024-09-13T14:29:09.670418+0000 mon.a (mon.0) 1044 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:29:10.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:10 smithi177 bash[24494]: audit 2024-09-13T14:29:09.676806+0000 mon.a (mon.0) 1045 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:29:10.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:10 smithi177 bash[24494]: audit 2024-09-13T14:29:09.677655+0000 mon.a (mon.0) 1046 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:29:10.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:10 smithi177 bash[24494]: audit 2024-09-13T14:29:09.677948+0000 mon.a (mon.0) 1047 : audit [DBG] from='mgr.? 172.21.15.99:0/3363456828' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:29:11.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:11 smithi099 bash[22656]: audit 2024-09-13T14:29:09.725125+0000 mgr.x (mgr.24637) 18 : audit [DBG] from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:11.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:11 smithi099 bash[22656]: cluster 2024-09-13T14:29:10.034950+0000 mgr.x (mgr.24637) 19 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 621 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 137 KiB/s rd, 1.0 MiB/s wr, 168 op/s 2024-09-13T14:29:11.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:11 smithi099 bash[22656]: cluster 2024-09-13T14:29:10.600954+0000 mon.a (mon.0) 1048 : cluster [DBG] mgrmap e26: x(active, since 6s), standbys: y 2024-09-13T14:29:11.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:11 smithi099 bash[22656]: cluster 2024-09-13T14:29:10.610569+0000 mon.a (mon.0) 1049 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-09-13T14:29:11.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:11 smithi099 bash[22656]: audit 2024-09-13T14:29:10.613272+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:29:11.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:11 smithi099 bash[32082]: audit 2024-09-13T14:29:09.725125+0000 mgr.x (mgr.24637) 18 : audit [DBG] from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:11.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:11 smithi099 bash[32082]: cluster 2024-09-13T14:29:10.034950+0000 mgr.x (mgr.24637) 19 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 621 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 137 KiB/s rd, 1.0 MiB/s wr, 168 op/s 2024-09-13T14:29:11.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:11 smithi099 bash[32082]: cluster 2024-09-13T14:29:10.600954+0000 mon.a (mon.0) 1048 : cluster [DBG] mgrmap e26: x(active, since 6s), standbys: y 2024-09-13T14:29:11.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:11 smithi099 bash[32082]: cluster 2024-09-13T14:29:10.610569+0000 mon.a (mon.0) 1049 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-09-13T14:29:11.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:11 smithi099 bash[32082]: audit 2024-09-13T14:29:10.613272+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:29:11.602 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:11 smithi177 bash[24494]: audit 2024-09-13T14:29:09.725125+0000 mgr.x (mgr.24637) 18 : audit [DBG] from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:11.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:11 smithi177 bash[24494]: cluster 2024-09-13T14:29:10.034950+0000 mgr.x (mgr.24637) 19 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 621 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 137 KiB/s rd, 1.0 MiB/s wr, 168 op/s 2024-09-13T14:29:11.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:11 smithi177 bash[24494]: cluster 2024-09-13T14:29:10.600954+0000 mon.a (mon.0) 1048 : cluster [DBG] mgrmap e26: x(active, since 6s), standbys: y 2024-09-13T14:29:11.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:11 smithi177 bash[24494]: cluster 2024-09-13T14:29:10.610569+0000 mon.a (mon.0) 1049 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-09-13T14:29:11.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:11 smithi177 bash[24494]: audit 2024-09-13T14:29:10.613272+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:29:11.630 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:11.683 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83886079~1 2024-09-13T14:29:11.704 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:29:12.423 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:12.567 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:12 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:13.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:12 smithi099 bash[22656]: cluster 2024-09-13T14:29:11.616823+0000 mon.a (mon.0) 1050 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-09-13T14:29:13.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:12 smithi099 bash[22656]: cluster 2024-09-13T14:29:12.035708+0000 mgr.x (mgr.24637) 20 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 378 KiB/s rd, 12 MiB/s wr, 456 op/s 2024-09-13T14:29:13.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:12 smithi099 bash[22656]: audit 2024-09-13T14:29:12.105537+0000 mon.a (mon.0) 1051 : audit [DBG] from='client.? 172.21.15.99:0/738732159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:13.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:12 smithi099 bash[22656]: audit 2024-09-13T14:29:12.419992+0000 mgr.x (mgr.24637) 21 : audit [DBG] from='client.25316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:13.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:12 smithi099 bash[32082]: cluster 2024-09-13T14:29:11.616823+0000 mon.a (mon.0) 1050 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-09-13T14:29:13.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:12 smithi099 bash[32082]: cluster 2024-09-13T14:29:12.035708+0000 mgr.x (mgr.24637) 20 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 378 KiB/s rd, 12 MiB/s wr, 456 op/s 2024-09-13T14:29:13.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:12 smithi099 bash[32082]: audit 2024-09-13T14:29:12.105537+0000 mon.a (mon.0) 1051 : audit [DBG] from='client.? 172.21.15.99:0/738732159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:13.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:12 smithi099 bash[32082]: audit 2024-09-13T14:29:12.419992+0000 mgr.x (mgr.24637) 21 : audit [DBG] from='client.25316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:13.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:12 smithi177 bash[24494]: cluster 2024-09-13T14:29:11.616823+0000 mon.a (mon.0) 1050 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-09-13T14:29:13.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:12 smithi177 bash[24494]: cluster 2024-09-13T14:29:12.035708+0000 mgr.x (mgr.24637) 20 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 378 KiB/s rd, 12 MiB/s wr, 456 op/s 2024-09-13T14:29:13.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:12 smithi177 bash[24494]: audit 2024-09-13T14:29:12.105537+0000 mon.a (mon.0) 1051 : audit [DBG] from='client.? 172.21.15.99:0/738732159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:13.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:12 smithi177 bash[24494]: audit 2024-09-13T14:29:12.419992+0000 mgr.x (mgr.24637) 21 : audit [DBG] from='client.25316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:14.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:13 smithi099 bash[22656]: cluster 2024-09-13T14:29:12.623386+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-09-13T14:29:14.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:13 smithi099 bash[22656]: audit 2024-09-13T14:29:12.745855+0000 mgr.x (mgr.24637) 22 : audit [DBG] from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:14.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:13 smithi099 bash[22656]: cluster 2024-09-13T14:29:13.576854+0000 mon.a (mon.0) 1053 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-09-13T14:29:14.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:13 smithi099 bash[32082]: cluster 2024-09-13T14:29:12.623386+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-09-13T14:29:14.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:13 smithi099 bash[32082]: audit 2024-09-13T14:29:12.745855+0000 mgr.x (mgr.24637) 22 : audit [DBG] from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:14.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:13 smithi099 bash[32082]: cluster 2024-09-13T14:29:13.576854+0000 mon.a (mon.0) 1053 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-09-13T14:29:14.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:13 smithi177 bash[24494]: cluster 2024-09-13T14:29:12.623386+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-09-13T14:29:14.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:13 smithi177 bash[24494]: audit 2024-09-13T14:29:12.745855+0000 mgr.x (mgr.24637) 22 : audit [DBG] from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:14.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:13 smithi177 bash[24494]: cluster 2024-09-13T14:29:13.576854+0000 mon.a (mon.0) 1053 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-09-13T14:29:15.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[32082]: audit 2024-09-13T14:29:13.671753+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[32082]: audit 2024-09-13T14:29:13.679207+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[32082]: cluster 2024-09-13T14:29:14.036417+0000 mgr.x (mgr.24637) 23 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 233 KiB/s rd, 11 MiB/s wr, 279 op/s 2024-09-13T14:29:15.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[32082]: audit 2024-09-13T14:29:14.115643+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[32082]: audit 2024-09-13T14:29:14.122629+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[32082]: cluster 2024-09-13T14:29:14.583393+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-09-13T14:29:15.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[22656]: audit 2024-09-13T14:29:13.671753+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[22656]: audit 2024-09-13T14:29:13.679207+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[22656]: cluster 2024-09-13T14:29:14.036417+0000 mgr.x (mgr.24637) 23 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 233 KiB/s rd, 11 MiB/s wr, 279 op/s 2024-09-13T14:29:15.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[22656]: audit 2024-09-13T14:29:14.115643+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[22656]: audit 2024-09-13T14:29:14.122629+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:14 smithi099 bash[22656]: cluster 2024-09-13T14:29:14.583393+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-09-13T14:29:15.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:14 smithi177 bash[24494]: audit 2024-09-13T14:29:13.671753+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:14 smithi177 bash[24494]: audit 2024-09-13T14:29:13.679207+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:14 smithi177 bash[24494]: cluster 2024-09-13T14:29:14.036417+0000 mgr.x (mgr.24637) 23 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 233 KiB/s rd, 11 MiB/s wr, 279 op/s 2024-09-13T14:29:15.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:14 smithi177 bash[24494]: audit 2024-09-13T14:29:14.115643+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:14 smithi177 bash[24494]: audit 2024-09-13T14:29:14.122629+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:15.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:14 smithi177 bash[24494]: cluster 2024-09-13T14:29:14.583393+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-09-13T14:29:15.400 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: after_prepare_snap 2024-09-13T14:29:15.494 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:15.852 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:15] "GET /metrics HTTP/1.1" 200 38564 "" "Prometheus/2.43.0" 2024-09-13T14:29:16.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.859611+0000 mon.a (mon.0) 1059 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.876005+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.881542+0000 mon.a (mon.0) 1061 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.883647+0000 mon.a (mon.0) 1062 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.883792+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.885279+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.885920+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.886825+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.887695+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.889263+0000 mon.b (mon.1) 61 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: cephadm 2024-09-13T14:29:14.890840+0000 mgr.x (mgr.24637) 24 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 2524M 2024-09-13T14:29:16.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:14.893980+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:15.157029+0000 mon.a (mon.0) 1066 : audit [DBG] from='client.? 172.21.15.99:0/3159417031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:16.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:15.287453+0000 mon.a (mon.0) 1067 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:15.296295+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[22656]: audit 2024-09-13T14:29:15.492055+0000 mgr.x (mgr.24637) 25 : audit [DBG] from='client.25334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:16.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.859611+0000 mon.a (mon.0) 1059 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.876005+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.881542+0000 mon.a (mon.0) 1061 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.883647+0000 mon.a (mon.0) 1062 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.883792+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.885279+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.885920+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.886825+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.887695+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.889263+0000 mon.b (mon.1) 61 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: cephadm 2024-09-13T14:29:14.890840+0000 mgr.x (mgr.24637) 24 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 2524M 2024-09-13T14:29:16.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:14.893980+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:15.157029+0000 mon.a (mon.0) 1066 : audit [DBG] from='client.? 172.21.15.99:0/3159417031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:16.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:15.287453+0000 mon.a (mon.0) 1067 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:15.296295+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:15 smithi099 bash[32082]: audit 2024-09-13T14:29:15.492055+0000 mgr.x (mgr.24637) 25 : audit [DBG] from='client.25334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:16.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.859611+0000 mon.a (mon.0) 1059 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.876005+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.881542+0000 mon.a (mon.0) 1061 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.883647+0000 mon.a (mon.0) 1062 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.883792+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.885279+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.885920+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.886825+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.887695+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.889263+0000 mon.b (mon.1) 61 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:16.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: cephadm 2024-09-13T14:29:14.890840+0000 mgr.x (mgr.24637) 24 : cephadm [INF] Adjusting osd_memory_target on smithi099 to 2524M 2024-09-13T14:29:16.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:14.893980+0000 mon.a (mon.0) 1065 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:15.157029+0000 mon.a (mon.0) 1066 : audit [DBG] from='client.? 172.21.15.99:0/3159417031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:16.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:15.287453+0000 mon.a (mon.0) 1067 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:15.296295+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:16.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:15 smithi177 bash[24494]: audit 2024-09-13T14:29:15.492055+0000 mgr.x (mgr.24637) 25 : audit [DBG] from='client.25334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:17.328 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 41943040 2024-09-13T14:29:17.409 INFO:tasks.workunit.client.0.smithi099.stdout:write: * 0~1000 2024-09-13T14:29:17.425 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:29:17.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:17 smithi099 bash[22656]: audit 2024-09-13T14:29:15.798369+0000 mgr.x (mgr.24637) 26 : audit [DBG] from='client.15447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:17.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:17 smithi099 bash[22656]: cluster 2024-09-13T14:29:16.037993+0000 mgr.x (mgr.24637) 27 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 11 KiB/s wr, 189 op/s 2024-09-13T14:29:17.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:17 smithi099 bash[22656]: cluster 2024-09-13T14:29:16.302074+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-09-13T14:29:17.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:17 smithi099 bash[32082]: audit 2024-09-13T14:29:15.798369+0000 mgr.x (mgr.24637) 26 : audit [DBG] from='client.15447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:17.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:17 smithi099 bash[32082]: cluster 2024-09-13T14:29:16.037993+0000 mgr.x (mgr.24637) 27 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 11 KiB/s wr, 189 op/s 2024-09-13T14:29:17.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:17 smithi099 bash[32082]: cluster 2024-09-13T14:29:16.302074+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-09-13T14:29:17.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:17 smithi177 bash[24494]: audit 2024-09-13T14:29:15.798369+0000 mgr.x (mgr.24637) 26 : audit [DBG] from='client.15447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:17 smithi177 bash[24494]: cluster 2024-09-13T14:29:16.037993+0000 mgr.x (mgr.24637) 27 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 11 KiB/s wr, 189 op/s 2024-09-13T14:29:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:17 smithi177 bash[24494]: cluster 2024-09-13T14:29:16.302074+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-09-13T14:29:17.689 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:29:18.501 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:18.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:18 smithi099 bash[32082]: cluster 2024-09-13T14:29:17.305980+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-09-13T14:29:18.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:18 smithi099 bash[32082]: audit 2024-09-13T14:29:18.171924+0000 mon.c (mon.2) 163 : audit [DBG] from='client.? 172.21.15.99:0/1875761892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:18.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:18 smithi099 bash[22656]: cluster 2024-09-13T14:29:17.305980+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-09-13T14:29:18.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:18 smithi099 bash[22656]: audit 2024-09-13T14:29:18.171924+0000 mon.c (mon.2) 163 : audit [DBG] from='client.? 172.21.15.99:0/1875761892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:18.602 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:18 smithi177 bash[24494]: cluster 2024-09-13T14:29:17.305980+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-09-13T14:29:18.602 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:18 smithi177 bash[24494]: audit 2024-09-13T14:29:18.171924+0000 mon.c (mon.2) 163 : audit [DBG] from='client.? 172.21.15.99:0/1875761892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:19.602 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:19 smithi177 bash[24494]: cluster 2024-09-13T14:29:18.038877+0000 mgr.x (mgr.24637) 28 : cluster [DBG] pgmap v23: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 11 KiB/s wr, 188 op/s 2024-09-13T14:29:19.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:19 smithi177 bash[24494]: cluster 2024-09-13T14:29:18.322256+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-09-13T14:29:19.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:19 smithi177 bash[24494]: audit 2024-09-13T14:29:18.498859+0000 mgr.x (mgr.24637) 29 : audit [DBG] from='client.15459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:19.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:19 smithi099 bash[22656]: cluster 2024-09-13T14:29:18.038877+0000 mgr.x (mgr.24637) 28 : cluster [DBG] pgmap v23: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 11 KiB/s wr, 188 op/s 2024-09-13T14:29:19.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:19 smithi099 bash[22656]: cluster 2024-09-13T14:29:18.322256+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-09-13T14:29:19.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:19 smithi099 bash[22656]: audit 2024-09-13T14:29:18.498859+0000 mgr.x (mgr.24637) 29 : audit [DBG] from='client.15459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:19.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:19 smithi099 bash[32082]: cluster 2024-09-13T14:29:18.038877+0000 mgr.x (mgr.24637) 28 : cluster [DBG] pgmap v23: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 11 KiB/s wr, 188 op/s 2024-09-13T14:29:19.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:19 smithi099 bash[32082]: cluster 2024-09-13T14:29:18.322256+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-09-13T14:29:19.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:19 smithi099 bash[32082]: audit 2024-09-13T14:29:18.498859+0000 mgr.x (mgr.24637) 29 : audit [DBG] from='client.15459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:20.632 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:20 smithi177 bash[24494]: audit 2024-09-13T14:29:18.810471+0000 mgr.x (mgr.24637) 30 : audit [DBG] from='client.15465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:20.632 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:20 smithi177 bash[24494]: cluster 2024-09-13T14:29:19.328940+0000 mon.a (mon.0) 1072 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-09-13T14:29:20.632 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:20 smithi177 bash[24494]: audit 2024-09-13T14:29:20.149726+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:20.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:20 smithi099 bash[22656]: audit 2024-09-13T14:29:18.810471+0000 mgr.x (mgr.24637) 30 : audit [DBG] from='client.15465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:20.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:20 smithi099 bash[22656]: cluster 2024-09-13T14:29:19.328940+0000 mon.a (mon.0) 1072 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-09-13T14:29:20.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:20 smithi099 bash[22656]: audit 2024-09-13T14:29:20.149726+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:20.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:20 smithi099 bash[32082]: audit 2024-09-13T14:29:18.810471+0000 mgr.x (mgr.24637) 30 : audit [DBG] from='client.15465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:20.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:20 smithi099 bash[32082]: cluster 2024-09-13T14:29:19.328940+0000 mon.a (mon.0) 1072 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-09-13T14:29:20.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:20 smithi099 bash[32082]: audit 2024-09-13T14:29:20.149726+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:20.930 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.SnapsSnapAfterPrepare (14678 ms) 2024-09-13T14:29:20.930 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.SnapsSnapExpandAfterPrepare 2024-09-13T14:29:20.984 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:21.005 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:29:21.097 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:29:21.532 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:21.805 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:21 smithi177 bash[24494]: cluster 2024-09-13T14:29:20.040380+0000 mgr.x (mgr.24637) 31 : cluster [DBG] pgmap v26: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s rd, 5.0 MiB/s wr, 242 op/s 2024-09-13T14:29:21.805 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:21 smithi177 bash[24494]: cluster 2024-09-13T14:29:20.357725+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-09-13T14:29:21.805 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:21 smithi177 bash[24494]: audit 2024-09-13T14:29:21.207953+0000 mon.a (mon.0) 1074 : audit [DBG] from='client.? 172.21.15.99:0/924339792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:21.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:21 smithi099 bash[22656]: cluster 2024-09-13T14:29:20.040380+0000 mgr.x (mgr.24637) 31 : cluster [DBG] pgmap v26: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s rd, 5.0 MiB/s wr, 242 op/s 2024-09-13T14:29:21.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:21 smithi099 bash[22656]: cluster 2024-09-13T14:29:20.357725+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-09-13T14:29:21.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:21 smithi099 bash[22656]: audit 2024-09-13T14:29:21.207953+0000 mon.a (mon.0) 1074 : audit [DBG] from='client.? 172.21.15.99:0/924339792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:21.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:21 smithi099 bash[32082]: cluster 2024-09-13T14:29:20.040380+0000 mgr.x (mgr.24637) 31 : cluster [DBG] pgmap v26: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s rd, 5.0 MiB/s wr, 242 op/s 2024-09-13T14:29:21.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:21 smithi099 bash[32082]: cluster 2024-09-13T14:29:20.357725+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-09-13T14:29:21.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:21 smithi099 bash[32082]: audit 2024-09-13T14:29:21.207953+0000 mon.a (mon.0) 1074 : audit [DBG] from='client.? 172.21.15.99:0/924339792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:22.366 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:22 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:22.396 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 8388609~8388609 2024-09-13T14:29:22.425 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:29:22.425 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:29:22.426 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:29:22.602 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:22 smithi177 bash[24494]: cluster 2024-09-13T14:29:21.356175+0000 mon.a (mon.0) 1075 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-09-13T14:29:22.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:22 smithi099 bash[22656]: cluster 2024-09-13T14:29:21.356175+0000 mon.a (mon.0) 1075 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-09-13T14:29:22.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:22 smithi099 bash[32082]: cluster 2024-09-13T14:29:21.356175+0000 mon.a (mon.0) 1075 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-09-13T14:29:23.602 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 8388609~1000 2024-09-13T14:29:23.642 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16777228~1000 2024-09-13T14:29:23.642 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:29:23.642 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 83886079~8388609 2024-09-13T14:29:23.689 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:29:23.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:23 smithi099 bash[22656]: audit 2024-09-13T14:29:21.528491+0000 mgr.x (mgr.24637) 32 : audit [DBG] from='client.15477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:23.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:23 smithi099 bash[22656]: audit 2024-09-13T14:29:21.833711+0000 mgr.x (mgr.24637) 33 : audit [DBG] from='client.25376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:23.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:23 smithi099 bash[22656]: cluster 2024-09-13T14:29:22.041197+0000 mgr.x (mgr.24637) 34 : cluster [DBG] pgmap v29: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 6.0 MiB/s wr, 449 op/s 2024-09-13T14:29:23.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:23 smithi099 bash[22656]: cluster 2024-09-13T14:29:22.380329+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-09-13T14:29:23.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:23 smithi099 bash[32082]: audit 2024-09-13T14:29:21.528491+0000 mgr.x (mgr.24637) 32 : audit [DBG] from='client.15477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:23.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:23 smithi099 bash[32082]: audit 2024-09-13T14:29:21.833711+0000 mgr.x (mgr.24637) 33 : audit [DBG] from='client.25376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:23.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:23 smithi099 bash[32082]: cluster 2024-09-13T14:29:22.041197+0000 mgr.x (mgr.24637) 34 : cluster [DBG] pgmap v29: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 6.0 MiB/s wr, 449 op/s 2024-09-13T14:29:23.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:23 smithi099 bash[32082]: cluster 2024-09-13T14:29:22.380329+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-09-13T14:29:23.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:23 smithi177 bash[24494]: audit 2024-09-13T14:29:21.528491+0000 mgr.x (mgr.24637) 32 : audit [DBG] from='client.15477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:23.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:23 smithi177 bash[24494]: audit 2024-09-13T14:29:21.833711+0000 mgr.x (mgr.24637) 33 : audit [DBG] from='client.25376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:23.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:23 smithi177 bash[24494]: cluster 2024-09-13T14:29:22.041197+0000 mgr.x (mgr.24637) 34 : cluster [DBG] pgmap v29: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 6.0 MiB/s wr, 449 op/s 2024-09-13T14:29:23.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:23 smithi177 bash[24494]: cluster 2024-09-13T14:29:22.380329+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-09-13T14:29:24.527 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:24.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:24 smithi099 bash[32082]: cluster 2024-09-13T14:29:23.392826+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-09-13T14:29:24.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:24 smithi099 bash[32082]: cluster 2024-09-13T14:29:23.588291+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-09-13T14:29:24.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:24 smithi099 bash[32082]: audit 2024-09-13T14:29:24.207944+0000 mon.c (mon.2) 164 : audit [DBG] from='client.? 172.21.15.99:0/3774558906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:24.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:24 smithi099 bash[22656]: cluster 2024-09-13T14:29:23.392826+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-09-13T14:29:24.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:24 smithi099 bash[22656]: cluster 2024-09-13T14:29:23.588291+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-09-13T14:29:24.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:24 smithi099 bash[22656]: audit 2024-09-13T14:29:24.207944+0000 mon.c (mon.2) 164 : audit [DBG] from='client.? 172.21.15.99:0/3774558906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:24.852 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:24 smithi177 bash[24494]: cluster 2024-09-13T14:29:23.392826+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-09-13T14:29:24.852 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:24 smithi177 bash[24494]: cluster 2024-09-13T14:29:23.588291+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-09-13T14:29:24.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:24 smithi177 bash[24494]: audit 2024-09-13T14:29:24.207944+0000 mon.c (mon.2) 164 : audit [DBG] from='client.? 172.21.15.99:0/3774558906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:25.603 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:25 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:25] "GET /metrics HTTP/1.1" 200 38909 "" "Prometheus/2.43.0" 2024-09-13T14:29:25.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:25 smithi177 bash[24494]: cluster 2024-09-13T14:29:24.041786+0000 mgr.x (mgr.24637) 35 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:29:25.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:25 smithi177 bash[24494]: audit 2024-09-13T14:29:24.523521+0000 mgr.x (mgr.24637) 36 : audit [DBG] from='client.15495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:25.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:25 smithi177 bash[24494]: cluster 2024-09-13T14:29:24.588865+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-09-13T14:29:25.613 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:25.669 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83886079~1 2024-09-13T14:29:25.690 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:29:25.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:25 smithi099 bash[32082]: cluster 2024-09-13T14:29:24.041786+0000 mgr.x (mgr.24637) 35 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:29:25.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:25 smithi099 bash[32082]: audit 2024-09-13T14:29:24.523521+0000 mgr.x (mgr.24637) 36 : audit [DBG] from='client.15495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:25.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:25 smithi099 bash[32082]: cluster 2024-09-13T14:29:24.588865+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-09-13T14:29:25.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:25 smithi099 bash[22656]: cluster 2024-09-13T14:29:24.041786+0000 mgr.x (mgr.24637) 35 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:29:25.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:25 smithi099 bash[22656]: audit 2024-09-13T14:29:24.523521+0000 mgr.x (mgr.24637) 36 : audit [DBG] from='client.15495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:25.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:25 smithi099 bash[22656]: cluster 2024-09-13T14:29:24.588865+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-09-13T14:29:26.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:24.837659+0000 mgr.x (mgr.24637) 37 : audit [DBG] from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:26.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: cluster 2024-09-13T14:29:25.595112+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-09-13T14:29:26.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.829481+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:26.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.845132+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:26.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.850194+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.852347+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.852472+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.854832+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.854863+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.856894+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.856923+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.859114+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.866995+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.873208+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[32082]: audit 2024-09-13T14:29:25.874777+0000 mon.b (mon.1) 68 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:26.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:24.837659+0000 mgr.x (mgr.24637) 37 : audit [DBG] from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:26.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: cluster 2024-09-13T14:29:25.595112+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-09-13T14:29:26.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.829481+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:26.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.845132+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:26.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.850194+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.852347+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.852472+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.854832+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.854863+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.856894+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.856923+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.859114+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.866995+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.873208+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:26.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:26 smithi099 bash[22656]: audit 2024-09-13T14:29:25.874777+0000 mon.b (mon.1) 68 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:26.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:24.837659+0000 mgr.x (mgr.24637) 37 : audit [DBG] from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:26.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: cluster 2024-09-13T14:29:25.595112+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-09-13T14:29:26.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.829481+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:26.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.845132+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:26.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.850194+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.852347+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.852472+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.854832+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.854863+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.856894+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.856923+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.859114+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:29:26.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.866995+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:26.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.873208+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:29:26.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:26 smithi177 bash[24494]: audit 2024-09-13T14:29:25.874777+0000 mon.b (mon.1) 68 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:29:27.468 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:27 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:27.469 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:29:27 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:27.469 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:29:27 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:27.469 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:29:27 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:27.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:27.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cephadm 2024-09-13T14:29:25.860972+0000 mgr.x (mgr.24637) 38 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 2524M 2024-09-13T14:29:27.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cephadm 2024-09-13T14:29:25.876691+0000 mgr.x (mgr.24637) 39 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:29:27.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cephadm 2024-09-13T14:29:25.877141+0000 mgr.x (mgr.24637) 40 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:29:27.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cluster 2024-09-13T14:29:26.042490+0000 mgr.x (mgr.24637) 41 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 13 MiB/s wr, 210 op/s 2024-09-13T14:29:27.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cephadm 2024-09-13T14:29:26.082029+0000 mgr.x (mgr.24637) 42 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:29:27.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cephadm 2024-09-13T14:29:26.082204+0000 mgr.x (mgr.24637) 43 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:29:27.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cephadm 2024-09-13T14:29:26.217117+0000 mgr.x (mgr.24637) 44 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:29:27.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cephadm 2024-09-13T14:29:26.226274+0000 mgr.x (mgr.24637) 45 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:29:27.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cephadm 2024-09-13T14:29:26.341094+0000 mgr.x (mgr.24637) 46 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:29:27.471 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cephadm 2024-09-13T14:29:26.365737+0000 mgr.x (mgr.24637) 47 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:29:27.471 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:29:27 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:27.471 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:27 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:27.472 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:27 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:27.472 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:27 smithi099 systemd[1]: Stopping Ceph node-exporter.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:29:27.472 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:27.472 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cephadm 2024-09-13T14:29:25.860972+0000 mgr.x (mgr.24637) 38 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 2524M 2024-09-13T14:29:27.472 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cephadm 2024-09-13T14:29:25.876691+0000 mgr.x (mgr.24637) 39 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:29:27.472 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cephadm 2024-09-13T14:29:25.877141+0000 mgr.x (mgr.24637) 40 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:29:27.472 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cluster 2024-09-13T14:29:26.042490+0000 mgr.x (mgr.24637) 41 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 13 MiB/s wr, 210 op/s 2024-09-13T14:29:27.472 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cephadm 2024-09-13T14:29:26.082029+0000 mgr.x (mgr.24637) 42 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cephadm 2024-09-13T14:29:26.082204+0000 mgr.x (mgr.24637) 43 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cephadm 2024-09-13T14:29:26.217117+0000 mgr.x (mgr.24637) 44 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cephadm 2024-09-13T14:29:26.226274+0000 mgr.x (mgr.24637) 45 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cephadm 2024-09-13T14:29:26.341094+0000 mgr.x (mgr.24637) 46 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cephadm 2024-09-13T14:29:26.365737+0000 mgr.x (mgr.24637) 47 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: audit 2024-09-13T14:29:26.476670+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: audit 2024-09-13T14:29:26.486767+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: audit 2024-09-13T14:29:26.509101+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: audit 2024-09-13T14:29:26.518939+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: audit 2024-09-13T14:29:26.529986+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: audit 2024-09-13T14:29:26.577761+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.542 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:27.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: audit 2024-09-13T14:29:26.587072+0000 mon.a (mon.0) 1094 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: audit 2024-09-13T14:29:26.594202+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: audit 2024-09-13T14:29:26.603518+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: cluster 2024-09-13T14:29:26.848757+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-09-13T14:29:27.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[32082]: audit 2024-09-13T14:29:27.222112+0000 mon.c (mon.2) 165 : audit [DBG] from='client.? 172.21.15.99:0/3064635919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:27.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: audit 2024-09-13T14:29:26.476670+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: audit 2024-09-13T14:29:26.486767+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: audit 2024-09-13T14:29:26.509101+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: audit 2024-09-13T14:29:26.518939+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: audit 2024-09-13T14:29:26.529986+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: audit 2024-09-13T14:29:26.577761+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: audit 2024-09-13T14:29:26.587072+0000 mon.a (mon.0) 1094 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: audit 2024-09-13T14:29:26.594202+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: audit 2024-09-13T14:29:26.603518+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: cluster 2024-09-13T14:29:26.848757+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-09-13T14:29:27.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:27 smithi099 bash[22656]: audit 2024-09-13T14:29:27.222112+0000 mon.c (mon.2) 165 : audit [DBG] from='client.? 172.21.15.99:0/3064635919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:27.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cephadm 2024-09-13T14:29:25.860972+0000 mgr.x (mgr.24637) 38 : cephadm [INF] Adjusting osd_memory_target on smithi177 to 2524M 2024-09-13T14:29:27.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cephadm 2024-09-13T14:29:25.876691+0000 mgr.x (mgr.24637) 39 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:29:27.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cephadm 2024-09-13T14:29:25.877141+0000 mgr.x (mgr.24637) 40 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:29:27.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cluster 2024-09-13T14:29:26.042490+0000 mgr.x (mgr.24637) 41 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 13 MiB/s wr, 210 op/s 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cephadm 2024-09-13T14:29:26.082029+0000 mgr.x (mgr.24637) 42 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cephadm 2024-09-13T14:29:26.082204+0000 mgr.x (mgr.24637) 43 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cephadm 2024-09-13T14:29:26.217117+0000 mgr.x (mgr.24637) 44 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cephadm 2024-09-13T14:29:26.226274+0000 mgr.x (mgr.24637) 45 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cephadm 2024-09-13T14:29:26.341094+0000 mgr.x (mgr.24637) 46 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cephadm 2024-09-13T14:29:26.365737+0000 mgr.x (mgr.24637) 47 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: audit 2024-09-13T14:29:26.476670+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: audit 2024-09-13T14:29:26.486767+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: audit 2024-09-13T14:29:26.509101+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: audit 2024-09-13T14:29:26.518939+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: audit 2024-09-13T14:29:26.529986+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: audit 2024-09-13T14:29:26.577761+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: audit 2024-09-13T14:29:26.587072+0000 mon.a (mon.0) 1094 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: audit 2024-09-13T14:29:26.594202+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: audit 2024-09-13T14:29:26.603518+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:27.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: cluster 2024-09-13T14:29:26.848757+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-09-13T14:29:27.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:27 smithi177 bash[24494]: audit 2024-09-13T14:29:27.222112+0000 mon.c (mon.2) 165 : audit [DBG] from='client.? 172.21.15.99:0/3064635919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:28.742 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:28 smithi099 bash[22656]: cephadm 2024-09-13T14:29:26.609185+0000 mgr.x (mgr.24637) 48 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:29:28.743 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:28 smithi099 bash[22656]: cephadm 2024-09-13T14:29:26.610014+0000 mgr.x (mgr.24637) 49 : cephadm [INF] Deploying daemon node-exporter.a on smithi099 2024-09-13T14:29:28.743 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:28 smithi099 bash[22656]: cluster 2024-09-13T14:29:27.847589+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-09-13T14:29:28.743 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:28 smithi099 bash[32082]: cephadm 2024-09-13T14:29:26.609185+0000 mgr.x (mgr.24637) 48 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:29:28.743 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:28 smithi099 bash[32082]: cephadm 2024-09-13T14:29:26.610014+0000 mgr.x (mgr.24637) 49 : cephadm [INF] Deploying daemon node-exporter.a on smithi099 2024-09-13T14:29:28.743 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:28 smithi099 bash[32082]: cluster 2024-09-13T14:29:27.847589+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-09-13T14:29:28.852 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:28 smithi177 bash[24494]: cephadm 2024-09-13T14:29:26.609185+0000 mgr.x (mgr.24637) 48 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T14:29:28.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:28 smithi177 bash[24494]: cephadm 2024-09-13T14:29:26.610014+0000 mgr.x (mgr.24637) 49 : cephadm [INF] Deploying daemon node-exporter.a on smithi099 2024-09-13T14:29:28.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:28 smithi177 bash[24494]: cluster 2024-09-13T14:29:27.847589+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-09-13T14:29:28.999 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:28 smithi099 bash[74169]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T14:29:29.293 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:29.294 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:29.294 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:29.294 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:29.295 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:29.295 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:29.295 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:29:29.296 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[74257]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T14:29:29.296 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T14:29:29.296 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: Stopped Ceph node-exporter.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:29:29.296 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:29.296 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: Started Ceph node-exporter.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:29:29.296 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:29.297 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:29 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:29.325 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: after_prepare_snap 2024-09-13T14:29:29.567 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[74311]: Unable to find image 'quay.io/prometheus/node-exporter:v1.7.0' locally 2024-09-13T14:29:29.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[32082]: audit 2024-09-13T14:29:27.540168+0000 mgr.x (mgr.24637) 50 : audit [DBG] from='client.15513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:29.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[32082]: audit 2024-09-13T14:29:27.831740+0000 mgr.x (mgr.24637) 51 : audit [DBG] from='client.25412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:29.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[32082]: cluster 2024-09-13T14:29:28.043281+0000 mgr.x (mgr.24637) 52 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-09-13T14:29:29.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[32082]: cluster 2024-09-13T14:29:28.586370+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-09-13T14:29:29.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[32082]: audit 2024-09-13T14:29:29.353459+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:29.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[32082]: audit 2024-09-13T14:29:29.363646+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:29.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[22656]: audit 2024-09-13T14:29:27.540168+0000 mgr.x (mgr.24637) 50 : audit [DBG] from='client.15513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:29.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[22656]: audit 2024-09-13T14:29:27.831740+0000 mgr.x (mgr.24637) 51 : audit [DBG] from='client.25412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:29.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[22656]: cluster 2024-09-13T14:29:28.043281+0000 mgr.x (mgr.24637) 52 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-09-13T14:29:29.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[22656]: cluster 2024-09-13T14:29:28.586370+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-09-13T14:29:29.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[22656]: audit 2024-09-13T14:29:29.353459+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:29.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[22656]: audit 2024-09-13T14:29:29.363646+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:29.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:29 smithi177 bash[24494]: audit 2024-09-13T14:29:27.540168+0000 mgr.x (mgr.24637) 50 : audit [DBG] from='client.15513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:29.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:29 smithi177 bash[24494]: audit 2024-09-13T14:29:27.831740+0000 mgr.x (mgr.24637) 51 : audit [DBG] from='client.25412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:29.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:29 smithi177 bash[24494]: cluster 2024-09-13T14:29:28.043281+0000 mgr.x (mgr.24637) 52 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-09-13T14:29:29.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:29 smithi177 bash[24494]: cluster 2024-09-13T14:29:28.586370+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-09-13T14:29:29.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:29 smithi177 bash[24494]: audit 2024-09-13T14:29:29.353459+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:29.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:29 smithi177 bash[24494]: audit 2024-09-13T14:29:29.363646+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:30.159 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:29 smithi099 bash[74311]: v1.7.0: Pulling from prometheus/node-exporter 2024-09-13T14:29:30.444 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[74311]: 2abcce694348: Pulling fs layer 2024-09-13T14:29:30.444 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[74311]: 455fd88e5221: Pulling fs layer 2024-09-13T14:29:30.445 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[74311]: 324153f2810a: Pulling fs layer 2024-09-13T14:29:30.445 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[74311]: 2abcce694348: Download complete 2024-09-13T14:29:30.543 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:30.817 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[74311]: 324153f2810a: Verifying Checksum 2024-09-13T14:29:30.817 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[74311]: 324153f2810a: Download complete 2024-09-13T14:29:30.817 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[74311]: 455fd88e5221: Download complete 2024-09-13T14:29:30.817 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[74311]: 2abcce694348: Pull complete 2024-09-13T14:29:30.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[22656]: cephadm 2024-09-13T14:29:29.371363+0000 mgr.x (mgr.24637) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:29:30.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[22656]: cephadm 2024-09-13T14:29:29.395410+0000 mgr.x (mgr.24637) 54 : cephadm [INF] Deploying daemon alertmanager.a on smithi099 2024-09-13T14:29:30.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[22656]: cluster 2024-09-13T14:29:29.850901+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-09-13T14:29:30.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[22656]: audit 2024-09-13T14:29:30.202915+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.99:0/3136796249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:30.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[32082]: cephadm 2024-09-13T14:29:29.371363+0000 mgr.x (mgr.24637) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:29:30.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[32082]: cephadm 2024-09-13T14:29:29.395410+0000 mgr.x (mgr.24637) 54 : cephadm [INF] Deploying daemon alertmanager.a on smithi099 2024-09-13T14:29:30.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[32082]: cluster 2024-09-13T14:29:29.850901+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-09-13T14:29:30.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:30 smithi099 bash[32082]: audit 2024-09-13T14:29:30.202915+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.99:0/3136796249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:30.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:30 smithi177 bash[24494]: cephadm 2024-09-13T14:29:29.371363+0000 mgr.x (mgr.24637) 53 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:29:30.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:30 smithi177 bash[24494]: cephadm 2024-09-13T14:29:29.395410+0000 mgr.x (mgr.24637) 54 : cephadm [INF] Deploying daemon alertmanager.a on smithi099 2024-09-13T14:29:30.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:30 smithi177 bash[24494]: cluster 2024-09-13T14:29:29.850901+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-09-13T14:29:30.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:30 smithi177 bash[24494]: audit 2024-09-13T14:29:30.202915+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.99:0/3136796249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:30.862 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:29:30.885 INFO:tasks.workunit.client.0.smithi099.stdout:write: * 83886080~1000 2024-09-13T14:29:30.898 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:29:31.086 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:29:31.473 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:31 smithi099 bash[74311]: 455fd88e5221: Pull complete 2024-09-13T14:29:31.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:31 smithi099 bash[32082]: cluster 2024-09-13T14:29:30.044648+0000 mgr.x (mgr.24637) 55 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 6.2 KiB/s wr, 198 op/s 2024-09-13T14:29:31.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:31 smithi099 bash[32082]: cluster 2024-09-13T14:29:30.852892+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-09-13T14:29:31.818 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:31 smithi099 bash[74311]: 324153f2810a: Pull complete 2024-09-13T14:29:31.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:31 smithi099 bash[22656]: cluster 2024-09-13T14:29:30.044648+0000 mgr.x (mgr.24637) 55 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 6.2 KiB/s wr, 198 op/s 2024-09-13T14:29:31.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:31 smithi099 bash[22656]: cluster 2024-09-13T14:29:30.852892+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-09-13T14:29:31.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:31 smithi177 bash[24494]: cluster 2024-09-13T14:29:30.044648+0000 mgr.x (mgr.24637) 55 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 6.2 KiB/s wr, 198 op/s 2024-09-13T14:29:31.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:31 smithi177 bash[24494]: cluster 2024-09-13T14:29:30.852892+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-09-13T14:29:32.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:32 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:32.318 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:32 smithi099 bash[74311]: Digest: sha256:4cb2b9019f1757be8482419002cb7afe028fdba35d47958829e4cfeaf6246d80 2024-09-13T14:29:32.318 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:32 smithi099 bash[74311]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.7.0 2024-09-13T14:29:32.817 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:32 smithi099 bash[32082]: audit 2024-09-13T14:29:30.541517+0000 mgr.x (mgr.24637) 56 : audit [DBG] from='client.15531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:32.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:32 smithi099 bash[32082]: audit 2024-09-13T14:29:30.874430+0000 mgr.x (mgr.24637) 57 : audit [DBG] from='client.25430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:32.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:32 smithi099 bash[32082]: cluster 2024-09-13T14:29:31.870024+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-09-13T14:29:32.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:32 smithi099 bash[22656]: audit 2024-09-13T14:29:30.541517+0000 mgr.x (mgr.24637) 56 : audit [DBG] from='client.15531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:32.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:32 smithi099 bash[22656]: audit 2024-09-13T14:29:30.874430+0000 mgr.x (mgr.24637) 57 : audit [DBG] from='client.25430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:32.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:32 smithi099 bash[22656]: cluster 2024-09-13T14:29:31.870024+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-09-13T14:29:32.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:32 smithi177 bash[24494]: audit 2024-09-13T14:29:30.541517+0000 mgr.x (mgr.24637) 56 : audit [DBG] from='client.15531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:32.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:32 smithi177 bash[24494]: audit 2024-09-13T14:29:30.874430+0000 mgr.x (mgr.24637) 57 : audit [DBG] from='client.25430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:32.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:32 smithi177 bash[24494]: cluster 2024-09-13T14:29:31.870024+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-09-13T14:29:33.705 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:33.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:33 smithi099 bash[22656]: cluster 2024-09-13T14:29:32.045446+0000 mgr.x (mgr.24637) 58 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-09-13T14:29:33.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:33 smithi099 bash[22656]: cluster 2024-09-13T14:29:32.881557+0000 mon.a (mon.0) 1105 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-09-13T14:29:33.817 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:33 smithi099 bash[22656]: audit 2024-09-13T14:29:33.258010+0000 mon.a (mon.0) 1106 : audit [DBG] from='client.? 172.21.15.99:0/1829377592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:33.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:33 smithi099 bash[32082]: cluster 2024-09-13T14:29:32.045446+0000 mgr.x (mgr.24637) 58 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-09-13T14:29:33.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:33 smithi099 bash[32082]: cluster 2024-09-13T14:29:32.881557+0000 mon.a (mon.0) 1105 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-09-13T14:29:33.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:33 smithi099 bash[32082]: audit 2024-09-13T14:29:33.258010+0000 mon.a (mon.0) 1106 : audit [DBG] from='client.? 172.21.15.99:0/1829377592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:33.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:33 smithi177 bash[24494]: cluster 2024-09-13T14:29:32.045446+0000 mgr.x (mgr.24637) 58 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-09-13T14:29:33.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:33 smithi177 bash[24494]: cluster 2024-09-13T14:29:32.881557+0000 mon.a (mon.0) 1105 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-09-13T14:29:33.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:33 smithi177 bash[24494]: audit 2024-09-13T14:29:33.258010+0000 mon.a (mon.0) 1106 : audit [DBG] from='client.? 172.21.15.99:0/1829377592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:34.593 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.SnapsSnapExpandAfterPrepare (13663 ms) 2024-09-13T14:29:34.594 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.Clone 2024-09-13T14:29:34.675 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:34.702 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~8388609 2024-09-13T14:29:34.806 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 16777218~8388609 2024-09-13T14:29:34.866 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap 2024-09-13T14:29:35.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:34 smithi099 bash[22656]: audit 2024-09-13T14:29:33.570729+0000 mgr.x (mgr.24637) 59 : audit [DBG] from='client.25442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:35.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:34 smithi099 bash[22656]: cluster 2024-09-13T14:29:33.882147+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-09-13T14:29:35.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:34 smithi099 bash[22656]: audit 2024-09-13T14:29:34.040803+0000 mgr.x (mgr.24637) 60 : audit [DBG] from='client.25448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:35.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:34 smithi099 bash[22656]: cluster 2024-09-13T14:29:34.046184+0000 mgr.x (mgr.24637) 61 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 5.5 KiB/s wr, 79 op/s 2024-09-13T14:29:35.185 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:34 smithi099 bash[32082]: audit 2024-09-13T14:29:33.570729+0000 mgr.x (mgr.24637) 59 : audit [DBG] from='client.25442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:35.185 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:34 smithi099 bash[32082]: cluster 2024-09-13T14:29:33.882147+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-09-13T14:29:35.185 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:34 smithi099 bash[32082]: audit 2024-09-13T14:29:34.040803+0000 mgr.x (mgr.24637) 60 : audit [DBG] from='client.25448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:35.185 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:34 smithi099 bash[32082]: cluster 2024-09-13T14:29:34.046184+0000 mgr.x (mgr.24637) 61 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 5.5 KiB/s wr, 79 op/s 2024-09-13T14:29:35.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:34 smithi177 bash[24494]: audit 2024-09-13T14:29:33.570729+0000 mgr.x (mgr.24637) 59 : audit [DBG] from='client.25442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:35.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:34 smithi177 bash[24494]: cluster 2024-09-13T14:29:33.882147+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-09-13T14:29:35.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:34 smithi177 bash[24494]: audit 2024-09-13T14:29:34.040803+0000 mgr.x (mgr.24637) 60 : audit [DBG] from='client.25448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:35.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:34 smithi177 bash[24494]: cluster 2024-09-13T14:29:34.046184+0000 mgr.x (mgr.24637) 61 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 5.5 KiB/s wr, 79 op/s 2024-09-13T14:29:35.568 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.275Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.275Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.276Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.276Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:29:35.569 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:29:35.570 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:29:35.571 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:29:35.571 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:29:35.571 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:29:35.571 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:29:35.571 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:29:35.571 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:29:35.571 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:29:35.571 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:29:35.571 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.277Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.278Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.278Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.278Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:29:35.572 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.278Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:29:35.573 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.278Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:29:35.573 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.278Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:29:35.573 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.278Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:29:35.573 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.278Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:29:35.573 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.278Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:29:35.573 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[74311]: ts=2024-09-13T14:29:35.278Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:29:35.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:35 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:35] "GET /metrics HTTP/1.1" 200 38907 "" "Prometheus/2.43.0" 2024-09-13T14:29:35.923 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap 2024-09-13T14:29:35.951 INFO:tasks.workunit.client.0.smithi099.stdout:clone image103 -> image106 2024-09-13T14:29:36.173 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:29:36.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[22656]: cluster 2024-09-13T14:29:34.889044+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-09-13T14:29:36.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[22656]: audit 2024-09-13T14:29:35.150015+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:36.244 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[32082]: cluster 2024-09-13T14:29:34.889044+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-09-13T14:29:36.245 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:35 smithi099 bash[32082]: audit 2024-09-13T14:29:35.150015+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:36.268 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:29:36.272 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap 2024-09-13T14:29:36.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:35 smithi177 bash[24494]: cluster 2024-09-13T14:29:34.889044+0000 mon.a (mon.0) 1108 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-09-13T14:29:36.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:35 smithi177 bash[24494]: audit 2024-09-13T14:29:35.150015+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:36.749 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:37.282 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:36 smithi099 bash[22656]: cluster 2024-09-13T14:29:35.901896+0000 mon.a (mon.0) 1109 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:29:37.282 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:36 smithi099 bash[22656]: cluster 2024-09-13T14:29:36.047069+0000 mgr.x (mgr.24637) 62 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 434 op/s 2024-09-13T14:29:37.282 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:36 smithi099 bash[22656]: audit 2024-09-13T14:29:36.435114+0000 mon.a (mon.0) 1110 : audit [DBG] from='client.? 172.21.15.99:0/1946822794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:37.283 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:36 smithi099 bash[32082]: cluster 2024-09-13T14:29:35.901896+0000 mon.a (mon.0) 1109 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:29:37.283 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:36 smithi099 bash[32082]: cluster 2024-09-13T14:29:36.047069+0000 mgr.x (mgr.24637) 62 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 434 op/s 2024-09-13T14:29:37.283 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:36 smithi099 bash[32082]: audit 2024-09-13T14:29:36.435114+0000 mon.a (mon.0) 1110 : audit [DBG] from='client.? 172.21.15.99:0/1946822794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:37.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:36 smithi177 bash[24494]: cluster 2024-09-13T14:29:35.901896+0000 mon.a (mon.0) 1109 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-09-13T14:29:37.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:36 smithi177 bash[24494]: cluster 2024-09-13T14:29:36.047069+0000 mgr.x (mgr.24637) 62 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 434 op/s 2024-09-13T14:29:37.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:36 smithi177 bash[24494]: audit 2024-09-13T14:29:36.435114+0000 mon.a (mon.0) 1110 : audit [DBG] from='client.? 172.21.15.99:0/1946822794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:37.943 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap 2024-09-13T14:29:37.970 INFO:tasks.workunit.client.0.smithi099.stdout:clone image106 -> image108 2024-09-13T14:29:38.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:37 smithi099 bash[22656]: audit 2024-09-13T14:29:36.746262+0000 mgr.x (mgr.24637) 63 : audit [DBG] from='client.25460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:38.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:37 smithi099 bash[22656]: cluster 2024-09-13T14:29:36.909223+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:29:38.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:37 smithi099 bash[22656]: audit 2024-09-13T14:29:37.067789+0000 mgr.x (mgr.24637) 64 : audit [DBG] from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:38.150 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:37 smithi099 bash[32082]: audit 2024-09-13T14:29:36.746262+0000 mgr.x (mgr.24637) 63 : audit [DBG] from='client.25460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:38.150 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:37 smithi099 bash[32082]: cluster 2024-09-13T14:29:36.909223+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:29:38.150 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:37 smithi099 bash[32082]: audit 2024-09-13T14:29:37.067789+0000 mgr.x (mgr.24637) 64 : audit [DBG] from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:38.257 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 1000~1000 2024-09-13T14:29:38.346 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 2010~1000 2024-09-13T14:29:38.350 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:29:38.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:37 smithi177 bash[24494]: audit 2024-09-13T14:29:36.746262+0000 mgr.x (mgr.24637) 63 : audit [DBG] from='client.25460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:38.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:37 smithi177 bash[24494]: cluster 2024-09-13T14:29:36.909223+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-09-13T14:29:38.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:37 smithi177 bash[24494]: audit 2024-09-13T14:29:37.067789+0000 mgr.x (mgr.24637) 64 : audit [DBG] from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:39.105 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:29:39.243 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:38 smithi099 bash[22656]: cluster 2024-09-13T14:29:37.920277+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:29:39.243 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:38 smithi099 bash[22656]: cluster 2024-09-13T14:29:38.047978+0000 mgr.x (mgr.24637) 65 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 434 op/s 2024-09-13T14:29:39.243 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:38 smithi099 bash[22656]: cluster 2024-09-13T14:29:38.596907+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:29:39.243 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:38 smithi099 bash[32082]: cluster 2024-09-13T14:29:37.920277+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:29:39.243 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:38 smithi099 bash[32082]: cluster 2024-09-13T14:29:38.047978+0000 mgr.x (mgr.24637) 65 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 434 op/s 2024-09-13T14:29:39.243 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:38 smithi099 bash[32082]: cluster 2024-09-13T14:29:38.596907+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:29:39.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:38 smithi177 bash[24494]: cluster 2024-09-13T14:29:37.920277+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-09-13T14:29:39.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:38 smithi177 bash[24494]: cluster 2024-09-13T14:29:38.047978+0000 mgr.x (mgr.24637) 65 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 14 MiB/s wr, 434 op/s 2024-09-13T14:29:39.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:38 smithi177 bash[24494]: cluster 2024-09-13T14:29:38.596907+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-09-13T14:29:39.463 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:29:39.786 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.Clone (5193 ms) 2024-09-13T14:29:39.786 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneParent 2024-09-13T14:29:39.798 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:39.839 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap 2024-09-13T14:29:40.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:39 smithi099 bash[22656]: audit 2024-09-13T14:29:39.464031+0000 mon.a (mon.0) 1114 : audit [DBG] from='client.? 172.21.15.99:0/429520979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:40.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:39 smithi099 bash[32082]: audit 2024-09-13T14:29:39.464031+0000 mon.a (mon.0) 1114 : audit [DBG] from='client.? 172.21.15.99:0/429520979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:40.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:39 smithi177 bash[24494]: audit 2024-09-13T14:29:39.464031+0000 mon.a (mon.0) 1114 : audit [DBG] from='client.? 172.21.15.99:0/429520979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:40.957 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap 2024-09-13T14:29:40.990 INFO:tasks.workunit.client.0.smithi099.stdout:clone image109 -> image112 2024-09-13T14:29:41.226 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:29:41.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:40 smithi099 bash[22656]: audit 2024-09-13T14:29:39.794142+0000 mgr.x (mgr.24637) 66 : audit [DBG] from='client.25478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:41.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:40 smithi099 bash[22656]: cluster 2024-09-13T14:29:39.932677+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:29:41.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:40 smithi099 bash[22656]: cluster 2024-09-13T14:29:40.049596+0000 mgr.x (mgr.24637) 67 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 777 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 5.0 MiB/s wr, 822 op/s 2024-09-13T14:29:41.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:40 smithi099 bash[22656]: audit 2024-09-13T14:29:40.116161+0000 mgr.x (mgr.24637) 68 : audit [DBG] from='client.15591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:41.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:40 smithi099 bash[32082]: audit 2024-09-13T14:29:39.794142+0000 mgr.x (mgr.24637) 66 : audit [DBG] from='client.25478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:41.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:40 smithi099 bash[32082]: cluster 2024-09-13T14:29:39.932677+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:29:41.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:40 smithi099 bash[32082]: cluster 2024-09-13T14:29:40.049596+0000 mgr.x (mgr.24637) 67 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 777 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 5.0 MiB/s wr, 822 op/s 2024-09-13T14:29:41.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:40 smithi099 bash[32082]: audit 2024-09-13T14:29:40.116161+0000 mgr.x (mgr.24637) 68 : audit [DBG] from='client.15591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:41.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:40 smithi177 bash[24494]: audit 2024-09-13T14:29:39.794142+0000 mgr.x (mgr.24637) 66 : audit [DBG] from='client.25478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:41.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:40 smithi177 bash[24494]: cluster 2024-09-13T14:29:39.932677+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-09-13T14:29:41.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:40 smithi177 bash[24494]: cluster 2024-09-13T14:29:40.049596+0000 mgr.x (mgr.24637) 67 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 777 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 5.0 MiB/s wr, 822 op/s 2024-09-13T14:29:41.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:40 smithi177 bash[24494]: audit 2024-09-13T14:29:40.116161+0000 mgr.x (mgr.24637) 68 : audit [DBG] from='client.15591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:41.616 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:29:41.875 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneParent (2089 ms) 2024-09-13T14:29:41.875 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneUpdateAfterPrepare 2024-09-13T14:29:41.907 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:41.907 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:41.908 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:41.908 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:29:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:41.908 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:29:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:41.909 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:29:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:41.909 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:29:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:41.909 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:41.909 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:41.910 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:41 smithi099 systemd[1]: Stopping Ceph alertmanager.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:29:41.939 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:41.957 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~10 2024-09-13T14:29:41.972 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap 2024-09-13T14:29:42.317 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:42 smithi099 bash[58469]: ts=2024-09-13T14:29:42.021Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:29:42.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:41 smithi099 bash[22656]: cluster 2024-09-13T14:29:40.945656+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:29:42.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:42 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:42.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:41 smithi099 bash[32082]: cluster 2024-09-13T14:29:40.945656+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:29:42.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:41 smithi177 bash[24494]: cluster 2024-09-13T14:29:40.945656+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-09-13T14:29:42.807 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:43.066 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:42 smithi099 bash[22656]: cluster 2024-09-13T14:29:42.050356+0000 mgr.x (mgr.24637) 69 : cluster [DBG] pgmap v59: 137 pgs: 137 active+clean; 777 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.9 MiB/s wr, 796 op/s 2024-09-13T14:29:43.066 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:42 smithi099 bash[22656]: audit 2024-09-13T14:29:42.485240+0000 mon.a (mon.0) 1117 : audit [DBG] from='client.? 172.21.15.99:0/657878426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:43.066 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:42 smithi099 bash[74898]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:29:43.066 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[75027]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:29:43.066 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:42 smithi099 bash[32082]: cluster 2024-09-13T14:29:42.050356+0000 mgr.x (mgr.24637) 69 : cluster [DBG] pgmap v59: 137 pgs: 137 active+clean; 777 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.9 MiB/s wr, 796 op/s 2024-09-13T14:29:43.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:42 smithi099 bash[32082]: audit 2024-09-13T14:29:42.485240+0000 mon.a (mon.0) 1117 : audit [DBG] from='client.? 172.21.15.99:0/657878426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:43.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:43.318 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:43.318 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:43.318 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:43.318 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:43.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:43.319 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:43.319 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:29:43.320 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: Stopped Ceph alertmanager.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:29:43.320 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:43.320 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:43 smithi099 systemd[1]: Started Ceph alertmanager.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:29:43.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:42 smithi177 bash[24494]: cluster 2024-09-13T14:29:42.050356+0000 mgr.x (mgr.24637) 69 : cluster [DBG] pgmap v59: 137 pgs: 137 active+clean; 777 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.9 MiB/s wr, 796 op/s 2024-09-13T14:29:43.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:42 smithi177 bash[24494]: audit 2024-09-13T14:29:42.485240+0000 mon.a (mon.0) 1117 : audit [DBG] from='client.? 172.21.15.99:0/657878426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:43.853 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:43 smithi177 bash[51021]: ts=2024-09-13T14:29:43.510Z caller=notifier.go:532 level=error component=notifier alertmanager=http://smithi099.front.sepia.ceph.com:9093/api/v2/alerts count=1 msg="Error sending alert" err="Post \"http://smithi099.front.sepia.ceph.com:9093/api/v2/alerts\": dial tcp 172.21.15.99:9093: connect: connection refused" 2024-09-13T14:29:43.983 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap 2024-09-13T14:29:44.007 INFO:tasks.workunit.client.0.smithi099.stdout:clone image113 -> image116 2024-09-13T14:29:44.228 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:29:44.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[22656]: audit 2024-09-13T14:29:42.802332+0000 mgr.x (mgr.24637) 70 : audit [DBG] from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:44.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[22656]: cluster 2024-09-13T14:29:42.957633+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:29:44.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[22656]: audit 2024-09-13T14:29:43.115914+0000 mgr.x (mgr.24637) 71 : audit [DBG] from='client.25502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:44.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[22656]: audit 2024-09-13T14:29:43.355214+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:44.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[22656]: audit 2024-09-13T14:29:43.366273+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:44.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[22656]: cephadm 2024-09-13T14:29:43.378334+0000 mgr.x (mgr.24637) 72 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:29:44.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[32082]: audit 2024-09-13T14:29:42.802332+0000 mgr.x (mgr.24637) 70 : audit [DBG] from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:44.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[32082]: cluster 2024-09-13T14:29:42.957633+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:29:44.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[32082]: audit 2024-09-13T14:29:43.115914+0000 mgr.x (mgr.24637) 71 : audit [DBG] from='client.25502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:44.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[32082]: audit 2024-09-13T14:29:43.355214+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:44.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[32082]: audit 2024-09-13T14:29:43.366273+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:44.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:43 smithi099 bash[32082]: cephadm 2024-09-13T14:29:43.378334+0000 mgr.x (mgr.24637) 72 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:29:44.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:43 smithi177 bash[24494]: audit 2024-09-13T14:29:42.802332+0000 mgr.x (mgr.24637) 70 : audit [DBG] from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:43 smithi177 bash[24494]: cluster 2024-09-13T14:29:42.957633+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-09-13T14:29:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:43 smithi177 bash[24494]: audit 2024-09-13T14:29:43.115914+0000 mgr.x (mgr.24637) 71 : audit [DBG] from='client.25502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:43 smithi177 bash[24494]: audit 2024-09-13T14:29:43.355214+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:43 smithi177 bash[24494]: audit 2024-09-13T14:29:43.366273+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:29:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:43 smithi177 bash[24494]: cephadm 2024-09-13T14:29:43.378334+0000 mgr.x (mgr.24637) 72 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:29:44.545 INFO:tasks.workunit.client.0.smithi099.stdout:write: Y 0~1 2024-09-13T14:29:44.593 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:29:44.729 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:29:44.818 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[75088]: ts=2024-09-13T14:29:44.668Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-13T14:29:44.818 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[75088]: ts=2024-09-13T14:29:44.668Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-13T14:29:44.818 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[75088]: ts=2024-09-13T14:29:44.670Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.99 port=9094 2024-09-13T14:29:44.818 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[75088]: ts=2024-09-13T14:29:44.671Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:29:44.818 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[75088]: ts=2024-09-13T14:29:44.703Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:29:44.818 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[75088]: ts=2024-09-13T14:29:44.703Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:29:44.818 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[75088]: ts=2024-09-13T14:29:44.705Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:29:44.818 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[75088]: ts=2024-09-13T14:29:44.705Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:29:44.963 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneUpdateAfterPrepare (3088 ms) 2024-09-13T14:29:44.963 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.TriggerAssertSnapcSeq 2024-09-13T14:29:45.047 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:45.077 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 41943040~10 2024-09-13T14:29:45.097 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:29:45.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[22656]: cephadm 2024-09-13T14:29:43.674813+0000 mgr.x (mgr.24637) 73 : cephadm [INF] Deploying daemon prometheus.a on smithi177 2024-09-13T14:29:45.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[22656]: cluster 2024-09-13T14:29:43.970078+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:29:45.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[22656]: cluster 2024-09-13T14:29:44.052486+0000 mgr.x (mgr.24637) 74 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 777 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.9 MiB/s wr, 799 op/s 2024-09-13T14:29:45.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[32082]: cephadm 2024-09-13T14:29:43.674813+0000 mgr.x (mgr.24637) 73 : cephadm [INF] Deploying daemon prometheus.a on smithi177 2024-09-13T14:29:45.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[32082]: cluster 2024-09-13T14:29:43.970078+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:29:45.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:44 smithi099 bash[32082]: cluster 2024-09-13T14:29:44.052486+0000 mgr.x (mgr.24637) 74 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 777 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.9 MiB/s wr, 799 op/s 2024-09-13T14:29:45.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:44 smithi177 bash[24494]: cephadm 2024-09-13T14:29:43.674813+0000 mgr.x (mgr.24637) 73 : cephadm [INF] Deploying daemon prometheus.a on smithi177 2024-09-13T14:29:45.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:44 smithi177 bash[24494]: cluster 2024-09-13T14:29:43.970078+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-09-13T14:29:45.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:44 smithi177 bash[24494]: cluster 2024-09-13T14:29:44.052486+0000 mgr.x (mgr.24637) 74 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 777 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.9 MiB/s wr, 799 op/s 2024-09-13T14:29:45.804 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:45.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:45 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:45] "GET /metrics HTTP/1.1" 200 38907 "" "Prometheus/2.43.0" 2024-09-13T14:29:46.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:45 smithi099 bash[22656]: audit 2024-09-13T14:29:45.488725+0000 mon.a (mon.0) 1122 : audit [DBG] from='client.? 172.21.15.99:0/1823740988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:46.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:45 smithi099 bash[32082]: audit 2024-09-13T14:29:45.488725+0000 mon.a (mon.0) 1122 : audit [DBG] from='client.? 172.21.15.99:0/1823740988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:46.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:45 smithi177 bash[24494]: audit 2024-09-13T14:29:45.488725+0000 mon.a (mon.0) 1122 : audit [DBG] from='client.? 172.21.15.99:0/1823740988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:46.999 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:46 smithi099 bash[75088]: ts=2024-09-13T14:29:46.672Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000205375s 2024-09-13T14:29:47.032 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 41943041~10 2024-09-13T14:29:47.043 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:29:47.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:47 smithi099 bash[22656]: audit 2024-09-13T14:29:45.799922+0000 mgr.x (mgr.24637) 75 : audit [DBG] from='client.25514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:47.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:47 smithi099 bash[22656]: cluster 2024-09-13T14:29:45.981692+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:29:47.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:47 smithi099 bash[22656]: cluster 2024-09-13T14:29:46.053292+0000 mgr.x (mgr.24637) 76 : cluster [DBG] pgmap v64: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 825 KiB/s wr, 561 op/s 2024-09-13T14:29:47.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:47 smithi099 bash[22656]: audit 2024-09-13T14:29:46.124214+0000 mgr.x (mgr.24637) 77 : audit [DBG] from='client.25520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:47.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:47 smithi099 bash[32082]: audit 2024-09-13T14:29:45.799922+0000 mgr.x (mgr.24637) 75 : audit [DBG] from='client.25514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:47.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:47 smithi099 bash[32082]: cluster 2024-09-13T14:29:45.981692+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:29:47.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:47 smithi099 bash[32082]: cluster 2024-09-13T14:29:46.053292+0000 mgr.x (mgr.24637) 76 : cluster [DBG] pgmap v64: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 825 KiB/s wr, 561 op/s 2024-09-13T14:29:47.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:47 smithi099 bash[32082]: audit 2024-09-13T14:29:46.124214+0000 mgr.x (mgr.24637) 77 : audit [DBG] from='client.25520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:47.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:47 smithi177 bash[24494]: audit 2024-09-13T14:29:45.799922+0000 mgr.x (mgr.24637) 75 : audit [DBG] from='client.25514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:47.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:47 smithi177 bash[24494]: cluster 2024-09-13T14:29:45.981692+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-09-13T14:29:47.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:47 smithi177 bash[24494]: cluster 2024-09-13T14:29:46.053292+0000 mgr.x (mgr.24637) 76 : cluster [DBG] pgmap v64: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.2 MiB/s rd, 825 KiB/s wr, 561 op/s 2024-09-13T14:29:47.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:47 smithi177 bash[24494]: audit 2024-09-13T14:29:46.124214+0000 mgr.x (mgr.24637) 77 : audit [DBG] from='client.25520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:48.281 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 41943042~10 2024-09-13T14:29:48.298 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:29:48.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:48 smithi099 bash[32082]: cluster 2024-09-13T14:29:47.014389+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:29:48.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:48 smithi099 bash[22656]: cluster 2024-09-13T14:29:47.014389+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:29:48.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:48 smithi177 bash[24494]: cluster 2024-09-13T14:29:47.014389+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-09-13T14:29:48.842 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:49.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:49 smithi099 bash[22656]: cluster 2024-09-13T14:29:48.022871+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-09-13T14:29:49.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:49 smithi099 bash[22656]: cluster 2024-09-13T14:29:48.054153+0000 mgr.x (mgr.24637) 78 : cluster [DBG] pgmap v67: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.0 MiB/s wr, 702 op/s 2024-09-13T14:29:49.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:49 smithi099 bash[22656]: audit 2024-09-13T14:29:48.517781+0000 mon.c (mon.2) 167 : audit [DBG] from='client.? 172.21.15.99:0/954665713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:49.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:49 smithi099 bash[32082]: cluster 2024-09-13T14:29:48.022871+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-09-13T14:29:49.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:49 smithi099 bash[32082]: cluster 2024-09-13T14:29:48.054153+0000 mgr.x (mgr.24637) 78 : cluster [DBG] pgmap v67: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.0 MiB/s wr, 702 op/s 2024-09-13T14:29:49.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:49 smithi099 bash[32082]: audit 2024-09-13T14:29:48.517781+0000 mon.c (mon.2) 167 : audit [DBG] from='client.? 172.21.15.99:0/954665713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:49.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:49 smithi177 bash[24494]: cluster 2024-09-13T14:29:48.022871+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-09-13T14:29:49.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:49 smithi177 bash[24494]: cluster 2024-09-13T14:29:48.054153+0000 mgr.x (mgr.24637) 78 : cluster [DBG] pgmap v67: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.0 MiB/s wr, 702 op/s 2024-09-13T14:29:49.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:49 smithi177 bash[24494]: audit 2024-09-13T14:29:48.517781+0000 mon.c (mon.2) 167 : audit [DBG] from='client.? 172.21.15.99:0/954665713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:50.061 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 41943040 2024-09-13T14:29:50.134 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:29:50.288 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:29:50.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:50 smithi099 bash[22656]: audit 2024-09-13T14:29:48.838772+0000 mgr.x (mgr.24637) 79 : audit [DBG] from='client.25532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:50.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:50 smithi099 bash[22656]: cluster 2024-09-13T14:29:49.031520+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-09-13T14:29:50.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:50 smithi099 bash[22656]: audit 2024-09-13T14:29:49.154780+0000 mgr.x (mgr.24637) 80 : audit [DBG] from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:50.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:50 smithi099 bash[32082]: audit 2024-09-13T14:29:48.838772+0000 mgr.x (mgr.24637) 79 : audit [DBG] from='client.25532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:50.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:50 smithi099 bash[32082]: cluster 2024-09-13T14:29:49.031520+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-09-13T14:29:50.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:50 smithi099 bash[32082]: audit 2024-09-13T14:29:49.154780+0000 mgr.x (mgr.24637) 80 : audit [DBG] from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:50.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:50 smithi177 bash[24494]: audit 2024-09-13T14:29:48.838772+0000 mgr.x (mgr.24637) 79 : audit [DBG] from='client.25532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:50.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:50 smithi177 bash[24494]: cluster 2024-09-13T14:29:49.031520+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-09-13T14:29:50.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:50 smithi177 bash[24494]: audit 2024-09-13T14:29:49.154780+0000 mgr.x (mgr.24637) 80 : audit [DBG] from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:51.273 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.TriggerAssertSnapcSeq (6310 ms) 2024-09-13T14:29:51.274 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.SnapTrimBeforePrepare 2024-09-13T14:29:51.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:51 smithi099 bash[32082]: cluster 2024-09-13T14:29:50.040236+0000 mon.a (mon.0) 1127 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-09-13T14:29:51.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:51 smithi099 bash[32082]: cluster 2024-09-13T14:29:50.055588+0000 mgr.x (mgr.24637) 81 : cluster [DBG] pgmap v70: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 114 KiB/s rd, 13 KiB/s wr, 155 op/s 2024-09-13T14:29:51.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:51 smithi099 bash[32082]: audit 2024-09-13T14:29:50.149808+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:51.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:51 smithi099 bash[22656]: cluster 2024-09-13T14:29:50.040236+0000 mon.a (mon.0) 1127 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-09-13T14:29:51.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:51 smithi099 bash[22656]: cluster 2024-09-13T14:29:50.055588+0000 mgr.x (mgr.24637) 81 : cluster [DBG] pgmap v70: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 114 KiB/s rd, 13 KiB/s wr, 155 op/s 2024-09-13T14:29:51.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:51 smithi099 bash[22656]: audit 2024-09-13T14:29:50.149808+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:51.321 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:51.339 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 41943040~10 2024-09-13T14:29:51.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:51 smithi177 bash[24494]: cluster 2024-09-13T14:29:50.040236+0000 mon.a (mon.0) 1127 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-09-13T14:29:51.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:51 smithi177 bash[24494]: cluster 2024-09-13T14:29:50.055588+0000 mgr.x (mgr.24637) 81 : cluster [DBG] pgmap v70: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 114 KiB/s rd, 13 KiB/s wr, 155 op/s 2024-09-13T14:29:51.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:51 smithi177 bash[24494]: audit 2024-09-13T14:29:50.149808+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:29:51.353 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:29:51.861 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:52.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:29:52 smithi099 bash[22909]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:29:52.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:52 smithi099 bash[22656]: cluster 2024-09-13T14:29:51.053737+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-09-13T14:29:52.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:52 smithi099 bash[22656]: audit 2024-09-13T14:29:51.539231+0000 mon.a (mon.0) 1129 : audit [DBG] from='client.? 172.21.15.99:0/2181692419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:52.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:52 smithi099 bash[32082]: cluster 2024-09-13T14:29:51.053737+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-09-13T14:29:52.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:52 smithi099 bash[32082]: audit 2024-09-13T14:29:51.539231+0000 mon.a (mon.0) 1129 : audit [DBG] from='client.? 172.21.15.99:0/2181692419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:52.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:52 smithi177 bash[24494]: cluster 2024-09-13T14:29:51.053737+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-09-13T14:29:52.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:52 smithi177 bash[24494]: audit 2024-09-13T14:29:51.539231+0000 mon.a (mon.0) 1129 : audit [DBG] from='client.? 172.21.15.99:0/2181692419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:53.119 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 41943040 2024-09-13T14:29:53.343 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:29:53.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:53 smithi177 bash[24494]: audit 2024-09-13T14:29:51.856287+0000 mgr.x (mgr.24637) 82 : audit [DBG] from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:53.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:53 smithi177 bash[24494]: cluster 2024-09-13T14:29:52.056504+0000 mgr.x (mgr.24637) 83 : cluster [DBG] pgmap v73: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 114 KiB/s rd, 13 KiB/s wr, 155 op/s 2024-09-13T14:29:53.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:53 smithi177 bash[24494]: cluster 2024-09-13T14:29:52.060277+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-09-13T14:29:53.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:53 smithi177 bash[24494]: audit 2024-09-13T14:29:52.182036+0000 mgr.x (mgr.24637) 84 : audit [DBG] from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:53.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:53 smithi099 bash[22656]: audit 2024-09-13T14:29:51.856287+0000 mgr.x (mgr.24637) 82 : audit [DBG] from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:53.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:53 smithi099 bash[22656]: cluster 2024-09-13T14:29:52.056504+0000 mgr.x (mgr.24637) 83 : cluster [DBG] pgmap v73: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 114 KiB/s rd, 13 KiB/s wr, 155 op/s 2024-09-13T14:29:53.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:53 smithi099 bash[22656]: cluster 2024-09-13T14:29:52.060277+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-09-13T14:29:53.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:53 smithi099 bash[22656]: audit 2024-09-13T14:29:52.182036+0000 mgr.x (mgr.24637) 84 : audit [DBG] from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:53.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:53 smithi099 bash[32082]: audit 2024-09-13T14:29:51.856287+0000 mgr.x (mgr.24637) 82 : audit [DBG] from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:53.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:53 smithi099 bash[32082]: cluster 2024-09-13T14:29:52.056504+0000 mgr.x (mgr.24637) 83 : cluster [DBG] pgmap v73: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 114 KiB/s rd, 13 KiB/s wr, 155 op/s 2024-09-13T14:29:53.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:53 smithi099 bash[32082]: cluster 2024-09-13T14:29:52.060277+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-09-13T14:29:53.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:53 smithi099 bash[32082]: audit 2024-09-13T14:29:52.182036+0000 mgr.x (mgr.24637) 84 : audit [DBG] from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:54.329 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:54.347 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:29:54.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:54 smithi177 bash[24494]: cluster 2024-09-13T14:29:53.081150+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-09-13T14:29:54.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:54 smithi099 bash[32082]: cluster 2024-09-13T14:29:53.081150+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-09-13T14:29:54.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:54 smithi099 bash[22656]: cluster 2024-09-13T14:29:53.081150+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-09-13T14:29:54.898 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:55.068 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:29:54 smithi099 bash[75088]: ts=2024-09-13T14:29:54.674Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002440804s 2024-09-13T14:29:55.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:55 smithi177 bash[24494]: cluster 2024-09-13T14:29:54.057459+0000 mgr.x (mgr.24637) 85 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 110 KiB/s rd, 12 KiB/s wr, 150 op/s 2024-09-13T14:29:55.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:55 smithi177 bash[24494]: cluster 2024-09-13T14:29:54.104525+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-09-13T14:29:55.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:55 smithi177 bash[24494]: audit 2024-09-13T14:29:54.580681+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.99:0/3513223257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:55.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:55 smithi099 bash[22656]: cluster 2024-09-13T14:29:54.057459+0000 mgr.x (mgr.24637) 85 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 110 KiB/s rd, 12 KiB/s wr, 150 op/s 2024-09-13T14:29:55.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:55 smithi099 bash[22656]: cluster 2024-09-13T14:29:54.104525+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-09-13T14:29:55.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:55 smithi099 bash[22656]: audit 2024-09-13T14:29:54.580681+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.99:0/3513223257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:55.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:55 smithi099 bash[32082]: cluster 2024-09-13T14:29:54.057459+0000 mgr.x (mgr.24637) 85 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 110 KiB/s rd, 12 KiB/s wr, 150 op/s 2024-09-13T14:29:55.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:55 smithi099 bash[32082]: cluster 2024-09-13T14:29:54.104525+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-09-13T14:29:55.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:55 smithi099 bash[32082]: audit 2024-09-13T14:29:54.580681+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.99:0/3513223257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:55.654 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:55 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:29:55] "GET /metrics HTTP/1.1" 200 38903 "" "Prometheus/2.43.0" 2024-09-13T14:29:56.150 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 41943040~10 2024-09-13T14:29:56.163 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap4 2024-09-13T14:29:56.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:56 smithi099 bash[22656]: audit 2024-09-13T14:29:54.895194+0000 mgr.x (mgr.24637) 86 : audit [DBG] from='client.15675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:56.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:56 smithi099 bash[22656]: cluster 2024-09-13T14:29:55.104498+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-09-13T14:29:56.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:56 smithi099 bash[22656]: audit 2024-09-13T14:29:55.222537+0000 mgr.x (mgr.24637) 87 : audit [DBG] from='client.25574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:56.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:56 smithi099 bash[32082]: audit 2024-09-13T14:29:54.895194+0000 mgr.x (mgr.24637) 86 : audit [DBG] from='client.15675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:56.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:56 smithi099 bash[32082]: cluster 2024-09-13T14:29:55.104498+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-09-13T14:29:56.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:56 smithi099 bash[32082]: audit 2024-09-13T14:29:55.222537+0000 mgr.x (mgr.24637) 87 : audit [DBG] from='client.25574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:56.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:56 smithi177 bash[24494]: audit 2024-09-13T14:29:54.895194+0000 mgr.x (mgr.24637) 86 : audit [DBG] from='client.15675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:56.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:56 smithi177 bash[24494]: cluster 2024-09-13T14:29:55.104498+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-09-13T14:29:56.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:56 smithi177 bash[24494]: audit 2024-09-13T14:29:55.222537+0000 mgr.x (mgr.24637) 87 : audit [DBG] from='client.25574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:57.378 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:57 smithi177 bash[24494]: cluster 2024-09-13T14:29:56.058300+0000 mgr.x (mgr.24637) 88 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 371 op/s 2024-09-13T14:29:57.378 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:57 smithi177 bash[24494]: cluster 2024-09-13T14:29:56.126488+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-09-13T14:29:57.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:57 smithi099 bash[22656]: cluster 2024-09-13T14:29:56.058300+0000 mgr.x (mgr.24637) 88 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 371 op/s 2024-09-13T14:29:57.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:57 smithi099 bash[22656]: cluster 2024-09-13T14:29:56.126488+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-09-13T14:29:57.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:57 smithi099 bash[32082]: cluster 2024-09-13T14:29:56.058300+0000 mgr.x (mgr.24637) 88 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 371 op/s 2024-09-13T14:29:57.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:57 smithi099 bash[32082]: cluster 2024-09-13T14:29:56.126488+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-09-13T14:29:57.911 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:29:58.153 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 41943040 2024-09-13T14:29:58.191 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:29:58.298 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:29:58.472 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:58 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:58.473 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:29:58 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:58.473 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:58.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[24494]: cluster 2024-09-13T14:29:57.133493+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:29:58.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[24494]: audit 2024-09-13T14:29:57.596466+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.99:0/557783445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:58.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:58 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:58.474 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:29:58 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:58.474 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:29:58 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:58.475 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:29:58 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:58.475 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:29:58 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:58.475 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:29:58 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:58.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:58 smithi099 bash[22656]: cluster 2024-09-13T14:29:57.133493+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:29:58.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:58 smithi099 bash[22656]: audit 2024-09-13T14:29:57.596466+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.99:0/557783445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:58.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:58 smithi099 bash[32082]: cluster 2024-09-13T14:29:57.133493+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-09-13T14:29:58.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:58 smithi099 bash[32082]: audit 2024-09-13T14:29:57.596466+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.99:0/557783445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:29:58.845 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 systemd[1]: Stopping Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:29:58.845 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.622Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:29:58.845 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.622Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:29:58.845 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.622Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:29:58.845 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.622Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:29:58.846 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.622Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:29:58.846 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.622Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:29:58.846 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.622Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:29:58.846 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.622Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:29:58.846 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.623Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:29:58.846 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.623Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:29:58.846 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.623Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:29:58.846 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:58 smithi177 bash[51021]: ts=2024-09-13T14:29:58.623Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:29:58.866 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.SnapTrimBeforePrepare (7593 ms) 2024-09-13T14:29:58.867 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.AbortInUseImage 2024-09-13T14:29:58.950 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:58.979 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:29:59.403 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.AbortInUseImage (536 ms) 2024-09-13T14:29:59.403 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.AbortWithoutSnapshots 2024-09-13T14:29:59.477 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:29:59.501 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:29:59.510 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:59 smithi177 bash[59446]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:29:59.510 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:59 smithi177 bash[24494]: audit 2024-09-13T14:29:57.908638+0000 mgr.x (mgr.24637) 89 : audit [DBG] from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.510 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:59 smithi177 bash[24494]: cluster 2024-09-13T14:29:58.059055+0000 mgr.x (mgr.24637) 90 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-09-13T14:29:59.510 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:59 smithi177 bash[24494]: cluster 2024-09-13T14:29:58.145558+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:29:59.511 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:59 smithi177 bash[24494]: audit 2024-09-13T14:29:58.216951+0000 mgr.x (mgr.24637) 91 : audit [DBG] from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.511 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:59 smithi177 bash[24494]: cluster 2024-09-13T14:29:58.601824+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:29:59.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:59 smithi099 bash[22656]: audit 2024-09-13T14:29:57.908638+0000 mgr.x (mgr.24637) 89 : audit [DBG] from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:59 smithi099 bash[22656]: cluster 2024-09-13T14:29:58.059055+0000 mgr.x (mgr.24637) 90 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-09-13T14:29:59.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:59 smithi099 bash[22656]: cluster 2024-09-13T14:29:58.145558+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:29:59.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:59 smithi099 bash[22656]: audit 2024-09-13T14:29:58.216951+0000 mgr.x (mgr.24637) 91 : audit [DBG] from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:29:59 smithi099 bash[22656]: cluster 2024-09-13T14:29:58.601824+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:29:59.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:59 smithi099 bash[32082]: audit 2024-09-13T14:29:57.908638+0000 mgr.x (mgr.24637) 89 : audit [DBG] from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:59 smithi099 bash[32082]: cluster 2024-09-13T14:29:58.059055+0000 mgr.x (mgr.24637) 90 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-09-13T14:29:59.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:59 smithi099 bash[32082]: cluster 2024-09-13T14:29:58.145558+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-09-13T14:29:59.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:59 smithi099 bash[32082]: audit 2024-09-13T14:29:58.216951+0000 mgr.x (mgr.24637) 91 : audit [DBG] from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:59.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:29:59 smithi099 bash[32082]: cluster 2024-09-13T14:29:58.601824+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-09-13T14:29:59.595 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:29:59.669 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:29:59.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:59.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:59.853 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:59.854 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:59.854 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:59.854 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:59.855 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:59.855 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:59 smithi177 bash[59504]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:29:59.855 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:29:59.855 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: Stopped Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:29:59.855 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:29:59.855 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: Started Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:29:59.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:29:59 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:00.038 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:30:00.106 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:30:00.196 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:30:00.473 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.AbortWithoutSnapshots (1069 ms) 2024-09-13T14:30:00.473 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.AbortWithSnapshots 2024-09-13T14:30:00.542 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:00.566 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:30:00.669 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:00.967 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:00.969 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:30:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:00.970 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:30:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:00.970 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:00.970 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:00.971 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:00.971 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:00.971 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:30:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:00.971 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:30:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:00.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 bash[24494]: audit 2024-09-13T14:29:59.811228+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:00.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 bash[24494]: audit 2024-09-13T14:29:59.822914+0000 mon.a (mon.0) 1139 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:00.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 bash[24494]: cephadm 2024-09-13T14:29:59.827753+0000 mgr.x (mgr.24637) 92 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:30:00.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 bash[24494]: cephadm 2024-09-13T14:29:59.828161+0000 mgr.x (mgr.24637) 93 : cephadm [INF] Deploying daemon node-exporter.b on smithi177 2024-09-13T14:30:00.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 bash[24494]: cluster 2024-09-13T14:30:00.000130+0000 mon.a (mon.0) 1140 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-13T14:30:00.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 bash[24494]: cluster 2024-09-13T14:30:00.000177+0000 mon.a (mon.0) 1141 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T14:30:00.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 bash[24494]: cluster 2024-09-13T14:30:00.000202+0000 mon.a (mon.0) 1142 : cluster [WRN] application not enabled on pool 'test-librbd-smithi099-69110-8' 2024-09-13T14:30:00.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 bash[24494]: cluster 2024-09-13T14:30:00.000229+0000 mon.a (mon.0) 1143 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T14:30:00.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 bash[24494]: cluster 2024-09-13T14:30:00.060706+0000 mgr.x (mgr.24637) 94 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 MiB/s rd, 8.0 MiB/s wr, 516 op/s 2024-09-13T14:30:00.973 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 bash[24494]: audit 2024-09-13T14:30:00.634840+0000 mon.a (mon.0) 1144 : audit [DBG] from='client.? 172.21.15.99:0/662945309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:00.973 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:00 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:01.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[22656]: audit 2024-09-13T14:29:59.811228+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:01.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[22656]: audit 2024-09-13T14:29:59.822914+0000 mon.a (mon.0) 1139 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:01.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[22656]: cephadm 2024-09-13T14:29:59.827753+0000 mgr.x (mgr.24637) 92 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:30:01.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[22656]: cephadm 2024-09-13T14:29:59.828161+0000 mgr.x (mgr.24637) 93 : cephadm [INF] Deploying daemon node-exporter.b on smithi177 2024-09-13T14:30:01.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[22656]: cluster 2024-09-13T14:30:00.000130+0000 mon.a (mon.0) 1140 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-13T14:30:01.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[22656]: cluster 2024-09-13T14:30:00.000177+0000 mon.a (mon.0) 1141 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T14:30:01.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[22656]: cluster 2024-09-13T14:30:00.000202+0000 mon.a (mon.0) 1142 : cluster [WRN] application not enabled on pool 'test-librbd-smithi099-69110-8' 2024-09-13T14:30:01.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[22656]: cluster 2024-09-13T14:30:00.000229+0000 mon.a (mon.0) 1143 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T14:30:01.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[22656]: cluster 2024-09-13T14:30:00.060706+0000 mgr.x (mgr.24637) 94 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 MiB/s rd, 8.0 MiB/s wr, 516 op/s 2024-09-13T14:30:01.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[22656]: audit 2024-09-13T14:30:00.634840+0000 mon.a (mon.0) 1144 : audit [DBG] from='client.? 172.21.15.99:0/662945309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:01.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[32082]: audit 2024-09-13T14:29:59.811228+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:01.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[32082]: audit 2024-09-13T14:29:59.822914+0000 mon.a (mon.0) 1139 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:01.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[32082]: cephadm 2024-09-13T14:29:59.827753+0000 mgr.x (mgr.24637) 92 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T14:30:01.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[32082]: cephadm 2024-09-13T14:29:59.828161+0000 mgr.x (mgr.24637) 93 : cephadm [INF] Deploying daemon node-exporter.b on smithi177 2024-09-13T14:30:01.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[32082]: cluster 2024-09-13T14:30:00.000130+0000 mon.a (mon.0) 1140 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-13T14:30:01.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[32082]: cluster 2024-09-13T14:30:00.000177+0000 mon.a (mon.0) 1141 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T14:30:01.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[32082]: cluster 2024-09-13T14:30:00.000202+0000 mon.a (mon.0) 1142 : cluster [WRN] application not enabled on pool 'test-librbd-smithi099-69110-8' 2024-09-13T14:30:01.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[32082]: cluster 2024-09-13T14:30:00.000229+0000 mon.a (mon.0) 1143 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T14:30:01.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[32082]: cluster 2024-09-13T14:30:00.060706+0000 mgr.x (mgr.24637) 94 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 MiB/s rd, 8.0 MiB/s wr, 516 op/s 2024-09-13T14:30:01.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:00 smithi099 bash[32082]: audit 2024-09-13T14:30:00.634840+0000 mon.a (mon.0) 1144 : audit [DBG] from='client.? 172.21.15.99:0/662945309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:01.228 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:00 smithi177 systemd[1]: Stopping Ceph node-exporter.b for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:30:01.818 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.518Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-13T14:30:01.818 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.518Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.518Z caller=main.go:623 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi177 (none))" 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.518Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.518Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.520Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.520Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.522Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.522Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.524Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.524Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.106µs 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.524Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.525Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-13T14:30:01.819 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.545Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-13T14:30:01.820 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.546Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-13T14:30:01.820 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.546Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=34.194µs wal_replay_duration=21.469048ms wbl_replay_duration=133ns total_replay_duration=21.523977ms 2024-09-13T14:30:01.820 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.549Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:30:01.820 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.549Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-13T14:30:01.820 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.549Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:30:01.820 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.569Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.280498ms db_storage=1.131µs remote_storage=1.209µs web_handler=403ns query_engine=1.079µs scrape=276.168µs scrape_sd=112.022µs notify=17.731µs notify_sd=17.125µs rules=18.330576ms tracing=7.522µs 2024-09-13T14:30:01.820 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.569Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-13T14:30:01.820 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[59561]: ts=2024-09-13T14:30:01.569Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:30:01.886 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 8388609~8388609 2024-09-13T14:30:01.930 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 0~1000 2024-09-13T14:30:01.930 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1010~1000 2024-09-13T14:30:01.930 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:30:02.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:01 smithi099 bash[22656]: cluster 2024-09-13T14:30:00.823239+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:30:02.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:01 smithi099 bash[22656]: audit 2024-09-13T14:30:00.963670+0000 mgr.x (mgr.24637) 95 : audit [DBG] from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:02.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:01 smithi099 bash[22656]: audit 2024-09-13T14:30:01.285651+0000 mgr.x (mgr.24637) 96 : audit [DBG] from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:02.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:01 smithi099 bash[32082]: cluster 2024-09-13T14:30:00.823239+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:30:02.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:01 smithi099 bash[32082]: audit 2024-09-13T14:30:00.963670+0000 mgr.x (mgr.24637) 95 : audit [DBG] from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:02.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:01 smithi099 bash[32082]: audit 2024-09-13T14:30:01.285651+0000 mgr.x (mgr.24637) 96 : audit [DBG] from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:02.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[24494]: cluster 2024-09-13T14:30:00.823239+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-09-13T14:30:02.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[24494]: audit 2024-09-13T14:30:00.963670+0000 mgr.x (mgr.24637) 95 : audit [DBG] from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:02.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:01 smithi177 bash[24494]: audit 2024-09-13T14:30:01.285651+0000 mgr.x (mgr.24637) 96 : audit [DBG] from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:02.353 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:02 smithi177 bash[59629]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T14:30:02.353 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:30:02.353 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:02 smithi177 bash[59747]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T14:30:02.353 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T14:30:02.353 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: Stopped Ceph node-exporter.b for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:30:02.853 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:02.853 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:02.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:02.854 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:02.854 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:02.855 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:02.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:02.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: Started Ceph node-exporter.b for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:30:02.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:02 smithi177 bash[59802]: Unable to find image 'quay.io/prometheus/node-exporter:v1.7.0' locally 2024-09-13T14:30:02.855 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:02.855 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:30:02 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:03.188 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:02 smithi177 bash[24494]: cluster 2024-09-13T14:30:01.868290+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:30:03.188 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:02 smithi177 bash[24494]: cluster 2024-09-13T14:30:02.061562+0000 mgr.x (mgr.24637) 97 : cluster [DBG] pgmap v87: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 MiB/s rd, 8.0 MiB/s wr, 516 op/s 2024-09-13T14:30:03.188 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:02 smithi177 bash[24494]: audit 2024-09-13T14:30:02.623529+0000 mon.a (mon.0) 1147 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:03.188 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:02 smithi177 bash[24494]: audit 2024-09-13T14:30:02.633586+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:03.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:02 smithi099 bash[32082]: cluster 2024-09-13T14:30:01.868290+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:30:03.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:02 smithi099 bash[32082]: cluster 2024-09-13T14:30:02.061562+0000 mgr.x (mgr.24637) 97 : cluster [DBG] pgmap v87: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 MiB/s rd, 8.0 MiB/s wr, 516 op/s 2024-09-13T14:30:03.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:02 smithi099 bash[32082]: audit 2024-09-13T14:30:02.623529+0000 mon.a (mon.0) 1147 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:03.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:02 smithi099 bash[32082]: audit 2024-09-13T14:30:02.633586+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:03.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:02 smithi099 bash[22656]: cluster 2024-09-13T14:30:01.868290+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:30:03.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:02 smithi099 bash[22656]: cluster 2024-09-13T14:30:02.061562+0000 mgr.x (mgr.24637) 97 : cluster [DBG] pgmap v87: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 MiB/s rd, 8.0 MiB/s wr, 516 op/s 2024-09-13T14:30:03.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:02 smithi099 bash[22656]: audit 2024-09-13T14:30:02.623529+0000 mon.a (mon.0) 1147 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:03.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:02 smithi099 bash[22656]: audit 2024-09-13T14:30:02.633586+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:03.543 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[59802]: v1.7.0: Pulling from prometheus/node-exporter 2024-09-13T14:30:03.544 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[59802]: 2abcce694348: Pulling fs layer 2024-09-13T14:30:03.544 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[59802]: 455fd88e5221: Pulling fs layer 2024-09-13T14:30:03.544 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[59802]: 324153f2810a: Pulling fs layer 2024-09-13T14:30:03.544 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[59802]: 455fd88e5221: Verifying Checksum 2024-09-13T14:30:03.618 INFO:tasks.workunit.client.0.smithi099.stdout:write: X 8388609~1000 2024-09-13T14:30:03.634 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16777228~1000 2024-09-13T14:30:03.634 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 167772160 2024-09-13T14:30:03.650 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 83886079~8388609 2024-09-13T14:30:03.685 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:30:03.867 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[59802]: 455fd88e5221: Download complete 2024-09-13T14:30:03.867 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[59802]: 2abcce694348: Verifying Checksum 2024-09-13T14:30:03.867 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[59802]: 2abcce694348: Download complete 2024-09-13T14:30:03.867 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[59802]: 324153f2810a: Verifying Checksum 2024-09-13T14:30:03.867 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[59802]: 324153f2810a: Download complete 2024-09-13T14:30:04.001 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:03 smithi099 bash[22656]: cephadm 2024-09-13T14:30:02.638585+0000 mgr.x (mgr.24637) 98 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:30:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:03 smithi099 bash[22656]: cephadm 2024-09-13T14:30:02.690172+0000 mgr.x (mgr.24637) 99 : cephadm [INF] Reconfiguring daemon grafana.a on smithi177 2024-09-13T14:30:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:03 smithi099 bash[22656]: cluster 2024-09-13T14:30:02.875860+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:30:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:03 smithi099 bash[22656]: cluster 2024-09-13T14:30:03.602411+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:30:04.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:03 smithi099 bash[22656]: audit 2024-09-13T14:30:03.675292+0000 mon.a (mon.0) 1151 : audit [DBG] from='client.? 172.21.15.99:0/2837593138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:04.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:03 smithi099 bash[32082]: cephadm 2024-09-13T14:30:02.638585+0000 mgr.x (mgr.24637) 98 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:30:04.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:03 smithi099 bash[32082]: cephadm 2024-09-13T14:30:02.690172+0000 mgr.x (mgr.24637) 99 : cephadm [INF] Reconfiguring daemon grafana.a on smithi177 2024-09-13T14:30:04.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:03 smithi099 bash[32082]: cluster 2024-09-13T14:30:02.875860+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:30:04.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:03 smithi099 bash[32082]: cluster 2024-09-13T14:30:03.602411+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:30:04.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:03 smithi099 bash[32082]: audit 2024-09-13T14:30:03.675292+0000 mon.a (mon.0) 1151 : audit [DBG] from='client.? 172.21.15.99:0/2837593138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:04.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[24494]: cephadm 2024-09-13T14:30:02.638585+0000 mgr.x (mgr.24637) 98 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-13T14:30:04.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[24494]: cephadm 2024-09-13T14:30:02.690172+0000 mgr.x (mgr.24637) 99 : cephadm [INF] Reconfiguring daemon grafana.a on smithi177 2024-09-13T14:30:04.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[24494]: cluster 2024-09-13T14:30:02.875860+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:30:04.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[24494]: cluster 2024-09-13T14:30:03.602411+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:30:04.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:03 smithi177 bash[24494]: audit 2024-09-13T14:30:03.675292+0000 mon.a (mon.0) 1151 : audit [DBG] from='client.? 172.21.15.99:0/2837593138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:05.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:04 smithi177 bash[24494]: audit 2024-09-13T14:30:03.997360+0000 mgr.x (mgr.24637) 100 : audit [DBG] from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:05.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:04 smithi177 bash[24494]: cluster 2024-09-13T14:30:04.062387+0000 mgr.x (mgr.24637) 101 : cluster [DBG] pgmap v90: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:30:05.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:04 smithi177 bash[24494]: audit 2024-09-13T14:30:04.289870+0000 mgr.x (mgr.24637) 102 : audit [DBG] from='client.25628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:05.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:04 smithi177 bash[24494]: cluster 2024-09-13T14:30:04.602905+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:30:05.104 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:04 smithi177 bash[59802]: 2abcce694348: Pull complete 2024-09-13T14:30:05.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:04 smithi099 bash[22656]: audit 2024-09-13T14:30:03.997360+0000 mgr.x (mgr.24637) 100 : audit [DBG] from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:05.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:04 smithi099 bash[22656]: cluster 2024-09-13T14:30:04.062387+0000 mgr.x (mgr.24637) 101 : cluster [DBG] pgmap v90: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:30:05.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:04 smithi099 bash[22656]: audit 2024-09-13T14:30:04.289870+0000 mgr.x (mgr.24637) 102 : audit [DBG] from='client.25628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:05.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:04 smithi099 bash[22656]: cluster 2024-09-13T14:30:04.602905+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:30:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:04 smithi099 bash[32082]: audit 2024-09-13T14:30:03.997360+0000 mgr.x (mgr.24637) 100 : audit [DBG] from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:04 smithi099 bash[32082]: cluster 2024-09-13T14:30:04.062387+0000 mgr.x (mgr.24637) 101 : cluster [DBG] pgmap v90: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:30:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:04 smithi099 bash[32082]: audit 2024-09-13T14:30:04.289870+0000 mgr.x (mgr.24637) 102 : audit [DBG] from='client.25628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:04 smithi099 bash[32082]: cluster 2024-09-13T14:30:04.602905+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:30:05.603 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:05 smithi177 bash[59802]: 455fd88e5221: Pull complete 2024-09-13T14:30:05.622 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:05.675 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83886079~1 2024-09-13T14:30:05.696 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:06.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:05 smithi177 bash[24494]: audit 2024-09-13T14:30:05.150040+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:06.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:05 smithi177 bash[24494]: cluster 2024-09-13T14:30:05.606492+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:30:06.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:05 smithi099 bash[22656]: audit 2024-09-13T14:30:05.150040+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:06.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:05 smithi099 bash[22656]: cluster 2024-09-13T14:30:05.606492+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:30:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:05 smithi099 bash[32082]: audit 2024-09-13T14:30:05.150040+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:06.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:05 smithi099 bash[32082]: cluster 2024-09-13T14:30:05.606492+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:30:06.973 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:07.312 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:06 smithi177 bash[59802]: 324153f2810a: Pull complete 2024-09-13T14:30:07.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:06 smithi177 bash[24494]: cluster 2024-09-13T14:30:06.063304+0000 mgr.x (mgr.24637) 103 : cluster [DBG] pgmap v93: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 25 MiB/s wr, 229 op/s 2024-09-13T14:30:07.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:06 smithi177 bash[24494]: cluster 2024-09-13T14:30:06.608528+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:30:07.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:06 smithi177 bash[24494]: audit 2024-09-13T14:30:06.668745+0000 mon.a (mon.0) 1155 : audit [DBG] from='client.? 172.21.15.99:0/1754197803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:07.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:06 smithi099 bash[32082]: cluster 2024-09-13T14:30:06.063304+0000 mgr.x (mgr.24637) 103 : cluster [DBG] pgmap v93: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 25 MiB/s wr, 229 op/s 2024-09-13T14:30:07.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:06 smithi099 bash[32082]: cluster 2024-09-13T14:30:06.608528+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:30:07.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:06 smithi099 bash[32082]: audit 2024-09-13T14:30:06.668745+0000 mon.a (mon.0) 1155 : audit [DBG] from='client.? 172.21.15.99:0/1754197803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:06 smithi099 bash[22656]: cluster 2024-09-13T14:30:06.063304+0000 mgr.x (mgr.24637) 103 : cluster [DBG] pgmap v93: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 25 MiB/s wr, 229 op/s 2024-09-13T14:30:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:06 smithi099 bash[22656]: cluster 2024-09-13T14:30:06.608528+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:30:07.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:06 smithi099 bash[22656]: audit 2024-09-13T14:30:06.668745+0000 mon.a (mon.0) 1155 : audit [DBG] from='client.? 172.21.15.99:0/1754197803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:07.603 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:07 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:30:07] "GET /metrics HTTP/1.1" 200 38903 "" "Prometheus/2.51.0" 2024-09-13T14:30:07.603 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:07 smithi177 bash[59802]: Digest: sha256:4cb2b9019f1757be8482419002cb7afe028fdba35d47958829e4cfeaf6246d80 2024-09-13T14:30:08.064 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:07 smithi177 bash[59802]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.7.0 2024-09-13T14:30:08.065 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:07 smithi177 bash[24494]: audit 2024-09-13T14:30:06.970545+0000 mgr.x (mgr.24637) 104 : audit [DBG] from='client.25640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.065 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:07 smithi177 bash[24494]: audit 2024-09-13T14:30:07.277096+0000 mgr.x (mgr.24637) 105 : audit [DBG] from='client.25646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.065 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:07 smithi177 bash[24494]: cluster 2024-09-13T14:30:07.610441+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:30:08.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:07 smithi099 bash[22656]: audit 2024-09-13T14:30:06.970545+0000 mgr.x (mgr.24637) 104 : audit [DBG] from='client.25640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:07 smithi099 bash[22656]: audit 2024-09-13T14:30:07.277096+0000 mgr.x (mgr.24637) 105 : audit [DBG] from='client.25646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:07 smithi099 bash[22656]: cluster 2024-09-13T14:30:07.610441+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:30:08.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:07 smithi099 bash[32082]: audit 2024-09-13T14:30:06.970545+0000 mgr.x (mgr.24637) 104 : audit [DBG] from='client.25640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:07 smithi099 bash[32082]: audit 2024-09-13T14:30:07.277096+0000 mgr.x (mgr.24637) 105 : audit [DBG] from='client.25646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:08.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:07 smithi099 bash[32082]: cluster 2024-09-13T14:30:07.610441+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:30:09.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:08 smithi177 bash[24494]: cluster 2024-09-13T14:30:08.064038+0000 mgr.x (mgr.24637) 106 : cluster [DBG] pgmap v96: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 25 MiB/s wr, 229 op/s 2024-09-13T14:30:09.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:08 smithi177 bash[24494]: cluster 2024-09-13T14:30:08.613855+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:30:09.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:08 smithi099 bash[22656]: cluster 2024-09-13T14:30:08.064038+0000 mgr.x (mgr.24637) 106 : cluster [DBG] pgmap v96: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 25 MiB/s wr, 229 op/s 2024-09-13T14:30:09.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:08 smithi099 bash[22656]: cluster 2024-09-13T14:30:08.613855+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:30:09.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:08 smithi099 bash[32082]: cluster 2024-09-13T14:30:08.064038+0000 mgr.x (mgr.24637) 106 : cluster [DBG] pgmap v96: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 25 MiB/s wr, 229 op/s 2024-09-13T14:30:09.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:08 smithi099 bash[32082]: cluster 2024-09-13T14:30:08.613855+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:30:09.437 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:30:09.562 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:30:09.854 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.567Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-13T14:30:09.854 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.567Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-13T14:30:09.854 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.568Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:30:09.854 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.568Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:30:09.854 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.568Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:30:09.854 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.568Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:30:09.854 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:30:09.854 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:30:09.855 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:30:09.856 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:30:09.856 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:30:09.856 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:30:09.856 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:30:09.856 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:30:09.856 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:30:09.856 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:30:09.856 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:30:09.856 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:30:09.857 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:30:09.858 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:30:09.858 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:30:09.858 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:30:09.858 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:30:09.858 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:30:09.858 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:30:09.858 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.569Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:30:09.858 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.570Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:30:09.858 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[59802]: ts=2024-09-13T14:30:09.570Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:30:09.936 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 0~8388609 2024-09-13T14:30:09.976 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 16777218~8388609 2024-09-13T14:30:10.069 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:10.090 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:30:10.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:09 smithi099 bash[22656]: audit 2024-09-13T14:30:09.709424+0000 mon.c (mon.2) 170 : audit [DBG] from='client.? 172.21.15.99:0/3182762108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:10.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:09 smithi099 bash[32082]: audit 2024-09-13T14:30:09.709424+0000 mon.c (mon.2) 170 : audit [DBG] from='client.? 172.21.15.99:0/3182762108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:10.322 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:09 smithi177 bash[24494]: audit 2024-09-13T14:30:09.709424+0000 mon.c (mon.2) 170 : audit [DBG] from='client.? 172.21.15.99:0/3182762108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:10.603 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:10 smithi177 systemd[1]: Stopping Ceph grafana.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:30:11.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:10 smithi177 bash[24494]: cluster 2024-09-13T14:30:09.915029+0000 mon.a (mon.0) 1158 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:30:11.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:10 smithi177 bash[24494]: audit 2024-09-13T14:30:10.058030+0000 mgr.x (mgr.24637) 107 : audit [DBG] from='client.25658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:10 smithi177 bash[24494]: cluster 2024-09-13T14:30:10.065828+0000 mgr.x (mgr.24637) 108 : cluster [DBG] pgmap v99: 137 pgs: 137 active+clean; 913 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 376 op/s 2024-09-13T14:30:11.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:10 smithi177 bash[24494]: audit 2024-09-13T14:30:10.393175+0000 mgr.x (mgr.24637) 109 : audit [DBG] from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:10 smithi099 bash[22656]: cluster 2024-09-13T14:30:09.915029+0000 mon.a (mon.0) 1158 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:30:11.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:10 smithi099 bash[22656]: audit 2024-09-13T14:30:10.058030+0000 mgr.x (mgr.24637) 107 : audit [DBG] from='client.25658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:10 smithi099 bash[22656]: cluster 2024-09-13T14:30:10.065828+0000 mgr.x (mgr.24637) 108 : cluster [DBG] pgmap v99: 137 pgs: 137 active+clean; 913 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 376 op/s 2024-09-13T14:30:11.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:10 smithi099 bash[22656]: audit 2024-09-13T14:30:10.393175+0000 mgr.x (mgr.24637) 109 : audit [DBG] from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:10 smithi099 bash[32082]: cluster 2024-09-13T14:30:09.915029+0000 mon.a (mon.0) 1158 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:30:11.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:10 smithi099 bash[32082]: audit 2024-09-13T14:30:10.058030+0000 mgr.x (mgr.24637) 107 : audit [DBG] from='client.25658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:10 smithi099 bash[32082]: cluster 2024-09-13T14:30:10.065828+0000 mgr.x (mgr.24637) 108 : cluster [DBG] pgmap v99: 137 pgs: 137 active+clean; 913 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 376 op/s 2024-09-13T14:30:11.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:10 smithi099 bash[32082]: audit 2024-09-13T14:30:10.393175+0000 mgr.x (mgr.24637) 109 : audit [DBG] from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:11.353 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[60058]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-grafana-a 2024-09-13T14:30:11.353 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:11 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:30:11.353 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[60117]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-grafana-a 2024-09-13T14:30:11.353 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:11 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T14:30:11.353 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:11 smithi177 systemd[1]: Stopped Ceph grafana.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:30:11.353 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:11 smithi177 systemd[1]: Started Ceph grafana.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:30:12.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: cluster 2024-09-13T14:30:10.925309+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: audit 2024-09-13T14:30:11.390393+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: audit 2024-09-13T14:30:11.400124+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: audit 2024-09-13T14:30:11.408040+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: audit 2024-09-13T14:30:11.408446+0000 mgr.x (mgr.24637) 110 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: audit 2024-09-13T14:30:11.411576+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: audit 2024-09-13T14:30:11.411951+0000 mgr.x (mgr.24637) 111 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: audit 2024-09-13T14:30:11.414888+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: audit 2024-09-13T14:30:11.415232+0000 mgr.x (mgr.24637) 112 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: audit 2024-09-13T14:30:11.456623+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: cephadm 2024-09-13T14:30:11.458201+0000 mgr.x (mgr.24637) 113 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[22656]: cephadm 2024-09-13T14:30:11.458756+0000 mgr.x (mgr.24637) 114 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:30:12.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: cluster 2024-09-13T14:30:10.925309+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: audit 2024-09-13T14:30:11.390393+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: audit 2024-09-13T14:30:11.400124+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: audit 2024-09-13T14:30:11.408040+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: audit 2024-09-13T14:30:11.408446+0000 mgr.x (mgr.24637) 110 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: audit 2024-09-13T14:30:11.411576+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: audit 2024-09-13T14:30:11.411951+0000 mgr.x (mgr.24637) 111 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: audit 2024-09-13T14:30:11.414888+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: audit 2024-09-13T14:30:11.415232+0000 mgr.x (mgr.24637) 112 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: audit 2024-09-13T14:30:11.456623+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: cephadm 2024-09-13T14:30:11.458201+0000 mgr.x (mgr.24637) 113 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:30:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:11 smithi099 bash[32082]: cephadm 2024-09-13T14:30:11.458756+0000 mgr.x (mgr.24637) 114 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:30:12.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: cluster 2024-09-13T14:30:10.925309+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:30:12.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: audit 2024-09-13T14:30:11.390393+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:12.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: audit 2024-09-13T14:30:11.400124+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:12.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: audit 2024-09-13T14:30:11.408040+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:30:12.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: audit 2024-09-13T14:30:11.408446+0000 mgr.x (mgr.24637) 110 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:30:12.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: audit 2024-09-13T14:30:11.411576+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:30:12.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: audit 2024-09-13T14:30:11.411951+0000 mgr.x (mgr.24637) 111 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:30:12.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: audit 2024-09-13T14:30:11.414888+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:30:12.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: audit 2024-09-13T14:30:11.415232+0000 mgr.x (mgr.24637) 112 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:30:12.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: audit 2024-09-13T14:30:11.456623+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:12.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: cephadm 2024-09-13T14:30:11.458201+0000 mgr.x (mgr.24637) 113 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:30:12.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:11 smithi177 bash[24494]: cephadm 2024-09-13T14:30:11.458756+0000 mgr.x (mgr.24637) 114 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:30:12.750 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741269182Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T14:30:12.750 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741502518Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T14:30:12.750 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741532286Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:30:12.750 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741547882Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:30:12.750 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.74156259Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:30:12.750 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741574144Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:30:12.750 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741585712Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:30:12.750 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741597938Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:30:12.750 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741609477Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:30:12.751 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741621382Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:30:12.751 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741633784Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:30:12.751 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741667968Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:30:12.751 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741674992Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:30:12.751 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=settings t=2024-09-13T14:30:12.741682417Z level=info msg="App mode production" 2024-09-13T14:30:12.751 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=sqlstore t=2024-09-13T14:30:12.741745656Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:30:12.751 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=sqlstore t=2024-09-13T14:30:12.741770581Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:30:12.751 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=migrator t=2024-09-13T14:30:12.758445971Z level=info msg="Starting DB migrations" 2024-09-13T14:30:13.095 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:13.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[24494]: cluster 2024-09-13T14:30:11.934025+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:30:13.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[24494]: cluster 2024-09-13T14:30:12.066633+0000 mgr.x (mgr.24637) 115 : cluster [DBG] pgmap v102: 137 pgs: 137 active+clean; 913 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 376 op/s 2024-09-13T14:30:13.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[24494]: audit 2024-09-13T14:30:12.765026+0000 mon.a (mon.0) 1163 : audit [DBG] from='client.? 172.21.15.99:0/3655008639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:13.103 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=migrator t=2024-09-13T14:30:12.764144997Z level=info msg="migrations completed" performed=0 skipped=481 duration=518.585µs 2024-09-13T14:30:13.103 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=sqlstore t=2024-09-13T14:30:12.764569002Z level=info msg="Created default organization" 2024-09-13T14:30:13.104 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=plugin.loader t=2024-09-13T14:30:12.872025442Z level=info msg="Plugin registered" pluginID=input 2024-09-13T14:30:13.104 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=plugin.loader t=2024-09-13T14:30:12.903615406Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T14:30:13.104 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=plugin.loader t=2024-09-13T14:30:12.903649064Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T14:30:13.104 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=secrets t=2024-09-13T14:30:12.903705585Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:30:13.104 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=query_data t=2024-09-13T14:30:12.905698415Z level=info msg="Query Service initialization" 2024-09-13T14:30:13.104 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:12 smithi177 bash[60145]: logger=live.push_http t=2024-09-13T14:30:12.908513806Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:30:13.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:12 smithi099 bash[22656]: cluster 2024-09-13T14:30:11.934025+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:30:13.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:12 smithi099 bash[22656]: cluster 2024-09-13T14:30:12.066633+0000 mgr.x (mgr.24637) 115 : cluster [DBG] pgmap v102: 137 pgs: 137 active+clean; 913 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 376 op/s 2024-09-13T14:30:13.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:12 smithi099 bash[22656]: audit 2024-09-13T14:30:12.765026+0000 mon.a (mon.0) 1163 : audit [DBG] from='client.? 172.21.15.99:0/3655008639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:13.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:12 smithi099 bash[32082]: cluster 2024-09-13T14:30:11.934025+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:30:13.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:12 smithi099 bash[32082]: cluster 2024-09-13T14:30:12.066633+0000 mgr.x (mgr.24637) 115 : cluster [DBG] pgmap v102: 137 pgs: 137 active+clean; 913 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 376 op/s 2024-09-13T14:30:13.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:12 smithi099 bash[32082]: audit 2024-09-13T14:30:12.765026+0000 mon.a (mon.0) 1163 : audit [DBG] from='client.? 172.21.15.99:0/3655008639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:14.191 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=infra.usagestats.collector t=2024-09-13T14:30:14.060374062Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:30:14.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:13 smithi177 bash[24494]: cluster 2024-09-13T14:30:12.941104+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:30:14.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:13 smithi177 bash[24494]: audit 2024-09-13T14:30:13.092373+0000 mgr.x (mgr.24637) 116 : audit [DBG] from='client.25676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:13 smithi177 bash[24494]: audit 2024-09-13T14:30:13.396770+0000 mgr.x (mgr.24637) 117 : audit [DBG] from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.192 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[59561]: ts=2024-09-13T14:30:14.150Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:14.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:13 smithi099 bash[32082]: cluster 2024-09-13T14:30:12.941104+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:30:14.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:13 smithi099 bash[32082]: audit 2024-09-13T14:30:13.092373+0000 mgr.x (mgr.24637) 116 : audit [DBG] from='client.25676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:13 smithi099 bash[32082]: audit 2024-09-13T14:30:13.396770+0000 mgr.x (mgr.24637) 117 : audit [DBG] from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:13 smithi099 bash[22656]: cluster 2024-09-13T14:30:12.941104+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:30:14.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:13 smithi099 bash[22656]: audit 2024-09-13T14:30:13.092373+0000 mgr.x (mgr.24637) 116 : audit [DBG] from='client.25676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:13 smithi099 bash[22656]: audit 2024-09-13T14:30:13.396770+0000 mgr.x (mgr.24637) 117 : audit [DBG] from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:14.484 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=provisioning.datasources t=2024-09-13T14:30:14.200222526Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-13T14:30:14.484 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=provisioning.datasources t=2024-09-13T14:30:14.200296357Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-13T14:30:14.484 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=provisioning.datasources t=2024-09-13T14:30:14.211743552Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:30:14.484 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=provisioning.alerting t=2024-09-13T14:30:14.440729356Z level=info msg="starting to provision alerting" 2024-09-13T14:30:14.484 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=provisioning.alerting t=2024-09-13T14:30:14.440805966Z level=info msg="finished to provision alerting" 2024-09-13T14:30:14.484 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=ngalert.state.manager t=2024-09-13T14:30:14.441129091Z level=info msg="Warming state cache for startup" 2024-09-13T14:30:14.484 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=grafanaStorageLogger t=2024-09-13T14:30:14.441367923Z level=info msg="storage starting" 2024-09-13T14:30:14.484 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=http.server t=2024-09-13T14:30:14.444557667Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:30:14.484 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=ngalert.state.manager t=2024-09-13T14:30:14.492831031Z level=info msg="State cache has been initialized" states=0 duration=51.701297ms 2024-09-13T14:30:14.603 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.AbortWithSnapshots (14130 ms) 2024-09-13T14:30:14.603 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneV1Parent 2024-09-13T14:30:14.680 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:14.706 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 0~10 2024-09-13T14:30:14.726 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:14.853 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:30:14.492927392Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T14:30:14.853 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[60145]: logger=ticker t=2024-09-13T14:30:14.493028338Z level=info msg=starting first_tick=2024-09-13T14:30:20Z 2024-09-13T14:30:15.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[22656]: cluster 2024-09-13T14:30:13.960262+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:30:15.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[22656]: cluster 2024-09-13T14:30:14.067178+0000 mgr.x (mgr.24637) 118 : cluster [DBG] pgmap v105: 137 pgs: 137 active+clean; 913 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:15.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[22656]: cephadm 2024-09-13T14:30:14.389408+0000 mgr.x (mgr.24637) 119 : cephadm [INF] Upgrade: Updating mgr.y 2024-09-13T14:30:15.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[22656]: audit 2024-09-13T14:30:14.400470+0000 mon.a (mon.0) 1166 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:15.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[22656]: audit 2024-09-13T14:30:14.403612+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:30:15.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[22656]: audit 2024-09-13T14:30:14.405748+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:30:15.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[22656]: audit 2024-09-13T14:30:14.408597+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:30:15.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[22656]: audit 2024-09-13T14:30:14.410559+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:15.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[22656]: cephadm 2024-09-13T14:30:14.412591+0000 mgr.x (mgr.24637) 120 : cephadm [INF] Deploying daemon mgr.y on smithi099 2024-09-13T14:30:15.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[32082]: cluster 2024-09-13T14:30:13.960262+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:30:15.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[32082]: cluster 2024-09-13T14:30:14.067178+0000 mgr.x (mgr.24637) 118 : cluster [DBG] pgmap v105: 137 pgs: 137 active+clean; 913 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:15.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[32082]: cephadm 2024-09-13T14:30:14.389408+0000 mgr.x (mgr.24637) 119 : cephadm [INF] Upgrade: Updating mgr.y 2024-09-13T14:30:15.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[32082]: audit 2024-09-13T14:30:14.400470+0000 mon.a (mon.0) 1166 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:15.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[32082]: audit 2024-09-13T14:30:14.403612+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:30:15.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[32082]: audit 2024-09-13T14:30:14.405748+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:30:15.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[32082]: audit 2024-09-13T14:30:14.408597+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:30:15.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[32082]: audit 2024-09-13T14:30:14.410559+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:15.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:14 smithi099 bash[32082]: cephadm 2024-09-13T14:30:14.412591+0000 mgr.x (mgr.24637) 120 : cephadm [INF] Deploying daemon mgr.y on smithi099 2024-09-13T14:30:15.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[24494]: cluster 2024-09-13T14:30:13.960262+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:30:15.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[24494]: cluster 2024-09-13T14:30:14.067178+0000 mgr.x (mgr.24637) 118 : cluster [DBG] pgmap v105: 137 pgs: 137 active+clean; 913 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:15.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[24494]: cephadm 2024-09-13T14:30:14.389408+0000 mgr.x (mgr.24637) 119 : cephadm [INF] Upgrade: Updating mgr.y 2024-09-13T14:30:15.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[24494]: audit 2024-09-13T14:30:14.400470+0000 mon.a (mon.0) 1166 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:15.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[24494]: audit 2024-09-13T14:30:14.403612+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:30:15.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[24494]: audit 2024-09-13T14:30:14.405748+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:30:15.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[24494]: audit 2024-09-13T14:30:14.408597+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:30:15.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[24494]: audit 2024-09-13T14:30:14.410559+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:15.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:14 smithi177 bash[24494]: cephadm 2024-09-13T14:30:14.412591+0000 mgr.x (mgr.24637) 120 : cephadm [INF] Deploying daemon mgr.y on smithi099 2024-09-13T14:30:16.085 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:16.103 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:15 smithi177 bash[59561]: ts=2024-09-13T14:30:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:16.103 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:15 smithi177 bash[59561]: ts=2024-09-13T14:30:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:16.418 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:16.495 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:16.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:16 smithi099 bash[22656]: cluster 2024-09-13T14:30:15.400017+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:30:16.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:16 smithi099 bash[22656]: audit 2024-09-13T14:30:15.766608+0000 mon.a (mon.0) 1169 : audit [DBG] from='client.? 172.21.15.99:0/2952104580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:16.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:16 smithi099 bash[32082]: cluster 2024-09-13T14:30:15.400017+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:30:16.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:16 smithi099 bash[32082]: audit 2024-09-13T14:30:15.766608+0000 mon.a (mon.0) 1169 : audit [DBG] from='client.? 172.21.15.99:0/2952104580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:16.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:16 smithi177 bash[24494]: cluster 2024-09-13T14:30:15.400017+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:30:16.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:16 smithi177 bash[24494]: audit 2024-09-13T14:30:15.766608+0000 mon.a (mon.0) 1169 : audit [DBG] from='client.? 172.21.15.99:0/2952104580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:17.353 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:16 smithi177 bash[59561]: ts=2024-09-13T14:30:16.952Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:17.697 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:17 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:17.697 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:30:17 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:17.698 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:30:17 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:17.698 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:30:17 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:17.698 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:30:17 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:17.699 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:30:17 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:17.699 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:30:17 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:17.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:17 smithi099 bash[32082]: cluster 2024-09-13T14:30:16.067891+0000 mgr.x (mgr.24637) 121 : cluster [DBG] pgmap v107: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s rd, 7.8 MiB/s wr, 207 op/s 2024-09-13T14:30:17.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:17 smithi099 bash[32082]: audit 2024-09-13T14:30:16.082185+0000 mgr.x (mgr.24637) 122 : audit [DBG] from='client.25694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:17 smithi099 bash[32082]: audit 2024-09-13T14:30:16.398337+0000 mgr.x (mgr.24637) 123 : audit [DBG] from='client.25700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:17 smithi099 bash[32082]: cluster 2024-09-13T14:30:16.405710+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:30:17.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:17 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:17.700 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:17 smithi099 bash[22656]: cluster 2024-09-13T14:30:16.067891+0000 mgr.x (mgr.24637) 121 : cluster [DBG] pgmap v107: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s rd, 7.8 MiB/s wr, 207 op/s 2024-09-13T14:30:17.700 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:17 smithi099 bash[22656]: audit 2024-09-13T14:30:16.082185+0000 mgr.x (mgr.24637) 122 : audit [DBG] from='client.25694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.700 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:17 smithi099 bash[22656]: audit 2024-09-13T14:30:16.398337+0000 mgr.x (mgr.24637) 123 : audit [DBG] from='client.25700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.700 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:17 smithi099 bash[22656]: cluster 2024-09-13T14:30:16.405710+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:30:17.700 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:17 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:17.738 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:30:17.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:17 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:30:17] "GET /metrics HTTP/1.1" 200 38903 "" "Prometheus/2.51.0" 2024-09-13T14:30:17.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:17 smithi177 bash[24494]: cluster 2024-09-13T14:30:16.067891+0000 mgr.x (mgr.24637) 121 : cluster [DBG] pgmap v107: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s rd, 7.8 MiB/s wr, 207 op/s 2024-09-13T14:30:17.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:17 smithi177 bash[24494]: audit 2024-09-13T14:30:16.082185+0000 mgr.x (mgr.24637) 122 : audit [DBG] from='client.25694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:17 smithi177 bash[24494]: audit 2024-09-13T14:30:16.398337+0000 mgr.x (mgr.24637) 123 : audit [DBG] from='client.25700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:17.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:17 smithi177 bash[24494]: cluster 2024-09-13T14:30:16.405710+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:30:17.875 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:30:18.061 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:17 smithi099 systemd[1]: Stopping Ceph mgr.y for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:30:18.581 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneV1Parent (3978 ms) 2024-09-13T14:30:18.581 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneV2Parent 2024-09-13T14:30:18.665 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:18.678 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:18 smithi099 bash[22656]: cluster 2024-09-13T14:30:17.408633+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:30:18.678 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:18 smithi099 bash[76164]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mgr-y 2024-09-13T14:30:18.678 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:30:18.678 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:18 smithi099 bash[76245]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mgr-y 2024-09-13T14:30:18.678 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:18 smithi099 bash[32082]: cluster 2024-09-13T14:30:17.408633+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:30:18.692 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 0~10 2024-09-13T14:30:18.707 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:18.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:18 smithi177 bash[24494]: cluster 2024-09-13T14:30:17.408633+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:30:18.946 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:18.947 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:18.947 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:18.948 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:18.948 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:18.948 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-13T14:30:18.948 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: Stopped Ceph mgr.y for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:30:18.948 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:18.949 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:18.949 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:18.950 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:30:19.165 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:19.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:18 smithi099 systemd[1]: Started Ceph mgr.y for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:30:19.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[22656]: cluster 2024-09-13T14:30:18.068667+0000 mgr.x (mgr.24637) 124 : cluster [DBG] pgmap v110: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s rd, 7.8 MiB/s wr, 208 op/s 2024-09-13T14:30:19.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[22656]: cluster 2024-09-13T14:30:18.423417+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:30:19.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[22656]: cluster 2024-09-13T14:30:18.602953+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[22656]: audit 2024-09-13T14:30:18.821844+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.99:0/3457766776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[22656]: audit 2024-09-13T14:30:19.013144+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[22656]: audit 2024-09-13T14:30:19.024032+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[22656]: audit 2024-09-13T14:30:19.031641+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[32082]: cluster 2024-09-13T14:30:18.068667+0000 mgr.x (mgr.24637) 124 : cluster [DBG] pgmap v110: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s rd, 7.8 MiB/s wr, 208 op/s 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[32082]: cluster 2024-09-13T14:30:18.423417+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[32082]: cluster 2024-09-13T14:30:18.602953+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[32082]: audit 2024-09-13T14:30:18.821844+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.99:0/3457766776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[32082]: audit 2024-09-13T14:30:19.013144+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[32082]: audit 2024-09-13T14:30:19.024032+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:19.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:19 smithi099 bash[32082]: audit 2024-09-13T14:30:19.031641+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:19.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:19 smithi177 bash[24494]: cluster 2024-09-13T14:30:18.068667+0000 mgr.x (mgr.24637) 124 : cluster [DBG] pgmap v110: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s rd, 7.8 MiB/s wr, 208 op/s 2024-09-13T14:30:19.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:19 smithi177 bash[24494]: cluster 2024-09-13T14:30:18.423417+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:30:19.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:19 smithi177 bash[24494]: cluster 2024-09-13T14:30:18.602953+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:30:19.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:19 smithi177 bash[24494]: audit 2024-09-13T14:30:18.821844+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.99:0/3457766776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:19.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:19 smithi177 bash[24494]: audit 2024-09-13T14:30:19.013144+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:19.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:19 smithi177 bash[24494]: audit 2024-09-13T14:30:19.024032+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:19.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:19 smithi177 bash[24494]: audit 2024-09-13T14:30:19.031641+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:20.603 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:20 smithi177 bash[59561]: ts=2024-09-13T14:30:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:20.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:20 smithi177 bash[24494]: audit 2024-09-13T14:30:19.162441+0000 mgr.x (mgr.24637) 125 : audit [DBG] from='client.15819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:20.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:20 smithi177 bash[24494]: audit 2024-09-13T14:30:19.465849+0000 mgr.x (mgr.24637) 126 : audit [DBG] from='client.15825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:20.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:20 smithi177 bash[24494]: cluster 2024-09-13T14:30:19.606258+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:30:20.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:20 smithi177 bash[24494]: audit 2024-09-13T14:30:20.150756+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:20.633 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:20.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:20 smithi099 bash[32082]: audit 2024-09-13T14:30:19.162441+0000 mgr.x (mgr.24637) 125 : audit [DBG] from='client.15819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:20.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:20 smithi099 bash[32082]: audit 2024-09-13T14:30:19.465849+0000 mgr.x (mgr.24637) 126 : audit [DBG] from='client.15825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:20.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:20 smithi099 bash[32082]: cluster 2024-09-13T14:30:19.606258+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:30:20.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:20 smithi099 bash[32082]: audit 2024-09-13T14:30:20.150756+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:20.690 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:20 smithi099 bash[22656]: audit 2024-09-13T14:30:19.162441+0000 mgr.x (mgr.24637) 125 : audit [DBG] from='client.15819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:20.690 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:20 smithi099 bash[22656]: audit 2024-09-13T14:30:19.465849+0000 mgr.x (mgr.24637) 126 : audit [DBG] from='client.15825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:20.691 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:20 smithi099 bash[22656]: cluster 2024-09-13T14:30:19.606258+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:30:20.691 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:20 smithi099 bash[22656]: audit 2024-09-13T14:30:20.150756+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:20.715 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:21.068 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:20 smithi099 bash[76324]: debug 2024-09-13T14:30:20.690+0000 7fd83450b640 1 -- 172.21.15.99:0/965821445 <== mon.2 v2:172.21.15.99:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f84ef8f0e0 con 0x55f84ef91800 2024-09-13T14:30:21.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:21 smithi099 bash[22656]: cluster 2024-09-13T14:30:20.070236+0000 mgr.x (mgr.24637) 127 : cluster [DBG] pgmap v114: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 377 KiB/s rd, 22 KiB/s wr, 475 op/s 2024-09-13T14:30:21.818 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:21 smithi099 bash[22656]: cluster 2024-09-13T14:30:20.618354+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:30:21.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:21 smithi099 bash[32082]: cluster 2024-09-13T14:30:20.070236+0000 mgr.x (mgr.24637) 127 : cluster [DBG] pgmap v114: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 377 KiB/s rd, 22 KiB/s wr, 475 op/s 2024-09-13T14:30:21.818 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:21 smithi099 bash[32082]: cluster 2024-09-13T14:30:20.618354+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:30:21.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:21 smithi177 bash[24494]: cluster 2024-09-13T14:30:20.070236+0000 mgr.x (mgr.24637) 127 : cluster [DBG] pgmap v114: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 377 KiB/s rd, 22 KiB/s wr, 475 op/s 2024-09-13T14:30:21.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:21 smithi177 bash[24494]: cluster 2024-09-13T14:30:20.618354+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:30:21.978 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:30:22.132 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:30:22.210 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:22.871 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneV2Parent (4291 ms) 2024-09-13T14:30:22.871 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneV1ParentAbort 2024-09-13T14:30:22.947 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:22.976 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 0~10 2024-09-13T14:30:22.999 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:23.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:22 smithi099 bash[32082]: cluster 2024-09-13T14:30:21.621923+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:30:23.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:22 smithi099 bash[32082]: audit 2024-09-13T14:30:21.851517+0000 mon.a (mon.0) 1179 : audit [DBG] from='client.? 172.21.15.99:0/1914550517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:23.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:22 smithi099 bash[32082]: cluster 2024-09-13T14:30:22.071034+0000 mgr.x (mgr.24637) 128 : cluster [DBG] pgmap v117: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 379 KiB/s rd, 22 KiB/s wr, 478 op/s 2024-09-13T14:30:23.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:22 smithi099 bash[32082]: audit 2024-09-13T14:30:22.208064+0000 mgr.x (mgr.24637) 129 : audit [DBG] from='client.25736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:23.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:22 smithi099 bash[32082]: audit 2024-09-13T14:30:22.517761+0000 mgr.x (mgr.24637) 130 : audit [DBG] from='client.15849 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:23.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:22 smithi099 bash[22656]: cluster 2024-09-13T14:30:21.621923+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:30:23.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:22 smithi099 bash[22656]: audit 2024-09-13T14:30:21.851517+0000 mon.a (mon.0) 1179 : audit [DBG] from='client.? 172.21.15.99:0/1914550517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:23.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:22 smithi099 bash[22656]: cluster 2024-09-13T14:30:22.071034+0000 mgr.x (mgr.24637) 128 : cluster [DBG] pgmap v117: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 379 KiB/s rd, 22 KiB/s wr, 478 op/s 2024-09-13T14:30:23.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:22 smithi099 bash[22656]: audit 2024-09-13T14:30:22.208064+0000 mgr.x (mgr.24637) 129 : audit [DBG] from='client.25736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:23.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:22 smithi099 bash[22656]: audit 2024-09-13T14:30:22.517761+0000 mgr.x (mgr.24637) 130 : audit [DBG] from='client.15849 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:23.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:22 smithi177 bash[24494]: cluster 2024-09-13T14:30:21.621923+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:30:23.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:22 smithi177 bash[24494]: audit 2024-09-13T14:30:21.851517+0000 mon.a (mon.0) 1179 : audit [DBG] from='client.? 172.21.15.99:0/1914550517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:23.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:22 smithi177 bash[24494]: cluster 2024-09-13T14:30:22.071034+0000 mgr.x (mgr.24637) 128 : cluster [DBG] pgmap v117: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 379 KiB/s rd, 22 KiB/s wr, 478 op/s 2024-09-13T14:30:23.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:22 smithi177 bash[24494]: audit 2024-09-13T14:30:22.208064+0000 mgr.x (mgr.24637) 129 : audit [DBG] from='client.25736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:23.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:22 smithi177 bash[24494]: audit 2024-09-13T14:30:22.517761+0000 mgr.x (mgr.24637) 130 : audit [DBG] from='client.15849 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:24.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:23 smithi099 bash[22656]: cluster 2024-09-13T14:30:22.638976+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:30:24.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:23 smithi099 bash[22656]: cluster 2024-09-13T14:30:23.608908+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:30:24.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:23 smithi099 bash[32082]: cluster 2024-09-13T14:30:22.638976+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:30:24.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:23 smithi099 bash[32082]: cluster 2024-09-13T14:30:23.608908+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:30:24.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:23 smithi177 bash[24494]: cluster 2024-09-13T14:30:22.638976+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:30:24.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:23 smithi177 bash[24494]: cluster 2024-09-13T14:30:23.608908+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-09-13T14:30:24.603 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:24 smithi177 bash[59561]: ts=2024-09-13T14:30:24.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:24.642 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:24.712 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:25.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:24 smithi099 bash[22656]: cluster 2024-09-13T14:30:24.071787+0000 mgr.x (mgr.24637) 131 : cluster [DBG] pgmap v120: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:25.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:24 smithi099 bash[32082]: cluster 2024-09-13T14:30:24.071787+0000 mgr.x (mgr.24637) 131 : cluster [DBG] pgmap v120: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:25.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:24 smithi177 bash[24494]: cluster 2024-09-13T14:30:24.071787+0000 mgr.x (mgr.24637) 131 : cluster [DBG] pgmap v120: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:25.242 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:25.843 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:30:26.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:25 smithi099 bash[22656]: cluster 2024-09-13T14:30:24.639369+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:30:26.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:25 smithi099 bash[22656]: audit 2024-09-13T14:30:24.913312+0000 mon.a (mon.0) 1183 : audit [DBG] from='client.? 172.21.15.99:0/1796940470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:26.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:25 smithi099 bash[22656]: audit 2024-09-13T14:30:25.159026+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:26.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:25 smithi099 bash[22656]: audit 2024-09-13T14:30:25.166373+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:26.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:25 smithi099 bash[22656]: audit 2024-09-13T14:30:25.238988+0000 mgr.x (mgr.24637) 132 : audit [DBG] from='client.25754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:26.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:25 smithi099 bash[32082]: cluster 2024-09-13T14:30:24.639369+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:30:26.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:25 smithi099 bash[32082]: audit 2024-09-13T14:30:24.913312+0000 mon.a (mon.0) 1183 : audit [DBG] from='client.? 172.21.15.99:0/1796940470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:26.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:25 smithi099 bash[32082]: audit 2024-09-13T14:30:25.159026+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:26.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:25 smithi099 bash[32082]: audit 2024-09-13T14:30:25.166373+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:26.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:25 smithi099 bash[32082]: audit 2024-09-13T14:30:25.238988+0000 mgr.x (mgr.24637) 132 : audit [DBG] from='client.25754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:26.103 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:25 smithi177 bash[59561]: ts=2024-09-13T14:30:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:26.103 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:25 smithi177 bash[59561]: ts=2024-09-13T14:30:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:26.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:25 smithi177 bash[24494]: cluster 2024-09-13T14:30:24.639369+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-09-13T14:30:26.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:25 smithi177 bash[24494]: audit 2024-09-13T14:30:24.913312+0000 mon.a (mon.0) 1183 : audit [DBG] from='client.? 172.21.15.99:0/1796940470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:26.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:25 smithi177 bash[24494]: audit 2024-09-13T14:30:25.159026+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:26.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:25 smithi177 bash[24494]: audit 2024-09-13T14:30:25.166373+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:26.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:25 smithi177 bash[24494]: audit 2024-09-13T14:30:25.238988+0000 mgr.x (mgr.24637) 132 : audit [DBG] from='client.25754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:26.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:26 smithi177 bash[24494]: audit 2024-09-13T14:30:25.546766+0000 mgr.x (mgr.24637) 133 : audit [DBG] from='client.15867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:26.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:26 smithi177 bash[24494]: cluster 2024-09-13T14:30:25.649076+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-09-13T14:30:26.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:26 smithi177 bash[24494]: audit 2024-09-13T14:30:25.832730+0000 mon.a (mon.0) 1187 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:26.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:26 smithi177 bash[24494]: audit 2024-09-13T14:30:25.840718+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:26.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:26 smithi177 bash[24494]: cluster 2024-09-13T14:30:26.072561+0000 mgr.x (mgr.24637) 134 : cluster [DBG] pgmap v123: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 529 op/s 2024-09-13T14:30:27.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:26 smithi099 bash[22656]: audit 2024-09-13T14:30:25.546766+0000 mgr.x (mgr.24637) 133 : audit [DBG] from='client.15867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:27.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:26 smithi099 bash[22656]: cluster 2024-09-13T14:30:25.649076+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-09-13T14:30:27.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:26 smithi099 bash[22656]: audit 2024-09-13T14:30:25.832730+0000 mon.a (mon.0) 1187 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:27.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:26 smithi099 bash[22656]: audit 2024-09-13T14:30:25.840718+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:27.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:26 smithi099 bash[22656]: cluster 2024-09-13T14:30:26.072561+0000 mgr.x (mgr.24637) 134 : cluster [DBG] pgmap v123: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 529 op/s 2024-09-13T14:30:27.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:26 smithi099 bash[32082]: audit 2024-09-13T14:30:25.546766+0000 mgr.x (mgr.24637) 133 : audit [DBG] from='client.15867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:26 smithi099 bash[32082]: cluster 2024-09-13T14:30:25.649076+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-09-13T14:30:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:26 smithi099 bash[32082]: audit 2024-09-13T14:30:25.832730+0000 mon.a (mon.0) 1187 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:26 smithi099 bash[32082]: audit 2024-09-13T14:30:25.840718+0000 mon.a (mon.0) 1188 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:27.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:26 smithi099 bash[32082]: cluster 2024-09-13T14:30:26.072561+0000 mgr.x (mgr.24637) 134 : cluster [DBG] pgmap v123: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 529 op/s 2024-09-13T14:30:27.078 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneV1ParentAbort (4207 ms) 2024-09-13T14:30:27.078 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneV2ParentAbort 2024-09-13T14:30:27.136 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:27.159 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 0~10 2024-09-13T14:30:27.177 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:27.353 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:26 smithi177 bash[59561]: ts=2024-09-13T14:30:26.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:27.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[76324]: [13/Sep/2024:14:30:27] ENGINE Bus STARTING 2024-09-13T14:30:27.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[76324]: CherryPy Checker: 2024-09-13T14:30:27.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[76324]: The Application mounted at '' has an empty config. 2024-09-13T14:30:27.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[76324]: [13/Sep/2024:14:30:27] ENGINE Serving on http://:::9283 2024-09-13T14:30:27.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[76324]: [13/Sep/2024:14:30:27] ENGINE Bus STARTED 2024-09-13T14:30:27.845 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:27 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:30:27] "GET /metrics HTTP/1.1" 200 38905 "" "Prometheus/2.51.0" 2024-09-13T14:30:28.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:27 smithi177 bash[24494]: cluster 2024-09-13T14:30:26.851110+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-09-13T14:30:28.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:27 smithi177 bash[24494]: audit 2024-09-13T14:30:27.025194+0000 mon.a (mon.0) 1190 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:28.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:27 smithi177 bash[24494]: audit 2024-09-13T14:30:27.036777+0000 mon.a (mon.0) 1191 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:28.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:27 smithi177 bash[24494]: cluster 2024-09-13T14:30:27.084103+0000 mon.a (mon.0) 1192 : cluster [DBG] Standby manager daemon y restarted 2024-09-13T14:30:28.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:27 smithi177 bash[24494]: cluster 2024-09-13T14:30:27.084251+0000 mon.a (mon.0) 1193 : cluster [DBG] Standby manager daemon y started 2024-09-13T14:30:28.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:27 smithi177 bash[24494]: audit 2024-09-13T14:30:27.086393+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:30:28.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:27 smithi177 bash[24494]: audit 2024-09-13T14:30:27.087655+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:28.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:27 smithi177 bash[24494]: audit 2024-09-13T14:30:27.090426+0000 mon.c (mon.2) 174 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:30:28.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:27 smithi177 bash[24494]: audit 2024-09-13T14:30:27.090871+0000 mon.c (mon.2) 175 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:28.188 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:28.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[22656]: cluster 2024-09-13T14:30:26.851110+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-09-13T14:30:28.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[22656]: audit 2024-09-13T14:30:27.025194+0000 mon.a (mon.0) 1190 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:28.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[22656]: audit 2024-09-13T14:30:27.036777+0000 mon.a (mon.0) 1191 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:28.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[22656]: cluster 2024-09-13T14:30:27.084103+0000 mon.a (mon.0) 1192 : cluster [DBG] Standby manager daemon y restarted 2024-09-13T14:30:28.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[22656]: cluster 2024-09-13T14:30:27.084251+0000 mon.a (mon.0) 1193 : cluster [DBG] Standby manager daemon y started 2024-09-13T14:30:28.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[22656]: audit 2024-09-13T14:30:27.086393+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:30:28.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[22656]: audit 2024-09-13T14:30:27.087655+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:28.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[22656]: audit 2024-09-13T14:30:27.090426+0000 mon.c (mon.2) 174 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:30:28.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[22656]: audit 2024-09-13T14:30:27.090871+0000 mon.c (mon.2) 175 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:28.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[32082]: cluster 2024-09-13T14:30:26.851110+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-09-13T14:30:28.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[32082]: audit 2024-09-13T14:30:27.025194+0000 mon.a (mon.0) 1190 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:28.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[32082]: audit 2024-09-13T14:30:27.036777+0000 mon.a (mon.0) 1191 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:28.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[32082]: cluster 2024-09-13T14:30:27.084103+0000 mon.a (mon.0) 1192 : cluster [DBG] Standby manager daemon y restarted 2024-09-13T14:30:28.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[32082]: cluster 2024-09-13T14:30:27.084251+0000 mon.a (mon.0) 1193 : cluster [DBG] Standby manager daemon y started 2024-09-13T14:30:28.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[32082]: audit 2024-09-13T14:30:27.086393+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:30:28.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[32082]: audit 2024-09-13T14:30:27.087655+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:28.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[32082]: audit 2024-09-13T14:30:27.090426+0000 mon.c (mon.2) 174 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:30:28.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:27 smithi099 bash[32082]: audit 2024-09-13T14:30:27.090871+0000 mon.c (mon.2) 175 : audit [DBG] from='mgr.? 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:28.627 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:28.767 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:29.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:28 smithi177 bash[24494]: audit 2024-09-13T14:30:27.877150+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.99:0/1531121947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:29.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:28 smithi177 bash[24494]: cluster 2024-09-13T14:30:28.042265+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-09-13T14:30:29.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:28 smithi177 bash[24494]: cluster 2024-09-13T14:30:28.059521+0000 mon.a (mon.0) 1195 : cluster [DBG] mgrmap e27: x(active, since 84s), standbys: y 2024-09-13T14:30:29.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:28 smithi177 bash[24494]: cluster 2024-09-13T14:30:28.073539+0000 mgr.x (mgr.24637) 135 : cluster [DBG] pgmap v126: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 529 op/s 2024-09-13T14:30:29.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:28 smithi177 bash[24494]: audit 2024-09-13T14:30:28.186092+0000 mgr.x (mgr.24637) 136 : audit [DBG] from='client.15879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:29.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:28 smithi177 bash[24494]: audit 2024-09-13T14:30:28.479751+0000 mgr.x (mgr.24637) 137 : audit [DBG] from='client.25778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:29.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:28 smithi177 bash[24494]: cluster 2024-09-13T14:30:28.611137+0000 mon.a (mon.0) 1196 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-09-13T14:30:29.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[22656]: audit 2024-09-13T14:30:27.877150+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.99:0/1531121947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:29.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[22656]: cluster 2024-09-13T14:30:28.042265+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[22656]: cluster 2024-09-13T14:30:28.059521+0000 mon.a (mon.0) 1195 : cluster [DBG] mgrmap e27: x(active, since 84s), standbys: y 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[22656]: cluster 2024-09-13T14:30:28.073539+0000 mgr.x (mgr.24637) 135 : cluster [DBG] pgmap v126: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 529 op/s 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[22656]: audit 2024-09-13T14:30:28.186092+0000 mgr.x (mgr.24637) 136 : audit [DBG] from='client.15879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[22656]: audit 2024-09-13T14:30:28.479751+0000 mgr.x (mgr.24637) 137 : audit [DBG] from='client.25778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[22656]: cluster 2024-09-13T14:30:28.611137+0000 mon.a (mon.0) 1196 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[32082]: audit 2024-09-13T14:30:27.877150+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.99:0/1531121947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[32082]: cluster 2024-09-13T14:30:28.042265+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[32082]: cluster 2024-09-13T14:30:28.059521+0000 mon.a (mon.0) 1195 : cluster [DBG] mgrmap e27: x(active, since 84s), standbys: y 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[32082]: cluster 2024-09-13T14:30:28.073539+0000 mgr.x (mgr.24637) 135 : cluster [DBG] pgmap v126: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 529 op/s 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[32082]: audit 2024-09-13T14:30:28.186092+0000 mgr.x (mgr.24637) 136 : audit [DBG] from='client.15879 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:29.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[32082]: audit 2024-09-13T14:30:28.479751+0000 mgr.x (mgr.24637) 137 : audit [DBG] from='client.25778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:29.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:28 smithi099 bash[32082]: cluster 2024-09-13T14:30:28.611137+0000 mon.a (mon.0) 1196 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-09-13T14:30:30.193 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:30:30.603 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:30 smithi177 bash[59561]: ts=2024-09-13T14:30:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:31.034 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneV2ParentAbort (3956 ms) 2024-09-13T14:30:31.034 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach 2024-09-13T14:30:31.092 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:31.119 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 0~10 2024-09-13T14:30:31.142 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:31.194 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:31.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:30 smithi099 bash[22656]: cluster 2024-09-13T14:30:29.861145+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-09-13T14:30:31.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:30 smithi099 bash[22656]: cluster 2024-09-13T14:30:30.075129+0000 mgr.x (mgr.24637) 138 : cluster [DBG] pgmap v129: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 306 KiB/s rd, 21 KiB/s wr, 392 op/s 2024-09-13T14:30:31.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:30 smithi099 bash[32082]: cluster 2024-09-13T14:30:29.861145+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-09-13T14:30:31.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:30 smithi099 bash[32082]: cluster 2024-09-13T14:30:30.075129+0000 mgr.x (mgr.24637) 138 : cluster [DBG] pgmap v129: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 306 KiB/s rd, 21 KiB/s wr, 392 op/s 2024-09-13T14:30:31.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:30 smithi177 bash[24494]: cluster 2024-09-13T14:30:29.861145+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-09-13T14:30:31.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:30 smithi177 bash[24494]: cluster 2024-09-13T14:30:30.075129+0000 mgr.x (mgr.24637) 138 : cluster [DBG] pgmap v129: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 306 KiB/s rd, 21 KiB/s wr, 392 op/s 2024-09-13T14:30:32.227 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:31 smithi099 bash[22656]: audit 2024-09-13T14:30:30.854923+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.99:0/3516395468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:32.227 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:31 smithi099 bash[22656]: cluster 2024-09-13T14:30:30.867127+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-09-13T14:30:32.227 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:31 smithi099 bash[22656]: audit 2024-09-13T14:30:31.189719+0000 mgr.x (mgr.24637) 139 : audit [DBG] from='client.25790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:32.227 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:31 smithi099 bash[22656]: audit 2024-09-13T14:30:31.502091+0000 mgr.x (mgr.24637) 140 : audit [DBG] from='client.15903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:32.228 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:31 smithi099 bash[32082]: audit 2024-09-13T14:30:30.854923+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.99:0/3516395468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:32.228 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:31 smithi099 bash[32082]: cluster 2024-09-13T14:30:30.867127+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-09-13T14:30:32.228 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:31 smithi099 bash[32082]: audit 2024-09-13T14:30:31.189719+0000 mgr.x (mgr.24637) 139 : audit [DBG] from='client.25790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:32.228 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:31 smithi099 bash[32082]: audit 2024-09-13T14:30:31.502091+0000 mgr.x (mgr.24637) 140 : audit [DBG] from='client.15903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:32.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:31 smithi177 bash[24494]: audit 2024-09-13T14:30:30.854923+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.99:0/3516395468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:32.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:31 smithi177 bash[24494]: cluster 2024-09-13T14:30:30.867127+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-09-13T14:30:32.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:31 smithi177 bash[24494]: audit 2024-09-13T14:30:31.189719+0000 mgr.x (mgr.24637) 139 : audit [DBG] from='client.25790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:32.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:31 smithi177 bash[24494]: audit 2024-09-13T14:30:31.502091+0000 mgr.x (mgr.24637) 140 : audit [DBG] from='client.15903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:32.911 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:33.055 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:33.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:32 smithi099 bash[22656]: cluster 2024-09-13T14:30:31.880910+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-09-13T14:30:33.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:32 smithi099 bash[22656]: cluster 2024-09-13T14:30:32.075934+0000 mgr.x (mgr.24637) 141 : cluster [DBG] pgmap v132: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 306 KiB/s rd, 21 KiB/s wr, 392 op/s 2024-09-13T14:30:33.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:32 smithi099 bash[32082]: cluster 2024-09-13T14:30:31.880910+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-09-13T14:30:33.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:32 smithi099 bash[32082]: cluster 2024-09-13T14:30:32.075934+0000 mgr.x (mgr.24637) 141 : cluster [DBG] pgmap v132: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 306 KiB/s rd, 21 KiB/s wr, 392 op/s 2024-09-13T14:30:33.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:32 smithi177 bash[24494]: cluster 2024-09-13T14:30:31.880910+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-09-13T14:30:33.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:32 smithi177 bash[24494]: cluster 2024-09-13T14:30:32.075934+0000 mgr.x (mgr.24637) 141 : cluster [DBG] pgmap v132: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 306 KiB/s rd, 21 KiB/s wr, 392 op/s 2024-09-13T14:30:33.875 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:30:34.058 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:33 smithi099 bash[22656]: cluster 2024-09-13T14:30:32.891550+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-09-13T14:30:34.058 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:33 smithi099 bash[22656]: cluster 2024-09-13T14:30:33.609081+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-09-13T14:30:34.058 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:33 smithi099 bash[32082]: cluster 2024-09-13T14:30:32.891550+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-09-13T14:30:34.058 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:33 smithi099 bash[32082]: cluster 2024-09-13T14:30:33.609081+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-09-13T14:30:34.147 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:33 smithi177 bash[24494]: cluster 2024-09-13T14:30:32.891550+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-09-13T14:30:34.147 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:33 smithi177 bash[24494]: cluster 2024-09-13T14:30:33.609081+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-09-13T14:30:34.226 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:34.603 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:34 smithi177 bash[59561]: ts=2024-09-13T14:30:34.150Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:35.149 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach (4115 ms) 2024-09-13T14:30:35.149 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixParentReattach 2024-09-13T14:30:35.231 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:35.260 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 0~10 2024-09-13T14:30:35.282 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:35.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:34 smithi099 bash[22656]: audit 2024-09-13T14:30:33.908579+0000 mon.a (mon.0) 1202 : audit [DBG] from='client.? 172.21.15.99:0/838083583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:35.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:34 smithi099 bash[22656]: cluster 2024-09-13T14:30:34.076682+0000 mgr.x (mgr.24637) 142 : cluster [DBG] pgmap v135: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:35.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:34 smithi099 bash[22656]: audit 2024-09-13T14:30:34.223451+0000 mgr.x (mgr.24637) 143 : audit [DBG] from='client.25805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:35.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:34 smithi099 bash[22656]: audit 2024-09-13T14:30:34.536589+0000 mgr.x (mgr.24637) 144 : audit [DBG] from='client.15921 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:35.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:34 smithi099 bash[32082]: audit 2024-09-13T14:30:33.908579+0000 mon.a (mon.0) 1202 : audit [DBG] from='client.? 172.21.15.99:0/838083583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:35.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:34 smithi099 bash[32082]: cluster 2024-09-13T14:30:34.076682+0000 mgr.x (mgr.24637) 142 : cluster [DBG] pgmap v135: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:35.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:34 smithi099 bash[32082]: audit 2024-09-13T14:30:34.223451+0000 mgr.x (mgr.24637) 143 : audit [DBG] from='client.25805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:35.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:34 smithi099 bash[32082]: audit 2024-09-13T14:30:34.536589+0000 mgr.x (mgr.24637) 144 : audit [DBG] from='client.15921 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:35.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:34 smithi177 bash[24494]: audit 2024-09-13T14:30:33.908579+0000 mon.a (mon.0) 1202 : audit [DBG] from='client.? 172.21.15.99:0/838083583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:35.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:34 smithi177 bash[24494]: cluster 2024-09-13T14:30:34.076682+0000 mgr.x (mgr.24637) 142 : cluster [DBG] pgmap v135: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:35.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:34 smithi177 bash[24494]: audit 2024-09-13T14:30:34.223451+0000 mgr.x (mgr.24637) 143 : audit [DBG] from='client.25805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:35.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:34 smithi177 bash[24494]: audit 2024-09-13T14:30:34.536589+0000 mgr.x (mgr.24637) 144 : audit [DBG] from='client.15921 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:36.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:35 smithi177 bash[24494]: cluster 2024-09-13T14:30:34.910985+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-09-13T14:30:36.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:35 smithi177 bash[24494]: audit 2024-09-13T14:30:35.151376+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:36.104 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:35 smithi177 bash[59561]: ts=2024-09-13T14:30:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:36.104 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:35 smithi177 bash[59561]: ts=2024-09-13T14:30:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:36.163 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:35 smithi099 bash[22656]: cluster 2024-09-13T14:30:34.910985+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-09-13T14:30:36.163 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:35 smithi099 bash[22656]: audit 2024-09-13T14:30:35.151376+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:36.163 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:35 smithi099 bash[32082]: cluster 2024-09-13T14:30:34.910985+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-09-13T14:30:36.163 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:35 smithi099 bash[32082]: audit 2024-09-13T14:30:35.151376+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:36.945 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:37.044 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:37.172 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:36 smithi099 bash[22656]: cluster 2024-09-13T14:30:35.915135+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-09-13T14:30:37.173 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:36 smithi099 bash[22656]: cluster 2024-09-13T14:30:36.077465+0000 mgr.x (mgr.24637) 145 : cluster [DBG] pgmap v138: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 351 KiB/s rd, 21 KiB/s wr, 448 op/s 2024-09-13T14:30:37.173 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:36 smithi099 bash[22656]: audit 2024-09-13T14:30:36.904224+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.99:0/3577961701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:37.173 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:36 smithi099 bash[32082]: cluster 2024-09-13T14:30:35.915135+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-09-13T14:30:37.173 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:36 smithi099 bash[32082]: cluster 2024-09-13T14:30:36.077465+0000 mgr.x (mgr.24637) 145 : cluster [DBG] pgmap v138: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 351 KiB/s rd, 21 KiB/s wr, 448 op/s 2024-09-13T14:30:37.173 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:36 smithi099 bash[32082]: audit 2024-09-13T14:30:36.904224+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.99:0/3577961701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:37.275 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:37.353 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:36 smithi177 bash[59561]: ts=2024-09-13T14:30:36.953Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:37.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:36 smithi177 bash[24494]: cluster 2024-09-13T14:30:35.915135+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-09-13T14:30:37.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:36 smithi177 bash[24494]: cluster 2024-09-13T14:30:36.077465+0000 mgr.x (mgr.24637) 145 : cluster [DBG] pgmap v138: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 351 KiB/s rd, 21 KiB/s wr, 448 op/s 2024-09-13T14:30:37.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:36 smithi177 bash[24494]: audit 2024-09-13T14:30:36.904224+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.99:0/3577961701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:37.853 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:37 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:30:37] "GET /metrics HTTP/1.1" 200 38905 "" "Prometheus/2.51.0" 2024-09-13T14:30:38.259 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:37 smithi177 bash[24494]: cluster 2024-09-13T14:30:36.925552+0000 mon.a (mon.0) 1205 : cluster [DBG] mgrmap e28: x(active, since 92s), standbys: y 2024-09-13T14:30:38.259 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:37 smithi177 bash[24494]: cluster 2024-09-13T14:30:36.925689+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-09-13T14:30:38.259 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:37 smithi177 bash[24494]: audit 2024-09-13T14:30:37.271933+0000 mgr.x (mgr.24637) 146 : audit [DBG] from='client.25823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:38.272 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:37 smithi099 bash[22656]: cluster 2024-09-13T14:30:36.925552+0000 mon.a (mon.0) 1205 : cluster [DBG] mgrmap e28: x(active, since 92s), standbys: y 2024-09-13T14:30:38.272 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:37 smithi099 bash[22656]: cluster 2024-09-13T14:30:36.925689+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-09-13T14:30:38.272 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:37 smithi099 bash[22656]: audit 2024-09-13T14:30:37.271933+0000 mgr.x (mgr.24637) 146 : audit [DBG] from='client.25823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:38.272 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:37 smithi099 bash[32082]: cluster 2024-09-13T14:30:36.925552+0000 mon.a (mon.0) 1205 : cluster [DBG] mgrmap e28: x(active, since 92s), standbys: y 2024-09-13T14:30:38.272 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:37 smithi099 bash[32082]: cluster 2024-09-13T14:30:36.925689+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-09-13T14:30:38.272 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:37 smithi099 bash[32082]: audit 2024-09-13T14:30:37.271933+0000 mgr.x (mgr.24637) 146 : audit [DBG] from='client.25823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:38.310 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:30:38.568 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[76324]: [13/Sep/2024:14:30:38] ENGINE Bus STOPPING 2024-09-13T14:30:38.603 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[52886]: debug 2024-09-13T14:30:38.263+0000 7ff3868b6640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T14:30:38.603 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[52886]: ignoring --setuser ceph since I am not root 2024-09-13T14:30:38.604 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[52886]: ignoring --setgroup ceph since I am not root 2024-09-13T14:30:38.604 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[52886]: debug 2024-09-13T14:30:38.327+0000 7fc8ca16f640 1 -- 172.21.15.177:0/4167635286 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5600ee8e90e0 con 0x5600ee8eb400 2024-09-13T14:30:38.822 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixParentReattach (3673 ms) 2024-09-13T14:30:38.822 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneV1ParentAbortRelinkNotNeeded 2024-09-13T14:30:38.888 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:38.910 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 0~10 2024-09-13T14:30:38.940 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:39.068 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[76324]: [13/Sep/2024:14:30:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:30:39.068 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[76324]: [13/Sep/2024:14:30:38] ENGINE Bus STOPPED 2024-09-13T14:30:39.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: cluster 2024-09-13T14:30:37.924495+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-09-13T14:30:39.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.049117+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:39.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.063392+0000 mon.a (mon.0) 1209 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:39.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.070538+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:39.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.072507+0000 mon.b (mon.1) 83 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:30:39.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.079486+0000 mon.a (mon.0) 1210 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:39.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.242065+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:39.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.246180+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: cluster 2024-09-13T14:30:38.247160+0000 mon.a (mon.0) 1212 : cluster [INF] Activating manager daemon y 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.248288+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: cluster 2024-09-13T14:30:38.262102+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.264803+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: cluster 2024-09-13T14:30:38.265093+0000 mon.a (mon.0) 1215 : cluster [DBG] mgrmap e29: y(active, starting, since 0.0179227s) 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.271372+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.271533+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.271669+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.272662+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.outtkv"}]: dispatch 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.272902+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.273656+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:30:39.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.274025+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: cluster 2024-09-13T14:30:37.924495+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.049117+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.063392+0000 mon.a (mon.0) 1209 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.070538+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.072507+0000 mon.b (mon.1) 83 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.079486+0000 mon.a (mon.0) 1210 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.242065+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.246180+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: cluster 2024-09-13T14:30:38.247160+0000 mon.a (mon.0) 1212 : cluster [INF] Activating manager daemon y 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.248288+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:30:39.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: cluster 2024-09-13T14:30:38.262102+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-09-13T14:30:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.264803+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:30:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: cluster 2024-09-13T14:30:38.265093+0000 mon.a (mon.0) 1215 : cluster [DBG] mgrmap e29: y(active, starting, since 0.0179227s) 2024-09-13T14:30:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.271372+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:30:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.271533+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:30:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.271669+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:30:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.272662+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.outtkv"}]: dispatch 2024-09-13T14:30:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.272902+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:30:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.273656+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:30:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.274025+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:30:39.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.274296+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:30:39.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.274508+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:30:39.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.274726+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:30:39.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.274937+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:30:39.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.275156+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:30:39.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.275356+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:30:39.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.275580+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:30:39.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.276731+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:30:39.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.276981+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:30:39.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: audit 2024-09-13T14:30:38.277441+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:30:39.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: cluster 2024-09-13T14:30:38.613718+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-09-13T14:30:39.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[22656]: cluster 2024-09-13T14:30:38.743241+0000 mon.a (mon.0) 1217 : cluster [INF] Manager daemon y is now available 2024-09-13T14:30:39.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.274296+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.274508+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.274726+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.274937+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.275156+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.275356+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.275580+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.276731+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.276981+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: audit 2024-09-13T14:30:38.277441+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: cluster 2024-09-13T14:30:38.613718+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-09-13T14:30:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:38 smithi099 bash[32082]: cluster 2024-09-13T14:30:38.743241+0000 mon.a (mon.0) 1217 : cluster [INF] Manager daemon y is now available 2024-09-13T14:30:39.239 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: cluster 2024-09-13T14:30:37.924495+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-09-13T14:30:39.239 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.049117+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:39.239 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.063392+0000 mon.a (mon.0) 1209 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.070538+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.072507+0000 mon.b (mon.1) 83 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.079486+0000 mon.a (mon.0) 1210 : audit [INF] from='mgr.24637 ' entity='mgr.x' 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.242065+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.246180+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: cluster 2024-09-13T14:30:38.247160+0000 mon.a (mon.0) 1212 : cluster [INF] Activating manager daemon y 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.248288+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.24637 172.21.15.177:0/354594214' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: cluster 2024-09-13T14:30:38.262102+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.264803+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.24637 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: cluster 2024-09-13T14:30:38.265093+0000 mon.a (mon.0) 1215 : cluster [DBG] mgrmap e29: y(active, starting, since 0.0179227s) 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.271372+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.271533+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:30:39.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.271669+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:30:39.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.272662+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.outtkv"}]: dispatch 2024-09-13T14:30:39.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.272902+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"}]: dispatch 2024-09-13T14:30:39.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.273656+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:30:39.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.274025+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:30:39.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.274296+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:30:39.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.274508+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:30:39.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.274726+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:30:39.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.274937+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:30:39.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.275156+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:30:39.242 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.275356+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:30:39.242 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.275580+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:30:39.242 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.276731+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:30:39.242 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.276981+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:30:39.242 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: audit 2024-09-13T14:30:38.277441+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:30:39.242 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: cluster 2024-09-13T14:30:38.613718+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-09-13T14:30:39.242 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:38 smithi177 bash[24494]: cluster 2024-09-13T14:30:38.743241+0000 mon.a (mon.0) 1217 : cluster [INF] Manager daemon y is now available 2024-09-13T14:30:39.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: debug 2024-09-13T14:30:39.130+0000 7fd7a4e2a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:39.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: debug 2024-09-13T14:30:39.130+0000 7fd7a4e2a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:39.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: debug 2024-09-13T14:30:39.130+0000 7fd7a4e2a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:39.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: debug 2024-09-13T14:30:39.130+0000 7fd7a4e2a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:39.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: debug 2024-09-13T14:30:39.130+0000 7fd7a4e2a640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:39.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: debug 2024-09-13T14:30:39.142+0000 7fd7a1623640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:39.570 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: debug 2024-09-13T14:30:39.142+0000 7fd7a1623640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:39.570 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: debug 2024-09-13T14:30:39.142+0000 7fd7a1623640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:39.570 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: debug 2024-09-13T14:30:39.142+0000 7fd7a1623640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:39.570 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: debug 2024-09-13T14:30:39.142+0000 7fd7a1623640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:30:39.930 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: [13/Sep/2024:14:30:39] ENGINE Bus STARTING 2024-09-13T14:30:39.930 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: [13/Sep/2024:14:30:39] ENGINE Serving on http://:::9283 2024-09-13T14:30:39.930 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[76324]: [13/Sep/2024:14:30:39] ENGINE Bus STARTED 2024-09-13T14:30:40.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:39 smithi177 bash[24494]: audit 2024-09-13T14:30:39.055495+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:40.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:39 smithi177 bash[24494]: audit 2024-09-13T14:30:39.085004+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:40.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:39 smithi177 bash[24494]: audit 2024-09-13T14:30:39.085657+0000 mon.b (mon.1) 105 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:40.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:39 smithi177 bash[24494]: audit 2024-09-13T14:30:39.092510+0000 mon.a (mon.0) 1218 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:40.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:39 smithi177 bash[24494]: audit 2024-09-13T14:30:39.094561+0000 mon.b (mon.1) 106 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:40.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:39 smithi177 bash[24494]: audit 2024-09-13T14:30:39.123845+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:40.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:39 smithi177 bash[24494]: audit 2024-09-13T14:30:39.125901+0000 mon.b (mon.1) 107 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:40.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:39 smithi177 bash[24494]: cluster 2024-09-13T14:30:39.619036+0000 mon.a (mon.0) 1220 : cluster [DBG] mgrmap e30: y(active, since 1.37187s) 2024-09-13T14:30:40.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:39 smithi177 bash[24494]: audit 2024-09-13T14:30:39.657559+0000 mgr.y (mgr.25724) 1 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:40.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:39 smithi177 bash[24494]: cluster 2024-09-13T14:30:39.659968+0000 mgr.y (mgr.25724) 2 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[32082]: audit 2024-09-13T14:30:39.055495+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[32082]: audit 2024-09-13T14:30:39.085004+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[32082]: audit 2024-09-13T14:30:39.085657+0000 mon.b (mon.1) 105 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[32082]: audit 2024-09-13T14:30:39.092510+0000 mon.a (mon.0) 1218 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[32082]: audit 2024-09-13T14:30:39.094561+0000 mon.b (mon.1) 106 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[32082]: audit 2024-09-13T14:30:39.123845+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[32082]: audit 2024-09-13T14:30:39.125901+0000 mon.b (mon.1) 107 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[32082]: cluster 2024-09-13T14:30:39.619036+0000 mon.a (mon.0) 1220 : cluster [DBG] mgrmap e30: y(active, since 1.37187s) 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[32082]: audit 2024-09-13T14:30:39.657559+0000 mgr.y (mgr.25724) 1 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:40.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[32082]: cluster 2024-09-13T14:30:39.659968+0000 mgr.y (mgr.25724) 2 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:40.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[22656]: audit 2024-09-13T14:30:39.055495+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:30:40.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[22656]: audit 2024-09-13T14:30:39.085004+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:40.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[22656]: audit 2024-09-13T14:30:39.085657+0000 mon.b (mon.1) 105 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:40.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[22656]: audit 2024-09-13T14:30:39.092510+0000 mon.a (mon.0) 1218 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:40.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[22656]: audit 2024-09-13T14:30:39.094561+0000 mon.b (mon.1) 106 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:30:40.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[22656]: audit 2024-09-13T14:30:39.123845+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:40.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[22656]: audit 2024-09-13T14:30:39.125901+0000 mon.b (mon.1) 107 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:30:40.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[22656]: cluster 2024-09-13T14:30:39.619036+0000 mon.a (mon.0) 1220 : cluster [DBG] mgrmap e30: y(active, since 1.37187s) 2024-09-13T14:30:40.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[22656]: audit 2024-09-13T14:30:39.657559+0000 mgr.y (mgr.25724) 1 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:30:40.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:39 smithi099 bash[22656]: cluster 2024-09-13T14:30:39.659968+0000 mgr.y (mgr.25724) 2 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:40.359 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:40.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: debug 2024-09-13T14:30:40.218+0000 7fd788b75640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:30:40] ENGINE Error in HTTPServer.serve 2024-09-13T14:30:40.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: Traceback (most recent call last): 2024-09-13T14:30:40.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:30:40.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: self._connections.run(self.expiration_interval) 2024-09-13T14:30:40.570 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:30:40.570 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: self._run(expiration_interval) 2024-09-13T14:30:40.570 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:30:40.570 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:30:40.570 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:30:40.570 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:30:40.571 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:30:40.571 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: s = self.context.wrap_socket( 2024-09-13T14:30:40.571 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:30:40.571 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: return self.sslsocket_class._create( 2024-09-13T14:30:40.571 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:30:40.571 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: self.do_handshake() 2024-09-13T14:30:40.572 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:30:40.572 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: self._sslobj.do_handshake() 2024-09-13T14:30:40.572 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[76324]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:30:40.603 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[59561]: ts=2024-09-13T14:30:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:40.982 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:41.103 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:41.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: cluster 2024-09-13T14:30:39.945154+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-09-13T14:30:41.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: audit 2024-09-13T14:30:40.070197+0000 mon.a (mon.0) 1222 : audit [DBG] from='client.? 172.21.15.99:0/1805190326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:41.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: cephadm 2024-09-13T14:30:40.102594+0000 mgr.y (mgr.25724) 3 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Bus STARTING 2024-09-13T14:30:41.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: cephadm 2024-09-13T14:30:40.221241+0000 mgr.y (mgr.25724) 4 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:30:41.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: cephadm 2024-09-13T14:30:40.224020+0000 mgr.y (mgr.25724) 5 : cephadm [ERR] [13/Sep/2024:14:30:40] ENGINE Error in HTTPServer.serve 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: Traceback (most recent call last): 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: self._connections.run(self.expiration_interval) 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: self._run(expiration_interval) 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: s = self.context.wrap_socket( 2024-09-13T14:30:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: return self.sslsocket_class._create( 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: self.do_handshake() 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: self._sslobj.do_handshake() 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: cluster 2024-09-13T14:30:40.273817+0000 mgr.y (mgr.25724) 6 : cluster [DBG] pgmap v6: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: cephadm 2024-09-13T14:30:40.322459+0000 mgr.y (mgr.25724) 7 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Serving on http://172.21.15.99:8765 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: cephadm 2024-09-13T14:30:40.322627+0000 mgr.y (mgr.25724) 8 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Bus STARTED 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: audit 2024-09-13T14:30:40.353916+0000 mgr.y (mgr.25724) 9 : audit [DBG] from='client.15987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[22656]: audit 2024-09-13T14:30:40.642665+0000 mgr.y (mgr.25724) 10 : audit [DBG] from='client.25883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:41.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: cluster 2024-09-13T14:30:39.945154+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: audit 2024-09-13T14:30:40.070197+0000 mon.a (mon.0) 1222 : audit [DBG] from='client.? 172.21.15.99:0/1805190326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: cephadm 2024-09-13T14:30:40.102594+0000 mgr.y (mgr.25724) 3 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Bus STARTING 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: cephadm 2024-09-13T14:30:40.221241+0000 mgr.y (mgr.25724) 4 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: cephadm 2024-09-13T14:30:40.224020+0000 mgr.y (mgr.25724) 5 : cephadm [ERR] [13/Sep/2024:14:30:40] ENGINE Error in HTTPServer.serve 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: Traceback (most recent call last): 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: self._connections.run(self.expiration_interval) 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: self._run(expiration_interval) 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:30:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: s = self.context.wrap_socket( 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: return self.sslsocket_class._create( 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: self.do_handshake() 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: self._sslobj.do_handshake() 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: cluster 2024-09-13T14:30:40.273817+0000 mgr.y (mgr.25724) 6 : cluster [DBG] pgmap v6: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: cephadm 2024-09-13T14:30:40.322459+0000 mgr.y (mgr.25724) 7 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Serving on http://172.21.15.99:8765 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: cephadm 2024-09-13T14:30:40.322627+0000 mgr.y (mgr.25724) 8 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Bus STARTED 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: audit 2024-09-13T14:30:40.353916+0000 mgr.y (mgr.25724) 9 : audit [DBG] from='client.15987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:40 smithi099 bash[32082]: audit 2024-09-13T14:30:40.642665+0000 mgr.y (mgr.25724) 10 : audit [DBG] from='client.25883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:41.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: cluster 2024-09-13T14:30:39.945154+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-09-13T14:30:41.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: audit 2024-09-13T14:30:40.070197+0000 mon.a (mon.0) 1222 : audit [DBG] from='client.? 172.21.15.99:0/1805190326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:41.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: cephadm 2024-09-13T14:30:40.102594+0000 mgr.y (mgr.25724) 3 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Bus STARTING 2024-09-13T14:30:41.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: cephadm 2024-09-13T14:30:40.221241+0000 mgr.y (mgr.25724) 4 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:30:41.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: cephadm 2024-09-13T14:30:40.224020+0000 mgr.y (mgr.25724) 5 : cephadm [ERR] [13/Sep/2024:14:30:40] ENGINE Error in HTTPServer.serve 2024-09-13T14:30:41.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: Traceback (most recent call last): 2024-09-13T14:30:41.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:30:41.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: self._connections.run(self.expiration_interval) 2024-09-13T14:30:41.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: self._run(expiration_interval) 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: s = self.context.wrap_socket( 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: return self.sslsocket_class._create( 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: self.do_handshake() 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:30:41.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: self._sslobj.do_handshake() 2024-09-13T14:30:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:30:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: cluster 2024-09-13T14:30:40.273817+0000 mgr.y (mgr.25724) 6 : cluster [DBG] pgmap v6: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: cephadm 2024-09-13T14:30:40.322459+0000 mgr.y (mgr.25724) 7 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Serving on http://172.21.15.99:8765 2024-09-13T14:30:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: cephadm 2024-09-13T14:30:40.322627+0000 mgr.y (mgr.25724) 8 : cephadm [INF] [13/Sep/2024:14:30:40] ENGINE Bus STARTED 2024-09-13T14:30:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: audit 2024-09-13T14:30:40.353916+0000 mgr.y (mgr.25724) 9 : audit [DBG] from='client.15987 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:40 smithi177 bash[24494]: audit 2024-09-13T14:30:40.642665+0000 mgr.y (mgr.25724) 10 : audit [DBG] from='client.25883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:42.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:41 smithi099 bash[22656]: cluster 2024-09-13T14:30:40.962374+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-09-13T14:30:42.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:41 smithi099 bash[32082]: cluster 2024-09-13T14:30:40.962374+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-09-13T14:30:42.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:41 smithi177 bash[24494]: cluster 2024-09-13T14:30:40.962374+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-09-13T14:30:42.370 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:30:43.154 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneV1ParentAbortRelinkNotNeeded (4331 ms) 2024-09-13T14:30:43.154 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach 2024-09-13T14:30:43.204 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:43.224 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 0~10 2024-09-13T14:30:43.241 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:43.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:42 smithi099 bash[22656]: cluster 2024-09-13T14:30:41.960130+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-09-13T14:30:43.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:42 smithi099 bash[22656]: cluster 2024-09-13T14:30:41.974829+0000 mon.a (mon.0) 1225 : cluster [DBG] mgrmap e31: y(active, since 3s) 2024-09-13T14:30:43.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:42 smithi099 bash[22656]: cluster 2024-09-13T14:30:42.274541+0000 mgr.y (mgr.25724) 11 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:42 smithi099 bash[32082]: cluster 2024-09-13T14:30:41.960130+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-09-13T14:30:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:42 smithi099 bash[32082]: cluster 2024-09-13T14:30:41.974829+0000 mon.a (mon.0) 1225 : cluster [DBG] mgrmap e31: y(active, since 3s) 2024-09-13T14:30:43.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:42 smithi099 bash[32082]: cluster 2024-09-13T14:30:42.274541+0000 mgr.y (mgr.25724) 11 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:43.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:42 smithi177 bash[24494]: cluster 2024-09-13T14:30:41.960130+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-09-13T14:30:43.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:42 smithi177 bash[24494]: cluster 2024-09-13T14:30:41.974829+0000 mon.a (mon.0) 1225 : cluster [DBG] mgrmap e31: y(active, since 3s) 2024-09-13T14:30:43.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:42 smithi177 bash[24494]: cluster 2024-09-13T14:30:42.274541+0000 mgr.y (mgr.25724) 11 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:43.368 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:44.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:43 smithi099 bash[22656]: cluster 2024-09-13T14:30:42.982682+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-09-13T14:30:44.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:43 smithi099 bash[22656]: audit 2024-09-13T14:30:43.032047+0000 mon.a (mon.0) 1227 : audit [DBG] from='client.? 172.21.15.99:0/1206455858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:44.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:43 smithi099 bash[22656]: audit 2024-09-13T14:30:43.361672+0000 mgr.y (mgr.25724) 12 : audit [DBG] from='client.16011 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:44.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:43 smithi099 bash[22656]: cluster 2024-09-13T14:30:43.615316+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-09-13T14:30:44.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:43 smithi099 bash[22656]: audit 2024-09-13T14:30:43.676636+0000 mgr.y (mgr.25724) 13 : audit [DBG] from='client.16017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:44.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:43 smithi099 bash[32082]: cluster 2024-09-13T14:30:42.982682+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-09-13T14:30:44.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:43 smithi099 bash[32082]: audit 2024-09-13T14:30:43.032047+0000 mon.a (mon.0) 1227 : audit [DBG] from='client.? 172.21.15.99:0/1206455858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:44.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:43 smithi099 bash[32082]: audit 2024-09-13T14:30:43.361672+0000 mgr.y (mgr.25724) 12 : audit [DBG] from='client.16011 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:44.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:43 smithi099 bash[32082]: cluster 2024-09-13T14:30:43.615316+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-09-13T14:30:44.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:43 smithi099 bash[32082]: audit 2024-09-13T14:30:43.676636+0000 mgr.y (mgr.25724) 13 : audit [DBG] from='client.16017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:43 smithi177 bash[24494]: cluster 2024-09-13T14:30:42.982682+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-09-13T14:30:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:43 smithi177 bash[24494]: audit 2024-09-13T14:30:43.032047+0000 mon.a (mon.0) 1227 : audit [DBG] from='client.? 172.21.15.99:0/1206455858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:44.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:43 smithi177 bash[24494]: audit 2024-09-13T14:30:43.361672+0000 mgr.y (mgr.25724) 12 : audit [DBG] from='client.16011 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:44.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:43 smithi177 bash[24494]: cluster 2024-09-13T14:30:43.615316+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-09-13T14:30:44.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:43 smithi177 bash[24494]: audit 2024-09-13T14:30:43.676636+0000 mgr.y (mgr.25724) 13 : audit [DBG] from='client.16017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:44.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:44 smithi177 bash[59561]: ts=2024-09-13T14:30:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:44.634 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:44.734 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:45.103 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:44 smithi177 bash[52886]: [13/Sep/2024:14:30:44] ENGINE Bus STARTING 2024-09-13T14:30:45.103 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:44 smithi177 bash[52886]: CherryPy Checker: 2024-09-13T14:30:45.103 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:44 smithi177 bash[52886]: The Application mounted at '' has an empty config. 2024-09-13T14:30:45.103 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:44 smithi177 bash[52886]: [13/Sep/2024:14:30:44] ENGINE Serving on http://:::9283 2024-09-13T14:30:45.103 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:44 smithi177 bash[52886]: [13/Sep/2024:14:30:44] ENGINE Bus STARTED 2024-09-13T14:30:45.887 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:30:46.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: cluster 2024-09-13T14:30:44.275279+0000 mgr.y (mgr.25724) 14 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:46.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: cluster 2024-09-13T14:30:44.618643+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: cluster 2024-09-13T14:30:44.625244+0000 mon.a (mon.0) 1230 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: audit 2024-09-13T14:30:44.626356+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: audit 2024-09-13T14:30:44.627455+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: audit 2024-09-13T14:30:44.629725+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: audit 2024-09-13T14:30:44.630056+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: audit 2024-09-13T14:30:45.426081+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: audit 2024-09-13T14:30:45.437924+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: audit 2024-09-13T14:30:45.543983+0000 mon.a (mon.0) 1233 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[22656]: audit 2024-09-13T14:30:45.556624+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: cluster 2024-09-13T14:30:44.275279+0000 mgr.y (mgr.25724) 14 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:46.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: cluster 2024-09-13T14:30:44.618643+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-09-13T14:30:46.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: cluster 2024-09-13T14:30:44.625244+0000 mon.a (mon.0) 1230 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:30:46.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: audit 2024-09-13T14:30:44.626356+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:30:46.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: audit 2024-09-13T14:30:44.627455+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:46.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: audit 2024-09-13T14:30:44.629725+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:30:46.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: audit 2024-09-13T14:30:44.630056+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:46.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: audit 2024-09-13T14:30:45.426081+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: audit 2024-09-13T14:30:45.437924+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: audit 2024-09-13T14:30:45.543983+0000 mon.a (mon.0) 1233 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:45 smithi099 bash[32082]: audit 2024-09-13T14:30:45.556624+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.103 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[59561]: ts=2024-09-13T14:30:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:46.103 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[59561]: ts=2024-09-13T14:30:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: cluster 2024-09-13T14:30:44.275279+0000 mgr.y (mgr.25724) 14 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: cluster 2024-09-13T14:30:44.618643+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: cluster 2024-09-13T14:30:44.625244+0000 mon.a (mon.0) 1230 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: audit 2024-09-13T14:30:44.626356+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: audit 2024-09-13T14:30:44.627455+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: audit 2024-09-13T14:30:44.629725+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: audit 2024-09-13T14:30:44.630056+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.? 172.21.15.177:0/1102445589' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: audit 2024-09-13T14:30:45.426081+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: audit 2024-09-13T14:30:45.437924+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: audit 2024-09-13T14:30:45.543983+0000 mon.a (mon.0) 1233 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:45 smithi177 bash[24494]: audit 2024-09-13T14:30:45.556624+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:46.382 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:46.639 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:46 smithi099 bash[22656]: cluster 2024-09-13T14:30:45.622383+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-09-13T14:30:46.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:46 smithi099 bash[22656]: cluster 2024-09-13T14:30:45.656070+0000 mon.a (mon.0) 1236 : cluster [DBG] mgrmap e32: y(active, since 7s), standbys: x 2024-09-13T14:30:46.640 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:46 smithi099 bash[32082]: cluster 2024-09-13T14:30:45.622383+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-09-13T14:30:46.640 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:46 smithi099 bash[32082]: cluster 2024-09-13T14:30:45.656070+0000 mon.a (mon.0) 1236 : cluster [DBG] mgrmap e32: y(active, since 7s), standbys: x 2024-09-13T14:30:46.804 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach (3650 ms) 2024-09-13T14:30:46.804 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixParentReattach 2024-09-13T14:30:46.865 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:46.888 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 0~10 2024-09-13T14:30:46.907 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:46.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:46 smithi177 bash[24494]: cluster 2024-09-13T14:30:45.622383+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-09-13T14:30:46.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:46 smithi177 bash[24494]: cluster 2024-09-13T14:30:45.656070+0000 mon.a (mon.0) 1236 : cluster [DBG] mgrmap e32: y(active, since 7s), standbys: x 2024-09-13T14:30:46.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:46 smithi177 bash[24494]: audit 2024-09-13T14:30:45.656700+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:30:46.944 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:46 smithi177 bash[24494]: audit 2024-09-13T14:30:46.061769+0000 mon.a (mon.0) 1237 : audit [DBG] from='client.? 172.21.15.99:0/761768212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:47.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:46 smithi099 bash[32082]: audit 2024-09-13T14:30:45.656700+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:30:47.068 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:46 smithi099 bash[32082]: audit 2024-09-13T14:30:46.061769+0000 mon.a (mon.0) 1237 : audit [DBG] from='client.? 172.21.15.99:0/761768212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:47.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:46 smithi099 bash[22656]: audit 2024-09-13T14:30:45.656700+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:30:47.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:46 smithi099 bash[22656]: audit 2024-09-13T14:30:46.061769+0000 mon.a (mon.0) 1237 : audit [DBG] from='client.? 172.21.15.99:0/761768212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:47.353 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:46 smithi177 bash[59561]: ts=2024-09-13T14:30:46.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:47.638 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:47 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:30:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-09-13T14:30:48.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[22656]: cluster 2024-09-13T14:30:46.276014+0000 mgr.y (mgr.25724) 15 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 324 KiB/s rd, 19 KiB/s wr, 377 op/s 2024-09-13T14:30:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[22656]: audit 2024-09-13T14:30:46.376260+0000 mgr.y (mgr.25724) 16 : audit [DBG] from='client.16029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[22656]: cluster 2024-09-13T14:30:46.651358+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-09-13T14:30:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[22656]: audit 2024-09-13T14:30:46.686258+0000 mgr.y (mgr.25724) 17 : audit [DBG] from='client.25925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[22656]: audit 2024-09-13T14:30:46.705074+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[22656]: audit 2024-09-13T14:30:46.713879+0000 mon.a (mon.0) 1240 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[22656]: audit 2024-09-13T14:30:46.743674+0000 mon.a (mon.0) 1241 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[22656]: audit 2024-09-13T14:30:46.753347+0000 mon.a (mon.0) 1242 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[32082]: cluster 2024-09-13T14:30:46.276014+0000 mgr.y (mgr.25724) 15 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 324 KiB/s rd, 19 KiB/s wr, 377 op/s 2024-09-13T14:30:48.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[32082]: audit 2024-09-13T14:30:46.376260+0000 mgr.y (mgr.25724) 16 : audit [DBG] from='client.16029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:48.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[32082]: cluster 2024-09-13T14:30:46.651358+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-09-13T14:30:48.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[32082]: audit 2024-09-13T14:30:46.686258+0000 mgr.y (mgr.25724) 17 : audit [DBG] from='client.25925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:48.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[32082]: audit 2024-09-13T14:30:46.705074+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[32082]: audit 2024-09-13T14:30:46.713879+0000 mon.a (mon.0) 1240 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[32082]: audit 2024-09-13T14:30:46.743674+0000 mon.a (mon.0) 1241 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:47 smithi099 bash[32082]: audit 2024-09-13T14:30:46.753347+0000 mon.a (mon.0) 1242 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:47 smithi177 bash[24494]: cluster 2024-09-13T14:30:46.276014+0000 mgr.y (mgr.25724) 15 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 324 KiB/s rd, 19 KiB/s wr, 377 op/s 2024-09-13T14:30:48.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:47 smithi177 bash[24494]: audit 2024-09-13T14:30:46.376260+0000 mgr.y (mgr.25724) 16 : audit [DBG] from='client.16029 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:48.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:47 smithi177 bash[24494]: cluster 2024-09-13T14:30:46.651358+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-09-13T14:30:48.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:47 smithi177 bash[24494]: audit 2024-09-13T14:30:46.686258+0000 mgr.y (mgr.25724) 17 : audit [DBG] from='client.25925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:48.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:47 smithi177 bash[24494]: audit 2024-09-13T14:30:46.705074+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:47 smithi177 bash[24494]: audit 2024-09-13T14:30:46.713879+0000 mon.a (mon.0) 1240 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:47 smithi177 bash[24494]: audit 2024-09-13T14:30:46.743674+0000 mon.a (mon.0) 1241 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:47 smithi177 bash[24494]: audit 2024-09-13T14:30:46.753347+0000 mon.a (mon.0) 1242 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:48.642 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:48.772 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:49.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:48 smithi099 bash[22656]: cluster 2024-09-13T14:30:47.759699+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-09-13T14:30:49.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:48 smithi099 bash[22656]: cluster 2024-09-13T14:30:48.276803+0000 mgr.y (mgr.25724) 18 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 324 KiB/s rd, 19 KiB/s wr, 377 op/s 2024-09-13T14:30:49.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:48 smithi099 bash[22656]: cluster 2024-09-13T14:30:48.618271+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-09-13T14:30:49.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:48 smithi099 bash[32082]: cluster 2024-09-13T14:30:47.759699+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-09-13T14:30:49.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:48 smithi099 bash[32082]: cluster 2024-09-13T14:30:48.276803+0000 mgr.y (mgr.25724) 18 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 324 KiB/s rd, 19 KiB/s wr, 377 op/s 2024-09-13T14:30:49.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:48 smithi099 bash[32082]: cluster 2024-09-13T14:30:48.618271+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-09-13T14:30:49.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:48 smithi177 bash[24494]: cluster 2024-09-13T14:30:47.759699+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-09-13T14:30:49.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:48 smithi177 bash[24494]: cluster 2024-09-13T14:30:48.276803+0000 mgr.y (mgr.25724) 18 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 324 KiB/s rd, 19 KiB/s wr, 377 op/s 2024-09-13T14:30:49.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:48 smithi177 bash[24494]: cluster 2024-09-13T14:30:48.618271+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-09-13T14:30:49.410 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:50.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:49 smithi099 bash[22656]: audit 2024-09-13T14:30:49.083230+0000 mon.c (mon.2) 179 : audit [DBG] from='client.? 172.21.15.99:0/2090073955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:50.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:49 smithi099 bash[22656]: audit 2024-09-13T14:30:49.403400+0000 mgr.y (mgr.25724) 19 : audit [DBG] from='client.16047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:50.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:49 smithi099 bash[32082]: audit 2024-09-13T14:30:49.083230+0000 mon.c (mon.2) 179 : audit [DBG] from='client.? 172.21.15.99:0/2090073955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:50.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:49 smithi099 bash[32082]: audit 2024-09-13T14:30:49.403400+0000 mgr.y (mgr.25724) 19 : audit [DBG] from='client.16047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:50.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:49 smithi177 bash[24494]: audit 2024-09-13T14:30:49.083230+0000 mon.c (mon.2) 179 : audit [DBG] from='client.? 172.21.15.99:0/2090073955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:50.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:49 smithi177 bash[24494]: audit 2024-09-13T14:30:49.403400+0000 mgr.y (mgr.25724) 19 : audit [DBG] from='client.16047 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:50.145 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:30:50.603 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:50 smithi177 bash[59561]: ts=2024-09-13T14:30:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:51.066 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixParentReattach (4262 ms) 2024-09-13T14:30:51.066 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.CloneV2ParentAbortRelinkNotNeeded 2024-09-13T14:30:51.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:50 smithi099 bash[22656]: audit 2024-09-13T14:30:49.716790+0000 mgr.y (mgr.25724) 20 : audit [DBG] from='client.16053 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:51.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:50 smithi099 bash[22656]: cluster 2024-09-13T14:30:49.783022+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-09-13T14:30:51.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:50 smithi099 bash[22656]: cluster 2024-09-13T14:30:50.277628+0000 mgr.y (mgr.25724) 21 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 25 KiB/s wr, 406 op/s 2024-09-13T14:30:51.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:50 smithi099 bash[32082]: audit 2024-09-13T14:30:49.716790+0000 mgr.y (mgr.25724) 20 : audit [DBG] from='client.16053 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:51.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:50 smithi099 bash[32082]: cluster 2024-09-13T14:30:49.783022+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-09-13T14:30:51.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:50 smithi099 bash[32082]: cluster 2024-09-13T14:30:50.277628+0000 mgr.y (mgr.25724) 21 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 25 KiB/s wr, 406 op/s 2024-09-13T14:30:51.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:50 smithi177 bash[24494]: audit 2024-09-13T14:30:49.716790+0000 mgr.y (mgr.25724) 20 : audit [DBG] from='client.16053 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:51.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:50 smithi177 bash[24494]: cluster 2024-09-13T14:30:49.783022+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-09-13T14:30:51.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:50 smithi177 bash[24494]: cluster 2024-09-13T14:30:50.277628+0000 mgr.y (mgr.25724) 21 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 25 KiB/s wr, 406 op/s 2024-09-13T14:30:51.149 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:51.179 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 0~10 2024-09-13T14:30:51.204 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:52.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:51 smithi099 bash[32082]: cluster 2024-09-13T14:30:50.801402+0000 mon.a (mon.0) 1246 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-09-13T14:30:52.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:51 smithi099 bash[22656]: cluster 2024-09-13T14:30:50.801402+0000 mon.a (mon.0) 1246 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-09-13T14:30:52.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:51 smithi177 bash[24494]: cluster 2024-09-13T14:30:50.801402+0000 mon.a (mon.0) 1246 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-09-13T14:30:52.416 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:52.825 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:52.979 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:30:53.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:52 smithi099 bash[22656]: cluster 2024-09-13T14:30:51.805196+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-09-13T14:30:53.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:52 smithi099 bash[22656]: audit 2024-09-13T14:30:52.098874+0000 mon.a (mon.0) 1248 : audit [DBG] from='client.? 172.21.15.99:0/479199240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:53.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:52 smithi099 bash[22656]: cluster 2024-09-13T14:30:52.278110+0000 mgr.y (mgr.25724) 22 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 25 KiB/s wr, 406 op/s 2024-09-13T14:30:53.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:52 smithi099 bash[22656]: audit 2024-09-13T14:30:52.410165+0000 mgr.y (mgr.25724) 23 : audit [DBG] from='client.16065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:53.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:52 smithi099 bash[32082]: cluster 2024-09-13T14:30:51.805196+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-09-13T14:30:53.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:52 smithi099 bash[32082]: audit 2024-09-13T14:30:52.098874+0000 mon.a (mon.0) 1248 : audit [DBG] from='client.? 172.21.15.99:0/479199240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:53.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:52 smithi099 bash[32082]: cluster 2024-09-13T14:30:52.278110+0000 mgr.y (mgr.25724) 22 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 25 KiB/s wr, 406 op/s 2024-09-13T14:30:53.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:52 smithi099 bash[32082]: audit 2024-09-13T14:30:52.410165+0000 mgr.y (mgr.25724) 23 : audit [DBG] from='client.16065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:53.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:52 smithi177 bash[24494]: cluster 2024-09-13T14:30:51.805196+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-09-13T14:30:53.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:52 smithi177 bash[24494]: audit 2024-09-13T14:30:52.098874+0000 mon.a (mon.0) 1248 : audit [DBG] from='client.? 172.21.15.99:0/479199240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:53.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:52 smithi177 bash[24494]: cluster 2024-09-13T14:30:52.278110+0000 mgr.y (mgr.25724) 22 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 313 KiB/s rd, 25 KiB/s wr, 406 op/s 2024-09-13T14:30:53.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:52 smithi177 bash[24494]: audit 2024-09-13T14:30:52.410165+0000 mgr.y (mgr.25724) 23 : audit [DBG] from='client.16065 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:53.992 INFO:tasks.workunit.client.0.smithi099.stdout:migration_abort 2024-09-13T14:30:54.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:53 smithi099 bash[22656]: audit 2024-09-13T14:30:52.727797+0000 mgr.y (mgr.25724) 24 : audit [DBG] from='client.25961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:54.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:53 smithi099 bash[22656]: cluster 2024-09-13T14:30:52.811076+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-09-13T14:30:54.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:53 smithi099 bash[22656]: cluster 2024-09-13T14:30:53.620360+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-09-13T14:30:54.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:53 smithi099 bash[32082]: audit 2024-09-13T14:30:52.727797+0000 mgr.y (mgr.25724) 24 : audit [DBG] from='client.25961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:54.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:53 smithi099 bash[32082]: cluster 2024-09-13T14:30:52.811076+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-09-13T14:30:54.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:53 smithi099 bash[32082]: cluster 2024-09-13T14:30:53.620360+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-09-13T14:30:54.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:53 smithi177 bash[24494]: audit 2024-09-13T14:30:52.727797+0000 mgr.y (mgr.25724) 24 : audit [DBG] from='client.25961 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:54.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:53 smithi177 bash[24494]: cluster 2024-09-13T14:30:52.811076+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-09-13T14:30:54.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:53 smithi177 bash[24494]: cluster 2024-09-13T14:30:53.620360+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-09-13T14:30:55.009 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.CloneV2ParentAbortRelinkNotNeeded (3943 ms) 2024-09-13T14:30:55.009 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.StressNoMigrate 2024-09-13T14:30:55.064 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:30:55.089 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 53862828~7454406 2024-09-13T14:30:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:54 smithi177 bash[24494]: audit 2024-09-13T14:30:54.085125+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:55.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:54 smithi177 bash[24494]: cluster 2024-09-13T14:30:54.278763+0000 mgr.y (mgr.25724) 25 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:55.182 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 20776411~1527654 2024-09-13T14:30:55.182 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 62794905~3019967 2024-09-13T14:30:55.222 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 70770876~935553 2024-09-13T14:30:55.223 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 36600121~428403 2024-09-13T14:30:55.236 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 62224544~7907049 2024-09-13T14:30:55.251 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 80088337~14058 2024-09-13T14:30:55.259 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 49096043~4402355 2024-09-13T14:30:55.263 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap0 2024-09-13T14:30:55.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:54 smithi099 bash[22656]: audit 2024-09-13T14:30:54.085125+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:55.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:54 smithi099 bash[22656]: cluster 2024-09-13T14:30:54.278763+0000 mgr.y (mgr.25724) 25 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:55.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:54 smithi099 bash[32082]: audit 2024-09-13T14:30:54.085125+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:30:55.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:54 smithi099 bash[32082]: cluster 2024-09-13T14:30:54.278763+0000 mgr.y (mgr.25724) 25 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:30:55.460 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:55 smithi177 bash[24494]: cluster 2024-09-13T14:30:54.834773+0000 mon.a (mon.0) 1251 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-09-13T14:30:56.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:55 smithi177 bash[24494]: audit 2024-09-13T14:30:55.143271+0000 mon.a (mon.0) 1252 : audit [DBG] from='client.? 172.21.15.99:0/2740180523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:56.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:55 smithi177 bash[24494]: audit 2024-09-13T14:30:55.452566+0000 mgr.y (mgr.25724) 26 : audit [DBG] from='client.25973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:56.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:55 smithi099 bash[22656]: cluster 2024-09-13T14:30:54.834773+0000 mon.a (mon.0) 1251 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-09-13T14:30:56.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:55 smithi099 bash[22656]: audit 2024-09-13T14:30:55.143271+0000 mon.a (mon.0) 1252 : audit [DBG] from='client.? 172.21.15.99:0/2740180523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:56.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:55 smithi099 bash[22656]: audit 2024-09-13T14:30:55.452566+0000 mgr.y (mgr.25724) 26 : audit [DBG] from='client.25973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:56.142 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:55 smithi099 bash[32082]: cluster 2024-09-13T14:30:54.834773+0000 mon.a (mon.0) 1251 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-09-13T14:30:56.142 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:55 smithi099 bash[32082]: audit 2024-09-13T14:30:55.143271+0000 mon.a (mon.0) 1252 : audit [DBG] from='client.? 172.21.15.99:0/2740180523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:56.142 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:55 smithi099 bash[32082]: audit 2024-09-13T14:30:55.452566+0000 mgr.y (mgr.25724) 26 : audit [DBG] from='client.25973 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:56.860 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap0 2024-09-13T14:30:56.879 INFO:tasks.workunit.client.0.smithi099.stdout:clone image157 -> image160 2024-09-13T14:30:57.016 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 105103355 2024-09-13T14:30:57.036 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 98223756~1850708 2024-09-13T14:30:57.071 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 75390662~6674582 2024-09-13T14:30:57.091 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 42277855~6965999 2024-09-13T14:30:57.104 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:30:56 smithi177 bash[59561]: ts=2024-09-13T14:30:56.952Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:30:57.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:56 smithi177 bash[24494]: audit 2024-09-13T14:30:55.749145+0000 mgr.y (mgr.25724) 27 : audit [DBG] from='client.25979 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:57.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:56 smithi177 bash[24494]: cluster 2024-09-13T14:30:55.836835+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-09-13T14:30:57.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:56 smithi177 bash[24494]: cluster 2024-09-13T14:30:56.279217+0000 mgr.y (mgr.25724) 28 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 361 KiB/s rd, 27 KiB/s wr, 462 op/s 2024-09-13T14:30:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:56 smithi099 bash[22656]: audit 2024-09-13T14:30:55.749145+0000 mgr.y (mgr.25724) 27 : audit [DBG] from='client.25979 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:56 smithi099 bash[22656]: cluster 2024-09-13T14:30:55.836835+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-09-13T14:30:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:56 smithi099 bash[22656]: cluster 2024-09-13T14:30:56.279217+0000 mgr.y (mgr.25724) 28 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 361 KiB/s rd, 27 KiB/s wr, 462 op/s 2024-09-13T14:30:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:56 smithi099 bash[32082]: audit 2024-09-13T14:30:55.749145+0000 mgr.y (mgr.25724) 27 : audit [DBG] from='client.25979 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:56 smithi099 bash[32082]: cluster 2024-09-13T14:30:55.836835+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-09-13T14:30:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:56 smithi099 bash[32082]: cluster 2024-09-13T14:30:56.279217+0000 mgr.y (mgr.25724) 28 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 361 KiB/s rd, 27 KiB/s wr, 462 op/s 2024-09-13T14:30:57.158 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 96832587~8270768 2024-09-13T14:30:57.161 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 69636678~5187690 2024-09-13T14:30:57.206 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 13237881~8351303 2024-09-13T14:30:57.224 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 70602739~7999711 2024-09-13T14:30:57.281 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 36673220~2428379 2024-09-13T14:30:57.296 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:30:57.846 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:30:57 smithi177 bash[52886]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:30:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-09-13T14:30:58.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:57 smithi177 bash[24494]: cluster 2024-09-13T14:30:56.855335+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-09-13T14:30:58.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:57 smithi099 bash[22656]: cluster 2024-09-13T14:30:56.855335+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-09-13T14:30:58.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:57 smithi099 bash[32082]: cluster 2024-09-13T14:30:56.855335+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-09-13T14:30:58.465 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:30:58.638 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:30:58.651 INFO:tasks.workunit.client.0.smithi099.stdout:clone image160 -> image162 2024-09-13T14:30:58.808 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 70878619~4545852 2024-09-13T14:30:58.889 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 98171924~6931431 2024-09-13T14:30:58.903 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 44006620~1305688 2024-09-13T14:30:58.971 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83868044~7605600 2024-09-13T14:30:58.985 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 11318033~6449466 2024-09-13T14:30:59.035 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 88472055~7279150 2024-09-13T14:30:59.042 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 64901054~2421446 2024-09-13T14:30:59.070 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 37227382~4096007 2024-09-13T14:30:59.084 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:30:59.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cluster 2024-09-13T14:30:57.862005+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-09-13T14:30:59.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.022785+0000 mon.a (mon.0) 1256 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.035749+0000 mon.a (mon.0) 1257 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.044059+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cephadm 2024-09-13T14:30:58.044722+0000 mgr.y (mgr.25724) 29 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cephadm 2024-09-13T14:30:58.045007+0000 mgr.y (mgr.25724) 30 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.045920+0000 mon.b (mon.1) 115 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.141688+0000 mon.a (mon.0) 1258 : audit [DBG] from='client.? 172.21.15.99:0/51994417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cephadm 2024-09-13T14:30:58.216763+0000 mgr.y (mgr.25724) 31 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cephadm 2024-09-13T14:30:58.227368+0000 mgr.y (mgr.25724) 32 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cluster 2024-09-13T14:30:58.279700+0000 mgr.y (mgr.25724) 33 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 361 KiB/s rd, 27 KiB/s wr, 462 op/s 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cephadm 2024-09-13T14:30:58.338733+0000 mgr.y (mgr.25724) 34 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cephadm 2024-09-13T14:30:58.395476+0000 mgr.y (mgr.25724) 35 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:30:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.458766+0000 mgr.y (mgr.25724) 36 : audit [DBG] from='client.25991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cephadm 2024-09-13T14:30:58.462190+0000 mgr.y (mgr.25724) 37 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cephadm 2024-09-13T14:30:58.541964+0000 mgr.y (mgr.25724) 38 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.586068+0000 mon.a (mon.0) 1259 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.598944+0000 mon.a (mon.0) 1260 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: cluster 2024-09-13T14:30:58.619447+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.671615+0000 mon.a (mon.0) 1262 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.679711+0000 mon.a (mon.0) 1263 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[22656]: audit 2024-09-13T14:30:58.688921+0000 mon.a (mon.0) 1264 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cluster 2024-09-13T14:30:57.862005+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.022785+0000 mon.a (mon.0) 1256 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.035749+0000 mon.a (mon.0) 1257 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.044059+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cephadm 2024-09-13T14:30:58.044722+0000 mgr.y (mgr.25724) 29 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cephadm 2024-09-13T14:30:58.045007+0000 mgr.y (mgr.25724) 30 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.045920+0000 mon.b (mon.1) 115 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.141688+0000 mon.a (mon.0) 1258 : audit [DBG] from='client.? 172.21.15.99:0/51994417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cephadm 2024-09-13T14:30:58.216763+0000 mgr.y (mgr.25724) 31 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cephadm 2024-09-13T14:30:58.227368+0000 mgr.y (mgr.25724) 32 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cluster 2024-09-13T14:30:58.279700+0000 mgr.y (mgr.25724) 33 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 361 KiB/s rd, 27 KiB/s wr, 462 op/s 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cephadm 2024-09-13T14:30:58.338733+0000 mgr.y (mgr.25724) 34 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cephadm 2024-09-13T14:30:58.395476+0000 mgr.y (mgr.25724) 35 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.458766+0000 mgr.y (mgr.25724) 36 : audit [DBG] from='client.25991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cephadm 2024-09-13T14:30:58.462190+0000 mgr.y (mgr.25724) 37 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cephadm 2024-09-13T14:30:58.541964+0000 mgr.y (mgr.25724) 38 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:30:59.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.586068+0000 mon.a (mon.0) 1259 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.598944+0000 mon.a (mon.0) 1260 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: cluster 2024-09-13T14:30:58.619447+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-09-13T14:30:59.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.671615+0000 mon.a (mon.0) 1262 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.679711+0000 mon.a (mon.0) 1263 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:30:58 smithi099 bash[32082]: audit 2024-09-13T14:30:58.688921+0000 mon.a (mon.0) 1264 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cluster 2024-09-13T14:30:57.862005+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-09-13T14:30:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.022785+0000 mon.a (mon.0) 1256 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.035749+0000 mon.a (mon.0) 1257 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.044059+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:30:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cephadm 2024-09-13T14:30:58.044722+0000 mgr.y (mgr.25724) 29 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:30:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cephadm 2024-09-13T14:30:58.045007+0000 mgr.y (mgr.25724) 30 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.045920+0000 mon.b (mon.1) 115 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.141688+0000 mon.a (mon.0) 1258 : audit [DBG] from='client.? 172.21.15.99:0/51994417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cephadm 2024-09-13T14:30:58.216763+0000 mgr.y (mgr.25724) 31 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cephadm 2024-09-13T14:30:58.227368+0000 mgr.y (mgr.25724) 32 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cluster 2024-09-13T14:30:58.279700+0000 mgr.y (mgr.25724) 33 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 361 KiB/s rd, 27 KiB/s wr, 462 op/s 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cephadm 2024-09-13T14:30:58.338733+0000 mgr.y (mgr.25724) 34 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cephadm 2024-09-13T14:30:58.395476+0000 mgr.y (mgr.25724) 35 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.458766+0000 mgr.y (mgr.25724) 36 : audit [DBG] from='client.25991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cephadm 2024-09-13T14:30:58.462190+0000 mgr.y (mgr.25724) 37 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cephadm 2024-09-13T14:30:58.541964+0000 mgr.y (mgr.25724) 38 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.586068+0000 mon.a (mon.0) 1259 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.598944+0000 mon.a (mon.0) 1260 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: cluster 2024-09-13T14:30:58.619447+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-09-13T14:30:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.671615+0000 mon.a (mon.0) 1262 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.679711+0000 mon.a (mon.0) 1263 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:30:59.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:30:58 smithi177 bash[24494]: audit 2024-09-13T14:30:58.688921+0000 mon.a (mon.0) 1264 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:00.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:00 smithi099 bash[22656]: cephadm 2024-09-13T14:30:58.727064+0000 mgr.y (mgr.25724) 39 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:31:00.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:00 smithi099 bash[22656]: audit 2024-09-13T14:30:58.791251+0000 mgr.y (mgr.25724) 40 : audit [DBG] from='client.25997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:00.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:00 smithi099 bash[22656]: cephadm 2024-09-13T14:30:58.980518+0000 mgr.y (mgr.25724) 41 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi177 2024-09-13T14:31:00.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:00 smithi099 bash[32082]: cephadm 2024-09-13T14:30:58.727064+0000 mgr.y (mgr.25724) 39 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:31:00.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:00 smithi099 bash[32082]: audit 2024-09-13T14:30:58.791251+0000 mgr.y (mgr.25724) 40 : audit [DBG] from='client.25997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:00.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:00 smithi099 bash[32082]: cephadm 2024-09-13T14:30:58.980518+0000 mgr.y (mgr.25724) 41 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi177 2024-09-13T14:31:00.353 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:00 smithi177 bash[59561]: ts=2024-09-13T14:31:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:00.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:00 smithi177 bash[24494]: cephadm 2024-09-13T14:30:58.727064+0000 mgr.y (mgr.25724) 39 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:31:00.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:00 smithi177 bash[24494]: audit 2024-09-13T14:30:58.791251+0000 mgr.y (mgr.25724) 40 : audit [DBG] from='client.25997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:00.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:00 smithi177 bash[24494]: cephadm 2024-09-13T14:30:58.980518+0000 mgr.y (mgr.25724) 41 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi177 2024-09-13T14:31:01.077 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap2 2024-09-13T14:31:01.096 INFO:tasks.workunit.client.0.smithi099.stdout:clone image162 -> image164 2024-09-13T14:31:01.315 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 3957171~1533746 2024-09-13T14:31:01.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:01 smithi099 bash[22656]: cluster 2024-09-13T14:31:00.045162+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-09-13T14:31:01.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:01 smithi099 bash[22656]: cluster 2024-09-13T14:31:00.280557+0000 mgr.y (mgr.25724) 42 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.2 MiB/s rd, 25 MiB/s wr, 530 op/s 2024-09-13T14:31:01.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:01 smithi099 bash[32082]: cluster 2024-09-13T14:31:00.045162+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-09-13T14:31:01.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:01 smithi099 bash[32082]: cluster 2024-09-13T14:31:00.280557+0000 mgr.y (mgr.25724) 42 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.2 MiB/s rd, 25 MiB/s wr, 530 op/s 2024-09-13T14:31:01.346 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 101961653~1577822 2024-09-13T14:31:01.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:01 smithi177 bash[24494]: cluster 2024-09-13T14:31:00.045162+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-09-13T14:31:01.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:01 smithi177 bash[24494]: cluster 2024-09-13T14:31:00.280557+0000 mgr.y (mgr.25724) 42 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.2 MiB/s rd, 25 MiB/s wr, 530 op/s 2024-09-13T14:31:01.357 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 77545968~1536947 2024-09-13T14:31:01.422 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 69256403~7622160 2024-09-13T14:31:01.448 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 49044519~5034392 2024-09-13T14:31:01.528 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 47085175~4758349 2024-09-13T14:31:01.533 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 50264221~8164959 2024-09-13T14:31:01.543 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:01.605 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 26450528~4946873 2024-09-13T14:31:01.619 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:31:01.853 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:01 smithi177 bash[59561]: ts=2024-09-13T14:31:01.554Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.177:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.177:8765: connect: connection refused" 2024-09-13T14:31:01.854 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:01 smithi177 bash[59561]: ts=2024-09-13T14:31:01.557Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.177:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.177:8765: connect: connection refused" 2024-09-13T14:31:01.854 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:01 smithi177 bash[59561]: ts=2024-09-13T14:31:01.558Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.177:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.177:8765: connect: connection refused" 2024-09-13T14:31:01.854 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:01 smithi177 bash[59561]: ts=2024-09-13T14:31:01.558Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.177:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.177:8765: connect: connection refused" 2024-09-13T14:31:01.854 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:01 smithi177 bash[59561]: ts=2024-09-13T14:31:01.560Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.177:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.177:8765: connect: connection refused" 2024-09-13T14:31:01.854 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:01 smithi177 bash[59561]: ts=2024-09-13T14:31:01.560Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.177:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.177:8765: connect: connection refused" 2024-09-13T14:31:02.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:02 smithi099 bash[22656]: cluster 2024-09-13T14:31:01.068597+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-09-13T14:31:02.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:02 smithi099 bash[22656]: audit 2024-09-13T14:31:01.190530+0000 mon.a (mon.0) 1267 : audit [DBG] from='client.? 172.21.15.99:0/2744225705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:02.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:02 smithi099 bash[22656]: audit 2024-09-13T14:31:01.532935+0000 mgr.y (mgr.25724) 43 : audit [DBG] from='client.16119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:02 smithi099 bash[32082]: cluster 2024-09-13T14:31:01.068597+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-09-13T14:31:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:02 smithi099 bash[32082]: audit 2024-09-13T14:31:01.190530+0000 mon.a (mon.0) 1267 : audit [DBG] from='client.? 172.21.15.99:0/2744225705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:02 smithi099 bash[32082]: audit 2024-09-13T14:31:01.532935+0000 mgr.y (mgr.25724) 43 : audit [DBG] from='client.16119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:02.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:02 smithi177 bash[24494]: cluster 2024-09-13T14:31:01.068597+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-09-13T14:31:02.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:02 smithi177 bash[24494]: audit 2024-09-13T14:31:01.190530+0000 mon.a (mon.0) 1267 : audit [DBG] from='client.? 172.21.15.99:0/2744225705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:02.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:02 smithi177 bash[24494]: audit 2024-09-13T14:31:01.532935+0000 mgr.y (mgr.25724) 43 : audit [DBG] from='client.16119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap3 2024-09-13T14:31:03.132 INFO:tasks.workunit.client.0.smithi099.stdout:clone image164 -> image166 2024-09-13T14:31:03.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:03 smithi177 bash[24494]: audit 2024-09-13T14:31:01.845736+0000 mgr.y (mgr.25724) 44 : audit [DBG] from='client.26012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:03.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:03 smithi177 bash[24494]: cluster 2024-09-13T14:31:02.078781+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-09-13T14:31:03.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:03 smithi177 bash[24494]: cluster 2024-09-13T14:31:02.281386+0000 mgr.y (mgr.25724) 45 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.2 MiB/s rd, 25 MiB/s wr, 530 op/s 2024-09-13T14:31:03.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:03 smithi099 bash[22656]: audit 2024-09-13T14:31:01.845736+0000 mgr.y (mgr.25724) 44 : audit [DBG] from='client.26012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:03.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:03 smithi099 bash[22656]: cluster 2024-09-13T14:31:02.078781+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-09-13T14:31:03.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:03 smithi099 bash[22656]: cluster 2024-09-13T14:31:02.281386+0000 mgr.y (mgr.25724) 45 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.2 MiB/s rd, 25 MiB/s wr, 530 op/s 2024-09-13T14:31:03.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:03 smithi099 bash[32082]: audit 2024-09-13T14:31:01.845736+0000 mgr.y (mgr.25724) 44 : audit [DBG] from='client.26012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:03.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:03 smithi099 bash[32082]: cluster 2024-09-13T14:31:02.078781+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-09-13T14:31:03.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:03 smithi099 bash[32082]: cluster 2024-09-13T14:31:02.281386+0000 mgr.y (mgr.25724) 45 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.2 MiB/s rd, 25 MiB/s wr, 530 op/s 2024-09-13T14:31:03.571 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 108926680 2024-09-13T14:31:03.904 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.StressNoMigrate (8895 ms) 2024-09-13T14:31:03.905 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.Stress 2024-09-13T14:31:03.966 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:31:03.983 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 76161954~7724126 2024-09-13T14:31:04.045 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 64425770~6815753 2024-09-13T14:31:04.046 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 6453690~2651048 2024-09-13T14:31:04.073 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 70501163~8126960 2024-09-13T14:31:04.080 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 67554637~2161089 2024-09-13T14:31:04.101 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 79384300~4501780 2024-09-13T14:31:04.113 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 39047854~1979519 2024-09-13T14:31:04.139 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 33866779~6412309 2024-09-13T14:31:04.142 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap0 2024-09-13T14:31:04.318 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[24494]: cluster 2024-09-13T14:31:03.092434+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-09-13T14:31:04.318 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[24494]: cluster 2024-09-13T14:31:03.625875+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-09-13T14:31:04.318 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 systemd[1]: Stopping Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:31:04.318 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.137Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:31:04.318 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.137Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:31:04.318 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.137Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:31:04.318 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.137Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:31:04.318 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.137Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:31:04.319 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.137Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-13T14:31:04.319 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.137Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:31:04.319 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.137Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-13T14:31:04.319 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.138Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-13T14:31:04.319 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.139Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:31:04.319 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.139Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-13T14:31:04.319 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[59561]: ts=2024-09-13T14:31:04.139Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-13T14:31:04.565 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:04.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:04 smithi099 bash[22656]: cluster 2024-09-13T14:31:03.092434+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-09-13T14:31:04.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:04 smithi099 bash[22656]: cluster 2024-09-13T14:31:03.625875+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-09-13T14:31:04.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:04 smithi099 bash[32082]: cluster 2024-09-13T14:31:03.092434+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-09-13T14:31:04.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:04 smithi099 bash[32082]: cluster 2024-09-13T14:31:03.625875+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-09-13T14:31:04.985 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[61350]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:31:04.985 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 bash[61411]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:31:04.985 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:31:04.985 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 systemd[1]: Stopped Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:31:04.985 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:04 smithi177 systemd[1]: Started Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:31:05.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:05 smithi177 bash[24494]: audit 2024-09-13T14:31:04.251537+0000 mon.a (mon.0) 1271 : audit [DBG] from='client.? 172.21.15.99:0/3681462200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:05.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:05 smithi177 bash[24494]: cluster 2024-09-13T14:31:04.281903+0000 mgr.y (mgr.25724) 46 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:31:05.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:05 smithi177 bash[24494]: audit 2024-09-13T14:31:04.557727+0000 mgr.y (mgr.25724) 47 : audit [DBG] from='client.16137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:05.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:05 smithi177 bash[24494]: audit 2024-09-13T14:31:04.914823+0000 mon.a (mon.0) 1272 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:05.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:05 smithi177 bash[24494]: audit 2024-09-13T14:31:04.922342+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:05.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:05 smithi177 bash[24494]: audit 2024-09-13T14:31:04.931590+0000 mon.b (mon.1) 116 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:05.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:05 smithi177 bash[24494]: audit 2024-09-13T14:31:04.983617+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:05.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[22656]: audit 2024-09-13T14:31:04.251537+0000 mon.a (mon.0) 1271 : audit [DBG] from='client.? 172.21.15.99:0/3681462200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:05.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[22656]: cluster 2024-09-13T14:31:04.281903+0000 mgr.y (mgr.25724) 46 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:31:05.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[22656]: audit 2024-09-13T14:31:04.557727+0000 mgr.y (mgr.25724) 47 : audit [DBG] from='client.16137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:05.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[22656]: audit 2024-09-13T14:31:04.914823+0000 mon.a (mon.0) 1272 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:05.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[22656]: audit 2024-09-13T14:31:04.922342+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:05.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[22656]: audit 2024-09-13T14:31:04.931590+0000 mon.b (mon.1) 116 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:05.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[22656]: audit 2024-09-13T14:31:04.983617+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:05.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[32082]: audit 2024-09-13T14:31:04.251537+0000 mon.a (mon.0) 1271 : audit [DBG] from='client.? 172.21.15.99:0/3681462200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:05.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[32082]: cluster 2024-09-13T14:31:04.281903+0000 mgr.y (mgr.25724) 46 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 1.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:31:05.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[32082]: audit 2024-09-13T14:31:04.557727+0000 mgr.y (mgr.25724) 47 : audit [DBG] from='client.16137 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:05.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[32082]: audit 2024-09-13T14:31:04.914823+0000 mon.a (mon.0) 1272 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:05.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[32082]: audit 2024-09-13T14:31:04.922342+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:05.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[32082]: audit 2024-09-13T14:31:04.931590+0000 mon.b (mon.1) 116 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:05.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:05 smithi099 bash[32082]: audit 2024-09-13T14:31:04.983617+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:06.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:06 smithi177 bash[24494]: audit 2024-09-13T14:31:04.871151+0000 mgr.y (mgr.25724) 48 : audit [DBG] from='client.16143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:06.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:06 smithi177 bash[24494]: audit 2024-09-13T14:31:04.929070+0000 mgr.y (mgr.25724) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:06.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:06 smithi177 bash[24494]: cluster 2024-09-13T14:31:05.104122+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2024-09-13T14:31:06.128 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap0 2024-09-13T14:31:06.160 INFO:tasks.workunit.client.0.smithi099.stdout:clone image167 -> image170 2024-09-13T14:31:06.401 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 33344244~7934508 2024-09-13T14:31:06.521 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 76705303~4598610 2024-09-13T14:31:06.539 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 16146845~4548161 2024-09-13T14:31:06.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:06 smithi099 bash[22656]: audit 2024-09-13T14:31:04.871151+0000 mgr.y (mgr.25724) 48 : audit [DBG] from='client.16143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:06.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:06 smithi099 bash[22656]: audit 2024-09-13T14:31:04.929070+0000 mgr.y (mgr.25724) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:06.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:06 smithi099 bash[22656]: cluster 2024-09-13T14:31:05.104122+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2024-09-13T14:31:06.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:06 smithi099 bash[32082]: audit 2024-09-13T14:31:04.871151+0000 mgr.y (mgr.25724) 48 : audit [DBG] from='client.16143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:06.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:06 smithi099 bash[32082]: audit 2024-09-13T14:31:04.929070+0000 mgr.y (mgr.25724) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:31:06.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:06 smithi099 bash[32082]: cluster 2024-09-13T14:31:05.104122+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2024-09-13T14:31:06.603 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 39018186~984183 2024-09-13T14:31:06.609 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 32857790~4316502 2024-09-13T14:31:06.649 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 60900711~5582280 2024-09-13T14:31:06.664 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 13306407~8058252 2024-09-13T14:31:06.729 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 56817045~5940961 2024-09-13T14:31:06.740 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:31:07.377 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[24494]: cluster 2024-09-13T14:31:06.113615+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-09-13T14:31:07.377 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[24494]: cluster 2024-09-13T14:31:06.282614+0000 mgr.y (mgr.25724) 50 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 20 MiB/s rd, 18 MiB/s wr, 567 op/s 2024-09-13T14:31:07.377 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.313Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-13T14:31:07.377 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.313Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-13T14:31:07.377 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.313Z caller=main.go:623 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi177 (none))" 2024-09-13T14:31:07.377 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.313Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:31:07.377 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.313Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.315Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.316Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.319Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.319Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.322Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.322Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.571µs 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.322Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.323Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.347Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.357Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.357Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.357Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=48.167µs wal_replay_duration=35.000371ms wbl_replay_duration=104ns total_replay_duration=35.089747ms 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.362Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:31:07.378 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.362Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-13T14:31:07.379 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.362Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:31:07.379 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.385Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.429442ms db_storage=982ns remote_storage=1.526µs web_handler=489ns query_engine=815ns scrape=335.803µs scrape_sd=154.708µs notify=18.684µs notify_sd=20.081µs rules=22.495396ms tracing=8.364µs 2024-09-13T14:31:07.379 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.385Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-13T14:31:07.379 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:07 smithi177 bash[61439]: ts=2024-09-13T14:31:07.385Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:31:07.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:07 smithi099 bash[22656]: cluster 2024-09-13T14:31:06.113615+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-09-13T14:31:07.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:07 smithi099 bash[22656]: cluster 2024-09-13T14:31:06.282614+0000 mgr.y (mgr.25724) 50 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 20 MiB/s rd, 18 MiB/s wr, 567 op/s 2024-09-13T14:31:07.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:07 smithi099 bash[32082]: cluster 2024-09-13T14:31:06.113615+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-09-13T14:31:07.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:07 smithi099 bash[32082]: cluster 2024-09-13T14:31:06.282614+0000 mgr.y (mgr.25724) 50 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 20 MiB/s rd, 18 MiB/s wr, 567 op/s 2024-09-13T14:31:07.584 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:08.165 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:31:08.190 INFO:tasks.workunit.client.0.smithi099.stdout:clone image170 -> image172 2024-09-13T14:31:08.486 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 96825168 2024-09-13T14:31:08.514 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 64293396~1973057 2024-09-13T14:31:08.559 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 31699151~6711569 2024-09-13T14:31:08.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:08 smithi099 bash[22656]: cluster 2024-09-13T14:31:07.126960+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-09-13T14:31:08.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:08 smithi099 bash[22656]: audit 2024-09-13T14:31:07.258141+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.99:0/3632512695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:08.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:08 smithi099 bash[22656]: audit 2024-09-13T14:31:07.576613+0000 mgr.y (mgr.25724) 51 : audit [DBG] from='client.16155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:08.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:08 smithi099 bash[32082]: cluster 2024-09-13T14:31:07.126960+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-09-13T14:31:08.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:08 smithi099 bash[32082]: audit 2024-09-13T14:31:07.258141+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.99:0/3632512695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:08.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:08 smithi099 bash[32082]: audit 2024-09-13T14:31:07.576613+0000 mgr.y (mgr.25724) 51 : audit [DBG] from='client.16155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:08.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:08 smithi177 bash[24494]: cluster 2024-09-13T14:31:07.126960+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-09-13T14:31:08.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:08 smithi177 bash[24494]: audit 2024-09-13T14:31:07.258141+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.99:0/3632512695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:08.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:08 smithi177 bash[24494]: audit 2024-09-13T14:31:07.576613+0000 mgr.y (mgr.25724) 51 : audit [DBG] from='client.16155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:08.604 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 88727687~8097481 2024-09-13T14:31:08.704 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 40457799~4398350 2024-09-13T14:31:08.718 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 52905290~1181150 2024-09-13T14:31:08.746 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 69831673~3844057 2024-09-13T14:31:08.800 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 2209165~4256598 2024-09-13T14:31:08.866 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 477965~5343979 2024-09-13T14:31:08.870 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:31:09.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:09 smithi177 bash[24494]: audit 2024-09-13T14:31:07.891236+0000 mgr.y (mgr.25724) 52 : audit [DBG] from='client.16161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:09.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:09 smithi177 bash[24494]: cluster 2024-09-13T14:31:08.138479+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-09-13T14:31:09.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:09 smithi177 bash[24494]: cephadm 2024-09-13T14:31:08.277297+0000 mgr.y (mgr.25724) 53 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T14:31:09.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:09 smithi177 bash[24494]: audit 2024-09-13T14:31:08.279086+0000 mon.a (mon.0) 1278 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:09.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:09 smithi177 bash[24494]: audit 2024-09-13T14:31:08.281199+0000 mon.b (mon.1) 118 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:09.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:09 smithi177 bash[24494]: cluster 2024-09-13T14:31:08.283354+0000 mgr.y (mgr.25724) 54 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 20 MiB/s rd, 18 MiB/s wr, 567 op/s 2024-09-13T14:31:09.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:09 smithi177 bash[24494]: audit 2024-09-13T14:31:08.284375+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:31:09.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:09 smithi177 bash[24494]: cephadm 2024-09-13T14:31:08.285162+0000 mgr.y (mgr.25724) 55 : cephadm [INF] Deploying daemon mgr.x on smithi177 2024-09-13T14:31:09.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:09 smithi177 bash[24494]: audit 2024-09-13T14:31:08.286142+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:09.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:09 smithi177 bash[24494]: audit 2024-09-13T14:31:09.085719+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:09.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[22656]: audit 2024-09-13T14:31:07.891236+0000 mgr.y (mgr.25724) 52 : audit [DBG] from='client.16161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:09.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[22656]: cluster 2024-09-13T14:31:08.138479+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-09-13T14:31:09.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[22656]: cephadm 2024-09-13T14:31:08.277297+0000 mgr.y (mgr.25724) 53 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T14:31:09.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[22656]: audit 2024-09-13T14:31:08.279086+0000 mon.a (mon.0) 1278 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:09.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[22656]: audit 2024-09-13T14:31:08.281199+0000 mon.b (mon.1) 118 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:09.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[22656]: cluster 2024-09-13T14:31:08.283354+0000 mgr.y (mgr.25724) 54 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 20 MiB/s rd, 18 MiB/s wr, 567 op/s 2024-09-13T14:31:09.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[22656]: audit 2024-09-13T14:31:08.284375+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:31:09.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[22656]: cephadm 2024-09-13T14:31:08.285162+0000 mgr.y (mgr.25724) 55 : cephadm [INF] Deploying daemon mgr.x on smithi177 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[22656]: audit 2024-09-13T14:31:08.286142+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[22656]: audit 2024-09-13T14:31:09.085719+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[32082]: audit 2024-09-13T14:31:07.891236+0000 mgr.y (mgr.25724) 52 : audit [DBG] from='client.16161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[32082]: cluster 2024-09-13T14:31:08.138479+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[32082]: cephadm 2024-09-13T14:31:08.277297+0000 mgr.y (mgr.25724) 53 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[32082]: audit 2024-09-13T14:31:08.279086+0000 mon.a (mon.0) 1278 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[32082]: audit 2024-09-13T14:31:08.281199+0000 mon.b (mon.1) 118 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[32082]: cluster 2024-09-13T14:31:08.283354+0000 mgr.y (mgr.25724) 54 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 20 MiB/s rd, 18 MiB/s wr, 567 op/s 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[32082]: audit 2024-09-13T14:31:08.284375+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[32082]: cephadm 2024-09-13T14:31:08.285162+0000 mgr.y (mgr.25724) 55 : cephadm [INF] Deploying daemon mgr.x on smithi177 2024-09-13T14:31:09.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[32082]: audit 2024-09-13T14:31:08.286142+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:09.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:09 smithi099 bash[32082]: audit 2024-09-13T14:31:09.085719+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:10.170 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap2 2024-09-13T14:31:10.204 INFO:tasks.workunit.client.0.smithi099.stdout:clone image172 -> image174 2024-09-13T14:31:10.409 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 68804280~6448612 2024-09-13T14:31:10.484 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 62611538~641502 2024-09-13T14:31:10.512 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 89121461~2450848 2024-09-13T14:31:10.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:10 smithi099 bash[32082]: cluster 2024-09-13T14:31:09.154616+0000 mon.a (mon.0) 1279 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-09-13T14:31:10.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:10 smithi099 bash[22656]: cluster 2024-09-13T14:31:09.154616+0000 mon.a (mon.0) 1279 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-09-13T14:31:10.570 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 77167303~5421615 2024-09-13T14:31:10.581 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:10 smithi177 bash[24494]: cluster 2024-09-13T14:31:09.154616+0000 mon.a (mon.0) 1279 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-09-13T14:31:10.581 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:10 smithi177 bash[61439]: ts=2024-09-13T14:31:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:10.585 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 40474980~374310 2024-09-13T14:31:10.617 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 94117488~1214744 2024-09-13T14:31:10.636 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:10.678 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 19252666~8068884 2024-09-13T14:31:10.770 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83866758~4173412 2024-09-13T14:31:10.779 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:31:11.421 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:31:11 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:11.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:11 smithi177 bash[24494]: cluster 2024-09-13T14:31:10.156053+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-09-13T14:31:11.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:11 smithi177 bash[24494]: audit 2024-09-13T14:31:10.283509+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.99:0/3141044248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:11.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:11 smithi177 bash[24494]: cluster 2024-09-13T14:31:10.283910+0000 mgr.y (mgr.25724) 56 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 4.5 MiB/s rd, 22 MiB/s wr, 414 op/s 2024-09-13T14:31:11.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:11 smithi177 bash[24494]: audit 2024-09-13T14:31:10.629183+0000 mgr.y (mgr.25724) 57 : audit [DBG] from='client.16173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:11.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:11 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:11.422 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:11 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:11.423 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:31:11 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:11.423 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:31:11 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:11.423 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:31:11 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:11.424 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:31:11 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:11.424 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:31:11 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:11.424 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:11 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:11.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:11 smithi099 bash[22656]: cluster 2024-09-13T14:31:10.156053+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-09-13T14:31:11.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:11 smithi099 bash[22656]: audit 2024-09-13T14:31:10.283509+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.99:0/3141044248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:11.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:11 smithi099 bash[22656]: cluster 2024-09-13T14:31:10.283910+0000 mgr.y (mgr.25724) 56 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 4.5 MiB/s rd, 22 MiB/s wr, 414 op/s 2024-09-13T14:31:11.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:11 smithi099 bash[22656]: audit 2024-09-13T14:31:10.629183+0000 mgr.y (mgr.25724) 57 : audit [DBG] from='client.16173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:11.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:11 smithi099 bash[32082]: cluster 2024-09-13T14:31:10.156053+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-09-13T14:31:11.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:11 smithi099 bash[32082]: audit 2024-09-13T14:31:10.283509+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.99:0/3141044248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:11.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:11 smithi099 bash[32082]: cluster 2024-09-13T14:31:10.283910+0000 mgr.y (mgr.25724) 56 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 4.5 MiB/s rd, 22 MiB/s wr, 414 op/s 2024-09-13T14:31:11.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:11 smithi099 bash[32082]: audit 2024-09-13T14:31:10.629183+0000 mgr.y (mgr.25724) 57 : audit [DBG] from='client.16173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:11.848 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:11 smithi177 systemd[1]: Stopping Ceph mgr.x for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:31:12.200 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap3 2024-09-13T14:31:12.226 INFO:tasks.workunit.client.0.smithi099.stdout:clone image174 -> image176 2024-09-13T14:31:12.459 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:12 smithi177 bash[24494]: audit 2024-09-13T14:31:10.931308+0000 mgr.y (mgr.25724) 58 : audit [DBG] from='client.16179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:12.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:12 smithi177 bash[24494]: cluster 2024-09-13T14:31:11.175324+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-09-13T14:31:12.460 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:12 smithi177 bash[61717]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:31:12.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:12 smithi099 bash[22656]: audit 2024-09-13T14:31:10.931308+0000 mgr.y (mgr.25724) 58 : audit [DBG] from='client.16179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:12.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:12 smithi099 bash[22656]: cluster 2024-09-13T14:31:11.175324+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-09-13T14:31:12.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:12 smithi099 bash[32082]: audit 2024-09-13T14:31:10.931308+0000 mgr.y (mgr.25724) 58 : audit [DBG] from='client.16179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:12.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:12 smithi099 bash[32082]: cluster 2024-09-13T14:31:11.175324+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-09-13T14:31:12.651 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 144755743 2024-09-13T14:31:12.678 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:31:12.759 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:12.760 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:12.760 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:12.760 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:31:12.760 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:12 smithi177 bash[61776]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:31:12.760 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T14:31:12.760 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: Stopped Ceph mgr.x for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:31:12.760 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:12.761 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:12.761 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:12.761 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:12.762 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:12.762 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:13.103 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:12 smithi177 systemd[1]: Started Ceph mgr.x for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:31:13.477 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:31:13.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:13 smithi099 bash[22656]: cluster 2024-09-13T14:31:12.185514+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-09-13T14:31:13.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:13 smithi099 bash[22656]: cluster 2024-09-13T14:31:12.284663+0000 mgr.y (mgr.25724) 59 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 4.5 MiB/s rd, 22 MiB/s wr, 414 op/s 2024-09-13T14:31:13.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:13 smithi099 bash[22656]: audit 2024-09-13T14:31:12.820589+0000 mon.a (mon.0) 1283 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:13.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:13 smithi099 bash[22656]: audit 2024-09-13T14:31:12.832822+0000 mon.a (mon.0) 1284 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:13.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:13 smithi099 bash[22656]: audit 2024-09-13T14:31:12.838819+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:13.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:13 smithi099 bash[32082]: cluster 2024-09-13T14:31:12.185514+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-09-13T14:31:13.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:13 smithi099 bash[32082]: cluster 2024-09-13T14:31:12.284663+0000 mgr.y (mgr.25724) 59 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 4.5 MiB/s rd, 22 MiB/s wr, 414 op/s 2024-09-13T14:31:13.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:13 smithi099 bash[32082]: audit 2024-09-13T14:31:12.820589+0000 mon.a (mon.0) 1283 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:13.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:13 smithi099 bash[32082]: audit 2024-09-13T14:31:12.832822+0000 mon.a (mon.0) 1284 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:13.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:13 smithi099 bash[32082]: audit 2024-09-13T14:31:12.838819+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:13.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:13 smithi177 bash[24494]: cluster 2024-09-13T14:31:12.185514+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-09-13T14:31:13.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:13 smithi177 bash[24494]: cluster 2024-09-13T14:31:12.284663+0000 mgr.y (mgr.25724) 59 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 4.5 MiB/s rd, 22 MiB/s wr, 414 op/s 2024-09-13T14:31:13.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:13 smithi177 bash[24494]: audit 2024-09-13T14:31:12.820589+0000 mon.a (mon.0) 1283 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:13.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:13 smithi177 bash[24494]: audit 2024-09-13T14:31:12.832822+0000 mon.a (mon.0) 1284 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:13.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:13 smithi177 bash[24494]: audit 2024-09-13T14:31:12.838819+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:13.697 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:13.853 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:31:14.401 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.Stress (10497 ms) 2024-09-13T14:31:14.401 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.Stress2 2024-09-13T14:31:14.470 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:31:14.493 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 17473189~1616375 2024-09-13T14:31:14.532 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 73435937~2341837 2024-09-13T14:31:14.533 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 61807462~4540030 2024-09-13T14:31:14.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:14 smithi099 bash[22656]: audit 2024-09-13T14:31:13.365447+0000 mon.a (mon.0) 1285 : audit [DBG] from='client.? 172.21.15.99:0/2804050439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:14.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:14 smithi099 bash[22656]: audit 2024-09-13T14:31:13.521190+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:14.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:14 smithi099 bash[22656]: audit 2024-09-13T14:31:13.530252+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:14.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:14 smithi099 bash[22656]: audit 2024-09-13T14:31:13.691639+0000 mgr.y (mgr.25724) 60 : audit [DBG] from='client.26078 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:14.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:14 smithi099 bash[32082]: audit 2024-09-13T14:31:13.365447+0000 mon.a (mon.0) 1285 : audit [DBG] from='client.? 172.21.15.99:0/2804050439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:14.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:14 smithi099 bash[32082]: audit 2024-09-13T14:31:13.521190+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:14.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:14 smithi099 bash[32082]: audit 2024-09-13T14:31:13.530252+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:14.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:14 smithi099 bash[32082]: audit 2024-09-13T14:31:13.691639+0000 mgr.y (mgr.25724) 60 : audit [DBG] from='client.26078 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:14.588 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 58505547~2195063 2024-09-13T14:31:14.593 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 24436709~42578 2024-09-13T14:31:14.603 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:14 smithi177 bash[61832]: debug 2024-09-13T14:31:14.310+0000 7f4894137640 1 -- 172.21.15.177:0/2054195522 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563feccad0e0 con 0x563feccaf800 2024-09-13T14:31:14.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:14 smithi177 bash[24494]: audit 2024-09-13T14:31:13.365447+0000 mon.a (mon.0) 1285 : audit [DBG] from='client.? 172.21.15.99:0/2804050439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:14.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:14 smithi177 bash[24494]: audit 2024-09-13T14:31:13.521190+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:14.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:14 smithi177 bash[24494]: audit 2024-09-13T14:31:13.530252+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:14.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:14 smithi177 bash[24494]: audit 2024-09-13T14:31:13.691639+0000 mgr.y (mgr.25724) 60 : audit [DBG] from='client.26078 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:14.607 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 21533402~7967321 2024-09-13T14:31:14.613 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 78166583~5719497 2024-09-13T14:31:14.677 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 3082860~5397689 2024-09-13T14:31:14.677 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap0 2024-09-13T14:31:15.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:15 smithi099 bash[22656]: audit 2024-09-13T14:31:14.008164+0000 mgr.y (mgr.25724) 61 : audit [DBG] from='client.16197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:15.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:15 smithi099 bash[22656]: cluster 2024-09-13T14:31:14.285410+0000 mgr.y (mgr.25724) 62 : cluster [DBG] pgmap v55: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.5 MiB/s rd, 17 MiB/s wr, 322 op/s 2024-09-13T14:31:15.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:15 smithi099 bash[32082]: audit 2024-09-13T14:31:14.008164+0000 mgr.y (mgr.25724) 61 : audit [DBG] from='client.16197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:15.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:15 smithi099 bash[32082]: cluster 2024-09-13T14:31:14.285410+0000 mgr.y (mgr.25724) 62 : cluster [DBG] pgmap v55: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.5 MiB/s rd, 17 MiB/s wr, 322 op/s 2024-09-13T14:31:15.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:15 smithi177 bash[24494]: audit 2024-09-13T14:31:14.008164+0000 mgr.y (mgr.25724) 61 : audit [DBG] from='client.16197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:15.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:15 smithi177 bash[24494]: cluster 2024-09-13T14:31:14.285410+0000 mgr.y (mgr.25724) 62 : cluster [DBG] pgmap v55: 137 pgs: 137 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.5 MiB/s rd, 17 MiB/s wr, 322 op/s 2024-09-13T14:31:16.227 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap0 2024-09-13T14:31:16.252 INFO:tasks.workunit.client.0.smithi099.stdout:clone image177 -> image180 2024-09-13T14:31:16.458 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 81764249~115867 2024-09-13T14:31:16.539 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 3634271~205349 2024-09-13T14:31:16.550 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 59503317~7481521 2024-09-13T14:31:16.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:16 smithi099 bash[22656]: cluster 2024-09-13T14:31:15.204111+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-09-13T14:31:16.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:16 smithi099 bash[32082]: cluster 2024-09-13T14:31:15.204111+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-09-13T14:31:16.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:16 smithi177 bash[24494]: cluster 2024-09-13T14:31:15.204111+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-09-13T14:31:16.624 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 52612363~3314548 2024-09-13T14:31:16.637 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 34872218~4956473 2024-09-13T14:31:16.681 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 75947623~7887348 2024-09-13T14:31:16.699 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 9503724~2478473 2024-09-13T14:31:16.727 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 55151958~1146124 2024-09-13T14:31:16.732 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:31:16.742 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:17.212 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:16 smithi177 bash[61439]: ts=2024-09-13T14:31:16.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:17.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:31:17] "GET /metrics HTTP/1.1" 200 38904 "" "Prometheus/2.51.0" 2024-09-13T14:31:17.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:17 smithi099 bash[22656]: cluster 2024-09-13T14:31:16.213021+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-09-13T14:31:17.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:17 smithi099 bash[22656]: cluster 2024-09-13T14:31:16.286181+0000 mgr.y (mgr.25724) 63 : cluster [DBG] pgmap v58: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 14 MiB/s wr, 1.06k op/s 2024-09-13T14:31:17.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:17 smithi099 bash[22656]: audit 2024-09-13T14:31:16.400286+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.99:0/776638353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:17.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:17 smithi099 bash[32082]: cluster 2024-09-13T14:31:16.213021+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-09-13T14:31:17.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:17 smithi099 bash[32082]: cluster 2024-09-13T14:31:16.286181+0000 mgr.y (mgr.25724) 63 : cluster [DBG] pgmap v58: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 14 MiB/s wr, 1.06k op/s 2024-09-13T14:31:17.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:17 smithi099 bash[32082]: audit 2024-09-13T14:31:16.400286+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.99:0/776638353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:17.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:17 smithi177 bash[24494]: cluster 2024-09-13T14:31:16.213021+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-09-13T14:31:17.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:17 smithi177 bash[24494]: cluster 2024-09-13T14:31:16.286181+0000 mgr.y (mgr.25724) 63 : cluster [DBG] pgmap v58: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 14 MiB/s wr, 1.06k op/s 2024-09-13T14:31:17.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:17 smithi177 bash[24494]: audit 2024-09-13T14:31:16.400286+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.99:0/776638353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:18.241 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap1 2024-09-13T14:31:18.265 INFO:tasks.workunit.client.0.smithi099.stdout:clone image180 -> image182 2024-09-13T14:31:18.539 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 9637253~7907517 2024-09-13T14:31:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:18 smithi099 bash[22656]: audit 2024-09-13T14:31:16.735983+0000 mgr.y (mgr.25724) 64 : audit [DBG] from='client.26096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:18 smithi099 bash[22656]: audit 2024-09-13T14:31:17.043496+0000 mgr.y (mgr.25724) 65 : audit [DBG] from='client.26102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:18.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:18 smithi099 bash[22656]: cluster 2024-09-13T14:31:17.223439+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-09-13T14:31:18.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:18 smithi099 bash[32082]: audit 2024-09-13T14:31:16.735983+0000 mgr.y (mgr.25724) 64 : audit [DBG] from='client.26096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:18.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:18 smithi099 bash[32082]: audit 2024-09-13T14:31:17.043496+0000 mgr.y (mgr.25724) 65 : audit [DBG] from='client.26102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:18.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:18 smithi099 bash[32082]: cluster 2024-09-13T14:31:17.223439+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-09-13T14:31:18.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:18 smithi177 bash[24494]: audit 2024-09-13T14:31:16.735983+0000 mgr.y (mgr.25724) 64 : audit [DBG] from='client.26096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:18.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:18 smithi177 bash[24494]: audit 2024-09-13T14:31:17.043496+0000 mgr.y (mgr.25724) 65 : audit [DBG] from='client.26102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:18.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:18 smithi177 bash[24494]: cluster 2024-09-13T14:31:17.223439+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-09-13T14:31:18.645 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 41504527~5471445 2024-09-13T14:31:18.689 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 72159022~519530 2024-09-13T14:31:18.706 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 6239027~4317673 2024-09-13T14:31:18.718 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 43269794~1497439 2024-09-13T14:31:18.735 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 67504432~933279 2024-09-13T14:31:18.746 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 76546618~1157793 2024-09-13T14:31:18.767 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 51694790~6662345 2024-09-13T14:31:18.782 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:31:19.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:19 smithi099 bash[22656]: cluster 2024-09-13T14:31:18.225107+0000 mon.a (mon.0) 1291 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-09-13T14:31:19.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:19 smithi099 bash[22656]: cluster 2024-09-13T14:31:18.287126+0000 mgr.y (mgr.25724) 66 : cluster [DBG] pgmap v61: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s rd, 18 MiB/s wr, 1.35k op/s 2024-09-13T14:31:19.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:19 smithi099 bash[22656]: audit 2024-09-13T14:31:18.998129+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:19.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:19 smithi099 bash[22656]: audit 2024-09-13T14:31:19.010047+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:19.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:19 smithi099 bash[32082]: cluster 2024-09-13T14:31:18.225107+0000 mon.a (mon.0) 1291 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-09-13T14:31:19.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:19 smithi099 bash[32082]: cluster 2024-09-13T14:31:18.287126+0000 mgr.y (mgr.25724) 66 : cluster [DBG] pgmap v61: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s rd, 18 MiB/s wr, 1.35k op/s 2024-09-13T14:31:19.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:19 smithi099 bash[32082]: audit 2024-09-13T14:31:18.998129+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:19.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:19 smithi099 bash[32082]: audit 2024-09-13T14:31:19.010047+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:19.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:19 smithi177 bash[24494]: cluster 2024-09-13T14:31:18.225107+0000 mon.a (mon.0) 1291 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-09-13T14:31:19.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:19 smithi177 bash[24494]: cluster 2024-09-13T14:31:18.287126+0000 mgr.y (mgr.25724) 66 : cluster [DBG] pgmap v61: 137 pgs: 137 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s rd, 18 MiB/s wr, 1.35k op/s 2024-09-13T14:31:19.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:19 smithi177 bash[24494]: audit 2024-09-13T14:31:18.998129+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:19.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:19 smithi177 bash[24494]: audit 2024-09-13T14:31:19.010047+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:19.747 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:20.265 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap2 2024-09-13T14:31:20.290 INFO:tasks.workunit.client.0.smithi099.stdout:clone image182 -> image184 2024-09-13T14:31:20.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:20 smithi099 bash[22656]: cluster 2024-09-13T14:31:19.243291+0000 mon.a (mon.0) 1294 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-09-13T14:31:20.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:20 smithi099 bash[22656]: audit 2024-09-13T14:31:19.429559+0000 mon.a (mon.0) 1295 : audit [DBG] from='client.? 172.21.15.99:0/3426124393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:20.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:20 smithi099 bash[22656]: audit 2024-09-13T14:31:20.168642+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:20.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:20 smithi099 bash[22656]: audit 2024-09-13T14:31:20.181975+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:20.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:20 smithi099 bash[32082]: cluster 2024-09-13T14:31:19.243291+0000 mon.a (mon.0) 1294 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-09-13T14:31:20.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:20 smithi099 bash[32082]: audit 2024-09-13T14:31:19.429559+0000 mon.a (mon.0) 1295 : audit [DBG] from='client.? 172.21.15.99:0/3426124393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:20.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:20 smithi099 bash[32082]: audit 2024-09-13T14:31:20.168642+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:20.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:20 smithi099 bash[32082]: audit 2024-09-13T14:31:20.181975+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:20.604 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:20 smithi177 bash[61439]: ts=2024-09-13T14:31:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:20.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:20 smithi177 bash[24494]: cluster 2024-09-13T14:31:19.243291+0000 mon.a (mon.0) 1294 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-09-13T14:31:20.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:20 smithi177 bash[24494]: audit 2024-09-13T14:31:19.429559+0000 mon.a (mon.0) 1295 : audit [DBG] from='client.? 172.21.15.99:0/3426124393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:20.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:20 smithi177 bash[24494]: audit 2024-09-13T14:31:20.168642+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:20.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:20 smithi177 bash[24494]: audit 2024-09-13T14:31:20.181975+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:20.660 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 144752539 2024-09-13T14:31:20.684 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 24813190~7997966 2024-09-13T14:31:20.768 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 59058788~7102676 2024-09-13T14:31:20.804 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 93388906~7534715 2024-09-13T14:31:20.865 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 86097302~3405617 2024-09-13T14:31:20.865 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 102450614~310343 2024-09-13T14:31:20.870 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 135725615~7703412 2024-09-13T14:31:20.870 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 27286864~7744836 2024-09-13T14:31:20.944 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 71680372~3079334 2024-09-13T14:31:20.961 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:31:21.104 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:20 smithi177 bash[61832]: [13/Sep/2024:14:31:20] ENGINE Bus STARTING 2024-09-13T14:31:21.104 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:20 smithi177 bash[61832]: CherryPy Checker: 2024-09-13T14:31:21.104 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:20 smithi177 bash[61832]: The Application mounted at '' has an empty config. 2024-09-13T14:31:21.104 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:20 smithi177 bash[61832]: [13/Sep/2024:14:31:20] ENGINE Serving on http://:::9283 2024-09-13T14:31:21.104 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:20 smithi177 bash[61832]: [13/Sep/2024:14:31:20] ENGINE Bus STARTED 2024-09-13T14:31:21.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[22656]: audit 2024-09-13T14:31:19.740378+0000 mgr.y (mgr.25724) 67 : audit [DBG] from='client.16227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:21.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[22656]: audit 2024-09-13T14:31:20.050405+0000 mgr.y (mgr.25724) 68 : audit [DBG] from='client.26120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:21.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[22656]: cluster 2024-09-13T14:31:20.255435+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-09-13T14:31:21.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[22656]: cluster 2024-09-13T14:31:20.288068+0000 mgr.y (mgr.25724) 69 : cluster [DBG] pgmap v64: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 8.1 MiB/s rd, 18 MiB/s wr, 391 op/s 2024-09-13T14:31:21.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[22656]: cluster 2024-09-13T14:31:20.661563+0000 mon.a (mon.0) 1299 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:31:21.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[22656]: cluster 2024-09-13T14:31:20.661962+0000 mon.a (mon.0) 1300 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:31:21.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[22656]: audit 2024-09-13T14:31:20.667457+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:31:21.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[22656]: audit 2024-09-13T14:31:20.669069+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:31:21.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[22656]: audit 2024-09-13T14:31:20.670553+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:31:21.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[22656]: audit 2024-09-13T14:31:20.670972+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:31:21.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[32082]: audit 2024-09-13T14:31:19.740378+0000 mgr.y (mgr.25724) 67 : audit [DBG] from='client.16227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:21.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[32082]: audit 2024-09-13T14:31:20.050405+0000 mgr.y (mgr.25724) 68 : audit [DBG] from='client.26120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:21.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[32082]: cluster 2024-09-13T14:31:20.255435+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-09-13T14:31:21.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[32082]: cluster 2024-09-13T14:31:20.288068+0000 mgr.y (mgr.25724) 69 : cluster [DBG] pgmap v64: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 8.1 MiB/s rd, 18 MiB/s wr, 391 op/s 2024-09-13T14:31:21.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[32082]: cluster 2024-09-13T14:31:20.661563+0000 mon.a (mon.0) 1299 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:31:21.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[32082]: cluster 2024-09-13T14:31:20.661962+0000 mon.a (mon.0) 1300 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:31:21.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[32082]: audit 2024-09-13T14:31:20.667457+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:31:21.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[32082]: audit 2024-09-13T14:31:20.669069+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:31:21.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[32082]: audit 2024-09-13T14:31:20.670553+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:31:21.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:21 smithi099 bash[32082]: audit 2024-09-13T14:31:20.670972+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:31:21.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:21 smithi177 bash[24494]: audit 2024-09-13T14:31:19.740378+0000 mgr.y (mgr.25724) 67 : audit [DBG] from='client.16227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:21.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:21 smithi177 bash[24494]: audit 2024-09-13T14:31:20.050405+0000 mgr.y (mgr.25724) 68 : audit [DBG] from='client.26120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:21.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:21 smithi177 bash[24494]: cluster 2024-09-13T14:31:20.255435+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-09-13T14:31:21.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:21 smithi177 bash[24494]: cluster 2024-09-13T14:31:20.288068+0000 mgr.y (mgr.25724) 69 : cluster [DBG] pgmap v64: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 8.1 MiB/s rd, 18 MiB/s wr, 391 op/s 2024-09-13T14:31:21.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:21 smithi177 bash[24494]: cluster 2024-09-13T14:31:20.661563+0000 mon.a (mon.0) 1299 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:31:21.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:21 smithi177 bash[24494]: cluster 2024-09-13T14:31:20.661962+0000 mon.a (mon.0) 1300 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:31:21.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:21 smithi177 bash[24494]: audit 2024-09-13T14:31:20.667457+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:31:21.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:21 smithi177 bash[24494]: audit 2024-09-13T14:31:20.669069+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:31:21.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:21 smithi177 bash[24494]: audit 2024-09-13T14:31:20.670553+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:31:21.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:21 smithi177 bash[24494]: audit 2024-09-13T14:31:20.670972+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.? 172.21.15.177:0/1972087143' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:31:22.292 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap3 2024-09-13T14:31:22.314 INFO:tasks.workunit.client.0.smithi099.stdout:clone image184 -> image186 2024-09-13T14:31:22.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:22 smithi099 bash[22656]: cluster 2024-09-13T14:31:21.280589+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-09-13T14:31:22.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:22 smithi099 bash[22656]: cluster 2024-09-13T14:31:21.286520+0000 mon.a (mon.0) 1302 : cluster [DBG] mgrmap e33: y(active, since 43s), standbys: x 2024-09-13T14:31:22.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:22 smithi099 bash[32082]: cluster 2024-09-13T14:31:21.280589+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-09-13T14:31:22.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:22 smithi099 bash[32082]: cluster 2024-09-13T14:31:21.286520+0000 mon.a (mon.0) 1302 : cluster [DBG] mgrmap e33: y(active, since 43s), standbys: x 2024-09-13T14:31:22.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:22 smithi177 bash[24494]: cluster 2024-09-13T14:31:21.280589+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-09-13T14:31:22.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:22 smithi177 bash[24494]: cluster 2024-09-13T14:31:21.286520+0000 mon.a (mon.0) 1302 : cluster [DBG] mgrmap e33: y(active, since 43s), standbys: x 2024-09-13T14:31:22.644 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:31:22.796 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:23.539 INFO:tasks.workunit.client.0.smithi099.stdout:write: a 57457451~3235389 2024-09-13T14:31:23.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:23 smithi099 bash[22656]: cluster 2024-09-13T14:31:22.270601+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-09-13T14:31:23.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:23 smithi099 bash[22656]: cluster 2024-09-13T14:31:22.288844+0000 mgr.y (mgr.25724) 70 : cluster [DBG] pgmap v67: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 8.1 MiB/s rd, 18 MiB/s wr, 391 op/s 2024-09-13T14:31:23.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:23 smithi099 bash[22656]: audit 2024-09-13T14:31:22.444167+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.99:0/2938585573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:23 smithi099 bash[32082]: cluster 2024-09-13T14:31:22.270601+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-09-13T14:31:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:23 smithi099 bash[32082]: cluster 2024-09-13T14:31:22.288844+0000 mgr.y (mgr.25724) 70 : cluster [DBG] pgmap v67: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 8.1 MiB/s rd, 18 MiB/s wr, 391 op/s 2024-09-13T14:31:23.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:23 smithi099 bash[32082]: audit 2024-09-13T14:31:22.444167+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.99:0/2938585573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:23.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:23 smithi177 bash[24494]: cluster 2024-09-13T14:31:22.270601+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-09-13T14:31:23.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:23 smithi177 bash[24494]: cluster 2024-09-13T14:31:22.288844+0000 mgr.y (mgr.25724) 70 : cluster [DBG] pgmap v67: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 8.1 MiB/s rd, 18 MiB/s wr, 391 op/s 2024-09-13T14:31:23.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:23 smithi177 bash[24494]: audit 2024-09-13T14:31:22.444167+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.99:0/2938585573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:23.622 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 69132396~1238323 2024-09-13T14:31:23.657 INFO:tasks.workunit.client.0.smithi099.stdout:write: a 20164903~8255286 2024-09-13T14:31:23.753 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 36027848~5719936 2024-09-13T14:31:23.804 INFO:tasks.workunit.client.0.smithi099.stdout:write: a 82384346~8296237 2024-09-13T14:31:23.897 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 120957197~5599243 2024-09-13T14:31:23.941 INFO:tasks.workunit.client.0.smithi099.stdout:write: a 44809234~840673 2024-09-13T14:31:23.987 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 1112333~5310255 2024-09-13T14:31:24.030 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: user0 2024-09-13T14:31:24.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:24 smithi099 bash[22656]: audit 2024-09-13T14:31:22.789427+0000 mgr.y (mgr.25724) 71 : audit [DBG] from='client.26132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:24.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:24 smithi099 bash[22656]: audit 2024-09-13T14:31:23.122451+0000 mgr.y (mgr.25724) 72 : audit [DBG] from='client.16251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:24.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:24 smithi099 bash[22656]: audit 2024-09-13T14:31:24.086444+0000 mon.b (mon.1) 127 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:24.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:24 smithi099 bash[32082]: audit 2024-09-13T14:31:22.789427+0000 mgr.y (mgr.25724) 71 : audit [DBG] from='client.26132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:24.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:24 smithi099 bash[32082]: audit 2024-09-13T14:31:23.122451+0000 mgr.y (mgr.25724) 72 : audit [DBG] from='client.16251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:24.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:24 smithi099 bash[32082]: audit 2024-09-13T14:31:24.086444+0000 mon.b (mon.1) 127 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:24.603 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:24 smithi177 bash[61439]: ts=2024-09-13T14:31:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:24.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:24 smithi177 bash[24494]: audit 2024-09-13T14:31:22.789427+0000 mgr.y (mgr.25724) 71 : audit [DBG] from='client.26132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:24.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:24 smithi177 bash[24494]: audit 2024-09-13T14:31:23.122451+0000 mgr.y (mgr.25724) 72 : audit [DBG] from='client.16251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:24.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:24 smithi177 bash[24494]: audit 2024-09-13T14:31:24.086444+0000 mon.b (mon.1) 127 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:25.321 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 199344345 2024-09-13T14:31:25.339 INFO:tasks.workunit.client.0.smithi099.stdout:write: b 28300825~7107569 2024-09-13T14:31:25.370 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 69643043~4717585 2024-09-13T14:31:25.370 INFO:tasks.workunit.client.0.smithi099.stdout:write: b 196970373~990073 2024-09-13T14:31:25.371 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 109398206~5025802 2024-09-13T14:31:25.372 INFO:tasks.workunit.client.0.smithi099.stdout:write: b 175877297~4595839 2024-09-13T14:31:25.392 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 87123031~7652625 2024-09-13T14:31:25.472 INFO:tasks.workunit.client.0.smithi099.stdout:write: b 119020678~7846021 2024-09-13T14:31:25.479 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 28906815~4612217 2024-09-13T14:31:25.517 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: user1 2024-09-13T14:31:25.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:25 smithi099 bash[22656]: cluster 2024-09-13T14:31:24.289627+0000 mgr.y (mgr.25724) 73 : cluster [DBG] pgmap v68: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 6.4 MiB/s rd, 15 MiB/s wr, 310 op/s 2024-09-13T14:31:25.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:25 smithi099 bash[22656]: cluster 2024-09-13T14:31:24.301032+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-09-13T14:31:25.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:25 smithi099 bash[32082]: cluster 2024-09-13T14:31:24.289627+0000 mgr.y (mgr.25724) 73 : cluster [DBG] pgmap v68: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 6.4 MiB/s rd, 15 MiB/s wr, 310 op/s 2024-09-13T14:31:25.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:25 smithi099 bash[32082]: cluster 2024-09-13T14:31:24.301032+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-09-13T14:31:25.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:25 smithi177 bash[24494]: cluster 2024-09-13T14:31:24.289627+0000 mgr.y (mgr.25724) 73 : cluster [DBG] pgmap v68: 137 pgs: 137 active+clean; 1.4 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 6.4 MiB/s rd, 15 MiB/s wr, 310 op/s 2024-09-13T14:31:25.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:25 smithi177 bash[24494]: cluster 2024-09-13T14:31:24.301032+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-09-13T14:31:25.895 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:26.103 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:25 smithi177 bash[61439]: ts=2024-09-13T14:31:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:26.104 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:25 smithi177 bash[61439]: ts=2024-09-13T14:31:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:26.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:26 smithi099 bash[22656]: cluster 2024-09-13T14:31:25.306301+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-09-13T14:31:26.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:26 smithi099 bash[22656]: audit 2024-09-13T14:31:25.587763+0000 mon.a (mon.0) 1306 : audit [DBG] from='client.? 172.21.15.99:0/2995388245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:26.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:26 smithi099 bash[32082]: cluster 2024-09-13T14:31:25.306301+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-09-13T14:31:26.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:26 smithi099 bash[32082]: audit 2024-09-13T14:31:25.587763+0000 mon.a (mon.0) 1306 : audit [DBG] from='client.? 172.21.15.99:0/2995388245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:26.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:26 smithi177 bash[24494]: cluster 2024-09-13T14:31:25.306301+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-09-13T14:31:26.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:26 smithi177 bash[24494]: audit 2024-09-13T14:31:25.587763+0000 mon.a (mon.0) 1306 : audit [DBG] from='client.? 172.21.15.99:0/2995388245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:27.207 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:26 smithi177 bash[61439]: ts=2024-09-13T14:31:26.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:27.315 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:31:27] "GET /metrics HTTP/1.1" 200 38906 "" "Prometheus/2.51.0" 2024-09-13T14:31:27.356 INFO:tasks.workunit.client.0.smithi099.stdout:write: c 110921913~4478896 2024-09-13T14:31:27.389 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 46878141~4392878 2024-09-13T14:31:27.391 INFO:tasks.workunit.client.0.smithi099.stdout:write: c 101308778~2237258 2024-09-13T14:31:27.397 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 127322906~2605356 2024-09-13T14:31:27.397 INFO:tasks.workunit.client.0.smithi099.stdout:write: c 168260516~3928022 2024-09-13T14:31:27.410 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 194522551~4821794 2024-09-13T14:31:27.410 INFO:tasks.workunit.client.0.smithi099.stdout:write: c 54726070~5561094 2024-09-13T14:31:27.480 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 61009990~2201726 2024-09-13T14:31:27.480 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: user2 2024-09-13T14:31:27.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:27 smithi099 bash[32082]: audit 2024-09-13T14:31:25.888368+0000 mgr.y (mgr.25724) 74 : audit [DBG] from='client.16263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:27.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:27 smithi099 bash[32082]: audit 2024-09-13T14:31:26.204466+0000 mgr.y (mgr.25724) 75 : audit [DBG] from='client.16269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:27.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:27 smithi099 bash[32082]: cluster 2024-09-13T14:31:26.290439+0000 mgr.y (mgr.25724) 76 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 31 MiB/s wr, 728 op/s 2024-09-13T14:31:27.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:27 smithi099 bash[32082]: cluster 2024-09-13T14:31:26.327809+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-09-13T14:31:27.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:27 smithi099 bash[22656]: audit 2024-09-13T14:31:25.888368+0000 mgr.y (mgr.25724) 74 : audit [DBG] from='client.16263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:27.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:27 smithi099 bash[22656]: audit 2024-09-13T14:31:26.204466+0000 mgr.y (mgr.25724) 75 : audit [DBG] from='client.16269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:27.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:27 smithi099 bash[22656]: cluster 2024-09-13T14:31:26.290439+0000 mgr.y (mgr.25724) 76 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 31 MiB/s wr, 728 op/s 2024-09-13T14:31:27.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:27 smithi099 bash[22656]: cluster 2024-09-13T14:31:26.327809+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-09-13T14:31:27.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:27 smithi177 bash[24494]: audit 2024-09-13T14:31:25.888368+0000 mgr.y (mgr.25724) 74 : audit [DBG] from='client.16263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:27.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:27 smithi177 bash[24494]: audit 2024-09-13T14:31:26.204466+0000 mgr.y (mgr.25724) 75 : audit [DBG] from='client.16269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:27.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:27 smithi177 bash[24494]: cluster 2024-09-13T14:31:26.290439+0000 mgr.y (mgr.25724) 76 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 31 MiB/s wr, 728 op/s 2024-09-13T14:31:27.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:27 smithi177 bash[24494]: cluster 2024-09-13T14:31:26.327809+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-09-13T14:31:28.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:28 smithi177 bash[24494]: cluster 2024-09-13T14:31:27.335808+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-09-13T14:31:28.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:28 smithi099 bash[22656]: cluster 2024-09-13T14:31:27.335808+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-09-13T14:31:28.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:28 smithi099 bash[32082]: cluster 2024-09-13T14:31:27.335808+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-09-13T14:31:28.911 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:29.373 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 229410030 2024-09-13T14:31:29.394 INFO:tasks.workunit.client.0.smithi099.stdout:write: d 221706109~7703921 2024-09-13T14:31:29.435 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 186400137~6256833 2024-09-13T14:31:29.435 INFO:tasks.workunit.client.0.smithi099.stdout:write: d 33328332~5846753 2024-09-13T14:31:29.454 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 120213242~7314241 2024-09-13T14:31:29.454 INFO:tasks.workunit.client.0.smithi099.stdout:write: d 126889925~5430776 2024-09-13T14:31:29.538 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 75424523~7749181 2024-09-13T14:31:29.538 INFO:tasks.workunit.client.0.smithi099.stdout:write: d 198966599~97981 2024-09-13T14:31:29.539 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 185394154~67534 2024-09-13T14:31:29.539 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: user3 2024-09-13T14:31:29.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:29 smithi177 bash[24494]: cluster 2024-09-13T14:31:28.291167+0000 mgr.y (mgr.25724) 77 : cluster [DBG] pgmap v74: 137 pgs: 137 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 31 MiB/s rd, 39 MiB/s wr, 914 op/s 2024-09-13T14:31:29.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:29 smithi177 bash[24494]: cluster 2024-09-13T14:31:28.345207+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-09-13T14:31:29.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:29 smithi177 bash[24494]: audit 2024-09-13T14:31:28.589444+0000 mon.a (mon.0) 1310 : audit [DBG] from='client.? 172.21.15.99:0/3289443229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:29.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:29 smithi099 bash[22656]: cluster 2024-09-13T14:31:28.291167+0000 mgr.y (mgr.25724) 77 : cluster [DBG] pgmap v74: 137 pgs: 137 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 31 MiB/s rd, 39 MiB/s wr, 914 op/s 2024-09-13T14:31:29.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:29 smithi099 bash[22656]: cluster 2024-09-13T14:31:28.345207+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-09-13T14:31:29.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:29 smithi099 bash[22656]: audit 2024-09-13T14:31:28.589444+0000 mon.a (mon.0) 1310 : audit [DBG] from='client.? 172.21.15.99:0/3289443229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:29.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:29 smithi099 bash[32082]: cluster 2024-09-13T14:31:28.291167+0000 mgr.y (mgr.25724) 77 : cluster [DBG] pgmap v74: 137 pgs: 137 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 31 MiB/s rd, 39 MiB/s wr, 914 op/s 2024-09-13T14:31:29.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:29 smithi099 bash[32082]: cluster 2024-09-13T14:31:28.345207+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-09-13T14:31:29.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:29 smithi099 bash[32082]: audit 2024-09-13T14:31:28.589444+0000 mon.a (mon.0) 1310 : audit [DBG] from='client.? 172.21.15.99:0/3289443229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:30.604 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:30 smithi177 bash[61439]: ts=2024-09-13T14:31:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:30.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:30 smithi177 bash[24494]: audit 2024-09-13T14:31:28.902851+0000 mgr.y (mgr.25724) 78 : audit [DBG] from='client.16281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:30.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:30 smithi177 bash[24494]: audit 2024-09-13T14:31:29.219870+0000 mgr.y (mgr.25724) 79 : audit [DBG] from='client.26174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:30.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:30 smithi177 bash[24494]: cluster 2024-09-13T14:31:29.361522+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-09-13T14:31:30.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:30 smithi099 bash[22656]: audit 2024-09-13T14:31:28.902851+0000 mgr.y (mgr.25724) 78 : audit [DBG] from='client.16281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:30 smithi099 bash[22656]: audit 2024-09-13T14:31:29.219870+0000 mgr.y (mgr.25724) 79 : audit [DBG] from='client.26174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:30 smithi099 bash[22656]: cluster 2024-09-13T14:31:29.361522+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-09-13T14:31:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:30 smithi099 bash[32082]: audit 2024-09-13T14:31:28.902851+0000 mgr.y (mgr.25724) 78 : audit [DBG] from='client.16281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:30 smithi099 bash[32082]: audit 2024-09-13T14:31:29.219870+0000 mgr.y (mgr.25724) 79 : audit [DBG] from='client.26174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:30 smithi099 bash[32082]: cluster 2024-09-13T14:31:29.361522+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-09-13T14:31:31.390 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 22941003~5735250 2024-09-13T14:31:31.398 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 24374815~2867625 2024-09-13T14:31:31.470 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 45882006~5735250 2024-09-13T14:31:31.472 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 47315818~2867625 2024-09-13T14:31:31.537 INFO:tasks.workunit.client.0.smithi099.stdout:write: 3 68823009~5735250 2024-09-13T14:31:31.539 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 70256821~2867625 2024-09-13T14:31:31.593 INFO:tasks.workunit.client.0.smithi099.stdout:write: 4 91764012~5735250 2024-09-13T14:31:31.595 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 93197824~2867625 2024-09-13T14:31:31.664 INFO:tasks.workunit.client.0.smithi099.stdout:write: 5 114705015~5735250 2024-09-13T14:31:31.756 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 116138827~2867625 2024-09-13T14:31:31.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[22656]: cluster 2024-09-13T14:31:30.292761+0000 mgr.y (mgr.25724) 80 : cluster [DBG] pgmap v77: 137 pgs: 137 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 9.5 MiB/s rd, 41 MiB/s wr, 479 op/s 2024-09-13T14:31:31.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[22656]: cluster 2024-09-13T14:31:30.375628+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2024-09-13T14:31:31.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[22656]: audit 2024-09-13T14:31:31.290327+0000 mon.a (mon.0) 1313 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:31.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[22656]: audit 2024-09-13T14:31:31.304381+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:31.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[22656]: audit 2024-09-13T14:31:31.311716+0000 mon.b (mon.1) 128 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:31.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[22656]: audit 2024-09-13T14:31:31.313509+0000 mon.b (mon.1) 129 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:31:31.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[22656]: audit 2024-09-13T14:31:31.320975+0000 mon.a (mon.0) 1315 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:31.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[32082]: cluster 2024-09-13T14:31:30.292761+0000 mgr.y (mgr.25724) 80 : cluster [DBG] pgmap v77: 137 pgs: 137 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 9.5 MiB/s rd, 41 MiB/s wr, 479 op/s 2024-09-13T14:31:31.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[32082]: cluster 2024-09-13T14:31:30.375628+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2024-09-13T14:31:31.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[32082]: audit 2024-09-13T14:31:31.290327+0000 mon.a (mon.0) 1313 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:31.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[32082]: audit 2024-09-13T14:31:31.304381+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:31.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[32082]: audit 2024-09-13T14:31:31.311716+0000 mon.b (mon.1) 128 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:31.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[32082]: audit 2024-09-13T14:31:31.313509+0000 mon.b (mon.1) 129 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:31:31.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:31 smithi099 bash[32082]: audit 2024-09-13T14:31:31.320975+0000 mon.a (mon.0) 1315 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:31.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:31 smithi177 bash[24494]: cluster 2024-09-13T14:31:30.292761+0000 mgr.y (mgr.25724) 80 : cluster [DBG] pgmap v77: 137 pgs: 137 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 9.5 MiB/s rd, 41 MiB/s wr, 479 op/s 2024-09-13T14:31:31.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:31 smithi177 bash[24494]: cluster 2024-09-13T14:31:30.375628+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2024-09-13T14:31:31.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:31 smithi177 bash[24494]: audit 2024-09-13T14:31:31.290327+0000 mon.a (mon.0) 1313 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:31.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:31 smithi177 bash[24494]: audit 2024-09-13T14:31:31.304381+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:31.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:31 smithi177 bash[24494]: audit 2024-09-13T14:31:31.311716+0000 mon.b (mon.1) 128 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:31.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:31 smithi177 bash[24494]: audit 2024-09-13T14:31:31.313509+0000 mon.b (mon.1) 129 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:31:31.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:31 smithi177 bash[24494]: audit 2024-09-13T14:31:31.320975+0000 mon.a (mon.0) 1315 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:32.023 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:32.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: cluster 2024-09-13T14:31:31.369339+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-13T14:31:32.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.415957+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:32.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: cephadm 2024-09-13T14:31:31.419974+0000 mgr.y (mgr.25724) 81 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-13T14:31:32.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.420650+0000 mon.b (mon.1) 131 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:31:32.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.430565+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:32.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.437200+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.439909+0000 mon.b (mon.1) 132 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.442288+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.453857+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.456226+0000 mon.b (mon.1) 133 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.459228+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: cephadm 2024-09-13T14:31:31.469198+0000 mgr.y (mgr.25724) 82 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.470260+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.471732+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[22656]: audit 2024-09-13T14:31:31.665853+0000 mon.a (mon.0) 1322 : audit [DBG] from='client.? 172.21.15.99:0/2137864905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: cluster 2024-09-13T14:31:31.369339+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-13T14:31:32.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.415957+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: cephadm 2024-09-13T14:31:31.419974+0000 mgr.y (mgr.25724) 81 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.420650+0000 mon.b (mon.1) 131 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.430565+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.437200+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.439909+0000 mon.b (mon.1) 132 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.442288+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.453857+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.456226+0000 mon.b (mon.1) 133 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.459228+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: cephadm 2024-09-13T14:31:31.469198+0000 mgr.y (mgr.25724) 82 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.470260+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.471732+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:31:32.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:32 smithi099 bash[32082]: audit 2024-09-13T14:31:31.665853+0000 mon.a (mon.0) 1322 : audit [DBG] from='client.? 172.21.15.99:0/2137864905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:32.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: cluster 2024-09-13T14:31:31.369339+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-13T14:31:32.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.415957+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:31:32.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: cephadm 2024-09-13T14:31:31.419974+0000 mgr.y (mgr.25724) 81 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-13T14:31:32.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.420650+0000 mon.b (mon.1) 131 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:31:32.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.430565+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:32.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.437200+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:31:32.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.439909+0000 mon.b (mon.1) 132 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:31:32.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.442288+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:31:32.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.453857+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:31:32.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.456226+0000 mon.b (mon.1) 133 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:31:32.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.459228+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.25724 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:31:32.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: cephadm 2024-09-13T14:31:31.469198+0000 mgr.y (mgr.25724) 82 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-13T14:31:32.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.470260+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:31:32.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.471732+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:31:32.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:32 smithi177 bash[24494]: audit 2024-09-13T14:31:31.665853+0000 mon.a (mon.0) 1322 : audit [DBG] from='client.? 172.21.15.99:0/2137864905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:33.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:33 smithi099 bash[22656]: audit 2024-09-13T14:31:32.014769+0000 mgr.y (mgr.25724) 83 : audit [DBG] from='client.16299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:33.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:33 smithi099 bash[22656]: cluster 2024-09-13T14:31:32.293486+0000 mgr.y (mgr.25724) 84 : cluster [DBG] pgmap v80: 137 pgs: 137 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 9.5 MiB/s rd, 41 MiB/s wr, 479 op/s 2024-09-13T14:31:33.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:33 smithi099 bash[22656]: audit 2024-09-13T14:31:32.360524+0000 mgr.y (mgr.25724) 85 : audit [DBG] from='client.26192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:33.792 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:33 smithi099 bash[32082]: audit 2024-09-13T14:31:32.014769+0000 mgr.y (mgr.25724) 83 : audit [DBG] from='client.16299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:33.792 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:33 smithi099 bash[32082]: cluster 2024-09-13T14:31:32.293486+0000 mgr.y (mgr.25724) 84 : cluster [DBG] pgmap v80: 137 pgs: 137 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 9.5 MiB/s rd, 41 MiB/s wr, 479 op/s 2024-09-13T14:31:33.792 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:33 smithi099 bash[32082]: audit 2024-09-13T14:31:32.360524+0000 mgr.y (mgr.25724) 85 : audit [DBG] from='client.26192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:33.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:33 smithi177 bash[24494]: audit 2024-09-13T14:31:32.014769+0000 mgr.y (mgr.25724) 83 : audit [DBG] from='client.16299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:33.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:33 smithi177 bash[24494]: cluster 2024-09-13T14:31:32.293486+0000 mgr.y (mgr.25724) 84 : cluster [DBG] pgmap v80: 137 pgs: 137 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 9.5 MiB/s rd, 41 MiB/s wr, 479 op/s 2024-09-13T14:31:33.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:33 smithi177 bash[24494]: audit 2024-09-13T14:31:32.360524+0000 mgr.y (mgr.25724) 85 : audit [DBG] from='client.26192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:34.002 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:31:34.457 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:31:34.604 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:34 smithi177 bash[61439]: ts=2024-09-13T14:31:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:35.112 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:35.733 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:35 smithi177 bash[24494]: cluster 2024-09-13T14:31:34.294226+0000 mgr.y (mgr.25724) 86 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 6.4 MiB/s rd, 27 MiB/s wr, 322 op/s 2024-09-13T14:31:35.733 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:35 smithi177 bash[24494]: audit 2024-09-13T14:31:34.770267+0000 mon.a (mon.0) 1323 : audit [DBG] from='client.? 172.21.15.99:0/15562742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:35.734 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:35 smithi177 bash[24494]: audit 2024-09-13T14:31:34.865617+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:35.734 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:35 smithi177 bash[24494]: audit 2024-09-13T14:31:34.872127+0000 mon.b (mon.1) 136 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:31:35.734 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:35 smithi177 bash[24494]: audit 2024-09-13T14:31:34.873238+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:31:35.734 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:35 smithi177 bash[24494]: audit 2024-09-13T14:31:34.874950+0000 mon.b (mon.1) 138 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:35.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[22656]: cluster 2024-09-13T14:31:34.294226+0000 mgr.y (mgr.25724) 86 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 6.4 MiB/s rd, 27 MiB/s wr, 322 op/s 2024-09-13T14:31:35.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[22656]: audit 2024-09-13T14:31:34.770267+0000 mon.a (mon.0) 1323 : audit [DBG] from='client.? 172.21.15.99:0/15562742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:35.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[22656]: audit 2024-09-13T14:31:34.865617+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:35.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[22656]: audit 2024-09-13T14:31:34.872127+0000 mon.b (mon.1) 136 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:31:35.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[22656]: audit 2024-09-13T14:31:34.873238+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:31:35.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[22656]: audit 2024-09-13T14:31:34.874950+0000 mon.b (mon.1) 138 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:35.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[32082]: cluster 2024-09-13T14:31:34.294226+0000 mgr.y (mgr.25724) 86 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 6.4 MiB/s rd, 27 MiB/s wr, 322 op/s 2024-09-13T14:31:35.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[32082]: audit 2024-09-13T14:31:34.770267+0000 mon.a (mon.0) 1323 : audit [DBG] from='client.? 172.21.15.99:0/15562742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:35.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[32082]: audit 2024-09-13T14:31:34.865617+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.25724 ' entity='mgr.y' 2024-09-13T14:31:35.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[32082]: audit 2024-09-13T14:31:34.872127+0000 mon.b (mon.1) 136 : audit [INF] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:31:35.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[32082]: audit 2024-09-13T14:31:34.873238+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:31:35.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:35 smithi099 bash[32082]: audit 2024-09-13T14:31:34.874950+0000 mon.b (mon.1) 138 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:31:36.104 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:35 smithi177 bash[61439]: ts=2024-09-13T14:31:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:36.104 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:35 smithi177 bash[61439]: ts=2024-09-13T14:31:35.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:36.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:36 smithi099 bash[22656]: cephadm 2024-09-13T14:31:34.856766+0000 mgr.y (mgr.25724) 87 : cephadm [INF] Upgrade: Updating mon.a 2024-09-13T14:31:36.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:36 smithi099 bash[22656]: cephadm 2024-09-13T14:31:34.873068+0000 mgr.y (mgr.25724) 88 : cephadm [INF] Deploying daemon mon.a on smithi099 2024-09-13T14:31:36.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:36 smithi099 bash[22656]: audit 2024-09-13T14:31:35.103656+0000 mgr.y (mgr.25724) 89 : audit [DBG] from='client.26204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:36.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:36 smithi099 bash[22656]: audit 2024-09-13T14:31:35.442809+0000 mgr.y (mgr.25724) 90 : audit [DBG] from='client.26210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:36.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:36 smithi099 bash[32082]: cephadm 2024-09-13T14:31:34.856766+0000 mgr.y (mgr.25724) 87 : cephadm [INF] Upgrade: Updating mon.a 2024-09-13T14:31:36.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:36 smithi099 bash[32082]: cephadm 2024-09-13T14:31:34.873068+0000 mgr.y (mgr.25724) 88 : cephadm [INF] Deploying daemon mon.a on smithi099 2024-09-13T14:31:36.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:36 smithi099 bash[32082]: audit 2024-09-13T14:31:35.103656+0000 mgr.y (mgr.25724) 89 : audit [DBG] from='client.26204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:36.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:36 smithi099 bash[32082]: audit 2024-09-13T14:31:35.442809+0000 mgr.y (mgr.25724) 90 : audit [DBG] from='client.26210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:36.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:36 smithi177 bash[24494]: cephadm 2024-09-13T14:31:34.856766+0000 mgr.y (mgr.25724) 87 : cephadm [INF] Upgrade: Updating mon.a 2024-09-13T14:31:36.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:36 smithi177 bash[24494]: cephadm 2024-09-13T14:31:34.873068+0000 mgr.y (mgr.25724) 88 : cephadm [INF] Deploying daemon mon.a on smithi099 2024-09-13T14:31:36.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:36 smithi177 bash[24494]: audit 2024-09-13T14:31:35.103656+0000 mgr.y (mgr.25724) 89 : audit [DBG] from='client.26204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:36.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:36 smithi177 bash[24494]: audit 2024-09-13T14:31:35.442809+0000 mgr.y (mgr.25724) 90 : audit [DBG] from='client.26210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:36.921 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.Stress2 (22520 ms) 2024-09-13T14:31:36.921 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMigration.StressLive 2024-09-13T14:31:36.974 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 83886080 2024-09-13T14:31:36.994 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 7578209~2304792 2024-09-13T14:31:37.026 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 35370763~3450938 2024-09-13T14:31:37.026 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 44819327~3117623 2024-09-13T14:31:37.060 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 14988996~6638125 2024-09-13T14:31:37.060 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 7527673~48773 2024-09-13T14:31:37.064 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 65482877~413506 2024-09-13T14:31:37.065 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 10033145~4746198 2024-09-13T14:31:37.102 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 47940153~6077935 2024-09-13T14:31:37.105 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap0 2024-09-13T14:31:37.278 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:31:37] "GET /metrics HTTP/1.1" 200 38906 "" "Prometheus/2.51.0" 2024-09-13T14:31:37.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:36 smithi177 bash[61439]: ts=2024-09-13T14:31:36.952Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:37.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:37 smithi099 bash[22656]: cluster 2024-09-13T14:31:36.295812+0000 mgr.y (mgr.25724) 91 : cluster [DBG] pgmap v82: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s rd, 33 MiB/s wr, 727 op/s 2024-09-13T14:31:37.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:37 smithi099 bash[32082]: cluster 2024-09-13T14:31:36.295812+0000 mgr.y (mgr.25724) 91 : cluster [DBG] pgmap v82: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s rd, 33 MiB/s wr, 727 op/s 2024-09-13T14:31:37.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:37 smithi177 bash[24494]: cluster 2024-09-13T14:31:36.295812+0000 mgr.y (mgr.25724) 91 : cluster [DBG] pgmap v82: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s rd, 33 MiB/s wr, 727 op/s 2024-09-13T14:31:38.133 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:38.133 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:31:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:38.134 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:38.134 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:31:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:38.134 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:38.135 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:31:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:38.135 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:31:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:38.135 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:31:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:38.135 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:31:38 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:38.159 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:38.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:38 smithi099 systemd[1]: Stopping Ceph mon.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:31:38.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:38 smithi099 bash[22656]: debug 2024-09-13T14:31:38.256+0000 7fea4fc0b640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T14:31:38.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:38 smithi099 bash[22656]: debug 2024-09-13T14:31:38.256+0000 7fea4fc0b640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T14:31:39.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:39 smithi099 bash[84440]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:31:39.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:39 smithi099 bash[84521]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:31:39.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T14:31:39.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: Stopped Ceph mon.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:31:39.819 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:39.819 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:39.820 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:39.820 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:39.820 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:39.820 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:39.821 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: Started Ceph mon.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:31:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:39 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:31:40.603 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:40 smithi177 bash[61439]: ts=2024-09-13T14:31:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:41.141 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:31:41.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.900+0000 7f068fe04c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:31:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.900+0000 7f068fe04c80 0 ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:31:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.904+0000 7f068fe04c80 0 pidfile_write: ignore empty --pid-file 2024-09-13T14:31:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.904+0000 7f068fe04c80 0 load: jerasure load: lrc 2024-09-13T14:31:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: RocksDB version: 7.9.2 2024-09-13T14:31:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Git sha 0 2024-09-13T14:31:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:31:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: DB SUMMARY 2024-09-13T14:31:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: DB Session ID: 1VYHDGKYNA6KKJAW7J6S 2024-09-13T14:31:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: CURRENT file: CURRENT 2024-09-13T14:31:41.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: MANIFEST file: MANIFEST-000015 size: 1939 Bytes 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000045.sst 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000043.log size: 1949952 ; 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.error_if_exists: 0 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.create_if_missing: 0 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.env: 0x5592840bf160 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.info_log: 0x55928622b880 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:31:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.statistics: (nil) 2024-09-13T14:31:41.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.use_fsync: 0 2024-09-13T14:31:41.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T14:31:41.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:31:41.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.db_log_dir: 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.wal_dir: 2024-09-13T14:31:41.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.write_buffer_manager: 0x55928622e960 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:31:41.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.unordered_write: 0 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.row_cache: None 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.wal_filter: None 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:31:41.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.two_write_queues: 0 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.wal_compression: 0 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.atomic_flush: 0 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:31:41.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_open_files: -1 2024-09-13T14:31:41.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Compression algorithms supported: 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: kZSTD supported: 0 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: kXpressCompression supported: 0 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: kZlibCompression supported: 1 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:31:41.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:31:41.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-13T14:31:41.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:31:41.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:31:41.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.merge_operator: 2024-09-13T14:31:41.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_filter: None 2024-09-13T14:31:41.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:31:41.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:31:41.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:31:41.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55928622b4e0) 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: cache_index_and_filter_blocks: 1 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: pin_top_level_index_and_filter: 1 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: index_type: 0 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: data_block_index_type: 0 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: index_shortening: 1 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: checksum: 4 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: no_block_cache: 0 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: block_cache: 0x559286220850 2024-09-13T14:31:41.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: block_cache_name: BinnedLRUCache 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: block_cache_options: 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: capacity : 536870912 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: num_shard_bits : 4 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: strict_capacity_limit : 0 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: high_pri_pool_ratio: 0.000 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: block_cache_compressed: (nil) 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: persistent_cache: (nil) 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: block_size: 4096 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: block_size_deviation: 10 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: block_restart_interval: 16 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: index_block_restart_interval: 1 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: metadata_block_size: 4096 2024-09-13T14:31:41.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: partition_filters: 0 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: use_delta_encoding: 1 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: filter_policy: bloomfilter 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: whole_key_filtering: 1 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: verify_compression: 0 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: read_amp_bytes_per_bit: 0 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: format_version: 5 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: enable_index_compression: 1 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: block_align: 0 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: max_auto_readahead_size: 262144 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: prepopulate_block_cache: 0 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: initial_auto_readahead_size: 8192 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: num_file_reads_for_auto_readahead: 2 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:31:41.333 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compression: NoCompression 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.num_levels: 7 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:31:41.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:31:41.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:31:41.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:31:41.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:31:41.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:31:41.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:31:41.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:31:41.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:31:41.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:31:41.336 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:31:41.337 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:31:41.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.bloom_locality: 0 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.ttl: 2592000 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:31:41.339 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.enable_blob_files: false 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.min_blob_size: 0 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068cbda640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 45.sst 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:31:41.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 47, last_sequence is 20186, log_number is 43,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 43 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9efea801-3da2-4bdd-8ce2-edd00894fe79 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726237900911533, "job": 1, "event": "recovery_started", "wal_files": [43]} 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.908+0000 7f068fe04c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #43 mode 2 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.916+0000 7f068fe04c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726237900918234, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 48, "file_size": 1838991, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 20187, "largest_seqno": 21357, "table_properties": {"data_size": 1832355, "index_size": 4251, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 12088, "raw_average_key_size": 23, "raw_value_size": 1820387, "raw_average_value_size": 3500, "num_data_blocks": 187, "num_entries": 520, "num_filter_entries": 520, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237900, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9efea801-3da2-4bdd-8ce2-edd00894fe79", "db_session_id": "1VYHDGKYNA6KKJAW7J6S", "orig_file_number": 48, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.916+0000 7f068fe04c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726237900918367, "job": 1, "event": "recovery_finished"} 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.916+0000 7f068fe04c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 50 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.916+0000 7f068fe04c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000043.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55928624ee00 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 4 rocksdb: DB pointer 0x559286352000 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f0685bcc640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:31:41.341 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f0685bcc640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: ** DB Stats ** 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: ** Compaction Stats [default] ** 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: L0 1/0 1.75 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 503.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: L6 1/0 10.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Sum 2/0 12.30 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 503.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T14:31:41.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 503.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: ** Compaction Stats [default] ** 2024-09-13T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 503.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Flush(GB): cumulative 0.002, interval 0.002 2024-09-13T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Cumulative compaction: 0.00 GB write, 145.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Interval compaction: 0.00 GB write, 145.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Block cache BinnedLRUCache@0x559286220850#7 capacity: 512.00 MB usage: 5.98 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] at bind addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 1 mon.a@-1(???) e3 preinit fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 0 mon.a@-1(???).mds e9 new map 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 0 mon.a@-1(???).mds e9 print_map 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: e9 2024-09-13T14:31:41.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: legacy client fscid: 1 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Filesystem 'foo' (1) 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: fs_name foo 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: epoch 9 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: created 2024-09-13T14:24:17.237575+0000 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: modified 2024-09-13T14:24:29.188501+0000 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: tableserver 0 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: root 0 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: session_timeout 60 2024-09-13T14:31:41.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: session_autoclose 300 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: max_file_size 1099511627776 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: max_xattr_size 65536 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: required_client_features {} 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: last_failure 0 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: last_failure_osd_epoch 65 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: max_mds 1 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: in 0 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: up {0=24403} 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: failed 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: damaged 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: stopped 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: data_pools [4] 2024-09-13T14:31:41.346 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: metadata_pool 3 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: inline_data disabled 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: balancer 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: bal_rank_mask -1 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: standby_count_wanted 1 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: qdb_cluster leader: 0 members: 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: [mds.foo.smithi177.outtkv{0:24403} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: Standby daemons: 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: [mds.foo.smithi099.yyrtld{-1:24488} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.99:6834/2117449908,v1:172.21.15.99:6835/2117449908] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 0 mon.a@-1(???).osd e336 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 0 mon.a@-1(???).osd e336 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 0 mon.a@-1(???).osd e336 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:31:41.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 0 mon.a@-1(???).osd e336 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:31:41.348 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:40 smithi099 bash[84576]: debug 2024-09-13T14:31:40.920+0000 7f068fe04c80 1 mon.a@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-13T14:31:41.650 INFO:tasks.workunit.client.0.smithi099.stdout:snap_protect: snap0 2024-09-13T14:31:41.663 INFO:tasks.workunit.client.0.smithi099.stdout:clone image187 -> image190 2024-09-13T14:31:41.775 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 47429165~3455258 2024-09-13T14:31:41.853 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 56923870~4665183 2024-09-13T14:31:41.866 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 56421871~2528293 2024-09-13T14:31:41.886 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 50597846~5952796 2024-09-13T14:31:41.891 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 76901129~6188204 2024-09-13T14:31:41.944 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 61010712~333732 2024-09-13T14:31:41.949 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 78135996~2249240 2024-09-13T14:31:41.969 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 47643218~1480057 2024-09-13T14:31:41.974 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap1 2024-09-13T14:31:42.069 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[76324]: ignoring --setuser ceph since I am not root 2024-09-13T14:31:42.069 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[76324]: ignoring --setgroup ceph since I am not root 2024-09-13T14:31:42.069 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[76324]: debug 2024-09-13T14:31:41.676+0000 7fd5bb050640 1 -- 172.21.15.99:0/2536247458 <== mon.0 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564708c350e0 con 0x564708c37800 2024-09-13T14:31:42.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: audit 2024-09-13T14:31:38.152180+0000 mgr.y (mgr.25724) 92 : audit [DBG] from='client.16335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:42.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: audit 2024-09-13T14:31:38.152180+0000 mgr.y (mgr.25724) 92 : audit [DBG] from='client.16335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:42.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:38.296258+0000 mgr.y (mgr.25724) 93 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 103 MiB/s rd, 7.9 MiB/s wr, 394 op/s 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:38.296258+0000 mgr.y (mgr.25724) 93 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 103 MiB/s rd, 7.9 MiB/s wr, 394 op/s 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: audit 2024-09-13T14:31:38.451265+0000 mgr.y (mgr.25724) 94 : audit [DBG] from='client.16341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: audit 2024-09-13T14:31:38.451265+0000 mgr.y (mgr.25724) 94 : audit [DBG] from='client.16341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: audit 2024-09-13T14:31:39.086798+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: audit 2024-09-13T14:31:39.086798+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:40.297933+0000 mgr.y (mgr.25724) 95 : cluster [DBG] pgmap v85: 137 pgs: 137 active+clean; 1.9 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 180 MiB/s rd, 9.2 MiB/s wr, 442 op/s 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:40.297933+0000 mgr.y (mgr.25724) 95 : cluster [DBG] pgmap v85: 137 pgs: 137 active+clean; 1.9 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 180 MiB/s rd, 9.2 MiB/s wr, 442 op/s 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: audit 2024-09-13T14:31:40.832296+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.99:0/2352043771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: audit 2024-09-13T14:31:40.832296+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.99:0/2352043771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.539935+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.539935+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.559423+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:31:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.559423+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:31:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570715+0000 mon.a (mon.0) 3 : cluster [DBG] monmap epoch 3 2024-09-13T14:31:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570715+0000 mon.a (mon.0) 3 : cluster [DBG] monmap epoch 3 2024-09-13T14:31:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570746+0000 mon.a (mon.0) 4 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:31:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570746+0000 mon.a (mon.0) 4 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:31:42.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: audit 2024-09-13T14:31:38.152180+0000 mgr.y (mgr.25724) 92 : audit [DBG] from='client.16335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:42.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:38.296258+0000 mgr.y (mgr.25724) 93 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 103 MiB/s rd, 7.9 MiB/s wr, 394 op/s 2024-09-13T14:31:42.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: audit 2024-09-13T14:31:38.451265+0000 mgr.y (mgr.25724) 94 : audit [DBG] from='client.16341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:42.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: audit 2024-09-13T14:31:39.086798+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:40.297933+0000 mgr.y (mgr.25724) 95 : cluster [DBG] pgmap v85: 137 pgs: 137 active+clean; 1.9 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 180 MiB/s rd, 9.2 MiB/s wr, 442 op/s 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: audit 2024-09-13T14:31:40.832296+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.99:0/2352043771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.539935+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.559423+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.570715+0000 mon.a (mon.0) 3 : cluster [DBG] monmap epoch 3 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.570746+0000 mon.a (mon.0) 4 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.570764+0000 mon.a (mon.0) 5 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.570779+0000 mon.a (mon.0) 6 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.570794+0000 mon.a (mon.0) 7 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.570811+0000 mon.a (mon.0) 8 : cluster [DBG] election_strategy: 1 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.570832+0000 mon.a (mon.0) 9 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.570850+0000 mon.a (mon.0) 10 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:31:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.570875+0000 mon.a (mon.0) 11 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:31:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.572070+0000 mon.a (mon.0) 12 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:31:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.572132+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-09-13T14:31:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.574588+0000 mon.a (mon.0) 14 : cluster [DBG] mgrmap e33: y(active, since 63s), standbys: x 2024-09-13T14:31:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.575738+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-13T14:31:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.575770+0000 mon.a (mon.0) 16 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T14:31:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.575791+0000 mon.a (mon.0) 17 : cluster [WRN] application not enabled on pool 'test-librbd-smithi099-69110-8' 2024-09-13T14:31:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.575809+0000 mon.a (mon.0) 18 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T14:31:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: audit 2024-09-13T14:31:41.603928+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25724 ' entity='' 2024-09-13T14:31:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.606067+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e34: y(active, since 63s), standbys: x 2024-09-13T14:31:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570764+0000 mon.a (mon.0) 5 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:31:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570764+0000 mon.a (mon.0) 5 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:31:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570779+0000 mon.a (mon.0) 6 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:31:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570779+0000 mon.a (mon.0) 6 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:31:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570794+0000 mon.a (mon.0) 7 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:31:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570794+0000 mon.a (mon.0) 7 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:31:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570811+0000 mon.a (mon.0) 8 : cluster [DBG] election_strategy: 1 2024-09-13T14:31:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570811+0000 mon.a (mon.0) 8 : cluster [DBG] election_strategy: 1 2024-09-13T14:31:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570832+0000 mon.a (mon.0) 9 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570832+0000 mon.a (mon.0) 9 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570850+0000 mon.a (mon.0) 10 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570850+0000 mon.a (mon.0) 10 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570875+0000 mon.a (mon.0) 11 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.570875+0000 mon.a (mon.0) 11 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.572070+0000 mon.a (mon.0) 12 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.572070+0000 mon.a (mon.0) 12 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.572132+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.572132+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.574588+0000 mon.a (mon.0) 14 : cluster [DBG] mgrmap e33: y(active, since 63s), standbys: x 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.574588+0000 mon.a (mon.0) 14 : cluster [DBG] mgrmap e33: y(active, since 63s), standbys: x 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.575738+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-13T14:31:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.575738+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-13T14:31:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.575770+0000 mon.a (mon.0) 16 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T14:31:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.575770+0000 mon.a (mon.0) 16 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T14:31:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.575791+0000 mon.a (mon.0) 17 : cluster [WRN] application not enabled on pool 'test-librbd-smithi099-69110-8' 2024-09-13T14:31:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.575791+0000 mon.a (mon.0) 17 : cluster [WRN] application not enabled on pool 'test-librbd-smithi099-69110-8' 2024-09-13T14:31:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.575809+0000 mon.a (mon.0) 18 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T14:31:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.575809+0000 mon.a (mon.0) 18 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T14:31:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: audit 2024-09-13T14:31:41.603928+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25724 ' entity='' 2024-09-13T14:31:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: audit 2024-09-13T14:31:41.603928+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25724 ' entity='' 2024-09-13T14:31:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.606067+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e34: y(active, since 63s), standbys: x 2024-09-13T14:31:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:41 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.606067+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e34: y(active, since 63s), standbys: x 2024-09-13T14:31:42.104 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[61832]: ignoring --setuser ceph since I am not root 2024-09-13T14:31:42.104 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[61832]: ignoring --setgroup ceph since I am not root 2024-09-13T14:31:42.104 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[61832]: debug 2024-09-13T14:31:41.681+0000 7fb0ff86c640 1 -- 172.21.15.177:0/1885763112 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5587256b10e0 con 0x5587256b3800 2024-09-13T14:31:42.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: audit 2024-09-13T14:31:38.152180+0000 mgr.y (mgr.25724) 92 : audit [DBG] from='client.16335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:42.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:38.296258+0000 mgr.y (mgr.25724) 93 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 103 MiB/s rd, 7.9 MiB/s wr, 394 op/s 2024-09-13T14:31:42.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: audit 2024-09-13T14:31:38.451265+0000 mgr.y (mgr.25724) 94 : audit [DBG] from='client.16341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: audit 2024-09-13T14:31:39.086798+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.25724 172.21.15.99:0/1237045805' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:40.297933+0000 mgr.y (mgr.25724) 95 : cluster [DBG] pgmap v85: 137 pgs: 137 active+clean; 1.9 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 180 MiB/s rd, 9.2 MiB/s wr, 442 op/s 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: audit 2024-09-13T14:31:40.832296+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.99:0/2352043771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.539935+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.559423+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.570715+0000 mon.a (mon.0) 3 : cluster [DBG] monmap epoch 3 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.570746+0000 mon.a (mon.0) 4 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.570764+0000 mon.a (mon.0) 5 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.570779+0000 mon.a (mon.0) 6 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.570794+0000 mon.a (mon.0) 7 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.570811+0000 mon.a (mon.0) 8 : cluster [DBG] election_strategy: 1 2024-09-13T14:31:42.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.570832+0000 mon.a (mon.0) 9 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.570850+0000 mon.a (mon.0) 10 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.570875+0000 mon.a (mon.0) 11 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.572070+0000 mon.a (mon.0) 12 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.572132+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.574588+0000 mon.a (mon.0) 14 : cluster [DBG] mgrmap e33: y(active, since 63s), standbys: x 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.575738+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.575770+0000 mon.a (mon.0) 16 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.575791+0000 mon.a (mon.0) 17 : cluster [WRN] application not enabled on pool 'test-librbd-smithi099-69110-8' 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.575809+0000 mon.a (mon.0) 18 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: audit 2024-09-13T14:31:41.603928+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25724 ' entity='' 2024-09-13T14:31:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:41 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.606067+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e34: y(active, since 63s), standbys: x 2024-09-13T14:31:43.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:42 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.643288+0000 mon.a (mon.0) 21 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-09-13T14:31:43.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:42 smithi099 bash[84576]: cluster 2024-09-13T14:31:41.643288+0000 mon.a (mon.0) 21 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-09-13T14:31:43.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:42 smithi099 bash[32082]: cluster 2024-09-13T14:31:41.643288+0000 mon.a (mon.0) 21 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-09-13T14:31:43.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:42 smithi177 bash[24494]: cluster 2024-09-13T14:31:41.643288+0000 mon.a (mon.0) 21 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-09-13T14:31:43.679 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 92462546 2024-09-13T14:31:43.704 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 52765393~2704373 2024-09-13T14:31:43.718 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 9292757~236631 2024-09-13T14:31:43.718 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 36608667~3222967 2024-09-13T14:31:43.724 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 70661362~6589637 2024-09-13T14:31:43.724 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 51336204~4688580 2024-09-13T14:31:43.765 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 9113124~4380266 2024-09-13T14:31:43.786 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 49930851~7653401 2024-09-13T14:31:43.798 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 9924889~6704706 2024-09-13T14:31:43.833 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap2 2024-09-13T14:31:43.867 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:31:43.896 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:43 smithi099 bash[32082]: cluster 2024-09-13T14:31:42.650889+0000 mon.a (mon.0) 22 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-09-13T14:31:43.896 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:43 smithi099 bash[84576]: cluster 2024-09-13T14:31:42.650889+0000 mon.a (mon.0) 22 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-09-13T14:31:43.896 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:43 smithi099 bash[84576]: cluster 2024-09-13T14:31:42.650889+0000 mon.a (mon.0) 22 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-09-13T14:31:44.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:43 smithi177 bash[24494]: cluster 2024-09-13T14:31:42.650889+0000 mon.a (mon.0) 22 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-09-13T14:31:44.604 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:44 smithi177 bash[61439]: ts=2024-09-13T14:31:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:44.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:31:44.916 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:44 smithi099 bash[32082]: cluster 2024-09-13T14:31:43.657710+0000 mon.a (mon.0) 23 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-09-13T14:31:44.916 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:44 smithi099 bash[32082]: audit 2024-09-13T14:31:43.862554+0000 mon.a (mon.0) 24 : audit [DBG] from='client.? 172.21.15.99:0/4026963524' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:31:44.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:44 smithi099 bash[84576]: cluster 2024-09-13T14:31:43.657710+0000 mon.a (mon.0) 23 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-09-13T14:31:44.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:44 smithi099 bash[84576]: cluster 2024-09-13T14:31:43.657710+0000 mon.a (mon.0) 23 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-09-13T14:31:44.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:44 smithi099 bash[84576]: audit 2024-09-13T14:31:43.862554+0000 mon.a (mon.0) 24 : audit [DBG] from='client.? 172.21.15.99:0/4026963524' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:31:44.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:44 smithi099 bash[84576]: audit 2024-09-13T14:31:43.862554+0000 mon.a (mon.0) 24 : audit [DBG] from='client.? 172.21.15.99:0/4026963524' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:31:45.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:44 smithi177 bash[24494]: cluster 2024-09-13T14:31:43.657710+0000 mon.a (mon.0) 23 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-09-13T14:31:45.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:44 smithi177 bash[24494]: audit 2024-09-13T14:31:43.862554+0000 mon.a (mon.0) 24 : audit [DBG] from='client.? 172.21.15.99:0/4026963524' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:31:45.709 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 125868879 2024-09-13T14:31:45.733 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 102471623~4272274 2024-09-13T14:31:45.766 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 4287934~4824861 2024-09-13T14:31:45.766 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 48579792~4496374 2024-09-13T14:31:45.779 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 47919471~7244531 2024-09-13T14:31:45.823 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 61553927~4385671 2024-09-13T14:31:45.838 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 54785900~2185504 2024-09-13T14:31:45.839 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 88801527~6015152 2024-09-13T14:31:45.863 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 24556289~4810161 2024-09-13T14:31:45.863 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: snap3 2024-09-13T14:31:46.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:45 smithi099 bash[32082]: cluster 2024-09-13T14:31:44.666264+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-09-13T14:31:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:45 smithi099 bash[84576]: cluster 2024-09-13T14:31:44.666264+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-09-13T14:31:46.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:45 smithi099 bash[84576]: cluster 2024-09-13T14:31:44.666264+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-09-13T14:31:46.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:45 smithi177 bash[24494]: cluster 2024-09-13T14:31:44.666264+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-09-13T14:31:46.104 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:45 smithi177 bash[61439]: ts=2024-09-13T14:31:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:46.104 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:45 smithi177 bash[61439]: ts=2024-09-13T14:31:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:46.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:46 smithi177 bash[24494]: cluster 2024-09-13T14:31:45.696981+0000 mon.a (mon.0) 26 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-09-13T14:31:47.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:46 smithi099 bash[32082]: cluster 2024-09-13T14:31:45.696981+0000 mon.a (mon.0) 26 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-09-13T14:31:47.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:46 smithi099 bash[84576]: cluster 2024-09-13T14:31:45.696981+0000 mon.a (mon.0) 26 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-09-13T14:31:47.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:46 smithi099 bash[84576]: cluster 2024-09-13T14:31:45.696981+0000 mon.a (mon.0) 26 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-09-13T14:31:47.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:46 smithi177 bash[61439]: ts=2024-09-13T14:31:46.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:47.735 INFO:tasks.workunit.client.0.smithi099.stdout:migration_prepare 2024-09-13T14:31:47.954 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:47 smithi177 bash[24494]: cluster 2024-09-13T14:31:46.700383+0000 mon.a (mon.0) 27 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-09-13T14:31:47.954 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:47 smithi177 bash[61832]: [13/Sep/2024:14:31:47] ENGINE Bus STARTING 2024-09-13T14:31:47.954 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:47 smithi177 bash[61832]: CherryPy Checker: 2024-09-13T14:31:47.954 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:47 smithi177 bash[61832]: The Application mounted at '' has an empty config. 2024-09-13T14:31:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:47 smithi099 bash[84576]: cluster 2024-09-13T14:31:46.700383+0000 mon.a (mon.0) 27 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-09-13T14:31:48.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:47 smithi099 bash[84576]: cluster 2024-09-13T14:31:46.700383+0000 mon.a (mon.0) 27 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-09-13T14:31:48.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:47 smithi099 bash[32082]: cluster 2024-09-13T14:31:46.700383+0000 mon.a (mon.0) 27 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-09-13T14:31:48.357 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:47 smithi177 bash[61832]: [13/Sep/2024:14:31:47] ENGINE Serving on http://:::9283 2024-09-13T14:31:48.357 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:31:47 smithi177 bash[61832]: [13/Sep/2024:14:31:47] ENGINE Bus STARTED 2024-09-13T14:31:48.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.296+0000 7fd5298ab640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:48.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.296+0000 7fd5298ab640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:48.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.296+0000 7fd5298ab640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:48.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.296+0000 7fd5298ab640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:48.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.296+0000 7fd5298ab640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:48.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.300+0000 7fd52c0b0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:48.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.300+0000 7fd52c0b0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:48.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.300+0000 7fd52c0b0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:48.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.300+0000 7fd52c0b0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:48.569 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.300+0000 7fd52c0b0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:31:49.037 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: debug 2024-09-13T14:31:48.928+0000 7fd55bc0f640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-13T14:31:49.037 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: [13/Sep/2024:14:31:48] ENGINE Bus STARTING 2024-09-13T14:31:49.037 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: CherryPy Checker: 2024-09-13T14:31:49.037 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[76324]: The Application mounted at '' has an empty config. 2024-09-13T14:31:49.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.713270+0000 mon.a (mon.0) 28 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-09-13T14:31:49.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.713270+0000 mon.a (mon.0) 28 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.853255+0000 mon.a (mon.0) 29 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.853255+0000 mon.a (mon.0) 29 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.853423+0000 mon.a (mon.0) 30 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.853423+0000 mon.a (mon.0) 30 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.858334+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.858334+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.859026+0000 mon.b (mon.1) 141 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.859026+0000 mon.b (mon.1) 141 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.861055+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.861055+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:31:49.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.862251+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.862251+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.884951+0000 mon.a (mon.0) 31 : cluster [INF] Active manager daemon y restarted 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.884951+0000 mon.a (mon.0) 31 : cluster [INF] Active manager daemon y restarted 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.885786+0000 mon.a (mon.0) 32 : cluster [INF] Activating manager daemon y 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.885786+0000 mon.a (mon.0) 32 : cluster [INF] Activating manager daemon y 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.894892+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.894892+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.899434+0000 mon.a (mon.0) 34 : cluster [DBG] mgrmap e35: y(active, starting, since 0.0138312s), standbys: x 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.899434+0000 mon.a (mon.0) 34 : cluster [DBG] mgrmap e35: y(active, starting, since 0.0138312s), standbys: x 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: cluster 2024-09-13T14:31:47.713270+0000 mon.a (mon.0) 28 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-09-13T14:31:49.039 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: cluster 2024-09-13T14:31:47.853255+0000 mon.a (mon.0) 29 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: cluster 2024-09-13T14:31:47.853423+0000 mon.a (mon.0) 30 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.858334+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.859026+0000 mon.b (mon.1) 141 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.861055+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.862251+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: cluster 2024-09-13T14:31:47.884951+0000 mon.a (mon.0) 31 : cluster [INF] Active manager daemon y restarted 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: cluster 2024-09-13T14:31:47.885786+0000 mon.a (mon.0) 32 : cluster [INF] Activating manager daemon y 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: cluster 2024-09-13T14:31:47.894892+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: cluster 2024-09-13T14:31:47.899434+0000 mon.a (mon.0) 34 : cluster [DBG] mgrmap e35: y(active, starting, since 0.0138312s), standbys: x 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.905770+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.905949+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.906227+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.906681+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi177.outtkv"} : dispatch 2024-09-13T14:31:49.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.907112+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"} : dispatch 2024-09-13T14:31:49.041 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.907811+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-13T14:31:49.041 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.908021+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-13T14:31:49.041 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.908266+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:31:49.041 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.908525+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:31:49.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.905770+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:31:49.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.905770+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:31:49.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.905949+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:31:49.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.905949+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:31:49.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.906227+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:31:49.041 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.906227+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.906681+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi177.outtkv"} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.906681+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi177.outtkv"} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.907112+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.907112+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.907811+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.907811+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.908021+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.908021+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.908266+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.908266+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.908525+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:31:49.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.908525+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.908759+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.909015+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.909259+0000 mon.a (mon.0) 46 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.909488+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.909695+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.909949+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.910192+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.910434+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:47.910899+0000 mon.a (mon.0) 52 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: cluster 2024-09-13T14:31:47.919415+0000 mon.a (mon.0) 53 : cluster [INF] Manager daemon y is now available 2024-09-13T14:31:49.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:48.210181+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:49.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:48.236995+0000 mon.a (mon.0) 55 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:49.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:48.241391+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:31:49.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:48.252795+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:31:49.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[32082]: audit 2024-09-13T14:31:48.282949+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:31:49.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.908759+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:31:49.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.908759+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:31:49.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.909015+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:31:49.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.909015+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:31:49.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.909259+0000 mon.a (mon.0) 46 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.909259+0000 mon.a (mon.0) 46 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.909488+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.909488+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.909695+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.909695+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.909949+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.909949+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.910192+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.910192+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.910434+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.910434+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.910899+0000 mon.a (mon.0) 52 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:47.910899+0000 mon.a (mon.0) 52 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T14:31:49.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.919415+0000 mon.a (mon.0) 53 : cluster [INF] Manager daemon y is now available 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: cluster 2024-09-13T14:31:47.919415+0000 mon.a (mon.0) 53 : cluster [INF] Manager daemon y is now available 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:48.210181+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:48.210181+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:48.236995+0000 mon.a (mon.0) 55 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:48.236995+0000 mon.a (mon.0) 55 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:48.241391+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:48.241391+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:48.252795+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:48.252795+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:48.282949+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:31:49.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:48 smithi099 bash[84576]: audit 2024-09-13T14:31:48.282949+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:31:49.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: cluster 2024-09-13T14:31:47.713270+0000 mon.a (mon.0) 28 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-09-13T14:31:49.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: cluster 2024-09-13T14:31:47.853255+0000 mon.a (mon.0) 29 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T14:31:49.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: cluster 2024-09-13T14:31:47.853423+0000 mon.a (mon.0) 30 : cluster [DBG] Standby manager daemon x started 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.858334+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.859026+0000 mon.b (mon.1) 141 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.861055+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.862251+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.? 172.21.15.177:0/2679617261' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: cluster 2024-09-13T14:31:47.884951+0000 mon.a (mon.0) 31 : cluster [INF] Active manager daemon y restarted 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: cluster 2024-09-13T14:31:47.885786+0000 mon.a (mon.0) 32 : cluster [INF] Activating manager daemon y 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: cluster 2024-09-13T14:31:47.894892+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: cluster 2024-09-13T14:31:47.899434+0000 mon.a (mon.0) 34 : cluster [DBG] mgrmap e35: y(active, starting, since 0.0138312s), standbys: x 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.905770+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.905949+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.906227+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.906681+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi177.outtkv"} : dispatch 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.907112+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi099.yyrtld"} : dispatch 2024-09-13T14:31:49.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.907811+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.908021+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.908266+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.908525+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.908759+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.909015+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.909259+0000 mon.a (mon.0) 46 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.909488+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.909695+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.909949+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.910192+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.910434+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:47.910899+0000 mon.a (mon.0) 52 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: cluster 2024-09-13T14:31:47.919415+0000 mon.a (mon.0) 53 : cluster [INF] Manager daemon y is now available 2024-09-13T14:31:49.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:48.210181+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:49.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:48.236995+0000 mon.a (mon.0) 55 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:49.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:48.241391+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:31:49.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:48.252795+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:31:49.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:48 smithi177 bash[24494]: audit 2024-09-13T14:31:48.282949+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:31:49.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: [13/Sep/2024:14:31:49] ENGINE Serving on http://:::9283 2024-09-13T14:31:49.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: [13/Sep/2024:14:31:49] ENGINE Bus STARTED 2024-09-13T14:31:49.660 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:50.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[32082]: cluster 2024-09-13T14:31:48.900208+0000 mon.a (mon.0) 59 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-09-13T14:31:50.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[32082]: cluster 2024-09-13T14:31:48.919364+0000 mon.a (mon.0) 60 : cluster [DBG] mgrmap e36: y(active, since 1.03376s), standbys: x 2024-09-13T14:31:50.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[32082]: cephadm 2024-09-13T14:31:49.610058+0000 mgr.y (mgr.26252) 3 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Bus STARTING 2024-09-13T14:31:50.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[84576]: cluster 2024-09-13T14:31:48.900208+0000 mon.a (mon.0) 59 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-09-13T14:31:50.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[84576]: cluster 2024-09-13T14:31:48.900208+0000 mon.a (mon.0) 59 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-09-13T14:31:50.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[84576]: cluster 2024-09-13T14:31:48.919364+0000 mon.a (mon.0) 60 : cluster [DBG] mgrmap e36: y(active, since 1.03376s), standbys: x 2024-09-13T14:31:50.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[84576]: cluster 2024-09-13T14:31:48.919364+0000 mon.a (mon.0) 60 : cluster [DBG] mgrmap e36: y(active, since 1.03376s), standbys: x 2024-09-13T14:31:50.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[84576]: cephadm 2024-09-13T14:31:49.610058+0000 mgr.y (mgr.26252) 3 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Bus STARTING 2024-09-13T14:31:50.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[84576]: cephadm 2024-09-13T14:31:49.610058+0000 mgr.y (mgr.26252) 3 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Bus STARTING 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: debug 2024-09-13T14:31:49.828+0000 7fd509efc640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:31:49] ENGINE Error in HTTPServer.serve 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: Traceback (most recent call last): 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: self._connections.run(self.expiration_interval) 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: self._run(expiration_interval) 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:31:50.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: s = self.context.wrap_socket( 2024-09-13T14:31:50.072 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:31:50.072 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: return self.sslsocket_class._create( 2024-09-13T14:31:50.072 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:31:50.072 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: self.do_handshake() 2024-09-13T14:31:50.072 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:31:50.072 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: self._sslobj.do_handshake() 2024-09-13T14:31:50.072 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:49 smithi099 bash[76324]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:31:50.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:49 smithi177 bash[24494]: cluster 2024-09-13T14:31:48.900208+0000 mon.a (mon.0) 59 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-09-13T14:31:50.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:49 smithi177 bash[24494]: cluster 2024-09-13T14:31:48.919364+0000 mon.a (mon.0) 60 : cluster [DBG] mgrmap e36: y(active, since 1.03376s), standbys: x 2024-09-13T14:31:50.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:49 smithi177 bash[24494]: cephadm 2024-09-13T14:31:49.610058+0000 mgr.y (mgr.26252) 3 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Bus STARTING 2024-09-13T14:31:50.604 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[61439]: ts=2024-09-13T14:31:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:51.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: cephadm 2024-09-13T14:31:49.712587+0000 mgr.y (mgr.26252) 4 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Serving on http://172.21.15.99:8765 2024-09-13T14:31:51.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: cephadm 2024-09-13T14:31:49.829865+0000 mgr.y (mgr.26252) 5 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:31:51.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: cephadm 2024-09-13T14:31:49.830098+0000 mgr.y (mgr.26252) 6 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Bus STARTED 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: cephadm 2024-09-13T14:31:49.831895+0000 mgr.y (mgr.26252) 7 : cephadm [ERR] [13/Sep/2024:14:31:49] ENGINE Error in HTTPServer.serve 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: Traceback (most recent call last): 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: self._connections.run(self.expiration_interval) 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: self._run(expiration_interval) 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:31:51.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: s = self.context.wrap_socket( 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: return self.sslsocket_class._create( 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: self.do_handshake() 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: self._sslobj.do_handshake() 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: cluster 2024-09-13T14:31:49.909951+0000 mgr.y (mgr.26252) 8 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: cluster 2024-09-13T14:31:49.928936+0000 mon.a (mon.0) 61 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-09-13T14:31:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[32082]: cluster 2024-09-13T14:31:50.913242+0000 mon.a (mon.0) 62 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2024-09-13T14:31:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cephadm 2024-09-13T14:31:49.712587+0000 mgr.y (mgr.26252) 4 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Serving on http://172.21.15.99:8765 2024-09-13T14:31:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cephadm 2024-09-13T14:31:49.712587+0000 mgr.y (mgr.26252) 4 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Serving on http://172.21.15.99:8765 2024-09-13T14:31:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cephadm 2024-09-13T14:31:49.829865+0000 mgr.y (mgr.26252) 5 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:31:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cephadm 2024-09-13T14:31:49.829865+0000 mgr.y (mgr.26252) 5 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:31:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cephadm 2024-09-13T14:31:49.830098+0000 mgr.y (mgr.26252) 6 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Bus STARTED 2024-09-13T14:31:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cephadm 2024-09-13T14:31:49.830098+0000 mgr.y (mgr.26252) 6 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Bus STARTED 2024-09-13T14:31:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cephadm 2024-09-13T14:31:49.831895+0000 mgr.y (mgr.26252) 7 : cephadm [ERR] [13/Sep/2024:14:31:49] ENGINE Error in HTTPServer.serve 2024-09-13T14:31:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: Traceback (most recent call last): 2024-09-13T14:31:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:31:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: self._connections.run(self.expiration_interval) 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: self._run(expiration_interval) 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: s = self.context.wrap_socket( 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: return self.sslsocket_class._create( 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: self.do_handshake() 2024-09-13T14:31:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: self._sslobj.do_handshake() 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cephadm 2024-09-13T14:31:49.831895+0000 mgr.y (mgr.26252) 7 : cephadm [ERR] [13/Sep/2024:14:31:49] ENGINE Error in HTTPServer.serve 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: Traceback (most recent call last): 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: self._connections.run(self.expiration_interval) 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: self._run(expiration_interval) 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: s = self.context.wrap_socket( 2024-09-13T14:31:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: return self.sslsocket_class._create( 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: self.do_handshake() 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: self._sslobj.do_handshake() 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cluster 2024-09-13T14:31:49.909951+0000 mgr.y (mgr.26252) 8 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cluster 2024-09-13T14:31:49.909951+0000 mgr.y (mgr.26252) 8 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cluster 2024-09-13T14:31:49.928936+0000 mon.a (mon.0) 61 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cluster 2024-09-13T14:31:49.928936+0000 mon.a (mon.0) 61 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cluster 2024-09-13T14:31:50.913242+0000 mon.a (mon.0) 62 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2024-09-13T14:31:51.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:50 smithi099 bash[84576]: cluster 2024-09-13T14:31:50.913242+0000 mon.a (mon.0) 62 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2024-09-13T14:31:51.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: cephadm 2024-09-13T14:31:49.712587+0000 mgr.y (mgr.26252) 4 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Serving on http://172.21.15.99:8765 2024-09-13T14:31:51.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: cephadm 2024-09-13T14:31:49.829865+0000 mgr.y (mgr.26252) 5 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Serving on https://172.21.15.99:7150 2024-09-13T14:31:51.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: cephadm 2024-09-13T14:31:49.830098+0000 mgr.y (mgr.26252) 6 : cephadm [INF] [13/Sep/2024:14:31:49] ENGINE Bus STARTED 2024-09-13T14:31:51.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: cephadm 2024-09-13T14:31:49.831895+0000 mgr.y (mgr.26252) 7 : cephadm [ERR] [13/Sep/2024:14:31:49] ENGINE Error in HTTPServer.serve 2024-09-13T14:31:51.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: Traceback (most recent call last): 2024-09-13T14:31:51.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:31:51.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: self._connections.run(self.expiration_interval) 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: self._run(expiration_interval) 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: s = self.context.wrap_socket( 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: return self.sslsocket_class._create( 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: self.do_handshake() 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: self._sslobj.do_handshake() 2024-09-13T14:31:51.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:31:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: cluster 2024-09-13T14:31:49.909951+0000 mgr.y (mgr.26252) 8 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: cluster 2024-09-13T14:31:49.928936+0000 mon.a (mon.0) 61 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-09-13T14:31:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:50 smithi177 bash[24494]: cluster 2024-09-13T14:31:50.913242+0000 mon.a (mon.0) 62 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2024-09-13T14:31:52.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:51 smithi099 bash[32082]: cluster 2024-09-13T14:31:50.944758+0000 mon.a (mon.0) 63 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-09-13T14:31:52.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:51 smithi099 bash[84576]: cluster 2024-09-13T14:31:50.944758+0000 mon.a (mon.0) 63 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-09-13T14:31:52.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:51 smithi099 bash[84576]: cluster 2024-09-13T14:31:50.944758+0000 mon.a (mon.0) 63 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-09-13T14:31:52.076 INFO:tasks.workunit.client.0.smithi099.stdout:migration_execute 2024-09-13T14:31:52.076 INFO:tasks.workunit.client.0.smithi099.stdout:write: a 88264836~6059621 2024-09-13T14:31:52.178 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 68818983~8109386 2024-09-13T14:31:52.237 INFO:tasks.workunit.client.0.smithi099.stdout:write: a 43146714~688487 2024-09-13T14:31:52.288 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 52458159~1133948 2024-09-13T14:31:52.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:51 smithi177 bash[24494]: cluster 2024-09-13T14:31:50.944758+0000 mon.a (mon.0) 63 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-09-13T14:31:52.376 INFO:tasks.workunit.client.0.smithi099.stdout:write: a 82210123~76398 2024-09-13T14:31:52.395 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 119696532~6172347 2024-09-13T14:31:52.407 INFO:tasks.workunit.client.0.smithi099.stdout:write: a 77275172~2023559 2024-09-13T14:31:52.423 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 115622011~6688637 2024-09-13T14:31:52.423 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: user0 2024-09-13T14:31:53.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:52 smithi099 bash[32082]: cluster 2024-09-13T14:31:51.910689+0000 mgr.y (mgr.26252) 9 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:53.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:52 smithi099 bash[32082]: cluster 2024-09-13T14:31:51.962047+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e38: y(active, since 4s), standbys: x 2024-09-13T14:31:53.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:52 smithi099 bash[84576]: cluster 2024-09-13T14:31:51.910689+0000 mgr.y (mgr.26252) 9 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:53.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:52 smithi099 bash[84576]: cluster 2024-09-13T14:31:51.910689+0000 mgr.y (mgr.26252) 9 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:53.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:52 smithi099 bash[84576]: cluster 2024-09-13T14:31:51.962047+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e38: y(active, since 4s), standbys: x 2024-09-13T14:31:53.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:52 smithi099 bash[84576]: cluster 2024-09-13T14:31:51.962047+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e38: y(active, since 4s), standbys: x 2024-09-13T14:31:53.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:52 smithi177 bash[24494]: cluster 2024-09-13T14:31:51.910689+0000 mgr.y (mgr.26252) 9 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:53.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:52 smithi177 bash[24494]: cluster 2024-09-13T14:31:51.962047+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e38: y(active, since 4s), standbys: x 2024-09-13T14:31:53.967 INFO:tasks.workunit.client.0.smithi099.stdout:write: b 30620219~2796403 2024-09-13T14:31:53.977 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 42009194~5517943 2024-09-13T14:31:53.977 INFO:tasks.workunit.client.0.smithi099.stdout:write: b 41945805~1530977 2024-09-13T14:31:53.999 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 48047546~2569731 2024-09-13T14:31:53.999 INFO:tasks.workunit.client.0.smithi099.stdout:write: b 107790643~6455266 2024-09-13T14:31:54.008 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 122928413~2526059 2024-09-13T14:31:54.008 INFO:tasks.workunit.client.0.smithi099.stdout:write: b 81086506~7394021 2024-09-13T14:31:54.023 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 62183943~3596263 2024-09-13T14:31:54.023 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: user1 2024-09-13T14:31:54.312 INFO:teuthology.orchestra.run.smithi099.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:31:54.312 INFO:teuthology.orchestra.run.smithi099.stdout:alertmanager.a smithi099 *:9093,9094 running (2m) 0s ago 10m 15.0M - 0.27.0 11f11916f8cd 45a2ce75edd3 2024-09-13T14:31:54.312 INFO:teuthology.orchestra.run.smithi099.stdout:grafana.a smithi177 *:3000 running (102s) 35s ago 9m 74.4M - 9.4.7 954c08fa6188 443ce4187a0d 2024-09-13T14:31:54.312 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi099.yyrtld smithi099 running (7m) 0s ago 7m 13.8M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 7bce2f3fd8d5 2024-09-13T14:31:54.312 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi177.outtkv smithi177 running (7m) 35s ago 7m 16.3M - 17.2.7-1627-gbbdadfa3 a994b00deb3a b976db2e3d1b 2024-09-13T14:31:54.312 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.x smithi177 *:8443,9283,8765 running (41s) 35s ago 18m 118M - 19.3.0-4697-gd58f7306 fec07016a2b0 24a222989f2c 2024-09-13T14:31:54.312 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.y smithi099 *:8443,9283,8765 running (95s) 0s ago 21m 481M - 19.3.0-4697-gd58f7306 fec07016a2b0 8ebc4af8542d 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:mon.a smithi099 running (14s) 0s ago 21m 34.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 7bae834be590 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:mon.b smithi177 running (19m) 35s ago 19m 50.9M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 660521332db7 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:mon.c smithi099 running (19m) 0s ago 19m 56.6M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a cd755a87885d 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.a smithi099 *:9100 running (2m) 0s ago 10m 7940k - 1.7.0 72c9c2088986 adc238b4fabf 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.b smithi177 *:9100 running (106s) 35s ago 10m 7468k - 1.7.0 72c9c2088986 0e333c475514 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0 smithi099 running (18m) 0s ago 18m 121M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 3334f7b9d02c 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1 smithi099 running (17m) 0s ago 17m 119M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 324b38b19317 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2 smithi099 running (16m) 0s ago 16m 148M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a e6fb2db24160 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3 smithi099 running (15m) 0s ago 15m 119M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 1fad69868d22 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4 smithi177 running (14m) 35s ago 14m 100M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a f54f8f87d384 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5 smithi177 running (13m) 35s ago 13m 112M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a f285f4fee6d3 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6 smithi177 running (12m) 35s ago 12m 128M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 2318776c78d4 2024-09-13T14:31:54.313 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7 smithi177 running (11m) 35s ago 11m 154M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a e2d33a0b16ca 2024-09-13T14:31:54.314 INFO:teuthology.orchestra.run.smithi099.stdout:prometheus.a smithi177 *:9095 running (49s) 35s ago 10m 36.1M - 2.51.0 1d3b7f56885b d360e28d5f8b 2024-09-13T14:31:54.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:53 smithi099 bash[32082]: cluster 2024-09-13T14:31:52.950561+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-09-13T14:31:54.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:53 smithi099 bash[32082]: audit 2024-09-13T14:31:53.481531+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:54.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:53 smithi099 bash[32082]: audit 2024-09-13T14:31:53.487286+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:54.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:53 smithi099 bash[84576]: cluster 2024-09-13T14:31:52.950561+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-09-13T14:31:54.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:53 smithi099 bash[84576]: cluster 2024-09-13T14:31:52.950561+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-09-13T14:31:54.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:53 smithi099 bash[84576]: audit 2024-09-13T14:31:53.481531+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:54.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:53 smithi099 bash[84576]: audit 2024-09-13T14:31:53.481531+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:54.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:53 smithi099 bash[84576]: audit 2024-09-13T14:31:53.487286+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:54.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:53 smithi099 bash[84576]: audit 2024-09-13T14:31:53.487286+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:54.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:53 smithi177 bash[24494]: cluster 2024-09-13T14:31:52.950561+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-09-13T14:31:54.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:53 smithi177 bash[24494]: audit 2024-09-13T14:31:53.481531+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:54.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:53 smithi177 bash[24494]: audit 2024-09-13T14:31:53.487286+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.148 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:31:55.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[32082]: cluster 2024-09-13T14:31:53.911390+0000 mgr.y (mgr.26252) 10 : cluster [DBG] pgmap v10: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:55.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[32082]: cluster 2024-09-13T14:31:53.961368+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-09-13T14:31:55.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[32082]: audit 2024-09-13T14:31:54.298842+0000 mgr.y (mgr.26252) 11 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:55.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[32082]: audit 2024-09-13T14:31:54.605956+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[32082]: audit 2024-09-13T14:31:54.613134+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[32082]: audit 2024-09-13T14:31:54.825509+0000 mon.a (mon.0) 71 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[32082]: audit 2024-09-13T14:31:54.835852+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: cluster 2024-09-13T14:31:53.911390+0000 mgr.y (mgr.26252) 10 : cluster [DBG] pgmap v10: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:55.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: cluster 2024-09-13T14:31:53.911390+0000 mgr.y (mgr.26252) 10 : cluster [DBG] pgmap v10: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:55.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: cluster 2024-09-13T14:31:53.961368+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-09-13T14:31:55.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: cluster 2024-09-13T14:31:53.961368+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-09-13T14:31:55.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: audit 2024-09-13T14:31:54.298842+0000 mgr.y (mgr.26252) 11 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:55.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: audit 2024-09-13T14:31:54.298842+0000 mgr.y (mgr.26252) 11 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:55.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: audit 2024-09-13T14:31:54.605956+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: audit 2024-09-13T14:31:54.605956+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: audit 2024-09-13T14:31:54.613134+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: audit 2024-09-13T14:31:54.613134+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: audit 2024-09-13T14:31:54.825509+0000 mon.a (mon.0) 71 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: audit 2024-09-13T14:31:54.825509+0000 mon.a (mon.0) 71 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: audit 2024-09-13T14:31:54.835852+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:54 smithi099 bash[84576]: audit 2024-09-13T14:31:54.835852+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:54 smithi177 bash[24494]: cluster 2024-09-13T14:31:53.911390+0000 mgr.y (mgr.26252) 10 : cluster [DBG] pgmap v10: 137 pgs: 137 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:31:55.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:54 smithi177 bash[24494]: cluster 2024-09-13T14:31:53.961368+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-09-13T14:31:55.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:54 smithi177 bash[24494]: audit 2024-09-13T14:31:54.298842+0000 mgr.y (mgr.26252) 11 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:55.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:54 smithi177 bash[24494]: audit 2024-09-13T14:31:54.605956+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:54 smithi177 bash[24494]: audit 2024-09-13T14:31:54.613134+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:54 smithi177 bash[24494]: audit 2024-09-13T14:31:54.825509+0000 mon.a (mon.0) 71 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:55.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:54 smithi177 bash[24494]: audit 2024-09-13T14:31:54.835852+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:56.010 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 147793441 2024-09-13T14:31:56.029 INFO:tasks.workunit.client.0.smithi099.stdout:write: c 99379131~4390728 2024-09-13T14:31:56.057 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 51701542~6599367 2024-09-13T14:31:56.057 INFO:tasks.workunit.client.0.smithi099.stdout:write: c 79929494~7504342 2024-09-13T14:31:56.087 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 105477651~4340738 2024-09-13T14:31:56.087 INFO:tasks.workunit.client.0.smithi099.stdout:write: c 81894223~7225282 2024-09-13T14:31:56.139 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 42441030~1898483 2024-09-13T14:31:56.139 INFO:tasks.workunit.client.0.smithi099.stdout:write: c 96646073~4670163 2024-09-13T14:31:56.141 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 132492222~2830400 2024-09-13T14:31:56.141 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: user2 2024-09-13T14:31:56.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:55 smithi099 bash[84576]: cluster 2024-09-13T14:31:54.968007+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-09-13T14:31:56.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:55 smithi099 bash[84576]: cluster 2024-09-13T14:31:54.968007+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-09-13T14:31:56.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:55 smithi099 bash[84576]: audit 2024-09-13T14:31:55.952390+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:56.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:55 smithi099 bash[84576]: audit 2024-09-13T14:31:55.952390+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:56.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:55 smithi099 bash[84576]: audit 2024-09-13T14:31:55.961849+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:56.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:55 smithi099 bash[84576]: audit 2024-09-13T14:31:55.961849+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:56.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:55 smithi099 bash[32082]: cluster 2024-09-13T14:31:54.968007+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-09-13T14:31:56.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:55 smithi099 bash[32082]: audit 2024-09-13T14:31:55.952390+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:56.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:55 smithi099 bash[32082]: audit 2024-09-13T14:31:55.961849+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:56.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:55 smithi177 bash[24494]: cluster 2024-09-13T14:31:54.968007+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-09-13T14:31:56.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:55 smithi177 bash[24494]: audit 2024-09-13T14:31:55.952390+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:56.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:55 smithi177 bash[24494]: audit 2024-09-13T14:31:55.961849+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:31:57.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:31:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:31:57] "GET /metrics HTTP/1.1" 200 36055 "" "Prometheus/2.51.0" 2024-09-13T14:31:57.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:56 smithi099 bash[84576]: cluster 2024-09-13T14:31:55.912164+0000 mgr.y (mgr.26252) 12 : cluster [DBG] pgmap v13: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 48 MiB/s rd, 13 MiB/s wr, 504 op/s 2024-09-13T14:31:57.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:56 smithi099 bash[84576]: cluster 2024-09-13T14:31:55.912164+0000 mgr.y (mgr.26252) 12 : cluster [DBG] pgmap v13: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 48 MiB/s rd, 13 MiB/s wr, 504 op/s 2024-09-13T14:31:57.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:56 smithi099 bash[84576]: cluster 2024-09-13T14:31:55.992916+0000 mon.a (mon.0) 76 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-09-13T14:31:57.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:56 smithi099 bash[84576]: cluster 2024-09-13T14:31:55.992916+0000 mon.a (mon.0) 76 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-09-13T14:31:57.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:56 smithi099 bash[32082]: cluster 2024-09-13T14:31:55.912164+0000 mgr.y (mgr.26252) 12 : cluster [DBG] pgmap v13: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 48 MiB/s rd, 13 MiB/s wr, 504 op/s 2024-09-13T14:31:57.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:56 smithi099 bash[32082]: cluster 2024-09-13T14:31:55.992916+0000 mon.a (mon.0) 76 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-09-13T14:31:57.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:57 smithi177 bash[24494]: cluster 2024-09-13T14:31:55.912164+0000 mgr.y (mgr.26252) 12 : cluster [DBG] pgmap v13: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 48 MiB/s rd, 13 MiB/s wr, 504 op/s 2024-09-13T14:31:57.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:57 smithi177 bash[24494]: cluster 2024-09-13T14:31:55.992916+0000 mon.a (mon.0) 76 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-09-13T14:31:57.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:31:56 smithi177 bash[61439]: ts=2024-09-13T14:31:56.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:31:58.019 INFO:tasks.workunit.client.0.smithi099.stdout:write: d 2685812~4361377 2024-09-13T14:31:58.048 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 136295004~7648843 2024-09-13T14:31:58.048 INFO:tasks.workunit.client.0.smithi099.stdout:write: d 85970107~306271 2024-09-13T14:31:58.049 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 2706517~4603335 2024-09-13T14:31:58.074 INFO:tasks.workunit.client.0.smithi099.stdout:write: d 141963096~5830345 2024-09-13T14:31:58.079 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 34374909~1439176 2024-09-13T14:31:58.079 INFO:tasks.workunit.client.0.smithi099.stdout:write: d 79646763~7829236 2024-09-13T14:31:58.102 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 6868572~5534707 2024-09-13T14:31:58.102 INFO:tasks.workunit.client.0.smithi099.stdout:snap_create: user3 2024-09-13T14:31:58.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:58 smithi099 bash[84576]: cluster 2024-09-13T14:31:56.992389+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-09-13T14:31:58.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:58 smithi099 bash[84576]: cluster 2024-09-13T14:31:56.992389+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-09-13T14:31:58.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:58 smithi099 bash[32082]: cluster 2024-09-13T14:31:56.992389+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-09-13T14:31:58.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:58 smithi177 bash[24494]: cluster 2024-09-13T14:31:56.992389+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-09-13T14:31:59.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:59 smithi099 bash[32082]: cluster 2024-09-13T14:31:57.912972+0000 mgr.y (mgr.26252) 13 : cluster [DBG] pgmap v16: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 60 MiB/s rd, 16 MiB/s wr, 627 op/s 2024-09-13T14:31:59.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:31:59 smithi099 bash[32082]: cluster 2024-09-13T14:31:58.002327+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-09-13T14:31:59.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:59 smithi099 bash[84576]: cluster 2024-09-13T14:31:57.912972+0000 mgr.y (mgr.26252) 13 : cluster [DBG] pgmap v16: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 60 MiB/s rd, 16 MiB/s wr, 627 op/s 2024-09-13T14:31:59.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:59 smithi099 bash[84576]: cluster 2024-09-13T14:31:57.912972+0000 mgr.y (mgr.26252) 13 : cluster [DBG] pgmap v16: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 60 MiB/s rd, 16 MiB/s wr, 627 op/s 2024-09-13T14:31:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:59 smithi099 bash[84576]: cluster 2024-09-13T14:31:58.002327+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-09-13T14:31:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:31:59 smithi099 bash[84576]: cluster 2024-09-13T14:31:58.002327+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-09-13T14:31:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:59 smithi177 bash[24494]: cluster 2024-09-13T14:31:57.912972+0000 mgr.y (mgr.26252) 13 : cluster [DBG] pgmap v16: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 60 MiB/s rd, 16 MiB/s wr, 627 op/s 2024-09-13T14:31:59.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:31:59 smithi177 bash[24494]: cluster 2024-09-13T14:31:58.002327+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-09-13T14:31:59.978 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:00.055 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 154737801 2024-09-13T14:32:00.103 INFO:tasks.workunit.client.0.smithi099.stdout:write: 1 15473780~3868445 2024-09-13T14:32:00.109 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 16440891~1934222 2024-09-13T14:32:00.134 INFO:tasks.workunit.client.0.smithi099.stdout:write: 2 30947560~3868445 2024-09-13T14:32:00.136 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 31914671~1934222 2024-09-13T14:32:00.174 INFO:tasks.workunit.client.0.smithi099.stdout:write: 3 46421340~3868445 2024-09-13T14:32:00.176 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 47388451~1934222 2024-09-13T14:32:00.205 INFO:tasks.workunit.client.0.smithi099.stdout:write: 4 61895120~3868445 2024-09-13T14:32:00.207 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 62862231~1934222 2024-09-13T14:32:00.234 INFO:tasks.workunit.client.0.smithi099.stdout:write: 5 77368900~3868445 2024-09-13T14:32:00.236 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 78336011~1934222 2024-09-13T14:32:00.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:00 smithi099 bash[84576]: cluster 2024-09-13T14:31:59.028222+0000 mon.a (mon.0) 79 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-09-13T14:32:00.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:00 smithi099 bash[84576]: cluster 2024-09-13T14:31:59.028222+0000 mon.a (mon.0) 79 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-09-13T14:32:00.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:00 smithi099 bash[32082]: cluster 2024-09-13T14:31:59.028222+0000 mon.a (mon.0) 79 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-09-13T14:32:00.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:00 smithi177 bash[24494]: cluster 2024-09-13T14:31:59.028222+0000 mon.a (mon.0) 79 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-09-13T14:32:00.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:00 smithi177 bash[61439]: ts=2024-09-13T14:32:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:00.394 INFO:tasks.workunit.client.0.smithi099.stdout:migration_commit 2024-09-13T14:32:01.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:01 smithi099 bash[32082]: cluster 2024-09-13T14:31:59.913745+0000 mgr.y (mgr.26252) 14 : cluster [DBG] pgmap v19: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 174 KiB/s rd, 29 MiB/s wr, 285 op/s 2024-09-13T14:32:01.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:01 smithi099 bash[32082]: cluster 2024-09-13T14:32:00.038614+0000 mon.a (mon.0) 80 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-09-13T14:32:01.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:01 smithi099 bash[84576]: cluster 2024-09-13T14:31:59.913745+0000 mgr.y (mgr.26252) 14 : cluster [DBG] pgmap v19: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 174 KiB/s rd, 29 MiB/s wr, 285 op/s 2024-09-13T14:32:01.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:01 smithi099 bash[84576]: cluster 2024-09-13T14:31:59.913745+0000 mgr.y (mgr.26252) 14 : cluster [DBG] pgmap v19: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 174 KiB/s rd, 29 MiB/s wr, 285 op/s 2024-09-13T14:32:01.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:01 smithi099 bash[84576]: cluster 2024-09-13T14:32:00.038614+0000 mon.a (mon.0) 80 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-09-13T14:32:01.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:01 smithi099 bash[84576]: cluster 2024-09-13T14:32:00.038614+0000 mon.a (mon.0) 80 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-09-13T14:32:01.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:01 smithi177 bash[24494]: cluster 2024-09-13T14:31:59.913745+0000 mgr.y (mgr.26252) 14 : cluster [DBG] pgmap v19: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 174 KiB/s rd, 29 MiB/s wr, 285 op/s 2024-09-13T14:32:01.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:01 smithi177 bash[24494]: cluster 2024-09-13T14:32:00.038614+0000 mon.a (mon.0) 80 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-09-13T14:32:02.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:02 smithi099 bash[84576]: cluster 2024-09-13T14:32:01.050906+0000 mon.a (mon.0) 81 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-09-13T14:32:02.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:02 smithi099 bash[84576]: cluster 2024-09-13T14:32:01.050906+0000 mon.a (mon.0) 81 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-09-13T14:32:02.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:02 smithi099 bash[32082]: cluster 2024-09-13T14:32:01.050906+0000 mon.a (mon.0) 81 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-09-13T14:32:02.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:02 smithi177 bash[24494]: cluster 2024-09-13T14:32:01.050906+0000 mon.a (mon.0) 81 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-09-13T14:32:03.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:03 smithi099 bash[32082]: cluster 2024-09-13T14:32:01.914491+0000 mgr.y (mgr.26252) 15 : cluster [DBG] pgmap v22: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 174 KiB/s rd, 29 MiB/s wr, 285 op/s 2024-09-13T14:32:03.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:03 smithi099 bash[32082]: cluster 2024-09-13T14:32:02.062890+0000 mon.a (mon.0) 82 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-09-13T14:32:03.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:03 smithi099 bash[84576]: cluster 2024-09-13T14:32:01.914491+0000 mgr.y (mgr.26252) 15 : cluster [DBG] pgmap v22: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 174 KiB/s rd, 29 MiB/s wr, 285 op/s 2024-09-13T14:32:03.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:03 smithi099 bash[84576]: cluster 2024-09-13T14:32:01.914491+0000 mgr.y (mgr.26252) 15 : cluster [DBG] pgmap v22: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 174 KiB/s rd, 29 MiB/s wr, 285 op/s 2024-09-13T14:32:03.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:03 smithi099 bash[84576]: cluster 2024-09-13T14:32:02.062890+0000 mon.a (mon.0) 82 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-09-13T14:32:03.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:03 smithi099 bash[84576]: cluster 2024-09-13T14:32:02.062890+0000 mon.a (mon.0) 82 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-09-13T14:32:03.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:03 smithi177 bash[24494]: cluster 2024-09-13T14:32:01.914491+0000 mgr.y (mgr.26252) 15 : cluster [DBG] pgmap v22: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 174 KiB/s rd, 29 MiB/s wr, 285 op/s 2024-09-13T14:32:03.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:03 smithi177 bash[24494]: cluster 2024-09-13T14:32:02.062890+0000 mon.a (mon.0) 82 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-09-13T14:32:04.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:04 smithi099 bash[84576]: cluster 2024-09-13T14:32:03.062145+0000 mon.a (mon.0) 83 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-09-13T14:32:04.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:04 smithi099 bash[84576]: cluster 2024-09-13T14:32:03.062145+0000 mon.a (mon.0) 83 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-09-13T14:32:04.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:04 smithi099 bash[84576]: audit 2024-09-13T14:32:03.252011+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:04.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:04 smithi099 bash[84576]: audit 2024-09-13T14:32:03.252011+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:04.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:04 smithi099 bash[32082]: cluster 2024-09-13T14:32:03.062145+0000 mon.a (mon.0) 83 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-09-13T14:32:04.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:04 smithi099 bash[32082]: audit 2024-09-13T14:32:03.252011+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:04.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:04 smithi177 bash[24494]: cluster 2024-09-13T14:32:03.062145+0000 mon.a (mon.0) 83 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-09-13T14:32:04.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:04 smithi177 bash[24494]: audit 2024-09-13T14:32:03.252011+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:04.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:04 smithi177 bash[61439]: ts=2024-09-13T14:32:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:04.384 INFO:teuthology.orchestra.run.smithi099.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:04.909 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMigration.StressLive (27987 ms) 2024-09-13T14:32:05.289 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:32:05.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[32082]: cluster 2024-09-13T14:32:03.915457+0000 mgr.y (mgr.26252) 16 : cluster [DBG] pgmap v25: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:05.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[32082]: audit 2024-09-13T14:32:04.379899+0000 mgr.y (mgr.26252) 17 : audit [DBG] from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:05.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[32082]: cephadm 2024-09-13T14:32:04.380441+0000 mgr.y (mgr.26252) 18 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:05.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[32082]: audit 2024-09-13T14:32:04.389092+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:05.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[84576]: cluster 2024-09-13T14:32:03.915457+0000 mgr.y (mgr.26252) 16 : cluster [DBG] pgmap v25: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:05.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[84576]: cluster 2024-09-13T14:32:03.915457+0000 mgr.y (mgr.26252) 16 : cluster [DBG] pgmap v25: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:05.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[84576]: audit 2024-09-13T14:32:04.379899+0000 mgr.y (mgr.26252) 17 : audit [DBG] from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:05.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[84576]: audit 2024-09-13T14:32:04.379899+0000 mgr.y (mgr.26252) 17 : audit [DBG] from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:05.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[84576]: cephadm 2024-09-13T14:32:04.380441+0000 mgr.y (mgr.26252) 18 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:05.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[84576]: cephadm 2024-09-13T14:32:04.380441+0000 mgr.y (mgr.26252) 18 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:05.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[84576]: audit 2024-09-13T14:32:04.389092+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:05.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:05 smithi099 bash[84576]: audit 2024-09-13T14:32:04.389092+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:05.733 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:05 smithi177 bash[24494]: cluster 2024-09-13T14:32:03.915457+0000 mgr.y (mgr.26252) 16 : cluster [DBG] pgmap v25: 137 pgs: 137 active+clean; 2.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:05.733 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:05 smithi177 bash[24494]: audit 2024-09-13T14:32:04.379899+0000 mgr.y (mgr.26252) 17 : audit [DBG] from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:05.733 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:05 smithi177 bash[24494]: cephadm 2024-09-13T14:32:04.380441+0000 mgr.y (mgr.26252) 18 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:32:05.733 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:05 smithi177 bash[24494]: audit 2024-09-13T14:32:04.389092+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:06.054 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:05 smithi177 bash[61439]: ts=2024-09-13T14:32:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:06.054 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:05 smithi177 bash[61439]: ts=2024-09-13T14:32:05.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:06.434 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 57 tests from TestMigration (290820 ms total) 2024-09-13T14:32:06.434 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:32:06.434 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 6 tests from TestGroup 2024-09-13T14:32:06.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:06 smithi177 bash[24494]: cluster 2024-09-13T14:32:05.399638+0000 mon.a (mon.0) 86 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-09-13T14:32:06.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:06 smithi099 bash[32082]: cluster 2024-09-13T14:32:05.399638+0000 mon.a (mon.0) 86 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-09-13T14:32:06.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:06 smithi099 bash[84576]: cluster 2024-09-13T14:32:05.399638+0000 mon.a (mon.0) 86 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-09-13T14:32:06.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:06 smithi099 bash[84576]: cluster 2024-09-13T14:32:05.399638+0000 mon.a (mon.0) 86 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-09-13T14:32:07.318 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:32:07] "GET /metrics HTTP/1.1" 200 38903 "" "Prometheus/2.51.0" 2024-09-13T14:32:07.353 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:06 smithi177 bash[61439]: ts=2024-09-13T14:32:06.952Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:07.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[32082]: cluster 2024-09-13T14:32:05.915983+0000 mgr.y (mgr.26252) 19 : cluster [DBG] pgmap v27: 105 pgs: 105 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:07.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[32082]: cluster 2024-09-13T14:32:06.406812+0000 mon.a (mon.0) 87 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:32:07.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[32082]: cluster 2024-09-13T14:32:06.406867+0000 mon.a (mon.0) 88 : cluster [INF] Cluster is now healthy 2024-09-13T14:32:07.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[32082]: cluster 2024-09-13T14:32:06.430124+0000 mon.a (mon.0) 89 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-09-13T14:32:07.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[32082]: audit 2024-09-13T14:32:06.952241+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:07.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[32082]: audit 2024-09-13T14:32:06.964103+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:07.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[32082]: audit 2024-09-13T14:32:06.968910+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:07.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[32082]: audit 2024-09-13T14:32:06.970794+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:07.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: cluster 2024-09-13T14:32:05.915983+0000 mgr.y (mgr.26252) 19 : cluster [DBG] pgmap v27: 105 pgs: 105 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:07.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: cluster 2024-09-13T14:32:05.915983+0000 mgr.y (mgr.26252) 19 : cluster [DBG] pgmap v27: 105 pgs: 105 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:07.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: cluster 2024-09-13T14:32:06.406812+0000 mon.a (mon.0) 87 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:32:07.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: cluster 2024-09-13T14:32:06.406812+0000 mon.a (mon.0) 87 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:32:07.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: cluster 2024-09-13T14:32:06.406867+0000 mon.a (mon.0) 88 : cluster [INF] Cluster is now healthy 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: cluster 2024-09-13T14:32:06.406867+0000 mon.a (mon.0) 88 : cluster [INF] Cluster is now healthy 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: cluster 2024-09-13T14:32:06.430124+0000 mon.a (mon.0) 89 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: cluster 2024-09-13T14:32:06.430124+0000 mon.a (mon.0) 89 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: audit 2024-09-13T14:32:06.952241+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: audit 2024-09-13T14:32:06.952241+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: audit 2024-09-13T14:32:06.964103+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: audit 2024-09-13T14:32:06.964103+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: audit 2024-09-13T14:32:06.968910+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: audit 2024-09-13T14:32:06.968910+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: audit 2024-09-13T14:32:06.970794+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:07 smithi099 bash[84576]: audit 2024-09-13T14:32:06.970794+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:07.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:07 smithi177 bash[24494]: cluster 2024-09-13T14:32:05.915983+0000 mgr.y (mgr.26252) 19 : cluster [DBG] pgmap v27: 105 pgs: 105 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:07.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:07 smithi177 bash[24494]: cluster 2024-09-13T14:32:06.406812+0000 mon.a (mon.0) 87 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:32:07.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:07 smithi177 bash[24494]: cluster 2024-09-13T14:32:06.406867+0000 mon.a (mon.0) 88 : cluster [INF] Cluster is now healthy 2024-09-13T14:32:07.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:07 smithi177 bash[24494]: cluster 2024-09-13T14:32:06.430124+0000 mon.a (mon.0) 89 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-09-13T14:32:07.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:07 smithi177 bash[24494]: audit 2024-09-13T14:32:06.952241+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:07.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:07 smithi177 bash[24494]: audit 2024-09-13T14:32:06.964103+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:07.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:07 smithi177 bash[24494]: audit 2024-09-13T14:32:06.968910+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:07.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:07 smithi177 bash[24494]: audit 2024-09-13T14:32:06.970794+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:08.456 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestGroup.group_create 2024-09-13T14:32:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:06.973367+0000 mgr.y (mgr.26252) 20 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:32:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:06.973367+0000 mgr.y (mgr.26252) 20 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:32:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:06.973873+0000 mgr.y (mgr.26252) 21 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:32:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:06.973873+0000 mgr.y (mgr.26252) 21 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:32:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.176257+0000 mgr.y (mgr.26252) 22 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.176257+0000 mgr.y (mgr.26252) 22 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.185529+0000 mgr.y (mgr.26252) 23 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.185529+0000 mgr.y (mgr.26252) 23 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.315706+0000 mgr.y (mgr.26252) 24 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:08.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.315706+0000 mgr.y (mgr.26252) 24 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.335146+0000 mgr.y (mgr.26252) 25 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.335146+0000 mgr.y (mgr.26252) 25 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cluster 2024-09-13T14:32:07.424193+0000 mon.a (mon.0) 94 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cluster 2024-09-13T14:32:07.424193+0000 mon.a (mon.0) 94 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.428640+0000 mon.c (mon.2) 185 : audit [INF] from='client.? 172.21.15.99:0/2589657178' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.428640+0000 mon.c (mon.2) 185 : audit [INF] from='client.? 172.21.15.99:0/2589657178' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.439558+0000 mon.a (mon.0) 95 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.439558+0000 mon.a (mon.0) 95 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.454728+0000 mgr.y (mgr.26252) 26 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.454728+0000 mgr.y (mgr.26252) 26 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:08.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: cephadm 2024-09-13T14:32:06.973367+0000 mgr.y (mgr.26252) 20 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: cephadm 2024-09-13T14:32:06.973873+0000 mgr.y (mgr.26252) 21 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: cephadm 2024-09-13T14:32:07.176257+0000 mgr.y (mgr.26252) 22 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: cephadm 2024-09-13T14:32:07.185529+0000 mgr.y (mgr.26252) 23 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: cephadm 2024-09-13T14:32:07.315706+0000 mgr.y (mgr.26252) 24 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: cephadm 2024-09-13T14:32:07.335146+0000 mgr.y (mgr.26252) 25 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: cluster 2024-09-13T14:32:07.424193+0000 mon.a (mon.0) 94 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:07.428640+0000 mon.c (mon.2) 185 : audit [INF] from='client.? 172.21.15.99:0/2589657178' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:07.439558+0000 mon.a (mon.0) 95 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: cephadm 2024-09-13T14:32:07.454728+0000 mgr.y (mgr.26252) 26 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: cephadm 2024-09-13T14:32:07.493251+0000 mgr.y (mgr.26252) 27 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:07.587431+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:07.592606+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:07.632990+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:07.641089+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:07.650545+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:07.772001+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:08.343338+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:08.353794+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:08.356495+0000 mon.a (mon.0) 104 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:08.358040+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[32082]: audit 2024-09-13T14:32:08.366003+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.493251+0000 mgr.y (mgr.26252) 27 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: cephadm 2024-09-13T14:32:07.493251+0000 mgr.y (mgr.26252) 27 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.587431+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.587431+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.592606+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.592606+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.632990+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.632990+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.641089+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.641089+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.650545+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.650545+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.772001+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:07.772001+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:08.343338+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:08.343338+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:08.353794+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:08.353794+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:08.356495+0000 mon.a (mon.0) 104 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:08.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:08.356495+0000 mon.a (mon.0) 104 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:08.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:08.358040+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:08.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:08.358040+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:08.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:08.366003+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:08 smithi099 bash[84576]: audit 2024-09-13T14:32:08.366003+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: cephadm 2024-09-13T14:32:06.973367+0000 mgr.y (mgr.26252) 20 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.conf 2024-09-13T14:32:08.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: cephadm 2024-09-13T14:32:06.973873+0000 mgr.y (mgr.26252) 21 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.conf 2024-09-13T14:32:08.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: cephadm 2024-09-13T14:32:07.176257+0000 mgr.y (mgr.26252) 22 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:08.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: cephadm 2024-09-13T14:32:07.185529+0000 mgr.y (mgr.26252) 23 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: cephadm 2024-09-13T14:32:07.315706+0000 mgr.y (mgr.26252) 24 : cephadm [INF] Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: cephadm 2024-09-13T14:32:07.335146+0000 mgr.y (mgr.26252) 25 : cephadm [INF] Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: cluster 2024-09-13T14:32:07.424193+0000 mon.a (mon.0) 94 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:07.428640+0000 mon.c (mon.2) 185 : audit [INF] from='client.? 172.21.15.99:0/2589657178' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:07.439558+0000 mon.a (mon.0) 95 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: cephadm 2024-09-13T14:32:07.454728+0000 mgr.y (mgr.26252) 26 : cephadm [INF] Updating smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: cephadm 2024-09-13T14:32:07.493251+0000 mgr.y (mgr.26252) 27 : cephadm [INF] Updating smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:07.587431+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:07.592606+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:07.632990+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:07.641089+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:07.650545+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:07.772001+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:08.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:08.343338+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:08.353794+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:08.356495+0000 mon.a (mon.0) 104 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:08.358040+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:08 smithi177 bash[24494]: audit 2024-09-13T14:32:08.366003+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:09.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[84576]: cluster 2024-09-13T14:32:07.916657+0000 mgr.y (mgr.26252) 28 : cluster [DBG] pgmap v30: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:09.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[84576]: cluster 2024-09-13T14:32:07.916657+0000 mgr.y (mgr.26252) 28 : cluster [DBG] pgmap v30: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:09.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[84576]: audit 2024-09-13T14:32:08.445478+0000 mon.a (mon.0) 107 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:32:09.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[84576]: audit 2024-09-13T14:32:08.445478+0000 mon.a (mon.0) 107 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:32:09.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[84576]: cluster 2024-09-13T14:32:08.458169+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-09-13T14:32:09.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[84576]: cluster 2024-09-13T14:32:08.458169+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-09-13T14:32:09.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[84576]: cluster 2024-09-13T14:32:08.646153+0000 mon.a (mon.0) 109 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:32:09.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[84576]: cluster 2024-09-13T14:32:08.646153+0000 mon.a (mon.0) 109 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:32:09.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[32082]: cluster 2024-09-13T14:32:07.916657+0000 mgr.y (mgr.26252) 28 : cluster [DBG] pgmap v30: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:09.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[32082]: audit 2024-09-13T14:32:08.445478+0000 mon.a (mon.0) 107 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:32:09.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[32082]: cluster 2024-09-13T14:32:08.458169+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-09-13T14:32:09.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:09 smithi099 bash[32082]: cluster 2024-09-13T14:32:08.646153+0000 mon.a (mon.0) 109 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:32:09.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:09 smithi177 bash[24494]: cluster 2024-09-13T14:32:07.916657+0000 mgr.y (mgr.26252) 28 : cluster [DBG] pgmap v30: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-09-13T14:32:09.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:09 smithi177 bash[24494]: audit 2024-09-13T14:32:08.445478+0000 mon.a (mon.0) 107 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:32:09.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:09 smithi177 bash[24494]: cluster 2024-09-13T14:32:08.458169+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-09-13T14:32:09.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:09 smithi177 bash[24494]: cluster 2024-09-13T14:32:08.646153+0000 mon.a (mon.0) 109 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:32:10.058 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:10.457 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:10 smithi177 bash[61439]: ts=2024-09-13T14:32:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:10.528 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestGroup.group_create (2071 ms) 2024-09-13T14:32:10.528 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestGroup.group_createPP 2024-09-13T14:32:10.571 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestGroup.group_createPP (44 ms) 2024-09-13T14:32:10.572 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestGroup.add_image 2024-09-13T14:32:10.665 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestGroup.add_image (94 ms) 2024-09-13T14:32:10.666 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestGroup.add_imagePP 2024-09-13T14:32:10.734 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestGroup.add_imagePP (69 ms) 2024-09-13T14:32:10.734 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestGroup.add_snapshot 2024-09-13T14:32:10.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:10 smithi099 bash[84576]: cluster 2024-09-13T14:32:09.468108+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-09-13T14:32:10.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:10 smithi099 bash[84576]: cluster 2024-09-13T14:32:09.468108+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-09-13T14:32:10.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:10 smithi099 bash[32082]: cluster 2024-09-13T14:32:09.468108+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-09-13T14:32:10.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:10 smithi177 bash[24494]: cluster 2024-09-13T14:32:09.468108+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-09-13T14:32:11.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[84576]: cluster 2024-09-13T14:32:09.917384+0000 mgr.y (mgr.26252) 29 : cluster [DBG] pgmap v33: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:11.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[84576]: cluster 2024-09-13T14:32:09.917384+0000 mgr.y (mgr.26252) 29 : cluster [DBG] pgmap v33: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:11.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[84576]: cluster 2024-09-13T14:32:10.460660+0000 mon.a (mon.0) 111 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:32:11.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[84576]: cluster 2024-09-13T14:32:10.460660+0000 mon.a (mon.0) 111 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:32:11.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[84576]: cluster 2024-09-13T14:32:10.460713+0000 mon.a (mon.0) 112 : cluster [INF] Cluster is now healthy 2024-09-13T14:32:11.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[84576]: cluster 2024-09-13T14:32:10.460713+0000 mon.a (mon.0) 112 : cluster [INF] Cluster is now healthy 2024-09-13T14:32:11.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[84576]: cluster 2024-09-13T14:32:10.476471+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-09-13T14:32:11.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[84576]: cluster 2024-09-13T14:32:10.476471+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-09-13T14:32:11.730 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[32082]: cluster 2024-09-13T14:32:09.917384+0000 mgr.y (mgr.26252) 29 : cluster [DBG] pgmap v33: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:11.730 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[32082]: cluster 2024-09-13T14:32:10.460660+0000 mon.a (mon.0) 111 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:32:11.731 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[32082]: cluster 2024-09-13T14:32:10.460713+0000 mon.a (mon.0) 112 : cluster [INF] Cluster is now healthy 2024-09-13T14:32:11.731 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:11 smithi099 bash[32082]: cluster 2024-09-13T14:32:10.476471+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-09-13T14:32:11.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:11 smithi177 bash[24494]: cluster 2024-09-13T14:32:09.917384+0000 mgr.y (mgr.26252) 29 : cluster [DBG] pgmap v33: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:11.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:11 smithi177 bash[24494]: cluster 2024-09-13T14:32:10.460660+0000 mon.a (mon.0) 111 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:32:11.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:11 smithi177 bash[24494]: cluster 2024-09-13T14:32:10.460713+0000 mon.a (mon.0) 112 : cluster [INF] Cluster is now healthy 2024-09-13T14:32:11.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:11 smithi177 bash[24494]: cluster 2024-09-13T14:32:10.476471+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-09-13T14:32:12.750 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:12 smithi099 bash[84576]: cluster 2024-09-13T14:32:11.483553+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-09-13T14:32:12.750 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:12 smithi099 bash[84576]: cluster 2024-09-13T14:32:11.483553+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-09-13T14:32:12.750 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:12 smithi099 bash[32082]: cluster 2024-09-13T14:32:11.483553+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-09-13T14:32:12.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:12 smithi177 bash[24494]: cluster 2024-09-13T14:32:11.483553+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-09-13T14:32:13.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:13 smithi099 bash[84576]: cluster 2024-09-13T14:32:11.918258+0000 mgr.y (mgr.26252) 30 : cluster [DBG] pgmap v36: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:13.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:13 smithi099 bash[84576]: cluster 2024-09-13T14:32:11.918258+0000 mgr.y (mgr.26252) 30 : cluster [DBG] pgmap v36: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:13.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:13 smithi099 bash[84576]: cluster 2024-09-13T14:32:12.499902+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-09-13T14:32:13.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:13 smithi099 bash[84576]: cluster 2024-09-13T14:32:12.499902+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-09-13T14:32:13.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:13 smithi099 bash[32082]: cluster 2024-09-13T14:32:11.918258+0000 mgr.y (mgr.26252) 30 : cluster [DBG] pgmap v36: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:13.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:13 smithi099 bash[32082]: cluster 2024-09-13T14:32:12.499902+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-09-13T14:32:13.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:13 smithi177 bash[24494]: cluster 2024-09-13T14:32:11.918258+0000 mgr.y (mgr.26252) 30 : cluster [DBG] pgmap v36: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:13.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:13 smithi177 bash[24494]: cluster 2024-09-13T14:32:12.499902+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-09-13T14:32:14.509 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:14 smithi177 bash[61439]: ts=2024-09-13T14:32:14.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:14.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:14 smithi099 bash[32082]: cluster 2024-09-13T14:32:13.510927+0000 mon.a (mon.0) 116 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-09-13T14:32:14.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:14 smithi099 bash[84576]: cluster 2024-09-13T14:32:13.510927+0000 mon.a (mon.0) 116 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-09-13T14:32:14.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:14 smithi099 bash[84576]: cluster 2024-09-13T14:32:13.510927+0000 mon.a (mon.0) 116 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-09-13T14:32:14.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:14 smithi177 bash[24494]: cluster 2024-09-13T14:32:13.510927+0000 mon.a (mon.0) 116 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-09-13T14:32:15.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:15 smithi099 bash[84576]: cluster 2024-09-13T14:32:13.918867+0000 mgr.y (mgr.26252) 31 : cluster [DBG] pgmap v39: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:15.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:15 smithi099 bash[84576]: cluster 2024-09-13T14:32:13.918867+0000 mgr.y (mgr.26252) 31 : cluster [DBG] pgmap v39: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:15.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:15 smithi099 bash[84576]: cluster 2024-09-13T14:32:14.520765+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-09-13T14:32:15.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:15 smithi099 bash[84576]: cluster 2024-09-13T14:32:14.520765+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-09-13T14:32:15.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:15 smithi099 bash[32082]: cluster 2024-09-13T14:32:13.918867+0000 mgr.y (mgr.26252) 31 : cluster [DBG] pgmap v39: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:15.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:15 smithi099 bash[32082]: cluster 2024-09-13T14:32:14.520765+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-09-13T14:32:15.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:15 smithi177 bash[24494]: cluster 2024-09-13T14:32:13.918867+0000 mgr.y (mgr.26252) 31 : cluster [DBG] pgmap v39: 105 pgs: 105 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:15.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:15 smithi177 bash[24494]: cluster 2024-09-13T14:32:14.520765+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-09-13T14:32:15.854 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:15 smithi177 bash[61439]: ts=2024-09-13T14:32:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:15.854 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:15 smithi177 bash[61439]: ts=2024-09-13T14:32:15.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:16.592 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestGroup.add_snapshot (5858 ms) 2024-09-13T14:32:16.592 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestGroup.add_snapshotPP 2024-09-13T14:32:16.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:16 smithi099 bash[32082]: cluster 2024-09-13T14:32:15.530702+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-09-13T14:32:16.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:16 smithi099 bash[84576]: cluster 2024-09-13T14:32:15.530702+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-09-13T14:32:16.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:16 smithi099 bash[84576]: cluster 2024-09-13T14:32:15.530702+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-09-13T14:32:16.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:16 smithi177 bash[24494]: cluster 2024-09-13T14:32:15.530702+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-09-13T14:32:17.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:32:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:32:17] "GET /metrics HTTP/1.1" 200 38903 "" "Prometheus/2.51.0" 2024-09-13T14:32:17.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:16 smithi177 bash[61439]: ts=2024-09-13T14:32:16.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:17.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:17 smithi099 bash[32082]: cluster 2024-09-13T14:32:15.919586+0000 mgr.y (mgr.26252) 32 : cluster [DBG] pgmap v42: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 18 KiB/s wr, 207 op/s 2024-09-13T14:32:17.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:17 smithi099 bash[32082]: cluster 2024-09-13T14:32:16.543409+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-09-13T14:32:17.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:17 smithi099 bash[84576]: cluster 2024-09-13T14:32:15.919586+0000 mgr.y (mgr.26252) 32 : cluster [DBG] pgmap v42: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 18 KiB/s wr, 207 op/s 2024-09-13T14:32:17.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:17 smithi099 bash[84576]: cluster 2024-09-13T14:32:15.919586+0000 mgr.y (mgr.26252) 32 : cluster [DBG] pgmap v42: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 18 KiB/s wr, 207 op/s 2024-09-13T14:32:17.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:17 smithi099 bash[84576]: cluster 2024-09-13T14:32:16.543409+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-09-13T14:32:17.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:17 smithi099 bash[84576]: cluster 2024-09-13T14:32:16.543409+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-09-13T14:32:17.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:17 smithi177 bash[24494]: cluster 2024-09-13T14:32:15.919586+0000 mgr.y (mgr.26252) 32 : cluster [DBG] pgmap v42: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 18 KiB/s wr, 207 op/s 2024-09-13T14:32:17.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:17 smithi177 bash[24494]: cluster 2024-09-13T14:32:16.543409+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-09-13T14:32:18.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:18 smithi099 bash[32082]: cluster 2024-09-13T14:32:17.548552+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-09-13T14:32:18.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:18 smithi099 bash[32082]: audit 2024-09-13T14:32:18.245746+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:18.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:18 smithi099 bash[32082]: audit 2024-09-13T14:32:18.247936+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:18.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:18 smithi099 bash[84576]: cluster 2024-09-13T14:32:17.548552+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-09-13T14:32:18.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:18 smithi099 bash[84576]: cluster 2024-09-13T14:32:17.548552+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-09-13T14:32:18.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:18 smithi099 bash[84576]: audit 2024-09-13T14:32:18.245746+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:18.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:18 smithi099 bash[84576]: audit 2024-09-13T14:32:18.245746+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:18.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:18 smithi099 bash[84576]: audit 2024-09-13T14:32:18.247936+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:18.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:18 smithi099 bash[84576]: audit 2024-09-13T14:32:18.247936+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:18.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:18 smithi177 bash[24494]: cluster 2024-09-13T14:32:17.548552+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-09-13T14:32:18.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:18 smithi177 bash[24494]: audit 2024-09-13T14:32:18.245746+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:32:18.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:18 smithi177 bash[24494]: audit 2024-09-13T14:32:18.247936+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:19.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:19 smithi177 bash[24494]: cluster 2024-09-13T14:32:17.920349+0000 mgr.y (mgr.26252) 33 : cluster [DBG] pgmap v45: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 18 KiB/s wr, 207 op/s 2024-09-13T14:32:19.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:19 smithi177 bash[24494]: cluster 2024-09-13T14:32:18.572117+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-09-13T14:32:20.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:19 smithi099 bash[32082]: cluster 2024-09-13T14:32:17.920349+0000 mgr.y (mgr.26252) 33 : cluster [DBG] pgmap v45: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 18 KiB/s wr, 207 op/s 2024-09-13T14:32:20.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:19 smithi099 bash[32082]: cluster 2024-09-13T14:32:18.572117+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-09-13T14:32:20.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:19 smithi099 bash[84576]: cluster 2024-09-13T14:32:17.920349+0000 mgr.y (mgr.26252) 33 : cluster [DBG] pgmap v45: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 18 KiB/s wr, 207 op/s 2024-09-13T14:32:20.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:19 smithi099 bash[84576]: cluster 2024-09-13T14:32:17.920349+0000 mgr.y (mgr.26252) 33 : cluster [DBG] pgmap v45: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 18 KiB/s wr, 207 op/s 2024-09-13T14:32:20.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:19 smithi099 bash[84576]: cluster 2024-09-13T14:32:18.572117+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-09-13T14:32:20.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:19 smithi099 bash[84576]: cluster 2024-09-13T14:32:18.572117+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-09-13T14:32:20.582 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:20 smithi177 bash[61439]: ts=2024-09-13T14:32:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:20.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:20 smithi177 bash[24494]: cluster 2024-09-13T14:32:19.573006+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-09-13T14:32:21.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:20 smithi099 bash[84576]: cluster 2024-09-13T14:32:19.573006+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-09-13T14:32:21.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:20 smithi099 bash[84576]: cluster 2024-09-13T14:32:19.573006+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-09-13T14:32:21.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:20 smithi099 bash[32082]: cluster 2024-09-13T14:32:19.573006+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-09-13T14:32:21.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:21 smithi177 bash[24494]: cluster 2024-09-13T14:32:19.921086+0000 mgr.y (mgr.26252) 34 : cluster [DBG] pgmap v48: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 140 op/s 2024-09-13T14:32:21.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:21 smithi177 bash[24494]: cluster 2024-09-13T14:32:20.594814+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-09-13T14:32:21.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:21 smithi177 bash[24494]: cluster 2024-09-13T14:32:20.947321+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-09-13T14:32:21.996 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestGroup.add_snapshotPP (5402 ms) 2024-09-13T14:32:22.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:21 smithi099 bash[84576]: cluster 2024-09-13T14:32:19.921086+0000 mgr.y (mgr.26252) 34 : cluster [DBG] pgmap v48: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 140 op/s 2024-09-13T14:32:22.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:21 smithi099 bash[84576]: cluster 2024-09-13T14:32:19.921086+0000 mgr.y (mgr.26252) 34 : cluster [DBG] pgmap v48: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 140 op/s 2024-09-13T14:32:22.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:21 smithi099 bash[84576]: cluster 2024-09-13T14:32:20.594814+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-09-13T14:32:22.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:21 smithi099 bash[84576]: cluster 2024-09-13T14:32:20.594814+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-09-13T14:32:22.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:21 smithi099 bash[84576]: cluster 2024-09-13T14:32:20.947321+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-09-13T14:32:22.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:21 smithi099 bash[84576]: cluster 2024-09-13T14:32:20.947321+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-09-13T14:32:22.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:21 smithi099 bash[32082]: cluster 2024-09-13T14:32:19.921086+0000 mgr.y (mgr.26252) 34 : cluster [DBG] pgmap v48: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 140 op/s 2024-09-13T14:32:22.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:21 smithi099 bash[32082]: cluster 2024-09-13T14:32:20.594814+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-09-13T14:32:22.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:21 smithi099 bash[32082]: cluster 2024-09-13T14:32:20.947321+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-09-13T14:32:22.961 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 6 tests from TestGroup (13538 ms total) 2024-09-13T14:32:22.961 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:32:22.962 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 31 tests from TestDeepCopy 2024-09-13T14:32:23.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:22 smithi099 bash[84576]: cluster 2024-09-13T14:32:21.921806+0000 mgr.y (mgr.26252) 35 : cluster [DBG] pgmap v51: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 140 op/s 2024-09-13T14:32:23.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:22 smithi099 bash[84576]: cluster 2024-09-13T14:32:21.921806+0000 mgr.y (mgr.26252) 35 : cluster [DBG] pgmap v51: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 140 op/s 2024-09-13T14:32:23.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:22 smithi099 bash[84576]: cluster 2024-09-13T14:32:21.951338+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-09-13T14:32:23.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:22 smithi099 bash[84576]: cluster 2024-09-13T14:32:21.951338+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-09-13T14:32:23.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:22 smithi099 bash[32082]: cluster 2024-09-13T14:32:21.921806+0000 mgr.y (mgr.26252) 35 : cluster [DBG] pgmap v51: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 140 op/s 2024-09-13T14:32:23.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:22 smithi099 bash[32082]: cluster 2024-09-13T14:32:21.951338+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-09-13T14:32:23.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:22 smithi177 bash[24494]: cluster 2024-09-13T14:32:21.921806+0000 mgr.y (mgr.26252) 35 : cluster [DBG] pgmap v51: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 140 op/s 2024-09-13T14:32:23.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:22 smithi177 bash[24494]: cluster 2024-09-13T14:32:21.951338+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-09-13T14:32:24.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:23 smithi099 bash[84576]: cluster 2024-09-13T14:32:22.955722+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-09-13T14:32:24.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:23 smithi099 bash[84576]: cluster 2024-09-13T14:32:22.955722+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-09-13T14:32:24.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:23 smithi099 bash[32082]: cluster 2024-09-13T14:32:22.955722+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-09-13T14:32:24.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:23 smithi177 bash[24494]: cluster 2024-09-13T14:32:22.955722+0000 mon.a (mon.0) 128 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-09-13T14:32:24.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:24 smithi177 bash[61439]: ts=2024-09-13T14:32:24.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:24.996 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Empty 2024-09-13T14:32:25.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[32082]: cluster 2024-09-13T14:32:23.922533+0000 mgr.y (mgr.26252) 36 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:25.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[32082]: cluster 2024-09-13T14:32:23.989177+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-09-13T14:32:25.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[32082]: audit 2024-09-13T14:32:24.002745+0000 mon.c (mon.2) 186 : audit [INF] from='client.? 172.21.15.99:0/2284505524' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:32:25.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[32082]: audit 2024-09-13T14:32:24.004143+0000 mon.a (mon.0) 130 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:32:25.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[84576]: cluster 2024-09-13T14:32:23.922533+0000 mgr.y (mgr.26252) 36 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:25.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[84576]: cluster 2024-09-13T14:32:23.922533+0000 mgr.y (mgr.26252) 36 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:25.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[84576]: cluster 2024-09-13T14:32:23.989177+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-09-13T14:32:25.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[84576]: cluster 2024-09-13T14:32:23.989177+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-09-13T14:32:25.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[84576]: audit 2024-09-13T14:32:24.002745+0000 mon.c (mon.2) 186 : audit [INF] from='client.? 172.21.15.99:0/2284505524' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:32:25.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[84576]: audit 2024-09-13T14:32:24.002745+0000 mon.c (mon.2) 186 : audit [INF] from='client.? 172.21.15.99:0/2284505524' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:32:25.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[84576]: audit 2024-09-13T14:32:24.004143+0000 mon.a (mon.0) 130 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:32:25.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:24 smithi099 bash[84576]: audit 2024-09-13T14:32:24.004143+0000 mon.a (mon.0) 130 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:32:25.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:24 smithi177 bash[24494]: cluster 2024-09-13T14:32:23.922533+0000 mgr.y (mgr.26252) 36 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:25.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:24 smithi177 bash[24494]: cluster 2024-09-13T14:32:23.989177+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-09-13T14:32:25.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:24 smithi177 bash[24494]: audit 2024-09-13T14:32:24.002745+0000 mon.c (mon.2) 186 : audit [INF] from='client.? 172.21.15.99:0/2284505524' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:32:25.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:24 smithi177 bash[24494]: audit 2024-09-13T14:32:24.004143+0000 mon.a (mon.0) 130 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:32:25.997 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:25 smithi177 bash[61439]: ts=2024-09-13T14:32:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:25.997 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:25 smithi177 bash[61439]: ts=2024-09-13T14:32:25.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:26.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:26 smithi099 bash[32082]: audit 2024-09-13T14:32:24.978623+0000 mon.a (mon.0) 131 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:32:26.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:26 smithi099 bash[32082]: cluster 2024-09-13T14:32:24.986937+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-13T14:32:26.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:26 smithi099 bash[84576]: audit 2024-09-13T14:32:24.978623+0000 mon.a (mon.0) 131 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:32:26.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:26 smithi099 bash[84576]: audit 2024-09-13T14:32:24.978623+0000 mon.a (mon.0) 131 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:32:26.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:26 smithi099 bash[84576]: cluster 2024-09-13T14:32:24.986937+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-13T14:32:26.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:26 smithi099 bash[84576]: cluster 2024-09-13T14:32:24.986937+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-13T14:32:26.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:26 smithi177 bash[24494]: audit 2024-09-13T14:32:24.978623+0000 mon.a (mon.0) 131 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:32:26.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:26 smithi177 bash[24494]: cluster 2024-09-13T14:32:24.986937+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-13T14:32:27.264 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Empty (2269 ms) 2024-09-13T14:32:27.265 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.NoSnaps 2024-09-13T14:32:27.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:32:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:32:27] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.51.0" 2024-09-13T14:32:27.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:27 smithi099 bash[32082]: cluster 2024-09-13T14:32:25.923203+0000 mgr.y (mgr.26252) 37 : cluster [DBG] pgmap v57: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:27.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:27 smithi099 bash[32082]: cluster 2024-09-13T14:32:26.008717+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-09-13T14:32:27.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:27 smithi099 bash[84576]: cluster 2024-09-13T14:32:25.923203+0000 mgr.y (mgr.26252) 37 : cluster [DBG] pgmap v57: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:27.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:27 smithi099 bash[84576]: cluster 2024-09-13T14:32:25.923203+0000 mgr.y (mgr.26252) 37 : cluster [DBG] pgmap v57: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:27.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:27 smithi099 bash[84576]: cluster 2024-09-13T14:32:26.008717+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-09-13T14:32:27.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:27 smithi099 bash[84576]: cluster 2024-09-13T14:32:26.008717+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-09-13T14:32:27.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:27 smithi177 bash[24494]: cluster 2024-09-13T14:32:25.923203+0000 mgr.y (mgr.26252) 37 : cluster [DBG] pgmap v57: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:27.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:27 smithi177 bash[24494]: cluster 2024-09-13T14:32:26.008717+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-09-13T14:32:27.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:26 smithi177 bash[61439]: ts=2024-09-13T14:32:26.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:27.752 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.NoSnaps (487 ms) 2024-09-13T14:32:27.752 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Snaps 2024-09-13T14:32:28.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:28 smithi099 bash[84576]: cluster 2024-09-13T14:32:27.009612+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-09-13T14:32:28.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:28 smithi099 bash[84576]: cluster 2024-09-13T14:32:27.009612+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-09-13T14:32:28.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:28 smithi099 bash[32082]: cluster 2024-09-13T14:32:27.009612+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-09-13T14:32:28.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:28 smithi177 bash[24494]: cluster 2024-09-13T14:32:27.009612+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-09-13T14:32:29.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:29 smithi099 bash[32082]: cluster 2024-09-13T14:32:27.923933+0000 mgr.y (mgr.26252) 38 : cluster [DBG] pgmap v60: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:29.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:29 smithi099 bash[32082]: cluster 2024-09-13T14:32:28.030991+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-09-13T14:32:29.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:29 smithi099 bash[84576]: cluster 2024-09-13T14:32:27.923933+0000 mgr.y (mgr.26252) 38 : cluster [DBG] pgmap v60: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:29.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:29 smithi099 bash[84576]: cluster 2024-09-13T14:32:27.923933+0000 mgr.y (mgr.26252) 38 : cluster [DBG] pgmap v60: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:29.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:29 smithi099 bash[84576]: cluster 2024-09-13T14:32:28.030991+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-09-13T14:32:29.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:29 smithi099 bash[84576]: cluster 2024-09-13T14:32:28.030991+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-09-13T14:32:29.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:29 smithi177 bash[24494]: cluster 2024-09-13T14:32:27.923933+0000 mgr.y (mgr.26252) 38 : cluster [DBG] pgmap v60: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:29.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:29 smithi177 bash[24494]: cluster 2024-09-13T14:32:28.030991+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-09-13T14:32:30.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:30 smithi099 bash[84576]: cluster 2024-09-13T14:32:29.033532+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-09-13T14:32:30.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:30 smithi099 bash[84576]: cluster 2024-09-13T14:32:29.033532+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-09-13T14:32:30.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:30 smithi099 bash[32082]: cluster 2024-09-13T14:32:29.033532+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-09-13T14:32:30.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:30 smithi177 bash[24494]: cluster 2024-09-13T14:32:29.033532+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-09-13T14:32:30.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:30 smithi177 bash[61439]: ts=2024-09-13T14:32:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:31.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:31 smithi099 bash[84576]: cluster 2024-09-13T14:32:29.924639+0000 mgr.y (mgr.26252) 39 : cluster [DBG] pgmap v63: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T14:32:31.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:31 smithi099 bash[84576]: cluster 2024-09-13T14:32:29.924639+0000 mgr.y (mgr.26252) 39 : cluster [DBG] pgmap v63: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T14:32:31.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:31 smithi099 bash[84576]: cluster 2024-09-13T14:32:30.057319+0000 mon.a (mon.0) 137 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-09-13T14:32:31.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:31 smithi099 bash[84576]: cluster 2024-09-13T14:32:30.057319+0000 mon.a (mon.0) 137 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-09-13T14:32:31.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:31 smithi099 bash[84576]: cluster 2024-09-13T14:32:30.953968+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-09-13T14:32:31.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:31 smithi099 bash[84576]: cluster 2024-09-13T14:32:30.953968+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-09-13T14:32:31.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:31 smithi099 bash[32082]: cluster 2024-09-13T14:32:29.924639+0000 mgr.y (mgr.26252) 39 : cluster [DBG] pgmap v63: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T14:32:31.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:31 smithi099 bash[32082]: cluster 2024-09-13T14:32:30.057319+0000 mon.a (mon.0) 137 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-09-13T14:32:31.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:31 smithi099 bash[32082]: cluster 2024-09-13T14:32:30.953968+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-09-13T14:32:31.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:31 smithi177 bash[24494]: cluster 2024-09-13T14:32:29.924639+0000 mgr.y (mgr.26252) 39 : cluster [DBG] pgmap v63: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T14:32:31.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:31 smithi177 bash[24494]: cluster 2024-09-13T14:32:30.057319+0000 mon.a (mon.0) 137 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-09-13T14:32:31.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:31 smithi177 bash[24494]: cluster 2024-09-13T14:32:30.953968+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-09-13T14:32:31.429 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Snaps (3678 ms) 2024-09-13T14:32:31.429 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.SnapDiscard 2024-09-13T14:32:33.307 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.SnapDiscard (1877 ms) 2024-09-13T14:32:33.307 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.CloneDiscard 2024-09-13T14:32:33.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:33 smithi099 bash[32082]: cluster 2024-09-13T14:32:31.925365+0000 mgr.y (mgr.26252) 40 : cluster [DBG] pgmap v66: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T14:32:33.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:33 smithi099 bash[32082]: cluster 2024-09-13T14:32:32.072650+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2024-09-13T14:32:33.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:33 smithi099 bash[84576]: cluster 2024-09-13T14:32:31.925365+0000 mgr.y (mgr.26252) 40 : cluster [DBG] pgmap v66: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T14:32:33.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:33 smithi099 bash[84576]: cluster 2024-09-13T14:32:31.925365+0000 mgr.y (mgr.26252) 40 : cluster [DBG] pgmap v66: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T14:32:33.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:33 smithi099 bash[84576]: cluster 2024-09-13T14:32:32.072650+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2024-09-13T14:32:33.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:33 smithi099 bash[84576]: cluster 2024-09-13T14:32:32.072650+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2024-09-13T14:32:33.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:33 smithi177 bash[24494]: cluster 2024-09-13T14:32:31.925365+0000 mgr.y (mgr.26252) 40 : cluster [DBG] pgmap v66: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T14:32:33.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:33 smithi177 bash[24494]: cluster 2024-09-13T14:32:32.072650+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2024-09-13T14:32:34.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:34 smithi177 bash[24494]: cluster 2024-09-13T14:32:33.094633+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-09-13T14:32:34.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:34 smithi177 bash[24494]: audit 2024-09-13T14:32:33.238968+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:34.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:34 smithi177 bash[61439]: ts=2024-09-13T14:32:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:34.509 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.CloneDiscard (1202 ms) 2024-09-13T14:32:34.509 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.CloneShrink 2024-09-13T14:32:34.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:34 smithi099 bash[84576]: cluster 2024-09-13T14:32:33.094633+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-09-13T14:32:34.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:34 smithi099 bash[84576]: cluster 2024-09-13T14:32:33.094633+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-09-13T14:32:34.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:34 smithi099 bash[84576]: audit 2024-09-13T14:32:33.238968+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:34.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:34 smithi099 bash[84576]: audit 2024-09-13T14:32:33.238968+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:34.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:34 smithi099 bash[32082]: cluster 2024-09-13T14:32:33.094633+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-09-13T14:32:34.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:34 smithi099 bash[32082]: audit 2024-09-13T14:32:33.238968+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:35.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:35 smithi177 bash[24494]: cluster 2024-09-13T14:32:33.926067+0000 mgr.y (mgr.26252) 41 : cluster [DBG] pgmap v69: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:35.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:35 smithi177 bash[24494]: cluster 2024-09-13T14:32:34.096350+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-09-13T14:32:35.502 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.CloneShrink (994 ms) 2024-09-13T14:32:35.502 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.CloneExpand 2024-09-13T14:32:35.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:35 smithi099 bash[84576]: cluster 2024-09-13T14:32:33.926067+0000 mgr.y (mgr.26252) 41 : cluster [DBG] pgmap v69: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:35.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:35 smithi099 bash[84576]: cluster 2024-09-13T14:32:33.926067+0000 mgr.y (mgr.26252) 41 : cluster [DBG] pgmap v69: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:35.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:35 smithi099 bash[84576]: cluster 2024-09-13T14:32:34.096350+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-09-13T14:32:35.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:35 smithi099 bash[84576]: cluster 2024-09-13T14:32:34.096350+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-09-13T14:32:35.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:35 smithi099 bash[32082]: cluster 2024-09-13T14:32:33.926067+0000 mgr.y (mgr.26252) 41 : cluster [DBG] pgmap v69: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:35.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:35 smithi099 bash[32082]: cluster 2024-09-13T14:32:34.096350+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-09-13T14:32:36.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:35 smithi177 bash[61439]: ts=2024-09-13T14:32:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:36.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:35 smithi177 bash[61439]: ts=2024-09-13T14:32:35.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:36.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:36 smithi099 bash[84576]: cluster 2024-09-13T14:32:35.107794+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-09-13T14:32:36.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:36 smithi099 bash[84576]: cluster 2024-09-13T14:32:35.107794+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-09-13T14:32:36.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:36 smithi099 bash[32082]: cluster 2024-09-13T14:32:35.107794+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-09-13T14:32:36.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:36 smithi177 bash[24494]: cluster 2024-09-13T14:32:35.107794+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-09-13T14:32:36.612 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.CloneExpand (1109 ms) 2024-09-13T14:32:36.612 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.CloneHideParent 2024-09-13T14:32:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:32:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:32:37] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.51.0" 2024-09-13T14:32:37.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:37 smithi099 bash[84576]: cluster 2024-09-13T14:32:35.926874+0000 mgr.y (mgr.26252) 42 : cluster [DBG] pgmap v72: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-13T14:32:37.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:37 smithi099 bash[84576]: cluster 2024-09-13T14:32:35.926874+0000 mgr.y (mgr.26252) 42 : cluster [DBG] pgmap v72: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-13T14:32:37.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:37 smithi099 bash[84576]: cluster 2024-09-13T14:32:36.121204+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-09-13T14:32:37.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:37 smithi099 bash[84576]: cluster 2024-09-13T14:32:36.121204+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-09-13T14:32:37.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:37 smithi099 bash[32082]: cluster 2024-09-13T14:32:35.926874+0000 mgr.y (mgr.26252) 42 : cluster [DBG] pgmap v72: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-13T14:32:37.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:37 smithi099 bash[32082]: cluster 2024-09-13T14:32:36.121204+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-09-13T14:32:37.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:37 smithi177 bash[24494]: cluster 2024-09-13T14:32:35.926874+0000 mgr.y (mgr.26252) 42 : cluster [DBG] pgmap v72: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-13T14:32:37.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:37 smithi177 bash[24494]: cluster 2024-09-13T14:32:36.121204+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-09-13T14:32:37.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:36 smithi177 bash[61439]: ts=2024-09-13T14:32:36.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:38.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:38 smithi099 bash[32082]: cluster 2024-09-13T14:32:37.121843+0000 mon.a (mon.0) 145 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-09-13T14:32:38.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:38 smithi099 bash[84576]: cluster 2024-09-13T14:32:37.121843+0000 mon.a (mon.0) 145 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-09-13T14:32:38.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:38 smithi099 bash[84576]: cluster 2024-09-13T14:32:37.121843+0000 mon.a (mon.0) 145 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-09-13T14:32:38.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:38 smithi177 bash[24494]: cluster 2024-09-13T14:32:37.121843+0000 mon.a (mon.0) 145 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-09-13T14:32:39.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:39 smithi099 bash[84576]: cluster 2024-09-13T14:32:37.927604+0000 mgr.y (mgr.26252) 43 : cluster [DBG] pgmap v75: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-13T14:32:39.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:39 smithi099 bash[84576]: cluster 2024-09-13T14:32:37.927604+0000 mgr.y (mgr.26252) 43 : cluster [DBG] pgmap v75: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-13T14:32:39.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:39 smithi099 bash[84576]: cluster 2024-09-13T14:32:38.141177+0000 mon.a (mon.0) 146 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2024-09-13T14:32:39.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:39 smithi099 bash[84576]: cluster 2024-09-13T14:32:38.141177+0000 mon.a (mon.0) 146 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2024-09-13T14:32:39.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:39 smithi099 bash[32082]: cluster 2024-09-13T14:32:37.927604+0000 mgr.y (mgr.26252) 43 : cluster [DBG] pgmap v75: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-13T14:32:39.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:39 smithi099 bash[32082]: cluster 2024-09-13T14:32:38.141177+0000 mon.a (mon.0) 146 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2024-09-13T14:32:39.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:39 smithi177 bash[24494]: cluster 2024-09-13T14:32:37.927604+0000 mgr.y (mgr.26252) 43 : cluster [DBG] pgmap v75: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-13T14:32:39.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:39 smithi177 bash[24494]: cluster 2024-09-13T14:32:38.141177+0000 mon.a (mon.0) 146 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2024-09-13T14:32:40.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:40 smithi099 bash[84576]: cluster 2024-09-13T14:32:39.141788+0000 mon.a (mon.0) 147 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2024-09-13T14:32:40.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:40 smithi099 bash[84576]: cluster 2024-09-13T14:32:39.141788+0000 mon.a (mon.0) 147 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2024-09-13T14:32:40.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:40 smithi099 bash[32082]: cluster 2024-09-13T14:32:39.141788+0000 mon.a (mon.0) 147 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2024-09-13T14:32:40.604 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:40 smithi177 bash[61439]: ts=2024-09-13T14:32:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:40.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:40 smithi177 bash[24494]: cluster 2024-09-13T14:32:39.141788+0000 mon.a (mon.0) 147 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2024-09-13T14:32:41.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:41 smithi099 bash[32082]: cluster 2024-09-13T14:32:39.928423+0000 mgr.y (mgr.26252) 44 : cluster [DBG] pgmap v78: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T14:32:41.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:41 smithi099 bash[32082]: cluster 2024-09-13T14:32:40.161583+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2024-09-13T14:32:41.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:41 smithi099 bash[84576]: cluster 2024-09-13T14:32:39.928423+0000 mgr.y (mgr.26252) 44 : cluster [DBG] pgmap v78: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T14:32:41.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:41 smithi099 bash[84576]: cluster 2024-09-13T14:32:39.928423+0000 mgr.y (mgr.26252) 44 : cluster [DBG] pgmap v78: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T14:32:41.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:41 smithi099 bash[84576]: cluster 2024-09-13T14:32:40.161583+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2024-09-13T14:32:41.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:41 smithi099 bash[84576]: cluster 2024-09-13T14:32:40.161583+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2024-09-13T14:32:41.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:41 smithi177 bash[24494]: cluster 2024-09-13T14:32:39.928423+0000 mgr.y (mgr.26252) 44 : cluster [DBG] pgmap v78: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T14:32:41.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:41 smithi177 bash[24494]: cluster 2024-09-13T14:32:40.161583+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2024-09-13T14:32:42.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:42 smithi099 bash[84576]: cluster 2024-09-13T14:32:41.174293+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2024-09-13T14:32:42.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:42 smithi099 bash[84576]: cluster 2024-09-13T14:32:41.174293+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2024-09-13T14:32:42.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:42 smithi099 bash[32082]: cluster 2024-09-13T14:32:41.174293+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2024-09-13T14:32:42.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:42 smithi177 bash[24494]: cluster 2024-09-13T14:32:41.174293+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2024-09-13T14:32:43.420 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.CloneHideParent (6808 ms) 2024-09-13T14:32:43.420 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Clone 2024-09-13T14:32:43.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:43 smithi099 bash[32082]: cluster 2024-09-13T14:32:41.929189+0000 mgr.y (mgr.26252) 45 : cluster [DBG] pgmap v81: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T14:32:43.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:43 smithi099 bash[32082]: cluster 2024-09-13T14:32:42.189417+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2024-09-13T14:32:43.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:43 smithi099 bash[84576]: cluster 2024-09-13T14:32:41.929189+0000 mgr.y (mgr.26252) 45 : cluster [DBG] pgmap v81: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T14:32:43.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:43 smithi099 bash[84576]: cluster 2024-09-13T14:32:41.929189+0000 mgr.y (mgr.26252) 45 : cluster [DBG] pgmap v81: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T14:32:43.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:43 smithi099 bash[84576]: cluster 2024-09-13T14:32:42.189417+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2024-09-13T14:32:43.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:43 smithi099 bash[84576]: cluster 2024-09-13T14:32:42.189417+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2024-09-13T14:32:43.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:43 smithi177 bash[24494]: cluster 2024-09-13T14:32:41.929189+0000 mgr.y (mgr.26252) 45 : cluster [DBG] pgmap v81: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T14:32:43.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:43 smithi177 bash[24494]: cluster 2024-09-13T14:32:42.189417+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2024-09-13T14:32:44.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:44 smithi099 bash[84576]: cluster 2024-09-13T14:32:43.196446+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2024-09-13T14:32:44.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:44 smithi099 bash[84576]: cluster 2024-09-13T14:32:43.196446+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2024-09-13T14:32:44.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:44 smithi099 bash[32082]: cluster 2024-09-13T14:32:43.196446+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2024-09-13T14:32:44.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:44 smithi177 bash[24494]: cluster 2024-09-13T14:32:43.196446+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2024-09-13T14:32:44.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:44 smithi177 bash[61439]: ts=2024-09-13T14:32:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:45.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:45 smithi099 bash[84576]: cluster 2024-09-13T14:32:43.929919+0000 mgr.y (mgr.26252) 46 : cluster [DBG] pgmap v84: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:45.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:45 smithi099 bash[84576]: cluster 2024-09-13T14:32:43.929919+0000 mgr.y (mgr.26252) 46 : cluster [DBG] pgmap v84: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:45.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:45 smithi099 bash[84576]: cluster 2024-09-13T14:32:44.211582+0000 mon.a (mon.0) 152 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2024-09-13T14:32:45.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:45 smithi099 bash[84576]: cluster 2024-09-13T14:32:44.211582+0000 mon.a (mon.0) 152 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2024-09-13T14:32:45.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:45 smithi099 bash[32082]: cluster 2024-09-13T14:32:43.929919+0000 mgr.y (mgr.26252) 46 : cluster [DBG] pgmap v84: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:45.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:45 smithi099 bash[32082]: cluster 2024-09-13T14:32:44.211582+0000 mon.a (mon.0) 152 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2024-09-13T14:32:45.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:45 smithi177 bash[24494]: cluster 2024-09-13T14:32:43.929919+0000 mgr.y (mgr.26252) 46 : cluster [DBG] pgmap v84: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:32:45.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:45 smithi177 bash[24494]: cluster 2024-09-13T14:32:44.211582+0000 mon.a (mon.0) 152 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2024-09-13T14:32:45.912 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Clone (2493 ms) 2024-09-13T14:32:45.912 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.CloneFlatten 2024-09-13T14:32:46.104 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:45 smithi177 bash[61439]: ts=2024-09-13T14:32:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:46.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:45 smithi177 bash[61439]: ts=2024-09-13T14:32:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:46.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:46 smithi099 bash[84576]: cluster 2024-09-13T14:32:45.224286+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2024-09-13T14:32:46.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:46 smithi099 bash[84576]: cluster 2024-09-13T14:32:45.224286+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2024-09-13T14:32:46.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:46 smithi099 bash[32082]: cluster 2024-09-13T14:32:45.224286+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2024-09-13T14:32:46.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:46 smithi177 bash[24494]: cluster 2024-09-13T14:32:45.224286+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2024-09-13T14:32:47.219 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:46 smithi177 bash[61439]: ts=2024-09-13T14:32:46.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:47.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:32:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:32:47] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.51.0" 2024-09-13T14:32:47.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:47 smithi099 bash[84576]: cluster 2024-09-13T14:32:45.930627+0000 mgr.y (mgr.26252) 47 : cluster [DBG] pgmap v87: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T14:32:47.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:47 smithi099 bash[84576]: cluster 2024-09-13T14:32:45.930627+0000 mgr.y (mgr.26252) 47 : cluster [DBG] pgmap v87: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T14:32:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:47 smithi099 bash[84576]: cluster 2024-09-13T14:32:46.229634+0000 mon.a (mon.0) 154 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2024-09-13T14:32:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:47 smithi099 bash[84576]: cluster 2024-09-13T14:32:46.229634+0000 mon.a (mon.0) 154 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2024-09-13T14:32:47.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:47 smithi099 bash[32082]: cluster 2024-09-13T14:32:45.930627+0000 mgr.y (mgr.26252) 47 : cluster [DBG] pgmap v87: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T14:32:47.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:47 smithi099 bash[32082]: cluster 2024-09-13T14:32:46.229634+0000 mon.a (mon.0) 154 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2024-09-13T14:32:47.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:47 smithi177 bash[24494]: cluster 2024-09-13T14:32:45.930627+0000 mgr.y (mgr.26252) 47 : cluster [DBG] pgmap v87: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T14:32:47.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:47 smithi177 bash[24494]: cluster 2024-09-13T14:32:46.229634+0000 mon.a (mon.0) 154 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2024-09-13T14:32:48.113 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.CloneFlatten (2201 ms) 2024-09-13T14:32:48.114 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Stress 2024-09-13T14:32:48.178 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 78632672~5253408 2024-09-13T14:32:48.212 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 24107900~3738262 2024-09-13T14:32:48.214 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 10563194~6678917 2024-09-13T14:32:48.249 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 29687654~3545411 2024-09-13T14:32:48.252 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 72815868~8004931 2024-09-13T14:32:48.279 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 34314733~5961006 2024-09-13T14:32:48.280 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 4476992~5147633 2024-09-13T14:32:48.304 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 45570637~6295008 2024-09-13T14:32:48.304 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap0 2024-09-13T14:32:48.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:48 smithi099 bash[32082]: cluster 2024-09-13T14:32:47.238167+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2024-09-13T14:32:48.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:48 smithi099 bash[84576]: cluster 2024-09-13T14:32:47.238167+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2024-09-13T14:32:48.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:48 smithi099 bash[84576]: cluster 2024-09-13T14:32:47.238167+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2024-09-13T14:32:48.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:48 smithi177 bash[24494]: cluster 2024-09-13T14:32:47.238167+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2024-09-13T14:32:49.302 INFO:tasks.workunit.client.0.smithi099.stdout:clone image236 -> image237 2024-09-13T14:32:49.421 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 28298030~1223296 2024-09-13T14:32:49.440 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 2417449~1264726 2024-09-13T14:32:49.448 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 40871138~3897349 2024-09-13T14:32:49.474 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 35513411~5510085 2024-09-13T14:32:49.481 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 46852039~1261633 2024-09-13T14:32:49.493 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 55726314~2851010 2024-09-13T14:32:49.500 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 41658850~998418 2024-09-13T14:32:49.507 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 80234152~3378704 2024-09-13T14:32:49.527 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap1 2024-09-13T14:32:49.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:49 smithi099 bash[32082]: cluster 2024-09-13T14:32:47.931308+0000 mgr.y (mgr.26252) 48 : cluster [DBG] pgmap v90: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T14:32:49.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:49 smithi099 bash[32082]: audit 2024-09-13T14:32:48.242511+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:49.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:49 smithi099 bash[84576]: cluster 2024-09-13T14:32:47.931308+0000 mgr.y (mgr.26252) 48 : cluster [DBG] pgmap v90: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T14:32:49.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:49 smithi099 bash[84576]: cluster 2024-09-13T14:32:47.931308+0000 mgr.y (mgr.26252) 48 : cluster [DBG] pgmap v90: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T14:32:49.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:49 smithi099 bash[84576]: audit 2024-09-13T14:32:48.242511+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:49.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:49 smithi099 bash[84576]: audit 2024-09-13T14:32:48.242511+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:49.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:49 smithi177 bash[24494]: cluster 2024-09-13T14:32:47.931308+0000 mgr.y (mgr.26252) 48 : cluster [DBG] pgmap v90: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T14:32:49.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:49 smithi177 bash[24494]: audit 2024-09-13T14:32:48.242511+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:50.307 INFO:tasks.workunit.client.0.smithi099.stdout:clone image237 -> image238 2024-09-13T14:32:50.467 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 90797346 2024-09-13T14:32:50.482 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 88594049~2203297 2024-09-13T14:32:50.508 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 34502458~6527589 2024-09-13T14:32:50.522 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 34406500~3763241 2024-09-13T14:32:50.543 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 62952898~4669327 2024-09-13T14:32:50.553 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 53670092~5194291 2024-09-13T14:32:50.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:50 smithi099 bash[84576]: cluster 2024-09-13T14:32:49.263889+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-09-13T14:32:50.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:50 smithi099 bash[84576]: cluster 2024-09-13T14:32:49.263889+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-09-13T14:32:50.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:50 smithi099 bash[32082]: cluster 2024-09-13T14:32:49.263889+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-09-13T14:32:50.592 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 76816327~1055364 2024-09-13T14:32:50.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:50 smithi177 bash[24494]: cluster 2024-09-13T14:32:49.263889+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-09-13T14:32:50.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:50 smithi177 bash[61439]: ts=2024-09-13T14:32:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:50.626 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 69768082~1131613 2024-09-13T14:32:50.633 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 44490204~8068516 2024-09-13T14:32:50.662 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap2 2024-09-13T14:32:51.295 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 71667829~670540 2024-09-13T14:32:51.300 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 28266123~8258247 2024-09-13T14:32:51.302 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 54543045~4506301 2024-09-13T14:32:51.303 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 8603127~6577957 2024-09-13T14:32:51.303 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 78372491~254402 2024-09-13T14:32:51.304 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 48773925~2879997 2024-09-13T14:32:51.305 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 3554943~1448687 2024-09-13T14:32:51.305 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 42957040~8114259 2024-09-13T14:32:51.398 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap3 2024-09-13T14:32:51.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:51 smithi099 bash[84576]: cluster 2024-09-13T14:32:49.932095+0000 mgr.y (mgr.26252) 49 : cluster [DBG] pgmap v92: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 32 MiB/s wr, 443 op/s 2024-09-13T14:32:51.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:51 smithi099 bash[84576]: cluster 2024-09-13T14:32:49.932095+0000 mgr.y (mgr.26252) 49 : cluster [DBG] pgmap v92: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 32 MiB/s wr, 443 op/s 2024-09-13T14:32:51.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:51 smithi099 bash[84576]: cluster 2024-09-13T14:32:50.276150+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2024-09-13T14:32:51.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:51 smithi099 bash[84576]: cluster 2024-09-13T14:32:50.276150+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2024-09-13T14:32:51.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:51 smithi099 bash[32082]: cluster 2024-09-13T14:32:49.932095+0000 mgr.y (mgr.26252) 49 : cluster [DBG] pgmap v92: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 32 MiB/s wr, 443 op/s 2024-09-13T14:32:51.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:51 smithi099 bash[32082]: cluster 2024-09-13T14:32:50.276150+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2024-09-13T14:32:51.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:51 smithi177 bash[24494]: cluster 2024-09-13T14:32:49.932095+0000 mgr.y (mgr.26252) 49 : cluster [DBG] pgmap v92: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 32 MiB/s wr, 443 op/s 2024-09-13T14:32:51.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:51 smithi177 bash[24494]: cluster 2024-09-13T14:32:50.276150+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2024-09-13T14:32:52.336 INFO:tasks.workunit.client.0.smithi099.stdout:clone image238 -> image239 2024-09-13T14:32:52.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:52 smithi099 bash[84576]: cluster 2024-09-13T14:32:51.275468+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2024-09-13T14:32:52.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:52 smithi099 bash[84576]: cluster 2024-09-13T14:32:51.275468+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2024-09-13T14:32:52.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:52 smithi099 bash[32082]: cluster 2024-09-13T14:32:51.275468+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2024-09-13T14:32:52.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:52 smithi177 bash[24494]: cluster 2024-09-13T14:32:51.275468+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2024-09-13T14:32:53.018 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Stress (4904 ms) 2024-09-13T14:32:53.018 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.NoSnaps_LargerDstObjSize 2024-09-13T14:32:53.494 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.NoSnaps_LargerDstObjSize (476 ms) 2024-09-13T14:32:53.494 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Snaps_LargerDstObjSize 2024-09-13T14:32:53.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:53 smithi099 bash[84576]: cluster 2024-09-13T14:32:51.932854+0000 mgr.y (mgr.26252) 50 : cluster [DBG] pgmap v95: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 32 MiB/s wr, 444 op/s 2024-09-13T14:32:53.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:53 smithi099 bash[84576]: cluster 2024-09-13T14:32:51.932854+0000 mgr.y (mgr.26252) 50 : cluster [DBG] pgmap v95: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 32 MiB/s wr, 444 op/s 2024-09-13T14:32:53.569 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:53 smithi099 bash[84576]: cluster 2024-09-13T14:32:52.300686+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-09-13T14:32:53.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:53 smithi099 bash[84576]: cluster 2024-09-13T14:32:52.300686+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-09-13T14:32:53.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:53 smithi099 bash[32082]: cluster 2024-09-13T14:32:51.932854+0000 mgr.y (mgr.26252) 50 : cluster [DBG] pgmap v95: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 32 MiB/s wr, 444 op/s 2024-09-13T14:32:53.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:53 smithi099 bash[32082]: cluster 2024-09-13T14:32:52.300686+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-09-13T14:32:53.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:53 smithi177 bash[24494]: cluster 2024-09-13T14:32:51.932854+0000 mgr.y (mgr.26252) 50 : cluster [DBG] pgmap v95: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 32 MiB/s wr, 444 op/s 2024-09-13T14:32:53.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:53 smithi177 bash[24494]: cluster 2024-09-13T14:32:52.300686+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-09-13T14:32:54.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:54 smithi177 bash[61439]: ts=2024-09-13T14:32:54.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:55.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:55 smithi177 bash[24494]: cluster 2024-09-13T14:32:53.933594+0000 mgr.y (mgr.26252) 51 : cluster [DBG] pgmap v97: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:55.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:55 smithi177 bash[24494]: cluster 2024-09-13T14:32:54.322878+0000 mon.a (mon.0) 161 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-09-13T14:32:55.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:55 smithi099 bash[32082]: cluster 2024-09-13T14:32:53.933594+0000 mgr.y (mgr.26252) 51 : cluster [DBG] pgmap v97: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:55.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:55 smithi099 bash[32082]: cluster 2024-09-13T14:32:54.322878+0000 mon.a (mon.0) 161 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-09-13T14:32:55.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:55 smithi099 bash[84576]: cluster 2024-09-13T14:32:53.933594+0000 mgr.y (mgr.26252) 51 : cluster [DBG] pgmap v97: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:55.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:55 smithi099 bash[84576]: cluster 2024-09-13T14:32:53.933594+0000 mgr.y (mgr.26252) 51 : cluster [DBG] pgmap v97: 105 pgs: 105 active+clean; 233 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:32:55.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:55 smithi099 bash[84576]: cluster 2024-09-13T14:32:54.322878+0000 mon.a (mon.0) 161 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-09-13T14:32:55.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:55 smithi099 bash[84576]: cluster 2024-09-13T14:32:54.322878+0000 mon.a (mon.0) 161 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-09-13T14:32:56.104 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:55 smithi177 bash[61439]: ts=2024-09-13T14:32:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:56.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:55 smithi177 bash[61439]: ts=2024-09-13T14:32:55.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:56.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:56 smithi177 bash[24494]: cluster 2024-09-13T14:32:55.323899+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-09-13T14:32:56.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:56 smithi099 bash[32082]: cluster 2024-09-13T14:32:55.323899+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-09-13T14:32:56.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:56 smithi099 bash[84576]: cluster 2024-09-13T14:32:55.323899+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-09-13T14:32:56.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:56 smithi099 bash[84576]: cluster 2024-09-13T14:32:55.323899+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-09-13T14:32:57.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:32:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:32:57] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.51.0" 2024-09-13T14:32:57.332 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:32:56 smithi177 bash[61439]: ts=2024-09-13T14:32:56.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:32:57.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:57 smithi177 bash[24494]: cluster 2024-09-13T14:32:55.934283+0000 mgr.y (mgr.26252) 52 : cluster [DBG] pgmap v100: 105 pgs: 105 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 19 MiB/s wr, 502 op/s 2024-09-13T14:32:57.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:57 smithi177 bash[24494]: cluster 2024-09-13T14:32:56.344110+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-09-13T14:32:57.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:57 smithi099 bash[84576]: cluster 2024-09-13T14:32:55.934283+0000 mgr.y (mgr.26252) 52 : cluster [DBG] pgmap v100: 105 pgs: 105 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 19 MiB/s wr, 502 op/s 2024-09-13T14:32:57.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:57 smithi099 bash[84576]: cluster 2024-09-13T14:32:55.934283+0000 mgr.y (mgr.26252) 52 : cluster [DBG] pgmap v100: 105 pgs: 105 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 19 MiB/s wr, 502 op/s 2024-09-13T14:32:57.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:57 smithi099 bash[84576]: cluster 2024-09-13T14:32:56.344110+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-09-13T14:32:57.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:57 smithi099 bash[84576]: cluster 2024-09-13T14:32:56.344110+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-09-13T14:32:57.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:57 smithi099 bash[32082]: cluster 2024-09-13T14:32:55.934283+0000 mgr.y (mgr.26252) 52 : cluster [DBG] pgmap v100: 105 pgs: 105 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 19 MiB/s wr, 502 op/s 2024-09-13T14:32:57.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:57 smithi099 bash[32082]: cluster 2024-09-13T14:32:56.344110+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-09-13T14:32:57.854 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Snaps_LargerDstObjSize (4360 ms) 2024-09-13T14:32:57.854 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Clone_LargerDstObjSize 2024-09-13T14:32:58.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:58 smithi177 bash[24494]: cluster 2024-09-13T14:32:57.351347+0000 mon.a (mon.0) 164 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-09-13T14:32:58.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:58 smithi099 bash[84576]: cluster 2024-09-13T14:32:57.351347+0000 mon.a (mon.0) 164 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-09-13T14:32:58.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:58 smithi099 bash[84576]: cluster 2024-09-13T14:32:57.351347+0000 mon.a (mon.0) 164 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-09-13T14:32:58.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:58 smithi099 bash[32082]: cluster 2024-09-13T14:32:57.351347+0000 mon.a (mon.0) 164 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-09-13T14:32:59.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:59 smithi177 bash[24494]: cluster 2024-09-13T14:32:57.934959+0000 mgr.y (mgr.26252) 53 : cluster [DBG] pgmap v103: 105 pgs: 105 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 22 MiB/s wr, 585 op/s 2024-09-13T14:32:59.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:32:59 smithi177 bash[24494]: cluster 2024-09-13T14:32:58.365824+0000 mon.a (mon.0) 165 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-09-13T14:32:59.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:59 smithi099 bash[32082]: cluster 2024-09-13T14:32:57.934959+0000 mgr.y (mgr.26252) 53 : cluster [DBG] pgmap v103: 105 pgs: 105 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 22 MiB/s wr, 585 op/s 2024-09-13T14:32:59.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:32:59 smithi099 bash[32082]: cluster 2024-09-13T14:32:58.365824+0000 mon.a (mon.0) 165 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-09-13T14:32:59.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:59 smithi099 bash[84576]: cluster 2024-09-13T14:32:57.934959+0000 mgr.y (mgr.26252) 53 : cluster [DBG] pgmap v103: 105 pgs: 105 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 22 MiB/s wr, 585 op/s 2024-09-13T14:32:59.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:59 smithi099 bash[84576]: cluster 2024-09-13T14:32:57.934959+0000 mgr.y (mgr.26252) 53 : cluster [DBG] pgmap v103: 105 pgs: 105 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 22 MiB/s wr, 585 op/s 2024-09-13T14:32:59.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:59 smithi099 bash[84576]: cluster 2024-09-13T14:32:58.365824+0000 mon.a (mon.0) 165 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-09-13T14:32:59.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:32:59 smithi099 bash[84576]: cluster 2024-09-13T14:32:58.365824+0000 mon.a (mon.0) 165 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-09-13T14:33:00.110 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Clone_LargerDstObjSize (2256 ms) 2024-09-13T14:33:00.110 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.CloneFlatten_LargerDstObjSize 2024-09-13T14:33:00.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:00 smithi177 bash[24494]: cluster 2024-09-13T14:32:59.372612+0000 mon.a (mon.0) 166 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-09-13T14:33:00.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:00 smithi177 bash[61439]: ts=2024-09-13T14:33:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:00.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:00 smithi099 bash[84576]: cluster 2024-09-13T14:32:59.372612+0000 mon.a (mon.0) 166 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-09-13T14:33:00.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:00 smithi099 bash[84576]: cluster 2024-09-13T14:32:59.372612+0000 mon.a (mon.0) 166 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-09-13T14:33:00.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:00 smithi099 bash[32082]: cluster 2024-09-13T14:32:59.372612+0000 mon.a (mon.0) 166 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-09-13T14:33:01.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:01 smithi099 bash[32082]: cluster 2024-09-13T14:32:59.935749+0000 mgr.y (mgr.26252) 54 : cluster [DBG] pgmap v106: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-13T14:33:01.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:01 smithi099 bash[32082]: cluster 2024-09-13T14:33:00.383055+0000 mon.a (mon.0) 167 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-09-13T14:33:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:01 smithi099 bash[84576]: cluster 2024-09-13T14:32:59.935749+0000 mgr.y (mgr.26252) 54 : cluster [DBG] pgmap v106: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-13T14:33:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:01 smithi099 bash[84576]: cluster 2024-09-13T14:32:59.935749+0000 mgr.y (mgr.26252) 54 : cluster [DBG] pgmap v106: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-13T14:33:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:01 smithi099 bash[84576]: cluster 2024-09-13T14:33:00.383055+0000 mon.a (mon.0) 167 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-09-13T14:33:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:01 smithi099 bash[84576]: cluster 2024-09-13T14:33:00.383055+0000 mon.a (mon.0) 167 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-09-13T14:33:01.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:01 smithi177 bash[24494]: cluster 2024-09-13T14:32:59.935749+0000 mgr.y (mgr.26252) 54 : cluster [DBG] pgmap v106: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-13T14:33:01.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:01 smithi177 bash[24494]: cluster 2024-09-13T14:33:00.383055+0000 mon.a (mon.0) 167 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-09-13T14:33:02.170 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.CloneFlatten_LargerDstObjSize (2060 ms) 2024-09-13T14:33:02.171 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Stress_LargerDstObjSize 2024-09-13T14:33:02.233 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 59514280~3681242 2024-09-13T14:33:02.264 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 2096423~4395882 2024-09-13T14:33:02.265 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 33749277~1925637 2024-09-13T14:33:02.280 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 47417740~1776331 2024-09-13T14:33:02.283 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 42327377~8279765 2024-09-13T14:33:02.317 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 18323664~6145240 2024-09-13T14:33:02.317 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 28501995~253976 2024-09-13T14:33:02.323 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 47461322~6052750 2024-09-13T14:33:02.326 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap0 2024-09-13T14:33:02.448 INFO:tasks.workunit.client.0.smithi099.stdout:clone image258 -> image259 2024-09-13T14:33:02.573 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 89658764 2024-09-13T14:33:02.588 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 66014175~4451550 2024-09-13T14:33:02.616 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 80768618~7696099 2024-09-13T14:33:02.623 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 83912638~1460938 2024-09-13T14:33:02.637 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 83550231~5487906 2024-09-13T14:33:02.647 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 79384073~2522035 2024-09-13T14:33:02.662 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 12628653~6951635 2024-09-13T14:33:02.672 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 73772851~7656099 2024-09-13T14:33:02.704 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 70459611~1869828 2024-09-13T14:33:02.707 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap1 2024-09-13T14:33:02.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:02 smithi099 bash[32082]: cluster 2024-09-13T14:33:01.382451+0000 mon.a (mon.0) 168 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-09-13T14:33:02.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:02 smithi099 bash[84576]: cluster 2024-09-13T14:33:01.382451+0000 mon.a (mon.0) 168 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-09-13T14:33:02.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:02 smithi099 bash[84576]: cluster 2024-09-13T14:33:01.382451+0000 mon.a (mon.0) 168 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-09-13T14:33:02.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:02 smithi177 bash[24494]: cluster 2024-09-13T14:33:01.382451+0000 mon.a (mon.0) 168 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-09-13T14:33:03.459 INFO:tasks.workunit.client.0.smithi099.stdout:clone image259 -> image260 2024-09-13T14:33:03.624 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 173528330 2024-09-13T14:33:03.641 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 96862956~6757389 2024-09-13T14:33:03.681 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 117356919~3524584 2024-09-13T14:33:03.682 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 55268764~3502278 2024-09-13T14:33:03.727 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 98705984~7850192 2024-09-13T14:33:03.736 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 161529213~908234 2024-09-13T14:33:03.746 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 86951972~3052674 2024-09-13T14:33:03.755 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 157831281~5438656 2024-09-13T14:33:03.781 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 167539920~5988410 2024-09-13T14:33:03.781 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap2 2024-09-13T14:33:03.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:03 smithi099 bash[84576]: cluster 2024-09-13T14:33:01.936358+0000 mgr.y (mgr.26252) 55 : cluster [DBG] pgmap v109: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-13T14:33:03.908 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:03 smithi099 bash[84576]: cluster 2024-09-13T14:33:01.936358+0000 mgr.y (mgr.26252) 55 : cluster [DBG] pgmap v109: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-13T14:33:03.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:03 smithi099 bash[84576]: cluster 2024-09-13T14:33:02.414307+0000 mon.a (mon.0) 169 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-09-13T14:33:03.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:03 smithi099 bash[84576]: cluster 2024-09-13T14:33:02.414307+0000 mon.a (mon.0) 169 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-09-13T14:33:03.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:03 smithi099 bash[84576]: audit 2024-09-13T14:33:03.241815+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:03.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:03 smithi099 bash[84576]: audit 2024-09-13T14:33:03.241815+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:03.909 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:03 smithi099 bash[32082]: cluster 2024-09-13T14:33:01.936358+0000 mgr.y (mgr.26252) 55 : cluster [DBG] pgmap v109: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-13T14:33:03.909 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:03 smithi099 bash[32082]: cluster 2024-09-13T14:33:02.414307+0000 mon.a (mon.0) 169 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-09-13T14:33:03.909 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:03 smithi099 bash[32082]: audit 2024-09-13T14:33:03.241815+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:03.910 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:03 smithi177 bash[24494]: cluster 2024-09-13T14:33:01.936358+0000 mgr.y (mgr.26252) 55 : cluster [DBG] pgmap v109: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-13T14:33:03.910 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:03 smithi177 bash[24494]: cluster 2024-09-13T14:33:02.414307+0000 mon.a (mon.0) 169 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-09-13T14:33:03.910 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:03 smithi177 bash[24494]: audit 2024-09-13T14:33:03.241815+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:04.422 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:04 smithi177 bash[61439]: ts=2024-09-13T14:33:04.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:04.468 INFO:tasks.workunit.client.0.smithi099.stdout:clone image260 -> image261 2024-09-13T14:33:04.659 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 92001818~49584 2024-09-13T14:33:04.673 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 154036798~7543587 2024-09-13T14:33:04.699 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 79532417~5520209 2024-09-13T14:33:04.743 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 104557968~3291933 2024-09-13T14:33:04.752 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 104827850~6887361 2024-09-13T14:33:04.781 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 167368381~6159949 2024-09-13T14:33:04.793 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 52895202~7796095 2024-09-13T14:33:04.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:04 smithi099 bash[84576]: cluster 2024-09-13T14:33:03.421070+0000 mon.a (mon.0) 171 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-09-13T14:33:04.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:04 smithi099 bash[84576]: cluster 2024-09-13T14:33:03.421070+0000 mon.a (mon.0) 171 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-09-13T14:33:04.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:04 smithi099 bash[32082]: cluster 2024-09-13T14:33:03.421070+0000 mon.a (mon.0) 171 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-09-13T14:33:04.846 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 40277497~2519256 2024-09-13T14:33:04.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:04 smithi177 bash[24494]: cluster 2024-09-13T14:33:03.421070+0000 mon.a (mon.0) 171 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-09-13T14:33:04.871 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap3 2024-09-13T14:33:05.482 INFO:tasks.workunit.client.0.smithi099.stdout:clone image261 -> image262 2024-09-13T14:33:05.734 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:05 smithi177 bash[24494]: cluster 2024-09-13T14:33:03.937023+0000 mgr.y (mgr.26252) 56 : cluster [DBG] pgmap v112: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:33:05.734 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:05 smithi177 bash[24494]: cluster 2024-09-13T14:33:04.431807+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-09-13T14:33:05.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:05 smithi099 bash[84576]: cluster 2024-09-13T14:33:03.937023+0000 mgr.y (mgr.26252) 56 : cluster [DBG] pgmap v112: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:33:05.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:05 smithi099 bash[84576]: cluster 2024-09-13T14:33:03.937023+0000 mgr.y (mgr.26252) 56 : cluster [DBG] pgmap v112: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:33:05.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:05 smithi099 bash[84576]: cluster 2024-09-13T14:33:04.431807+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-09-13T14:33:05.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:05 smithi099 bash[84576]: cluster 2024-09-13T14:33:04.431807+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-09-13T14:33:05.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:05 smithi099 bash[32082]: cluster 2024-09-13T14:33:03.937023+0000 mgr.y (mgr.26252) 56 : cluster [DBG] pgmap v112: 105 pgs: 105 active+clean; 401 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:33:05.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:05 smithi099 bash[32082]: cluster 2024-09-13T14:33:04.431807+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-09-13T14:33:06.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:05 smithi177 bash[61439]: ts=2024-09-13T14:33:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:06.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:05 smithi177 bash[61439]: ts=2024-09-13T14:33:05.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:06.390 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Stress_LargerDstObjSize (4220 ms) 2024-09-13T14:33:06.391 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.NoSnaps_SmallerDstObjSize 2024-09-13T14:33:06.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:06 smithi099 bash[84576]: cluster 2024-09-13T14:33:05.434135+0000 mon.a (mon.0) 173 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-09-13T14:33:06.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:06 smithi099 bash[84576]: cluster 2024-09-13T14:33:05.434135+0000 mon.a (mon.0) 173 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-09-13T14:33:06.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:06 smithi099 bash[32082]: cluster 2024-09-13T14:33:05.434135+0000 mon.a (mon.0) 173 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-09-13T14:33:06.824 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.NoSnaps_SmallerDstObjSize (433 ms) 2024-09-13T14:33:06.824 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Snaps_SmallerDstObjSize 2024-09-13T14:33:06.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:06 smithi177 bash[24494]: cluster 2024-09-13T14:33:05.434135+0000 mon.a (mon.0) 173 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-09-13T14:33:07.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:33:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:33:07] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.51.0" 2024-09-13T14:33:07.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:06 smithi177 bash[61439]: ts=2024-09-13T14:33:06.958Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:07.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:07 smithi099 bash[84576]: cluster 2024-09-13T14:33:05.937781+0000 mgr.y (mgr.26252) 57 : cluster [DBG] pgmap v115: 105 pgs: 105 active+clean; 588 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 45 MiB/s wr, 642 op/s 2024-09-13T14:33:07.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:07 smithi099 bash[84576]: cluster 2024-09-13T14:33:05.937781+0000 mgr.y (mgr.26252) 57 : cluster [DBG] pgmap v115: 105 pgs: 105 active+clean; 588 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 45 MiB/s wr, 642 op/s 2024-09-13T14:33:07.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:07 smithi099 bash[32082]: cluster 2024-09-13T14:33:05.937781+0000 mgr.y (mgr.26252) 57 : cluster [DBG] pgmap v115: 105 pgs: 105 active+clean; 588 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 45 MiB/s wr, 642 op/s 2024-09-13T14:33:07.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:07 smithi177 bash[24494]: cluster 2024-09-13T14:33:05.937781+0000 mgr.y (mgr.26252) 57 : cluster [DBG] pgmap v115: 105 pgs: 105 active+clean; 588 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 45 MiB/s wr, 642 op/s 2024-09-13T14:33:08.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:08 smithi099 bash[32082]: cluster 2024-09-13T14:33:07.456673+0000 mon.a (mon.0) 174 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-09-13T14:33:08.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:08 smithi099 bash[84576]: cluster 2024-09-13T14:33:07.456673+0000 mon.a (mon.0) 174 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-09-13T14:33:08.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:08 smithi099 bash[84576]: cluster 2024-09-13T14:33:07.456673+0000 mon.a (mon.0) 174 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-09-13T14:33:08.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:08 smithi177 bash[24494]: cluster 2024-09-13T14:33:07.456673+0000 mon.a (mon.0) 174 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-09-13T14:33:09.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:09 smithi099 bash[32082]: cluster 2024-09-13T14:33:07.938439+0000 mgr.y (mgr.26252) 58 : cluster [DBG] pgmap v117: 105 pgs: 105 active+clean; 588 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 40 MiB/s wr, 568 op/s 2024-09-13T14:33:09.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:09 smithi099 bash[32082]: cluster 2024-09-13T14:33:08.480851+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-09-13T14:33:09.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:09 smithi099 bash[32082]: audit 2024-09-13T14:33:08.510503+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:09.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:09 smithi099 bash[84576]: cluster 2024-09-13T14:33:07.938439+0000 mgr.y (mgr.26252) 58 : cluster [DBG] pgmap v117: 105 pgs: 105 active+clean; 588 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 40 MiB/s wr, 568 op/s 2024-09-13T14:33:09.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:09 smithi099 bash[84576]: cluster 2024-09-13T14:33:07.938439+0000 mgr.y (mgr.26252) 58 : cluster [DBG] pgmap v117: 105 pgs: 105 active+clean; 588 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 40 MiB/s wr, 568 op/s 2024-09-13T14:33:09.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:09 smithi099 bash[84576]: cluster 2024-09-13T14:33:08.480851+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-09-13T14:33:09.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:09 smithi099 bash[84576]: cluster 2024-09-13T14:33:08.480851+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-09-13T14:33:09.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:09 smithi099 bash[84576]: audit 2024-09-13T14:33:08.510503+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:09.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:09 smithi099 bash[84576]: audit 2024-09-13T14:33:08.510503+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:09.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:09 smithi177 bash[24494]: cluster 2024-09-13T14:33:07.938439+0000 mgr.y (mgr.26252) 58 : cluster [DBG] pgmap v117: 105 pgs: 105 active+clean; 588 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 40 MiB/s wr, 568 op/s 2024-09-13T14:33:09.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:09 smithi177 bash[24494]: cluster 2024-09-13T14:33:08.480851+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-09-13T14:33:09.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:09 smithi177 bash[24494]: audit 2024-09-13T14:33:08.510503+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:10.489 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:10 smithi177 bash[61439]: ts=2024-09-13T14:33:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:10.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:10 smithi099 bash[84576]: cluster 2024-09-13T14:33:09.481007+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2024-09-13T14:33:10.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:10 smithi099 bash[84576]: cluster 2024-09-13T14:33:09.481007+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2024-09-13T14:33:10.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:10 smithi099 bash[32082]: cluster 2024-09-13T14:33:09.481007+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2024-09-13T14:33:10.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:10 smithi177 bash[24494]: cluster 2024-09-13T14:33:09.481007+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2024-09-13T14:33:10.956 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Snaps_SmallerDstObjSize (4132 ms) 2024-09-13T14:33:10.956 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Clone_SmallerDstObjSize 2024-09-13T14:33:11.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:11 smithi099 bash[32082]: cluster 2024-09-13T14:33:09.939267+0000 mgr.y (mgr.26252) 59 : cluster [DBG] pgmap v120: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 33 MiB/s rd, 17 MiB/s wr, 498 op/s 2024-09-13T14:33:11.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:11 smithi099 bash[32082]: cluster 2024-09-13T14:33:10.504721+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2024-09-13T14:33:11.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:11 smithi099 bash[84576]: cluster 2024-09-13T14:33:09.939267+0000 mgr.y (mgr.26252) 59 : cluster [DBG] pgmap v120: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 33 MiB/s rd, 17 MiB/s wr, 498 op/s 2024-09-13T14:33:11.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:11 smithi099 bash[84576]: cluster 2024-09-13T14:33:09.939267+0000 mgr.y (mgr.26252) 59 : cluster [DBG] pgmap v120: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 33 MiB/s rd, 17 MiB/s wr, 498 op/s 2024-09-13T14:33:11.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:11 smithi099 bash[84576]: cluster 2024-09-13T14:33:10.504721+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2024-09-13T14:33:11.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:11 smithi099 bash[84576]: cluster 2024-09-13T14:33:10.504721+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2024-09-13T14:33:11.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:11 smithi177 bash[24494]: cluster 2024-09-13T14:33:09.939267+0000 mgr.y (mgr.26252) 59 : cluster [DBG] pgmap v120: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 33 MiB/s rd, 17 MiB/s wr, 498 op/s 2024-09-13T14:33:11.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:11 smithi177 bash[24494]: cluster 2024-09-13T14:33:10.504721+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2024-09-13T14:33:12.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:12 smithi099 bash[84576]: cluster 2024-09-13T14:33:11.510744+0000 mon.a (mon.0) 179 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2024-09-13T14:33:12.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:12 smithi099 bash[84576]: cluster 2024-09-13T14:33:11.510744+0000 mon.a (mon.0) 179 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2024-09-13T14:33:12.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:12 smithi099 bash[32082]: cluster 2024-09-13T14:33:11.510744+0000 mon.a (mon.0) 179 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2024-09-13T14:33:12.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:12 smithi177 bash[24494]: cluster 2024-09-13T14:33:11.510744+0000 mon.a (mon.0) 179 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2024-09-13T14:33:13.184 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Clone_SmallerDstObjSize (2228 ms) 2024-09-13T14:33:13.184 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.CloneFlatten_SmallerDstObjSize 2024-09-13T14:33:13.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:13 smithi099 bash[84576]: cluster 2024-09-13T14:33:11.939924+0000 mgr.y (mgr.26252) 60 : cluster [DBG] pgmap v123: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 20 MiB/s wr, 562 op/s 2024-09-13T14:33:13.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:13 smithi099 bash[84576]: cluster 2024-09-13T14:33:11.939924+0000 mgr.y (mgr.26252) 60 : cluster [DBG] pgmap v123: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 20 MiB/s wr, 562 op/s 2024-09-13T14:33:13.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:13 smithi099 bash[84576]: cluster 2024-09-13T14:33:12.523116+0000 mon.a (mon.0) 180 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2024-09-13T14:33:13.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:13 smithi099 bash[84576]: cluster 2024-09-13T14:33:12.523116+0000 mon.a (mon.0) 180 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2024-09-13T14:33:13.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:13 smithi099 bash[32082]: cluster 2024-09-13T14:33:11.939924+0000 mgr.y (mgr.26252) 60 : cluster [DBG] pgmap v123: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 20 MiB/s wr, 562 op/s 2024-09-13T14:33:13.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:13 smithi099 bash[32082]: cluster 2024-09-13T14:33:12.523116+0000 mon.a (mon.0) 180 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2024-09-13T14:33:13.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:13 smithi177 bash[24494]: cluster 2024-09-13T14:33:11.939924+0000 mgr.y (mgr.26252) 60 : cluster [DBG] pgmap v123: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 20 MiB/s wr, 562 op/s 2024-09-13T14:33:13.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:13 smithi177 bash[24494]: cluster 2024-09-13T14:33:12.523116+0000 mon.a (mon.0) 180 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2024-09-13T14:33:14.532 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:14 smithi177 bash[61439]: ts=2024-09-13T14:33:14.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:14.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:14 smithi177 bash[24494]: cluster 2024-09-13T14:33:13.524138+0000 mon.a (mon.0) 181 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2024-09-13T14:33:14.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:14 smithi177 bash[24494]: audit 2024-09-13T14:33:14.346146+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:14.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:14 smithi177 bash[24494]: audit 2024-09-13T14:33:14.350482+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:14.903 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:14 smithi099 bash[84576]: cluster 2024-09-13T14:33:13.524138+0000 mon.a (mon.0) 181 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2024-09-13T14:33:14.903 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:14 smithi099 bash[84576]: cluster 2024-09-13T14:33:13.524138+0000 mon.a (mon.0) 181 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2024-09-13T14:33:14.903 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:14 smithi099 bash[84576]: audit 2024-09-13T14:33:14.346146+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:14.903 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:14 smithi099 bash[84576]: audit 2024-09-13T14:33:14.346146+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:14.903 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:14 smithi099 bash[84576]: audit 2024-09-13T14:33:14.350482+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:14.903 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:14 smithi099 bash[84576]: audit 2024-09-13T14:33:14.350482+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:14.903 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:14 smithi099 bash[32082]: cluster 2024-09-13T14:33:13.524138+0000 mon.a (mon.0) 181 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2024-09-13T14:33:14.903 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:14 smithi099 bash[32082]: audit 2024-09-13T14:33:14.346146+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:14.903 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:14 smithi099 bash[32082]: audit 2024-09-13T14:33:14.350482+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.192 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.CloneFlatten_SmallerDstObjSize (2008 ms) 2024-09-13T14:33:15.193 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Stress_SmallerDstObjSize 2024-09-13T14:33:15.243 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 76267214~7618866 2024-09-13T14:33:15.273 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 5911176~1852021 2024-09-13T14:33:15.275 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 18059554~6541489 2024-09-13T14:33:15.299 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 76600639~3510978 2024-09-13T14:33:15.302 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 42086963~2173593 2024-09-13T14:33:15.317 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 67851031~1152752 2024-09-13T14:33:15.319 INFO:tasks.workunit.client.0.smithi099.stdout:write: A 30343018~7687510 2024-09-13T14:33:15.344 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 37870867~6262376 2024-09-13T14:33:15.347 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap0 2024-09-13T14:33:15.352 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade resume' 2024-09-13T14:33:15.684 INFO:tasks.workunit.client.0.smithi099.stdout:clone image281 -> image282 2024-09-13T14:33:15.802 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 4736368~2815188 2024-09-13T14:33:15.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[32082]: cluster 2024-09-13T14:33:13.940395+0000 mgr.y (mgr.26252) 61 : cluster [DBG] pgmap v126: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:33:15.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[32082]: cluster 2024-09-13T14:33:14.544472+0000 mon.a (mon.0) 184 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2024-09-13T14:33:15.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[32082]: audit 2024-09-13T14:33:14.629565+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[32082]: audit 2024-09-13T14:33:14.640523+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[32082]: audit 2024-09-13T14:33:15.356916+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:15.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[32082]: audit 2024-09-13T14:33:15.357942+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:15.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[32082]: audit 2024-09-13T14:33:15.364251+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: cluster 2024-09-13T14:33:13.940395+0000 mgr.y (mgr.26252) 61 : cluster [DBG] pgmap v126: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:33:15.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: cluster 2024-09-13T14:33:13.940395+0000 mgr.y (mgr.26252) 61 : cluster [DBG] pgmap v126: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:33:15.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: cluster 2024-09-13T14:33:14.544472+0000 mon.a (mon.0) 184 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2024-09-13T14:33:15.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: cluster 2024-09-13T14:33:14.544472+0000 mon.a (mon.0) 184 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2024-09-13T14:33:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: audit 2024-09-13T14:33:14.629565+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: audit 2024-09-13T14:33:14.629565+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: audit 2024-09-13T14:33:14.640523+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: audit 2024-09-13T14:33:14.640523+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: audit 2024-09-13T14:33:15.356916+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: audit 2024-09-13T14:33:15.356916+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: audit 2024-09-13T14:33:15.357942+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: audit 2024-09-13T14:33:15.357942+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: audit 2024-09-13T14:33:15.364251+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:15 smithi099 bash[84576]: audit 2024-09-13T14:33:15.364251+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.835 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 36933046~522370 2024-09-13T14:33:15.855 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:15 smithi177 bash[61439]: ts=2024-09-13T14:33:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:15.855 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:15 smithi177 bash[61439]: ts=2024-09-13T14:33:15.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:15.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:15 smithi177 bash[24494]: cluster 2024-09-13T14:33:13.940395+0000 mgr.y (mgr.26252) 61 : cluster [DBG] pgmap v126: 105 pgs: 105 active+clean; 688 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:33:15.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:15 smithi177 bash[24494]: cluster 2024-09-13T14:33:14.544472+0000 mon.a (mon.0) 184 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2024-09-13T14:33:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:15 smithi177 bash[24494]: audit 2024-09-13T14:33:14.629565+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:15 smithi177 bash[24494]: audit 2024-09-13T14:33:14.640523+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:15 smithi177 bash[24494]: audit 2024-09-13T14:33:15.356916+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:15 smithi177 bash[24494]: audit 2024-09-13T14:33:15.357942+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:15 smithi177 bash[24494]: audit 2024-09-13T14:33:15.364251+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:15.874 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 5051100~4143855 2024-09-13T14:33:15.896 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 23853142~5897870 2024-09-13T14:33:15.935 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 45473837~5705244 2024-09-13T14:33:15.964 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 64541228~5397949 2024-09-13T14:33:15.973 INFO:tasks.workunit.client.0.smithi099.stdout:write: B 29794031~2330144 2024-09-13T14:33:15.985 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 4182165~6923805 2024-09-13T14:33:15.991 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap1 2024-09-13T14:33:16.669 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 106127799 2024-09-13T14:33:16.685 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 61737849~5661754 2024-09-13T14:33:16.692 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 15345406~7638096 2024-09-13T14:33:16.708 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 98795193~7332606 2024-09-13T14:33:16.708 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 100087399~5104580 2024-09-13T14:33:16.777 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 72216497~6254031 2024-09-13T14:33:16.777 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 76113923~4201712 2024-09-13T14:33:16.805 INFO:tasks.workunit.client.0.smithi099.stdout:write: C 38179130~3847596 2024-09-13T14:33:16.805 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 28846066~1138821 2024-09-13T14:33:16.839 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap2 2024-09-13T14:33:16.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:16 smithi177 bash[24494]: cluster 2024-09-13T14:33:15.645143+0000 mon.a (mon.0) 190 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2024-09-13T14:33:17.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:16 smithi099 bash[32082]: cluster 2024-09-13T14:33:15.645143+0000 mon.a (mon.0) 190 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2024-09-13T14:33:17.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:16 smithi099 bash[84576]: cluster 2024-09-13T14:33:15.645143+0000 mon.a (mon.0) 190 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2024-09-13T14:33:17.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:16 smithi099 bash[84576]: cluster 2024-09-13T14:33:15.645143+0000 mon.a (mon.0) 190 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2024-09-13T14:33:17.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:33:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:33:17] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.51.0" 2024-09-13T14:33:17.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:16 smithi177 bash[61439]: ts=2024-09-13T14:33:16.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:17.698 INFO:tasks.workunit.client.0.smithi099.stdout:clone image282 -> image283 2024-09-13T14:33:17.860 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 77488840~4735473 2024-09-13T14:33:17.928 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 84147449~3506558 2024-09-13T14:33:17.934 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 2090043~389986 2024-09-13T14:33:17.942 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 63116587~2207806 2024-09-13T14:33:17.970 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 88886774~7862727 2024-09-13T14:33:18.010 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 71037807~7168970 2024-09-13T14:33:18.022 INFO:tasks.workunit.client.0.smithi099.stdout:write: D 46385156~3465866 2024-09-13T14:33:18.061 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 28648592~2342053 2024-09-13T14:33:18.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:17 smithi099 bash[84576]: cluster 2024-09-13T14:33:15.941016+0000 mgr.y (mgr.26252) 62 : cluster [DBG] pgmap v129: 105 pgs: 105 active+clean; 760 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 467 op/s 2024-09-13T14:33:18.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:17 smithi099 bash[84576]: cluster 2024-09-13T14:33:15.941016+0000 mgr.y (mgr.26252) 62 : cluster [DBG] pgmap v129: 105 pgs: 105 active+clean; 760 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 467 op/s 2024-09-13T14:33:18.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:17 smithi099 bash[84576]: cluster 2024-09-13T14:33:16.654340+0000 mon.a (mon.0) 191 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2024-09-13T14:33:18.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:17 smithi099 bash[84576]: cluster 2024-09-13T14:33:16.654340+0000 mon.a (mon.0) 191 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2024-09-13T14:33:18.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:17 smithi099 bash[32082]: cluster 2024-09-13T14:33:15.941016+0000 mgr.y (mgr.26252) 62 : cluster [DBG] pgmap v129: 105 pgs: 105 active+clean; 760 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 467 op/s 2024-09-13T14:33:18.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:17 smithi099 bash[32082]: cluster 2024-09-13T14:33:16.654340+0000 mon.a (mon.0) 191 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2024-09-13T14:33:18.091 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap3 2024-09-13T14:33:18.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:17 smithi177 bash[24494]: cluster 2024-09-13T14:33:15.941016+0000 mgr.y (mgr.26252) 62 : cluster [DBG] pgmap v129: 105 pgs: 105 active+clean; 760 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 467 op/s 2024-09-13T14:33:18.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:17 smithi177 bash[24494]: cluster 2024-09-13T14:33:16.654340+0000 mon.a (mon.0) 191 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2024-09-13T14:33:18.708 INFO:tasks.workunit.client.0.smithi099.stdout:clone image283 -> image284 2024-09-13T14:33:18.900 INFO:tasks.workunit.client.0.smithi099.stdout:resize: 110753343 2024-09-13T14:33:19.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:18 smithi099 bash[32082]: cluster 2024-09-13T14:33:17.663258+0000 mon.a (mon.0) 192 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2024-09-13T14:33:19.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:18 smithi099 bash[32082]: audit 2024-09-13T14:33:18.242259+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:19.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:18 smithi099 bash[84576]: cluster 2024-09-13T14:33:17.663258+0000 mon.a (mon.0) 192 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2024-09-13T14:33:19.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:18 smithi099 bash[84576]: cluster 2024-09-13T14:33:17.663258+0000 mon.a (mon.0) 192 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2024-09-13T14:33:19.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:18 smithi099 bash[84576]: audit 2024-09-13T14:33:18.242259+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:19.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:18 smithi099 bash[84576]: audit 2024-09-13T14:33:18.242259+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:19.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:18 smithi177 bash[24494]: cluster 2024-09-13T14:33:17.663258+0000 mon.a (mon.0) 192 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2024-09-13T14:33:19.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:18 smithi177 bash[24494]: audit 2024-09-13T14:33:18.242259+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:19.475 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Stress_SmallerDstObjSize (4283 ms) 2024-09-13T14:33:19.476 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-09-13T14:33:19.534 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:19.700 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize (224 ms) 2024-09-13T14:33:19.700 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-09-13T14:33:19.749 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:19.910 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Snaps_StrippingLargerDstObjSize (211 ms) 2024-09-13T14:33:19.910 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-09-13T14:33:19.971 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:20.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:19 smithi099 bash[32082]: cluster 2024-09-13T14:33:17.941848+0000 mgr.y (mgr.26252) 63 : cluster [DBG] pgmap v132: 105 pgs: 105 active+clean; 760 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 467 op/s 2024-09-13T14:33:20.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:19 smithi099 bash[32082]: cluster 2024-09-13T14:33:18.668134+0000 mon.a (mon.0) 194 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2024-09-13T14:33:20.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:19 smithi099 bash[84576]: cluster 2024-09-13T14:33:17.941848+0000 mgr.y (mgr.26252) 63 : cluster [DBG] pgmap v132: 105 pgs: 105 active+clean; 760 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 467 op/s 2024-09-13T14:33:20.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:19 smithi099 bash[84576]: cluster 2024-09-13T14:33:17.941848+0000 mgr.y (mgr.26252) 63 : cluster [DBG] pgmap v132: 105 pgs: 105 active+clean; 760 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 467 op/s 2024-09-13T14:33:20.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:19 smithi099 bash[84576]: cluster 2024-09-13T14:33:18.668134+0000 mon.a (mon.0) 194 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2024-09-13T14:33:20.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:19 smithi099 bash[84576]: cluster 2024-09-13T14:33:18.668134+0000 mon.a (mon.0) 194 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2024-09-13T14:33:20.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:19 smithi177 bash[24494]: cluster 2024-09-13T14:33:17.941848+0000 mgr.y (mgr.26252) 63 : cluster [DBG] pgmap v132: 105 pgs: 105 active+clean; 760 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 467 op/s 2024-09-13T14:33:20.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:19 smithi177 bash[24494]: cluster 2024-09-13T14:33:18.668134+0000 mon.a (mon.0) 194 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2024-09-13T14:33:20.140 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:20.141 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Clone_StrippingLargerDstObjSize (231 ms) 2024-09-13T14:33:20.141 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-09-13T14:33:20.184 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:20.303 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize (162 ms) 2024-09-13T14:33:20.303 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-09-13T14:33:20.353 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:20.524 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Stress_StrippingLargerDstObjSize (221 ms) 2024-09-13T14:33:20.525 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-09-13T14:33:20.583 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:20.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:20 smithi177 bash[61439]: ts=2024-09-13T14:33:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:20.753 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize (229 ms) 2024-09-13T14:33:20.753 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-09-13T14:33:20.805 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:20.979 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize (226 ms) 2024-09-13T14:33:20.979 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-09-13T14:33:21.043 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:21.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:20 smithi099 bash[32082]: cluster 2024-09-13T14:33:19.686965+0000 mon.a (mon.0) 195 : cluster [DBG] mgrmap e39: y(active, since 91s), standbys: x 2024-09-13T14:33:21.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:20 smithi099 bash[84576]: cluster 2024-09-13T14:33:19.686965+0000 mon.a (mon.0) 195 : cluster [DBG] mgrmap e39: y(active, since 91s), standbys: x 2024-09-13T14:33:21.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:20 smithi099 bash[84576]: cluster 2024-09-13T14:33:19.686965+0000 mon.a (mon.0) 195 : cluster [DBG] mgrmap e39: y(active, since 91s), standbys: x 2024-09-13T14:33:21.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:20 smithi177 bash[24494]: cluster 2024-09-13T14:33:19.686965+0000 mon.a (mon.0) 195 : cluster [DBG] mgrmap e39: y(active, since 91s), standbys: x 2024-09-13T14:33:21.214 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Clone_StrippingSmallerDstObjSize (234 ms) 2024-09-13T14:33:21.214 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-09-13T14:33:21.272 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:21.441 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize (227 ms) 2024-09-13T14:33:21.441 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-09-13T14:33:21.489 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:21.648 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestDeepCopy.Stress_StrippingSmallerDstObjSize (207 ms) 2024-09-13T14:33:21.733 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 31 tests from TestDeepCopy (56652 ms total) 2024-09-13T14:33:21.734 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:33:21.734 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 51 tests from TestMirroring 2024-09-13T14:33:21.957 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:21 smithi099 bash[32082]: cluster 2024-09-13T14:33:19.943563+0000 mgr.y (mgr.26252) 64 : cluster [DBG] pgmap v134: 105 pgs: 105 active+clean; 977 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 44 MiB/s wr, 600 op/s 2024-09-13T14:33:21.957 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:21 smithi099 bash[84576]: cluster 2024-09-13T14:33:19.943563+0000 mgr.y (mgr.26252) 64 : cluster [DBG] pgmap v134: 105 pgs: 105 active+clean; 977 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 44 MiB/s wr, 600 op/s 2024-09-13T14:33:21.957 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:21 smithi099 bash[84576]: cluster 2024-09-13T14:33:19.943563+0000 mgr.y (mgr.26252) 64 : cluster [DBG] pgmap v134: 105 pgs: 105 active+clean; 977 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 44 MiB/s wr, 600 op/s 2024-09-13T14:33:22.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:21 smithi177 bash[24494]: cluster 2024-09-13T14:33:19.943563+0000 mgr.y (mgr.26252) 64 : cluster [DBG] pgmap v134: 105 pgs: 105 active+clean; 977 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 37 MiB/s rd, 44 MiB/s wr, 600 op/s 2024-09-13T14:33:23.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:22 smithi099 bash[32082]: cluster 2024-09-13T14:33:21.724092+0000 mon.a (mon.0) 196 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2024-09-13T14:33:23.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:22 smithi099 bash[32082]: cluster 2024-09-13T14:33:21.944004+0000 mgr.y (mgr.26252) 65 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:33:23.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:22 smithi099 bash[84576]: cluster 2024-09-13T14:33:21.724092+0000 mon.a (mon.0) 196 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2024-09-13T14:33:23.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:22 smithi099 bash[84576]: cluster 2024-09-13T14:33:21.724092+0000 mon.a (mon.0) 196 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2024-09-13T14:33:23.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:22 smithi099 bash[84576]: cluster 2024-09-13T14:33:21.944004+0000 mgr.y (mgr.26252) 65 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:33:23.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:22 smithi099 bash[84576]: cluster 2024-09-13T14:33:21.944004+0000 mgr.y (mgr.26252) 65 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:33:23.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:22 smithi177 bash[24494]: cluster 2024-09-13T14:33:21.724092+0000 mon.a (mon.0) 196 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2024-09-13T14:33:23.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:22 smithi177 bash[24494]: cluster 2024-09-13T14:33:21.944004+0000 mgr.y (mgr.26252) 65 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-09-13T14:33:23.736 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage 2024-09-13T14:33:24.034 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:23 smithi099 bash[32082]: cluster 2024-09-13T14:33:22.737049+0000 mon.a (mon.0) 197 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2024-09-13T14:33:24.034 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:23 smithi099 bash[32082]: audit 2024-09-13T14:33:22.741837+0000 mon.c (mon.2) 187 : audit [INF] from='client.? 172.21.15.99:0/3289303752' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:33:24.034 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:23 smithi099 bash[32082]: audit 2024-09-13T14:33:22.747752+0000 mon.a (mon.0) 198 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:24.034 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:23 smithi099 bash[84576]: cluster 2024-09-13T14:33:22.737049+0000 mon.a (mon.0) 197 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2024-09-13T14:33:24.035 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:23 smithi099 bash[84576]: cluster 2024-09-13T14:33:22.737049+0000 mon.a (mon.0) 197 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2024-09-13T14:33:24.035 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:23 smithi099 bash[84576]: audit 2024-09-13T14:33:22.741837+0000 mon.c (mon.2) 187 : audit [INF] from='client.? 172.21.15.99:0/3289303752' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:33:24.035 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:23 smithi099 bash[84576]: audit 2024-09-13T14:33:22.741837+0000 mon.c (mon.2) 187 : audit [INF] from='client.? 172.21.15.99:0/3289303752' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:33:24.035 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:23 smithi099 bash[84576]: audit 2024-09-13T14:33:22.747752+0000 mon.a (mon.0) 198 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:24.035 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:23 smithi099 bash[84576]: audit 2024-09-13T14:33:22.747752+0000 mon.a (mon.0) 198 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:24.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:23 smithi177 bash[24494]: cluster 2024-09-13T14:33:22.737049+0000 mon.a (mon.0) 197 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2024-09-13T14:33:24.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:23 smithi177 bash[24494]: audit 2024-09-13T14:33:22.741837+0000 mon.c (mon.2) 187 : audit [INF] from='client.? 172.21.15.99:0/3289303752' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:33:24.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:23 smithi177 bash[24494]: audit 2024-09-13T14:33:22.747752+0000 mon.a (mon.0) 198 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:33:24.413 INFO:teuthology.orchestra.run.smithi099.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:33:24.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[61439]: ts=2024-09-13T14:33:24.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:25.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:23.731365+0000 mon.a (mon.0) 199 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:25.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:23.731365+0000 mon.a (mon.0) 199 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: cluster 2024-09-13T14:33:23.739854+0000 mon.a (mon.0) 200 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: cluster 2024-09-13T14:33:23.739854+0000 mon.a (mon.0) 200 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: cluster 2024-09-13T14:33:23.945028+0000 mgr.y (mgr.26252) 66 : cluster [DBG] pgmap v139: 105 pgs: 18 unknown, 87 active+clean; 582 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: cluster 2024-09-13T14:33:23.945028+0000 mgr.y (mgr.26252) 66 : cluster [DBG] pgmap v139: 105 pgs: 18 unknown, 87 active+clean; 582 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.409868+0000 mgr.y (mgr.26252) 67 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.409868+0000 mgr.y (mgr.26252) 67 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: cephadm 2024-09-13T14:33:24.410264+0000 mgr.y (mgr.26252) 68 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: cephadm 2024-09-13T14:33:24.410264+0000 mgr.y (mgr.26252) 68 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.416786+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.416786+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.418193+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:25.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.418193+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:25.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.420262+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:25.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.420262+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:25.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.421270+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:25.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.421270+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:25.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:23.731365+0000 mon.a (mon.0) 199 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:25.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: cluster 2024-09-13T14:33:23.739854+0000 mon.a (mon.0) 200 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2024-09-13T14:33:25.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: cluster 2024-09-13T14:33:23.945028+0000 mgr.y (mgr.26252) 66 : cluster [DBG] pgmap v139: 105 pgs: 18 unknown, 87 active+clean; 582 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:33:25.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:24.409868+0000 mgr.y (mgr.26252) 67 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:25.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: cephadm 2024-09-13T14:33:24.410264+0000 mgr.y (mgr.26252) 68 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:33:25.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:24.416786+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:25.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:24.418193+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:25.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:24.420262+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:25.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:24.421270+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:25.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:24.429011+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:25.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:24.509949+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:25.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:24.511974+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:33:25.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:24.513159+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:33:25.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: audit 2024-09-13T14:33:24.513801+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-13T14:33:25.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[32082]: cephadm 2024-09-13T14:33:24.514301+0000 mgr.y (mgr.26252) 69 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-13T14:33:25.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.429011+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.429011+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.509949+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.509949+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.511974+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.511974+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.513159+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.513159+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.513801+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: audit 2024-09-13T14:33:24.513801+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: cephadm 2024-09-13T14:33:24.514301+0000 mgr.y (mgr.26252) 69 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-13T14:33:25.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:24 smithi099 bash[84576]: cephadm 2024-09-13T14:33:24.514301+0000 mgr.y (mgr.26252) 69 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-13T14:33:25.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:23.731365+0000 mon.a (mon.0) 199 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:33:25.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: cluster 2024-09-13T14:33:23.739854+0000 mon.a (mon.0) 200 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2024-09-13T14:33:25.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: cluster 2024-09-13T14:33:23.945028+0000 mgr.y (mgr.26252) 66 : cluster [DBG] pgmap v139: 105 pgs: 18 unknown, 87 active+clean; 582 KiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T14:33:25.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:24.409868+0000 mgr.y (mgr.26252) 67 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:33:25.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: cephadm 2024-09-13T14:33:24.410264+0000 mgr.y (mgr.26252) 68 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:33:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:24.416786+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:24.418193+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:24.420262+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:24.421270+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:33:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:24.429011+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:24.509949+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:24.511974+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:33:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:24.513159+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:33:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: audit 2024-09-13T14:33:24.513801+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-13T14:33:25.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:24 smithi177 bash[24494]: cephadm 2024-09-13T14:33:24.514301+0000 mgr.y (mgr.26252) 69 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-13T14:33:25.529 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for majority of mons to upgrade' 2024-09-13T14:33:25.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:25 smithi099 bash[84576]: cluster 2024-09-13T14:33:24.770855+0000 mon.a (mon.0) 210 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-09-13T14:33:25.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:25 smithi099 bash[84576]: cluster 2024-09-13T14:33:24.770855+0000 mon.a (mon.0) 210 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-09-13T14:33:25.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:25 smithi099 bash[32082]: cluster 2024-09-13T14:33:24.770855+0000 mon.a (mon.0) 210 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-09-13T14:33:25.962 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage (2225 ms) 2024-09-13T14:33:25.962 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool 2024-09-13T14:33:26.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:25 smithi177 bash[24494]: cluster 2024-09-13T14:33:24.770855+0000 mon.a (mon.0) 210 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-09-13T14:33:26.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:25 smithi177 bash[61439]: ts=2024-09-13T14:33:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:26.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:25 smithi177 bash[61439]: ts=2024-09-13T14:33:25.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:26.188 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool (226 ms) 2024-09-13T14:33:26.189 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled 2024-09-13T14:33:26.392 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled (203 ms) 2024-09-13T14:33:26.392 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage 2024-09-13T14:33:26.623 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage (230 ms) 2024-09-13T14:33:26.623 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap 2024-09-13T14:33:26.824 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap (201 ms) 2024-09-13T14:33:26.824 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModePool 2024-09-13T14:33:27.006 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModePool (182 ms) 2024-09-13T14:33:27.006 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled 2024-09-13T14:33:27.043 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:26 smithi099 bash[84576]: cluster 2024-09-13T14:33:25.767347+0000 mon.a (mon.0) 211 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-09-13T14:33:27.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:26 smithi099 bash[84576]: cluster 2024-09-13T14:33:25.767347+0000 mon.a (mon.0) 211 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-09-13T14:33:27.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:26 smithi099 bash[84576]: cluster 2024-09-13T14:33:25.945662+0000 mgr.y (mgr.26252) 70 : cluster [DBG] pgmap v142: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:27.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:26 smithi099 bash[84576]: cluster 2024-09-13T14:33:25.945662+0000 mgr.y (mgr.26252) 70 : cluster [DBG] pgmap v142: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:27.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:26 smithi099 bash[32082]: cluster 2024-09-13T14:33:25.767347+0000 mon.a (mon.0) 211 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-09-13T14:33:27.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:26 smithi099 bash[32082]: cluster 2024-09-13T14:33:25.945662+0000 mgr.y (mgr.26252) 70 : cluster [DBG] pgmap v142: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:27.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:26 smithi177 bash[24494]: cluster 2024-09-13T14:33:25.767347+0000 mon.a (mon.0) 211 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-09-13T14:33:27.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:26 smithi177 bash[24494]: cluster 2024-09-13T14:33:25.945662+0000 mgr.y (mgr.26252) 70 : cluster [DBG] pgmap v142: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:27.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:26 smithi177 bash[61439]: ts=2024-09-13T14:33:26.952Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:27.220 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled (214 ms) 2024-09-13T14:33:27.220 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.DisableImageMirrorWithPeer 2024-09-13T14:33:27.220 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:27.220 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.DisableImageMirrorWithPeer (0 ms) 2024-09-13T14:33:27.220 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.DisableJournalingWithPeer 2024-09-13T14:33:27.220 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:33:27.220 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.DisableJournalingWithPeer (0 ms) 2024-09-13T14:33:27.220 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling 2024-09-13T14:33:27.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:33:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:33:27] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.51.0" 2024-09-13T14:33:27.368 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling (149 ms) 2024-09-13T14:33:27.368 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling 2024-09-13T14:33:27.549 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling (180 ms) 2024-09-13T14:33:27.549 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling 2024-09-13T14:33:28.888 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling (1339 ms) 2024-09-13T14:33:28.888 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock 2024-09-13T14:33:29.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[32082]: cluster 2024-09-13T14:33:27.782068+0000 mon.a (mon.0) 212 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-09-13T14:33:29.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[32082]: cluster 2024-09-13T14:33:27.946303+0000 mgr.y (mgr.26252) 71 : cluster [DBG] pgmap v144: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:29.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[32082]: cephadm 2024-09-13T14:33:28.003009+0000 mgr.y (mgr.26252) 72 : cephadm [INF] Upgrade: Updating mon.c 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[32082]: audit 2024-09-13T14:33:28.007485+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[32082]: audit 2024-09-13T14:33:28.009786+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[32082]: audit 2024-09-13T14:33:28.010629+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[32082]: audit 2024-09-13T14:33:28.011466+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[32082]: cephadm 2024-09-13T14:33:28.012500+0000 mgr.y (mgr.26252) 73 : cephadm [INF] Deploying daemon mon.c on smithi099 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: cluster 2024-09-13T14:33:27.782068+0000 mon.a (mon.0) 212 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: cluster 2024-09-13T14:33:27.782068+0000 mon.a (mon.0) 212 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: cluster 2024-09-13T14:33:27.946303+0000 mgr.y (mgr.26252) 71 : cluster [DBG] pgmap v144: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: cluster 2024-09-13T14:33:27.946303+0000 mgr.y (mgr.26252) 71 : cluster [DBG] pgmap v144: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: cephadm 2024-09-13T14:33:28.003009+0000 mgr.y (mgr.26252) 72 : cephadm [INF] Upgrade: Updating mon.c 2024-09-13T14:33:29.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: cephadm 2024-09-13T14:33:28.003009+0000 mgr.y (mgr.26252) 72 : cephadm [INF] Upgrade: Updating mon.c 2024-09-13T14:33:29.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: audit 2024-09-13T14:33:28.007485+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:29.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: audit 2024-09-13T14:33:28.007485+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:29.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: audit 2024-09-13T14:33:28.009786+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:33:29.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: audit 2024-09-13T14:33:28.009786+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:33:29.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: audit 2024-09-13T14:33:28.010629+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:33:29.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: audit 2024-09-13T14:33:28.010629+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:33:29.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: audit 2024-09-13T14:33:28.011466+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:29.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: audit 2024-09-13T14:33:28.011466+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:29.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: cephadm 2024-09-13T14:33:28.012500+0000 mgr.y (mgr.26252) 73 : cephadm [INF] Deploying daemon mon.c on smithi099 2024-09-13T14:33:29.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:28 smithi099 bash[84576]: cephadm 2024-09-13T14:33:28.012500+0000 mgr.y (mgr.26252) 73 : cephadm [INF] Deploying daemon mon.c on smithi099 2024-09-13T14:33:29.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:28 smithi177 bash[24494]: cluster 2024-09-13T14:33:27.782068+0000 mon.a (mon.0) 212 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-09-13T14:33:29.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:28 smithi177 bash[24494]: cluster 2024-09-13T14:33:27.946303+0000 mgr.y (mgr.26252) 71 : cluster [DBG] pgmap v144: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:33:29.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:28 smithi177 bash[24494]: cephadm 2024-09-13T14:33:28.003009+0000 mgr.y (mgr.26252) 72 : cephadm [INF] Upgrade: Updating mon.c 2024-09-13T14:33:29.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:28 smithi177 bash[24494]: audit 2024-09-13T14:33:28.007485+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:29.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:28 smithi177 bash[24494]: audit 2024-09-13T14:33:28.009786+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:33:29.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:28 smithi177 bash[24494]: audit 2024-09-13T14:33:28.010629+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:33:29.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:28 smithi177 bash[24494]: audit 2024-09-13T14:33:28.011466+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:33:29.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:28 smithi177 bash[24494]: cephadm 2024-09-13T14:33:28.012500+0000 mgr.y (mgr.26252) 73 : cephadm [INF] Deploying daemon mon.c on smithi099 2024-09-13T14:33:30.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:29 smithi099 bash[84576]: cluster 2024-09-13T14:33:28.795767+0000 mon.a (mon.0) 217 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-09-13T14:33:30.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:29 smithi099 bash[84576]: cluster 2024-09-13T14:33:28.795767+0000 mon.a (mon.0) 217 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-09-13T14:33:30.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:29 smithi099 bash[32082]: cluster 2024-09-13T14:33:28.795767+0000 mon.a (mon.0) 217 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-09-13T14:33:30.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:29 smithi177 bash[24494]: cluster 2024-09-13T14:33:28.795767+0000 mon.a (mon.0) 217 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-09-13T14:33:30.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:30 smithi177 bash[61439]: ts=2024-09-13T14:33:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:30.915 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock (2027 ms) 2024-09-13T14:33:30.915 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeDisabled 2024-09-13T14:33:31.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:30 smithi099 bash[32082]: cluster 2024-09-13T14:33:29.795066+0000 mon.a (mon.0) 218 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-09-13T14:33:31.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:30 smithi099 bash[32082]: cluster 2024-09-13T14:33:29.946831+0000 mgr.y (mgr.26252) 74 : cluster [DBG] pgmap v147: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 53 KiB/s wr, 569 op/s 2024-09-13T14:33:31.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:30 smithi099 bash[84576]: cluster 2024-09-13T14:33:29.795066+0000 mon.a (mon.0) 218 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-09-13T14:33:31.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:30 smithi099 bash[84576]: cluster 2024-09-13T14:33:29.795066+0000 mon.a (mon.0) 218 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-09-13T14:33:31.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:30 smithi099 bash[84576]: cluster 2024-09-13T14:33:29.946831+0000 mgr.y (mgr.26252) 74 : cluster [DBG] pgmap v147: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 53 KiB/s wr, 569 op/s 2024-09-13T14:33:31.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:30 smithi099 bash[84576]: cluster 2024-09-13T14:33:29.946831+0000 mgr.y (mgr.26252) 74 : cluster [DBG] pgmap v147: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 53 KiB/s wr, 569 op/s 2024-09-13T14:33:31.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:30 smithi177 bash[24494]: cluster 2024-09-13T14:33:29.795066+0000 mon.a (mon.0) 218 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-09-13T14:33:31.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:30 smithi177 bash[24494]: cluster 2024-09-13T14:33:29.946831+0000 mgr.y (mgr.26252) 74 : cluster [DBG] pgmap v147: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 53 KiB/s wr, 569 op/s 2024-09-13T14:33:31.116 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeDisabled (202 ms) 2024-09-13T14:33:31.116 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage 2024-09-13T14:33:31.252 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage (135 ms) 2024-09-13T14:33:31.252 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool 2024-09-13T14:33:31.355 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:31 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:31.355 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:33:31 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:31.355 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:33:31 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:31.356 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:33:31 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:31.356 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:33:31 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:31.356 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:33:31 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:31.357 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:33:31 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:31.357 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:33:31 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:31.357 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:31 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:31.357 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:31 smithi099 systemd[1]: Stopping Ceph mon.c for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:33:31.358 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:31.427 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool (176 ms) 2024-09-13T14:33:31.427 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling 2024-09-13T14:33:31.582 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling (155 ms) 2024-09-13T14:33:31.583 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling 2024-09-13T14:33:31.760 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling (178 ms) 2024-09-13T14:33:31.761 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeDisabled 2024-09-13T14:33:31.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:31 smithi099 bash[32082]: debug 2024-09-13T14:33:31.357+0000 7f84b40a3640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T14:33:31.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:31 smithi099 bash[32082]: debug 2024-09-13T14:33:31.357+0000 7f84b40a3640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-13T14:33:32.008 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeDisabled (247 ms) 2024-09-13T14:33:32.008 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage 2024-09-13T14:33:32.255 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage (248 ms) 2024-09-13T14:33:32.255 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled 2024-09-13T14:33:32.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:32 smithi099 bash[87012]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-c 2024-09-13T14:33:33.243 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:33:33 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:33.244 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:33:33 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:33.244 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:33:33 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:33.244 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:33:33 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:33.244 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:33:33 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:33.245 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:33 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:33.245 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:33:33 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:33.245 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:33:33 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:33.246 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:32 smithi099 bash[87092]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-c 2024-09-13T14:33:33.246 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:32 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-13T14:33:33.246 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:32 smithi099 systemd[1]: Stopped Ceph mon.c for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:33:33.246 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:33 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:33:33.569 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:33 smithi099 systemd[1]: Started Ceph mon.c for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:33:34.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:34 smithi177 bash[61439]: ts=2024-09-13T14:33:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:35.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.505+0000 7ff50de71c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:33:35.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.505+0000 7ff50de71c80 0 ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:33:35.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.505+0000 7ff50de71c80 0 pidfile_write: ignore empty --pid-file 2024-09-13T14:33:35.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 0 load: jerasure load: lrc 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: RocksDB version: 7.9.2 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Git sha 0 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: DB SUMMARY 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: DB Session ID: 6VO3QDNBJ6VCI4ZJDN72 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: CURRENT file: CURRENT 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 2058 Bytes 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000042.sst 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000040.log size: 2701055 ; 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.error_if_exists: 0 2024-09-13T14:33:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.create_if_missing: 0 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.env: 0x55e44c824160 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.info_log: 0x55e44de49880 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.statistics: (nil) 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.use_fsync: 0 2024-09-13T14:33:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T14:33:35.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:33:35.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:33:35.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:33:35.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:33:35.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T14:33:35.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:33:35.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:33:35.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T14:33:35.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:33:35.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:33:35.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.db_log_dir: 2024-09-13T14:33:35.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.wal_dir: 2024-09-13T14:33:35.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:33:35.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:33:35.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:33:35.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.write_buffer_manager: 0x55e44de4c960 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:33:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.unordered_write: 0 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.row_cache: None 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.wal_filter: None 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:33:35.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.two_write_queues: 0 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.wal_compression: 0 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.atomic_flush: 0 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:33:35.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:33:35.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_open_files: -1 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Compression algorithms supported: 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: kZSTD supported: 0 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: kXpressCompression supported: 0 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: kZlibCompression supported: 1 2024-09-13T14:33:35.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:33:35.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T14:33:35.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:33:35.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:33:35.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-13T14:33:35.831 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:33:35.831 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:33:35.831 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.merge_operator: 2024-09-13T14:33:35.831 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_filter: None 2024-09-13T14:33:35.831 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:33:35.831 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e44de494e0) 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: cache_index_and_filter_blocks: 1 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: pin_top_level_index_and_filter: 1 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: index_type: 0 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: data_block_index_type: 0 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: index_shortening: 1 2024-09-13T14:33:35.832 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: checksum: 4 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: no_block_cache: 0 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: block_cache: 0x55e44de3e850 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: block_cache_name: BinnedLRUCache 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: block_cache_options: 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: capacity : 536870912 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: num_shard_bits : 4 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: strict_capacity_limit : 0 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: high_pri_pool_ratio: 0.000 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: block_cache_compressed: (nil) 2024-09-13T14:33:35.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: persistent_cache: (nil) 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: block_size: 4096 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: block_size_deviation: 10 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: block_restart_interval: 16 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: index_block_restart_interval: 1 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: metadata_block_size: 4096 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: partition_filters: 0 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: use_delta_encoding: 1 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: filter_policy: bloomfilter 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: whole_key_filtering: 1 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: verify_compression: 0 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: read_amp_bytes_per_bit: 0 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: format_version: 5 2024-09-13T14:33:35.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: enable_index_compression: 1 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: block_align: 0 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: max_auto_readahead_size: 262144 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: prepopulate_block_cache: 0 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: initial_auto_readahead_size: 8192 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: num_file_reads_for_auto_readahead: 2 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compression: NoCompression 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.num_levels: 7 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:33:35.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:33:35.836 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:33:35.837 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:33:35.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:33:35.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.bloom_locality: 0 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:33:35.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.ttl: 2592000 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.enable_blob_files: false 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.min_blob_size: 0 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:33:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50ac47640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 42.sst 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 44, last_sequence is 24414, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.509+0000 7ff50de71c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.513+0000 7ff50de71c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6623fa0e-6c65-4beb-a183-460ada02465e 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.513+0000 7ff50de71c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726238015513985, "job": 1, "event": "recovery_started", "wal_files": [40]} 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.513+0000 7ff50de71c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726238015521902, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 1766798, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 24419, "largest_seqno": 25688, "table_properties": {"data_size": 1760129, "index_size": 4412, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10142, "raw_average_key_size": 22, "raw_value_size": 1748952, "raw_average_value_size": 3793, "num_data_blocks": 192, "num_entries": 461, "num_filter_entries": 461, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726238015, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6623fa0e-6c65-4beb-a183-460ada02465e", "db_session_id": "6VO3QDNBJ6VCI4ZJDN72", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726238015522015, "job": 1, "event": "recovery_finished"} 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:33:35.842 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e44de6ce00 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 4 rocksdb: DB pointer 0x55e44df70000 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff503c39640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff503c39640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: ** DB Stats ** 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: ** Compaction Stats [default] ** 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:33:35.843 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: L0 1/0 1.68 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 504.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: L6 1/0 12.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Sum 2/0 14.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 504.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 504.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: ** Compaction Stats [default] ** 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 504.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:33:35.844 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Flush(GB): cumulative 0.002, interval 0.002 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Cumulative compaction: 0.00 GB write, 154.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Interval compaction: 0.00 GB write, 154.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Block cache BinnedLRUCache@0x55e44de3e850#7 capacity: 512.00 MB usage: 5.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] at bind addrs [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 1 mon.c@-1(???) e3 preinit fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 0 mon.c@-1(???).mds e9 new map 2024-09-13T14:33:35.845 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.521+0000 7ff50de71c80 0 mon.c@-1(???).mds e9 print_map 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: e9 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: legacy client fscid: 1 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Filesystem 'foo' (1) 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: fs_name foo 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: epoch 9 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: created 2024-09-13T14:24:17.237575+0000 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: modified 2024-09-13T14:24:29.188501+0000 2024-09-13T14:33:35.846 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: tableserver 0 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: root 0 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: session_timeout 60 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: session_autoclose 300 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: max_file_size 1099511627776 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: max_xattr_size 65536 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: required_client_features {} 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: last_failure 0 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: last_failure_osd_epoch 65 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: max_mds 1 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: in 0 2024-09-13T14:33:35.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: up {0=24403} 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: failed 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: damaged 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: stopped 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: data_pools [4] 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: metadata_pool 3 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: inline_data disabled 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: balancer 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: bal_rank_mask -1 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: standby_count_wanted 1 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: qdb_cluster leader: 0 members: 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: [mds.foo.smithi177.outtkv{0:24403} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: 2024-09-13T14:33:35.848 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: 2024-09-13T14:33:35.849 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: Standby daemons: 2024-09-13T14:33:35.849 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: 2024-09-13T14:33:35.849 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: [mds.foo.smithi099.yyrtld{-1:24488} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.99:6834/2117449908,v1:172.21.15.99:6835/2117449908] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:33:35.849 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.525+0000 7ff50de71c80 0 mon.c@-1(???).osd e439 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:33:35.849 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.525+0000 7ff50de71c80 0 mon.c@-1(???).osd e439 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:33:35.849 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.525+0000 7ff50de71c80 0 mon.c@-1(???).osd e439 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:33:35.849 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.525+0000 7ff50de71c80 0 mon.c@-1(???).osd e439 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:33:35.849 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:35 smithi099 bash[87147]: debug 2024-09-13T14:33:35.525+0000 7ff50de71c80 1 mon.c@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2024-09-13T14:33:36.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:35 smithi177 bash[61439]: ts=2024-09-13T14:33:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:36.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:35 smithi177 bash[61439]: ts=2024-09-13T14:33:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:36.880 INFO:teuthology.orchestra.run.smithi099.stdout:wait for majority of mons to upgrade 2024-09-13T14:33:37.142 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:33:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:33:37] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-09-13T14:33:37.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:36 smithi177 bash[61439]: ts=2024-09-13T14:33:36.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:37.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:33:40.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:40 smithi177 bash[61439]: ts=2024-09-13T14:33:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:41.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:40 smithi099 bash[84576]: cluster 2024-09-13T14:33:30.828202+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-13T14:33:41.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:40 smithi099 bash[84576]: cluster 2024-09-13T14:33:30.828202+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-13T14:33:41.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:40 smithi099 bash[84576]: cluster 2024-09-13T14:33:30.969764+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:33:41.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:40 smithi099 bash[84576]: cluster 2024-09-13T14:33:30.969764+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:33:41.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:40 smithi177 bash[24494]: cluster 2024-09-13T14:33:30.828202+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-13T14:33:41.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:40 smithi177 bash[24494]: cluster 2024-09-13T14:33:30.969764+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:33:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:31.947469+0000 mgr.y (mgr.26252) 75 : cluster [DBG] pgmap v150: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 55 KiB/s wr, 595 op/s 2024-09-13T14:33:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:31.947469+0000 mgr.y (mgr.26252) 75 : cluster [DBG] pgmap v150: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 55 KiB/s wr, 595 op/s 2024-09-13T14:33:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: audit 2024-09-13T14:33:33.242882+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: audit 2024-09-13T14:33:33.242882+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:42.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:33.948356+0000 mgr.y (mgr.26252) 76 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 48 KiB/s wr, 563 op/s 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:33.948356+0000 mgr.y (mgr.26252) 76 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 48 KiB/s wr, 563 op/s 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:35.739790+0000 mon.a (mon.0) 222 : cluster [INF] mon.a calling monitor election 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:35.739790+0000 mon.a (mon.0) 222 : cluster [INF] mon.a calling monitor election 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:35.740211+0000 mon.b (mon.1) 144 : cluster [INF] mon.b calling monitor election 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:35.740211+0000 mon.b (mon.1) 144 : cluster [INF] mon.b calling monitor election 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:35.949435+0000 mgr.y (mgr.26252) 77 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 24 KiB/s wr, 293 op/s 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:35.949435+0000 mgr.y (mgr.26252) 77 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 24 KiB/s wr, 293 op/s 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:37.949959+0000 mgr.y (mgr.26252) 78 : cluster [DBG] pgmap v153: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 225 op/s 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:37.949959+0000 mgr.y (mgr.26252) 78 : cluster [DBG] pgmap v153: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 225 op/s 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:39.951290+0000 mgr.y (mgr.26252) 79 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 135 KiB/s rd, 16 KiB/s wr, 198 op/s 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:39.951290+0000 mgr.y (mgr.26252) 79 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 135 KiB/s rd, 16 KiB/s wr, 198 op/s 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.759032+0000 mon.a (mon.0) 223 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:33:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.759032+0000 mon.a (mon.0) 223 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:33:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769484+0000 mon.a (mon.0) 224 : cluster [DBG] monmap epoch 3 2024-09-13T14:33:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769484+0000 mon.a (mon.0) 224 : cluster [DBG] monmap epoch 3 2024-09-13T14:33:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:30.828202+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-13T14:33:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:30.828202+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-13T14:33:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:30.969764+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:33:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:30.969764+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:33:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:31.947469+0000 mgr.y (mgr.26252) 75 : cluster [DBG] pgmap v150: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 55 KiB/s wr, 595 op/s 2024-09-13T14:33:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:31.947469+0000 mgr.y (mgr.26252) 75 : cluster [DBG] pgmap v150: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 55 KiB/s wr, 595 op/s 2024-09-13T14:33:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: audit 2024-09-13T14:33:33.242882+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: audit 2024-09-13T14:33:33.242882+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:33.948356+0000 mgr.y (mgr.26252) 76 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 48 KiB/s wr, 563 op/s 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:33.948356+0000 mgr.y (mgr.26252) 76 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 48 KiB/s wr, 563 op/s 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:35.739790+0000 mon.a (mon.0) 222 : cluster [INF] mon.a calling monitor election 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:35.739790+0000 mon.a (mon.0) 222 : cluster [INF] mon.a calling monitor election 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:35.740211+0000 mon.b (mon.1) 144 : cluster [INF] mon.b calling monitor election 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:35.740211+0000 mon.b (mon.1) 144 : cluster [INF] mon.b calling monitor election 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:35.949435+0000 mgr.y (mgr.26252) 77 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 24 KiB/s wr, 293 op/s 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:35.949435+0000 mgr.y (mgr.26252) 77 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 24 KiB/s wr, 293 op/s 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:37.949959+0000 mgr.y (mgr.26252) 78 : cluster [DBG] pgmap v153: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 225 op/s 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:37.949959+0000 mgr.y (mgr.26252) 78 : cluster [DBG] pgmap v153: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 225 op/s 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:39.951290+0000 mgr.y (mgr.26252) 79 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 135 KiB/s rd, 16 KiB/s wr, 198 op/s 2024-09-13T14:33:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:39.951290+0000 mgr.y (mgr.26252) 79 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 135 KiB/s rd, 16 KiB/s wr, 198 op/s 2024-09-13T14:33:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769520+0000 mon.a (mon.0) 225 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769520+0000 mon.a (mon.0) 225 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769580+0000 mon.a (mon.0) 226 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:33:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769580+0000 mon.a (mon.0) 226 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:33:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769621+0000 mon.a (mon.0) 227 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:33:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769621+0000 mon.a (mon.0) 227 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:33:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769655+0000 mon.a (mon.0) 228 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:33:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769655+0000 mon.a (mon.0) 228 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769688+0000 mon.a (mon.0) 229 : cluster [DBG] election_strategy: 1 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769688+0000 mon.a (mon.0) 229 : cluster [DBG] election_strategy: 1 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769725+0000 mon.a (mon.0) 230 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769725+0000 mon.a (mon.0) 230 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769759+0000 mon.a (mon.0) 231 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769759+0000 mon.a (mon.0) 231 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769807+0000 mon.a (mon.0) 232 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.769807+0000 mon.a (mon.0) 232 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.794935+0000 mon.a (mon.0) 233 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.794935+0000 mon.a (mon.0) 233 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.795011+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.795011+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:33:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.795866+0000 mon.a (mon.0) 235 : cluster [DBG] mgrmap e39: y(active, since 112s), standbys: x 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.795866+0000 mon.a (mon.0) 235 : cluster [DBG] mgrmap e39: y(active, since 112s), standbys: x 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.796195+0000 mon.a (mon.0) 236 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.796195+0000 mon.a (mon.0) 236 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.807966+0000 mon.a (mon.0) 237 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.807966+0000 mon.a (mon.0) 237 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.808005+0000 mon.a (mon.0) 238 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.808005+0000 mon.a (mon.0) 238 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.808047+0000 mon.a (mon.0) 239 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] is down (out of quorum) 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: cluster 2024-09-13T14:33:40.808047+0000 mon.a (mon.0) 239 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] is down (out of quorum) 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: audit 2024-09-13T14:33:40.813079+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: audit 2024-09-13T14:33:40.813079+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:42.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: audit 2024-09-13T14:33:40.822557+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:42.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: audit 2024-09-13T14:33:40.822557+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:42.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: audit 2024-09-13T14:33:40.825232+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:42.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[84576]: audit 2024-09-13T14:33:40.825232+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:42.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.759032+0000 mon.a (mon.0) 223 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:33:42.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.759032+0000 mon.a (mon.0) 223 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:33:42.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769484+0000 mon.a (mon.0) 224 : cluster [DBG] monmap epoch 3 2024-09-13T14:33:42.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769484+0000 mon.a (mon.0) 224 : cluster [DBG] monmap epoch 3 2024-09-13T14:33:42.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769520+0000 mon.a (mon.0) 225 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769520+0000 mon.a (mon.0) 225 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769580+0000 mon.a (mon.0) 226 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769580+0000 mon.a (mon.0) 226 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769621+0000 mon.a (mon.0) 227 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769621+0000 mon.a (mon.0) 227 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769655+0000 mon.a (mon.0) 228 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769655+0000 mon.a (mon.0) 228 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769688+0000 mon.a (mon.0) 229 : cluster [DBG] election_strategy: 1 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769688+0000 mon.a (mon.0) 229 : cluster [DBG] election_strategy: 1 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769725+0000 mon.a (mon.0) 230 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769725+0000 mon.a (mon.0) 230 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769759+0000 mon.a (mon.0) 231 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:33:42.078 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769759+0000 mon.a (mon.0) 231 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769807+0000 mon.a (mon.0) 232 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.769807+0000 mon.a (mon.0) 232 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.794935+0000 mon.a (mon.0) 233 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.794935+0000 mon.a (mon.0) 233 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.795011+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.795011+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.795866+0000 mon.a (mon.0) 235 : cluster [DBG] mgrmap e39: y(active, since 112s), standbys: x 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.795866+0000 mon.a (mon.0) 235 : cluster [DBG] mgrmap e39: y(active, since 112s), standbys: x 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.796195+0000 mon.a (mon.0) 236 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.796195+0000 mon.a (mon.0) 236 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.807966+0000 mon.a (mon.0) 237 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.807966+0000 mon.a (mon.0) 237 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-13T14:33:42.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.808005+0000 mon.a (mon.0) 238 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T14:33:42.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.808005+0000 mon.a (mon.0) 238 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T14:33:42.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.808047+0000 mon.a (mon.0) 239 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] is down (out of quorum) 2024-09-13T14:33:42.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: cluster 2024-09-13T14:33:40.808047+0000 mon.a (mon.0) 239 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] is down (out of quorum) 2024-09-13T14:33:42.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: audit 2024-09-13T14:33:40.813079+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:42.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: audit 2024-09-13T14:33:40.813079+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:42.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: audit 2024-09-13T14:33:40.822557+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:42.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: audit 2024-09-13T14:33:40.822557+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:42.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: audit 2024-09-13T14:33:40.825232+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:42.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:41 smithi099 bash[87147]: audit 2024-09-13T14:33:40.825232+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:31.947469+0000 mgr.y (mgr.26252) 75 : cluster [DBG] pgmap v150: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 55 KiB/s wr, 595 op/s 2024-09-13T14:33:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: audit 2024-09-13T14:33:33.242882+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:33.948356+0000 mgr.y (mgr.26252) 76 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 48 KiB/s wr, 563 op/s 2024-09-13T14:33:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:35.739790+0000 mon.a (mon.0) 222 : cluster [INF] mon.a calling monitor election 2024-09-13T14:33:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:35.740211+0000 mon.b (mon.1) 144 : cluster [INF] mon.b calling monitor election 2024-09-13T14:33:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:35.949435+0000 mgr.y (mgr.26252) 77 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 24 KiB/s wr, 293 op/s 2024-09-13T14:33:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:37.949959+0000 mgr.y (mgr.26252) 78 : cluster [DBG] pgmap v153: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 225 op/s 2024-09-13T14:33:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:39.951290+0000 mgr.y (mgr.26252) 79 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 135 KiB/s rd, 16 KiB/s wr, 198 op/s 2024-09-13T14:33:42.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.759032+0000 mon.a (mon.0) 223 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.769484+0000 mon.a (mon.0) 224 : cluster [DBG] monmap epoch 3 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.769520+0000 mon.a (mon.0) 225 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.769580+0000 mon.a (mon.0) 226 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.769621+0000 mon.a (mon.0) 227 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.769655+0000 mon.a (mon.0) 228 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.769688+0000 mon.a (mon.0) 229 : cluster [DBG] election_strategy: 1 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.769725+0000 mon.a (mon.0) 230 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.769759+0000 mon.a (mon.0) 231 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.769807+0000 mon.a (mon.0) 232 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.794935+0000 mon.a (mon.0) 233 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.795011+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T14:33:42.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.795866+0000 mon.a (mon.0) 235 : cluster [DBG] mgrmap e39: y(active, since 112s), standbys: x 2024-09-13T14:33:42.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.796195+0000 mon.a (mon.0) 236 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T14:33:42.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.807966+0000 mon.a (mon.0) 237 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-09-13T14:33:42.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.808005+0000 mon.a (mon.0) 238 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T14:33:42.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: cluster 2024-09-13T14:33:40.808047+0000 mon.a (mon.0) 239 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] is down (out of quorum) 2024-09-13T14:33:42.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: audit 2024-09-13T14:33:40.813079+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:42.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: audit 2024-09-13T14:33:40.822557+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:42.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:41 smithi177 bash[24494]: audit 2024-09-13T14:33:40.825232+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:42.451 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:35.736751+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:35.736751+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.945068+0000 mon.a (mon.0) 244 : cluster [INF] mon.a calling monitor election 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.945068+0000 mon.a (mon.0) 244 : cluster [INF] mon.a calling monitor election 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.947603+0000 mon.b (mon.1) 145 : cluster [INF] mon.b calling monitor election 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.947603+0000 mon.b (mon.1) 145 : cluster [INF] mon.b calling monitor election 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.951894+0000 mgr.y (mgr.26252) 80 : cluster [DBG] pgmap v156: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 123 KiB/s rd, 14 KiB/s wr, 180 op/s 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.951894+0000 mgr.y (mgr.26252) 80 : cluster [DBG] pgmap v156: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 123 KiB/s rd, 14 KiB/s wr, 180 op/s 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.984340+0000 mon.a (mon.0) 245 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.984340+0000 mon.a (mon.0) 245 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:33:42.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991381+0000 mon.a (mon.0) 246 : cluster [DBG] monmap epoch 3 2024-09-13T14:33:42.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991381+0000 mon.a (mon.0) 246 : cluster [DBG] monmap epoch 3 2024-09-13T14:33:42.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991420+0000 mon.a (mon.0) 247 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:42.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991420+0000 mon.a (mon.0) 247 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:42.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991455+0000 mon.a (mon.0) 248 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:33:42.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991455+0000 mon.a (mon.0) 248 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:33:42.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:35.736751+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T14:33:42.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:35.736751+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T14:33:42.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.945068+0000 mon.a (mon.0) 244 : cluster [INF] mon.a calling monitor election 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.945068+0000 mon.a (mon.0) 244 : cluster [INF] mon.a calling monitor election 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.947603+0000 mon.b (mon.1) 145 : cluster [INF] mon.b calling monitor election 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.947603+0000 mon.b (mon.1) 145 : cluster [INF] mon.b calling monitor election 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.951894+0000 mgr.y (mgr.26252) 80 : cluster [DBG] pgmap v156: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 123 KiB/s rd, 14 KiB/s wr, 180 op/s 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.951894+0000 mgr.y (mgr.26252) 80 : cluster [DBG] pgmap v156: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 123 KiB/s rd, 14 KiB/s wr, 180 op/s 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.984340+0000 mon.a (mon.0) 245 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.984340+0000 mon.a (mon.0) 245 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991381+0000 mon.a (mon.0) 246 : cluster [DBG] monmap epoch 3 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991381+0000 mon.a (mon.0) 246 : cluster [DBG] monmap epoch 3 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991420+0000 mon.a (mon.0) 247 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991420+0000 mon.a (mon.0) 247 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991455+0000 mon.a (mon.0) 248 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991455+0000 mon.a (mon.0) 248 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:33:42.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991480+0000 mon.a (mon.0) 249 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991480+0000 mon.a (mon.0) 249 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991507+0000 mon.a (mon.0) 250 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991507+0000 mon.a (mon.0) 250 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991530+0000 mon.a (mon.0) 251 : cluster [DBG] election_strategy: 1 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991530+0000 mon.a (mon.0) 251 : cluster [DBG] election_strategy: 1 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991546+0000 mon.a (mon.0) 252 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991546+0000 mon.a (mon.0) 252 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991558+0000 mon.a (mon.0) 253 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991558+0000 mon.a (mon.0) 253 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991576+0000 mon.a (mon.0) 254 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:33:42.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.991576+0000 mon.a (mon.0) 254 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:33:42.925 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled (10669 ms) 2024-09-13T14:33:42.926 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModePool 2024-09-13T14:33:43.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.992351+0000 mon.a (mon.0) 255 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:33:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.992351+0000 mon.a (mon.0) 255 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:33:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.992415+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-13T14:33:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.992415+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-13T14:33:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.993193+0000 mon.a (mon.0) 257 : cluster [DBG] mgrmap e39: y(active, since 114s), standbys: x 2024-09-13T14:33:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.993193+0000 mon.a (mon.0) 257 : cluster [DBG] mgrmap e39: y(active, since 114s), standbys: x 2024-09-13T14:33:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.993513+0000 mon.a (mon.0) 258 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:33:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.993513+0000 mon.a (mon.0) 258 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:33:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.993560+0000 mon.a (mon.0) 259 : cluster [INF] Cluster is now healthy 2024-09-13T14:33:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:41.993560+0000 mon.a (mon.0) 259 : cluster [INF] Cluster is now healthy 2024-09-13T14:33:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:42.012835+0000 mon.a (mon.0) 260 : cluster [INF] overall HEALTH_OK 2024-09-13T14:33:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[84576]: cluster 2024-09-13T14:33:42.012835+0000 mon.a (mon.0) 260 : cluster [INF] overall HEALTH_OK 2024-09-13T14:33:43.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991480+0000 mon.a (mon.0) 249 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:33:43.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991480+0000 mon.a (mon.0) 249 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:33:43.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991507+0000 mon.a (mon.0) 250 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:33:43.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991507+0000 mon.a (mon.0) 250 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:33:43.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991530+0000 mon.a (mon.0) 251 : cluster [DBG] election_strategy: 1 2024-09-13T14:33:43.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991530+0000 mon.a (mon.0) 251 : cluster [DBG] election_strategy: 1 2024-09-13T14:33:43.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991546+0000 mon.a (mon.0) 252 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:33:43.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991546+0000 mon.a (mon.0) 252 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991558+0000 mon.a (mon.0) 253 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991558+0000 mon.a (mon.0) 253 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991576+0000 mon.a (mon.0) 254 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.991576+0000 mon.a (mon.0) 254 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.992351+0000 mon.a (mon.0) 255 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.992351+0000 mon.a (mon.0) 255 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.992415+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.992415+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.993193+0000 mon.a (mon.0) 257 : cluster [DBG] mgrmap e39: y(active, since 114s), standbys: x 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.993193+0000 mon.a (mon.0) 257 : cluster [DBG] mgrmap e39: y(active, since 114s), standbys: x 2024-09-13T14:33:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.993513+0000 mon.a (mon.0) 258 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:33:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.993513+0000 mon.a (mon.0) 258 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:33:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.993560+0000 mon.a (mon.0) 259 : cluster [INF] Cluster is now healthy 2024-09-13T14:33:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:41.993560+0000 mon.a (mon.0) 259 : cluster [INF] Cluster is now healthy 2024-09-13T14:33:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:42.012835+0000 mon.a (mon.0) 260 : cluster [INF] overall HEALTH_OK 2024-09-13T14:33:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:42 smithi099 bash[87147]: cluster 2024-09-13T14:33:42.012835+0000 mon.a (mon.0) 260 : cluster [INF] overall HEALTH_OK 2024-09-13T14:33:43.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:35.736751+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.945068+0000 mon.a (mon.0) 244 : cluster [INF] mon.a calling monitor election 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.947603+0000 mon.b (mon.1) 145 : cluster [INF] mon.b calling monitor election 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.951894+0000 mgr.y (mgr.26252) 80 : cluster [DBG] pgmap v156: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 123 KiB/s rd, 14 KiB/s wr, 180 op/s 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.984340+0000 mon.a (mon.0) 245 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.991381+0000 mon.a (mon.0) 246 : cluster [DBG] monmap epoch 3 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.991420+0000 mon.a (mon.0) 247 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.991455+0000 mon.a (mon.0) 248 : cluster [DBG] last_changed 2024-09-13T14:12:32.888208+0000 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.991480+0000 mon.a (mon.0) 249 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.991507+0000 mon.a (mon.0) 250 : cluster [DBG] min_mon_release 17 (quincy) 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.991530+0000 mon.a (mon.0) 251 : cluster [DBG] election_strategy: 1 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.991546+0000 mon.a (mon.0) 252 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:33:43.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.991558+0000 mon.a (mon.0) 253 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:33:43.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.991576+0000 mon.a (mon.0) 254 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:33:43.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.992351+0000 mon.a (mon.0) 255 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:33:43.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.992415+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-13T14:33:43.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.993193+0000 mon.a (mon.0) 257 : cluster [DBG] mgrmap e39: y(active, since 114s), standbys: x 2024-09-13T14:33:43.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.993513+0000 mon.a (mon.0) 258 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T14:33:43.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:41.993560+0000 mon.a (mon.0) 259 : cluster [INF] Cluster is now healthy 2024-09-13T14:33:43.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:42 smithi177 bash[24494]: cluster 2024-09-13T14:33:42.012835+0000 mon.a (mon.0) 260 : cluster [INF] overall HEALTH_OK 2024-09-13T14:33:43.127 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModePool (202 ms) 2024-09-13T14:33:43.127 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModePool 2024-09-13T14:33:43.360 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModePool (232 ms) 2024-09-13T14:33:43.360 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModeImage 2024-09-13T14:33:43.578 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModeImage (218 ms) 2024-09-13T14:33:43.578 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode 2024-09-13T14:33:43.939 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode (361 ms) 2024-09-13T14:33:43.939 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode 2024-09-13T14:33:44.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:43 smithi099 bash[84576]: cluster 2024-09-13T14:33:42.805249+0000 mon.a (mon.0) 261 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-13T14:33:44.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:43 smithi099 bash[84576]: cluster 2024-09-13T14:33:42.805249+0000 mon.a (mon.0) 261 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-13T14:33:44.042 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:43 smithi099 bash[87147]: cluster 2024-09-13T14:33:42.805249+0000 mon.a (mon.0) 261 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-13T14:33:44.042 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:43 smithi099 bash[87147]: cluster 2024-09-13T14:33:42.805249+0000 mon.a (mon.0) 261 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-13T14:33:44.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:43 smithi177 bash[24494]: cluster 2024-09-13T14:33:42.805249+0000 mon.a (mon.0) 261 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-13T14:33:44.194 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode (255 ms) 2024-09-13T14:33:44.194 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode 2024-09-13T14:33:44.413 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode (219 ms) 2024-09-13T14:33:44.413 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode 2024-09-13T14:33:44.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:44 smithi177 bash[61439]: ts=2024-09-13T14:33:44.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:44.771 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode (359 ms) 2024-09-13T14:33:44.772 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode 2024-09-13T14:33:45.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:44 smithi099 bash[87147]: cluster 2024-09-13T14:33:43.953238+0000 mgr.y (mgr.26252) 81 : cluster [DBG] pgmap v158: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s rd, 18 KiB/s wr, 184 op/s 2024-09-13T14:33:45.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:44 smithi099 bash[87147]: cluster 2024-09-13T14:33:43.953238+0000 mgr.y (mgr.26252) 81 : cluster [DBG] pgmap v158: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s rd, 18 KiB/s wr, 184 op/s 2024-09-13T14:33:45.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:44 smithi099 bash[84576]: cluster 2024-09-13T14:33:43.953238+0000 mgr.y (mgr.26252) 81 : cluster [DBG] pgmap v158: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s rd, 18 KiB/s wr, 184 op/s 2024-09-13T14:33:45.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:44 smithi099 bash[84576]: cluster 2024-09-13T14:33:43.953238+0000 mgr.y (mgr.26252) 81 : cluster [DBG] pgmap v158: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s rd, 18 KiB/s wr, 184 op/s 2024-09-13T14:33:45.078 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode (306 ms) 2024-09-13T14:33:45.078 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode 2024-09-13T14:33:45.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:44 smithi177 bash[24494]: cluster 2024-09-13T14:33:43.953238+0000 mgr.y (mgr.26252) 81 : cluster [DBG] pgmap v158: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s rd, 18 KiB/s wr, 184 op/s 2024-09-13T14:33:45.372 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode (294 ms) 2024-09-13T14:33:45.372 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageEnabled 2024-09-13T14:33:45.530 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:33:45 smithi099 bash[76324]: debug 2024-09-13T14:33:45.532+0000 7fd55bc0f640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-13T14:33:45.557 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageEnabled (186 ms) 2024-09-13T14:33:45.557 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDisabled 2024-09-13T14:33:45.685 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDisabled (128 ms) 2024-09-13T14:33:45.686 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.RemoveImage_With_ImageWithoutJournal 2024-09-13T14:33:45.788 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.RemoveImage_With_ImageWithoutJournal (102 ms) 2024-09-13T14:33:45.788 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDemoted 2024-09-13T14:33:45.999 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDemoted (212 ms) 2024-09-13T14:33:45.999 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.TrashMoveRestore_PoolMode 2024-09-13T14:33:46.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:45 smithi177 bash[61439]: ts=2024-09-13T14:33:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:46.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:45 smithi177 bash[61439]: ts=2024-09-13T14:33:45.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:46.224 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.TrashMoveRestore_PoolMode (225 ms) 2024-09-13T14:33:46.224 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled 2024-09-13T14:33:46.384 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled (159 ms) 2024-09-13T14:33:46.384 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled 2024-09-13T14:33:46.613 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled (229 ms) 2024-09-13T14:33:46.613 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.MirrorStatusList 2024-09-13T14:33:46.731 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.MirrorStatusList (118 ms) 2024-09-13T14:33:46.731 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.RemoveBootstrapped 2024-09-13T14:33:46.874 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.RemoveBootstrapped (144 ms) 2024-09-13T14:33:46.874 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.AioPromoteDemote 2024-09-13T14:33:46.945 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:33:47.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:33:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:33:47] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-09-13T14:33:47.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[84576]: cluster 2024-09-13T14:33:45.953736+0000 mgr.y (mgr.26252) 82 : cluster [DBG] pgmap v159: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s rd, 18 KiB/s wr, 184 op/s 2024-09-13T14:33:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[84576]: cluster 2024-09-13T14:33:45.953736+0000 mgr.y (mgr.26252) 82 : cluster [DBG] pgmap v159: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s rd, 18 KiB/s wr, 184 op/s 2024-09-13T14:33:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[84576]: cluster 2024-09-13T14:33:45.971046+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-13T14:33:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[84576]: cluster 2024-09-13T14:33:45.971046+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-13T14:33:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[84576]: audit 2024-09-13T14:33:46.220354+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[84576]: audit 2024-09-13T14:33:46.220354+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[84576]: audit 2024-09-13T14:33:46.227097+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[84576]: audit 2024-09-13T14:33:46.227097+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[84576]: audit 2024-09-13T14:33:46.938906+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.99:0/1538209379' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:33:47.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[84576]: audit 2024-09-13T14:33:46.938906+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.99:0/1538209379' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:33:47.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[87147]: cluster 2024-09-13T14:33:45.953736+0000 mgr.y (mgr.26252) 82 : cluster [DBG] pgmap v159: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s rd, 18 KiB/s wr, 184 op/s 2024-09-13T14:33:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[87147]: cluster 2024-09-13T14:33:45.953736+0000 mgr.y (mgr.26252) 82 : cluster [DBG] pgmap v159: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s rd, 18 KiB/s wr, 184 op/s 2024-09-13T14:33:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[87147]: cluster 2024-09-13T14:33:45.971046+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-13T14:33:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[87147]: cluster 2024-09-13T14:33:45.971046+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-13T14:33:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[87147]: audit 2024-09-13T14:33:46.220354+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[87147]: audit 2024-09-13T14:33:46.220354+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[87147]: audit 2024-09-13T14:33:46.227097+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[87147]: audit 2024-09-13T14:33:46.227097+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[87147]: audit 2024-09-13T14:33:46.938906+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.99:0/1538209379' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:33:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:46 smithi099 bash[87147]: audit 2024-09-13T14:33:46.938906+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.99:0/1538209379' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:33:47.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:46 smithi177 bash[61439]: ts=2024-09-13T14:33:46.952Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:47.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:46 smithi177 bash[24494]: cluster 2024-09-13T14:33:45.953736+0000 mgr.y (mgr.26252) 82 : cluster [DBG] pgmap v159: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s rd, 18 KiB/s wr, 184 op/s 2024-09-13T14:33:47.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:46 smithi177 bash[24494]: cluster 2024-09-13T14:33:45.971046+0000 mon.a (mon.0) 262 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-13T14:33:47.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:46 smithi177 bash[24494]: audit 2024-09-13T14:33:46.220354+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:47.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:46 smithi177 bash[24494]: audit 2024-09-13T14:33:46.227097+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:47.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:46 smithi177 bash[24494]: audit 2024-09-13T14:33:46.938906+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.99:0/1538209379' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:33:47.399 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.AioPromoteDemote (524 ms) 2024-09-13T14:33:47.399 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.AioGetInfo 2024-09-13T14:33:47.952 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.AioGetInfo (554 ms) 2024-09-13T14:33:47.953 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.AioGetStatus 2024-09-13T14:33:47.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:33:48.300 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.AioGetStatus (347 ms) 2024-09-13T14:33:48.300 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.SiteName 2024-09-13T14:33:48.309 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.SiteName (10 ms) 2024-09-13T14:33:48.309 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.Bootstrap 2024-09-13T14:33:48.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:47.463908+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:47.463908+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:47.470051+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:47.470051+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.242063+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.242063+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.309895+0000 mon.a (mon.0) 268 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.309895+0000 mon.a (mon.0) 268 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.310254+0000 mon.a (mon.0) 269 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.310254+0000 mon.a (mon.0) 269 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.310504+0000 mon.a (mon.0) 270 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.310504+0000 mon.a (mon.0) 270 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.314644+0000 mon.a (mon.0) 271 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T14:33:48.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.314644+0000 mon.a (mon.0) 271 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.315112+0000 mon.a (mon.0) 272 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[84576]: audit 2024-09-13T14:33:48.315112+0000 mon.a (mon.0) 272 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:47.463908+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:47.463908+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:47.470051+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:47.470051+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.242063+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.242063+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.309895+0000 mon.a (mon.0) 268 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.309895+0000 mon.a (mon.0) 268 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' 2024-09-13T14:33:48.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.310254+0000 mon.a (mon.0) 269 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.310254+0000 mon.a (mon.0) 269 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.310504+0000 mon.a (mon.0) 270 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.310504+0000 mon.a (mon.0) 270 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.314644+0000 mon.a (mon.0) 271 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T14:33:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.314644+0000 mon.a (mon.0) 271 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T14:33:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.315112+0000 mon.a (mon.0) 272 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:48 smithi099 bash[87147]: audit 2024-09-13T14:33:48.315112+0000 mon.a (mon.0) 272 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:48 smithi177 bash[24494]: audit 2024-09-13T14:33:47.463908+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:48.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:48 smithi177 bash[24494]: audit 2024-09-13T14:33:47.470051+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:33:48.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:48 smithi177 bash[24494]: audit 2024-09-13T14:33:48.242063+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:48.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:48 smithi177 bash[24494]: audit 2024-09-13T14:33:48.309895+0000 mon.a (mon.0) 268 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' 2024-09-13T14:33:48.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:48 smithi177 bash[24494]: audit 2024-09-13T14:33:48.310254+0000 mon.a (mon.0) 269 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:48 smithi177 bash[24494]: audit 2024-09-13T14:33:48.310504+0000 mon.a (mon.0) 270 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:48.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:48 smithi177 bash[24494]: audit 2024-09-13T14:33:48.314644+0000 mon.a (mon.0) 271 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T14:33:48.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:48 smithi177 bash[24494]: audit 2024-09-13T14:33:48.315112+0000 mon.a (mon.0) 272 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:49.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:49 smithi099 bash[84576]: cluster 2024-09-13T14:33:47.954083+0000 mgr.y (mgr.26252) 83 : cluster [DBG] pgmap v161: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 168 KiB/s rd, 23 KiB/s wr, 239 op/s 2024-09-13T14:33:49.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:49 smithi099 bash[84576]: cluster 2024-09-13T14:33:47.954083+0000 mgr.y (mgr.26252) 83 : cluster [DBG] pgmap v161: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 168 KiB/s rd, 23 KiB/s wr, 239 op/s 2024-09-13T14:33:49.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:49 smithi099 bash[87147]: cluster 2024-09-13T14:33:47.954083+0000 mgr.y (mgr.26252) 83 : cluster [DBG] pgmap v161: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 168 KiB/s rd, 23 KiB/s wr, 239 op/s 2024-09-13T14:33:49.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:49 smithi099 bash[87147]: cluster 2024-09-13T14:33:47.954083+0000 mgr.y (mgr.26252) 83 : cluster [DBG] pgmap v161: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 168 KiB/s rd, 23 KiB/s wr, 239 op/s 2024-09-13T14:33:49.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:49 smithi177 bash[24494]: cluster 2024-09-13T14:33:47.954083+0000 mgr.y (mgr.26252) 83 : cluster [DBG] pgmap v161: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 168 KiB/s rd, 23 KiB/s wr, 239 op/s 2024-09-13T14:33:49.913 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.Bootstrap (1603 ms) 2024-09-13T14:33:49.913 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.PeerDirection 2024-09-13T14:33:49.925 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.PeerDirection (13 ms) 2024-09-13T14:33:49.925 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.Snapshot 2024-09-13T14:33:50.472 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:50 smithi177 bash[61439]: ts=2024-09-13T14:33:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:50.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.892522+0000 mon.a (mon.0) 273 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-09-13T14:33:50.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.892522+0000 mon.a (mon.0) 273 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-09-13T14:33:50.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.899145+0000 mon.a (mon.0) 274 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' 2024-09-13T14:33:50.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.899145+0000 mon.a (mon.0) 274 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' 2024-09-13T14:33:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.899805+0000 mon.a (mon.0) 275 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-09-13T14:33:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.899805+0000 mon.a (mon.0) 275 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-09-13T14:33:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.902455+0000 mon.a (mon.0) 276 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T14:33:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.902455+0000 mon.a (mon.0) 276 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T14:33:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.906910+0000 mon.a (mon.0) 277 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.906910+0000 mon.a (mon.0) 277 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.924207+0000 mon.a (mon.0) 278 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/1ab0abb2-e6c9-481c-bd50-5f88cc0abf60"} : dispatch 2024-09-13T14:33:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.924207+0000 mon.a (mon.0) 278 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/1ab0abb2-e6c9-481c-bd50-5f88cc0abf60"} : dispatch 2024-09-13T14:33:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.927956+0000 mon.a (mon.0) 279 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/1ab0abb2-e6c9-481c-bd50-5f88cc0abf60"}]': finished 2024-09-13T14:33:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[84576]: audit 2024-09-13T14:33:49.927956+0000 mon.a (mon.0) 279 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/1ab0abb2-e6c9-481c-bd50-5f88cc0abf60"}]': finished 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.892522+0000 mon.a (mon.0) 273 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.892522+0000 mon.a (mon.0) 273 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.899145+0000 mon.a (mon.0) 274 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.899145+0000 mon.a (mon.0) 274 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.899805+0000 mon.a (mon.0) 275 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.899805+0000 mon.a (mon.0) 275 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.902455+0000 mon.a (mon.0) 276 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.902455+0000 mon.a (mon.0) 276 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.906910+0000 mon.a (mon.0) 277 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.906910+0000 mon.a (mon.0) 277 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:50.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.924207+0000 mon.a (mon.0) 278 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/1ab0abb2-e6c9-481c-bd50-5f88cc0abf60"} : dispatch 2024-09-13T14:33:50.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.924207+0000 mon.a (mon.0) 278 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/1ab0abb2-e6c9-481c-bd50-5f88cc0abf60"} : dispatch 2024-09-13T14:33:50.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.927956+0000 mon.a (mon.0) 279 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/1ab0abb2-e6c9-481c-bd50-5f88cc0abf60"}]': finished 2024-09-13T14:33:50.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:50 smithi099 bash[87147]: audit 2024-09-13T14:33:49.927956+0000 mon.a (mon.0) 279 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/1ab0abb2-e6c9-481c-bd50-5f88cc0abf60"}]': finished 2024-09-13T14:33:50.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:50 smithi177 bash[24494]: audit 2024-09-13T14:33:49.892522+0000 mon.a (mon.0) 273 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-09-13T14:33:50.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:50 smithi177 bash[24494]: audit 2024-09-13T14:33:49.899145+0000 mon.a (mon.0) 274 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' 2024-09-13T14:33:50.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:50 smithi177 bash[24494]: audit 2024-09-13T14:33:49.899805+0000 mon.a (mon.0) 275 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-09-13T14:33:50.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:50 smithi177 bash[24494]: audit 2024-09-13T14:33:49.902455+0000 mon.a (mon.0) 276 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T14:33:50.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:50 smithi177 bash[24494]: audit 2024-09-13T14:33:49.906910+0000 mon.a (mon.0) 277 : audit [DBG] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-13T14:33:50.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:50 smithi177 bash[24494]: audit 2024-09-13T14:33:49.924207+0000 mon.a (mon.0) 278 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/1ab0abb2-e6c9-481c-bd50-5f88cc0abf60"} : dispatch 2024-09-13T14:33:50.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:50 smithi177 bash[24494]: audit 2024-09-13T14:33:49.927956+0000 mon.a (mon.0) 279 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/1ab0abb2-e6c9-481c-bd50-5f88cc0abf60"}]': finished 2024-09-13T14:33:51.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:51 smithi099 bash[84576]: cluster 2024-09-13T14:33:49.954944+0000 mgr.y (mgr.26252) 84 : cluster [DBG] pgmap v162: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 948 KiB/s rd, 148 KiB/s wr, 1.30k op/s 2024-09-13T14:33:51.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:51 smithi099 bash[84576]: cluster 2024-09-13T14:33:49.954944+0000 mgr.y (mgr.26252) 84 : cluster [DBG] pgmap v162: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 948 KiB/s rd, 148 KiB/s wr, 1.30k op/s 2024-09-13T14:33:51.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:51 smithi099 bash[84576]: cluster 2024-09-13T14:33:50.942871+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:33:51.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:51 smithi099 bash[84576]: cluster 2024-09-13T14:33:50.942871+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:33:51.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:51 smithi099 bash[87147]: cluster 2024-09-13T14:33:49.954944+0000 mgr.y (mgr.26252) 84 : cluster [DBG] pgmap v162: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 948 KiB/s rd, 148 KiB/s wr, 1.30k op/s 2024-09-13T14:33:51.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:51 smithi099 bash[87147]: cluster 2024-09-13T14:33:49.954944+0000 mgr.y (mgr.26252) 84 : cluster [DBG] pgmap v162: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 948 KiB/s rd, 148 KiB/s wr, 1.30k op/s 2024-09-13T14:33:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:51 smithi099 bash[87147]: cluster 2024-09-13T14:33:50.942871+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:33:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:51 smithi099 bash[87147]: cluster 2024-09-13T14:33:50.942871+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:33:51.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:51 smithi177 bash[24494]: cluster 2024-09-13T14:33:49.954944+0000 mgr.y (mgr.26252) 84 : cluster [DBG] pgmap v162: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 948 KiB/s rd, 148 KiB/s wr, 1.30k op/s 2024-09-13T14:33:51.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:51 smithi177 bash[24494]: cluster 2024-09-13T14:33:50.942871+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-13T14:33:52.549 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:33:53.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:52 smithi099 bash[84576]: cluster 2024-09-13T14:33:51.945304+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:33:53.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:52 smithi099 bash[84576]: cluster 2024-09-13T14:33:51.945304+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:33:53.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:52 smithi099 bash[84576]: cluster 2024-09-13T14:33:51.955777+0000 mgr.y (mgr.26252) 85 : cluster [DBG] pgmap v165: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 174 KiB/s wr, 1.48k op/s 2024-09-13T14:33:53.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:52 smithi099 bash[84576]: cluster 2024-09-13T14:33:51.955777+0000 mgr.y (mgr.26252) 85 : cluster [DBG] pgmap v165: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 174 KiB/s wr, 1.48k op/s 2024-09-13T14:33:53.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:52 smithi099 bash[87147]: cluster 2024-09-13T14:33:51.945304+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:33:53.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:52 smithi099 bash[87147]: cluster 2024-09-13T14:33:51.945304+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:33:53.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:52 smithi099 bash[87147]: cluster 2024-09-13T14:33:51.955777+0000 mgr.y (mgr.26252) 85 : cluster [DBG] pgmap v165: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 174 KiB/s wr, 1.48k op/s 2024-09-13T14:33:53.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:52 smithi099 bash[87147]: cluster 2024-09-13T14:33:51.955777+0000 mgr.y (mgr.26252) 85 : cluster [DBG] pgmap v165: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 174 KiB/s wr, 1.48k op/s 2024-09-13T14:33:53.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:52 smithi177 bash[24494]: cluster 2024-09-13T14:33:51.945304+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-13T14:33:53.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:52 smithi177 bash[24494]: cluster 2024-09-13T14:33:51.955777+0000 mgr.y (mgr.26252) 85 : cluster [DBG] pgmap v165: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 174 KiB/s wr, 1.48k op/s 2024-09-13T14:33:54.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:53 smithi099 bash[84576]: cluster 2024-09-13T14:33:52.956324+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:33:54.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:53 smithi099 bash[84576]: cluster 2024-09-13T14:33:52.956324+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:33:54.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:53 smithi099 bash[87147]: cluster 2024-09-13T14:33:52.956324+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:33:54.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:53 smithi099 bash[87147]: cluster 2024-09-13T14:33:52.956324+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:33:54.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:53 smithi177 bash[24494]: cluster 2024-09-13T14:33:52.956324+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-13T14:33:54.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:54 smithi177 bash[61439]: ts=2024-09-13T14:33:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:55.211 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:54 smithi099 bash[84576]: cluster 2024-09-13T14:33:53.956795+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:33:55.211 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:54 smithi099 bash[84576]: cluster 2024-09-13T14:33:53.956795+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:33:55.212 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:54 smithi099 bash[84576]: cluster 2024-09-13T14:33:53.957651+0000 mgr.y (mgr.26252) 86 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 22 KiB/s wr, 336 op/s 2024-09-13T14:33:55.212 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:54 smithi099 bash[84576]: cluster 2024-09-13T14:33:53.957651+0000 mgr.y (mgr.26252) 86 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 22 KiB/s wr, 336 op/s 2024-09-13T14:33:55.212 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:54 smithi099 bash[87147]: cluster 2024-09-13T14:33:53.956795+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:33:55.212 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:54 smithi099 bash[87147]: cluster 2024-09-13T14:33:53.956795+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:33:55.212 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:54 smithi099 bash[87147]: cluster 2024-09-13T14:33:53.957651+0000 mgr.y (mgr.26252) 86 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 22 KiB/s wr, 336 op/s 2024-09-13T14:33:55.212 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:54 smithi099 bash[87147]: cluster 2024-09-13T14:33:53.957651+0000 mgr.y (mgr.26252) 86 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 22 KiB/s wr, 336 op/s 2024-09-13T14:33:55.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:54 smithi177 bash[24494]: cluster 2024-09-13T14:33:53.956795+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-13T14:33:55.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:54 smithi177 bash[24494]: cluster 2024-09-13T14:33:53.957651+0000 mgr.y (mgr.26252) 86 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 22 KiB/s wr, 336 op/s 2024-09-13T14:33:56.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:55 smithi177 bash[24494]: cluster 2024-09-13T14:33:54.976464+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:33:56.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:55 smithi177 bash[61439]: ts=2024-09-13T14:33:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:56.106 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:55 smithi177 bash[61439]: ts=2024-09-13T14:33:55.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:56.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:55 smithi099 bash[84576]: cluster 2024-09-13T14:33:54.976464+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:33:56.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:55 smithi099 bash[84576]: cluster 2024-09-13T14:33:54.976464+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:33:56.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:55 smithi099 bash[87147]: cluster 2024-09-13T14:33:54.976464+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:33:56.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:55 smithi099 bash[87147]: cluster 2024-09-13T14:33:54.976464+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-13T14:33:57.319 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:33:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:33:57] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.51.0" 2024-09-13T14:33:57.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:56 smithi099 bash[84576]: cluster 2024-09-13T14:33:55.958374+0000 mgr.y (mgr.26252) 87 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 248 KiB/s rd, 22 KiB/s wr, 334 op/s 2024-09-13T14:33:57.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:56 smithi099 bash[84576]: cluster 2024-09-13T14:33:55.958374+0000 mgr.y (mgr.26252) 87 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 248 KiB/s rd, 22 KiB/s wr, 334 op/s 2024-09-13T14:33:57.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:56 smithi099 bash[84576]: cluster 2024-09-13T14:33:55.976299+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:33:57.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:56 smithi099 bash[84576]: cluster 2024-09-13T14:33:55.976299+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:33:57.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:56 smithi099 bash[87147]: cluster 2024-09-13T14:33:55.958374+0000 mgr.y (mgr.26252) 87 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 248 KiB/s rd, 22 KiB/s wr, 334 op/s 2024-09-13T14:33:57.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:56 smithi099 bash[87147]: cluster 2024-09-13T14:33:55.958374+0000 mgr.y (mgr.26252) 87 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 248 KiB/s rd, 22 KiB/s wr, 334 op/s 2024-09-13T14:33:57.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:56 smithi099 bash[87147]: cluster 2024-09-13T14:33:55.976299+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:33:57.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:56 smithi099 bash[87147]: cluster 2024-09-13T14:33:55.976299+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:33:57.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:56 smithi177 bash[24494]: cluster 2024-09-13T14:33:55.958374+0000 mgr.y (mgr.26252) 87 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 248 KiB/s rd, 22 KiB/s wr, 334 op/s 2024-09-13T14:33:57.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:56 smithi177 bash[24494]: cluster 2024-09-13T14:33:55.976299+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-13T14:33:57.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:33:56 smithi177 bash[61439]: ts=2024-09-13T14:33:56.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:33:58.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:58 smithi099 bash[84576]: cluster 2024-09-13T14:33:57.001688+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:33:58.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:58 smithi099 bash[84576]: cluster 2024-09-13T14:33:57.001688+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:33:58.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:58 smithi099 bash[87147]: cluster 2024-09-13T14:33:57.001688+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:33:58.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:58 smithi099 bash[87147]: cluster 2024-09-13T14:33:57.001688+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:33:58.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:58 smithi177 bash[24494]: cluster 2024-09-13T14:33:57.001688+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-13T14:33:59.293 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:59 smithi099 bash[84576]: cluster 2024-09-13T14:33:57.959019+0000 mgr.y (mgr.26252) 88 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 184 KiB/s rd, 20 KiB/s wr, 251 op/s 2024-09-13T14:33:59.293 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:59 smithi099 bash[84576]: cluster 2024-09-13T14:33:57.959019+0000 mgr.y (mgr.26252) 88 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 184 KiB/s rd, 20 KiB/s wr, 251 op/s 2024-09-13T14:33:59.293 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:59 smithi099 bash[84576]: cluster 2024-09-13T14:33:58.014585+0000 mon.a (mon.0) 287 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:33:59.293 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:33:59 smithi099 bash[84576]: cluster 2024-09-13T14:33:58.014585+0000 mon.a (mon.0) 287 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:33:59.293 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:59 smithi099 bash[87147]: cluster 2024-09-13T14:33:57.959019+0000 mgr.y (mgr.26252) 88 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 184 KiB/s rd, 20 KiB/s wr, 251 op/s 2024-09-13T14:33:59.293 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:59 smithi099 bash[87147]: cluster 2024-09-13T14:33:57.959019+0000 mgr.y (mgr.26252) 88 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 184 KiB/s rd, 20 KiB/s wr, 251 op/s 2024-09-13T14:33:59.293 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:59 smithi099 bash[87147]: cluster 2024-09-13T14:33:58.014585+0000 mon.a (mon.0) 287 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:33:59.293 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:33:59 smithi099 bash[87147]: cluster 2024-09-13T14:33:58.014585+0000 mon.a (mon.0) 287 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:33:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:59 smithi177 bash[24494]: cluster 2024-09-13T14:33:57.959019+0000 mgr.y (mgr.26252) 88 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 184 KiB/s rd, 20 KiB/s wr, 251 op/s 2024-09-13T14:33:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:33:59 smithi177 bash[24494]: cluster 2024-09-13T14:33:58.014585+0000 mon.a (mon.0) 287 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-09-13T14:33:59.642 INFO:teuthology.orchestra.run.smithi099.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:alertmanager.a smithi099 *:9093,9094 running (4m) 13s ago 12m 14.7M - 0.27.0 11f11916f8cd 45a2ce75edd3 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:grafana.a smithi177 *:3000 running (3m) 45s ago 11m 75.0M - 9.4.7 954c08fa6188 443ce4187a0d 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi099.yyrtld smithi099 running (9m) 13s ago 9m 14.2M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 7bce2f3fd8d5 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi177.outtkv smithi177 running (9m) 45s ago 9m 17.4M - 17.2.7-1627-gbbdadfa3 a994b00deb3a b976db2e3d1b 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.x smithi177 *:8443,9283,8765 running (2m) 45s ago 21m 438M - 19.3.0-4697-gd58f7306 fec07016a2b0 24a222989f2c 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.y smithi099 *:8443,9283,8765 running (3m) 13s ago 23m 497M - 19.3.0-4697-gd58f7306 fec07016a2b0 8ebc4af8542d 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:mon.a smithi099 running (2m) 13s ago 23m 46.3M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 7bae834be590 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:mon.b smithi177 running (21m) 45s ago 21m 57.7M 2048M 17.2.7-1627-gbbdadfa3 a994b00deb3a 660521332db7 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:mon.c smithi099 running (26s) 13s ago 21m 19.5M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 8e92ca9c3d4b 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.a smithi099 *:9100 running (4m) 13s ago 12m 8611k - 1.7.0 72c9c2088986 adc238b4fabf 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.b smithi177 *:9100 running (3m) 45s ago 12m 7936k - 1.7.0 72c9c2088986 0e333c475514 2024-09-13T14:33:59.643 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0 smithi099 running (20m) 13s ago 20m 129M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 3334f7b9d02c 2024-09-13T14:33:59.644 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1 smithi099 running (19m) 13s ago 19m 116M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 324b38b19317 2024-09-13T14:33:59.644 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2 smithi099 running (18m) 13s ago 18m 132M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a e6fb2db24160 2024-09-13T14:33:59.644 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3 smithi099 running (17m) 13s ago 17m 119M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 1fad69868d22 2024-09-13T14:33:59.644 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4 smithi177 running (16m) 45s ago 16m 138M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a f54f8f87d384 2024-09-13T14:33:59.644 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5 smithi177 running (15m) 45s ago 15m 138M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a f285f4fee6d3 2024-09-13T14:33:59.644 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6 smithi177 running (14m) 45s ago 14m 155M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 2318776c78d4 2024-09-13T14:33:59.644 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7 smithi177 running (14m) 45s ago 14m 153M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a e2d33a0b16ca 2024-09-13T14:33:59.644 INFO:teuthology.orchestra.run.smithi099.stdout:prometheus.a smithi177 *:9095 running (2m) 45s ago 12m 45.3M - 2.51.0 1d3b7f56885b d360e28d5f8b 2024-09-13T14:34:00.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:00 smithi099 bash[84576]: cluster 2024-09-13T14:33:59.022937+0000 mon.a (mon.0) 288 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:34:00.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:00 smithi099 bash[84576]: cluster 2024-09-13T14:33:59.022937+0000 mon.a (mon.0) 288 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:34:00.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:00 smithi099 bash[84576]: audit 2024-09-13T14:33:59.631743+0000 mgr.y (mgr.26252) 89 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:00.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:00 smithi099 bash[84576]: audit 2024-09-13T14:33:59.631743+0000 mgr.y (mgr.26252) 89 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:00.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:00 smithi099 bash[87147]: cluster 2024-09-13T14:33:59.022937+0000 mon.a (mon.0) 288 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:34:00.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:00 smithi099 bash[87147]: cluster 2024-09-13T14:33:59.022937+0000 mon.a (mon.0) 288 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:34:00.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:00 smithi099 bash[87147]: audit 2024-09-13T14:33:59.631743+0000 mgr.y (mgr.26252) 89 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:00.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:00 smithi099 bash[87147]: audit 2024-09-13T14:33:59.631743+0000 mgr.y (mgr.26252) 89 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:00.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:00 smithi177 bash[61439]: ts=2024-09-13T14:34:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:00.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:00 smithi177 bash[24494]: cluster 2024-09-13T14:33:59.022937+0000 mon.a (mon.0) 288 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-09-13T14:34:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:00 smithi177 bash[24494]: audit 2024-09-13T14:33:59.631743+0000 mgr.y (mgr.26252) 89 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:00.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:34:01.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:01 smithi099 bash[84576]: cluster 2024-09-13T14:33:59.959504+0000 mgr.y (mgr.26252) 90 : cluster [DBG] pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-09-13T14:34:01.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:01 smithi099 bash[84576]: cluster 2024-09-13T14:33:59.959504+0000 mgr.y (mgr.26252) 90 : cluster [DBG] pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-09-13T14:34:01.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:01 smithi099 bash[84576]: cluster 2024-09-13T14:34:00.029922+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:34:01.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:01 smithi099 bash[84576]: cluster 2024-09-13T14:34:00.029922+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:34:01.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:01 smithi099 bash[87147]: cluster 2024-09-13T14:33:59.959504+0000 mgr.y (mgr.26252) 90 : cluster [DBG] pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-09-13T14:34:01.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:01 smithi099 bash[87147]: cluster 2024-09-13T14:33:59.959504+0000 mgr.y (mgr.26252) 90 : cluster [DBG] pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-09-13T14:34:01.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:01 smithi099 bash[87147]: cluster 2024-09-13T14:34:00.029922+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:34:01.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:01 smithi099 bash[87147]: cluster 2024-09-13T14:34:00.029922+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:34:01.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:01 smithi177 bash[24494]: cluster 2024-09-13T14:33:59.959504+0000 mgr.y (mgr.26252) 90 : cluster [DBG] pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-09-13T14:34:01.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:01 smithi177 bash[24494]: cluster 2024-09-13T14:34:00.029922+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-09-13T14:34:02.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: cluster 2024-09-13T14:34:01.044267+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:34:02.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: cluster 2024-09-13T14:34:01.044267+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.341483+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.341483+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.351056+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.351056+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.352823+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.352823+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.354401+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.354401+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.366544+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.366544+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.517183+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:02.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.517183+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.520483+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.520483+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.523252+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.523252+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.524842+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: audit 2024-09-13T14:34:01.524842+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: cephadm 2024-09-13T14:34:01.525953+0000 mgr.y (mgr.26252) 91 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[84576]: cephadm 2024-09-13T14:34:01.525953+0000 mgr.y (mgr.26252) 91 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: cluster 2024-09-13T14:34:01.044267+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: cluster 2024-09-13T14:34:01.044267+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:34:02.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.341483+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.341483+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.351056+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.351056+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.352823+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.352823+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.354401+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.354401+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.366544+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.366544+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.517183+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.517183+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.520483+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.520483+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:02.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.523252+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:34:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.523252+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:34:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.524842+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:34:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: audit 2024-09-13T14:34:01.524842+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:34:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: cephadm 2024-09-13T14:34:01.525953+0000 mgr.y (mgr.26252) 91 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-13T14:34:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:02 smithi099 bash[87147]: cephadm 2024-09-13T14:34:01.525953+0000 mgr.y (mgr.26252) 91 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-13T14:34:02.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: cluster 2024-09-13T14:34:01.044267+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-09-13T14:34:02.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: audit 2024-09-13T14:34:01.341483+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: audit 2024-09-13T14:34:01.351056+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: audit 2024-09-13T14:34:01.352823+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: audit 2024-09-13T14:34:01.354401+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: audit 2024-09-13T14:34:01.366544+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:02.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: audit 2024-09-13T14:34:01.517183+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:02.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: audit 2024-09-13T14:34:01.520483+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:02.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: audit 2024-09-13T14:34:01.523252+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:34:02.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: audit 2024-09-13T14:34:01.524842+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:34:02.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:02 smithi177 bash[24494]: cephadm 2024-09-13T14:34:01.525953+0000 mgr.y (mgr.26252) 91 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-13T14:34:03.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:03 smithi099 bash[84576]: cluster 2024-09-13T14:34:01.960159+0000 mgr.y (mgr.26252) 92 : cluster [DBG] pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-09-13T14:34:03.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:03 smithi099 bash[84576]: cluster 2024-09-13T14:34:01.960159+0000 mgr.y (mgr.26252) 92 : cluster [DBG] pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-09-13T14:34:03.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:03 smithi099 bash[84576]: cluster 2024-09-13T14:34:02.062156+0000 mon.a (mon.0) 300 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:34:03.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:03 smithi099 bash[84576]: cluster 2024-09-13T14:34:02.062156+0000 mon.a (mon.0) 300 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:34:03.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:03 smithi099 bash[87147]: cluster 2024-09-13T14:34:01.960159+0000 mgr.y (mgr.26252) 92 : cluster [DBG] pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-09-13T14:34:03.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:03 smithi099 bash[87147]: cluster 2024-09-13T14:34:01.960159+0000 mgr.y (mgr.26252) 92 : cluster [DBG] pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-09-13T14:34:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:03 smithi099 bash[87147]: cluster 2024-09-13T14:34:02.062156+0000 mon.a (mon.0) 300 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:34:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:03 smithi099 bash[87147]: cluster 2024-09-13T14:34:02.062156+0000 mon.a (mon.0) 300 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:34:03.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:03 smithi177 bash[24494]: cluster 2024-09-13T14:34:01.960159+0000 mgr.y (mgr.26252) 92 : cluster [DBG] pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 7.0 KiB/s wr, 85 op/s 2024-09-13T14:34:03.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:03 smithi177 bash[24494]: cluster 2024-09-13T14:34:02.062156+0000 mon.a (mon.0) 300 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-09-13T14:34:04.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:04 smithi177 bash[24494]: cluster 2024-09-13T14:34:03.067519+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:34:04.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:04 smithi177 bash[24494]: audit 2024-09-13T14:34:03.243191+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:04.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:04 smithi099 bash[84576]: cluster 2024-09-13T14:34:03.067519+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:34:04.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:04 smithi099 bash[84576]: cluster 2024-09-13T14:34:03.067519+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:34:04.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:04 smithi099 bash[84576]: audit 2024-09-13T14:34:03.243191+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:04.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:04 smithi099 bash[84576]: audit 2024-09-13T14:34:03.243191+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:04.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:04 smithi099 bash[87147]: cluster 2024-09-13T14:34:03.067519+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:34:04.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:04 smithi099 bash[87147]: cluster 2024-09-13T14:34:03.067519+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-09-13T14:34:04.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:04 smithi099 bash[87147]: audit 2024-09-13T14:34:03.243191+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:04.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:04 smithi099 bash[87147]: audit 2024-09-13T14:34:03.243191+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:04.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:04 smithi177 bash[61439]: ts=2024-09-13T14:34:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: cluster 2024-09-13T14:34:03.961450+0000 mgr.y (mgr.26252) 93 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-13T14:34:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: cluster 2024-09-13T14:34:03.961450+0000 mgr.y (mgr.26252) 93 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-13T14:34:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: cluster 2024-09-13T14:34:04.077658+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:34:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: cluster 2024-09-13T14:34:04.077658+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:34:05.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: audit 2024-09-13T14:34:04.774126+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: audit 2024-09-13T14:34:04.774126+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: audit 2024-09-13T14:34:04.778136+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: audit 2024-09-13T14:34:04.778136+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: audit 2024-09-13T14:34:04.779657+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: audit 2024-09-13T14:34:04.779657+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: audit 2024-09-13T14:34:04.781668+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:05.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[84576]: audit 2024-09-13T14:34:04.781668+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:05.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: cluster 2024-09-13T14:34:03.961450+0000 mgr.y (mgr.26252) 93 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-13T14:34:05.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: cluster 2024-09-13T14:34:03.961450+0000 mgr.y (mgr.26252) 93 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-13T14:34:05.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: cluster 2024-09-13T14:34:04.077658+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:34:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: cluster 2024-09-13T14:34:04.077658+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:34:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: audit 2024-09-13T14:34:04.774126+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: audit 2024-09-13T14:34:04.774126+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: audit 2024-09-13T14:34:04.778136+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: audit 2024-09-13T14:34:04.778136+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: audit 2024-09-13T14:34:04.779657+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: audit 2024-09-13T14:34:04.779657+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: audit 2024-09-13T14:34:04.781668+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:05.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:05 smithi099 bash[87147]: audit 2024-09-13T14:34:04.781668+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:05.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:05 smithi177 bash[24494]: cluster 2024-09-13T14:34:03.961450+0000 mgr.y (mgr.26252) 93 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-13T14:34:05.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:05 smithi177 bash[24494]: cluster 2024-09-13T14:34:04.077658+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-09-13T14:34:05.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:05 smithi177 bash[24494]: audit 2024-09-13T14:34:04.774126+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:05.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:05 smithi177 bash[24494]: audit 2024-09-13T14:34:04.778136+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:05.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:05 smithi177 bash[24494]: audit 2024-09-13T14:34:04.779657+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:05.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:05 smithi177 bash[24494]: audit 2024-09-13T14:34:04.781668+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:05.456 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:06.087 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:05 smithi177 bash[61439]: ts=2024-09-13T14:34:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:06.087 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:05 smithi177 bash[61439]: ts=2024-09-13T14:34:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:06.161 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.Snapshot (16235 ms) 2024-09-13T14:34:06.162 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.SnapshotRemoveOnDisable 2024-09-13T14:34:06.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:06 smithi177 bash[24494]: cephadm 2024-09-13T14:34:04.766943+0000 mgr.y (mgr.26252) 94 : cephadm [INF] Upgrade: Updating mon.b 2024-09-13T14:34:06.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:06 smithi177 bash[24494]: cephadm 2024-09-13T14:34:04.783630+0000 mgr.y (mgr.26252) 95 : cephadm [INF] Deploying daemon mon.b on smithi177 2024-09-13T14:34:06.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:06 smithi177 bash[24494]: cluster 2024-09-13T14:34:05.087369+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:34:06.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:06 smithi177 bash[24494]: cluster 2024-09-13T14:34:05.980467+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:34:06.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[84576]: cephadm 2024-09-13T14:34:04.766943+0000 mgr.y (mgr.26252) 94 : cephadm [INF] Upgrade: Updating mon.b 2024-09-13T14:34:06.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[84576]: cephadm 2024-09-13T14:34:04.766943+0000 mgr.y (mgr.26252) 94 : cephadm [INF] Upgrade: Updating mon.b 2024-09-13T14:34:06.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[84576]: cephadm 2024-09-13T14:34:04.783630+0000 mgr.y (mgr.26252) 95 : cephadm [INF] Deploying daemon mon.b on smithi177 2024-09-13T14:34:06.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[84576]: cephadm 2024-09-13T14:34:04.783630+0000 mgr.y (mgr.26252) 95 : cephadm [INF] Deploying daemon mon.b on smithi177 2024-09-13T14:34:06.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[84576]: cluster 2024-09-13T14:34:05.087369+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:34:06.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[84576]: cluster 2024-09-13T14:34:05.087369+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:34:06.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[84576]: cluster 2024-09-13T14:34:05.980467+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:34:06.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[84576]: cluster 2024-09-13T14:34:05.980467+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:34:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[87147]: cephadm 2024-09-13T14:34:04.766943+0000 mgr.y (mgr.26252) 94 : cephadm [INF] Upgrade: Updating mon.b 2024-09-13T14:34:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[87147]: cephadm 2024-09-13T14:34:04.766943+0000 mgr.y (mgr.26252) 94 : cephadm [INF] Upgrade: Updating mon.b 2024-09-13T14:34:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[87147]: cephadm 2024-09-13T14:34:04.783630+0000 mgr.y (mgr.26252) 95 : cephadm [INF] Deploying daemon mon.b on smithi177 2024-09-13T14:34:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[87147]: cephadm 2024-09-13T14:34:04.783630+0000 mgr.y (mgr.26252) 95 : cephadm [INF] Deploying daemon mon.b on smithi177 2024-09-13T14:34:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[87147]: cluster 2024-09-13T14:34:05.087369+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:34:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[87147]: cluster 2024-09-13T14:34:05.087369+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-09-13T14:34:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[87147]: cluster 2024-09-13T14:34:05.980467+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:34:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:06 smithi099 bash[87147]: cluster 2024-09-13T14:34:05.980467+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-09-13T14:34:07.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:34:07] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-09-13T14:34:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[84576]: cluster 2024-09-13T14:34:05.962126+0000 mgr.y (mgr.26252) 96 : cluster [DBG] pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-13T14:34:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[84576]: cluster 2024-09-13T14:34:05.962126+0000 mgr.y (mgr.26252) 96 : cluster [DBG] pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-13T14:34:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[84576]: audit 2024-09-13T14:34:06.152779+0000 mon.a (mon.0) 310 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/dc1a2f3e-d249-42b8-9fbc-2f1ac2421e84"} : dispatch 2024-09-13T14:34:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[84576]: audit 2024-09-13T14:34:06.152779+0000 mon.a (mon.0) 310 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/dc1a2f3e-d249-42b8-9fbc-2f1ac2421e84"} : dispatch 2024-09-13T14:34:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[84576]: audit 2024-09-13T14:34:06.156999+0000 mon.a (mon.0) 311 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/dc1a2f3e-d249-42b8-9fbc-2f1ac2421e84"}]': finished 2024-09-13T14:34:07.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[84576]: audit 2024-09-13T14:34:06.156999+0000 mon.a (mon.0) 311 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/dc1a2f3e-d249-42b8-9fbc-2f1ac2421e84"}]': finished 2024-09-13T14:34:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[87147]: cluster 2024-09-13T14:34:05.962126+0000 mgr.y (mgr.26252) 96 : cluster [DBG] pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-13T14:34:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[87147]: cluster 2024-09-13T14:34:05.962126+0000 mgr.y (mgr.26252) 96 : cluster [DBG] pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-13T14:34:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[87147]: audit 2024-09-13T14:34:06.152779+0000 mon.a (mon.0) 310 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/dc1a2f3e-d249-42b8-9fbc-2f1ac2421e84"} : dispatch 2024-09-13T14:34:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[87147]: audit 2024-09-13T14:34:06.152779+0000 mon.a (mon.0) 310 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/dc1a2f3e-d249-42b8-9fbc-2f1ac2421e84"} : dispatch 2024-09-13T14:34:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[87147]: audit 2024-09-13T14:34:06.156999+0000 mon.a (mon.0) 311 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/dc1a2f3e-d249-42b8-9fbc-2f1ac2421e84"}]': finished 2024-09-13T14:34:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:07 smithi099 bash[87147]: audit 2024-09-13T14:34:06.156999+0000 mon.a (mon.0) 311 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/dc1a2f3e-d249-42b8-9fbc-2f1ac2421e84"}]': finished 2024-09-13T14:34:07.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:06 smithi177 bash[61439]: ts=2024-09-13T14:34:06.953Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:07 smithi177 bash[24494]: cluster 2024-09-13T14:34:05.962126+0000 mgr.y (mgr.26252) 96 : cluster [DBG] pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-13T14:34:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:07 smithi177 bash[24494]: audit 2024-09-13T14:34:06.152779+0000 mon.a (mon.0) 310 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/dc1a2f3e-d249-42b8-9fbc-2f1ac2421e84"} : dispatch 2024-09-13T14:34:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:07 smithi177 bash[24494]: audit 2024-09-13T14:34:06.156999+0000 mon.a (mon.0) 311 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/dc1a2f3e-d249-42b8-9fbc-2f1ac2421e84"}]': finished 2024-09-13T14:34:07.886 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:34:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:07.886 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:34:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:07.886 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:07.887 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:34:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:07.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:07.887 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:34:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:07.888 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:34:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:07.888 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:34:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:07.888 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:34:07 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:08.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:07 smithi177 systemd[1]: Stopping Ceph mon.b for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:34:08.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:08 smithi177 bash[24494]: debug 2024-09-13T14:34:08.007+0000 7fae10b61640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T14:34:08.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:08 smithi177 bash[24494]: debug 2024-09-13T14:34:08.007+0000 7fae10b61640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T14:34:09.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:08 smithi177 bash[68478]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-b 2024-09-13T14:34:09.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:08 smithi177 bash[68538]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-b 2024-09-13T14:34:09.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T14:34:09.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: Stopped Ceph mon.b for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:34:09.605 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:09.606 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:09.606 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:09.606 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:09.606 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:09.607 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:09.607 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:09.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:09.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: Started Ceph mon.b for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:34:09.608 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:34:09 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:34:10.487 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[61439]: ts=2024-09-13T14:34:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:10.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.551+0000 7f67f1a3ac80 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:34:10.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.551+0000 7f67f1a3ac80 0 ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:34:10.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.551+0000 7f67f1a3ac80 0 pidfile_write: ignore empty --pid-file 2024-09-13T14:34:10.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 0 load: jerasure load: lrc 2024-09-13T14:34:10.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: RocksDB version: 7.9.2 2024-09-13T14:34:10.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Git sha 0 2024-09-13T14:34:10.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: DB SUMMARY 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: DB Session ID: PO82NEQQM0ZLU016WHMT 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: CURRENT file: CURRENT 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 2058 Bytes 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000042.sst 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000040.log size: 4040890 ; 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.error_if_exists: 0 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.create_if_missing: 0 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:34:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:34:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.env: 0x561118435160 2024-09-13T14:34:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:34:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.info_log: 0x56111a589880 2024-09-13T14:34:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:34:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.statistics: (nil) 2024-09-13T14:34:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.use_fsync: 0 2024-09-13T14:34:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:34:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.db_log_dir: 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.wal_dir: 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.write_buffer_manager: 0x56111a58c960 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:34:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.unordered_write: 0 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:34:10.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.row_cache: None 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.wal_filter: None 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.two_write_queues: 0 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.wal_compression: 0 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.atomic_flush: 0 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T14:34:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:34:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_open_files: -1 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Compression algorithms supported: 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: kZSTD supported: 0 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: kXpressCompression supported: 0 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T14:34:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:34:10.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T14:34:10.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: kZlibCompression supported: 1 2024-09-13T14:34:10.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:34:10.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T14:34:10.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:34:10.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:34:10.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-13T14:34:10.866 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:34:10.866 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:34:10.866 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.merge_operator: 2024-09-13T14:34:10.866 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_filter: None 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56111a5894e0) 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: cache_index_and_filter_blocks: 1 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: pin_top_level_index_and_filter: 1 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: index_type: 0 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: data_block_index_type: 0 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: index_shortening: 1 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:34:10.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: checksum: 4 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: no_block_cache: 0 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: block_cache: 0x56111a57e850 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: block_cache_name: BinnedLRUCache 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: block_cache_options: 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: capacity : 536870912 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: num_shard_bits : 4 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: strict_capacity_limit : 0 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: high_pri_pool_ratio: 0.000 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: block_cache_compressed: (nil) 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: persistent_cache: (nil) 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: block_size: 4096 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: block_size_deviation: 10 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: block_restart_interval: 16 2024-09-13T14:34:10.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: index_block_restart_interval: 1 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: metadata_block_size: 4096 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: partition_filters: 0 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: use_delta_encoding: 1 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: filter_policy: bloomfilter 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: whole_key_filtering: 1 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: verify_compression: 0 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: read_amp_bytes_per_bit: 0 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: format_version: 5 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: enable_index_compression: 1 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: block_align: 0 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: max_auto_readahead_size: 262144 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: prepopulate_block_cache: 0 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: initial_auto_readahead_size: 8192 2024-09-13T14:34:10.869 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: num_file_reads_for_auto_readahead: 2 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compression: NoCompression 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.num_levels: 7 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:34:10.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:34:10.871 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:34:10.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:34:10.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:34:10.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.bloom_locality: 0 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.ttl: 2592000 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.enable_blob_files: false 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.min_blob_size: 0 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:34:10.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67f1a3ac80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.555+0000 7f67ee810640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 42.sst 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.559+0000 7f67f1a3ac80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.559+0000 7f67f1a3ac80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 44, last_sequence is 24509, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.559+0000 7f67f1a3ac80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.559+0000 7f67f1a3ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 73fd8e4e-c0fb-4e2d-ba08-bb478df7d478 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.559+0000 7f67f1a3ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726238050561071, "job": 1, "event": "recovery_started", "wal_files": [40]} 2024-09-13T14:34:10.876 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.559+0000 7f67f1a3ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2024-09-13T14:34:10.877 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.571+0000 7f67f1a3ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726238050573320, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 2593033, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 24514, "largest_seqno": 26577, "table_properties": {"data_size": 2583351, "index_size": 6652, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1989, "raw_key_size": 17495, "raw_average_key_size": 22, "raw_value_size": 2565502, "raw_average_value_size": 3323, "num_data_blocks": 290, "num_entries": 772, "num_filter_entries": 772, "num_deletions": 6, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726238050, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "73fd8e4e-c0fb-4e2d-ba08-bb478df7d478", "db_session_id": "PO82NEQQM0ZLU016WHMT", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:34:10.934 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.571+0000 7f67f1a3ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726238050573479, "job": 1, "event": "recovery_finished"} 2024-09-13T14:34:10.934 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.571+0000 7f67f1a3ac80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2024-09-13T14:34:10.934 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.571+0000 7f67f1a3ac80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:34:10.934 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.575+0000 7f67f1a3ac80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:34:10.934 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.575+0000 7f67f1a3ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56111a5ace00 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.575+0000 7f67f1a3ac80 4 rocksdb: DB pointer 0x56111a6b0000 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.575+0000 7f67e7802640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.575+0000 7f67e7802640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: ** DB Stats ** 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:34:10.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:34:10.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: ** Compaction Stats [default] ** 2024-09-13T14:34:10.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:34:10.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:34:10.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: L0 1/0 2.47 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 506.8 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T14:34:10.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: L6 1/0 12.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:34:10.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Sum 2/0 15.42 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 506.8 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T14:34:10.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 506.8 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T14:34:10.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: ** Compaction Stats [default] ** 2024-09-13T14:34:10.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:34:10.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 506.8 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Flush(GB): cumulative 0.002, interval 0.002 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Cumulative compaction: 0.00 GB write, 126.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Interval compaction: 0.00 GB write, 126.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:34:10.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:34:10.938 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Block cache BinnedLRUCache@0x56111a57e850#7 capacity: 512.00 MB usage: 8.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2024-09-13T14:34:10.938 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,6.58 KB,0.00125468%) Misc(1,0.00 KB,0%) 2024-09-13T14:34:10.938 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:34:10.938 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.575+0000 7f67f1a3ac80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] at bind addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:34:10.938 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.579+0000 7f67f1a3ac80 1 mon.b@-1(???) e3 preinit fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:34:10.938 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.579+0000 7f67f1a3ac80 0 mon.b@-1(???).mds e9 new map 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.579+0000 7f67f1a3ac80 0 mon.b@-1(???).mds e9 print_map 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: e9 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: legacy client fscid: 1 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Filesystem 'foo' (1) 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: fs_name foo 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: epoch 9 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: created 2024-09-13T14:24:17.237575+0000 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: modified 2024-09-13T14:24:29.188501+0000 2024-09-13T14:34:10.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: tableserver 0 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: root 0 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: session_timeout 60 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: session_autoclose 300 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: max_file_size 1099511627776 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: max_xattr_size 65536 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: required_client_features {} 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: last_failure 0 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: last_failure_osd_epoch 65 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: max_mds 1 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: in 0 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: up {0=24403} 2024-09-13T14:34:10.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: failed 2024-09-13T14:34:10.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: damaged 2024-09-13T14:34:10.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: stopped 2024-09-13T14:34:10.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: data_pools [4] 2024-09-13T14:34:10.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: metadata_pool 3 2024-09-13T14:34:10.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: inline_data disabled 2024-09-13T14:34:10.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: balancer 2024-09-13T14:34:10.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: bal_rank_mask -1 2024-09-13T14:34:10.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: standby_count_wanted 1 2024-09-13T14:34:10.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: qdb_cluster leader: 0 members: 2024-09-13T14:34:10.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: [mds.foo.smithi177.outtkv{0:24403} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.177:6832/2731985651,v1:172.21.15.177:6833/2731985651] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:34:10.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: 2024-09-13T14:34:10.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: 2024-09-13T14:34:10.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: Standby daemons: 2024-09-13T14:34:10.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: 2024-09-13T14:34:10.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: [mds.foo.smithi099.yyrtld{-1:24488} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.99:6834/2117449908,v1:172.21.15.99:6835/2117449908] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:34:10.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.579+0000 7f67f1a3ac80 0 mon.b@-1(???).osd e459 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:34:10.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.579+0000 7f67f1a3ac80 0 mon.b@-1(???).osd e459 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:34:10.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.579+0000 7f67f1a3ac80 0 mon.b@-1(???).osd e459 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:34:10.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.579+0000 7f67f1a3ac80 0 mon.b@-1(???).osd e459 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:34:10.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:10 smithi177 bash[68594]: debug 2024-09-13T14:34:10.579+0000 7f67f1a3ac80 1 mon.b@-1(???).paxosservice(auth 1..25) refresh upgraded, format 0 -> 3 2024-09-13T14:34:11.279 INFO:teuthology.orchestra.run.smithi099.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:12.067 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:34:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.239437+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.239437+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.239930+0000 mon.c (mon.2) 3 : cluster [INF] mon.c calling monitor election 2024-09-13T14:34:12.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.239930+0000 mon.c (mon.2) 3 : cluster [INF] mon.c calling monitor election 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.240044+0000 mon.a (mon.0) 329 : cluster [INF] mon.a calling monitor election 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.240044+0000 mon.a (mon.0) 329 : cluster [INF] mon.a calling monitor election 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.242248+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.242248+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.242513+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.242513+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.242813+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.242813+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.261297+0000 mon.a (mon.0) 332 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.261297+0000 mon.a (mon.0) 332 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266753+0000 mon.a (mon.0) 333 : cluster [DBG] monmap epoch 4 2024-09-13T14:34:12.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266753+0000 mon.a (mon.0) 333 : cluster [DBG] monmap epoch 4 2024-09-13T14:34:12.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266774+0000 mon.a (mon.0) 334 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:34:12.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266774+0000 mon.a (mon.0) 334 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:34:12.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266784+0000 mon.a (mon.0) 335 : cluster [DBG] last_changed 2024-09-13T14:34:11.210453+0000 2024-09-13T14:34:12.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266784+0000 mon.a (mon.0) 335 : cluster [DBG] last_changed 2024-09-13T14:34:11.210453+0000 2024-09-13T14:34:12.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266795+0000 mon.a (mon.0) 336 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:34:12.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266795+0000 mon.a (mon.0) 336 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:34:12.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266812+0000 mon.a (mon.0) 337 : cluster [DBG] min_mon_release 19 (squid) 2024-09-13T14:34:12.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266812+0000 mon.a (mon.0) 337 : cluster [DBG] min_mon_release 19 (squid) 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.239437+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.239437+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.239930+0000 mon.c (mon.2) 3 : cluster [INF] mon.c calling monitor election 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.239930+0000 mon.c (mon.2) 3 : cluster [INF] mon.c calling monitor election 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.240044+0000 mon.a (mon.0) 329 : cluster [INF] mon.a calling monitor election 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.240044+0000 mon.a (mon.0) 329 : cluster [INF] mon.a calling monitor election 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.242248+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.242248+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.242513+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.242513+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.242813+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T14:34:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.242813+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.261297+0000 mon.a (mon.0) 332 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.261297+0000 mon.a (mon.0) 332 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266753+0000 mon.a (mon.0) 333 : cluster [DBG] monmap epoch 4 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266753+0000 mon.a (mon.0) 333 : cluster [DBG] monmap epoch 4 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266774+0000 mon.a (mon.0) 334 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266774+0000 mon.a (mon.0) 334 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266784+0000 mon.a (mon.0) 335 : cluster [DBG] last_changed 2024-09-13T14:34:11.210453+0000 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266784+0000 mon.a (mon.0) 335 : cluster [DBG] last_changed 2024-09-13T14:34:11.210453+0000 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266795+0000 mon.a (mon.0) 336 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266795+0000 mon.a (mon.0) 336 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266812+0000 mon.a (mon.0) 337 : cluster [DBG] min_mon_release 19 (squid) 2024-09-13T14:34:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266812+0000 mon.a (mon.0) 337 : cluster [DBG] min_mon_release 19 (squid) 2024-09-13T14:34:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266823+0000 mon.a (mon.0) 338 : cluster [DBG] election_strategy: 1 2024-09-13T14:34:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266823+0000 mon.a (mon.0) 338 : cluster [DBG] election_strategy: 1 2024-09-13T14:34:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266835+0000 mon.a (mon.0) 339 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:34:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266835+0000 mon.a (mon.0) 339 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:34:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266846+0000 mon.a (mon.0) 340 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:34:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266846+0000 mon.a (mon.0) 340 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:34:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266858+0000 mon.a (mon.0) 341 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:34:12.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.266858+0000 mon.a (mon.0) 341 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:34:12.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.267494+0000 mon.a (mon.0) 342 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:34:12.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.267494+0000 mon.a (mon.0) 342 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:34:12.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.267528+0000 mon.a (mon.0) 343 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:34:12.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.267528+0000 mon.a (mon.0) 343 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:34:12.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.267921+0000 mon.a (mon.0) 344 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T14:34:12.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.267921+0000 mon.a (mon.0) 344 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T14:34:12.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.268085+0000 mon.a (mon.0) 345 : cluster [INF] overall HEALTH_OK 2024-09-13T14:34:12.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.268085+0000 mon.a (mon.0) 345 : cluster [INF] overall HEALTH_OK 2024-09-13T14:34:12.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.274320+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.274320+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.282967+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.282967+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.291889+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.291889+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.293642+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:12.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[87147]: audit 2024-09-13T14:34:11.293642+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:12.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266823+0000 mon.a (mon.0) 338 : cluster [DBG] election_strategy: 1 2024-09-13T14:34:12.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266823+0000 mon.a (mon.0) 338 : cluster [DBG] election_strategy: 1 2024-09-13T14:34:12.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266835+0000 mon.a (mon.0) 339 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:34:12.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266835+0000 mon.a (mon.0) 339 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:34:12.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266846+0000 mon.a (mon.0) 340 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:34:12.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266846+0000 mon.a (mon.0) 340 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:34:12.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266858+0000 mon.a (mon.0) 341 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:34:12.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.266858+0000 mon.a (mon.0) 341 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:34:12.328 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.267494+0000 mon.a (mon.0) 342 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.267494+0000 mon.a (mon.0) 342 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.267528+0000 mon.a (mon.0) 343 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.267528+0000 mon.a (mon.0) 343 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.267921+0000 mon.a (mon.0) 344 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.267921+0000 mon.a (mon.0) 344 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.268085+0000 mon.a (mon.0) 345 : cluster [INF] overall HEALTH_OK 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.268085+0000 mon.a (mon.0) 345 : cluster [INF] overall HEALTH_OK 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.274320+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.274320+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.282967+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.282967+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.291889+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.291889+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.293642+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:12.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:12 smithi099 bash[84576]: audit 2024-09-13T14:34:11.293642+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.239437+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.239437+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:34:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.239930+0000 mon.c (mon.2) 3 : cluster [INF] mon.c calling monitor election 2024-09-13T14:34:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.239930+0000 mon.c (mon.2) 3 : cluster [INF] mon.c calling monitor election 2024-09-13T14:34:12.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.240044+0000 mon.a (mon.0) 329 : cluster [INF] mon.a calling monitor election 2024-09-13T14:34:12.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.240044+0000 mon.a (mon.0) 329 : cluster [INF] mon.a calling monitor election 2024-09-13T14:34:12.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.242248+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:12.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.242248+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:34:12.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.242513+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:12.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.242513+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:34:12.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.242813+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T14:34:12.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.242813+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T14:34:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.261297+0000 mon.a (mon.0) 332 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.261297+0000 mon.a (mon.0) 332 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:34:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266753+0000 mon.a (mon.0) 333 : cluster [DBG] monmap epoch 4 2024-09-13T14:34:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266753+0000 mon.a (mon.0) 333 : cluster [DBG] monmap epoch 4 2024-09-13T14:34:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266774+0000 mon.a (mon.0) 334 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:34:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266774+0000 mon.a (mon.0) 334 : cluster [DBG] fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:34:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266784+0000 mon.a (mon.0) 335 : cluster [DBG] last_changed 2024-09-13T14:34:11.210453+0000 2024-09-13T14:34:12.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266784+0000 mon.a (mon.0) 335 : cluster [DBG] last_changed 2024-09-13T14:34:11.210453+0000 2024-09-13T14:34:12.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266795+0000 mon.a (mon.0) 336 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:34:12.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266795+0000 mon.a (mon.0) 336 : cluster [DBG] created 2024-09-13T14:10:16.718964+0000 2024-09-13T14:34:12.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266812+0000 mon.a (mon.0) 337 : cluster [DBG] min_mon_release 19 (squid) 2024-09-13T14:34:12.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266812+0000 mon.a (mon.0) 337 : cluster [DBG] min_mon_release 19 (squid) 2024-09-13T14:34:12.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266823+0000 mon.a (mon.0) 338 : cluster [DBG] election_strategy: 1 2024-09-13T14:34:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266823+0000 mon.a (mon.0) 338 : cluster [DBG] election_strategy: 1 2024-09-13T14:34:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266835+0000 mon.a (mon.0) 339 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:34:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266835+0000 mon.a (mon.0) 339 : cluster [DBG] 0: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.a 2024-09-13T14:34:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266846+0000 mon.a (mon.0) 340 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:34:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266846+0000 mon.a (mon.0) 340 : cluster [DBG] 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.b 2024-09-13T14:34:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266858+0000 mon.a (mon.0) 341 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:34:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.266858+0000 mon.a (mon.0) 341 : cluster [DBG] 2: [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0] mon.c 2024-09-13T14:34:12.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.267494+0000 mon.a (mon.0) 342 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:34:12.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.267494+0000 mon.a (mon.0) 342 : cluster [DBG] fsmap foo:1 {0=foo.smithi177.outtkv=up:active} 1 up:standby 2024-09-13T14:34:12.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.267528+0000 mon.a (mon.0) 343 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:34:12.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.267528+0000 mon.a (mon.0) 343 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-09-13T14:34:12.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.267921+0000 mon.a (mon.0) 344 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T14:34:12.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.267921+0000 mon.a (mon.0) 344 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T14:34:12.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.268085+0000 mon.a (mon.0) 345 : cluster [INF] overall HEALTH_OK 2024-09-13T14:34:12.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.268085+0000 mon.a (mon.0) 345 : cluster [INF] overall HEALTH_OK 2024-09-13T14:34:12.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.274320+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.274320+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.282967+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.282967+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.291889+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.291889+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:12.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.293642+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:12.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:12 smithi177 bash[68594]: audit 2024-09-13T14:34:11.293642+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:13.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:13 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.964858+0000 mgr.y (mgr.26252) 101 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 4.1 KiB/s wr, 53 op/s 2024-09-13T14:34:13.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:13 smithi099 bash[84576]: cluster 2024-09-13T14:34:11.964858+0000 mgr.y (mgr.26252) 101 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 4.1 KiB/s wr, 53 op/s 2024-09-13T14:34:13.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:13 smithi099 bash[84576]: cluster 2024-09-13T14:34:12.255925+0000 mon.a (mon.0) 350 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:34:13.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:13 smithi099 bash[84576]: cluster 2024-09-13T14:34:12.255925+0000 mon.a (mon.0) 350 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:34:13.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:13 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.964858+0000 mgr.y (mgr.26252) 101 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 4.1 KiB/s wr, 53 op/s 2024-09-13T14:34:13.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:13 smithi099 bash[87147]: cluster 2024-09-13T14:34:11.964858+0000 mgr.y (mgr.26252) 101 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 4.1 KiB/s wr, 53 op/s 2024-09-13T14:34:13.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:13 smithi099 bash[87147]: cluster 2024-09-13T14:34:12.255925+0000 mon.a (mon.0) 350 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:34:13.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:13 smithi099 bash[87147]: cluster 2024-09-13T14:34:12.255925+0000 mon.a (mon.0) 350 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:34:13.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:13 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.964858+0000 mgr.y (mgr.26252) 101 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 4.1 KiB/s wr, 53 op/s 2024-09-13T14:34:13.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:13 smithi177 bash[68594]: cluster 2024-09-13T14:34:11.964858+0000 mgr.y (mgr.26252) 101 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 4.1 KiB/s wr, 53 op/s 2024-09-13T14:34:13.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:13 smithi177 bash[68594]: cluster 2024-09-13T14:34:12.255925+0000 mon.a (mon.0) 350 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:34:13.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:13 smithi177 bash[68594]: cluster 2024-09-13T14:34:12.255925+0000 mon.a (mon.0) 350 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-09-13T14:34:14.478 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.SnapshotRemoveOnDisable (8317 ms) 2024-09-13T14:34:14.479 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.SnapshotUnlinkPeer 2024-09-13T14:34:14.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:14 smithi099 bash[84576]: cluster 2024-09-13T14:34:13.299246+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:34:14.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:14 smithi099 bash[84576]: cluster 2024-09-13T14:34:13.299246+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:34:14.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:14 smithi099 bash[87147]: cluster 2024-09-13T14:34:13.299246+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:34:14.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:14 smithi099 bash[87147]: cluster 2024-09-13T14:34:13.299246+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:34:14.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:14 smithi177 bash[68594]: cluster 2024-09-13T14:34:13.299246+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:34:14.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:14 smithi177 bash[68594]: cluster 2024-09-13T14:34:13.299246+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-09-13T14:34:14.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:14 smithi177 bash[61439]: ts=2024-09-13T14:34:14.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:15.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[84576]: cluster 2024-09-13T14:34:13.966166+0000 mgr.y (mgr.26252) 102 : cluster [DBG] pgmap v193: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 72 op/s 2024-09-13T14:34:15.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[84576]: cluster 2024-09-13T14:34:13.966166+0000 mgr.y (mgr.26252) 102 : cluster [DBG] pgmap v193: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 72 op/s 2024-09-13T14:34:15.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[84576]: cluster 2024-09-13T14:34:14.313491+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:34:15.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[84576]: cluster 2024-09-13T14:34:14.313491+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:34:15.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[84576]: audit 2024-09-13T14:34:14.466816+0000 mon.a (mon.0) 353 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"} : dispatch 2024-09-13T14:34:15.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[84576]: audit 2024-09-13T14:34:14.466816+0000 mon.a (mon.0) 353 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"} : dispatch 2024-09-13T14:34:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[84576]: audit 2024-09-13T14:34:14.472953+0000 mon.a (mon.0) 354 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"}]': finished 2024-09-13T14:34:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[84576]: audit 2024-09-13T14:34:14.472953+0000 mon.a (mon.0) 354 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"}]': finished 2024-09-13T14:34:15.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[87147]: cluster 2024-09-13T14:34:13.966166+0000 mgr.y (mgr.26252) 102 : cluster [DBG] pgmap v193: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 72 op/s 2024-09-13T14:34:15.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[87147]: cluster 2024-09-13T14:34:13.966166+0000 mgr.y (mgr.26252) 102 : cluster [DBG] pgmap v193: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 72 op/s 2024-09-13T14:34:15.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[87147]: cluster 2024-09-13T14:34:14.313491+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:34:15.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[87147]: cluster 2024-09-13T14:34:14.313491+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:34:15.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[87147]: audit 2024-09-13T14:34:14.466816+0000 mon.a (mon.0) 353 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"} : dispatch 2024-09-13T14:34:15.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[87147]: audit 2024-09-13T14:34:14.466816+0000 mon.a (mon.0) 353 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"} : dispatch 2024-09-13T14:34:15.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[87147]: audit 2024-09-13T14:34:14.472953+0000 mon.a (mon.0) 354 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"}]': finished 2024-09-13T14:34:15.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[87147]: audit 2024-09-13T14:34:14.472953+0000 mon.a (mon.0) 354 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"}]': finished 2024-09-13T14:34:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:15 smithi177 bash[68594]: cluster 2024-09-13T14:34:13.966166+0000 mgr.y (mgr.26252) 102 : cluster [DBG] pgmap v193: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 72 op/s 2024-09-13T14:34:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:15 smithi177 bash[68594]: cluster 2024-09-13T14:34:13.966166+0000 mgr.y (mgr.26252) 102 : cluster [DBG] pgmap v193: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 72 op/s 2024-09-13T14:34:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:15 smithi177 bash[68594]: cluster 2024-09-13T14:34:14.313491+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:34:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:15 smithi177 bash[68594]: cluster 2024-09-13T14:34:14.313491+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-09-13T14:34:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:15 smithi177 bash[68594]: audit 2024-09-13T14:34:14.466816+0000 mon.a (mon.0) 353 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"} : dispatch 2024-09-13T14:34:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:15 smithi177 bash[68594]: audit 2024-09-13T14:34:14.466816+0000 mon.a (mon.0) 353 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"} : dispatch 2024-09-13T14:34:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:15 smithi177 bash[68594]: audit 2024-09-13T14:34:14.472953+0000 mon.a (mon.0) 354 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"}]': finished 2024-09-13T14:34:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:15 smithi177 bash[68594]: audit 2024-09-13T14:34:14.472953+0000 mon.a (mon.0) 354 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ecb8ef1a-ba37-432b-9e76-600587b4df66"}]': finished 2024-09-13T14:34:16.070 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:34:15 smithi099 bash[76324]: debug 2024-09-13T14:34:15.583+0000 7fd55bc0f640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-13T14:34:16.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:15 smithi177 bash[61439]: ts=2024-09-13T14:34:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:16.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:15 smithi177 bash[61439]: ts=2024-09-13T14:34:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:16.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:16 smithi099 bash[84576]: cluster 2024-09-13T14:34:15.479540+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:34:16.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:16 smithi099 bash[84576]: cluster 2024-09-13T14:34:15.479540+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:34:16.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:16 smithi099 bash[84576]: cluster 2024-09-13T14:34:15.986656+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-13T14:34:16.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:16 smithi099 bash[84576]: cluster 2024-09-13T14:34:15.986656+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-13T14:34:16.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:16 smithi099 bash[87147]: cluster 2024-09-13T14:34:15.479540+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:34:16.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:16 smithi099 bash[87147]: cluster 2024-09-13T14:34:15.479540+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:34:16.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:16 smithi099 bash[87147]: cluster 2024-09-13T14:34:15.986656+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-13T14:34:16.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:16 smithi099 bash[87147]: cluster 2024-09-13T14:34:15.986656+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-13T14:34:16.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:16 smithi177 bash[68594]: cluster 2024-09-13T14:34:15.479540+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:34:16.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:16 smithi177 bash[68594]: cluster 2024-09-13T14:34:15.479540+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-13T14:34:16.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:16 smithi177 bash[68594]: cluster 2024-09-13T14:34:15.986656+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-13T14:34:16.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:16 smithi177 bash[68594]: cluster 2024-09-13T14:34:15.986656+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-13T14:34:16.867 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:17.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:34:17] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-09-13T14:34:17.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:16 smithi177 bash[61439]: ts=2024-09-13T14:34:16.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:17.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:17 smithi177 bash[68594]: cluster 2024-09-13T14:34:15.966853+0000 mgr.y (mgr.26252) 103 : cluster [DBG] pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 3.5 KiB/s wr, 31 op/s 2024-09-13T14:34:17.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:17 smithi177 bash[68594]: cluster 2024-09-13T14:34:15.966853+0000 mgr.y (mgr.26252) 103 : cluster [DBG] pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 3.5 KiB/s wr, 31 op/s 2024-09-13T14:34:17.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:17 smithi177 bash[68594]: cluster 2024-09-13T14:34:17.005602+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-13T14:34:17.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:17 smithi177 bash[68594]: cluster 2024-09-13T14:34:17.005602+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-13T14:34:17.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:17 smithi177 bash[68594]: audit 2024-09-13T14:34:17.405661+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:17.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:17 smithi177 bash[68594]: audit 2024-09-13T14:34:17.405661+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:17.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:17 smithi177 bash[68594]: audit 2024-09-13T14:34:17.415715+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:17.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:17 smithi177 bash[68594]: audit 2024-09-13T14:34:17.415715+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:18.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[84576]: cluster 2024-09-13T14:34:15.966853+0000 mgr.y (mgr.26252) 103 : cluster [DBG] pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 3.5 KiB/s wr, 31 op/s 2024-09-13T14:34:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[84576]: cluster 2024-09-13T14:34:15.966853+0000 mgr.y (mgr.26252) 103 : cluster [DBG] pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 3.5 KiB/s wr, 31 op/s 2024-09-13T14:34:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[84576]: cluster 2024-09-13T14:34:17.005602+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-13T14:34:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[84576]: cluster 2024-09-13T14:34:17.005602+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-13T14:34:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[84576]: audit 2024-09-13T14:34:17.405661+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[84576]: audit 2024-09-13T14:34:17.405661+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[84576]: audit 2024-09-13T14:34:17.415715+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[84576]: audit 2024-09-13T14:34:17.415715+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:18.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[87147]: cluster 2024-09-13T14:34:15.966853+0000 mgr.y (mgr.26252) 103 : cluster [DBG] pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 3.5 KiB/s wr, 31 op/s 2024-09-13T14:34:18.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[87147]: cluster 2024-09-13T14:34:15.966853+0000 mgr.y (mgr.26252) 103 : cluster [DBG] pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 3.5 KiB/s wr, 31 op/s 2024-09-13T14:34:18.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[87147]: cluster 2024-09-13T14:34:17.005602+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-13T14:34:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[87147]: cluster 2024-09-13T14:34:17.005602+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-13T14:34:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[87147]: audit 2024-09-13T14:34:17.405661+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[87147]: audit 2024-09-13T14:34:17.405661+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[87147]: audit 2024-09-13T14:34:17.415715+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:17 smithi099 bash[87147]: audit 2024-09-13T14:34:17.415715+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:18 smithi099 bash[84576]: cluster 2024-09-13T14:34:17.967482+0000 mgr.y (mgr.26252) 104 : cluster [DBG] pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:19.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:18 smithi099 bash[84576]: cluster 2024-09-13T14:34:17.967482+0000 mgr.y (mgr.26252) 104 : cluster [DBG] pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:18 smithi099 bash[84576]: cluster 2024-09-13T14:34:18.003638+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:18 smithi099 bash[84576]: cluster 2024-09-13T14:34:18.003638+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:18 smithi099 bash[84576]: audit 2024-09-13T14:34:18.243984+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:18 smithi099 bash[84576]: audit 2024-09-13T14:34:18.243984+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:18 smithi099 bash[84576]: audit 2024-09-13T14:34:18.621390+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:18 smithi099 bash[84576]: audit 2024-09-13T14:34:18.621390+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:18 smithi099 bash[84576]: audit 2024-09-13T14:34:18.626489+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:18 smithi099 bash[84576]: audit 2024-09-13T14:34:18.626489+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:19 smithi099 bash[87147]: cluster 2024-09-13T14:34:17.967482+0000 mgr.y (mgr.26252) 104 : cluster [DBG] pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:19 smithi099 bash[87147]: cluster 2024-09-13T14:34:17.967482+0000 mgr.y (mgr.26252) 104 : cluster [DBG] pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:19.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:19 smithi099 bash[87147]: cluster 2024-09-13T14:34:18.003638+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-13T14:34:19.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:19 smithi099 bash[87147]: cluster 2024-09-13T14:34:18.003638+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-13T14:34:19.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:19 smithi099 bash[87147]: audit 2024-09-13T14:34:18.243984+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:19.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:19 smithi099 bash[87147]: audit 2024-09-13T14:34:18.243984+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:19.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:19 smithi099 bash[87147]: audit 2024-09-13T14:34:18.621390+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:19 smithi099 bash[87147]: audit 2024-09-13T14:34:18.621390+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:19 smithi099 bash[87147]: audit 2024-09-13T14:34:18.626489+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:19 smithi099 bash[87147]: audit 2024-09-13T14:34:18.626489+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:19 smithi177 bash[68594]: cluster 2024-09-13T14:34:17.967482+0000 mgr.y (mgr.26252) 104 : cluster [DBG] pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:19.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:19 smithi177 bash[68594]: cluster 2024-09-13T14:34:17.967482+0000 mgr.y (mgr.26252) 104 : cluster [DBG] pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:19.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:19 smithi177 bash[68594]: cluster 2024-09-13T14:34:18.003638+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-13T14:34:19.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:19 smithi177 bash[68594]: cluster 2024-09-13T14:34:18.003638+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-13T14:34:19.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:19 smithi177 bash[68594]: audit 2024-09-13T14:34:18.243984+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:19.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:19 smithi177 bash[68594]: audit 2024-09-13T14:34:18.243984+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:19.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:19 smithi177 bash[68594]: audit 2024-09-13T14:34:18.621390+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:19 smithi177 bash[68594]: audit 2024-09-13T14:34:18.621390+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:19 smithi177 bash[68594]: audit 2024-09-13T14:34:18.626489+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:19.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:19 smithi177 bash[68594]: audit 2024-09-13T14:34:18.626489+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:20.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[84576]: cluster 2024-09-13T14:34:19.014578+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-13T14:34:20.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[84576]: cluster 2024-09-13T14:34:19.014578+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-13T14:34:20.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[84576]: audit 2024-09-13T14:34:19.054463+0000 mon.a (mon.0) 365 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"} : dispatch 2024-09-13T14:34:20.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[84576]: audit 2024-09-13T14:34:19.054463+0000 mon.a (mon.0) 365 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"} : dispatch 2024-09-13T14:34:20.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[84576]: audit 2024-09-13T14:34:19.062847+0000 mon.a (mon.0) 366 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"}]': finished 2024-09-13T14:34:20.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[84576]: audit 2024-09-13T14:34:19.062847+0000 mon.a (mon.0) 366 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"}]': finished 2024-09-13T14:34:20.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[87147]: cluster 2024-09-13T14:34:19.014578+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-13T14:34:20.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[87147]: cluster 2024-09-13T14:34:19.014578+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-13T14:34:20.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[87147]: audit 2024-09-13T14:34:19.054463+0000 mon.a (mon.0) 365 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"} : dispatch 2024-09-13T14:34:20.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[87147]: audit 2024-09-13T14:34:19.054463+0000 mon.a (mon.0) 365 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"} : dispatch 2024-09-13T14:34:20.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[87147]: audit 2024-09-13T14:34:19.062847+0000 mon.a (mon.0) 366 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"}]': finished 2024-09-13T14:34:20.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:20 smithi099 bash[87147]: audit 2024-09-13T14:34:19.062847+0000 mon.a (mon.0) 366 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"}]': finished 2024-09-13T14:34:20.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:20 smithi177 bash[61439]: ts=2024-09-13T14:34:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:20.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:20 smithi177 bash[68594]: cluster 2024-09-13T14:34:19.014578+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-13T14:34:20.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:20 smithi177 bash[68594]: cluster 2024-09-13T14:34:19.014578+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-13T14:34:20.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:20 smithi177 bash[68594]: audit 2024-09-13T14:34:19.054463+0000 mon.a (mon.0) 365 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"} : dispatch 2024-09-13T14:34:20.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:20 smithi177 bash[68594]: audit 2024-09-13T14:34:19.054463+0000 mon.a (mon.0) 365 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"} : dispatch 2024-09-13T14:34:20.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:20 smithi177 bash[68594]: audit 2024-09-13T14:34:19.062847+0000 mon.a (mon.0) 366 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"}]': finished 2024-09-13T14:34:20.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:20 smithi177 bash[68594]: audit 2024-09-13T14:34:19.062847+0000 mon.a (mon.0) 366 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/65c2f7b8-7969-4c59-b4c3-2442fa8365c2"}]': finished 2024-09-13T14:34:21.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[84576]: cluster 2024-09-13T14:34:19.968299+0000 mgr.y (mgr.26252) 105 : cluster [DBG] pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:21.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[84576]: cluster 2024-09-13T14:34:19.968299+0000 mgr.y (mgr.26252) 105 : cluster [DBG] pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:21.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[84576]: cluster 2024-09-13T14:34:20.074548+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-13T14:34:21.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[84576]: cluster 2024-09-13T14:34:20.074548+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-13T14:34:21.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[84576]: cluster 2024-09-13T14:34:20.992051+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-13T14:34:21.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[84576]: cluster 2024-09-13T14:34:20.992051+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-13T14:34:21.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[87147]: cluster 2024-09-13T14:34:19.968299+0000 mgr.y (mgr.26252) 105 : cluster [DBG] pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:21.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[87147]: cluster 2024-09-13T14:34:19.968299+0000 mgr.y (mgr.26252) 105 : cluster [DBG] pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:21.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[87147]: cluster 2024-09-13T14:34:20.074548+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-13T14:34:21.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[87147]: cluster 2024-09-13T14:34:20.074548+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-13T14:34:21.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[87147]: cluster 2024-09-13T14:34:20.992051+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-13T14:34:21.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:21 smithi099 bash[87147]: cluster 2024-09-13T14:34:20.992051+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-13T14:34:21.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:21 smithi177 bash[68594]: cluster 2024-09-13T14:34:19.968299+0000 mgr.y (mgr.26252) 105 : cluster [DBG] pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:21.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:21 smithi177 bash[68594]: cluster 2024-09-13T14:34:19.968299+0000 mgr.y (mgr.26252) 105 : cluster [DBG] pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:21.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:21 smithi177 bash[68594]: cluster 2024-09-13T14:34:20.074548+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-13T14:34:21.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:21 smithi177 bash[68594]: cluster 2024-09-13T14:34:20.074548+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-13T14:34:21.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:21 smithi177 bash[68594]: cluster 2024-09-13T14:34:20.992051+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-13T14:34:21.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:21 smithi177 bash[68594]: cluster 2024-09-13T14:34:20.992051+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-13T14:34:23.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:22 smithi099 bash[84576]: cluster 2024-09-13T14:34:21.969024+0000 mgr.y (mgr.26252) 106 : cluster [DBG] pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:23.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:22 smithi099 bash[84576]: cluster 2024-09-13T14:34:21.969024+0000 mgr.y (mgr.26252) 106 : cluster [DBG] pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:23.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:22 smithi099 bash[84576]: cluster 2024-09-13T14:34:21.995511+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-13T14:34:23.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:22 smithi099 bash[84576]: cluster 2024-09-13T14:34:21.995511+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-13T14:34:23.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:22 smithi099 bash[87147]: cluster 2024-09-13T14:34:21.969024+0000 mgr.y (mgr.26252) 106 : cluster [DBG] pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:23.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:22 smithi099 bash[87147]: cluster 2024-09-13T14:34:21.969024+0000 mgr.y (mgr.26252) 106 : cluster [DBG] pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:23.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:22 smithi099 bash[87147]: cluster 2024-09-13T14:34:21.995511+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-13T14:34:23.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:22 smithi099 bash[87147]: cluster 2024-09-13T14:34:21.995511+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-13T14:34:23.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:22 smithi177 bash[68594]: cluster 2024-09-13T14:34:21.969024+0000 mgr.y (mgr.26252) 106 : cluster [DBG] pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:23.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:22 smithi177 bash[68594]: cluster 2024-09-13T14:34:21.969024+0000 mgr.y (mgr.26252) 106 : cluster [DBG] pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-13T14:34:23.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:22 smithi177 bash[68594]: cluster 2024-09-13T14:34:21.995511+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-13T14:34:23.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:22 smithi177 bash[68594]: cluster 2024-09-13T14:34:21.995511+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-13T14:34:24.251 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.SnapshotUnlinkPeer (9773 ms) 2024-09-13T14:34:24.251 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.SnapshotImageState 2024-09-13T14:34:24.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:24 smithi177 bash[68594]: cluster 2024-09-13T14:34:23.080185+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-13T14:34:24.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:24 smithi177 bash[68594]: cluster 2024-09-13T14:34:23.080185+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-13T14:34:24.356 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:24 smithi177 bash[61439]: ts=2024-09-13T14:34:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:24.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:24 smithi099 bash[84576]: cluster 2024-09-13T14:34:23.080185+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-13T14:34:24.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:24 smithi099 bash[84576]: cluster 2024-09-13T14:34:23.080185+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-13T14:34:24.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:24 smithi099 bash[87147]: cluster 2024-09-13T14:34:23.080185+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-13T14:34:24.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:24 smithi099 bash[87147]: cluster 2024-09-13T14:34:23.080185+0000 mon.a (mon.0) 370 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-13T14:34:25.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: cluster 2024-09-13T14:34:23.970332+0000 mgr.y (mgr.26252) 107 : cluster [DBG] pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: cluster 2024-09-13T14:34:23.970332+0000 mgr.y (mgr.26252) 107 : cluster [DBG] pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: cluster 2024-09-13T14:34:24.093649+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: cluster 2024-09-13T14:34:24.093649+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: audit 2024-09-13T14:34:24.224965+0000 mon.a (mon.0) 372 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"} : dispatch 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: audit 2024-09-13T14:34:24.224965+0000 mon.a (mon.0) 372 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"} : dispatch 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: audit 2024-09-13T14:34:24.231174+0000 mon.a (mon.0) 373 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"}]': finished 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: audit 2024-09-13T14:34:24.231174+0000 mon.a (mon.0) 373 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"}]': finished 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: audit 2024-09-13T14:34:24.236861+0000 mon.a (mon.0) 374 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"} : dispatch 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: audit 2024-09-13T14:34:24.236861+0000 mon.a (mon.0) 374 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"} : dispatch 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: audit 2024-09-13T14:34:24.245045+0000 mon.a (mon.0) 375 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"}]': finished 2024-09-13T14:34:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[68594]: audit 2024-09-13T14:34:24.245045+0000 mon.a (mon.0) 375 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"}]': finished 2024-09-13T14:34:25.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: cluster 2024-09-13T14:34:23.970332+0000 mgr.y (mgr.26252) 107 : cluster [DBG] pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: cluster 2024-09-13T14:34:23.970332+0000 mgr.y (mgr.26252) 107 : cluster [DBG] pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: cluster 2024-09-13T14:34:24.093649+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: cluster 2024-09-13T14:34:24.093649+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: audit 2024-09-13T14:34:24.224965+0000 mon.a (mon.0) 372 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"} : dispatch 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: audit 2024-09-13T14:34:24.224965+0000 mon.a (mon.0) 372 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"} : dispatch 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: audit 2024-09-13T14:34:24.231174+0000 mon.a (mon.0) 373 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"}]': finished 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: audit 2024-09-13T14:34:24.231174+0000 mon.a (mon.0) 373 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"}]': finished 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: audit 2024-09-13T14:34:24.236861+0000 mon.a (mon.0) 374 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"} : dispatch 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: audit 2024-09-13T14:34:24.236861+0000 mon.a (mon.0) 374 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"} : dispatch 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: audit 2024-09-13T14:34:24.245045+0000 mon.a (mon.0) 375 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"}]': finished 2024-09-13T14:34:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[84576]: audit 2024-09-13T14:34:24.245045+0000 mon.a (mon.0) 375 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"}]': finished 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: cluster 2024-09-13T14:34:23.970332+0000 mgr.y (mgr.26252) 107 : cluster [DBG] pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: cluster 2024-09-13T14:34:23.970332+0000 mgr.y (mgr.26252) 107 : cluster [DBG] pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: cluster 2024-09-13T14:34:24.093649+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: cluster 2024-09-13T14:34:24.093649+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: audit 2024-09-13T14:34:24.224965+0000 mon.a (mon.0) 372 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"} : dispatch 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: audit 2024-09-13T14:34:24.224965+0000 mon.a (mon.0) 372 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"} : dispatch 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: audit 2024-09-13T14:34:24.231174+0000 mon.a (mon.0) 373 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"}]': finished 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: audit 2024-09-13T14:34:24.231174+0000 mon.a (mon.0) 373 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e76303b4-ca80-4a9e-901b-19eb327bf854"}]': finished 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: audit 2024-09-13T14:34:24.236861+0000 mon.a (mon.0) 374 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"} : dispatch 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: audit 2024-09-13T14:34:24.236861+0000 mon.a (mon.0) 374 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"} : dispatch 2024-09-13T14:34:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: audit 2024-09-13T14:34:24.245045+0000 mon.a (mon.0) 375 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"}]': finished 2024-09-13T14:34:25.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:25 smithi099 bash[87147]: audit 2024-09-13T14:34:24.245045+0000 mon.a (mon.0) 375 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4ea7538b-e59f-4845-b241-3ff334a658a0"}]': finished 2024-09-13T14:34:26.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[61439]: ts=2024-09-13T14:34:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:26.105 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:25 smithi177 bash[61439]: ts=2024-09-13T14:34:25.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:26.597 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:26 smithi099 bash[84576]: cluster 2024-09-13T14:34:25.257729+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-13T14:34:26.597 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:26 smithi099 bash[84576]: cluster 2024-09-13T14:34:25.257729+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-13T14:34:26.597 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:26 smithi099 bash[84576]: cluster 2024-09-13T14:34:25.994406+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:34:26.597 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:26 smithi099 bash[84576]: cluster 2024-09-13T14:34:25.994406+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:34:26.598 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:26 smithi099 bash[87147]: cluster 2024-09-13T14:34:25.257729+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-13T14:34:26.598 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:26 smithi099 bash[87147]: cluster 2024-09-13T14:34:25.257729+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-13T14:34:26.598 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:26 smithi099 bash[87147]: cluster 2024-09-13T14:34:25.994406+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:34:26.598 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:26 smithi099 bash[87147]: cluster 2024-09-13T14:34:25.994406+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:34:26.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:26 smithi177 bash[68594]: cluster 2024-09-13T14:34:25.257729+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-13T14:34:26.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:26 smithi177 bash[68594]: cluster 2024-09-13T14:34:25.257729+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-13T14:34:26.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:26 smithi177 bash[68594]: cluster 2024-09-13T14:34:25.994406+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:34:26.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:26 smithi177 bash[68594]: cluster 2024-09-13T14:34:25.994406+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-13T14:34:27.253 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:26 smithi177 bash[61439]: ts=2024-09-13T14:34:26.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:27.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:34:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:34:27] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.51.0" 2024-09-13T14:34:27.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:27 smithi099 bash[84576]: cluster 2024-09-13T14:34:25.971050+0000 mgr.y (mgr.26252) 108 : cluster [DBG] pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:27.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:27 smithi099 bash[84576]: cluster 2024-09-13T14:34:25.971050+0000 mgr.y (mgr.26252) 108 : cluster [DBG] pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:27.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:27 smithi099 bash[87147]: cluster 2024-09-13T14:34:25.971050+0000 mgr.y (mgr.26252) 108 : cluster [DBG] pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:27.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:27 smithi099 bash[87147]: cluster 2024-09-13T14:34:25.971050+0000 mgr.y (mgr.26252) 108 : cluster [DBG] pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:27.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:27 smithi177 bash[68594]: cluster 2024-09-13T14:34:25.971050+0000 mgr.y (mgr.26252) 108 : cluster [DBG] pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:27.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:27 smithi177 bash[68594]: cluster 2024-09-13T14:34:25.971050+0000 mgr.y (mgr.26252) 108 : cluster [DBG] pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-13T14:34:28.357 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.SnapshotImageState (4105 ms) 2024-09-13T14:34:28.357 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.SnapshotPromoteDemote 2024-09-13T14:34:28.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:28 smithi099 bash[84576]: cluster 2024-09-13T14:34:27.270784+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:34:28.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:28 smithi099 bash[84576]: cluster 2024-09-13T14:34:27.270784+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:34:28.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:28 smithi099 bash[87147]: cluster 2024-09-13T14:34:27.270784+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:34:28.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:28 smithi099 bash[87147]: cluster 2024-09-13T14:34:27.270784+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:34:28.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:28 smithi177 bash[68594]: cluster 2024-09-13T14:34:27.270784+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:34:28.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:28 smithi177 bash[68594]: cluster 2024-09-13T14:34:27.270784+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-13T14:34:29.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:29 smithi099 bash[84576]: cluster 2024-09-13T14:34:27.971738+0000 mgr.y (mgr.26252) 109 : cluster [DBG] pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:29.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:29 smithi099 bash[84576]: cluster 2024-09-13T14:34:27.971738+0000 mgr.y (mgr.26252) 109 : cluster [DBG] pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:29.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:29 smithi099 bash[84576]: cluster 2024-09-13T14:34:28.282289+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:34:29.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:29 smithi099 bash[84576]: cluster 2024-09-13T14:34:28.282289+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:34:29.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:29 smithi099 bash[87147]: cluster 2024-09-13T14:34:27.971738+0000 mgr.y (mgr.26252) 109 : cluster [DBG] pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:29.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:29 smithi099 bash[87147]: cluster 2024-09-13T14:34:27.971738+0000 mgr.y (mgr.26252) 109 : cluster [DBG] pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:29.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:29 smithi099 bash[87147]: cluster 2024-09-13T14:34:28.282289+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:34:29.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:29 smithi099 bash[87147]: cluster 2024-09-13T14:34:28.282289+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:34:29.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:29 smithi177 bash[68594]: cluster 2024-09-13T14:34:27.971738+0000 mgr.y (mgr.26252) 109 : cluster [DBG] pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:29.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:29 smithi177 bash[68594]: cluster 2024-09-13T14:34:27.971738+0000 mgr.y (mgr.26252) 109 : cluster [DBG] pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:29.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:29 smithi177 bash[68594]: cluster 2024-09-13T14:34:28.282289+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:34:29.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:29 smithi177 bash[68594]: cluster 2024-09-13T14:34:28.282289+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: cluster 2024-09-13T14:34:29.283071+0000 mon.a (mon.0) 380 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: cluster 2024-09-13T14:34:29.283071+0000 mon.a (mon.0) 380 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.303410+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.303410+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.326254+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.326254+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.328166+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.328166+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.329657+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.329657+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:30.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.341720+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.341720+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: cephadm 2024-09-13T14:34:29.385232+0000 mgr.y (mgr.26252) 110 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: cephadm 2024-09-13T14:34:29.385232+0000 mgr.y (mgr.26252) 110 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.385647+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.385647+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.386540+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.386540+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.387296+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: audit 2024-09-13T14:34:29.387296+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: cephadm 2024-09-13T14:34:29.388127+0000 mgr.y (mgr.26252) 111 : cephadm [INF] Reconfiguring daemon mon.a on smithi099 2024-09-13T14:34:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[84576]: cephadm 2024-09-13T14:34:29.388127+0000 mgr.y (mgr.26252) 111 : cephadm [INF] Reconfiguring daemon mon.a on smithi099 2024-09-13T14:34:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: cluster 2024-09-13T14:34:29.283071+0000 mon.a (mon.0) 380 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:34:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: cluster 2024-09-13T14:34:29.283071+0000 mon.a (mon.0) 380 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:34:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.303410+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.303410+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.326254+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.326254+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.328166+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.328166+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.329657+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.329657+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:30.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.341720+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.341720+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: cephadm 2024-09-13T14:34:29.385232+0000 mgr.y (mgr.26252) 110 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-13T14:34:30.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: cephadm 2024-09-13T14:34:29.385232+0000 mgr.y (mgr.26252) 110 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-13T14:34:30.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.385647+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:30.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.385647+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:30.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.386540+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:30.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.386540+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:30.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.387296+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: audit 2024-09-13T14:34:29.387296+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: cephadm 2024-09-13T14:34:29.388127+0000 mgr.y (mgr.26252) 111 : cephadm [INF] Reconfiguring daemon mon.a on smithi099 2024-09-13T14:34:30.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:30 smithi099 bash[87147]: cephadm 2024-09-13T14:34:29.388127+0000 mgr.y (mgr.26252) 111 : cephadm [INF] Reconfiguring daemon mon.a on smithi099 2024-09-13T14:34:30.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[61439]: ts=2024-09-13T14:34:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: cluster 2024-09-13T14:34:29.283071+0000 mon.a (mon.0) 380 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:34:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: cluster 2024-09-13T14:34:29.283071+0000 mon.a (mon.0) 380 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-13T14:34:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.303410+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.303410+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.326254+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.326254+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.328166+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.328166+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.329657+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.329657+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.341720+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.341720+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: cephadm 2024-09-13T14:34:29.385232+0000 mgr.y (mgr.26252) 110 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: cephadm 2024-09-13T14:34:29.385232+0000 mgr.y (mgr.26252) 110 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.385647+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.385647+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.386540+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.386540+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.387296+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: audit 2024-09-13T14:34:29.387296+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: cephadm 2024-09-13T14:34:29.388127+0000 mgr.y (mgr.26252) 111 : cephadm [INF] Reconfiguring daemon mon.a on smithi099 2024-09-13T14:34:30.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:30 smithi177 bash[68594]: cephadm 2024-09-13T14:34:29.388127+0000 mgr.y (mgr.26252) 111 : cephadm [INF] Reconfiguring daemon mon.a on smithi099 2024-09-13T14:34:31.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:31 smithi177 bash[68594]: cluster 2024-09-13T14:34:29.972407+0000 mgr.y (mgr.26252) 112 : cluster [DBG] pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:31.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:31 smithi177 bash[68594]: cluster 2024-09-13T14:34:29.972407+0000 mgr.y (mgr.26252) 112 : cluster [DBG] pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:31.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:31 smithi177 bash[68594]: cluster 2024-09-13T14:34:30.351052+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:34:31.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:31 smithi177 bash[68594]: cluster 2024-09-13T14:34:30.351052+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:34:31.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:31 smithi177 bash[68594]: cluster 2024-09-13T14:34:30.998600+0000 mon.a (mon.0) 390 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:34:31.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:31 smithi177 bash[68594]: cluster 2024-09-13T14:34:30.998600+0000 mon.a (mon.0) 390 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:34:31.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[87147]: cluster 2024-09-13T14:34:29.972407+0000 mgr.y (mgr.26252) 112 : cluster [DBG] pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:31.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[87147]: cluster 2024-09-13T14:34:29.972407+0000 mgr.y (mgr.26252) 112 : cluster [DBG] pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:31.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[87147]: cluster 2024-09-13T14:34:30.351052+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:34:31.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[87147]: cluster 2024-09-13T14:34:30.351052+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:34:31.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[87147]: cluster 2024-09-13T14:34:30.998600+0000 mon.a (mon.0) 390 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:34:31.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[87147]: cluster 2024-09-13T14:34:30.998600+0000 mon.a (mon.0) 390 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:34:31.751 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[84576]: cluster 2024-09-13T14:34:29.972407+0000 mgr.y (mgr.26252) 112 : cluster [DBG] pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:31.751 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[84576]: cluster 2024-09-13T14:34:29.972407+0000 mgr.y (mgr.26252) 112 : cluster [DBG] pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:31.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[84576]: cluster 2024-09-13T14:34:30.351052+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:34:31.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[84576]: cluster 2024-09-13T14:34:30.351052+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-13T14:34:31.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[84576]: cluster 2024-09-13T14:34:30.998600+0000 mon.a (mon.0) 390 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:34:31.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:31 smithi099 bash[84576]: cluster 2024-09-13T14:34:30.998600+0000 mon.a (mon.0) 390 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-09-13T14:34:33.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: cluster 2024-09-13T14:34:31.973107+0000 mgr.y (mgr.26252) 113 : cluster [DBG] pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:33.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: cluster 2024-09-13T14:34:31.973107+0000 mgr.y (mgr.26252) 113 : cluster [DBG] pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:33.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: cluster 2024-09-13T14:34:32.000223+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:34:33.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: cluster 2024-09-13T14:34:32.000223+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:34:33.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: audit 2024-09-13T14:34:32.743957+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: audit 2024-09-13T14:34:32.743957+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: audit 2024-09-13T14:34:32.751724+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: audit 2024-09-13T14:34:32.751724+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: audit 2024-09-13T14:34:32.753852+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:33.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: audit 2024-09-13T14:34:32.753852+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:33.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: audit 2024-09-13T14:34:32.755077+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:33.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: audit 2024-09-13T14:34:32.755077+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:33.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: audit 2024-09-13T14:34:32.756203+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:33.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[84576]: audit 2024-09-13T14:34:32.756203+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:33.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: cluster 2024-09-13T14:34:31.973107+0000 mgr.y (mgr.26252) 113 : cluster [DBG] pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:33.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: cluster 2024-09-13T14:34:31.973107+0000 mgr.y (mgr.26252) 113 : cluster [DBG] pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:33.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: cluster 2024-09-13T14:34:32.000223+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: cluster 2024-09-13T14:34:32.000223+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: audit 2024-09-13T14:34:32.743957+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: audit 2024-09-13T14:34:32.743957+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: audit 2024-09-13T14:34:32.751724+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: audit 2024-09-13T14:34:32.751724+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: audit 2024-09-13T14:34:32.753852+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: audit 2024-09-13T14:34:32.753852+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: audit 2024-09-13T14:34:32.755077+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: audit 2024-09-13T14:34:32.755077+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: audit 2024-09-13T14:34:32.756203+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:33.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:32 smithi099 bash[87147]: audit 2024-09-13T14:34:32.756203+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: cluster 2024-09-13T14:34:31.973107+0000 mgr.y (mgr.26252) 113 : cluster [DBG] pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: cluster 2024-09-13T14:34:31.973107+0000 mgr.y (mgr.26252) 113 : cluster [DBG] pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T14:34:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: cluster 2024-09-13T14:34:32.000223+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:34:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: cluster 2024-09-13T14:34:32.000223+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-09-13T14:34:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: audit 2024-09-13T14:34:32.743957+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: audit 2024-09-13T14:34:32.743957+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: audit 2024-09-13T14:34:32.751724+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: audit 2024-09-13T14:34:32.751724+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: audit 2024-09-13T14:34:32.753852+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:33.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: audit 2024-09-13T14:34:32.753852+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:33.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: audit 2024-09-13T14:34:32.755077+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:33.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: audit 2024-09-13T14:34:32.755077+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:33.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: audit 2024-09-13T14:34:32.756203+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:33.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:33 smithi177 bash[68594]: audit 2024-09-13T14:34:32.756203+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:34.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[84576]: cephadm 2024-09-13T14:34:32.753418+0000 mgr.y (mgr.26252) 114 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:34:34.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[84576]: cephadm 2024-09-13T14:34:32.753418+0000 mgr.y (mgr.26252) 114 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:34:34.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[84576]: cephadm 2024-09-13T14:34:32.757247+0000 mgr.y (mgr.26252) 115 : cephadm [INF] Reconfiguring daemon mgr.y on smithi099 2024-09-13T14:34:34.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[84576]: cephadm 2024-09-13T14:34:32.757247+0000 mgr.y (mgr.26252) 115 : cephadm [INF] Reconfiguring daemon mgr.y on smithi099 2024-09-13T14:34:34.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[84576]: cluster 2024-09-13T14:34:33.011253+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:34:34.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[84576]: cluster 2024-09-13T14:34:33.011253+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:34:34.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[84576]: audit 2024-09-13T14:34:33.243776+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:34.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[84576]: audit 2024-09-13T14:34:33.243776+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:34.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[87147]: cephadm 2024-09-13T14:34:32.753418+0000 mgr.y (mgr.26252) 114 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:34:34.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[87147]: cephadm 2024-09-13T14:34:32.753418+0000 mgr.y (mgr.26252) 114 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:34:34.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[87147]: cephadm 2024-09-13T14:34:32.757247+0000 mgr.y (mgr.26252) 115 : cephadm [INF] Reconfiguring daemon mgr.y on smithi099 2024-09-13T14:34:34.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[87147]: cephadm 2024-09-13T14:34:32.757247+0000 mgr.y (mgr.26252) 115 : cephadm [INF] Reconfiguring daemon mgr.y on smithi099 2024-09-13T14:34:34.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[87147]: cluster 2024-09-13T14:34:33.011253+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:34:34.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[87147]: cluster 2024-09-13T14:34:33.011253+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:34:34.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[87147]: audit 2024-09-13T14:34:33.243776+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:34.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:34 smithi099 bash[87147]: audit 2024-09-13T14:34:33.243776+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:34.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:34 smithi177 bash[61439]: ts=2024-09-13T14:34:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:34.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:34 smithi177 bash[68594]: cephadm 2024-09-13T14:34:32.753418+0000 mgr.y (mgr.26252) 114 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:34:34.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:34 smithi177 bash[68594]: cephadm 2024-09-13T14:34:32.753418+0000 mgr.y (mgr.26252) 114 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:34:34.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:34 smithi177 bash[68594]: cephadm 2024-09-13T14:34:32.757247+0000 mgr.y (mgr.26252) 115 : cephadm [INF] Reconfiguring daemon mgr.y on smithi099 2024-09-13T14:34:34.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:34 smithi177 bash[68594]: cephadm 2024-09-13T14:34:32.757247+0000 mgr.y (mgr.26252) 115 : cephadm [INF] Reconfiguring daemon mgr.y on smithi099 2024-09-13T14:34:34.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:34 smithi177 bash[68594]: cluster 2024-09-13T14:34:33.011253+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:34:34.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:34 smithi177 bash[68594]: cluster 2024-09-13T14:34:33.011253+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-09-13T14:34:34.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:34 smithi177 bash[68594]: audit 2024-09-13T14:34:33.243776+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:34.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:34 smithi177 bash[68594]: audit 2024-09-13T14:34:33.243776+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:35.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:35 smithi099 bash[84576]: cluster 2024-09-13T14:34:33.974360+0000 mgr.y (mgr.26252) 116 : cluster [DBG] pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:35.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:35 smithi099 bash[84576]: cluster 2024-09-13T14:34:33.974360+0000 mgr.y (mgr.26252) 116 : cluster [DBG] pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:35.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:35 smithi099 bash[84576]: cluster 2024-09-13T14:34:34.025679+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:34:35.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:35 smithi099 bash[84576]: cluster 2024-09-13T14:34:34.025679+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:34:35.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:35 smithi099 bash[87147]: cluster 2024-09-13T14:34:33.974360+0000 mgr.y (mgr.26252) 116 : cluster [DBG] pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:35.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:35 smithi099 bash[87147]: cluster 2024-09-13T14:34:33.974360+0000 mgr.y (mgr.26252) 116 : cluster [DBG] pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:35.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:35 smithi099 bash[87147]: cluster 2024-09-13T14:34:34.025679+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:34:35.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:35 smithi099 bash[87147]: cluster 2024-09-13T14:34:34.025679+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:34:35.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:35 smithi177 bash[68594]: cluster 2024-09-13T14:34:33.974360+0000 mgr.y (mgr.26252) 116 : cluster [DBG] pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:35.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:35 smithi177 bash[68594]: cluster 2024-09-13T14:34:33.974360+0000 mgr.y (mgr.26252) 116 : cluster [DBG] pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:35.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:35 smithi177 bash[68594]: cluster 2024-09-13T14:34:34.025679+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:34:35.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:35 smithi177 bash[68594]: cluster 2024-09-13T14:34:34.025679+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-09-13T14:34:36.026 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:35 smithi177 bash[61439]: ts=2024-09-13T14:34:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:36.026 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:35 smithi177 bash[61439]: ts=2024-09-13T14:34:35.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: cluster 2024-09-13T14:34:35.033076+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:34:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: cluster 2024-09-13T14:34:35.033076+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:34:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: audit 2024-09-13T14:34:35.689398+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: audit 2024-09-13T14:34:35.689398+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: audit 2024-09-13T14:34:35.701006+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: audit 2024-09-13T14:34:35.701006+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: cephadm 2024-09-13T14:34:35.702597+0000 mgr.y (mgr.26252) 117 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T14:34:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: cephadm 2024-09-13T14:34:35.702597+0000 mgr.y (mgr.26252) 117 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T14:34:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: audit 2024-09-13T14:34:35.702983+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: audit 2024-09-13T14:34:35.702983+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: audit 2024-09-13T14:34:35.704035+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: audit 2024-09-13T14:34:35.704035+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: audit 2024-09-13T14:34:35.705022+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: audit 2024-09-13T14:34:35.705022+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: cephadm 2024-09-13T14:34:35.706361+0000 mgr.y (mgr.26252) 118 : cephadm [INF] Reconfiguring daemon mon.c on smithi099 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[84576]: cephadm 2024-09-13T14:34:35.706361+0000 mgr.y (mgr.26252) 118 : cephadm [INF] Reconfiguring daemon mon.c on smithi099 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: cluster 2024-09-13T14:34:35.033076+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: cluster 2024-09-13T14:34:35.033076+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: audit 2024-09-13T14:34:35.689398+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: audit 2024-09-13T14:34:35.689398+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: audit 2024-09-13T14:34:35.701006+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: audit 2024-09-13T14:34:35.701006+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: cephadm 2024-09-13T14:34:35.702597+0000 mgr.y (mgr.26252) 117 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: cephadm 2024-09-13T14:34:35.702597+0000 mgr.y (mgr.26252) 117 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: audit 2024-09-13T14:34:35.702983+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: audit 2024-09-13T14:34:35.702983+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: audit 2024-09-13T14:34:35.704035+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: audit 2024-09-13T14:34:35.704035+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: audit 2024-09-13T14:34:35.705022+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: audit 2024-09-13T14:34:35.705022+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: cephadm 2024-09-13T14:34:35.706361+0000 mgr.y (mgr.26252) 118 : cephadm [INF] Reconfiguring daemon mon.c on smithi099 2024-09-13T14:34:36.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:36 smithi099 bash[87147]: cephadm 2024-09-13T14:34:35.706361+0000 mgr.y (mgr.26252) 118 : cephadm [INF] Reconfiguring daemon mon.c on smithi099 2024-09-13T14:34:36.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: cluster 2024-09-13T14:34:35.033076+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:34:36.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: cluster 2024-09-13T14:34:35.033076+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-09-13T14:34:36.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: audit 2024-09-13T14:34:35.689398+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: audit 2024-09-13T14:34:35.689398+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: audit 2024-09-13T14:34:35.701006+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: audit 2024-09-13T14:34:35.701006+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:36.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: cephadm 2024-09-13T14:34:35.702597+0000 mgr.y (mgr.26252) 117 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T14:34:36.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: cephadm 2024-09-13T14:34:35.702597+0000 mgr.y (mgr.26252) 117 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T14:34:36.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: audit 2024-09-13T14:34:35.702983+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:36.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: audit 2024-09-13T14:34:35.702983+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:36.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: audit 2024-09-13T14:34:35.704035+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:36.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: audit 2024-09-13T14:34:35.704035+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:36.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: audit 2024-09-13T14:34:35.705022+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:36.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: audit 2024-09-13T14:34:35.705022+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:36.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: cephadm 2024-09-13T14:34:35.706361+0000 mgr.y (mgr.26252) 118 : cephadm [INF] Reconfiguring daemon mon.c on smithi099 2024-09-13T14:34:36.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[68594]: cephadm 2024-09-13T14:34:35.706361+0000 mgr.y (mgr.26252) 118 : cephadm [INF] Reconfiguring daemon mon.c on smithi099 2024-09-13T14:34:37.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:34:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:34:37] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.51.0" 2024-09-13T14:34:37.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:37 smithi099 bash[84576]: cluster 2024-09-13T14:34:35.974854+0000 mgr.y (mgr.26252) 119 : cluster [DBG] pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:37.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:37 smithi099 bash[84576]: cluster 2024-09-13T14:34:35.974854+0000 mgr.y (mgr.26252) 119 : cluster [DBG] pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:37.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:37 smithi099 bash[84576]: cluster 2024-09-13T14:34:36.042334+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:34:37.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:37 smithi099 bash[84576]: cluster 2024-09-13T14:34:36.042334+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:34:37.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:37 smithi099 bash[87147]: cluster 2024-09-13T14:34:35.974854+0000 mgr.y (mgr.26252) 119 : cluster [DBG] pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:37.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:37 smithi099 bash[87147]: cluster 2024-09-13T14:34:35.974854+0000 mgr.y (mgr.26252) 119 : cluster [DBG] pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:37.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:37 smithi099 bash[87147]: cluster 2024-09-13T14:34:36.042334+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:34:37.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:37 smithi099 bash[87147]: cluster 2024-09-13T14:34:36.042334+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:34:37.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:36 smithi177 bash[61439]: ts=2024-09-13T14:34:36.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:37.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:37 smithi177 bash[68594]: cluster 2024-09-13T14:34:35.974854+0000 mgr.y (mgr.26252) 119 : cluster [DBG] pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:37.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:37 smithi177 bash[68594]: cluster 2024-09-13T14:34:35.974854+0000 mgr.y (mgr.26252) 119 : cluster [DBG] pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T14:34:37.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:37 smithi177 bash[68594]: cluster 2024-09-13T14:34:36.042334+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:34:37.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:37 smithi177 bash[68594]: cluster 2024-09-13T14:34:36.042334+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-09-13T14:34:38.195 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.SnapshotPromoteDemote (9838 ms) 2024-09-13T14:34:38.195 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestMirroring.AioSnapshotCreate 2024-09-13T14:34:38.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:38 smithi099 bash[84576]: cluster 2024-09-13T14:34:37.055865+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:34:38.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:38 smithi099 bash[84576]: cluster 2024-09-13T14:34:37.055865+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:34:38.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:38 smithi099 bash[87147]: cluster 2024-09-13T14:34:37.055865+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:34:38.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:38 smithi099 bash[87147]: cluster 2024-09-13T14:34:37.055865+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:34:38.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:38 smithi177 bash[68594]: cluster 2024-09-13T14:34:37.055865+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:34:38.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:38 smithi177 bash[68594]: cluster 2024-09-13T14:34:37.055865+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2024-09-13T14:34:39.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: cluster 2024-09-13T14:34:37.975522+0000 mgr.y (mgr.26252) 120 : cluster [DBG] pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:39.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: cluster 2024-09-13T14:34:37.975522+0000 mgr.y (mgr.26252) 120 : cluster [DBG] pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: cluster 2024-09-13T14:34:38.067112+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: cluster 2024-09-13T14:34:38.067112+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.181148+0000 mon.a (mon.0) 409 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"} : dispatch 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.181148+0000 mon.a (mon.0) 409 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"} : dispatch 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.188422+0000 mon.a (mon.0) 410 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"}]': finished 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.188422+0000 mon.a (mon.0) 410 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"}]': finished 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.659646+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.659646+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.667315+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.667315+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: cephadm 2024-09-13T14:34:38.669134+0000 mgr.y (mgr.26252) 121 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: cephadm 2024-09-13T14:34:38.669134+0000 mgr.y (mgr.26252) 121 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:34:39.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.669707+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.669707+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.670935+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: audit 2024-09-13T14:34:38.670935+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: cephadm 2024-09-13T14:34:38.675649+0000 mgr.y (mgr.26252) 122 : cephadm [INF] Reconfiguring daemon osd.0 on smithi099 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[87147]: cephadm 2024-09-13T14:34:38.675649+0000 mgr.y (mgr.26252) 122 : cephadm [INF] Reconfiguring daemon osd.0 on smithi099 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: cluster 2024-09-13T14:34:37.975522+0000 mgr.y (mgr.26252) 120 : cluster [DBG] pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: cluster 2024-09-13T14:34:37.975522+0000 mgr.y (mgr.26252) 120 : cluster [DBG] pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: cluster 2024-09-13T14:34:38.067112+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: cluster 2024-09-13T14:34:38.067112+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.181148+0000 mon.a (mon.0) 409 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"} : dispatch 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.181148+0000 mon.a (mon.0) 409 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"} : dispatch 2024-09-13T14:34:39.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.188422+0000 mon.a (mon.0) 410 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"}]': finished 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.188422+0000 mon.a (mon.0) 410 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"}]': finished 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.659646+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.659646+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.667315+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.667315+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: cephadm 2024-09-13T14:34:38.669134+0000 mgr.y (mgr.26252) 121 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: cephadm 2024-09-13T14:34:38.669134+0000 mgr.y (mgr.26252) 121 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.669707+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.669707+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.670935+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: audit 2024-09-13T14:34:38.670935+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: cephadm 2024-09-13T14:34:38.675649+0000 mgr.y (mgr.26252) 122 : cephadm [INF] Reconfiguring daemon osd.0 on smithi099 2024-09-13T14:34:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:39 smithi099 bash[84576]: cephadm 2024-09-13T14:34:38.675649+0000 mgr.y (mgr.26252) 122 : cephadm [INF] Reconfiguring daemon osd.0 on smithi099 2024-09-13T14:34:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: cluster 2024-09-13T14:34:37.975522+0000 mgr.y (mgr.26252) 120 : cluster [DBG] pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: cluster 2024-09-13T14:34:37.975522+0000 mgr.y (mgr.26252) 120 : cluster [DBG] pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: cluster 2024-09-13T14:34:38.067112+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:34:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: cluster 2024-09-13T14:34:38.067112+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2024-09-13T14:34:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.181148+0000 mon.a (mon.0) 409 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"} : dispatch 2024-09-13T14:34:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.181148+0000 mon.a (mon.0) 409 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"} : dispatch 2024-09-13T14:34:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.188422+0000 mon.a (mon.0) 410 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"}]': finished 2024-09-13T14:34:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.188422+0000 mon.a (mon.0) 410 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/f330f32d-98cf-4eed-9abd-5ef4227ebcff"}]': finished 2024-09-13T14:34:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.659646+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.659646+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.667315+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.667315+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: cephadm 2024-09-13T14:34:38.669134+0000 mgr.y (mgr.26252) 121 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: cephadm 2024-09-13T14:34:38.669134+0000 mgr.y (mgr.26252) 121 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.669707+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.669707+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.670935+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: audit 2024-09-13T14:34:38.670935+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: cephadm 2024-09-13T14:34:38.675649+0000 mgr.y (mgr.26252) 122 : cephadm [INF] Reconfiguring daemon osd.0 on smithi099 2024-09-13T14:34:39.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:39 smithi177 bash[68594]: cephadm 2024-09-13T14:34:38.675649+0000 mgr.y (mgr.26252) 122 : cephadm [INF] Reconfiguring daemon osd.0 on smithi099 2024-09-13T14:34:40.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:40 smithi099 bash[84576]: cluster 2024-09-13T14:34:39.192322+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:34:40.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:40 smithi099 bash[84576]: cluster 2024-09-13T14:34:39.192322+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:34:40.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:40 smithi099 bash[87147]: cluster 2024-09-13T14:34:39.192322+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:34:40.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:40 smithi099 bash[87147]: cluster 2024-09-13T14:34:39.192322+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:34:40.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:40 smithi177 bash[68594]: cluster 2024-09-13T14:34:39.192322+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:34:40.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:40 smithi177 bash[68594]: cluster 2024-09-13T14:34:39.192322+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2024-09-13T14:34:40.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:40 smithi177 bash[61439]: ts=2024-09-13T14:34:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:41.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[87147]: cluster 2024-09-13T14:34:39.976230+0000 mgr.y (mgr.26252) 123 : cluster [DBG] pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:41.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[87147]: cluster 2024-09-13T14:34:39.976230+0000 mgr.y (mgr.26252) 123 : cluster [DBG] pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:41.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[87147]: cluster 2024-09-13T14:34:40.203286+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:34:41.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[87147]: cluster 2024-09-13T14:34:40.203286+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:34:41.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[87147]: cluster 2024-09-13T14:34:41.001679+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:34:41.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[87147]: cluster 2024-09-13T14:34:41.001679+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:34:41.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[84576]: cluster 2024-09-13T14:34:39.976230+0000 mgr.y (mgr.26252) 123 : cluster [DBG] pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:41.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[84576]: cluster 2024-09-13T14:34:39.976230+0000 mgr.y (mgr.26252) 123 : cluster [DBG] pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:41.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[84576]: cluster 2024-09-13T14:34:40.203286+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:34:41.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[84576]: cluster 2024-09-13T14:34:40.203286+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:34:41.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[84576]: cluster 2024-09-13T14:34:41.001679+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:34:41.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:41 smithi099 bash[84576]: cluster 2024-09-13T14:34:41.001679+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:34:41.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:41 smithi177 bash[68594]: cluster 2024-09-13T14:34:39.976230+0000 mgr.y (mgr.26252) 123 : cluster [DBG] pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:41.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:41 smithi177 bash[68594]: cluster 2024-09-13T14:34:39.976230+0000 mgr.y (mgr.26252) 123 : cluster [DBG] pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:41.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:41 smithi177 bash[68594]: cluster 2024-09-13T14:34:40.203286+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:34:41.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:41 smithi177 bash[68594]: cluster 2024-09-13T14:34:40.203286+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2024-09-13T14:34:41.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:41 smithi177 bash[68594]: cluster 2024-09-13T14:34:41.001679+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:34:41.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:41 smithi177 bash[68594]: cluster 2024-09-13T14:34:41.001679+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2024-09-13T14:34:42.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: audit 2024-09-13T14:34:41.606080+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:42.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: audit 2024-09-13T14:34:41.606080+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:42.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: audit 2024-09-13T14:34:41.616210+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:42.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: audit 2024-09-13T14:34:41.616210+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:42.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: cephadm 2024-09-13T14:34:41.618006+0000 mgr.y (mgr.26252) 124 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:34:42.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: cephadm 2024-09-13T14:34:41.618006+0000 mgr.y (mgr.26252) 124 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:34:42.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: audit 2024-09-13T14:34:41.618441+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:34:42.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: audit 2024-09-13T14:34:41.618441+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:34:42.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: audit 2024-09-13T14:34:41.619647+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:42.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: audit 2024-09-13T14:34:41.619647+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:42.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: cephadm 2024-09-13T14:34:41.624284+0000 mgr.y (mgr.26252) 125 : cephadm [INF] Reconfiguring daemon osd.1 on smithi099 2024-09-13T14:34:42.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: cephadm 2024-09-13T14:34:41.624284+0000 mgr.y (mgr.26252) 125 : cephadm [INF] Reconfiguring daemon osd.1 on smithi099 2024-09-13T14:34:42.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: cluster 2024-09-13T14:34:42.001591+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:34:42.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:42 smithi177 bash[68594]: cluster 2024-09-13T14:34:42.001591+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:34:43.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: audit 2024-09-13T14:34:41.606080+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: audit 2024-09-13T14:34:41.606080+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: audit 2024-09-13T14:34:41.616210+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: audit 2024-09-13T14:34:41.616210+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: cephadm 2024-09-13T14:34:41.618006+0000 mgr.y (mgr.26252) 124 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: cephadm 2024-09-13T14:34:41.618006+0000 mgr.y (mgr.26252) 124 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: audit 2024-09-13T14:34:41.618441+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: audit 2024-09-13T14:34:41.618441+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: audit 2024-09-13T14:34:41.619647+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: audit 2024-09-13T14:34:41.619647+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: cephadm 2024-09-13T14:34:41.624284+0000 mgr.y (mgr.26252) 125 : cephadm [INF] Reconfiguring daemon osd.1 on smithi099 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: cephadm 2024-09-13T14:34:41.624284+0000 mgr.y (mgr.26252) 125 : cephadm [INF] Reconfiguring daemon osd.1 on smithi099 2024-09-13T14:34:43.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: cluster 2024-09-13T14:34:42.001591+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[84576]: cluster 2024-09-13T14:34:42.001591+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: audit 2024-09-13T14:34:41.606080+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: audit 2024-09-13T14:34:41.606080+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: audit 2024-09-13T14:34:41.616210+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: audit 2024-09-13T14:34:41.616210+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: cephadm 2024-09-13T14:34:41.618006+0000 mgr.y (mgr.26252) 124 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: cephadm 2024-09-13T14:34:41.618006+0000 mgr.y (mgr.26252) 124 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: audit 2024-09-13T14:34:41.618441+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: audit 2024-09-13T14:34:41.618441+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: audit 2024-09-13T14:34:41.619647+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:43.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: audit 2024-09-13T14:34:41.619647+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: cephadm 2024-09-13T14:34:41.624284+0000 mgr.y (mgr.26252) 125 : cephadm [INF] Reconfiguring daemon osd.1 on smithi099 2024-09-13T14:34:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: cephadm 2024-09-13T14:34:41.624284+0000 mgr.y (mgr.26252) 125 : cephadm [INF] Reconfiguring daemon osd.1 on smithi099 2024-09-13T14:34:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: cluster 2024-09-13T14:34:42.001591+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:34:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:42 smithi099 bash[87147]: cluster 2024-09-13T14:34:42.001591+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2024-09-13T14:34:43.994 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:43 smithi099 bash[84576]: cluster 2024-09-13T14:34:41.976892+0000 mgr.y (mgr.26252) 126 : cluster [DBG] pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:43.994 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:43 smithi099 bash[84576]: cluster 2024-09-13T14:34:41.976892+0000 mgr.y (mgr.26252) 126 : cluster [DBG] pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:43.994 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:43 smithi099 bash[84576]: cluster 2024-09-13T14:34:43.006276+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:34:43.994 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:43 smithi099 bash[84576]: cluster 2024-09-13T14:34:43.006276+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:34:43.995 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:43 smithi099 bash[87147]: cluster 2024-09-13T14:34:41.976892+0000 mgr.y (mgr.26252) 126 : cluster [DBG] pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:43.995 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:43 smithi099 bash[87147]: cluster 2024-09-13T14:34:41.976892+0000 mgr.y (mgr.26252) 126 : cluster [DBG] pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:43.995 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:43 smithi099 bash[87147]: cluster 2024-09-13T14:34:43.006276+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:34:43.995 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:43 smithi099 bash[87147]: cluster 2024-09-13T14:34:43.006276+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:34:44.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:43 smithi177 bash[68594]: cluster 2024-09-13T14:34:41.976892+0000 mgr.y (mgr.26252) 126 : cluster [DBG] pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:44.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:43 smithi177 bash[68594]: cluster 2024-09-13T14:34:41.976892+0000 mgr.y (mgr.26252) 126 : cluster [DBG] pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T14:34:44.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:43 smithi177 bash[68594]: cluster 2024-09-13T14:34:43.006276+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:34:44.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:43 smithi177 bash[68594]: cluster 2024-09-13T14:34:43.006276+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e491: 8 total, 8 up, 8 in 2024-09-13T14:34:44.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:44 smithi177 bash[61439]: ts=2024-09-13T14:34:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:45.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: cluster 2024-09-13T14:34:43.978227+0000 mgr.y (mgr.26252) 127 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:45.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: cluster 2024-09-13T14:34:43.978227+0000 mgr.y (mgr.26252) 127 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:45.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: cluster 2024-09-13T14:34:44.012161+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:34:45.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: cluster 2024-09-13T14:34:44.012161+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:34:45.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: audit 2024-09-13T14:34:44.596517+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: audit 2024-09-13T14:34:44.596517+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: audit 2024-09-13T14:34:44.602807+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: audit 2024-09-13T14:34:44.602807+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: cephadm 2024-09-13T14:34:44.604256+0000 mgr.y (mgr.26252) 128 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: cephadm 2024-09-13T14:34:44.604256+0000 mgr.y (mgr.26252) 128 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: audit 2024-09-13T14:34:44.604667+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: audit 2024-09-13T14:34:44.604667+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: audit 2024-09-13T14:34:44.605833+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: audit 2024-09-13T14:34:44.605833+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: cephadm 2024-09-13T14:34:44.610169+0000 mgr.y (mgr.26252) 129 : cephadm [INF] Reconfiguring daemon osd.2 on smithi099 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[84576]: cephadm 2024-09-13T14:34:44.610169+0000 mgr.y (mgr.26252) 129 : cephadm [INF] Reconfiguring daemon osd.2 on smithi099 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: cluster 2024-09-13T14:34:43.978227+0000 mgr.y (mgr.26252) 127 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: cluster 2024-09-13T14:34:43.978227+0000 mgr.y (mgr.26252) 127 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:45.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: cluster 2024-09-13T14:34:44.012161+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: cluster 2024-09-13T14:34:44.012161+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: audit 2024-09-13T14:34:44.596517+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: audit 2024-09-13T14:34:44.596517+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: audit 2024-09-13T14:34:44.602807+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: audit 2024-09-13T14:34:44.602807+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: cephadm 2024-09-13T14:34:44.604256+0000 mgr.y (mgr.26252) 128 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: cephadm 2024-09-13T14:34:44.604256+0000 mgr.y (mgr.26252) 128 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: audit 2024-09-13T14:34:44.604667+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: audit 2024-09-13T14:34:44.604667+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: audit 2024-09-13T14:34:44.605833+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: audit 2024-09-13T14:34:44.605833+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: cephadm 2024-09-13T14:34:44.610169+0000 mgr.y (mgr.26252) 129 : cephadm [INF] Reconfiguring daemon osd.2 on smithi099 2024-09-13T14:34:45.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:45 smithi099 bash[87147]: cephadm 2024-09-13T14:34:44.610169+0000 mgr.y (mgr.26252) 129 : cephadm [INF] Reconfiguring daemon osd.2 on smithi099 2024-09-13T14:34:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: cluster 2024-09-13T14:34:43.978227+0000 mgr.y (mgr.26252) 127 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: cluster 2024-09-13T14:34:43.978227+0000 mgr.y (mgr.26252) 127 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: cluster 2024-09-13T14:34:44.012161+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:34:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: cluster 2024-09-13T14:34:44.012161+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2024-09-13T14:34:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: audit 2024-09-13T14:34:44.596517+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: audit 2024-09-13T14:34:44.596517+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: audit 2024-09-13T14:34:44.602807+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: audit 2024-09-13T14:34:44.602807+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:45.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: cephadm 2024-09-13T14:34:44.604256+0000 mgr.y (mgr.26252) 128 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:34:45.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: cephadm 2024-09-13T14:34:44.604256+0000 mgr.y (mgr.26252) 128 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:34:45.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: audit 2024-09-13T14:34:44.604667+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:34:45.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: audit 2024-09-13T14:34:44.604667+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:34:45.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: audit 2024-09-13T14:34:44.605833+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:45.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: audit 2024-09-13T14:34:44.605833+0000 mon.a (mon.0) 428 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:45.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: cephadm 2024-09-13T14:34:44.610169+0000 mgr.y (mgr.26252) 129 : cephadm [INF] Reconfiguring daemon osd.2 on smithi099 2024-09-13T14:34:45.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[68594]: cephadm 2024-09-13T14:34:44.610169+0000 mgr.y (mgr.26252) 129 : cephadm [INF] Reconfiguring daemon osd.2 on smithi099 2024-09-13T14:34:46.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[61439]: ts=2024-09-13T14:34:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:46.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:45 smithi177 bash[61439]: ts=2024-09-13T14:34:45.742Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:46.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:46 smithi099 bash[84576]: cluster 2024-09-13T14:34:45.026066+0000 mon.a (mon.0) 429 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:34:46.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:46 smithi099 bash[84576]: cluster 2024-09-13T14:34:45.026066+0000 mon.a (mon.0) 429 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:34:46.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:46 smithi099 bash[87147]: cluster 2024-09-13T14:34:45.026066+0000 mon.a (mon.0) 429 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:34:46.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:46 smithi099 bash[87147]: cluster 2024-09-13T14:34:45.026066+0000 mon.a (mon.0) 429 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:34:46.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:46 smithi177 bash[68594]: cluster 2024-09-13T14:34:45.026066+0000 mon.a (mon.0) 429 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:34:46.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:46 smithi177 bash[68594]: cluster 2024-09-13T14:34:45.026066+0000 mon.a (mon.0) 429 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2024-09-13T14:34:47.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:34:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:34:47] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.51.0" 2024-09-13T14:34:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:47 smithi099 bash[87147]: cluster 2024-09-13T14:34:45.979023+0000 mgr.y (mgr.26252) 130 : cluster [DBG] pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:47 smithi099 bash[87147]: cluster 2024-09-13T14:34:45.979023+0000 mgr.y (mgr.26252) 130 : cluster [DBG] pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:47.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:47 smithi099 bash[87147]: cluster 2024-09-13T14:34:46.036940+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:34:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:47 smithi099 bash[87147]: cluster 2024-09-13T14:34:46.036940+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:34:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:47 smithi099 bash[84576]: cluster 2024-09-13T14:34:45.979023+0000 mgr.y (mgr.26252) 130 : cluster [DBG] pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:47 smithi099 bash[84576]: cluster 2024-09-13T14:34:45.979023+0000 mgr.y (mgr.26252) 130 : cluster [DBG] pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:47 smithi099 bash[84576]: cluster 2024-09-13T14:34:46.036940+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:34:47.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:47 smithi099 bash[84576]: cluster 2024-09-13T14:34:46.036940+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:34:47.356 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:46 smithi177 bash[61439]: ts=2024-09-13T14:34:46.951Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi099\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi099\", job=\"node\", machine=\"x86_64\", nodename=\"smithi099\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:47.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:47 smithi177 bash[68594]: cluster 2024-09-13T14:34:45.979023+0000 mgr.y (mgr.26252) 130 : cluster [DBG] pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:47.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:47 smithi177 bash[68594]: cluster 2024-09-13T14:34:45.979023+0000 mgr.y (mgr.26252) 130 : cluster [DBG] pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T14:34:47.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:47 smithi177 bash[68594]: cluster 2024-09-13T14:34:46.036940+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:34:47.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:47 smithi177 bash[68594]: cluster 2024-09-13T14:34:46.036940+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2024-09-13T14:34:48.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: cluster 2024-09-13T14:34:47.046520+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:34:48.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: cluster 2024-09-13T14:34:47.046520+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:34:48.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: audit 2024-09-13T14:34:47.443334+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: audit 2024-09-13T14:34:47.443334+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: audit 2024-09-13T14:34:47.452055+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: audit 2024-09-13T14:34:47.452055+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: cephadm 2024-09-13T14:34:47.453362+0000 mgr.y (mgr.26252) 131 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:34:48.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: cephadm 2024-09-13T14:34:47.453362+0000 mgr.y (mgr.26252) 131 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:34:48.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: audit 2024-09-13T14:34:47.453742+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:34:48.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: audit 2024-09-13T14:34:47.453742+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: audit 2024-09-13T14:34:47.454976+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: audit 2024-09-13T14:34:47.454976+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: cephadm 2024-09-13T14:34:47.459489+0000 mgr.y (mgr.26252) 132 : cephadm [INF] Reconfiguring daemon osd.3 on smithi099 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[84576]: cephadm 2024-09-13T14:34:47.459489+0000 mgr.y (mgr.26252) 132 : cephadm [INF] Reconfiguring daemon osd.3 on smithi099 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: cluster 2024-09-13T14:34:47.046520+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: cluster 2024-09-13T14:34:47.046520+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: audit 2024-09-13T14:34:47.443334+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: audit 2024-09-13T14:34:47.443334+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: audit 2024-09-13T14:34:47.452055+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: audit 2024-09-13T14:34:47.452055+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: cephadm 2024-09-13T14:34:47.453362+0000 mgr.y (mgr.26252) 131 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:34:48.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: cephadm 2024-09-13T14:34:47.453362+0000 mgr.y (mgr.26252) 131 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:34:48.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: audit 2024-09-13T14:34:47.453742+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:34:48.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: audit 2024-09-13T14:34:47.453742+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:34:48.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: audit 2024-09-13T14:34:47.454976+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:48.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: audit 2024-09-13T14:34:47.454976+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:48.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: cephadm 2024-09-13T14:34:47.459489+0000 mgr.y (mgr.26252) 132 : cephadm [INF] Reconfiguring daemon osd.3 on smithi099 2024-09-13T14:34:48.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:48 smithi099 bash[87147]: cephadm 2024-09-13T14:34:47.459489+0000 mgr.y (mgr.26252) 132 : cephadm [INF] Reconfiguring daemon osd.3 on smithi099 2024-09-13T14:34:48.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: cluster 2024-09-13T14:34:47.046520+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:34:48.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: cluster 2024-09-13T14:34:47.046520+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2024-09-13T14:34:48.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: audit 2024-09-13T14:34:47.443334+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: audit 2024-09-13T14:34:47.443334+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: audit 2024-09-13T14:34:47.452055+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: audit 2024-09-13T14:34:47.452055+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:48.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: cephadm 2024-09-13T14:34:47.453362+0000 mgr.y (mgr.26252) 131 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:34:48.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: cephadm 2024-09-13T14:34:47.453362+0000 mgr.y (mgr.26252) 131 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:34:48.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: audit 2024-09-13T14:34:47.453742+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:34:48.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: audit 2024-09-13T14:34:47.453742+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:34:48.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: audit 2024-09-13T14:34:47.454976+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:48.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: audit 2024-09-13T14:34:47.454976+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:48.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: cephadm 2024-09-13T14:34:47.459489+0000 mgr.y (mgr.26252) 132 : cephadm [INF] Reconfiguring daemon osd.3 on smithi099 2024-09-13T14:34:48.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:48 smithi177 bash[68594]: cephadm 2024-09-13T14:34:47.459489+0000 mgr.y (mgr.26252) 132 : cephadm [INF] Reconfiguring daemon osd.3 on smithi099 2024-09-13T14:34:49.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[84576]: cluster 2024-09-13T14:34:47.979502+0000 mgr.y (mgr.26252) 133 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:49.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[84576]: cluster 2024-09-13T14:34:47.979502+0000 mgr.y (mgr.26252) 133 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:49.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[84576]: cluster 2024-09-13T14:34:48.058165+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2024-09-13T14:34:49.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[84576]: cluster 2024-09-13T14:34:48.058165+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2024-09-13T14:34:49.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[84576]: audit 2024-09-13T14:34:48.244160+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:49.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[84576]: audit 2024-09-13T14:34:48.244160+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:49.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[87147]: cluster 2024-09-13T14:34:47.979502+0000 mgr.y (mgr.26252) 133 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:49.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[87147]: cluster 2024-09-13T14:34:47.979502+0000 mgr.y (mgr.26252) 133 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:49.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[87147]: cluster 2024-09-13T14:34:48.058165+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2024-09-13T14:34:49.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[87147]: cluster 2024-09-13T14:34:48.058165+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2024-09-13T14:34:49.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[87147]: audit 2024-09-13T14:34:48.244160+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:49.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:49 smithi099 bash[87147]: audit 2024-09-13T14:34:48.244160+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:49 smithi177 bash[68594]: cluster 2024-09-13T14:34:47.979502+0000 mgr.y (mgr.26252) 133 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:49 smithi177 bash[68594]: cluster 2024-09-13T14:34:47.979502+0000 mgr.y (mgr.26252) 133 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:49 smithi177 bash[68594]: cluster 2024-09-13T14:34:48.058165+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2024-09-13T14:34:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:49 smithi177 bash[68594]: cluster 2024-09-13T14:34:48.058165+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2024-09-13T14:34:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:49 smithi177 bash[68594]: audit 2024-09-13T14:34:48.244160+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:49.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:49 smithi177 bash[68594]: audit 2024-09-13T14:34:48.244160+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:50.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:50 smithi177 bash[68594]: cluster 2024-09-13T14:34:49.070926+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2024-09-13T14:34:50.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:50 smithi177 bash[68594]: cluster 2024-09-13T14:34:49.070926+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2024-09-13T14:34:50.356 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:50 smithi177 bash[61439]: ts=2024-09-13T14:34:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:50.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:50 smithi099 bash[84576]: cluster 2024-09-13T14:34:49.070926+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2024-09-13T14:34:50.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:50 smithi099 bash[84576]: cluster 2024-09-13T14:34:49.070926+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2024-09-13T14:34:50.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:50 smithi099 bash[87147]: cluster 2024-09-13T14:34:49.070926+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2024-09-13T14:34:50.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:50 smithi099 bash[87147]: cluster 2024-09-13T14:34:49.070926+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2024-09-13T14:34:51.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: cluster 2024-09-13T14:34:49.980143+0000 mgr.y (mgr.26252) 134 : cluster [DBG] pgmap v247: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:51.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: cluster 2024-09-13T14:34:49.980143+0000 mgr.y (mgr.26252) 134 : cluster [DBG] pgmap v247: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:51.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: cluster 2024-09-13T14:34:49.980143+0000 mgr.y (mgr.26252) 134 : cluster [DBG] pgmap v247: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: cluster 2024-09-13T14:34:50.083095+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: cluster 2024-09-13T14:34:50.083095+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: audit 2024-09-13T14:34:50.321456+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: audit 2024-09-13T14:34:50.321456+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: audit 2024-09-13T14:34:50.329074+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: audit 2024-09-13T14:34:50.329074+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: cephadm 2024-09-13T14:34:50.331140+0000 mgr.y (mgr.26252) 135 : cephadm [INF] Reconfiguring mds.foo.smithi099.yyrtld (monmap changed)... 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: cephadm 2024-09-13T14:34:50.331140+0000 mgr.y (mgr.26252) 135 : cephadm [INF] Reconfiguring mds.foo.smithi099.yyrtld (monmap changed)... 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: audit 2024-09-13T14:34:50.331687+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: audit 2024-09-13T14:34:50.331687+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:51.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: audit 2024-09-13T14:34:50.333128+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:51.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: audit 2024-09-13T14:34:50.333128+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:51.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: cephadm 2024-09-13T14:34:50.334517+0000 mgr.y (mgr.26252) 136 : cephadm [INF] Reconfiguring daemon mds.foo.smithi099.yyrtld on smithi099 2024-09-13T14:34:51.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:51 smithi177 bash[68594]: cephadm 2024-09-13T14:34:50.334517+0000 mgr.y (mgr.26252) 136 : cephadm [INF] Reconfiguring daemon mds.foo.smithi099.yyrtld on smithi099 2024-09-13T14:34:51.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: cluster 2024-09-13T14:34:49.980143+0000 mgr.y (mgr.26252) 134 : cluster [DBG] pgmap v247: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:51.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: cluster 2024-09-13T14:34:49.980143+0000 mgr.y (mgr.26252) 134 : cluster [DBG] pgmap v247: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: cluster 2024-09-13T14:34:50.083095+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: cluster 2024-09-13T14:34:50.083095+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: audit 2024-09-13T14:34:50.321456+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: audit 2024-09-13T14:34:50.321456+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: audit 2024-09-13T14:34:50.329074+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: audit 2024-09-13T14:34:50.329074+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: cephadm 2024-09-13T14:34:50.331140+0000 mgr.y (mgr.26252) 135 : cephadm [INF] Reconfiguring mds.foo.smithi099.yyrtld (monmap changed)... 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: cephadm 2024-09-13T14:34:50.331140+0000 mgr.y (mgr.26252) 135 : cephadm [INF] Reconfiguring mds.foo.smithi099.yyrtld (monmap changed)... 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: audit 2024-09-13T14:34:50.331687+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: audit 2024-09-13T14:34:50.331687+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: audit 2024-09-13T14:34:50.333128+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: audit 2024-09-13T14:34:50.333128+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: cephadm 2024-09-13T14:34:50.334517+0000 mgr.y (mgr.26252) 136 : cephadm [INF] Reconfiguring daemon mds.foo.smithi099.yyrtld on smithi099 2024-09-13T14:34:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[87147]: cephadm 2024-09-13T14:34:50.334517+0000 mgr.y (mgr.26252) 136 : cephadm [INF] Reconfiguring daemon mds.foo.smithi099.yyrtld on smithi099 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: cluster 2024-09-13T14:34:49.980143+0000 mgr.y (mgr.26252) 134 : cluster [DBG] pgmap v247: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: cluster 2024-09-13T14:34:50.083095+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: cluster 2024-09-13T14:34:50.083095+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: audit 2024-09-13T14:34:50.321456+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: audit 2024-09-13T14:34:50.321456+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: audit 2024-09-13T14:34:50.329074+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: audit 2024-09-13T14:34:50.329074+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: cephadm 2024-09-13T14:34:50.331140+0000 mgr.y (mgr.26252) 135 : cephadm [INF] Reconfiguring mds.foo.smithi099.yyrtld (monmap changed)... 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: cephadm 2024-09-13T14:34:50.331140+0000 mgr.y (mgr.26252) 135 : cephadm [INF] Reconfiguring mds.foo.smithi099.yyrtld (monmap changed)... 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: audit 2024-09-13T14:34:50.331687+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: audit 2024-09-13T14:34:50.331687+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.yyrtld", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:34:51.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: audit 2024-09-13T14:34:50.333128+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:51.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: audit 2024-09-13T14:34:50.333128+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:51.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: cephadm 2024-09-13T14:34:50.334517+0000 mgr.y (mgr.26252) 136 : cephadm [INF] Reconfiguring daemon mds.foo.smithi099.yyrtld on smithi099 2024-09-13T14:34:51.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:51 smithi099 bash[84576]: cephadm 2024-09-13T14:34:50.334517+0000 mgr.y (mgr.26252) 136 : cephadm [INF] Reconfiguring daemon mds.foo.smithi099.yyrtld on smithi099 2024-09-13T14:34:52.354 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:52 smithi099 bash[87147]: cluster 2024-09-13T14:34:51.089976+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2024-09-13T14:34:52.354 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:52 smithi099 bash[87147]: cluster 2024-09-13T14:34:51.089976+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2024-09-13T14:34:52.354 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:52 smithi099 bash[84576]: cluster 2024-09-13T14:34:51.089976+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2024-09-13T14:34:52.354 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:52 smithi099 bash[84576]: cluster 2024-09-13T14:34:51.089976+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2024-09-13T14:34:52.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:52 smithi177 bash[68594]: cluster 2024-09-13T14:34:51.089976+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2024-09-13T14:34:52.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:52 smithi177 bash[68594]: cluster 2024-09-13T14:34:51.089976+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2024-09-13T14:34:53.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:53 smithi177 bash[68594]: cluster 2024-09-13T14:34:51.980837+0000 mgr.y (mgr.26252) 137 : cluster [DBG] pgmap v250: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:53 smithi177 bash[68594]: cluster 2024-09-13T14:34:51.980837+0000 mgr.y (mgr.26252) 137 : cluster [DBG] pgmap v250: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:53 smithi177 bash[68594]: cluster 2024-09-13T14:34:52.103185+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2024-09-13T14:34:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:53 smithi177 bash[68594]: cluster 2024-09-13T14:34:52.103185+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2024-09-13T14:34:53.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:53 smithi099 bash[84576]: cluster 2024-09-13T14:34:51.980837+0000 mgr.y (mgr.26252) 137 : cluster [DBG] pgmap v250: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:53.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:53 smithi099 bash[84576]: cluster 2024-09-13T14:34:51.980837+0000 mgr.y (mgr.26252) 137 : cluster [DBG] pgmap v250: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:53 smithi099 bash[84576]: cluster 2024-09-13T14:34:52.103185+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2024-09-13T14:34:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:53 smithi099 bash[84576]: cluster 2024-09-13T14:34:52.103185+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2024-09-13T14:34:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:53 smithi099 bash[87147]: cluster 2024-09-13T14:34:51.980837+0000 mgr.y (mgr.26252) 137 : cluster [DBG] pgmap v250: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:53 smithi099 bash[87147]: cluster 2024-09-13T14:34:51.980837+0000 mgr.y (mgr.26252) 137 : cluster [DBG] pgmap v250: 105 pgs: 105 active+clean; 584 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-13T14:34:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:53 smithi099 bash[87147]: cluster 2024-09-13T14:34:52.103185+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2024-09-13T14:34:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:53 smithi099 bash[87147]: cluster 2024-09-13T14:34:52.103185+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2024-09-13T14:34:54.356 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[61439]: ts=2024-09-13T14:34:54.150Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.99\", device_class=\"ssd\", hostname=\"smithi099\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.99\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:34:54.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: cluster 2024-09-13T14:34:53.114803+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e501: 8 total, 8 up, 8 in 2024-09-13T14:34:54.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: cluster 2024-09-13T14:34:53.114803+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e501: 8 total, 8 up, 8 in 2024-09-13T14:34:54.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: audit 2024-09-13T14:34:53.320577+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: audit 2024-09-13T14:34:53.320577+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: audit 2024-09-13T14:34:53.329215+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: audit 2024-09-13T14:34:53.329215+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: cephadm 2024-09-13T14:34:53.330824+0000 mgr.y (mgr.26252) 138 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T14:34:54.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: cephadm 2024-09-13T14:34:53.330824+0000 mgr.y (mgr.26252) 138 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T14:34:54.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: audit 2024-09-13T14:34:53.331237+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:54.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: audit 2024-09-13T14:34:53.331237+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:54.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: audit 2024-09-13T14:34:53.332443+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:54.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: audit 2024-09-13T14:34:53.332443+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:54.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: audit 2024-09-13T14:34:53.333585+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:54.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: audit 2024-09-13T14:34:53.333585+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:54.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: cephadm 2024-09-13T14:34:53.334882+0000 mgr.y (mgr.26252) 139 : cephadm [INF] Reconfiguring daemon mon.b on smithi177 2024-09-13T14:34:54.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:54 smithi177 bash[68594]: cephadm 2024-09-13T14:34:53.334882+0000 mgr.y (mgr.26252) 139 : cephadm [INF] Reconfiguring daemon mon.b on smithi177 2024-09-13T14:34:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: cluster 2024-09-13T14:34:53.114803+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e501: 8 total, 8 up, 8 in 2024-09-13T14:34:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: cluster 2024-09-13T14:34:53.114803+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e501: 8 total, 8 up, 8 in 2024-09-13T14:34:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: audit 2024-09-13T14:34:53.320577+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: audit 2024-09-13T14:34:53.320577+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: audit 2024-09-13T14:34:53.329215+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: audit 2024-09-13T14:34:53.329215+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: cephadm 2024-09-13T14:34:53.330824+0000 mgr.y (mgr.26252) 138 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T14:34:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: cephadm 2024-09-13T14:34:53.330824+0000 mgr.y (mgr.26252) 138 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: audit 2024-09-13T14:34:53.331237+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: audit 2024-09-13T14:34:53.331237+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: audit 2024-09-13T14:34:53.332443+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: audit 2024-09-13T14:34:53.332443+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: audit 2024-09-13T14:34:53.333585+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: audit 2024-09-13T14:34:53.333585+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: cephadm 2024-09-13T14:34:53.334882+0000 mgr.y (mgr.26252) 139 : cephadm [INF] Reconfiguring daemon mon.b on smithi177 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[84576]: cephadm 2024-09-13T14:34:53.334882+0000 mgr.y (mgr.26252) 139 : cephadm [INF] Reconfiguring daemon mon.b on smithi177 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: cluster 2024-09-13T14:34:53.114803+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e501: 8 total, 8 up, 8 in 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: cluster 2024-09-13T14:34:53.114803+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e501: 8 total, 8 up, 8 in 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: audit 2024-09-13T14:34:53.320577+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: audit 2024-09-13T14:34:53.320577+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: audit 2024-09-13T14:34:53.329215+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: audit 2024-09-13T14:34:53.329215+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: cephadm 2024-09-13T14:34:53.330824+0000 mgr.y (mgr.26252) 138 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: cephadm 2024-09-13T14:34:53.330824+0000 mgr.y (mgr.26252) 138 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: audit 2024-09-13T14:34:53.331237+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: audit 2024-09-13T14:34:53.331237+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: audit 2024-09-13T14:34:53.332443+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: audit 2024-09-13T14:34:53.332443+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: audit 2024-09-13T14:34:53.333585+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: audit 2024-09-13T14:34:53.333585+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: cephadm 2024-09-13T14:34:53.334882+0000 mgr.y (mgr.26252) 139 : cephadm [INF] Reconfiguring daemon mon.b on smithi177 2024-09-13T14:34:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:54 smithi099 bash[87147]: cephadm 2024-09-13T14:34:53.334882+0000 mgr.y (mgr.26252) 139 : cephadm [INF] Reconfiguring daemon mon.b on smithi177 2024-09-13T14:34:55.372 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:55 smithi177 bash[68594]: cluster 2024-09-13T14:34:53.982151+0000 mgr.y (mgr.26252) 140 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:55.372 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:55 smithi177 bash[68594]: cluster 2024-09-13T14:34:53.982151+0000 mgr.y (mgr.26252) 140 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:55.372 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:55 smithi177 bash[68594]: cluster 2024-09-13T14:34:54.116219+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-13T14:34:55.372 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:55 smithi177 bash[68594]: cluster 2024-09-13T14:34:54.116219+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-13T14:34:55.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:55 smithi099 bash[84576]: cluster 2024-09-13T14:34:53.982151+0000 mgr.y (mgr.26252) 140 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:55 smithi099 bash[84576]: cluster 2024-09-13T14:34:53.982151+0000 mgr.y (mgr.26252) 140 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:55 smithi099 bash[84576]: cluster 2024-09-13T14:34:54.116219+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-13T14:34:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:55 smithi099 bash[84576]: cluster 2024-09-13T14:34:54.116219+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-13T14:34:55.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:55 smithi099 bash[87147]: cluster 2024-09-13T14:34:53.982151+0000 mgr.y (mgr.26252) 140 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:55.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:55 smithi099 bash[87147]: cluster 2024-09-13T14:34:53.982151+0000 mgr.y (mgr.26252) 140 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:55.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:55 smithi099 bash[87147]: cluster 2024-09-13T14:34:54.116219+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-13T14:34:55.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:55 smithi099 bash[87147]: cluster 2024-09-13T14:34:54.116219+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-13T14:34:56.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:56 smithi099 bash[84576]: cluster 2024-09-13T14:34:55.136426+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-13T14:34:56.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:56 smithi099 bash[84576]: cluster 2024-09-13T14:34:55.136426+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-13T14:34:56.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:56 smithi099 bash[84576]: cluster 2024-09-13T14:34:56.007494+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-13T14:34:56.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:56 smithi099 bash[84576]: cluster 2024-09-13T14:34:56.007494+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-13T14:34:56.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:56 smithi099 bash[87147]: cluster 2024-09-13T14:34:55.136426+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-13T14:34:56.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:56 smithi099 bash[87147]: cluster 2024-09-13T14:34:55.136426+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-13T14:34:56.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:56 smithi099 bash[87147]: cluster 2024-09-13T14:34:56.007494+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-13T14:34:56.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:56 smithi099 bash[87147]: cluster 2024-09-13T14:34:56.007494+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-13T14:34:56.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:56 smithi177 bash[68594]: cluster 2024-09-13T14:34:55.136426+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-13T14:34:56.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:56 smithi177 bash[68594]: cluster 2024-09-13T14:34:55.136426+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-13T14:34:56.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:56 smithi177 bash[68594]: cluster 2024-09-13T14:34:56.007494+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-13T14:34:56.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:56 smithi177 bash[68594]: cluster 2024-09-13T14:34:56.007494+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-13T14:34:57.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:34:57] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.51.0" 2024-09-13T14:34:57.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: cluster 2024-09-13T14:34:55.982877+0000 mgr.y (mgr.26252) 141 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:57.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: cluster 2024-09-13T14:34:55.982877+0000 mgr.y (mgr.26252) 141 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:57.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: audit 2024-09-13T14:34:56.263284+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: audit 2024-09-13T14:34:56.263284+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: audit 2024-09-13T14:34:56.273667+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: audit 2024-09-13T14:34:56.273667+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: cephadm 2024-09-13T14:34:56.275782+0000 mgr.y (mgr.26252) 142 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: cephadm 2024-09-13T14:34:56.275782+0000 mgr.y (mgr.26252) 142 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: audit 2024-09-13T14:34:56.276422+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: audit 2024-09-13T14:34:56.276422+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: audit 2024-09-13T14:34:56.278230+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: audit 2024-09-13T14:34:56.278230+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: audit 2024-09-13T14:34:56.279770+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: audit 2024-09-13T14:34:56.279770+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: cephadm 2024-09-13T14:34:56.281559+0000 mgr.y (mgr.26252) 143 : cephadm [INF] Reconfiguring daemon mgr.x on smithi177 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: cephadm 2024-09-13T14:34:56.281559+0000 mgr.y (mgr.26252) 143 : cephadm [INF] Reconfiguring daemon mgr.x on smithi177 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: cluster 2024-09-13T14:34:57.008995+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-13T14:34:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[84576]: cluster 2024-09-13T14:34:57.008995+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: cluster 2024-09-13T14:34:55.982877+0000 mgr.y (mgr.26252) 141 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: cluster 2024-09-13T14:34:55.982877+0000 mgr.y (mgr.26252) 141 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: audit 2024-09-13T14:34:56.263284+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: audit 2024-09-13T14:34:56.263284+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: audit 2024-09-13T14:34:56.273667+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: audit 2024-09-13T14:34:56.273667+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: cephadm 2024-09-13T14:34:56.275782+0000 mgr.y (mgr.26252) 142 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: cephadm 2024-09-13T14:34:56.275782+0000 mgr.y (mgr.26252) 142 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: audit 2024-09-13T14:34:56.276422+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: audit 2024-09-13T14:34:56.276422+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: audit 2024-09-13T14:34:56.278230+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: audit 2024-09-13T14:34:56.278230+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: audit 2024-09-13T14:34:56.279770+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: audit 2024-09-13T14:34:56.279770+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: cephadm 2024-09-13T14:34:56.281559+0000 mgr.y (mgr.26252) 143 : cephadm [INF] Reconfiguring daemon mgr.x on smithi177 2024-09-13T14:34:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: cephadm 2024-09-13T14:34:56.281559+0000 mgr.y (mgr.26252) 143 : cephadm [INF] Reconfiguring daemon mgr.x on smithi177 2024-09-13T14:34:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: cluster 2024-09-13T14:34:57.008995+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-13T14:34:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:57 smithi099 bash[87147]: cluster 2024-09-13T14:34:57.008995+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-13T14:34:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: cluster 2024-09-13T14:34:55.982877+0000 mgr.y (mgr.26252) 141 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: cluster 2024-09-13T14:34:55.982877+0000 mgr.y (mgr.26252) 141 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-13T14:34:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: audit 2024-09-13T14:34:56.263284+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: audit 2024-09-13T14:34:56.263284+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: audit 2024-09-13T14:34:56.273667+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: audit 2024-09-13T14:34:56.273667+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: cephadm 2024-09-13T14:34:56.275782+0000 mgr.y (mgr.26252) 142 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: cephadm 2024-09-13T14:34:56.275782+0000 mgr.y (mgr.26252) 142 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: audit 2024-09-13T14:34:56.276422+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: audit 2024-09-13T14:34:56.276422+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: audit 2024-09-13T14:34:56.278230+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: audit 2024-09-13T14:34:56.278230+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: audit 2024-09-13T14:34:56.279770+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: audit 2024-09-13T14:34:56.279770+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: cephadm 2024-09-13T14:34:56.281559+0000 mgr.y (mgr.26252) 143 : cephadm [INF] Reconfiguring daemon mgr.x on smithi177 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: cephadm 2024-09-13T14:34:56.281559+0000 mgr.y (mgr.26252) 143 : cephadm [INF] Reconfiguring daemon mgr.x on smithi177 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: cluster 2024-09-13T14:34:57.008995+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-13T14:34:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:57 smithi177 bash[68594]: cluster 2024-09-13T14:34:57.008995+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-13T14:34:59.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:59 smithi099 bash[84576]: cluster 2024-09-13T14:34:57.983580+0000 mgr.y (mgr.26252) 144 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:59 smithi099 bash[84576]: cluster 2024-09-13T14:34:57.983580+0000 mgr.y (mgr.26252) 144 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:59 smithi099 bash[84576]: cluster 2024-09-13T14:34:58.011046+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-13T14:34:59.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:34:59 smithi099 bash[84576]: cluster 2024-09-13T14:34:58.011046+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-13T14:34:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:59 smithi099 bash[87147]: cluster 2024-09-13T14:34:57.983580+0000 mgr.y (mgr.26252) 144 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:59 smithi099 bash[87147]: cluster 2024-09-13T14:34:57.983580+0000 mgr.y (mgr.26252) 144 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:59 smithi099 bash[87147]: cluster 2024-09-13T14:34:58.011046+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-13T14:34:59.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:34:59 smithi099 bash[87147]: cluster 2024-09-13T14:34:58.011046+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-13T14:34:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:59 smithi177 bash[68594]: cluster 2024-09-13T14:34:57.983580+0000 mgr.y (mgr.26252) 144 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:59 smithi177 bash[68594]: cluster 2024-09-13T14:34:57.983580+0000 mgr.y (mgr.26252) 144 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:34:59.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:59 smithi177 bash[68594]: cluster 2024-09-13T14:34:58.011046+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-13T14:34:59.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:34:59 smithi177 bash[68594]: cluster 2024-09-13T14:34:58.011046+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-13T14:35:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: cluster 2024-09-13T14:34:59.024264+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-13T14:35:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: cluster 2024-09-13T14:34:59.024264+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-13T14:35:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: audit 2024-09-13T14:34:59.041352+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: audit 2024-09-13T14:34:59.041352+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: audit 2024-09-13T14:34:59.056650+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: audit 2024-09-13T14:34:59.056650+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: cephadm 2024-09-13T14:34:59.059353+0000 mgr.y (mgr.26252) 145 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:35:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: cephadm 2024-09-13T14:34:59.059353+0000 mgr.y (mgr.26252) 145 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:35:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: audit 2024-09-13T14:34:59.059964+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: audit 2024-09-13T14:34:59.059964+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: audit 2024-09-13T14:34:59.061557+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: audit 2024-09-13T14:34:59.061557+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: cephadm 2024-09-13T14:34:59.066999+0000 mgr.y (mgr.26252) 146 : cephadm [INF] Reconfiguring daemon osd.4 on smithi177 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[84576]: cephadm 2024-09-13T14:34:59.066999+0000 mgr.y (mgr.26252) 146 : cephadm [INF] Reconfiguring daemon osd.4 on smithi177 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: cluster 2024-09-13T14:34:59.024264+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: cluster 2024-09-13T14:34:59.024264+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: audit 2024-09-13T14:34:59.041352+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: audit 2024-09-13T14:34:59.041352+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: audit 2024-09-13T14:34:59.056650+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: audit 2024-09-13T14:34:59.056650+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: cephadm 2024-09-13T14:34:59.059353+0000 mgr.y (mgr.26252) 145 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:35:00.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: cephadm 2024-09-13T14:34:59.059353+0000 mgr.y (mgr.26252) 145 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:35:00.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: audit 2024-09-13T14:34:59.059964+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:35:00.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: audit 2024-09-13T14:34:59.059964+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:35:00.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: audit 2024-09-13T14:34:59.061557+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:00.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: audit 2024-09-13T14:34:59.061557+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:00.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: cephadm 2024-09-13T14:34:59.066999+0000 mgr.y (mgr.26252) 146 : cephadm [INF] Reconfiguring daemon osd.4 on smithi177 2024-09-13T14:35:00.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:00 smithi099 bash[87147]: cephadm 2024-09-13T14:34:59.066999+0000 mgr.y (mgr.26252) 146 : cephadm [INF] Reconfiguring daemon osd.4 on smithi177 2024-09-13T14:35:00.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[61439]: ts=2024-09-13T14:35:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: cluster 2024-09-13T14:34:59.024264+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-13T14:35:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: cluster 2024-09-13T14:34:59.024264+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-13T14:35:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: audit 2024-09-13T14:34:59.041352+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: audit 2024-09-13T14:34:59.041352+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: audit 2024-09-13T14:34:59.056650+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: audit 2024-09-13T14:34:59.056650+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: cephadm 2024-09-13T14:34:59.059353+0000 mgr.y (mgr.26252) 145 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:35:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: cephadm 2024-09-13T14:34:59.059353+0000 mgr.y (mgr.26252) 145 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:35:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: audit 2024-09-13T14:34:59.059964+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:35:00.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: audit 2024-09-13T14:34:59.059964+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:35:00.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: audit 2024-09-13T14:34:59.061557+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:00.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: audit 2024-09-13T14:34:59.061557+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:00.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: cephadm 2024-09-13T14:34:59.066999+0000 mgr.y (mgr.26252) 146 : cephadm [INF] Reconfiguring daemon osd.4 on smithi177 2024-09-13T14:35:00.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:00 smithi177 bash[68594]: cephadm 2024-09-13T14:34:59.066999+0000 mgr.y (mgr.26252) 146 : cephadm [INF] Reconfiguring daemon osd.4 on smithi177 2024-09-13T14:35:01.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:01 smithi177 bash[68594]: cluster 2024-09-13T14:34:59.984298+0000 mgr.y (mgr.26252) 147 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:01.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:01 smithi177 bash[68594]: cluster 2024-09-13T14:34:59.984298+0000 mgr.y (mgr.26252) 147 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:01.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:01 smithi177 bash[68594]: cluster 2024-09-13T14:35:00.065877+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:35:01.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:01 smithi177 bash[68594]: cluster 2024-09-13T14:35:00.065877+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:35:01.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:01 smithi177 bash[68594]: cluster 2024-09-13T14:35:01.008005+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:35:01.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:01 smithi177 bash[68594]: cluster 2024-09-13T14:35:01.008005+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:35:01.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[84576]: cluster 2024-09-13T14:34:59.984298+0000 mgr.y (mgr.26252) 147 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:01.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[84576]: cluster 2024-09-13T14:34:59.984298+0000 mgr.y (mgr.26252) 147 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:01.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[84576]: cluster 2024-09-13T14:35:00.065877+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:35:01.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[84576]: cluster 2024-09-13T14:35:00.065877+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:35:01.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[84576]: cluster 2024-09-13T14:35:01.008005+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:35:01.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[84576]: cluster 2024-09-13T14:35:01.008005+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:35:01.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[87147]: cluster 2024-09-13T14:34:59.984298+0000 mgr.y (mgr.26252) 147 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:01.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[87147]: cluster 2024-09-13T14:34:59.984298+0000 mgr.y (mgr.26252) 147 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:01.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[87147]: cluster 2024-09-13T14:35:00.065877+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:35:01.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[87147]: cluster 2024-09-13T14:35:00.065877+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-09-13T14:35:01.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[87147]: cluster 2024-09-13T14:35:01.008005+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:35:01.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:01 smithi099 bash[87147]: cluster 2024-09-13T14:35:01.008005+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-09-13T14:35:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: audit 2024-09-13T14:35:01.943861+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: audit 2024-09-13T14:35:01.943861+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: audit 2024-09-13T14:35:01.956046+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: audit 2024-09-13T14:35:01.956046+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: cephadm 2024-09-13T14:35:01.958234+0000 mgr.y (mgr.26252) 148 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:35:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: cephadm 2024-09-13T14:35:01.958234+0000 mgr.y (mgr.26252) 148 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:35:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: audit 2024-09-13T14:35:01.958785+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:35:03.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: audit 2024-09-13T14:35:01.958785+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:35:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: audit 2024-09-13T14:35:01.960384+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: audit 2024-09-13T14:35:01.960384+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: cephadm 2024-09-13T14:35:01.965698+0000 mgr.y (mgr.26252) 149 : cephadm [INF] Reconfiguring daemon osd.5 on smithi177 2024-09-13T14:35:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: cephadm 2024-09-13T14:35:01.965698+0000 mgr.y (mgr.26252) 149 : cephadm [INF] Reconfiguring daemon osd.5 on smithi177 2024-09-13T14:35:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: cluster 2024-09-13T14:35:01.985063+0000 mgr.y (mgr.26252) 150 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: cluster 2024-09-13T14:35:01.985063+0000 mgr.y (mgr.26252) 150 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: cluster 2024-09-13T14:35:02.070922+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:35:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[87147]: cluster 2024-09-13T14:35:02.070922+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:35:03.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: audit 2024-09-13T14:35:01.943861+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: audit 2024-09-13T14:35:01.943861+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: audit 2024-09-13T14:35:01.956046+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: audit 2024-09-13T14:35:01.956046+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: cephadm 2024-09-13T14:35:01.958234+0000 mgr.y (mgr.26252) 148 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: cephadm 2024-09-13T14:35:01.958234+0000 mgr.y (mgr.26252) 148 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: audit 2024-09-13T14:35:01.958785+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: audit 2024-09-13T14:35:01.958785+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: audit 2024-09-13T14:35:01.960384+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: audit 2024-09-13T14:35:01.960384+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: cephadm 2024-09-13T14:35:01.965698+0000 mgr.y (mgr.26252) 149 : cephadm [INF] Reconfiguring daemon osd.5 on smithi177 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: cephadm 2024-09-13T14:35:01.965698+0000 mgr.y (mgr.26252) 149 : cephadm [INF] Reconfiguring daemon osd.5 on smithi177 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: cluster 2024-09-13T14:35:01.985063+0000 mgr.y (mgr.26252) 150 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: cluster 2024-09-13T14:35:01.985063+0000 mgr.y (mgr.26252) 150 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: cluster 2024-09-13T14:35:02.070922+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:35:03.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:02 smithi099 bash[84576]: cluster 2024-09-13T14:35:02.070922+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:35:03.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: audit 2024-09-13T14:35:01.943861+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: audit 2024-09-13T14:35:01.943861+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: audit 2024-09-13T14:35:01.956046+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: audit 2024-09-13T14:35:01.956046+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:03.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: cephadm 2024-09-13T14:35:01.958234+0000 mgr.y (mgr.26252) 148 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:35:03.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: cephadm 2024-09-13T14:35:01.958234+0000 mgr.y (mgr.26252) 148 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:35:03.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: audit 2024-09-13T14:35:01.958785+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:35:03.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: audit 2024-09-13T14:35:01.958785+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:35:03.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: audit 2024-09-13T14:35:01.960384+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:03.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: audit 2024-09-13T14:35:01.960384+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:03.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: cephadm 2024-09-13T14:35:01.965698+0000 mgr.y (mgr.26252) 149 : cephadm [INF] Reconfiguring daemon osd.5 on smithi177 2024-09-13T14:35:03.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: cephadm 2024-09-13T14:35:01.965698+0000 mgr.y (mgr.26252) 149 : cephadm [INF] Reconfiguring daemon osd.5 on smithi177 2024-09-13T14:35:03.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: cluster 2024-09-13T14:35:01.985063+0000 mgr.y (mgr.26252) 150 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:03.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: cluster 2024-09-13T14:35:01.985063+0000 mgr.y (mgr.26252) 150 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:03.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: cluster 2024-09-13T14:35:02.070922+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:35:03.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:02 smithi177 bash[68594]: cluster 2024-09-13T14:35:02.070922+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-09-13T14:35:04.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:04 smithi177 bash[68594]: cluster 2024-09-13T14:35:03.072695+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:35:04.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:04 smithi177 bash[68594]: cluster 2024-09-13T14:35:03.072695+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:35:04.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:04 smithi177 bash[68594]: audit 2024-09-13T14:35:03.245335+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:04.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:04 smithi177 bash[68594]: audit 2024-09-13T14:35:03.245335+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:04.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:04 smithi099 bash[84576]: cluster 2024-09-13T14:35:03.072695+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:35:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:04 smithi099 bash[84576]: cluster 2024-09-13T14:35:03.072695+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:35:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:04 smithi099 bash[84576]: audit 2024-09-13T14:35:03.245335+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:04 smithi099 bash[84576]: audit 2024-09-13T14:35:03.245335+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:04.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:04 smithi099 bash[87147]: cluster 2024-09-13T14:35:03.072695+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:35:04.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:04 smithi099 bash[87147]: cluster 2024-09-13T14:35:03.072695+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-09-13T14:35:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:04 smithi099 bash[87147]: audit 2024-09-13T14:35:03.245335+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:04 smithi099 bash[87147]: audit 2024-09-13T14:35:03.245335+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:05.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: cluster 2024-09-13T14:35:03.986422+0000 mgr.y (mgr.26252) 151 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: cluster 2024-09-13T14:35:03.986422+0000 mgr.y (mgr.26252) 151 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: cluster 2024-09-13T14:35:04.082977+0000 mon.a (mon.0) 476 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: cluster 2024-09-13T14:35:04.082977+0000 mon.a (mon.0) 476 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: audit 2024-09-13T14:35:04.719238+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: audit 2024-09-13T14:35:04.719238+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: audit 2024-09-13T14:35:04.729361+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: audit 2024-09-13T14:35:04.729361+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: audit 2024-09-13T14:35:04.732063+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: audit 2024-09-13T14:35:04.732063+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: audit 2024-09-13T14:35:04.733683+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:05.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:05 smithi177 bash[68594]: audit 2024-09-13T14:35:04.733683+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: cluster 2024-09-13T14:35:03.986422+0000 mgr.y (mgr.26252) 151 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: cluster 2024-09-13T14:35:03.986422+0000 mgr.y (mgr.26252) 151 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: cluster 2024-09-13T14:35:04.082977+0000 mon.a (mon.0) 476 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:35:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: cluster 2024-09-13T14:35:04.082977+0000 mon.a (mon.0) 476 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:35:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: audit 2024-09-13T14:35:04.719238+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: audit 2024-09-13T14:35:04.719238+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: audit 2024-09-13T14:35:04.729361+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: audit 2024-09-13T14:35:04.729361+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: audit 2024-09-13T14:35:04.732063+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:35:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: audit 2024-09-13T14:35:04.732063+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:35:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: audit 2024-09-13T14:35:04.733683+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[84576]: audit 2024-09-13T14:35:04.733683+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: cluster 2024-09-13T14:35:03.986422+0000 mgr.y (mgr.26252) 151 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: cluster 2024-09-13T14:35:03.986422+0000 mgr.y (mgr.26252) 151 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: cluster 2024-09-13T14:35:04.082977+0000 mon.a (mon.0) 476 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:35:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: cluster 2024-09-13T14:35:04.082977+0000 mon.a (mon.0) 476 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-09-13T14:35:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: audit 2024-09-13T14:35:04.719238+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: audit 2024-09-13T14:35:04.719238+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: audit 2024-09-13T14:35:04.729361+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: audit 2024-09-13T14:35:04.729361+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: audit 2024-09-13T14:35:04.732063+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:35:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: audit 2024-09-13T14:35:04.732063+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:35:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: audit 2024-09-13T14:35:04.733683+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:05 smithi099 bash[87147]: audit 2024-09-13T14:35:04.733683+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:06.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:06 smithi177 bash[68594]: cephadm 2024-09-13T14:35:04.731476+0000 mgr.y (mgr.26252) 152 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:35:06.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:06 smithi177 bash[68594]: cephadm 2024-09-13T14:35:04.731476+0000 mgr.y (mgr.26252) 152 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:35:06.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:06 smithi177 bash[68594]: cephadm 2024-09-13T14:35:04.738477+0000 mgr.y (mgr.26252) 153 : cephadm [INF] Reconfiguring daemon osd.6 on smithi177 2024-09-13T14:35:06.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:06 smithi177 bash[68594]: cephadm 2024-09-13T14:35:04.738477+0000 mgr.y (mgr.26252) 153 : cephadm [INF] Reconfiguring daemon osd.6 on smithi177 2024-09-13T14:35:06.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:06 smithi177 bash[68594]: cluster 2024-09-13T14:35:05.094849+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:35:06.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:06 smithi177 bash[68594]: cluster 2024-09-13T14:35:05.094849+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:35:06.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:06 smithi177 bash[68594]: cluster 2024-09-13T14:35:06.012543+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:35:06.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:06 smithi177 bash[68594]: cluster 2024-09-13T14:35:06.012543+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:35:06.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[87147]: cephadm 2024-09-13T14:35:04.731476+0000 mgr.y (mgr.26252) 152 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[87147]: cephadm 2024-09-13T14:35:04.731476+0000 mgr.y (mgr.26252) 152 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[87147]: cephadm 2024-09-13T14:35:04.738477+0000 mgr.y (mgr.26252) 153 : cephadm [INF] Reconfiguring daemon osd.6 on smithi177 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[87147]: cephadm 2024-09-13T14:35:04.738477+0000 mgr.y (mgr.26252) 153 : cephadm [INF] Reconfiguring daemon osd.6 on smithi177 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[87147]: cluster 2024-09-13T14:35:05.094849+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[87147]: cluster 2024-09-13T14:35:05.094849+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[87147]: cluster 2024-09-13T14:35:06.012543+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[87147]: cluster 2024-09-13T14:35:06.012543+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[84576]: cephadm 2024-09-13T14:35:04.731476+0000 mgr.y (mgr.26252) 152 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[84576]: cephadm 2024-09-13T14:35:04.731476+0000 mgr.y (mgr.26252) 152 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[84576]: cephadm 2024-09-13T14:35:04.738477+0000 mgr.y (mgr.26252) 153 : cephadm [INF] Reconfiguring daemon osd.6 on smithi177 2024-09-13T14:35:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[84576]: cephadm 2024-09-13T14:35:04.738477+0000 mgr.y (mgr.26252) 153 : cephadm [INF] Reconfiguring daemon osd.6 on smithi177 2024-09-13T14:35:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[84576]: cluster 2024-09-13T14:35:05.094849+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:35:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[84576]: cluster 2024-09-13T14:35:05.094849+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-09-13T14:35:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[84576]: cluster 2024-09-13T14:35:06.012543+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:35:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:06 smithi099 bash[84576]: cluster 2024-09-13T14:35:06.012543+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-09-13T14:35:07.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:35:07] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-09-13T14:35:07.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:07 smithi099 bash[84576]: cluster 2024-09-13T14:35:05.987153+0000 mgr.y (mgr.26252) 154 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:07.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:07 smithi099 bash[84576]: cluster 2024-09-13T14:35:05.987153+0000 mgr.y (mgr.26252) 154 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:07.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:07 smithi099 bash[84576]: cluster 2024-09-13T14:35:07.013745+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:35:07.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:07 smithi099 bash[84576]: cluster 2024-09-13T14:35:07.013745+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:35:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:07 smithi099 bash[87147]: cluster 2024-09-13T14:35:05.987153+0000 mgr.y (mgr.26252) 154 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:07 smithi099 bash[87147]: cluster 2024-09-13T14:35:05.987153+0000 mgr.y (mgr.26252) 154 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:07.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:07 smithi099 bash[87147]: cluster 2024-09-13T14:35:07.013745+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:35:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:07 smithi099 bash[87147]: cluster 2024-09-13T14:35:07.013745+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:35:07.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:07 smithi177 bash[68594]: cluster 2024-09-13T14:35:05.987153+0000 mgr.y (mgr.26252) 154 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:07.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:07 smithi177 bash[68594]: cluster 2024-09-13T14:35:05.987153+0000 mgr.y (mgr.26252) 154 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-13T14:35:07.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:07 smithi177 bash[68594]: cluster 2024-09-13T14:35:07.013745+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:35:07.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:07 smithi177 bash[68594]: cluster 2024-09-13T14:35:07.013745+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-09-13T14:35:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: audit 2024-09-13T14:35:07.464476+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: audit 2024-09-13T14:35:07.464476+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: audit 2024-09-13T14:35:07.475141+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: audit 2024-09-13T14:35:07.475141+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: cephadm 2024-09-13T14:35:07.483812+0000 mgr.y (mgr.26252) 155 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:35:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: cephadm 2024-09-13T14:35:07.483812+0000 mgr.y (mgr.26252) 155 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:35:08.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: audit 2024-09-13T14:35:07.484442+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: audit 2024-09-13T14:35:07.484442+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: audit 2024-09-13T14:35:07.486076+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: audit 2024-09-13T14:35:07.486076+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: cephadm 2024-09-13T14:35:07.491005+0000 mgr.y (mgr.26252) 156 : cephadm [INF] Reconfiguring daemon osd.7 on smithi177 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: cephadm 2024-09-13T14:35:07.491005+0000 mgr.y (mgr.26252) 156 : cephadm [INF] Reconfiguring daemon osd.7 on smithi177 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: cluster 2024-09-13T14:35:08.109839+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[84576]: cluster 2024-09-13T14:35:08.109839+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: audit 2024-09-13T14:35:07.464476+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: audit 2024-09-13T14:35:07.464476+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: audit 2024-09-13T14:35:07.475141+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: audit 2024-09-13T14:35:07.475141+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: cephadm 2024-09-13T14:35:07.483812+0000 mgr.y (mgr.26252) 155 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:35:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: cephadm 2024-09-13T14:35:07.483812+0000 mgr.y (mgr.26252) 155 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:35:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: audit 2024-09-13T14:35:07.484442+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:35:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: audit 2024-09-13T14:35:07.484442+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:35:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: audit 2024-09-13T14:35:07.486076+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: audit 2024-09-13T14:35:07.486076+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: cephadm 2024-09-13T14:35:07.491005+0000 mgr.y (mgr.26252) 156 : cephadm [INF] Reconfiguring daemon osd.7 on smithi177 2024-09-13T14:35:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: cephadm 2024-09-13T14:35:07.491005+0000 mgr.y (mgr.26252) 156 : cephadm [INF] Reconfiguring daemon osd.7 on smithi177 2024-09-13T14:35:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: cluster 2024-09-13T14:35:08.109839+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:35:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:08 smithi099 bash[87147]: cluster 2024-09-13T14:35:08.109839+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:35:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: audit 2024-09-13T14:35:07.464476+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: audit 2024-09-13T14:35:07.464476+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: audit 2024-09-13T14:35:07.475141+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: audit 2024-09-13T14:35:07.475141+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: cephadm 2024-09-13T14:35:07.483812+0000 mgr.y (mgr.26252) 155 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:35:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: cephadm 2024-09-13T14:35:07.483812+0000 mgr.y (mgr.26252) 155 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:35:08.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: audit 2024-09-13T14:35:07.484442+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:35:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: audit 2024-09-13T14:35:07.484442+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:35:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: audit 2024-09-13T14:35:07.486076+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: audit 2024-09-13T14:35:07.486076+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: cephadm 2024-09-13T14:35:07.491005+0000 mgr.y (mgr.26252) 156 : cephadm [INF] Reconfiguring daemon osd.7 on smithi177 2024-09-13T14:35:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: cephadm 2024-09-13T14:35:07.491005+0000 mgr.y (mgr.26252) 156 : cephadm [INF] Reconfiguring daemon osd.7 on smithi177 2024-09-13T14:35:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: cluster 2024-09-13T14:35:08.109839+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:35:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:08 smithi177 bash[68594]: cluster 2024-09-13T14:35:08.109839+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-09-13T14:35:09.226 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestMirroring.AioSnapshotCreate (31032 ms) 2024-09-13T14:35:09.656 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:09 smithi177 bash[68594]: cluster 2024-09-13T14:35:07.987920+0000 mgr.y (mgr.26252) 157 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:09.656 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:09 smithi177 bash[68594]: cluster 2024-09-13T14:35:07.987920+0000 mgr.y (mgr.26252) 157 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:09.656 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:09 smithi177 bash[68594]: cluster 2024-09-13T14:35:09.112026+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-09-13T14:35:09.657 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:09 smithi177 bash[68594]: cluster 2024-09-13T14:35:09.112026+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-09-13T14:35:09.657 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:09 smithi177 bash[68594]: audit 2024-09-13T14:35:09.210787+0000 mon.a (mon.0) 490 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"} : dispatch 2024-09-13T14:35:09.657 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:09 smithi177 bash[68594]: audit 2024-09-13T14:35:09.210787+0000 mon.a (mon.0) 490 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"} : dispatch 2024-09-13T14:35:09.657 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:09 smithi177 bash[68594]: audit 2024-09-13T14:35:09.219128+0000 mon.a (mon.0) 491 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"}]': finished 2024-09-13T14:35:09.657 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:09 smithi177 bash[68594]: audit 2024-09-13T14:35:09.219128+0000 mon.a (mon.0) 491 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"}]': finished 2024-09-13T14:35:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[84576]: cluster 2024-09-13T14:35:07.987920+0000 mgr.y (mgr.26252) 157 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[84576]: cluster 2024-09-13T14:35:07.987920+0000 mgr.y (mgr.26252) 157 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[84576]: cluster 2024-09-13T14:35:09.112026+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-09-13T14:35:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[84576]: cluster 2024-09-13T14:35:09.112026+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-09-13T14:35:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[84576]: audit 2024-09-13T14:35:09.210787+0000 mon.a (mon.0) 490 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"} : dispatch 2024-09-13T14:35:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[84576]: audit 2024-09-13T14:35:09.210787+0000 mon.a (mon.0) 490 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"} : dispatch 2024-09-13T14:35:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[84576]: audit 2024-09-13T14:35:09.219128+0000 mon.a (mon.0) 491 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"}]': finished 2024-09-13T14:35:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[84576]: audit 2024-09-13T14:35:09.219128+0000 mon.a (mon.0) 491 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"}]': finished 2024-09-13T14:35:09.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[87147]: cluster 2024-09-13T14:35:07.987920+0000 mgr.y (mgr.26252) 157 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[87147]: cluster 2024-09-13T14:35:07.987920+0000 mgr.y (mgr.26252) 157 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[87147]: cluster 2024-09-13T14:35:09.112026+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-09-13T14:35:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[87147]: cluster 2024-09-13T14:35:09.112026+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-09-13T14:35:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[87147]: audit 2024-09-13T14:35:09.210787+0000 mon.a (mon.0) 490 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"} : dispatch 2024-09-13T14:35:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[87147]: audit 2024-09-13T14:35:09.210787+0000 mon.a (mon.0) 490 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"} : dispatch 2024-09-13T14:35:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[87147]: audit 2024-09-13T14:35:09.219128+0000 mon.a (mon.0) 491 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"}]': finished 2024-09-13T14:35:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:09 smithi099 bash[87147]: audit 2024-09-13T14:35:09.219128+0000 mon.a (mon.0) 491 : audit [INF] from='client.26348 172.21.15.99:0/3289303752' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/98de6bf8-08f9-43b8-8ee2-6ef73c3b243a"}]': finished 2024-09-13T14:35:10.233 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 51 tests from TestMirroring (105489 ms total) 2024-09-13T14:35:10.234 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:35:10.234 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 34 tests from TestInternal 2024-09-13T14:35:10.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:35:10 smithi177 bash[61439]: ts=2024-09-13T14:35:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:11.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: cluster 2024-09-13T14:35:09.988785+0000 mgr.y (mgr.26252) 158 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:11.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: cluster 2024-09-13T14:35:09.988785+0000 mgr.y (mgr.26252) 158 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:11.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: cluster 2024-09-13T14:35:10.226646+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-09-13T14:35:11.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: cluster 2024-09-13T14:35:10.226646+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-09-13T14:35:11.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: audit 2024-09-13T14:35:10.247827+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: audit 2024-09-13T14:35:10.247827+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: audit 2024-09-13T14:35:10.263494+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: audit 2024-09-13T14:35:10.263494+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: cephadm 2024-09-13T14:35:10.267963+0000 mgr.y (mgr.26252) 159 : cephadm [INF] Reconfiguring mds.foo.smithi177.outtkv (monmap changed)... 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: cephadm 2024-09-13T14:35:10.267963+0000 mgr.y (mgr.26252) 159 : cephadm [INF] Reconfiguring mds.foo.smithi177.outtkv (monmap changed)... 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: audit 2024-09-13T14:35:10.268643+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: audit 2024-09-13T14:35:10.268643+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: audit 2024-09-13T14:35:10.270450+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: audit 2024-09-13T14:35:10.270450+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: cephadm 2024-09-13T14:35:10.272252+0000 mgr.y (mgr.26252) 160 : cephadm [INF] Reconfiguring daemon mds.foo.smithi177.outtkv on smithi177 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[84576]: cephadm 2024-09-13T14:35:10.272252+0000 mgr.y (mgr.26252) 160 : cephadm [INF] Reconfiguring daemon mds.foo.smithi177.outtkv on smithi177 2024-09-13T14:35:11.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: cluster 2024-09-13T14:35:09.988785+0000 mgr.y (mgr.26252) 158 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: cluster 2024-09-13T14:35:09.988785+0000 mgr.y (mgr.26252) 158 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: cluster 2024-09-13T14:35:10.226646+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: cluster 2024-09-13T14:35:10.226646+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: audit 2024-09-13T14:35:10.247827+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: audit 2024-09-13T14:35:10.247827+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: audit 2024-09-13T14:35:10.263494+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: audit 2024-09-13T14:35:10.263494+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: cephadm 2024-09-13T14:35:10.267963+0000 mgr.y (mgr.26252) 159 : cephadm [INF] Reconfiguring mds.foo.smithi177.outtkv (monmap changed)... 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: cephadm 2024-09-13T14:35:10.267963+0000 mgr.y (mgr.26252) 159 : cephadm [INF] Reconfiguring mds.foo.smithi177.outtkv (monmap changed)... 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: audit 2024-09-13T14:35:10.268643+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: audit 2024-09-13T14:35:10.268643+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: audit 2024-09-13T14:35:10.270450+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: audit 2024-09-13T14:35:10.270450+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:11.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: cephadm 2024-09-13T14:35:10.272252+0000 mgr.y (mgr.26252) 160 : cephadm [INF] Reconfiguring daemon mds.foo.smithi177.outtkv on smithi177 2024-09-13T14:35:11.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:11 smithi099 bash[87147]: cephadm 2024-09-13T14:35:10.272252+0000 mgr.y (mgr.26252) 160 : cephadm [INF] Reconfiguring daemon mds.foo.smithi177.outtkv on smithi177 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: cluster 2024-09-13T14:35:09.988785+0000 mgr.y (mgr.26252) 158 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: cluster 2024-09-13T14:35:09.988785+0000 mgr.y (mgr.26252) 158 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: cluster 2024-09-13T14:35:10.226646+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: cluster 2024-09-13T14:35:10.226646+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: audit 2024-09-13T14:35:10.247827+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: audit 2024-09-13T14:35:10.247827+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: audit 2024-09-13T14:35:10.263494+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: audit 2024-09-13T14:35:10.263494+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: cephadm 2024-09-13T14:35:10.267963+0000 mgr.y (mgr.26252) 159 : cephadm [INF] Reconfiguring mds.foo.smithi177.outtkv (monmap changed)... 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: cephadm 2024-09-13T14:35:10.267963+0000 mgr.y (mgr.26252) 159 : cephadm [INF] Reconfiguring mds.foo.smithi177.outtkv (monmap changed)... 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: audit 2024-09-13T14:35:10.268643+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:35:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: audit 2024-09-13T14:35:10.268643+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.outtkv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:35:11.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: audit 2024-09-13T14:35:10.270450+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:11.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: audit 2024-09-13T14:35:10.270450+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:11.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: cephadm 2024-09-13T14:35:10.272252+0000 mgr.y (mgr.26252) 160 : cephadm [INF] Reconfiguring daemon mds.foo.smithi177.outtkv on smithi177 2024-09-13T14:35:11.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:11 smithi177 bash[68594]: cephadm 2024-09-13T14:35:10.272252+0000 mgr.y (mgr.26252) 160 : cephadm [INF] Reconfiguring daemon mds.foo.smithi177.outtkv on smithi177 2024-09-13T14:35:12.284 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.OpenByID 2024-09-13T14:35:12.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:12 smithi099 bash[84576]: cluster 2024-09-13T14:35:11.271035+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-09-13T14:35:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:12 smithi099 bash[84576]: cluster 2024-09-13T14:35:11.271035+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-09-13T14:35:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:12 smithi099 bash[84576]: audit 2024-09-13T14:35:11.274764+0000 mon.a (mon.0) 498 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:12 smithi099 bash[84576]: audit 2024-09-13T14:35:11.274764+0000 mon.a (mon.0) 498 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:12.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:12 smithi099 bash[87147]: cluster 2024-09-13T14:35:11.271035+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-09-13T14:35:12.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:12 smithi099 bash[87147]: cluster 2024-09-13T14:35:11.271035+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-09-13T14:35:12.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:12 smithi099 bash[87147]: audit 2024-09-13T14:35:11.274764+0000 mon.a (mon.0) 498 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:12.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:12 smithi099 bash[87147]: audit 2024-09-13T14:35:11.274764+0000 mon.a (mon.0) 498 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:12 smithi177 bash[68594]: cluster 2024-09-13T14:35:11.271035+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-09-13T14:35:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:12 smithi177 bash[68594]: cluster 2024-09-13T14:35:11.271035+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-09-13T14:35:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:12 smithi177 bash[68594]: audit 2024-09-13T14:35:11.274764+0000 mon.a (mon.0) 498 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:12 smithi177 bash[68594]: audit 2024-09-13T14:35:11.274764+0000 mon.a (mon.0) 498 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: cluster 2024-09-13T14:35:11.989655+0000 mgr.y (mgr.26252) 161 : cluster [DBG] pgmap v280: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: cluster 2024-09-13T14:35:11.989655+0000 mgr.y (mgr.26252) 161 : cluster [DBG] pgmap v280: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: cluster 2024-09-13T14:35:12.260217+0000 mon.a (mon.0) 499 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: cluster 2024-09-13T14:35:12.260217+0000 mon.a (mon.0) 499 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: audit 2024-09-13T14:35:12.269514+0000 mon.a (mon.0) 500 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: audit 2024-09-13T14:35:12.269514+0000 mon.a (mon.0) 500 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: cluster 2024-09-13T14:35:12.279530+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: cluster 2024-09-13T14:35:12.279530+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: audit 2024-09-13T14:35:12.701286+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: audit 2024-09-13T14:35:12.701286+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: audit 2024-09-13T14:35:12.714183+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: audit 2024-09-13T14:35:12.714183+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: audit 2024-09-13T14:35:12.803864+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[84576]: audit 2024-09-13T14:35:12.803864+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: cluster 2024-09-13T14:35:11.989655+0000 mgr.y (mgr.26252) 161 : cluster [DBG] pgmap v280: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: cluster 2024-09-13T14:35:11.989655+0000 mgr.y (mgr.26252) 161 : cluster [DBG] pgmap v280: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: cluster 2024-09-13T14:35:12.260217+0000 mon.a (mon.0) 499 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: cluster 2024-09-13T14:35:12.260217+0000 mon.a (mon.0) 499 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:13.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: audit 2024-09-13T14:35:12.269514+0000 mon.a (mon.0) 500 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:13.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: audit 2024-09-13T14:35:12.269514+0000 mon.a (mon.0) 500 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:13.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: cluster 2024-09-13T14:35:12.279530+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-13T14:35:13.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: cluster 2024-09-13T14:35:12.279530+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-13T14:35:13.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: audit 2024-09-13T14:35:12.701286+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: audit 2024-09-13T14:35:12.701286+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: audit 2024-09-13T14:35:12.714183+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: audit 2024-09-13T14:35:12.714183+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: audit 2024-09-13T14:35:12.803864+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:13.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:13 smithi099 bash[87147]: audit 2024-09-13T14:35:12.803864+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:13.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: cluster 2024-09-13T14:35:11.989655+0000 mgr.y (mgr.26252) 161 : cluster [DBG] pgmap v280: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:13.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: cluster 2024-09-13T14:35:11.989655+0000 mgr.y (mgr.26252) 161 : cluster [DBG] pgmap v280: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:13.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: cluster 2024-09-13T14:35:12.260217+0000 mon.a (mon.0) 499 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:13.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: cluster 2024-09-13T14:35:12.260217+0000 mon.a (mon.0) 499 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:13.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: audit 2024-09-13T14:35:12.269514+0000 mon.a (mon.0) 500 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:13.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: audit 2024-09-13T14:35:12.269514+0000 mon.a (mon.0) 500 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:13.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: cluster 2024-09-13T14:35:12.279530+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-13T14:35:13.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: cluster 2024-09-13T14:35:12.279530+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-13T14:35:13.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: audit 2024-09-13T14:35:12.701286+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: audit 2024-09-13T14:35:12.701286+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: audit 2024-09-13T14:35:12.714183+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: audit 2024-09-13T14:35:12.714183+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:13.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: audit 2024-09-13T14:35:12.803864+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:13.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:13 smithi177 bash[68594]: audit 2024-09-13T14:35:12.803864+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:14.358 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.OpenByID (2074 ms) 2024-09-13T14:35:14.358 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.OpenSnapDNE 2024-09-13T14:35:14.395 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.OpenSnapDNE (37 ms) 2024-09-13T14:35:14.396 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.IsExclusiveLockOwner 2024-09-13T14:35:14.435 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.IsExclusiveLockOwner (40 ms) 2024-09-13T14:35:14.435 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.ResizeLocksImage 2024-09-13T14:35:14.491 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.ResizeLocksImage (56 ms) 2024-09-13T14:35:14.491 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.ResizeFailsToLockImage 2024-09-13T14:35:14.536 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.ResizeFailsToLockImage (45 ms) 2024-09-13T14:35:14.536 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.SnapCreateLocksImage 2024-09-13T14:35:14.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:14 smithi099 bash[84576]: cluster 2024-09-13T14:35:13.292046+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-13T14:35:14.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:14 smithi099 bash[84576]: cluster 2024-09-13T14:35:13.292046+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-13T14:35:14.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:14 smithi099 bash[87147]: cluster 2024-09-13T14:35:13.292046+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-13T14:35:14.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:14 smithi099 bash[87147]: cluster 2024-09-13T14:35:13.292046+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-13T14:35:14.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:14 smithi177 bash[68594]: cluster 2024-09-13T14:35:13.292046+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-13T14:35:14.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:14 smithi177 bash[68594]: cluster 2024-09-13T14:35:13.292046+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-13T14:35:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[84576]: cluster 2024-09-13T14:35:13.990871+0000 mgr.y (mgr.26252) 162 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[84576]: cluster 2024-09-13T14:35:13.990871+0000 mgr.y (mgr.26252) 162 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[84576]: cluster 2024-09-13T14:35:14.282818+0000 mon.a (mon.0) 506 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[84576]: cluster 2024-09-13T14:35:14.282818+0000 mon.a (mon.0) 506 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[84576]: cluster 2024-09-13T14:35:14.282865+0000 mon.a (mon.0) 507 : cluster [INF] Cluster is now healthy 2024-09-13T14:35:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[84576]: cluster 2024-09-13T14:35:14.282865+0000 mon.a (mon.0) 507 : cluster [INF] Cluster is now healthy 2024-09-13T14:35:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[84576]: cluster 2024-09-13T14:35:14.307324+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-13T14:35:15.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[84576]: cluster 2024-09-13T14:35:14.307324+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-13T14:35:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[87147]: cluster 2024-09-13T14:35:13.990871+0000 mgr.y (mgr.26252) 162 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[87147]: cluster 2024-09-13T14:35:13.990871+0000 mgr.y (mgr.26252) 162 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[87147]: cluster 2024-09-13T14:35:14.282818+0000 mon.a (mon.0) 506 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[87147]: cluster 2024-09-13T14:35:14.282818+0000 mon.a (mon.0) 506 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[87147]: cluster 2024-09-13T14:35:14.282865+0000 mon.a (mon.0) 507 : cluster [INF] Cluster is now healthy 2024-09-13T14:35:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[87147]: cluster 2024-09-13T14:35:14.282865+0000 mon.a (mon.0) 507 : cluster [INF] Cluster is now healthy 2024-09-13T14:35:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[87147]: cluster 2024-09-13T14:35:14.307324+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-13T14:35:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:15 smithi099 bash[87147]: cluster 2024-09-13T14:35:14.307324+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-13T14:35:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:15 smithi177 bash[68594]: cluster 2024-09-13T14:35:13.990871+0000 mgr.y (mgr.26252) 162 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:15 smithi177 bash[68594]: cluster 2024-09-13T14:35:13.990871+0000 mgr.y (mgr.26252) 162 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:15 smithi177 bash[68594]: cluster 2024-09-13T14:35:14.282818+0000 mon.a (mon.0) 506 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:15 smithi177 bash[68594]: cluster 2024-09-13T14:35:14.282818+0000 mon.a (mon.0) 506 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:15 smithi177 bash[68594]: cluster 2024-09-13T14:35:14.282865+0000 mon.a (mon.0) 507 : cluster [INF] Cluster is now healthy 2024-09-13T14:35:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:15 smithi177 bash[68594]: cluster 2024-09-13T14:35:14.282865+0000 mon.a (mon.0) 507 : cluster [INF] Cluster is now healthy 2024-09-13T14:35:15.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:15 smithi177 bash[68594]: cluster 2024-09-13T14:35:14.307324+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-13T14:35:15.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:15 smithi177 bash[68594]: cluster 2024-09-13T14:35:14.307324+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-13T14:35:16.345 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.SnapCreateLocksImage (1809 ms) 2024-09-13T14:35:16.345 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.SnapCreateFailsToLockImage 2024-09-13T14:35:16.400 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.SnapCreateFailsToLockImage (55 ms) 2024-09-13T14:35:16.400 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.SnapRollbackLocksImage 2024-09-13T14:35:16.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:16 smithi099 bash[84576]: cluster 2024-09-13T14:35:15.313428+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-09-13T14:35:16.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:16 smithi099 bash[84576]: cluster 2024-09-13T14:35:15.313428+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-09-13T14:35:16.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:16 smithi099 bash[87147]: cluster 2024-09-13T14:35:15.313428+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-09-13T14:35:16.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:16 smithi099 bash[87147]: cluster 2024-09-13T14:35:15.313428+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-09-13T14:35:16.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:16 smithi177 bash[68594]: cluster 2024-09-13T14:35:15.313428+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-09-13T14:35:16.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:16 smithi177 bash[68594]: cluster 2024-09-13T14:35:15.313428+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-09-13T14:35:17.316 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:35:17] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-09-13T14:35:17.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:17 smithi099 bash[84576]: cluster 2024-09-13T14:35:15.991541+0000 mgr.y (mgr.26252) 163 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:35:17.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:17 smithi099 bash[84576]: cluster 2024-09-13T14:35:15.991541+0000 mgr.y (mgr.26252) 163 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:35:17.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:17 smithi099 bash[84576]: cluster 2024-09-13T14:35:16.326770+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-09-13T14:35:17.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:17 smithi099 bash[84576]: cluster 2024-09-13T14:35:16.326770+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-09-13T14:35:17.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:17 smithi099 bash[87147]: cluster 2024-09-13T14:35:15.991541+0000 mgr.y (mgr.26252) 163 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:35:17.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:17 smithi099 bash[87147]: cluster 2024-09-13T14:35:15.991541+0000 mgr.y (mgr.26252) 163 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:35:17.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:17 smithi099 bash[87147]: cluster 2024-09-13T14:35:16.326770+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-09-13T14:35:17.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:17 smithi099 bash[87147]: cluster 2024-09-13T14:35:16.326770+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-09-13T14:35:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:17 smithi177 bash[68594]: cluster 2024-09-13T14:35:15.991541+0000 mgr.y (mgr.26252) 163 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:35:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:17 smithi177 bash[68594]: cluster 2024-09-13T14:35:15.991541+0000 mgr.y (mgr.26252) 163 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T14:35:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:17 smithi177 bash[68594]: cluster 2024-09-13T14:35:16.326770+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-09-13T14:35:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:17 smithi177 bash[68594]: cluster 2024-09-13T14:35:16.326770+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-09-13T14:35:18.360 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.SnapRollbackLocksImage (1960 ms) 2024-09-13T14:35:18.360 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.SnapRollbackFailsToLockImage 2024-09-13T14:35:18.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:18 smithi177 bash[68594]: cluster 2024-09-13T14:35:17.333162+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2024-09-13T14:35:18.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:18 smithi177 bash[68594]: cluster 2024-09-13T14:35:17.333162+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2024-09-13T14:35:18.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:18 smithi177 bash[68594]: audit 2024-09-13T14:35:18.244567+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:18.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:18 smithi177 bash[68594]: audit 2024-09-13T14:35:18.244567+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:18.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:18 smithi099 bash[84576]: cluster 2024-09-13T14:35:17.333162+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2024-09-13T14:35:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:18 smithi099 bash[84576]: cluster 2024-09-13T14:35:17.333162+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2024-09-13T14:35:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:18 smithi099 bash[84576]: audit 2024-09-13T14:35:18.244567+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:18 smithi099 bash[84576]: audit 2024-09-13T14:35:18.244567+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:18 smithi099 bash[87147]: cluster 2024-09-13T14:35:17.333162+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2024-09-13T14:35:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:18 smithi099 bash[87147]: cluster 2024-09-13T14:35:17.333162+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2024-09-13T14:35:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:18 smithi099 bash[87147]: audit 2024-09-13T14:35:18.244567+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:18 smithi099 bash[87147]: audit 2024-09-13T14:35:18.244567+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:19.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: cluster 2024-09-13T14:35:17.991976+0000 mgr.y (mgr.26252) 164 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:19.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: cluster 2024-09-13T14:35:17.991976+0000 mgr.y (mgr.26252) 164 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:19.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: cluster 2024-09-13T14:35:18.343544+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2024-09-13T14:35:19.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: cluster 2024-09-13T14:35:18.343544+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2024-09-13T14:35:19.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: audit 2024-09-13T14:35:18.620994+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: audit 2024-09-13T14:35:18.620994+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: audit 2024-09-13T14:35:18.626507+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: audit 2024-09-13T14:35:18.626507+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: audit 2024-09-13T14:35:19.254015+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:19.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: audit 2024-09-13T14:35:19.254015+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:19.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: audit 2024-09-13T14:35:19.255270+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:19.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: audit 2024-09-13T14:35:19.255270+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:19.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: audit 2024-09-13T14:35:19.264160+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:19 smithi177 bash[68594]: audit 2024-09-13T14:35:19.264160+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: cluster 2024-09-13T14:35:17.991976+0000 mgr.y (mgr.26252) 164 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: cluster 2024-09-13T14:35:17.991976+0000 mgr.y (mgr.26252) 164 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: cluster 2024-09-13T14:35:18.343544+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: cluster 2024-09-13T14:35:18.343544+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: audit 2024-09-13T14:35:18.620994+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: audit 2024-09-13T14:35:18.620994+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: audit 2024-09-13T14:35:18.626507+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: audit 2024-09-13T14:35:18.626507+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: audit 2024-09-13T14:35:19.254015+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: audit 2024-09-13T14:35:19.254015+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: audit 2024-09-13T14:35:19.255270+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: audit 2024-09-13T14:35:19.255270+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: audit 2024-09-13T14:35:19.264160+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[84576]: audit 2024-09-13T14:35:19.264160+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: cluster 2024-09-13T14:35:17.991976+0000 mgr.y (mgr.26252) 164 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: cluster 2024-09-13T14:35:17.991976+0000 mgr.y (mgr.26252) 164 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: cluster 2024-09-13T14:35:18.343544+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: cluster 2024-09-13T14:35:18.343544+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: audit 2024-09-13T14:35:18.620994+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: audit 2024-09-13T14:35:18.620994+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: audit 2024-09-13T14:35:18.626507+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: audit 2024-09-13T14:35:18.626507+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: audit 2024-09-13T14:35:19.254015+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: audit 2024-09-13T14:35:19.254015+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: audit 2024-09-13T14:35:19.255270+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: audit 2024-09-13T14:35:19.255270+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:19.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: audit 2024-09-13T14:35:19.264160+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:19.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:19 smithi099 bash[87147]: audit 2024-09-13T14:35:19.264160+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:20.396 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.SnapRollbackFailsToLockImage (2036 ms) 2024-09-13T14:35:20.396 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.SnapSetReleasesLock 2024-09-13T14:35:20.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:35:20 smithi177 bash[61439]: ts=2024-09-13T14:35:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:20 smithi177 bash[68594]: cluster 2024-09-13T14:35:19.346665+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2024-09-13T14:35:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:20 smithi177 bash[68594]: cluster 2024-09-13T14:35:19.346665+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2024-09-13T14:35:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:20 smithi099 bash[84576]: cluster 2024-09-13T14:35:19.346665+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2024-09-13T14:35:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:20 smithi099 bash[84576]: cluster 2024-09-13T14:35:19.346665+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2024-09-13T14:35:20.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:20 smithi099 bash[87147]: cluster 2024-09-13T14:35:19.346665+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2024-09-13T14:35:20.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:20 smithi099 bash[87147]: cluster 2024-09-13T14:35:19.346665+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2024-09-13T14:35:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[84576]: cluster 2024-09-13T14:35:19.992643+0000 mgr.y (mgr.26252) 165 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[84576]: cluster 2024-09-13T14:35:19.992643+0000 mgr.y (mgr.26252) 165 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[84576]: cluster 2024-09-13T14:35:20.371186+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2024-09-13T14:35:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[84576]: cluster 2024-09-13T14:35:20.371186+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2024-09-13T14:35:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[84576]: cluster 2024-09-13T14:35:21.015410+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2024-09-13T14:35:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[84576]: cluster 2024-09-13T14:35:21.015410+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2024-09-13T14:35:21.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[87147]: cluster 2024-09-13T14:35:19.992643+0000 mgr.y (mgr.26252) 165 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[87147]: cluster 2024-09-13T14:35:19.992643+0000 mgr.y (mgr.26252) 165 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[87147]: cluster 2024-09-13T14:35:20.371186+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2024-09-13T14:35:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[87147]: cluster 2024-09-13T14:35:20.371186+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2024-09-13T14:35:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[87147]: cluster 2024-09-13T14:35:21.015410+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2024-09-13T14:35:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:21 smithi099 bash[87147]: cluster 2024-09-13T14:35:21.015410+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2024-09-13T14:35:21.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:21 smithi177 bash[68594]: cluster 2024-09-13T14:35:19.992643+0000 mgr.y (mgr.26252) 165 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:21.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:21 smithi177 bash[68594]: cluster 2024-09-13T14:35:19.992643+0000 mgr.y (mgr.26252) 165 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:21.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:21 smithi177 bash[68594]: cluster 2024-09-13T14:35:20.371186+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2024-09-13T14:35:21.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:21 smithi177 bash[68594]: cluster 2024-09-13T14:35:20.371186+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2024-09-13T14:35:21.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:21 smithi177 bash[68594]: cluster 2024-09-13T14:35:21.015410+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2024-09-13T14:35:21.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:21 smithi177 bash[68594]: cluster 2024-09-13T14:35:21.015410+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2024-09-13T14:35:22.043 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.SnapSetReleasesLock (1647 ms) 2024-09-13T14:35:22.043 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.FlattenLocksImage 2024-09-13T14:35:22.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade resume' 2024-09-13T14:35:23.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:23 smithi099 bash[84576]: cluster 2024-09-13T14:35:21.993312+0000 mgr.y (mgr.26252) 166 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:23.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:23 smithi099 bash[84576]: cluster 2024-09-13T14:35:21.993312+0000 mgr.y (mgr.26252) 166 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:23.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:23 smithi099 bash[84576]: cluster 2024-09-13T14:35:22.019865+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e530: 8 total, 8 up, 8 in 2024-09-13T14:35:23.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:23 smithi099 bash[84576]: cluster 2024-09-13T14:35:22.019865+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e530: 8 total, 8 up, 8 in 2024-09-13T14:35:23.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:23 smithi099 bash[87147]: cluster 2024-09-13T14:35:21.993312+0000 mgr.y (mgr.26252) 166 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:23.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:23 smithi099 bash[87147]: cluster 2024-09-13T14:35:21.993312+0000 mgr.y (mgr.26252) 166 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:23.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:23 smithi099 bash[87147]: cluster 2024-09-13T14:35:22.019865+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e530: 8 total, 8 up, 8 in 2024-09-13T14:35:23.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:23 smithi099 bash[87147]: cluster 2024-09-13T14:35:22.019865+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e530: 8 total, 8 up, 8 in 2024-09-13T14:35:23.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:23 smithi177 bash[68594]: cluster 2024-09-13T14:35:21.993312+0000 mgr.y (mgr.26252) 166 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:23.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:23 smithi177 bash[68594]: cluster 2024-09-13T14:35:21.993312+0000 mgr.y (mgr.26252) 166 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-13T14:35:23.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:23 smithi177 bash[68594]: cluster 2024-09-13T14:35:22.019865+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e530: 8 total, 8 up, 8 in 2024-09-13T14:35:23.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:23 smithi177 bash[68594]: cluster 2024-09-13T14:35:22.019865+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e530: 8 total, 8 up, 8 in 2024-09-13T14:35:24.092 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.FlattenLocksImage (2049 ms) 2024-09-13T14:35:24.092 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.FlattenFailsToLockImage 2024-09-13T14:35:24.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:24 smithi099 bash[87147]: cluster 2024-09-13T14:35:23.022329+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e531: 8 total, 8 up, 8 in 2024-09-13T14:35:24.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:24 smithi099 bash[87147]: cluster 2024-09-13T14:35:23.022329+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e531: 8 total, 8 up, 8 in 2024-09-13T14:35:24.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:24 smithi099 bash[87147]: audit 2024-09-13T14:35:23.311648+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:24.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:24 smithi099 bash[87147]: audit 2024-09-13T14:35:23.311648+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:24.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:24 smithi099 bash[84576]: cluster 2024-09-13T14:35:23.022329+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e531: 8 total, 8 up, 8 in 2024-09-13T14:35:24.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:24 smithi099 bash[84576]: cluster 2024-09-13T14:35:23.022329+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e531: 8 total, 8 up, 8 in 2024-09-13T14:35:24.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:24 smithi099 bash[84576]: audit 2024-09-13T14:35:23.311648+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:24.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:24 smithi099 bash[84576]: audit 2024-09-13T14:35:23.311648+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:24.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:24 smithi177 bash[68594]: cluster 2024-09-13T14:35:23.022329+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e531: 8 total, 8 up, 8 in 2024-09-13T14:35:24.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:24 smithi177 bash[68594]: cluster 2024-09-13T14:35:23.022329+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e531: 8 total, 8 up, 8 in 2024-09-13T14:35:24.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:24 smithi177 bash[68594]: audit 2024-09-13T14:35:23.311648+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:24.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:24 smithi177 bash[68594]: audit 2024-09-13T14:35:23.311648+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:25.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:25 smithi099 bash[84576]: cluster 2024-09-13T14:35:23.994624+0000 mgr.y (mgr.26252) 167 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:25.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:25 smithi099 bash[84576]: cluster 2024-09-13T14:35:23.994624+0000 mgr.y (mgr.26252) 167 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:25.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:25 smithi099 bash[84576]: cluster 2024-09-13T14:35:24.046873+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e532: 8 total, 8 up, 8 in 2024-09-13T14:35:25.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:25 smithi099 bash[84576]: cluster 2024-09-13T14:35:24.046873+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e532: 8 total, 8 up, 8 in 2024-09-13T14:35:25.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:25 smithi099 bash[87147]: cluster 2024-09-13T14:35:23.994624+0000 mgr.y (mgr.26252) 167 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:25.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:25 smithi099 bash[87147]: cluster 2024-09-13T14:35:23.994624+0000 mgr.y (mgr.26252) 167 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:25.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:25 smithi099 bash[87147]: cluster 2024-09-13T14:35:24.046873+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e532: 8 total, 8 up, 8 in 2024-09-13T14:35:25.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:25 smithi099 bash[87147]: cluster 2024-09-13T14:35:24.046873+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e532: 8 total, 8 up, 8 in 2024-09-13T14:35:25.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:25 smithi177 bash[68594]: cluster 2024-09-13T14:35:23.994624+0000 mgr.y (mgr.26252) 167 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:25 smithi177 bash[68594]: cluster 2024-09-13T14:35:23.994624+0000 mgr.y (mgr.26252) 167 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:25 smithi177 bash[68594]: cluster 2024-09-13T14:35:24.046873+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e532: 8 total, 8 up, 8 in 2024-09-13T14:35:25.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:25 smithi177 bash[68594]: cluster 2024-09-13T14:35:24.046873+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e532: 8 total, 8 up, 8 in 2024-09-13T14:35:26.059 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.FlattenFailsToLockImage (1967 ms) 2024-09-13T14:35:26.059 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklisted 2024-09-13T14:35:26.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:26 smithi099 bash[87147]: cluster 2024-09-13T14:35:25.053785+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e533: 8 total, 8 up, 8 in 2024-09-13T14:35:26.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:26 smithi099 bash[87147]: cluster 2024-09-13T14:35:25.053785+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e533: 8 total, 8 up, 8 in 2024-09-13T14:35:26.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:26 smithi099 bash[87147]: cluster 2024-09-13T14:35:26.017280+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e534: 8 total, 8 up, 8 in 2024-09-13T14:35:26.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:26 smithi099 bash[87147]: cluster 2024-09-13T14:35:26.017280+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e534: 8 total, 8 up, 8 in 2024-09-13T14:35:26.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:26 smithi099 bash[84576]: cluster 2024-09-13T14:35:25.053785+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e533: 8 total, 8 up, 8 in 2024-09-13T14:35:26.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:26 smithi099 bash[84576]: cluster 2024-09-13T14:35:25.053785+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e533: 8 total, 8 up, 8 in 2024-09-13T14:35:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:26 smithi099 bash[84576]: cluster 2024-09-13T14:35:26.017280+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e534: 8 total, 8 up, 8 in 2024-09-13T14:35:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:26 smithi099 bash[84576]: cluster 2024-09-13T14:35:26.017280+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e534: 8 total, 8 up, 8 in 2024-09-13T14:35:26.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:26 smithi177 bash[68594]: cluster 2024-09-13T14:35:25.053785+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e533: 8 total, 8 up, 8 in 2024-09-13T14:35:26.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:26 smithi177 bash[68594]: cluster 2024-09-13T14:35:25.053785+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e533: 8 total, 8 up, 8 in 2024-09-13T14:35:26.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:26 smithi177 bash[68594]: cluster 2024-09-13T14:35:26.017280+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e534: 8 total, 8 up, 8 in 2024-09-13T14:35:26.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:26 smithi177 bash[68594]: cluster 2024-09-13T14:35:26.017280+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e534: 8 total, 8 up, 8 in 2024-09-13T14:35:26.845 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:27.084 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklisted (1024 ms) 2024-09-13T14:35:27.084 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklistedWatch 2024-09-13T14:35:27.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:35:27] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.51.0" 2024-09-13T14:35:27.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[84576]: cluster 2024-09-13T14:35:25.995322+0000 mgr.y (mgr.26252) 168 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:27.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[84576]: cluster 2024-09-13T14:35:25.995322+0000 mgr.y (mgr.26252) 168 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:27.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[84576]: audit 2024-09-13T14:35:26.138710+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.99:0/3556003775' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[84576]: audit 2024-09-13T14:35:26.138710+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.99:0/3556003775' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[84576]: audit 2024-09-13T14:35:26.139180+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[84576]: audit 2024-09-13T14:35:26.139180+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[87147]: cluster 2024-09-13T14:35:25.995322+0000 mgr.y (mgr.26252) 168 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[87147]: cluster 2024-09-13T14:35:25.995322+0000 mgr.y (mgr.26252) 168 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[87147]: audit 2024-09-13T14:35:26.138710+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.99:0/3556003775' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[87147]: audit 2024-09-13T14:35:26.138710+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.99:0/3556003775' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[87147]: audit 2024-09-13T14:35:26.139180+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:27 smithi099 bash[87147]: audit 2024-09-13T14:35:26.139180+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:27 smithi177 bash[68594]: cluster 2024-09-13T14:35:25.995322+0000 mgr.y (mgr.26252) 168 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:27.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:27 smithi177 bash[68594]: cluster 2024-09-13T14:35:25.995322+0000 mgr.y (mgr.26252) 168 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-13T14:35:27.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:27 smithi177 bash[68594]: audit 2024-09-13T14:35:26.138710+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.99:0/3556003775' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:27 smithi177 bash[68594]: audit 2024-09-13T14:35:26.138710+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.99:0/3556003775' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:27 smithi177 bash[68594]: audit 2024-09-13T14:35:26.139180+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:27.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:27 smithi177 bash[68594]: audit 2024-09-13T14:35:26.139180+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"} : dispatch 2024-09-13T14:35:28.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:28 smithi177 bash[68594]: audit 2024-09-13T14:35:27.064711+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"}]': finished 2024-09-13T14:35:28.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:28 smithi177 bash[68594]: audit 2024-09-13T14:35:27.064711+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"}]': finished 2024-09-13T14:35:28.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:28 smithi177 bash[68594]: cluster 2024-09-13T14:35:27.074131+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e535: 8 total, 8 up, 8 in 2024-09-13T14:35:28.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:28 smithi177 bash[68594]: cluster 2024-09-13T14:35:27.074131+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e535: 8 total, 8 up, 8 in 2024-09-13T14:35:28.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:28 smithi177 bash[68594]: audit 2024-09-13T14:35:27.164806+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.99:0/313522629' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:28 smithi177 bash[68594]: audit 2024-09-13T14:35:27.164806+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.99:0/313522629' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:28 smithi177 bash[68594]: audit 2024-09-13T14:35:27.165187+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:28 smithi177 bash[68594]: audit 2024-09-13T14:35:27.165187+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.395 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[84576]: audit 2024-09-13T14:35:27.064711+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"}]': finished 2024-09-13T14:35:28.395 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[84576]: audit 2024-09-13T14:35:27.064711+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"}]': finished 2024-09-13T14:35:28.395 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[84576]: cluster 2024-09-13T14:35:27.074131+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e535: 8 total, 8 up, 8 in 2024-09-13T14:35:28.396 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[84576]: cluster 2024-09-13T14:35:27.074131+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e535: 8 total, 8 up, 8 in 2024-09-13T14:35:28.396 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[84576]: audit 2024-09-13T14:35:27.164806+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.99:0/313522629' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.396 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[84576]: audit 2024-09-13T14:35:27.164806+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.99:0/313522629' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.396 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[84576]: audit 2024-09-13T14:35:27.165187+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.396 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[84576]: audit 2024-09-13T14:35:27.165187+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[87147]: audit 2024-09-13T14:35:27.064711+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"}]': finished 2024-09-13T14:35:28.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[87147]: audit 2024-09-13T14:35:27.064711+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3556003775"}]': finished 2024-09-13T14:35:28.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[87147]: cluster 2024-09-13T14:35:27.074131+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e535: 8 total, 8 up, 8 in 2024-09-13T14:35:28.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[87147]: cluster 2024-09-13T14:35:27.074131+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e535: 8 total, 8 up, 8 in 2024-09-13T14:35:28.397 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[87147]: audit 2024-09-13T14:35:27.164806+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.99:0/313522629' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.397 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[87147]: audit 2024-09-13T14:35:27.164806+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.99:0/313522629' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.397 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[87147]: audit 2024-09-13T14:35:27.165187+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:28.397 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:28 smithi099 bash[87147]: audit 2024-09-13T14:35:27.165187+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"} : dispatch 2024-09-13T14:35:29.093 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklistedWatch (2010 ms) 2024-09-13T14:35:29.093 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.AioWriteRequestsLock 2024-09-13T14:35:29.155 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.AioWriteRequestsLock (62 ms) 2024-09-13T14:35:29.155 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.AioDiscardRequestsLock 2024-09-13T14:35:29.206 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.AioDiscardRequestsLock (50 ms) 2024-09-13T14:35:29.206 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.CancelAsyncResize 2024-09-13T14:35:29.308 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.CancelAsyncResize (103 ms) 2024-09-13T14:35:29.308 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.MultipleResize 2024-09-13T14:35:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:29 smithi177 bash[68594]: cluster 2024-09-13T14:35:27.996054+0000 mgr.y (mgr.26252) 169 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:29 smithi177 bash[68594]: cluster 2024-09-13T14:35:27.996054+0000 mgr.y (mgr.26252) 169 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:29 smithi177 bash[68594]: audit 2024-09-13T14:35:28.078711+0000 mon.a (mon.0) 532 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"}]': finished 2024-09-13T14:35:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:29 smithi177 bash[68594]: audit 2024-09-13T14:35:28.078711+0000 mon.a (mon.0) 532 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"}]': finished 2024-09-13T14:35:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:29 smithi177 bash[68594]: cluster 2024-09-13T14:35:28.091226+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e536: 8 total, 8 up, 8 in 2024-09-13T14:35:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:29 smithi177 bash[68594]: cluster 2024-09-13T14:35:28.091226+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e536: 8 total, 8 up, 8 in 2024-09-13T14:35:29.433 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.MultipleResize (125 ms) 2024-09-13T14:35:29.433 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.Metadata 2024-09-13T14:35:29.497 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.Metadata (64 ms) 2024-09-13T14:35:29.497 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.MetadataConfApply 2024-09-13T14:35:29.525 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[84576]: cluster 2024-09-13T14:35:27.996054+0000 mgr.y (mgr.26252) 169 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:29.526 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[84576]: cluster 2024-09-13T14:35:27.996054+0000 mgr.y (mgr.26252) 169 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:29.526 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[84576]: audit 2024-09-13T14:35:28.078711+0000 mon.a (mon.0) 532 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"}]': finished 2024-09-13T14:35:29.526 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[84576]: audit 2024-09-13T14:35:28.078711+0000 mon.a (mon.0) 532 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"}]': finished 2024-09-13T14:35:29.526 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[84576]: cluster 2024-09-13T14:35:28.091226+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e536: 8 total, 8 up, 8 in 2024-09-13T14:35:29.526 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[84576]: cluster 2024-09-13T14:35:28.091226+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e536: 8 total, 8 up, 8 in 2024-09-13T14:35:29.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[87147]: cluster 2024-09-13T14:35:27.996054+0000 mgr.y (mgr.26252) 169 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:29.527 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[87147]: cluster 2024-09-13T14:35:27.996054+0000 mgr.y (mgr.26252) 169 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:29.527 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[87147]: audit 2024-09-13T14:35:28.078711+0000 mon.a (mon.0) 532 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"}]': finished 2024-09-13T14:35:29.527 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[87147]: audit 2024-09-13T14:35:28.078711+0000 mon.a (mon.0) 532 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/313522629"}]': finished 2024-09-13T14:35:29.527 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[87147]: cluster 2024-09-13T14:35:28.091226+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e536: 8 total, 8 up, 8 in 2024-09-13T14:35:29.527 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:29 smithi099 bash[87147]: cluster 2024-09-13T14:35:28.091226+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e536: 8 total, 8 up, 8 in 2024-09-13T14:35:29.540 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.MetadataConfApply (43 ms) 2024-09-13T14:35:29.541 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.SnapshotCopyup 2024-09-13T14:35:30.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:35:30 smithi177 bash[61439]: ts=2024-09-13T14:35:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:31.193 INFO:teuthology.orchestra.run.smithi099.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:31.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:31 smithi177 bash[68594]: cluster 2024-09-13T14:35:29.997432+0000 mgr.y (mgr.26252) 170 : cluster [DBG] pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 127 op/s 2024-09-13T14:35:31.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:31 smithi177 bash[68594]: cluster 2024-09-13T14:35:29.997432+0000 mgr.y (mgr.26252) 170 : cluster [DBG] pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 127 op/s 2024-09-13T14:35:31.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:31 smithi177 bash[68594]: cluster 2024-09-13T14:35:30.111243+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e537: 8 total, 8 up, 8 in 2024-09-13T14:35:31.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:31 smithi177 bash[68594]: cluster 2024-09-13T14:35:30.111243+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e537: 8 total, 8 up, 8 in 2024-09-13T14:35:31.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:31 smithi177 bash[68594]: cluster 2024-09-13T14:35:31.015873+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e538: 8 total, 8 up, 8 in 2024-09-13T14:35:31.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:31 smithi177 bash[68594]: cluster 2024-09-13T14:35:31.015873+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e538: 8 total, 8 up, 8 in 2024-09-13T14:35:31.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[84576]: cluster 2024-09-13T14:35:29.997432+0000 mgr.y (mgr.26252) 170 : cluster [DBG] pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 127 op/s 2024-09-13T14:35:31.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[84576]: cluster 2024-09-13T14:35:29.997432+0000 mgr.y (mgr.26252) 170 : cluster [DBG] pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 127 op/s 2024-09-13T14:35:31.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[84576]: cluster 2024-09-13T14:35:30.111243+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e537: 8 total, 8 up, 8 in 2024-09-13T14:35:31.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[84576]: cluster 2024-09-13T14:35:30.111243+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e537: 8 total, 8 up, 8 in 2024-09-13T14:35:31.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[84576]: cluster 2024-09-13T14:35:31.015873+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e538: 8 total, 8 up, 8 in 2024-09-13T14:35:31.484 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[84576]: cluster 2024-09-13T14:35:31.015873+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e538: 8 total, 8 up, 8 in 2024-09-13T14:35:31.484 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[87147]: cluster 2024-09-13T14:35:29.997432+0000 mgr.y (mgr.26252) 170 : cluster [DBG] pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 127 op/s 2024-09-13T14:35:31.485 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[87147]: cluster 2024-09-13T14:35:29.997432+0000 mgr.y (mgr.26252) 170 : cluster [DBG] pgmap v306: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 127 op/s 2024-09-13T14:35:31.485 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[87147]: cluster 2024-09-13T14:35:30.111243+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e537: 8 total, 8 up, 8 in 2024-09-13T14:35:31.485 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[87147]: cluster 2024-09-13T14:35:30.111243+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e537: 8 total, 8 up, 8 in 2024-09-13T14:35:31.485 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[87147]: cluster 2024-09-13T14:35:31.015873+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e538: 8 total, 8 up, 8 in 2024-09-13T14:35:31.485 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:31 smithi099 bash[87147]: cluster 2024-09-13T14:35:31.015873+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e538: 8 total, 8 up, 8 in 2024-09-13T14:35:32.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for all mons to upgrade' 2024-09-13T14:35:32.183 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.SnapshotCopyup (2642 ms) 2024-09-13T14:35:32.183 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.SnapshotCopyupZeros 2024-09-13T14:35:32.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[84576]: audit 2024-09-13T14:35:31.190881+0000 mgr.y (mgr.26252) 171 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:32.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[84576]: audit 2024-09-13T14:35:31.190881+0000 mgr.y (mgr.26252) 171 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:32.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[84576]: cephadm 2024-09-13T14:35:31.191269+0000 mgr.y (mgr.26252) 172 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:32.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[84576]: cephadm 2024-09-13T14:35:31.191269+0000 mgr.y (mgr.26252) 172 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:32.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[84576]: audit 2024-09-13T14:35:31.196343+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:32.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[84576]: audit 2024-09-13T14:35:31.196343+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:32.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[84576]: audit 2024-09-13T14:35:31.197475+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:32.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[84576]: audit 2024-09-13T14:35:31.197475+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[84576]: cluster 2024-09-13T14:35:32.020965+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e539: 8 total, 8 up, 8 in 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[84576]: cluster 2024-09-13T14:35:32.020965+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e539: 8 total, 8 up, 8 in 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[87147]: audit 2024-09-13T14:35:31.190881+0000 mgr.y (mgr.26252) 171 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[87147]: audit 2024-09-13T14:35:31.190881+0000 mgr.y (mgr.26252) 171 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[87147]: cephadm 2024-09-13T14:35:31.191269+0000 mgr.y (mgr.26252) 172 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[87147]: cephadm 2024-09-13T14:35:31.191269+0000 mgr.y (mgr.26252) 172 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[87147]: audit 2024-09-13T14:35:31.196343+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[87147]: audit 2024-09-13T14:35:31.196343+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:32.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[87147]: audit 2024-09-13T14:35:31.197475+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:32.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[87147]: audit 2024-09-13T14:35:31.197475+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:32.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[87147]: cluster 2024-09-13T14:35:32.020965+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e539: 8 total, 8 up, 8 in 2024-09-13T14:35:32.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:32 smithi099 bash[87147]: cluster 2024-09-13T14:35:32.020965+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e539: 8 total, 8 up, 8 in 2024-09-13T14:35:32.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:32 smithi177 bash[68594]: audit 2024-09-13T14:35:31.190881+0000 mgr.y (mgr.26252) 171 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:32.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:32 smithi177 bash[68594]: audit 2024-09-13T14:35:31.190881+0000 mgr.y (mgr.26252) 171 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:32.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:32 smithi177 bash[68594]: cephadm 2024-09-13T14:35:31.191269+0000 mgr.y (mgr.26252) 172 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:32.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:32 smithi177 bash[68594]: cephadm 2024-09-13T14:35:31.191269+0000 mgr.y (mgr.26252) 172 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:32.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:32 smithi177 bash[68594]: audit 2024-09-13T14:35:31.196343+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:32.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:32 smithi177 bash[68594]: audit 2024-09-13T14:35:31.196343+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:32.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:32 smithi177 bash[68594]: audit 2024-09-13T14:35:31.197475+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:32.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:32 smithi177 bash[68594]: audit 2024-09-13T14:35:31.197475+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:32.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:32 smithi177 bash[68594]: cluster 2024-09-13T14:35:32.020965+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e539: 8 total, 8 up, 8 in 2024-09-13T14:35:32.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:32 smithi177 bash[68594]: cluster 2024-09-13T14:35:32.020965+0000 mon.a (mon.0) 538 : cluster [DBG] osdmap e539: 8 total, 8 up, 8 in 2024-09-13T14:35:33.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:33 smithi099 bash[84576]: cluster 2024-09-13T14:35:31.998145+0000 mgr.y (mgr.26252) 173 : cluster [DBG] pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 128 op/s 2024-09-13T14:35:33.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:33 smithi099 bash[84576]: cluster 2024-09-13T14:35:31.998145+0000 mgr.y (mgr.26252) 173 : cluster [DBG] pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 128 op/s 2024-09-13T14:35:33.605 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:33 smithi099 bash[87147]: cluster 2024-09-13T14:35:31.998145+0000 mgr.y (mgr.26252) 173 : cluster [DBG] pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 128 op/s 2024-09-13T14:35:33.605 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:33 smithi099 bash[87147]: cluster 2024-09-13T14:35:31.998145+0000 mgr.y (mgr.26252) 173 : cluster [DBG] pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 128 op/s 2024-09-13T14:35:33.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:33 smithi177 bash[68594]: cluster 2024-09-13T14:35:31.998145+0000 mgr.y (mgr.26252) 173 : cluster [DBG] pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 128 op/s 2024-09-13T14:35:33.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:33 smithi177 bash[68594]: cluster 2024-09-13T14:35:31.998145+0000 mgr.y (mgr.26252) 173 : cluster [DBG] pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 6.5 KiB/s wr, 128 op/s 2024-09-13T14:35:34.281 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.SnapshotCopyupZeros (2098 ms) 2024-09-13T14:35:34.281 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.SnapshotCopyupZerosMigration 2024-09-13T14:35:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:34 smithi099 bash[84576]: cluster 2024-09-13T14:35:33.220692+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-09-13T14:35:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:34 smithi099 bash[84576]: cluster 2024-09-13T14:35:33.220692+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-09-13T14:35:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:34 smithi099 bash[84576]: audit 2024-09-13T14:35:33.245850+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:34 smithi099 bash[84576]: audit 2024-09-13T14:35:33.245850+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:34.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:34 smithi099 bash[87147]: cluster 2024-09-13T14:35:33.220692+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-09-13T14:35:34.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:34 smithi099 bash[87147]: cluster 2024-09-13T14:35:33.220692+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-09-13T14:35:34.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:34 smithi099 bash[87147]: audit 2024-09-13T14:35:33.245850+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:34.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:34 smithi099 bash[87147]: audit 2024-09-13T14:35:33.245850+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:34.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:34 smithi177 bash[68594]: cluster 2024-09-13T14:35:33.220692+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-09-13T14:35:34.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:34 smithi177 bash[68594]: cluster 2024-09-13T14:35:33.220692+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-09-13T14:35:34.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:34 smithi177 bash[68594]: audit 2024-09-13T14:35:33.245850+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:34.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:34 smithi177 bash[68594]: audit 2024-09-13T14:35:33.245850+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:35.286 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.SnapshotCopyupZerosMigration (1005 ms) 2024-09-13T14:35:35.286 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.ResizeCopyup 2024-09-13T14:35:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:35 smithi099 bash[84576]: cluster 2024-09-13T14:35:33.999391+0000 mgr.y (mgr.26252) 174 : cluster [DBG] pgmap v312: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:35 smithi099 bash[84576]: cluster 2024-09-13T14:35:33.999391+0000 mgr.y (mgr.26252) 174 : cluster [DBG] pgmap v312: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:35 smithi099 bash[84576]: cluster 2024-09-13T14:35:34.230841+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-09-13T14:35:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:35 smithi099 bash[84576]: cluster 2024-09-13T14:35:34.230841+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-09-13T14:35:35.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:35 smithi099 bash[87147]: cluster 2024-09-13T14:35:33.999391+0000 mgr.y (mgr.26252) 174 : cluster [DBG] pgmap v312: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:35.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:35 smithi099 bash[87147]: cluster 2024-09-13T14:35:33.999391+0000 mgr.y (mgr.26252) 174 : cluster [DBG] pgmap v312: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:35.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:35 smithi099 bash[87147]: cluster 2024-09-13T14:35:34.230841+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-09-13T14:35:35.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:35 smithi099 bash[87147]: cluster 2024-09-13T14:35:34.230841+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-09-13T14:35:35.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:35 smithi177 bash[68594]: cluster 2024-09-13T14:35:33.999391+0000 mgr.y (mgr.26252) 174 : cluster [DBG] pgmap v312: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:35.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:35 smithi177 bash[68594]: cluster 2024-09-13T14:35:33.999391+0000 mgr.y (mgr.26252) 174 : cluster [DBG] pgmap v312: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:35.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:35 smithi177 bash[68594]: cluster 2024-09-13T14:35:34.230841+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-09-13T14:35:35.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:35 smithi177 bash[68594]: cluster 2024-09-13T14:35:34.230841+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-09-13T14:35:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:36 smithi099 bash[84576]: cluster 2024-09-13T14:35:35.232864+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-09-13T14:35:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:36 smithi099 bash[84576]: cluster 2024-09-13T14:35:35.232864+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-09-13T14:35:36.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:36 smithi099 bash[87147]: cluster 2024-09-13T14:35:35.232864+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-09-13T14:35:36.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:36 smithi099 bash[87147]: cluster 2024-09-13T14:35:35.232864+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-09-13T14:35:36.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:36 smithi177 bash[68594]: cluster 2024-09-13T14:35:35.232864+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-09-13T14:35:36.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:36 smithi177 bash[68594]: cluster 2024-09-13T14:35:35.232864+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-09-13T14:35:36.857 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:37.257 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:35:37] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.51.0" 2024-09-13T14:35:37.361 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.ResizeCopyup (2075 ms) 2024-09-13T14:35:37.362 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.DiscardCopyup 2024-09-13T14:35:37.381 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:35:37.381 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.DiscardCopyup (20 ms) 2024-09-13T14:35:37.381 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.ImageOptions 2024-09-13T14:35:37.400 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.ImageOptions (19 ms) 2024-09-13T14:35:37.400 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.WriteFullCopyup 2024-09-13T14:35:37.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:37 smithi099 bash[84576]: cluster 2024-09-13T14:35:36.000096+0000 mgr.y (mgr.26252) 175 : cluster [DBG] pgmap v315: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:37.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:37 smithi099 bash[84576]: cluster 2024-09-13T14:35:36.000096+0000 mgr.y (mgr.26252) 175 : cluster [DBG] pgmap v315: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:37.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:37 smithi099 bash[84576]: cluster 2024-09-13T14:35:36.252571+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-09-13T14:35:37.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:37 smithi099 bash[84576]: cluster 2024-09-13T14:35:36.252571+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-09-13T14:35:37.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:37 smithi099 bash[87147]: cluster 2024-09-13T14:35:36.000096+0000 mgr.y (mgr.26252) 175 : cluster [DBG] pgmap v315: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:37.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:37 smithi099 bash[87147]: cluster 2024-09-13T14:35:36.000096+0000 mgr.y (mgr.26252) 175 : cluster [DBG] pgmap v315: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:37.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:37 smithi099 bash[87147]: cluster 2024-09-13T14:35:36.252571+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-09-13T14:35:37.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:37 smithi099 bash[87147]: cluster 2024-09-13T14:35:36.252571+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-09-13T14:35:37.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:37 smithi177 bash[68594]: cluster 2024-09-13T14:35:36.000096+0000 mgr.y (mgr.26252) 175 : cluster [DBG] pgmap v315: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:37.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:37 smithi177 bash[68594]: cluster 2024-09-13T14:35:36.000096+0000 mgr.y (mgr.26252) 175 : cluster [DBG] pgmap v315: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-13T14:35:37.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:37 smithi177 bash[68594]: cluster 2024-09-13T14:35:36.252571+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-09-13T14:35:37.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:37 smithi177 bash[68594]: cluster 2024-09-13T14:35:36.252571+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-09-13T14:35:38.519 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: cluster 2024-09-13T14:35:37.256252+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-09-13T14:35:38.519 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: cluster 2024-09-13T14:35:37.256252+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-09-13T14:35:38.519 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:37.281302+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.519 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:37.281302+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.519 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:37.294792+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:37.294792+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:37.972931+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:37.972931+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:37.974558+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:37.974558+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:37.985541+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:37.985541+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.090332+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.090332+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.093513+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.093513+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.521 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.095991+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.521 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.095991+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.521 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.104304+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.521 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.104304+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.521 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.108393+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:35:38.521 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: cluster 2024-09-13T14:35:37.256252+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-09-13T14:35:38.521 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: cluster 2024-09-13T14:35:37.256252+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-09-13T14:35:38.521 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:37.281302+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:37.281302+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:37.294792+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:37.294792+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:37.972931+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:37.972931+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:37.974558+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:37.974558+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:37.985541+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:37.985541+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.090332+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.090332+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.093513+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.522 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.093513+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.095991+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.095991+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.104304+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.104304+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.108393+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:35:38.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.108393+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:35:38.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.114161+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:35:38.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.114161+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:35:38.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.118052+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:35:38.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.118052+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:35:38.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.123468+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:35:38.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.123468+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:35:38.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.127506+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:35:38.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.127506+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:35:38.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.133023+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:35:38.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.133023+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:35:38.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.137668+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.137668+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.144445+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.144445+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.148954+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[84576]: audit 2024-09-13T14:35:38.148954+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.108393+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.114161+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.114161+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.118052+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.118052+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.123468+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.123468+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.127506+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:35:38.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.127506+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:35:38.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.133023+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:35:38.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.133023+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:35:38.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.137668+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.137668+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.144445+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.144445+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.148954+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:35:38.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:38 smithi099 bash[87147]: audit 2024-09-13T14:35:38.148954+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:35:38.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: cluster 2024-09-13T14:35:37.256252+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-09-13T14:35:38.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: cluster 2024-09-13T14:35:37.256252+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:37.281302+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:37.281302+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:37.294792+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:37.294792+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:37.972931+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:37.972931+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:37.974558+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:37.974558+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:37.985541+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:37.985541+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.090332+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.090332+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.093513+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.093513+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.095991+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.095991+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.104304+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.104304+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.108393+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:35:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.108393+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.114161+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.114161+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.118052+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.118052+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.123468+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.123468+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.127506+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.127506+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.133023+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.133023+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.137668+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.137668+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.144445+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.144445+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:38.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.148954+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:35:38.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:38 smithi177 bash[68594]: audit 2024-09-13T14:35:38.148954+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:35:39.444 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: cluster 2024-09-13T14:35:38.000729+0000 mgr.y (mgr.26252) 176 : cluster [DBG] pgmap v318: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:39.444 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: cluster 2024-09-13T14:35:38.000729+0000 mgr.y (mgr.26252) 176 : cluster [DBG] pgmap v318: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:39.444 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: cephadm 2024-09-13T14:35:38.097372+0000 mgr.y (mgr.26252) 177 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-13T14:35:39.444 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: cephadm 2024-09-13T14:35:38.097372+0000 mgr.y (mgr.26252) 177 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-13T14:35:39.444 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: cephadm 2024-09-13T14:35:38.139174+0000 mgr.y (mgr.26252) 178 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-13T14:35:39.444 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: cluster 2024-09-13T14:35:38.000729+0000 mgr.y (mgr.26252) 176 : cluster [DBG] pgmap v318: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:39.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: cluster 2024-09-13T14:35:38.000729+0000 mgr.y (mgr.26252) 176 : cluster [DBG] pgmap v318: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:39.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: cluster 2024-09-13T14:35:38.000729+0000 mgr.y (mgr.26252) 176 : cluster [DBG] pgmap v318: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:39.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: cephadm 2024-09-13T14:35:38.097372+0000 mgr.y (mgr.26252) 177 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-13T14:35:39.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: cephadm 2024-09-13T14:35:38.097372+0000 mgr.y (mgr.26252) 177 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-13T14:35:39.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: cephadm 2024-09-13T14:35:38.139174+0000 mgr.y (mgr.26252) 178 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-13T14:35:39.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: cephadm 2024-09-13T14:35:38.139174+0000 mgr.y (mgr.26252) 178 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-13T14:35:39.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: audit 2024-09-13T14:35:38.149192+0000 mgr.y (mgr.26252) 179 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:35:39.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: audit 2024-09-13T14:35:38.149192+0000 mgr.y (mgr.26252) 179 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:35:39.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: cephadm 2024-09-13T14:35:38.150633+0000 mgr.y (mgr.26252) 180 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-13T14:35:39.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: cephadm 2024-09-13T14:35:38.150633+0000 mgr.y (mgr.26252) 180 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-13T14:35:39.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: cluster 2024-09-13T14:35:38.297590+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-09-13T14:35:39.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:39 smithi177 bash[68594]: cluster 2024-09-13T14:35:38.297590+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-09-13T14:35:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: cephadm 2024-09-13T14:35:38.139174+0000 mgr.y (mgr.26252) 178 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-13T14:35:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: audit 2024-09-13T14:35:38.149192+0000 mgr.y (mgr.26252) 179 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:35:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: audit 2024-09-13T14:35:38.149192+0000 mgr.y (mgr.26252) 179 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:35:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: cephadm 2024-09-13T14:35:38.150633+0000 mgr.y (mgr.26252) 180 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-13T14:35:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: cephadm 2024-09-13T14:35:38.150633+0000 mgr.y (mgr.26252) 180 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-13T14:35:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: cluster 2024-09-13T14:35:38.297590+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-09-13T14:35:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[84576]: cluster 2024-09-13T14:35:38.297590+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: cluster 2024-09-13T14:35:38.000729+0000 mgr.y (mgr.26252) 176 : cluster [DBG] pgmap v318: 105 pgs: 105 active+clean; 586 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: cephadm 2024-09-13T14:35:38.097372+0000 mgr.y (mgr.26252) 177 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: cephadm 2024-09-13T14:35:38.097372+0000 mgr.y (mgr.26252) 177 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: cephadm 2024-09-13T14:35:38.139174+0000 mgr.y (mgr.26252) 178 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: cephadm 2024-09-13T14:35:38.139174+0000 mgr.y (mgr.26252) 178 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: audit 2024-09-13T14:35:38.149192+0000 mgr.y (mgr.26252) 179 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: audit 2024-09-13T14:35:38.149192+0000 mgr.y (mgr.26252) 179 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: cephadm 2024-09-13T14:35:38.150633+0000 mgr.y (mgr.26252) 180 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: cephadm 2024-09-13T14:35:38.150633+0000 mgr.y (mgr.26252) 180 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: cluster 2024-09-13T14:35:38.297590+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-09-13T14:35:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:39 smithi099 bash[87147]: cluster 2024-09-13T14:35:38.297590+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-09-13T14:35:40.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:40 smithi177 bash[68594]: cluster 2024-09-13T14:35:39.381004+0000 mon.a (mon.0) 564 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-09-13T14:35:40.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:40 smithi177 bash[68594]: cluster 2024-09-13T14:35:39.381004+0000 mon.a (mon.0) 564 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-09-13T14:35:40.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:35:40 smithi177 bash[61439]: ts=2024-09-13T14:35:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:40.702 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:40 smithi099 bash[84576]: cluster 2024-09-13T14:35:39.381004+0000 mon.a (mon.0) 564 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-09-13T14:35:40.703 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:40 smithi099 bash[84576]: cluster 2024-09-13T14:35:39.381004+0000 mon.a (mon.0) 564 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-09-13T14:35:40.703 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:40 smithi099 bash[87147]: cluster 2024-09-13T14:35:39.381004+0000 mon.a (mon.0) 564 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-09-13T14:35:40.703 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:40 smithi099 bash[87147]: cluster 2024-09-13T14:35:39.381004+0000 mon.a (mon.0) 564 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-09-13T14:35:41.426 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.WriteFullCopyup (4026 ms) 2024-09-13T14:35:41.426 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.DiffIterateCloneOverwrite 2024-09-13T14:35:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:41 smithi099 bash[84576]: cluster 2024-09-13T14:35:40.001432+0000 mgr.y (mgr.26252) 181 : cluster [DBG] pgmap v321: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:41 smithi099 bash[84576]: cluster 2024-09-13T14:35:40.001432+0000 mgr.y (mgr.26252) 181 : cluster [DBG] pgmap v321: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:41 smithi099 bash[84576]: cluster 2024-09-13T14:35:40.391699+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-09-13T14:35:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:41 smithi099 bash[84576]: cluster 2024-09-13T14:35:40.391699+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-09-13T14:35:41.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:41 smithi099 bash[87147]: cluster 2024-09-13T14:35:40.001432+0000 mgr.y (mgr.26252) 181 : cluster [DBG] pgmap v321: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:41.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:41 smithi099 bash[87147]: cluster 2024-09-13T14:35:40.001432+0000 mgr.y (mgr.26252) 181 : cluster [DBG] pgmap v321: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:41.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:41 smithi099 bash[87147]: cluster 2024-09-13T14:35:40.391699+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-09-13T14:35:41.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:41 smithi099 bash[87147]: cluster 2024-09-13T14:35:40.391699+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-09-13T14:35:41.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:41 smithi177 bash[68594]: cluster 2024-09-13T14:35:40.001432+0000 mgr.y (mgr.26252) 181 : cluster [DBG] pgmap v321: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:41.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:41 smithi177 bash[68594]: cluster 2024-09-13T14:35:40.001432+0000 mgr.y (mgr.26252) 181 : cluster [DBG] pgmap v321: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:41.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:41 smithi177 bash[68594]: cluster 2024-09-13T14:35:40.391699+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-09-13T14:35:41.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:41 smithi177 bash[68594]: cluster 2024-09-13T14:35:40.391699+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-09-13T14:35:42.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:42 smithi099 bash[84576]: cluster 2024-09-13T14:35:41.392422+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-13T14:35:42.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:42 smithi099 bash[84576]: cluster 2024-09-13T14:35:41.392422+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-13T14:35:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:42 smithi099 bash[87147]: cluster 2024-09-13T14:35:41.392422+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-13T14:35:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:42 smithi099 bash[87147]: cluster 2024-09-13T14:35:41.392422+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-13T14:35:42.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:42 smithi177 bash[68594]: cluster 2024-09-13T14:35:41.392422+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-13T14:35:42.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:42 smithi177 bash[68594]: cluster 2024-09-13T14:35:41.392422+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-13T14:35:43.389 INFO:teuthology.orchestra.run.smithi099.stdout:wait for all mons to upgrade 2024-09-13T14:35:43.448 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.DiffIterateCloneOverwrite (2022 ms) 2024-09-13T14:35:43.448 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.TestCoR 2024-09-13T14:35:43.463 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING due to disabled rbd_copy_on_read 2024-09-13T14:35:43.463 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.TestCoR (15 ms) 2024-09-13T14:35:43.463 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.FlattenNoEmptyObjects 2024-09-13T14:35:43.692 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:43 smithi099 bash[84576]: cluster 2024-09-13T14:35:42.001850+0000 mgr.y (mgr.26252) 182 : cluster [DBG] pgmap v324: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:43.693 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:43 smithi099 bash[84576]: cluster 2024-09-13T14:35:42.001850+0000 mgr.y (mgr.26252) 182 : cluster [DBG] pgmap v324: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:43.693 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:43 smithi099 bash[84576]: cluster 2024-09-13T14:35:42.419542+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-13T14:35:43.693 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:43 smithi099 bash[84576]: cluster 2024-09-13T14:35:42.419542+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-13T14:35:43.693 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:43 smithi099 bash[87147]: cluster 2024-09-13T14:35:42.001850+0000 mgr.y (mgr.26252) 182 : cluster [DBG] pgmap v324: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:43.693 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:43 smithi099 bash[87147]: cluster 2024-09-13T14:35:42.001850+0000 mgr.y (mgr.26252) 182 : cluster [DBG] pgmap v324: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:43.693 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:43 smithi099 bash[87147]: cluster 2024-09-13T14:35:42.419542+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-13T14:35:43.693 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:43 smithi099 bash[87147]: cluster 2024-09-13T14:35:42.419542+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-13T14:35:43.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:43 smithi177 bash[68594]: cluster 2024-09-13T14:35:42.001850+0000 mgr.y (mgr.26252) 182 : cluster [DBG] pgmap v324: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:43.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:43 smithi177 bash[68594]: cluster 2024-09-13T14:35:42.001850+0000 mgr.y (mgr.26252) 182 : cluster [DBG] pgmap v324: 105 pgs: 105 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-13T14:35:43.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:43 smithi177 bash[68594]: cluster 2024-09-13T14:35:42.419542+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-13T14:35:43.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:43 smithi177 bash[68594]: cluster 2024-09-13T14:35:42.419542+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-13T14:35:43.869 INFO:tasks.workunit.client.0.smithi099.stdout:made parent image "image395": 4194304K (1024 * 4K) 2024-09-13T14:35:43.869 INFO:tasks.workunit.client.0.smithi099.stdout:generated random write map: 2024-09-13T14:35:43.869 INFO:tasks.workunit.client.0.smithi099.stdout: [10 , 42869 ] 2024-09-13T14:35:43.869 INFO:tasks.workunit.client.0.smithi099.stdout: [17 , 69986 ] 2024-09-13T14:35:43.869 INFO:tasks.workunit.client.0.smithi099.stdout: [22 , 93633 ] 2024-09-13T14:35:43.869 INFO:tasks.workunit.client.0.smithi099.stdout: [23 , 96842 ] 2024-09-13T14:35:43.869 INFO:tasks.workunit.client.0.smithi099.stdout: [26 , 109462 ] 2024-09-13T14:35:43.869 INFO:tasks.workunit.client.0.smithi099.stdout: [49 , 203616 ] 2024-09-13T14:35:43.869 INFO:tasks.workunit.client.0.smithi099.stdout: [67 , 276301 ] 2024-09-13T14:35:43.869 INFO:tasks.workunit.client.0.smithi099.stdout: [71 , 293978 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [82 , 336166 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [109 , 449937 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [115 , 473877 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [120 , 492219 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [140 , 573546 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [144 , 593277 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [151 , 622005 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [174 , 716205 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [175 , 717062 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [196 , 802962 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [231 , 946976 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [240 , 985470 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [253 , 1036337 ] 2024-09-13T14:35:43.870 INFO:tasks.workunit.client.0.smithi099.stdout: [258 , 1059890 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [259 , 1061293 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [277 , 1138117 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [288 , 1181011 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [325 , 1331630 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [327 , 1340990 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [329 , 1349532 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [344 , 1411919 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [349 , 1432106 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [356 , 1458232 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [385 , 1579163 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [394 , 1616013 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [395 , 1619206 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [406 , 1663973 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [408 , 1673841 ] 2024-09-13T14:35:43.871 INFO:tasks.workunit.client.0.smithi099.stdout: [418 , 1715270 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [422 , 1732022 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [428 , 1755377 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [487 , 1998294 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [490 , 2008237 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [492 , 2017132 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [493 , 2021918 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [497 , 2037371 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [509 , 2087756 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [520 , 2131451 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [521 , 2136638 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [524 , 2147540 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [542 , 2221045 ] 2024-09-13T14:35:43.872 INFO:tasks.workunit.client.0.smithi099.stdout: [555 , 2273463 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [558 , 2288622 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [559 , 2292034 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [564 , 2311062 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [567 , 2322765 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [576 , 2362789 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [577 , 2365160 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [599 , 2454014 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [622 , 2549678 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [630 , 2582723 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [670 , 2744432 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [676 , 2770146 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [713 , 2923522 ] 2024-09-13T14:35:43.873 INFO:tasks.workunit.client.0.smithi099.stdout: [714 , 2927290 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [716 , 2933739 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [720 , 2949493 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [721 , 2956111 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [725 , 2970548 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [736 , 3014918 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [758 , 3106188 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [762 , 3123163 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [771 , 3161190 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [777 , 3184032 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [798 , 3268716 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [803 , 3289785 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [804 , 3295621 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [808 , 3313107 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [814 , 3337720 ] 2024-09-13T14:35:43.874 INFO:tasks.workunit.client.0.smithi099.stdout: [839 , 3438044 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [855 , 3503945 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [859 , 3518521 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [865 , 3546579 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [872 , 3574637 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [877 , 3593378 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [886 , 3629929 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [891 , 3652086 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [923 , 3782667 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [927 , 3799584 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [934 , 3825762 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [951 , 3897286 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [973 , 3987411 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [983 , 4028952 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [991 , 4061816 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [996 , 4080853 ] 2024-09-13T14:35:43.875 INFO:tasks.workunit.client.0.smithi099.stdout: [1003 , 4109918 ] 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: [1009 , 4133864 ] 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: [1016 , 4164678 ] 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout:write data based on random map 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-10 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-17 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-22 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-23 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-26 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-49 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-67 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-71 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-82 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-109 2024-09-13T14:35:43.876 INFO:tasks.workunit.client.0.smithi099.stdout: write object-115 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-120 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-140 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-144 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-151 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-174 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-175 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-196 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-231 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-240 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-253 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-258 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-259 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-277 2024-09-13T14:35:43.877 INFO:tasks.workunit.client.0.smithi099.stdout: write object-288 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-325 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-327 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-329 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-344 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-349 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-356 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-385 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-394 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-395 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-406 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-408 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-418 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-422 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-428 2024-09-13T14:35:43.878 INFO:tasks.workunit.client.0.smithi099.stdout: write object-487 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-490 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-492 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-493 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-497 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-509 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-520 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-521 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-524 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-542 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-555 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-558 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-559 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-564 2024-09-13T14:35:43.879 INFO:tasks.workunit.client.0.smithi099.stdout: write object-567 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-576 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-577 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-599 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-622 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-630 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-670 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-676 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-713 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-714 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-716 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-720 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-721 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-725 2024-09-13T14:35:43.880 INFO:tasks.workunit.client.0.smithi099.stdout: write object-736 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-758 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-762 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-771 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-777 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-798 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-803 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-804 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-808 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-814 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-839 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-855 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-859 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-865 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-872 2024-09-13T14:35:43.881 INFO:tasks.workunit.client.0.smithi099.stdout: write object-877 2024-09-13T14:35:43.882 INFO:tasks.workunit.client.0.smithi099.stdout: write object-886 2024-09-13T14:35:43.882 INFO:tasks.workunit.client.0.smithi099.stdout: write object-891 2024-09-13T14:35:43.882 INFO:tasks.workunit.client.0.smithi099.stdout: write object-923 2024-09-13T14:35:43.882 INFO:tasks.workunit.client.0.smithi099.stdout: write object-927 2024-09-13T14:35:43.882 INFO:tasks.workunit.client.0.smithi099.stdout: write object-934 2024-09-13T14:35:44.817 INFO:tasks.workunit.client.0.smithi099.stdout: write object-951 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout: write object-973 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout: write object-983 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout: write object-991 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout: write object-996 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout: write object-1003 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout: write object-1009 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout: write object-1016 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout:verify written data by reading 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout: read object-10 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout:made snapshot "image395@parent_snap" and protect it 2024-09-13T14:35:44.818 INFO:tasks.workunit.client.0.smithi099.stdout:made and opened clone "image396" 2024-09-13T14:35:44.819 INFO:tasks.workunit.client.0.smithi099.stdout:flattening clone: "image396" 2024-09-13T14:35:44.819 INFO:tasks.workunit.client.0.smithi099.stdout:check whether child image has the same set of objects as parent 2024-09-13T14:35:44.819 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.FlattenNoEmptyObjects (1354 ms) 2024-09-13T14:35:44.819 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.PoolMetadataConfApply 2024-09-13T14:35:44.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: cluster 2024-09-13T14:35:43.421554+0000 mon.a (mon.0) 568 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: cluster 2024-09-13T14:35:43.421554+0000 mon.a (mon.0) 568 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: cephadm 2024-09-13T14:35:43.464286+0000 mgr.y (mgr.26252) 183 : cephadm [INF] Upgrade: Updating osd.0 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: cephadm 2024-09-13T14:35:43.464286+0000 mgr.y (mgr.26252) 183 : cephadm [INF] Upgrade: Updating osd.0 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: audit 2024-09-13T14:35:43.469203+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: audit 2024-09-13T14:35:43.469203+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: audit 2024-09-13T14:35:43.472088+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: audit 2024-09-13T14:35:43.472088+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: audit 2024-09-13T14:35:43.472993+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: audit 2024-09-13T14:35:43.472993+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: cephadm 2024-09-13T14:35:43.475410+0000 mgr.y (mgr.26252) 184 : cephadm [INF] Deploying daemon osd.0 on smithi099 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[87147]: cephadm 2024-09-13T14:35:43.475410+0000 mgr.y (mgr.26252) 184 : cephadm [INF] Deploying daemon osd.0 on smithi099 2024-09-13T14:35:44.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: cluster 2024-09-13T14:35:43.421554+0000 mon.a (mon.0) 568 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: cluster 2024-09-13T14:35:43.421554+0000 mon.a (mon.0) 568 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: cephadm 2024-09-13T14:35:43.464286+0000 mgr.y (mgr.26252) 183 : cephadm [INF] Upgrade: Updating osd.0 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: cephadm 2024-09-13T14:35:43.464286+0000 mgr.y (mgr.26252) 183 : cephadm [INF] Upgrade: Updating osd.0 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: audit 2024-09-13T14:35:43.469203+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: audit 2024-09-13T14:35:43.469203+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: audit 2024-09-13T14:35:43.472088+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: audit 2024-09-13T14:35:43.472088+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: audit 2024-09-13T14:35:43.472993+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: audit 2024-09-13T14:35:43.472993+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: cephadm 2024-09-13T14:35:43.475410+0000 mgr.y (mgr.26252) 184 : cephadm [INF] Deploying daemon osd.0 on smithi099 2024-09-13T14:35:44.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[84576]: cephadm 2024-09-13T14:35:43.475410+0000 mgr.y (mgr.26252) 184 : cephadm [INF] Deploying daemon osd.0 on smithi099 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: cluster 2024-09-13T14:35:43.421554+0000 mon.a (mon.0) 568 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: cluster 2024-09-13T14:35:43.421554+0000 mon.a (mon.0) 568 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: cephadm 2024-09-13T14:35:43.464286+0000 mgr.y (mgr.26252) 183 : cephadm [INF] Upgrade: Updating osd.0 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: cephadm 2024-09-13T14:35:43.464286+0000 mgr.y (mgr.26252) 183 : cephadm [INF] Upgrade: Updating osd.0 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: audit 2024-09-13T14:35:43.469203+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: audit 2024-09-13T14:35:43.469203+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: audit 2024-09-13T14:35:43.472088+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: audit 2024-09-13T14:35:43.472088+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: audit 2024-09-13T14:35:43.472993+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: audit 2024-09-13T14:35:43.472993+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:44.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: cephadm 2024-09-13T14:35:43.475410+0000 mgr.y (mgr.26252) 184 : cephadm [INF] Deploying daemon osd.0 on smithi099 2024-09-13T14:35:44.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:44 smithi177 bash[68594]: cephadm 2024-09-13T14:35:43.475410+0000 mgr.y (mgr.26252) 184 : cephadm [INF] Deploying daemon osd.0 on smithi099 2024-09-13T14:35:44.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:35:44.905 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.PoolMetadataConfApply (88 ms) 2024-09-13T14:35:44.905 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.Sparsify 2024-09-13T14:35:44.941 INFO:tasks.workunit.client.0.smithi099.stdout:sparsify_supported=1 2024-09-13T14:35:44.941 INFO:tasks.workunit.client.0.smithi099.stdout:sparse_read_supported=1 2024-09-13T14:35:44.988 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.Sparsify (83 ms) 2024-09-13T14:35:44.988 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.SparsifyClone 2024-09-13T14:35:45.024 INFO:tasks.workunit.client.0.smithi099.stdout:sparsify_supported=1 2024-09-13T14:35:45.163 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:44 smithi099 bash[76324]: [13/Sep/2024:14:35:44] ENGINE Bus STOPPING 2024-09-13T14:35:45.163 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[76324]: [13/Sep/2024:14:35:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:35:45.163 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[76324]: [13/Sep/2024:14:35:45] ENGINE Bus STOPPED 2024-09-13T14:35:45.163 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[76324]: [13/Sep/2024:14:35:45] ENGINE Bus STARTING 2024-09-13T14:35:45.163 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[76324]: [13/Sep/2024:14:35:45] ENGINE Serving on http://:::9283 2024-09-13T14:35:45.163 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[76324]: [13/Sep/2024:14:35:45] ENGINE Bus STARTED 2024-09-13T14:35:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[84576]: cluster 2024-09-13T14:35:44.002502+0000 mgr.y (mgr.26252) 185 : cluster [DBG] pgmap v327: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[84576]: cluster 2024-09-13T14:35:44.002502+0000 mgr.y (mgr.26252) 185 : cluster [DBG] pgmap v327: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[84576]: cluster 2024-09-13T14:35:44.485448+0000 mon.a (mon.0) 572 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-13T14:35:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[84576]: cluster 2024-09-13T14:35:44.485448+0000 mon.a (mon.0) 572 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-13T14:35:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[84576]: audit 2024-09-13T14:35:44.845762+0000 mon.a (mon.0) 573 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' 2024-09-13T14:35:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[84576]: audit 2024-09-13T14:35:44.845762+0000 mon.a (mon.0) 573 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' 2024-09-13T14:35:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[84576]: audit 2024-09-13T14:35:45.062822+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:35:45.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[84576]: audit 2024-09-13T14:35:45.062822+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:35:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[87147]: cluster 2024-09-13T14:35:44.002502+0000 mgr.y (mgr.26252) 185 : cluster [DBG] pgmap v327: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[87147]: cluster 2024-09-13T14:35:44.002502+0000 mgr.y (mgr.26252) 185 : cluster [DBG] pgmap v327: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[87147]: cluster 2024-09-13T14:35:44.485448+0000 mon.a (mon.0) 572 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-13T14:35:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[87147]: cluster 2024-09-13T14:35:44.485448+0000 mon.a (mon.0) 572 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-13T14:35:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[87147]: audit 2024-09-13T14:35:44.845762+0000 mon.a (mon.0) 573 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' 2024-09-13T14:35:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[87147]: audit 2024-09-13T14:35:44.845762+0000 mon.a (mon.0) 573 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' 2024-09-13T14:35:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[87147]: audit 2024-09-13T14:35:45.062822+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:35:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:45 smithi099 bash[87147]: audit 2024-09-13T14:35:45.062822+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:35:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:45 smithi177 bash[68594]: cluster 2024-09-13T14:35:44.002502+0000 mgr.y (mgr.26252) 185 : cluster [DBG] pgmap v327: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:45 smithi177 bash[68594]: cluster 2024-09-13T14:35:44.002502+0000 mgr.y (mgr.26252) 185 : cluster [DBG] pgmap v327: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:45 smithi177 bash[68594]: cluster 2024-09-13T14:35:44.485448+0000 mon.a (mon.0) 572 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-13T14:35:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:45 smithi177 bash[68594]: cluster 2024-09-13T14:35:44.485448+0000 mon.a (mon.0) 572 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-13T14:35:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:45 smithi177 bash[68594]: audit 2024-09-13T14:35:44.845762+0000 mon.a (mon.0) 573 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' 2024-09-13T14:35:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:45 smithi177 bash[68594]: audit 2024-09-13T14:35:44.845762+0000 mon.a (mon.0) 573 : audit [INF] from='client.? 172.21.15.99:0/2468331726' entity='client.admin' 2024-09-13T14:35:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:45 smithi177 bash[68594]: audit 2024-09-13T14:35:45.062822+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:35:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:45 smithi177 bash[68594]: audit 2024-09-13T14:35:45.062822+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:35:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[84576]: audit 2024-09-13T14:35:45.063559+0000 mgr.y (mgr.26252) 186 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:35:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[84576]: audit 2024-09-13T14:35:45.063559+0000 mgr.y (mgr.26252) 186 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:35:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[84576]: cluster 2024-09-13T14:35:45.852569+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-13T14:35:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[84576]: cluster 2024-09-13T14:35:45.852569+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-13T14:35:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[84576]: cluster 2024-09-13T14:35:46.021815+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-13T14:35:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[84576]: cluster 2024-09-13T14:35:46.021815+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-13T14:35:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[87147]: audit 2024-09-13T14:35:45.063559+0000 mgr.y (mgr.26252) 186 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:35:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[87147]: audit 2024-09-13T14:35:45.063559+0000 mgr.y (mgr.26252) 186 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:35:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[87147]: cluster 2024-09-13T14:35:45.852569+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-13T14:35:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[87147]: cluster 2024-09-13T14:35:45.852569+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-13T14:35:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[87147]: cluster 2024-09-13T14:35:46.021815+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-13T14:35:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:46 smithi099 bash[87147]: cluster 2024-09-13T14:35:46.021815+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-13T14:35:46.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:46 smithi177 bash[68594]: audit 2024-09-13T14:35:45.063559+0000 mgr.y (mgr.26252) 186 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:35:46.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:46 smithi177 bash[68594]: audit 2024-09-13T14:35:45.063559+0000 mgr.y (mgr.26252) 186 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:35:46.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:46 smithi177 bash[68594]: cluster 2024-09-13T14:35:45.852569+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-13T14:35:46.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:46 smithi177 bash[68594]: cluster 2024-09-13T14:35:45.852569+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-13T14:35:46.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:46 smithi177 bash[68594]: cluster 2024-09-13T14:35:46.021815+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-13T14:35:46.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:46 smithi177 bash[68594]: cluster 2024-09-13T14:35:46.021815+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-13T14:35:47.050 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.SparsifyClone (2062 ms) 2024-09-13T14:35:47.050 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestInternal.MissingDataPool 2024-09-13T14:35:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:35:47] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.51.0" 2024-09-13T14:35:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:47 smithi099 bash[84576]: cluster 2024-09-13T14:35:46.002870+0000 mgr.y (mgr.26252) 187 : cluster [DBG] pgmap v330: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:47 smithi099 bash[84576]: cluster 2024-09-13T14:35:46.002870+0000 mgr.y (mgr.26252) 187 : cluster [DBG] pgmap v330: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:47 smithi099 bash[84576]: cluster 2024-09-13T14:35:47.023606+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-13T14:35:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:47 smithi099 bash[84576]: cluster 2024-09-13T14:35:47.023606+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-13T14:35:47.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:47 smithi099 bash[87147]: cluster 2024-09-13T14:35:46.002870+0000 mgr.y (mgr.26252) 187 : cluster [DBG] pgmap v330: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:47 smithi099 bash[87147]: cluster 2024-09-13T14:35:46.002870+0000 mgr.y (mgr.26252) 187 : cluster [DBG] pgmap v330: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:47 smithi099 bash[87147]: cluster 2024-09-13T14:35:47.023606+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-13T14:35:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:47 smithi099 bash[87147]: cluster 2024-09-13T14:35:47.023606+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-13T14:35:47.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:47 smithi177 bash[68594]: cluster 2024-09-13T14:35:46.002870+0000 mgr.y (mgr.26252) 187 : cluster [DBG] pgmap v330: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:47.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:47 smithi177 bash[68594]: cluster 2024-09-13T14:35:46.002870+0000 mgr.y (mgr.26252) 187 : cluster [DBG] pgmap v330: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.1 MiB/s wr, 424 op/s 2024-09-13T14:35:47.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:47 smithi177 bash[68594]: cluster 2024-09-13T14:35:47.023606+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-13T14:35:47.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:47 smithi177 bash[68594]: cluster 2024-09-13T14:35:47.023606+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-13T14:35:48.272 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestInternal.MissingDataPool (1222 ms) 2024-09-13T14:35:49.048 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 34 tests from TestInternal (35988 ms total) 2024-09-13T14:35:49.049 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:35:49.049 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 17 tests from TestImageWatcher 2024-09-13T14:35:49.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[84576]: cluster 2024-09-13T14:35:48.003488+0000 mgr.y (mgr.26252) 188 : cluster [DBG] pgmap v333: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:49.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[84576]: cluster 2024-09-13T14:35:48.003488+0000 mgr.y (mgr.26252) 188 : cluster [DBG] pgmap v333: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:49.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[84576]: cluster 2024-09-13T14:35:48.028784+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-13T14:35:49.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[84576]: cluster 2024-09-13T14:35:48.028784+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-13T14:35:49.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[84576]: audit 2024-09-13T14:35:48.245851+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:49.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[84576]: audit 2024-09-13T14:35:48.245851+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:49.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[87147]: cluster 2024-09-13T14:35:48.003488+0000 mgr.y (mgr.26252) 188 : cluster [DBG] pgmap v333: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:49.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[87147]: cluster 2024-09-13T14:35:48.003488+0000 mgr.y (mgr.26252) 188 : cluster [DBG] pgmap v333: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:49.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[87147]: cluster 2024-09-13T14:35:48.028784+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-13T14:35:49.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[87147]: cluster 2024-09-13T14:35:48.028784+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-13T14:35:49.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[87147]: audit 2024-09-13T14:35:48.245851+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:49.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:49 smithi099 bash[87147]: audit 2024-09-13T14:35:48.245851+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:49 smithi177 bash[68594]: cluster 2024-09-13T14:35:48.003488+0000 mgr.y (mgr.26252) 188 : cluster [DBG] pgmap v333: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:49 smithi177 bash[68594]: cluster 2024-09-13T14:35:48.003488+0000 mgr.y (mgr.26252) 188 : cluster [DBG] pgmap v333: 105 pgs: 105 active+clean; 4.8 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:49 smithi177 bash[68594]: cluster 2024-09-13T14:35:48.028784+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-13T14:35:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:49 smithi177 bash[68594]: cluster 2024-09-13T14:35:48.028784+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-13T14:35:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:49 smithi177 bash[68594]: audit 2024-09-13T14:35:48.245851+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:49.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:49 smithi177 bash[68594]: audit 2024-09-13T14:35:48.245851+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:50.041 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:35:49 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:50.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:49 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:50.042 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:49 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:50.042 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:49 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:50.043 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:49 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:50.043 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:35:49 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:50.043 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:35:49 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:50.044 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:35:49 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:50.044 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:35:49 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:50.077 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:50.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:50 smithi099 bash[84576]: cluster 2024-09-13T14:35:49.040167+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-13T14:35:50.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:50 smithi099 bash[84576]: cluster 2024-09-13T14:35:49.040167+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-13T14:35:50.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:50 smithi099 bash[87147]: cluster 2024-09-13T14:35:49.040167+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-13T14:35:50.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:50 smithi099 bash[87147]: cluster 2024-09-13T14:35:49.040167+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-13T14:35:50.322 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:50 smithi099 systemd[1]: Stopping Ceph osd.0 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:35:50.322 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:50 smithi099 bash[35105]: debug 2024-09-13T14:35:50.288+0000 7fdaf58e9640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T14:35:50.322 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:50 smithi099 bash[35105]: debug 2024-09-13T14:35:50.288+0000 7fdaf58e9640 -1 osd.0 557 *** Got signal Terminated *** 2024-09-13T14:35:50.322 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:50 smithi099 bash[35105]: debug 2024-09-13T14:35:50.288+0000 7fdaf58e9640 -1 osd.0 557 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:35:50.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:35:50 smithi177 bash[61439]: ts=2024-09-13T14:35:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:35:50.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:50 smithi177 bash[68594]: cluster 2024-09-13T14:35:49.040167+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-13T14:35:50.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:50 smithi177 bash[68594]: cluster 2024-09-13T14:35:49.040167+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-13T14:35:51.057 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifyHeaderUpdate 2024-09-13T14:35:51.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[84576]: cluster 2024-09-13T14:35:50.004078+0000 mgr.y (mgr.26252) 189 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:51.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[84576]: cluster 2024-09-13T14:35:50.004078+0000 mgr.y (mgr.26252) 189 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:51.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[84576]: cluster 2024-09-13T14:35:50.056998+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-13T14:35:51.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[84576]: cluster 2024-09-13T14:35:50.056998+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[84576]: audit 2024-09-13T14:35:50.063261+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.99:0/4109218706' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[84576]: audit 2024-09-13T14:35:50.063261+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.99:0/4109218706' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[84576]: audit 2024-09-13T14:35:50.067879+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[84576]: audit 2024-09-13T14:35:50.067879+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[84576]: cluster 2024-09-13T14:35:50.292746+0000 mon.a (mon.0) 583 : cluster [INF] osd.0 marked itself down and dead 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[84576]: cluster 2024-09-13T14:35:50.292746+0000 mon.a (mon.0) 583 : cluster [INF] osd.0 marked itself down and dead 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[87147]: cluster 2024-09-13T14:35:50.004078+0000 mgr.y (mgr.26252) 189 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[87147]: cluster 2024-09-13T14:35:50.004078+0000 mgr.y (mgr.26252) 189 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[87147]: cluster 2024-09-13T14:35:50.056998+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[87147]: cluster 2024-09-13T14:35:50.056998+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-13T14:35:51.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[87147]: audit 2024-09-13T14:35:50.063261+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.99:0/4109218706' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[87147]: audit 2024-09-13T14:35:50.063261+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.99:0/4109218706' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[87147]: audit 2024-09-13T14:35:50.067879+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[87147]: audit 2024-09-13T14:35:50.067879+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[87147]: cluster 2024-09-13T14:35:50.292746+0000 mon.a (mon.0) 583 : cluster [INF] osd.0 marked itself down and dead 2024-09-13T14:35:51.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:51 smithi099 bash[87147]: cluster 2024-09-13T14:35:50.292746+0000 mon.a (mon.0) 583 : cluster [INF] osd.0 marked itself down and dead 2024-09-13T14:35:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:51 smithi177 bash[68594]: cluster 2024-09-13T14:35:50.004078+0000 mgr.y (mgr.26252) 189 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:51 smithi177 bash[68594]: cluster 2024-09-13T14:35:50.004078+0000 mgr.y (mgr.26252) 189 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:51 smithi177 bash[68594]: cluster 2024-09-13T14:35:50.056998+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-13T14:35:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:51 smithi177 bash[68594]: cluster 2024-09-13T14:35:50.056998+0000 mon.a (mon.0) 581 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-13T14:35:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:51 smithi177 bash[68594]: audit 2024-09-13T14:35:50.063261+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.99:0/4109218706' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:51 smithi177 bash[68594]: audit 2024-09-13T14:35:50.063261+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.99:0/4109218706' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:51 smithi177 bash[68594]: audit 2024-09-13T14:35:50.067879+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:51 smithi177 bash[68594]: audit 2024-09-13T14:35:50.067879+0000 mon.a (mon.0) 582 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:51.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:51 smithi177 bash[68594]: cluster 2024-09-13T14:35:50.292746+0000 mon.a (mon.0) 583 : cluster [INF] osd.0 marked itself down and dead 2024-09-13T14:35:51.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:51 smithi177 bash[68594]: cluster 2024-09-13T14:35:50.292746+0000 mon.a (mon.0) 583 : cluster [INF] osd.0 marked itself down and dead 2024-09-13T14:35:52.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[84576]: cluster 2024-09-13T14:35:51.044564+0000 mon.a (mon.0) 584 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:52.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[84576]: cluster 2024-09-13T14:35:51.044564+0000 mon.a (mon.0) 584 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:52.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[84576]: audit 2024-09-13T14:35:51.047103+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:52.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[84576]: audit 2024-09-13T14:35:51.047103+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:52.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[84576]: cluster 2024-09-13T14:35:51.056228+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e558: 8 total, 7 up, 8 in 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[84576]: cluster 2024-09-13T14:35:51.056228+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e558: 8 total, 7 up, 8 in 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[84576]: cluster 2024-09-13T14:35:52.058365+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e559: 8 total, 7 up, 8 in 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[84576]: cluster 2024-09-13T14:35:52.058365+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e559: 8 total, 7 up, 8 in 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[87147]: cluster 2024-09-13T14:35:51.044564+0000 mon.a (mon.0) 584 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[87147]: cluster 2024-09-13T14:35:51.044564+0000 mon.a (mon.0) 584 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[87147]: audit 2024-09-13T14:35:51.047103+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[87147]: audit 2024-09-13T14:35:51.047103+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[87147]: cluster 2024-09-13T14:35:51.056228+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e558: 8 total, 7 up, 8 in 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[87147]: cluster 2024-09-13T14:35:51.056228+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e558: 8 total, 7 up, 8 in 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[87147]: cluster 2024-09-13T14:35:52.058365+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e559: 8 total, 7 up, 8 in 2024-09-13T14:35:52.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[87147]: cluster 2024-09-13T14:35:52.058365+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e559: 8 total, 7 up, 8 in 2024-09-13T14:35:52.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:52 smithi177 bash[68594]: cluster 2024-09-13T14:35:51.044564+0000 mon.a (mon.0) 584 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:52.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:52 smithi177 bash[68594]: cluster 2024-09-13T14:35:51.044564+0000 mon.a (mon.0) 584 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:52.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:52 smithi177 bash[68594]: audit 2024-09-13T14:35:51.047103+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:52.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:52 smithi177 bash[68594]: audit 2024-09-13T14:35:51.047103+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:52.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:52 smithi177 bash[68594]: cluster 2024-09-13T14:35:51.056228+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e558: 8 total, 7 up, 8 in 2024-09-13T14:35:52.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:52 smithi177 bash[68594]: cluster 2024-09-13T14:35:51.056228+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e558: 8 total, 7 up, 8 in 2024-09-13T14:35:52.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:52 smithi177 bash[68594]: cluster 2024-09-13T14:35:52.058365+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e559: 8 total, 7 up, 8 in 2024-09-13T14:35:52.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:52 smithi177 bash[68594]: cluster 2024-09-13T14:35:52.058365+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e559: 8 total, 7 up, 8 in 2024-09-13T14:35:53.070 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:52 smithi099 bash[93403]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-0 2024-09-13T14:35:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:53 smithi177 bash[68594]: cluster 2024-09-13T14:35:52.004741+0000 mgr.y (mgr.26252) 190 : cluster [DBG] pgmap v339: 105 pgs: 11 stale+active+clean, 32 unknown, 62 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:53 smithi177 bash[68594]: cluster 2024-09-13T14:35:52.004741+0000 mgr.y (mgr.26252) 190 : cluster [DBG] pgmap v339: 105 pgs: 11 stale+active+clean, 32 unknown, 62 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:53 smithi177 bash[68594]: cluster 2024-09-13T14:35:53.059799+0000 mon.a (mon.0) 588 : cluster [DBG] osdmap e560: 8 total, 7 up, 8 in 2024-09-13T14:35:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:53 smithi177 bash[68594]: cluster 2024-09-13T14:35:53.059799+0000 mon.a (mon.0) 588 : cluster [DBG] osdmap e560: 8 total, 7 up, 8 in 2024-09-13T14:35:53.571 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:53 smithi099 bash[93530]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-0 2024-09-13T14:35:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:53 smithi099 bash[84576]: cluster 2024-09-13T14:35:52.004741+0000 mgr.y (mgr.26252) 190 : cluster [DBG] pgmap v339: 105 pgs: 11 stale+active+clean, 32 unknown, 62 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:53 smithi099 bash[84576]: cluster 2024-09-13T14:35:52.004741+0000 mgr.y (mgr.26252) 190 : cluster [DBG] pgmap v339: 105 pgs: 11 stale+active+clean, 32 unknown, 62 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:53 smithi099 bash[84576]: cluster 2024-09-13T14:35:53.059799+0000 mon.a (mon.0) 588 : cluster [DBG] osdmap e560: 8 total, 7 up, 8 in 2024-09-13T14:35:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:53 smithi099 bash[84576]: cluster 2024-09-13T14:35:53.059799+0000 mon.a (mon.0) 588 : cluster [DBG] osdmap e560: 8 total, 7 up, 8 in 2024-09-13T14:35:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:53 smithi099 bash[87147]: cluster 2024-09-13T14:35:52.004741+0000 mgr.y (mgr.26252) 190 : cluster [DBG] pgmap v339: 105 pgs: 11 stale+active+clean, 32 unknown, 62 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:53 smithi099 bash[87147]: cluster 2024-09-13T14:35:52.004741+0000 mgr.y (mgr.26252) 190 : cluster [DBG] pgmap v339: 105 pgs: 11 stale+active+clean, 32 unknown, 62 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:35:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:53 smithi099 bash[87147]: cluster 2024-09-13T14:35:53.059799+0000 mon.a (mon.0) 588 : cluster [DBG] osdmap e560: 8 total, 7 up, 8 in 2024-09-13T14:35:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:53 smithi099 bash[87147]: cluster 2024-09-13T14:35:53.059799+0000 mon.a (mon.0) 588 : cluster [DBG] osdmap e560: 8 total, 7 up, 8 in 2024-09-13T14:35:54.121 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifyHeaderUpdate (3064 ms) 2024-09-13T14:35:54.121 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifyFlatten 2024-09-13T14:35:54.173 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifyFlatten (51 ms) 2024-09-13T14:35:54.173 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifyResize 2024-09-13T14:35:54.225 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifyResize (52 ms) 2024-09-13T14:35:54.225 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifyRebuildObjectMap 2024-09-13T14:35:54.276 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifyRebuildObjectMap (51 ms) 2024-09-13T14:35:54.276 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifyUpdateFeatures 2024-09-13T14:35:54.331 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifyUpdateFeatures (56 ms) 2024-09-13T14:35:54.332 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifySnapCreate 2024-09-13T14:35:54.364 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifySnapCreate (33 ms) 2024-09-13T14:35:54.365 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifySnapCreateError 2024-09-13T14:35:54.413 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifySnapCreateError (48 ms) 2024-09-13T14:35:54.413 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifySnapRename 2024-09-13T14:35:54.467 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifySnapRename (54 ms) 2024-09-13T14:35:54.467 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifySnapRenameError 2024-09-13T14:35:54.524 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifySnapRenameError (57 ms) 2024-09-13T14:35:54.524 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifySnapRemove 2024-09-13T14:35:54.575 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifySnapRemove (52 ms) 2024-09-13T14:35:54.575 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifySnapProtect 2024-09-13T14:35:54.622 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifySnapProtect (47 ms) 2024-09-13T14:35:54.622 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifySnapUnprotect 2024-09-13T14:35:54.671 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifySnapUnprotect (49 ms) 2024-09-13T14:35:54.672 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifyRename 2024-09-13T14:35:54.720 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifyRename (49 ms) 2024-09-13T14:35:54.720 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifyAsyncTimedOut 2024-09-13T14:35:54.771 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifyAsyncTimedOut (50 ms) 2024-09-13T14:35:54.771 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifyAsyncError 2024-09-13T14:35:54.812 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifyAsyncError (41 ms) 2024-09-13T14:35:54.812 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifyAsyncCompleteError 2024-09-13T14:35:54.863 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifyAsyncCompleteError (52 ms) 2024-09-13T14:35:54.863 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestImageWatcher.NotifyAsyncRequestTimedOut 2024-09-13T14:35:54.912 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestImageWatcher.NotifyAsyncRequestTimedOut (49 ms) 2024-09-13T14:35:55.101 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 17 tests from TestImageWatcher (3855 ms total) 2024-09-13T14:35:55.102 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:35:55.102 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 160 tests from TestLibRBD 2024-09-13T14:35:55.156 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.CreateAndStat 2024-09-13T14:35:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[84576]: cluster 2024-09-13T14:35:54.006175+0000 mgr.y (mgr.26252) 191 : cluster [DBG] pgmap v342: 105 pgs: 35 active+undersized, 7 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 13/81 objects degraded (16.049%) 2024-09-13T14:35:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[84576]: cluster 2024-09-13T14:35:54.006175+0000 mgr.y (mgr.26252) 191 : cluster [DBG] pgmap v342: 105 pgs: 35 active+undersized, 7 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 13/81 objects degraded (16.049%) 2024-09-13T14:35:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[84576]: cluster 2024-09-13T14:35:54.065349+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e561: 8 total, 7 up, 8 in 2024-09-13T14:35:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[84576]: cluster 2024-09-13T14:35:54.065349+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e561: 8 total, 7 up, 8 in 2024-09-13T14:35:55.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[84576]: cluster 2024-09-13T14:35:54.085758+0000 mon.a (mon.0) 590 : cluster [WRN] Health check failed: Degraded data redundancy: 13/81 objects degraded (16.049%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:55.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[84576]: cluster 2024-09-13T14:35:54.085758+0000 mon.a (mon.0) 590 : cluster [WRN] Health check failed: Degraded data redundancy: 13/81 objects degraded (16.049%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:55.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[87147]: cluster 2024-09-13T14:35:54.006175+0000 mgr.y (mgr.26252) 191 : cluster [DBG] pgmap v342: 105 pgs: 35 active+undersized, 7 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 13/81 objects degraded (16.049%) 2024-09-13T14:35:55.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[87147]: cluster 2024-09-13T14:35:54.006175+0000 mgr.y (mgr.26252) 191 : cluster [DBG] pgmap v342: 105 pgs: 35 active+undersized, 7 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 13/81 objects degraded (16.049%) 2024-09-13T14:35:55.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[87147]: cluster 2024-09-13T14:35:54.065349+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e561: 8 total, 7 up, 8 in 2024-09-13T14:35:55.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[87147]: cluster 2024-09-13T14:35:54.065349+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e561: 8 total, 7 up, 8 in 2024-09-13T14:35:55.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[87147]: cluster 2024-09-13T14:35:54.085758+0000 mon.a (mon.0) 590 : cluster [WRN] Health check failed: Degraded data redundancy: 13/81 objects degraded (16.049%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:55.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:55 smithi099 bash[87147]: cluster 2024-09-13T14:35:54.085758+0000 mon.a (mon.0) 590 : cluster [WRN] Health check failed: Degraded data redundancy: 13/81 objects degraded (16.049%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:55 smithi177 bash[68594]: cluster 2024-09-13T14:35:54.006175+0000 mgr.y (mgr.26252) 191 : cluster [DBG] pgmap v342: 105 pgs: 35 active+undersized, 7 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 13/81 objects degraded (16.049%) 2024-09-13T14:35:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:55 smithi177 bash[68594]: cluster 2024-09-13T14:35:54.006175+0000 mgr.y (mgr.26252) 191 : cluster [DBG] pgmap v342: 105 pgs: 35 active+undersized, 7 active+undersized+degraded, 63 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 13/81 objects degraded (16.049%) 2024-09-13T14:35:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:55 smithi177 bash[68594]: cluster 2024-09-13T14:35:54.065349+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e561: 8 total, 7 up, 8 in 2024-09-13T14:35:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:55 smithi177 bash[68594]: cluster 2024-09-13T14:35:54.065349+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e561: 8 total, 7 up, 8 in 2024-09-13T14:35:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:55 smithi177 bash[68594]: cluster 2024-09-13T14:35:54.085758+0000 mon.a (mon.0) 590 : cluster [WRN] Health check failed: Degraded data redundancy: 13/81 objects degraded (16.049%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:55 smithi177 bash[68594]: cluster 2024-09-13T14:35:54.085758+0000 mon.a (mon.0) 590 : cluster [WRN] Health check failed: Degraded data redundancy: 13/81 objects degraded (16.049%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:55.594 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:35:56.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:56 smithi177 bash[68594]: cluster 2024-09-13T14:35:55.097957+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e562: 8 total, 7 up, 8 in 2024-09-13T14:35:56.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:56 smithi177 bash[68594]: cluster 2024-09-13T14:35:55.097957+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e562: 8 total, 7 up, 8 in 2024-09-13T14:35:56.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:56 smithi177 bash[68594]: audit 2024-09-13T14:35:55.585129+0000 mon.a (mon.0) 592 : audit [DBG] from='client.? 172.21.15.99:0/1737029274' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:35:56.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:56 smithi177 bash[68594]: audit 2024-09-13T14:35:55.585129+0000 mon.a (mon.0) 592 : audit [DBG] from='client.? 172.21.15.99:0/1737029274' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:35:56.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:56 smithi099 bash[84576]: cluster 2024-09-13T14:35:55.097957+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e562: 8 total, 7 up, 8 in 2024-09-13T14:35:56.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:56 smithi099 bash[84576]: cluster 2024-09-13T14:35:55.097957+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e562: 8 total, 7 up, 8 in 2024-09-13T14:35:56.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:56 smithi099 bash[84576]: audit 2024-09-13T14:35:55.585129+0000 mon.a (mon.0) 592 : audit [DBG] from='client.? 172.21.15.99:0/1737029274' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:35:56.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:56 smithi099 bash[84576]: audit 2024-09-13T14:35:55.585129+0000 mon.a (mon.0) 592 : audit [DBG] from='client.? 172.21.15.99:0/1737029274' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:35:56.391 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:56 smithi099 bash[87147]: cluster 2024-09-13T14:35:55.097957+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e562: 8 total, 7 up, 8 in 2024-09-13T14:35:56.392 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:56 smithi099 bash[87147]: cluster 2024-09-13T14:35:55.097957+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e562: 8 total, 7 up, 8 in 2024-09-13T14:35:56.392 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:56 smithi099 bash[87147]: audit 2024-09-13T14:35:55.585129+0000 mon.a (mon.0) 592 : audit [DBG] from='client.? 172.21.15.99:0/1737029274' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:35:56.392 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:56 smithi099 bash[87147]: audit 2024-09-13T14:35:55.585129+0000 mon.a (mon.0) 592 : audit [DBG] from='client.? 172.21.15.99:0/1737029274' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-13T14:35:57.121 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:57.148 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:35:57.321 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:57.321 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:57.321 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:57.322 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:57.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:35:57] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-09-13T14:35:57.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:57.322 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:57.323 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T14:35:57.323 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: Stopped Ceph osd.0 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:35:57.323 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[87147]: cluster 2024-09-13T14:35:56.006931+0000 mgr.y (mgr.26252) 192 : cluster [DBG] pgmap v345: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[87147]: cluster 2024-09-13T14:35:56.006931+0000 mgr.y (mgr.26252) 192 : cluster [DBG] pgmap v345: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[87147]: cluster 2024-09-13T14:35:56.114222+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e563: 8 total, 7 up, 8 in 2024-09-13T14:35:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[87147]: cluster 2024-09-13T14:35:56.114222+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e563: 8 total, 7 up, 8 in 2024-09-13T14:35:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[87147]: audit 2024-09-13T14:35:56.120088+0000 mon.a (mon.0) 594 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:57.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[87147]: audit 2024-09-13T14:35:56.120088+0000 mon.a (mon.0) 594 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[84576]: cluster 2024-09-13T14:35:56.006931+0000 mgr.y (mgr.26252) 192 : cluster [DBG] pgmap v345: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[84576]: cluster 2024-09-13T14:35:56.006931+0000 mgr.y (mgr.26252) 192 : cluster [DBG] pgmap v345: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[84576]: cluster 2024-09-13T14:35:56.114222+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e563: 8 total, 7 up, 8 in 2024-09-13T14:35:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[84576]: cluster 2024-09-13T14:35:56.114222+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e563: 8 total, 7 up, 8 in 2024-09-13T14:35:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[84576]: audit 2024-09-13T14:35:56.120088+0000 mon.a (mon.0) 594 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:57 smithi099 bash[84576]: audit 2024-09-13T14:35:56.120088+0000 mon.a (mon.0) 594 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:35:57.448 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:57 smithi099 systemd[1]: Started Ceph osd.0 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:35:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:57 smithi177 bash[68594]: cluster 2024-09-13T14:35:56.006931+0000 mgr.y (mgr.26252) 192 : cluster [DBG] pgmap v345: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:57 smithi177 bash[68594]: cluster 2024-09-13T14:35:56.006931+0000 mgr.y (mgr.26252) 192 : cluster [DBG] pgmap v345: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:57 smithi177 bash[68594]: cluster 2024-09-13T14:35:56.114222+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e563: 8 total, 7 up, 8 in 2024-09-13T14:35:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:57 smithi177 bash[68594]: cluster 2024-09-13T14:35:56.114222+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e563: 8 total, 7 up, 8 in 2024-09-13T14:35:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:57 smithi177 bash[68594]: audit 2024-09-13T14:35:56.120088+0000 mon.a (mon.0) 594 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:57.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:57 smithi177 bash[68594]: audit 2024-09-13T14:35:56.120088+0000 mon.a (mon.0) 594 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:58.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[84576]: audit 2024-09-13T14:35:57.106485+0000 mon.a (mon.0) 595 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:58.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[84576]: audit 2024-09-13T14:35:57.106485+0000 mon.a (mon.0) 595 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:58.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[84576]: cluster 2024-09-13T14:35:57.111455+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[84576]: cluster 2024-09-13T14:35:57.111455+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[84576]: audit 2024-09-13T14:35:57.388620+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[84576]: audit 2024-09-13T14:35:57.388620+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[84576]: audit 2024-09-13T14:35:57.394834+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[84576]: audit 2024-09-13T14:35:57.394834+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[84576]: audit 2024-09-13T14:35:57.395941+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[84576]: audit 2024-09-13T14:35:57.395941+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[87147]: audit 2024-09-13T14:35:57.106485+0000 mon.a (mon.0) 595 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[87147]: audit 2024-09-13T14:35:57.106485+0000 mon.a (mon.0) 595 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[87147]: cluster 2024-09-13T14:35:57.111455+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2024-09-13T14:35:58.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[87147]: cluster 2024-09-13T14:35:57.111455+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2024-09-13T14:35:58.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[87147]: audit 2024-09-13T14:35:57.388620+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[87147]: audit 2024-09-13T14:35:57.388620+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[87147]: audit 2024-09-13T14:35:57.394834+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[87147]: audit 2024-09-13T14:35:57.394834+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[87147]: audit 2024-09-13T14:35:57.395941+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:58.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[87147]: audit 2024-09-13T14:35:57.395941+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:58 smithi177 bash[68594]: audit 2024-09-13T14:35:57.106485+0000 mon.a (mon.0) 595 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:58 smithi177 bash[68594]: audit 2024-09-13T14:35:57.106485+0000 mon.a (mon.0) 595 : audit [INF] from='client.? 172.21.15.99:0/57771665' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:58 smithi177 bash[68594]: cluster 2024-09-13T14:35:57.111455+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2024-09-13T14:35:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:58 smithi177 bash[68594]: cluster 2024-09-13T14:35:57.111455+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2024-09-13T14:35:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:58 smithi177 bash[68594]: audit 2024-09-13T14:35:57.388620+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:58 smithi177 bash[68594]: audit 2024-09-13T14:35:57.388620+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:58 smithi177 bash[68594]: audit 2024-09-13T14:35:57.394834+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:58 smithi177 bash[68594]: audit 2024-09-13T14:35:57.394834+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:35:58.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:58 smithi177 bash[68594]: audit 2024-09-13T14:35:57.395941+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:58.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:58 smithi177 bash[68594]: audit 2024-09-13T14:35:57.395941+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:58.974 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[93850]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:58.974 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[93850]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:58.975 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[93850]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:35:58.975 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[93850]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:58.975 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[93850]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:59.204 INFO:tasks.workunit.client.0.smithi099.stdout:image has size 2097152 and order 22 2024-09-13T14:35:59.204 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.CreateAndStat (4048 ms) 2024-09-13T14:35:59.204 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.CreateWithSameDataPool 2024-09-13T14:35:59.204 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.232 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.CreateWithSameDataPool (28 ms) 2024-09-13T14:35:59.232 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.CreateAndStatPP 2024-09-13T14:35:59.232 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.260 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.CreateAndStatPP (29 ms) 2024-09-13T14:35:59.260 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.GetId 2024-09-13T14:35:59.260 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.302 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[93850]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:35:59.302 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:58 smithi099 bash[93850]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T14:35:59.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[84576]: cluster 2024-09-13T14:35:58.007813+0000 mgr.y (mgr.26252) 193 : cluster [DBG] pgmap v348: 105 pgs: 32 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:59.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[84576]: cluster 2024-09-13T14:35:58.007813+0000 mgr.y (mgr.26252) 193 : cluster [DBG] pgmap v348: 105 pgs: 32 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:59.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[84576]: cluster 2024-09-13T14:35:58.135964+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2024-09-13T14:35:59.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[84576]: cluster 2024-09-13T14:35:58.135964+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2024-09-13T14:35:59.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[87147]: cluster 2024-09-13T14:35:58.007813+0000 mgr.y (mgr.26252) 193 : cluster [DBG] pgmap v348: 105 pgs: 32 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:59.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[87147]: cluster 2024-09-13T14:35:58.007813+0000 mgr.y (mgr.26252) 193 : cluster [DBG] pgmap v348: 105 pgs: 32 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:59.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[87147]: cluster 2024-09-13T14:35:58.135964+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2024-09-13T14:35:59.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[87147]: cluster 2024-09-13T14:35:58.135964+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2024-09-13T14:35:59.312 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.GetId (52 ms) 2024-09-13T14:35:59.312 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.GetIdPP 2024-09-13T14:35:59.312 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.359 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.GetIdPP (47 ms) 2024-09-13T14:35:59.359 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefix 2024-09-13T14:35:59.360 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.386 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.GetBlockNamePrefix (27 ms) 2024-09-13T14:35:59.386 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefixPP 2024-09-13T14:35:59.386 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.413 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.GetBlockNamePrefixPP (26 ms) 2024-09-13T14:35:59.413 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestGetCreateTimestamp 2024-09-13T14:35:59.413 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.438 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestGetCreateTimestamp (26 ms) 2024-09-13T14:35:59.438 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.GetCreateTimestampPP 2024-09-13T14:35:59.438 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.462 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.GetCreateTimestampPP (24 ms) 2024-09-13T14:35:59.463 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.OpenAio 2024-09-13T14:35:59.463 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.485 INFO:tasks.workunit.client.0.smithi099.stdout:image has size 2097152 and order 22 2024-09-13T14:35:59.485 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.OpenAio (23 ms) 2024-09-13T14:35:59.485 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.OpenAioFail 2024-09-13T14:35:59.489 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.OpenAioFail (4 ms) 2024-09-13T14:35:59.489 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.OpenAioPP 2024-09-13T14:35:59.489 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.526 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.OpenAioPP (37 ms) 2024-09-13T14:35:59.527 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.OpenAioFailPP 2024-09-13T14:35:59.530 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.OpenAioFailPP (4 ms) 2024-09-13T14:35:59.530 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ResizeAndStat 2024-09-13T14:35:59.530 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.571 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[93850]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:35:59.571 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[93850]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:35:59.571 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[93850]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:35:59.571 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[93850]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:35:59.571 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:35:59 smithi099 bash[93850]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T14:35:59.593 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ResizeAndStat (62 ms) 2024-09-13T14:35:59.593 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ResizeAndStatPP 2024-09-13T14:35:59.593 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:59 smithi177 bash[68594]: cluster 2024-09-13T14:35:58.007813+0000 mgr.y (mgr.26252) 193 : cluster [DBG] pgmap v348: 105 pgs: 32 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:59.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:59 smithi177 bash[68594]: cluster 2024-09-13T14:35:58.007813+0000 mgr.y (mgr.26252) 193 : cluster [DBG] pgmap v348: 105 pgs: 32 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-09-13T14:35:59.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:59 smithi177 bash[68594]: cluster 2024-09-13T14:35:58.135964+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2024-09-13T14:35:59.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:35:59 smithi177 bash[68594]: cluster 2024-09-13T14:35:58.135964+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2024-09-13T14:35:59.665 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ResizeAndStatPP (73 ms) 2024-09-13T14:35:59.665 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResize 2024-09-13T14:35:59.665 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.749 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.UpdateWatchAndResize (83 ms) 2024-09-13T14:35:59.749 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResizePP 2024-09-13T14:35:59.749 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:35:59.830 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.UpdateWatchAndResizePP (82 ms) 2024-09-13T14:35:59.831 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCreateLsDelete 2024-09-13T14:36:00.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:00 smithi099 bash[84576]: cluster 2024-09-13T14:35:59.160326+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2024-09-13T14:36:00.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:00 smithi099 bash[84576]: cluster 2024-09-13T14:35:59.160326+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2024-09-13T14:36:00.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:00 smithi099 bash[87147]: cluster 2024-09-13T14:35:59.160326+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2024-09-13T14:36:00.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:00 smithi099 bash[87147]: cluster 2024-09-13T14:35:59.160326+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2024-09-13T14:36:00.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:00 smithi177 bash[68594]: cluster 2024-09-13T14:35:59.160326+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2024-09-13T14:36:00.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:00 smithi177 bash[68594]: cluster 2024-09-13T14:35:59.160326+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2024-09-13T14:36:00.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:36:00 smithi177 bash[61439]: ts=2024-09-13T14:36:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:01.209 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:01.449 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[84576]: cluster 2024-09-13T14:36:00.008589+0000 mgr.y (mgr.26252) 194 : cluster [DBG] pgmap v351: 105 pgs: 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:01.449 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[84576]: cluster 2024-09-13T14:36:00.008589+0000 mgr.y (mgr.26252) 194 : cluster [DBG] pgmap v351: 105 pgs: 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:01.449 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[84576]: cluster 2024-09-13T14:36:00.152230+0000 mon.a (mon.0) 602 : cluster [WRN] Health check update: Degraded data redundancy: 12/81 objects degraded (14.815%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:01.449 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[84576]: cluster 2024-09-13T14:36:00.152230+0000 mon.a (mon.0) 602 : cluster [WRN] Health check update: Degraded data redundancy: 12/81 objects degraded (14.815%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:01.449 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[84576]: cluster 2024-09-13T14:36:00.165594+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2024-09-13T14:36:01.449 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[84576]: cluster 2024-09-13T14:36:00.165594+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2024-09-13T14:36:01.449 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[84576]: audit 2024-09-13T14:36:00.180896+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.99:0/3113716634' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.449 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[84576]: audit 2024-09-13T14:36:00.180896+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.99:0/3113716634' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.450 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[84576]: audit 2024-09-13T14:36:00.181255+0000 mon.a (mon.0) 604 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.450 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[84576]: audit 2024-09-13T14:36:00.181255+0000 mon.a (mon.0) 604 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[87147]: cluster 2024-09-13T14:36:00.008589+0000 mgr.y (mgr.26252) 194 : cluster [DBG] pgmap v351: 105 pgs: 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:01.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[87147]: cluster 2024-09-13T14:36:00.008589+0000 mgr.y (mgr.26252) 194 : cluster [DBG] pgmap v351: 105 pgs: 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:01.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[87147]: cluster 2024-09-13T14:36:00.152230+0000 mon.a (mon.0) 602 : cluster [WRN] Health check update: Degraded data redundancy: 12/81 objects degraded (14.815%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:01.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[87147]: cluster 2024-09-13T14:36:00.152230+0000 mon.a (mon.0) 602 : cluster [WRN] Health check update: Degraded data redundancy: 12/81 objects degraded (14.815%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:01.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[87147]: cluster 2024-09-13T14:36:00.165594+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2024-09-13T14:36:01.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[87147]: cluster 2024-09-13T14:36:00.165594+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2024-09-13T14:36:01.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[87147]: audit 2024-09-13T14:36:00.180896+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.99:0/3113716634' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[87147]: audit 2024-09-13T14:36:00.180896+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.99:0/3113716634' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.451 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[87147]: audit 2024-09-13T14:36:00.181255+0000 mon.a (mon.0) 604 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.451 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[87147]: audit 2024-09-13T14:36:00.181255+0000 mon.a (mon.0) 604 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:01 smithi177 bash[68594]: cluster 2024-09-13T14:36:00.008589+0000 mgr.y (mgr.26252) 194 : cluster [DBG] pgmap v351: 105 pgs: 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:01 smithi177 bash[68594]: cluster 2024-09-13T14:36:00.008589+0000 mgr.y (mgr.26252) 194 : cluster [DBG] pgmap v351: 105 pgs: 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:01 smithi177 bash[68594]: cluster 2024-09-13T14:36:00.152230+0000 mon.a (mon.0) 602 : cluster [WRN] Health check update: Degraded data redundancy: 12/81 objects degraded (14.815%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:01 smithi177 bash[68594]: cluster 2024-09-13T14:36:00.152230+0000 mon.a (mon.0) 602 : cluster [WRN] Health check update: Degraded data redundancy: 12/81 objects degraded (14.815%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:01 smithi177 bash[68594]: cluster 2024-09-13T14:36:00.165594+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2024-09-13T14:36:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:01 smithi177 bash[68594]: cluster 2024-09-13T14:36:00.165594+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2024-09-13T14:36:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:01 smithi177 bash[68594]: audit 2024-09-13T14:36:00.180896+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.99:0/3113716634' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:01 smithi177 bash[68594]: audit 2024-09-13T14:36:00.180896+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.99:0/3113716634' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:01 smithi177 bash[68594]: audit 2024-09-13T14:36:00.181255+0000 mon.a (mon.0) 604 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:01 smithi177 bash[68594]: audit 2024-09-13T14:36:00.181255+0000 mon.a (mon.0) 604 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:01.821 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:36:01 smithi099 bash[94102]: debug 2024-09-13T14:36:01.532+0000 7fdfec1aa640 1 -- 172.21.15.99:0/2784239272 <== mon.2 v2:172.21.15.99:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559a37a82960 con 0x559a37aa3c00 2024-09-13T14:36:02.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:02 smithi099 bash[84576]: audit 2024-09-13T14:36:01.164332+0000 mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:02.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:02 smithi099 bash[84576]: audit 2024-09-13T14:36:01.164332+0000 mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:02.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:02 smithi099 bash[84576]: cluster 2024-09-13T14:36:01.171002+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2024-09-13T14:36:02.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:02 smithi099 bash[84576]: cluster 2024-09-13T14:36:01.171002+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2024-09-13T14:36:02.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:02 smithi099 bash[87147]: audit 2024-09-13T14:36:01.164332+0000 mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:02.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:02 smithi099 bash[87147]: audit 2024-09-13T14:36:01.164332+0000 mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:02.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:02 smithi099 bash[87147]: cluster 2024-09-13T14:36:01.171002+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2024-09-13T14:36:02.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:02 smithi099 bash[87147]: cluster 2024-09-13T14:36:01.171002+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2024-09-13T14:36:02.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:02 smithi177 bash[68594]: audit 2024-09-13T14:36:01.164332+0000 mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:02 smithi177 bash[68594]: audit 2024-09-13T14:36:01.164332+0000 mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:02 smithi177 bash[68594]: cluster 2024-09-13T14:36:01.171002+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2024-09-13T14:36:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:02 smithi177 bash[68594]: cluster 2024-09-13T14:36:01.171002+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2024-09-13T14:36:03.241 INFO:tasks.workunit.client.0.smithi099.stdout:image: image18 2024-09-13T14:36:03.241 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image18 2024-09-13T14:36:03.241 INFO:tasks.workunit.client.0.smithi099.stdout:found image18 2024-09-13T14:36:03.241 INFO:tasks.workunit.client.0.smithi099.stdout:erased image18 2024-09-13T14:36:03.242 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:03.321 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:36:03 smithi099 bash[94102]: debug 2024-09-13T14:36:03.059+0000 7fdfeea11640 -1 Falling back to public interface 2024-09-13T14:36:03.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:03 smithi099 bash[84576]: cluster 2024-09-13T14:36:02.009513+0000 mgr.y (mgr.26252) 195 : cluster [DBG] pgmap v354: 137 pgs: 32 unknown, 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:03.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:03 smithi099 bash[84576]: cluster 2024-09-13T14:36:02.009513+0000 mgr.y (mgr.26252) 195 : cluster [DBG] pgmap v354: 137 pgs: 32 unknown, 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:03.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:03 smithi099 bash[84576]: cluster 2024-09-13T14:36:02.194799+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2024-09-13T14:36:03.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:03 smithi099 bash[84576]: cluster 2024-09-13T14:36:02.194799+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2024-09-13T14:36:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:03 smithi099 bash[87147]: cluster 2024-09-13T14:36:02.009513+0000 mgr.y (mgr.26252) 195 : cluster [DBG] pgmap v354: 137 pgs: 32 unknown, 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:03 smithi099 bash[87147]: cluster 2024-09-13T14:36:02.009513+0000 mgr.y (mgr.26252) 195 : cluster [DBG] pgmap v354: 137 pgs: 32 unknown, 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:03 smithi099 bash[87147]: cluster 2024-09-13T14:36:02.194799+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2024-09-13T14:36:03.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:03 smithi099 bash[87147]: cluster 2024-09-13T14:36:02.194799+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2024-09-13T14:36:03.414 INFO:tasks.workunit.client.0.smithi099.stdout:image: image18 2024-09-13T14:36:03.414 INFO:tasks.workunit.client.0.smithi099.stdout:image: image19 2024-09-13T14:36:03.414 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image18 2024-09-13T14:36:03.414 INFO:tasks.workunit.client.0.smithi099.stdout:found image18 2024-09-13T14:36:03.414 INFO:tasks.workunit.client.0.smithi099.stdout:erased image18 2024-09-13T14:36:03.414 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image19 2024-09-13T14:36:03.415 INFO:tasks.workunit.client.0.smithi099.stdout:found image19 2024-09-13T14:36:03.415 INFO:tasks.workunit.client.0.smithi099.stdout:erased image19 2024-09-13T14:36:03.415 INFO:tasks.workunit.client.0.smithi099.stdout:image: image19 2024-09-13T14:36:03.415 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image19 2024-09-13T14:36:03.415 INFO:tasks.workunit.client.0.smithi099.stdout:found image19 2024-09-13T14:36:03.415 INFO:tasks.workunit.client.0.smithi099.stdout:erased image19 2024-09-13T14:36:03.415 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCreateLsDelete (3584 ms) 2024-09-13T14:36:03.415 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCreateLsDeletePP 2024-09-13T14:36:03.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:03 smithi177 bash[68594]: cluster 2024-09-13T14:36:02.009513+0000 mgr.y (mgr.26252) 195 : cluster [DBG] pgmap v354: 137 pgs: 32 unknown, 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:03.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:03 smithi177 bash[68594]: cluster 2024-09-13T14:36:02.009513+0000 mgr.y (mgr.26252) 195 : cluster [DBG] pgmap v354: 137 pgs: 32 unknown, 30 active+undersized, 7 active+undersized+degraded, 68 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 12/81 objects degraded (14.815%) 2024-09-13T14:36:03.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:03 smithi177 bash[68594]: cluster 2024-09-13T14:36:02.194799+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2024-09-13T14:36:03.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:03 smithi177 bash[68594]: cluster 2024-09-13T14:36:02.194799+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2024-09-13T14:36:03.752 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:04.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[87147]: cluster 2024-09-13T14:36:03.205293+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2024-09-13T14:36:04.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[87147]: cluster 2024-09-13T14:36:03.205293+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2024-09-13T14:36:04.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[87147]: audit 2024-09-13T14:36:03.253715+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:04.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[87147]: audit 2024-09-13T14:36:03.253715+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:04.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[87147]: audit 2024-09-13T14:36:03.256012+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:04.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[87147]: audit 2024-09-13T14:36:03.256012+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[84576]: cluster 2024-09-13T14:36:03.205293+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2024-09-13T14:36:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[84576]: cluster 2024-09-13T14:36:03.205293+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2024-09-13T14:36:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[84576]: audit 2024-09-13T14:36:03.253715+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[84576]: audit 2024-09-13T14:36:03.253715+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[84576]: audit 2024-09-13T14:36:03.256012+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:04 smithi099 bash[84576]: audit 2024-09-13T14:36:03.256012+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:04.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:04 smithi177 bash[68594]: cluster 2024-09-13T14:36:03.205293+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2024-09-13T14:36:04.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:04 smithi177 bash[68594]: cluster 2024-09-13T14:36:03.205293+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2024-09-13T14:36:04.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:04 smithi177 bash[68594]: audit 2024-09-13T14:36:03.253715+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:04.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:04 smithi177 bash[68594]: audit 2024-09-13T14:36:03.253715+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:04.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:04 smithi177 bash[68594]: audit 2024-09-13T14:36:03.256012+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:04.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:04 smithi177 bash[68594]: audit 2024-09-13T14:36:03.256012+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:05.275 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: cluster 2024-09-13T14:36:04.010421+0000 mgr.y (mgr.26252) 196 : cluster [DBG] pgmap v357: 137 pgs: 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: cluster 2024-09-13T14:36:04.010421+0000 mgr.y (mgr.26252) 196 : cluster [DBG] pgmap v357: 137 pgs: 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: cluster 2024-09-13T14:36:04.261840+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e571: 8 total, 7 up, 8 in 2024-09-13T14:36:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: cluster 2024-09-13T14:36:04.261840+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e571: 8 total, 7 up, 8 in 2024-09-13T14:36:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: audit 2024-09-13T14:36:04.266497+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.99:0/756971896' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: audit 2024-09-13T14:36:04.266497+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.99:0/756971896' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: audit 2024-09-13T14:36:04.266941+0000 mon.a (mon.0) 612 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: audit 2024-09-13T14:36:04.266941+0000 mon.a (mon.0) 612 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: audit 2024-09-13T14:36:04.439326+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: audit 2024-09-13T14:36:04.439326+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: audit 2024-09-13T14:36:04.445166+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[84576]: audit 2024-09-13T14:36:04.445166+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: cluster 2024-09-13T14:36:04.010421+0000 mgr.y (mgr.26252) 196 : cluster [DBG] pgmap v357: 137 pgs: 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: cluster 2024-09-13T14:36:04.010421+0000 mgr.y (mgr.26252) 196 : cluster [DBG] pgmap v357: 137 pgs: 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: cluster 2024-09-13T14:36:04.261840+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e571: 8 total, 7 up, 8 in 2024-09-13T14:36:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: cluster 2024-09-13T14:36:04.261840+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e571: 8 total, 7 up, 8 in 2024-09-13T14:36:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: audit 2024-09-13T14:36:04.266497+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.99:0/756971896' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: audit 2024-09-13T14:36:04.266497+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.99:0/756971896' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: audit 2024-09-13T14:36:04.266941+0000 mon.a (mon.0) 612 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: audit 2024-09-13T14:36:04.266941+0000 mon.a (mon.0) 612 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: audit 2024-09-13T14:36:04.439326+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: audit 2024-09-13T14:36:04.439326+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: audit 2024-09-13T14:36:04.445166+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:05 smithi099 bash[87147]: audit 2024-09-13T14:36:04.445166+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: cluster 2024-09-13T14:36:04.010421+0000 mgr.y (mgr.26252) 196 : cluster [DBG] pgmap v357: 137 pgs: 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: cluster 2024-09-13T14:36:04.010421+0000 mgr.y (mgr.26252) 196 : cluster [DBG] pgmap v357: 137 pgs: 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: cluster 2024-09-13T14:36:04.261840+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e571: 8 total, 7 up, 8 in 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: cluster 2024-09-13T14:36:04.261840+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e571: 8 total, 7 up, 8 in 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: audit 2024-09-13T14:36:04.266497+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.99:0/756971896' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: audit 2024-09-13T14:36:04.266497+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.99:0/756971896' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: audit 2024-09-13T14:36:04.266941+0000 mon.a (mon.0) 612 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: audit 2024-09-13T14:36:04.266941+0000 mon.a (mon.0) 612 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: audit 2024-09-13T14:36:04.439326+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: audit 2024-09-13T14:36:04.439326+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: audit 2024-09-13T14:36:04.445166+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:05 smithi177 bash[68594]: audit 2024-09-13T14:36:04.445166+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[84576]: audit 2024-09-13T14:36:05.257487+0000 mon.a (mon.0) 615 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[84576]: audit 2024-09-13T14:36:05.257487+0000 mon.a (mon.0) 615 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[84576]: cluster 2024-09-13T14:36:05.260666+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e572: 8 total, 7 up, 8 in 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[84576]: cluster 2024-09-13T14:36:05.260666+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e572: 8 total, 7 up, 8 in 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[84576]: audit 2024-09-13T14:36:05.591487+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[84576]: audit 2024-09-13T14:36:05.591487+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[84576]: audit 2024-09-13T14:36:05.597114+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[84576]: audit 2024-09-13T14:36:05.597114+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[84576]: cluster 2024-09-13T14:36:06.019765+0000 mon.a (mon.0) 619 : cluster [WRN] Health check update: Degraded data redundancy: 34/237 objects degraded (14.346%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[84576]: cluster 2024-09-13T14:36:06.019765+0000 mon.a (mon.0) 619 : cluster [WRN] Health check update: Degraded data redundancy: 34/237 objects degraded (14.346%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[87147]: audit 2024-09-13T14:36:05.257487+0000 mon.a (mon.0) 615 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[87147]: audit 2024-09-13T14:36:05.257487+0000 mon.a (mon.0) 615 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:06.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[87147]: cluster 2024-09-13T14:36:05.260666+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e572: 8 total, 7 up, 8 in 2024-09-13T14:36:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[87147]: cluster 2024-09-13T14:36:05.260666+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e572: 8 total, 7 up, 8 in 2024-09-13T14:36:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[87147]: audit 2024-09-13T14:36:05.591487+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[87147]: audit 2024-09-13T14:36:05.591487+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[87147]: audit 2024-09-13T14:36:05.597114+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[87147]: audit 2024-09-13T14:36:05.597114+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[87147]: cluster 2024-09-13T14:36:06.019765+0000 mon.a (mon.0) 619 : cluster [WRN] Health check update: Degraded data redundancy: 34/237 objects degraded (14.346%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:06 smithi099 bash[87147]: cluster 2024-09-13T14:36:06.019765+0000 mon.a (mon.0) 619 : cluster [WRN] Health check update: Degraded data redundancy: 34/237 objects degraded (14.346%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:06 smithi177 bash[68594]: audit 2024-09-13T14:36:05.257487+0000 mon.a (mon.0) 615 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:06 smithi177 bash[68594]: audit 2024-09-13T14:36:05.257487+0000 mon.a (mon.0) 615 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:06 smithi177 bash[68594]: cluster 2024-09-13T14:36:05.260666+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e572: 8 total, 7 up, 8 in 2024-09-13T14:36:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:06 smithi177 bash[68594]: cluster 2024-09-13T14:36:05.260666+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e572: 8 total, 7 up, 8 in 2024-09-13T14:36:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:06 smithi177 bash[68594]: audit 2024-09-13T14:36:05.591487+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:06 smithi177 bash[68594]: audit 2024-09-13T14:36:05.591487+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:06 smithi177 bash[68594]: audit 2024-09-13T14:36:05.597114+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:06 smithi177 bash[68594]: audit 2024-09-13T14:36:05.597114+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:06 smithi177 bash[68594]: cluster 2024-09-13T14:36:06.019765+0000 mon.a (mon.0) 619 : cluster [WRN] Health check update: Degraded data redundancy: 34/237 objects degraded (14.346%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:06.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:06 smithi177 bash[68594]: cluster 2024-09-13T14:36:06.019765+0000 mon.a (mon.0) 619 : cluster [WRN] Health check update: Degraded data redundancy: 34/237 objects degraded (14.346%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:07.294 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[94102]: debug 2024-09-13T14:36:07.031+0000 7fdfeea11640 -1 osd.0 558 log_to_monitors true 2024-09-13T14:36:07.295 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:36:07] "GET /metrics HTTP/1.1" 200 38922 "" "Prometheus/2.51.0" 2024-09-13T14:36:07.329 INFO:tasks.workunit.client.0.smithi099.stdout:num images is: 1 2024-09-13T14:36:07.329 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:36:07.330 INFO:tasks.workunit.client.0.smithi099.stdout:image: image20 2024-09-13T14:36:07.330 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image20 2024-09-13T14:36:07.330 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:07.358 INFO:tasks.workunit.client.0.smithi099.stdout:num images is: 2 2024-09-13T14:36:07.358 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:36:07.358 INFO:tasks.workunit.client.0.smithi099.stdout:image: image20 2024-09-13T14:36:07.358 INFO:tasks.workunit.client.0.smithi099.stdout:image: image21 2024-09-13T14:36:07.358 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image20 2024-09-13T14:36:07.358 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image21 2024-09-13T14:36:07.502 INFO:tasks.workunit.client.0.smithi099.stdout:num images is: 1 2024-09-13T14:36:07.502 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:36:07.502 INFO:tasks.workunit.client.0.smithi099.stdout:image: image21 2024-09-13T14:36:07.503 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image21 2024-09-13T14:36:07.503 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCreateLsDeletePP (4088 ms) 2024-09-13T14:36:07.503 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCopy 2024-09-13T14:36:07.571 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[94102]: debug 2024-09-13T14:36:07.335+0000 7fdfe67bb640 -1 osd.0 558 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:36:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[84576]: cluster 2024-09-13T14:36:06.011273+0000 mgr.y (mgr.26252) 197 : cluster [DBG] pgmap v360: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[84576]: cluster 2024-09-13T14:36:06.011273+0000 mgr.y (mgr.26252) 197 : cluster [DBG] pgmap v360: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[84576]: cluster 2024-09-13T14:36:06.272860+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e573: 8 total, 7 up, 8 in 2024-09-13T14:36:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[84576]: cluster 2024-09-13T14:36:06.272860+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e573: 8 total, 7 up, 8 in 2024-09-13T14:36:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[84576]: audit 2024-09-13T14:36:07.037864+0000 mon.c (mon.2) 9 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[84576]: audit 2024-09-13T14:36:07.037864+0000 mon.c (mon.2) 9 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[84576]: audit 2024-09-13T14:36:07.038157+0000 mon.a (mon.0) 621 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[84576]: audit 2024-09-13T14:36:07.038157+0000 mon.a (mon.0) 621 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[87147]: cluster 2024-09-13T14:36:06.011273+0000 mgr.y (mgr.26252) 197 : cluster [DBG] pgmap v360: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:07.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[87147]: cluster 2024-09-13T14:36:06.011273+0000 mgr.y (mgr.26252) 197 : cluster [DBG] pgmap v360: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:07.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[87147]: cluster 2024-09-13T14:36:06.272860+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e573: 8 total, 7 up, 8 in 2024-09-13T14:36:07.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[87147]: cluster 2024-09-13T14:36:06.272860+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e573: 8 total, 7 up, 8 in 2024-09-13T14:36:07.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[87147]: audit 2024-09-13T14:36:07.037864+0000 mon.c (mon.2) 9 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[87147]: audit 2024-09-13T14:36:07.037864+0000 mon.c (mon.2) 9 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[87147]: audit 2024-09-13T14:36:07.038157+0000 mon.a (mon.0) 621 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:07 smithi099 bash[87147]: audit 2024-09-13T14:36:07.038157+0000 mon.a (mon.0) 621 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:07 smithi177 bash[68594]: cluster 2024-09-13T14:36:06.011273+0000 mgr.y (mgr.26252) 197 : cluster [DBG] pgmap v360: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:07 smithi177 bash[68594]: cluster 2024-09-13T14:36:06.011273+0000 mgr.y (mgr.26252) 197 : cluster [DBG] pgmap v360: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 23 KiB/s wr, 291 op/s; 34/237 objects degraded (14.346%) 2024-09-13T14:36:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:07 smithi177 bash[68594]: cluster 2024-09-13T14:36:06.272860+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e573: 8 total, 7 up, 8 in 2024-09-13T14:36:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:07 smithi177 bash[68594]: cluster 2024-09-13T14:36:06.272860+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e573: 8 total, 7 up, 8 in 2024-09-13T14:36:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:07 smithi177 bash[68594]: audit 2024-09-13T14:36:07.037864+0000 mon.c (mon.2) 9 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:07 smithi177 bash[68594]: audit 2024-09-13T14:36:07.037864+0000 mon.c (mon.2) 9 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:07 smithi177 bash[68594]: audit 2024-09-13T14:36:07.038157+0000 mon.a (mon.0) 621 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:07 smithi177 bash[68594]: audit 2024-09-13T14:36:07.038157+0000 mon.a (mon.0) 621 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:36:07.872 INFO:teuthology.orchestra.run.smithi099.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:alertmanager.a smithi099 *:9093,9094 running (6m) 3s ago 14m 14.6M - 0.27.0 11f11916f8cd 45a2ce75edd3 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:grafana.a smithi177 *:3000 running (5m) 30s ago 14m 75.5M - 9.4.7 954c08fa6188 443ce4187a0d 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi099.yyrtld smithi099 running (11m) 3s ago 11m 14.6M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 7bce2f3fd8d5 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi177.outtkv smithi177 running (11m) 30s ago 11m 18.8M - 17.2.7-1627-gbbdadfa3 a994b00deb3a b976db2e3d1b 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.x smithi177 *:8443,9283,8765 running (4m) 30s ago 23m 440M - 19.3.0-4697-gd58f7306 fec07016a2b0 24a222989f2c 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.y smithi099 *:8443,9283,8765 running (5m) 3s ago 25m 505M - 19.3.0-4697-gd58f7306 fec07016a2b0 8ebc4af8542d 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:mon.a smithi099 running (4m) 3s ago 25m 58.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 7bae834be590 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:mon.b smithi177 running (118s) 30s ago 23m 39.1M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 6ddaaf41777b 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:mon.c smithi099 running (2m) 3s ago 23m 43.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 8e92ca9c3d4b 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.a smithi099 *:9100 running (6m) 3s ago 14m 8808k - 1.7.0 72c9c2088986 adc238b4fabf 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.b smithi177 *:9100 running (6m) 30s ago 14m 8168k - 1.7.0 72c9c2088986 0e333c475514 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0 smithi099 running (7s) 3s ago 22m - 2524M 19.3.0-4697-gd58f7306 fec07016a2b0 ebecc5a7ecad 2024-09-13T14:36:07.873 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1 smithi099 running (21m) 3s ago 21m 125M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 324b38b19317 2024-09-13T14:36:07.874 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2 smithi099 running (20m) 3s ago 20m 139M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a e6fb2db24160 2024-09-13T14:36:07.874 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3 smithi099 running (19m) 3s ago 19m 128M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 1fad69868d22 2024-09-13T14:36:07.874 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4 smithi177 running (18m) 30s ago 18m 117M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a f54f8f87d384 2024-09-13T14:36:07.874 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5 smithi177 running (17m) 30s ago 18m 113M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a f285f4fee6d3 2024-09-13T14:36:07.874 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6 smithi177 running (17m) 30s ago 17m 135M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 2318776c78d4 2024-09-13T14:36:07.874 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7 smithi177 running (16m) 30s ago 16m 136M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a e2d33a0b16ca 2024-09-13T14:36:07.874 INFO:teuthology.orchestra.run.smithi099.stdout:prometheus.a smithi177 *:9095 running (5m) 30s ago 15m 49.2M - 2.51.0 1d3b7f56885b d360e28d5f8b 2024-09-13T14:36:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[84576]: audit 2024-09-13T14:36:07.280513+0000 mon.a (mon.0) 622 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:36:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[84576]: audit 2024-09-13T14:36:07.280513+0000 mon.a (mon.0) 622 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:36:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[84576]: cluster 2024-09-13T14:36:07.290935+0000 mon.a (mon.0) 623 : cluster [DBG] osdmap e574: 8 total, 7 up, 8 in 2024-09-13T14:36:08.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[84576]: cluster 2024-09-13T14:36:07.290935+0000 mon.a (mon.0) 623 : cluster [DBG] osdmap e574: 8 total, 7 up, 8 in 2024-09-13T14:36:08.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[84576]: audit 2024-09-13T14:36:07.306326+0000 mon.c (mon.2) 10 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[84576]: audit 2024-09-13T14:36:07.306326+0000 mon.c (mon.2) 10 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[84576]: audit 2024-09-13T14:36:07.306980+0000 mon.a (mon.0) 624 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[84576]: audit 2024-09-13T14:36:07.306980+0000 mon.a (mon.0) 624 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[87147]: audit 2024-09-13T14:36:07.280513+0000 mon.a (mon.0) 622 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:36:08.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[87147]: audit 2024-09-13T14:36:07.280513+0000 mon.a (mon.0) 622 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:36:08.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[87147]: cluster 2024-09-13T14:36:07.290935+0000 mon.a (mon.0) 623 : cluster [DBG] osdmap e574: 8 total, 7 up, 8 in 2024-09-13T14:36:08.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[87147]: cluster 2024-09-13T14:36:07.290935+0000 mon.a (mon.0) 623 : cluster [DBG] osdmap e574: 8 total, 7 up, 8 in 2024-09-13T14:36:08.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[87147]: audit 2024-09-13T14:36:07.306326+0000 mon.c (mon.2) 10 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[87147]: audit 2024-09-13T14:36:07.306326+0000 mon.c (mon.2) 10 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[87147]: audit 2024-09-13T14:36:07.306980+0000 mon.a (mon.0) 624 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:08 smithi099 bash[87147]: audit 2024-09-13T14:36:07.306980+0000 mon.a (mon.0) 624 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:08 smithi177 bash[68594]: audit 2024-09-13T14:36:07.280513+0000 mon.a (mon.0) 622 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:36:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:08 smithi177 bash[68594]: audit 2024-09-13T14:36:07.280513+0000 mon.a (mon.0) 622 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:36:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:08 smithi177 bash[68594]: cluster 2024-09-13T14:36:07.290935+0000 mon.a (mon.0) 623 : cluster [DBG] osdmap e574: 8 total, 7 up, 8 in 2024-09-13T14:36:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:08 smithi177 bash[68594]: cluster 2024-09-13T14:36:07.290935+0000 mon.a (mon.0) 623 : cluster [DBG] osdmap e574: 8 total, 7 up, 8 in 2024-09-13T14:36:08.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:08 smithi177 bash[68594]: audit 2024-09-13T14:36:07.306326+0000 mon.c (mon.2) 10 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:08 smithi177 bash[68594]: audit 2024-09-13T14:36:07.306326+0000 mon.c (mon.2) 10 : audit [INF] from='osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:08 smithi177 bash[68594]: audit 2024-09-13T14:36:07.306980+0000 mon.a (mon.0) 624 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:08 smithi177 bash[68594]: audit 2024-09-13T14:36:07.306980+0000 mon.a (mon.0) 624 : audit [INF] from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:36:08.742 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:36:09.335 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: audit 2024-09-13T14:36:07.839640+0000 mgr.y (mgr.26252) 198 : audit [DBG] from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: audit 2024-09-13T14:36:07.839640+0000 mgr.y (mgr.26252) 198 : audit [DBG] from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: cluster 2024-09-13T14:36:08.011833+0000 mgr.y (mgr.26252) 199 : cluster [DBG] pgmap v363: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 34/237 objects degraded (14.346%) 2024-09-13T14:36:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: cluster 2024-09-13T14:36:08.011833+0000 mgr.y (mgr.26252) 199 : cluster [DBG] pgmap v363: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 34/237 objects degraded (14.346%) 2024-09-13T14:36:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: cluster 2024-09-13T14:36:08.299756+0000 mon.a (mon.0) 625 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:36:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: cluster 2024-09-13T14:36:08.299756+0000 mon.a (mon.0) 625 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:36:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: cluster 2024-09-13T14:36:08.322265+0000 mon.a (mon.0) 626 : cluster [INF] osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480] boot 2024-09-13T14:36:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: cluster 2024-09-13T14:36:08.322265+0000 mon.a (mon.0) 626 : cluster [INF] osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480] boot 2024-09-13T14:36:09.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: cluster 2024-09-13T14:36:08.322414+0000 mon.a (mon.0) 627 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-13T14:36:09.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: cluster 2024-09-13T14:36:08.322414+0000 mon.a (mon.0) 627 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-13T14:36:09.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: audit 2024-09-13T14:36:08.323916+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:36:09.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: audit 2024-09-13T14:36:08.323916+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:36:09.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: audit 2024-09-13T14:36:08.326583+0000 mon.a (mon.0) 629 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:09.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[84576]: audit 2024-09-13T14:36:08.326583+0000 mon.a (mon.0) 629 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:09.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: audit 2024-09-13T14:36:07.839640+0000 mgr.y (mgr.26252) 198 : audit [DBG] from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:09.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: audit 2024-09-13T14:36:07.839640+0000 mgr.y (mgr.26252) 198 : audit [DBG] from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:09.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: cluster 2024-09-13T14:36:08.011833+0000 mgr.y (mgr.26252) 199 : cluster [DBG] pgmap v363: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 34/237 objects degraded (14.346%) 2024-09-13T14:36:09.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: cluster 2024-09-13T14:36:08.011833+0000 mgr.y (mgr.26252) 199 : cluster [DBG] pgmap v363: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 34/237 objects degraded (14.346%) 2024-09-13T14:36:09.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: cluster 2024-09-13T14:36:08.299756+0000 mon.a (mon.0) 625 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:36:09.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: cluster 2024-09-13T14:36:08.299756+0000 mon.a (mon.0) 625 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:36:09.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: cluster 2024-09-13T14:36:08.322265+0000 mon.a (mon.0) 626 : cluster [INF] osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480] boot 2024-09-13T14:36:09.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: cluster 2024-09-13T14:36:08.322265+0000 mon.a (mon.0) 626 : cluster [INF] osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480] boot 2024-09-13T14:36:09.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: cluster 2024-09-13T14:36:08.322414+0000 mon.a (mon.0) 627 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-13T14:36:09.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: cluster 2024-09-13T14:36:08.322414+0000 mon.a (mon.0) 627 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-13T14:36:09.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: audit 2024-09-13T14:36:08.323916+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:36:09.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: audit 2024-09-13T14:36:08.323916+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:36:09.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: audit 2024-09-13T14:36:08.326583+0000 mon.a (mon.0) 629 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:09.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:09 smithi099 bash[87147]: audit 2024-09-13T14:36:08.326583+0000 mon.a (mon.0) 629 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:09.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: audit 2024-09-13T14:36:07.839640+0000 mgr.y (mgr.26252) 198 : audit [DBG] from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:09.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: audit 2024-09-13T14:36:07.839640+0000 mgr.y (mgr.26252) 198 : audit [DBG] from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:09.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: cluster 2024-09-13T14:36:08.011833+0000 mgr.y (mgr.26252) 199 : cluster [DBG] pgmap v363: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 34/237 objects degraded (14.346%) 2024-09-13T14:36:09.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: cluster 2024-09-13T14:36:08.011833+0000 mgr.y (mgr.26252) 199 : cluster [DBG] pgmap v363: 169 pgs: 32 unknown, 29 active+undersized, 16 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 34/237 objects degraded (14.346%) 2024-09-13T14:36:09.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: cluster 2024-09-13T14:36:08.299756+0000 mon.a (mon.0) 625 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:36:09.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: cluster 2024-09-13T14:36:08.299756+0000 mon.a (mon.0) 625 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:36:09.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: cluster 2024-09-13T14:36:08.322265+0000 mon.a (mon.0) 626 : cluster [INF] osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480] boot 2024-09-13T14:36:09.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: cluster 2024-09-13T14:36:08.322265+0000 mon.a (mon.0) 626 : cluster [INF] osd.0 [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480] boot 2024-09-13T14:36:09.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: cluster 2024-09-13T14:36:08.322414+0000 mon.a (mon.0) 627 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-13T14:36:09.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: cluster 2024-09-13T14:36:08.322414+0000 mon.a (mon.0) 627 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-13T14:36:09.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: audit 2024-09-13T14:36:08.323916+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:36:09.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: audit 2024-09-13T14:36:08.323916+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:36:09.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: audit 2024-09-13T14:36:08.326583+0000 mon.a (mon.0) 629 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:09.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:09 smithi177 bash[68594]: audit 2024-09-13T14:36:08.326583+0000 mon.a (mon.0) 629 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:10.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:36:10 smithi177 bash[61439]: ts=2024-09-13T14:36:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:10.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:10 smithi177 bash[68594]: audit 2024-09-13T14:36:09.319448+0000 mon.a (mon.0) 630 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:10.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:10 smithi177 bash[68594]: audit 2024-09-13T14:36:09.319448+0000 mon.a (mon.0) 630 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:10.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:10 smithi177 bash[68594]: cluster 2024-09-13T14:36:09.324781+0000 mon.a (mon.0) 631 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-13T14:36:10.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:10 smithi177 bash[68594]: cluster 2024-09-13T14:36:09.324781+0000 mon.a (mon.0) 631 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-13T14:36:10.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:10 smithi099 bash[84576]: audit 2024-09-13T14:36:09.319448+0000 mon.a (mon.0) 630 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:10.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:10 smithi099 bash[84576]: audit 2024-09-13T14:36:09.319448+0000 mon.a (mon.0) 630 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:10.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:10 smithi099 bash[84576]: cluster 2024-09-13T14:36:09.324781+0000 mon.a (mon.0) 631 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-13T14:36:10.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:10 smithi099 bash[84576]: cluster 2024-09-13T14:36:09.324781+0000 mon.a (mon.0) 631 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-13T14:36:10.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:10 smithi099 bash[87147]: audit 2024-09-13T14:36:09.319448+0000 mon.a (mon.0) 630 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:10.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:10 smithi099 bash[87147]: audit 2024-09-13T14:36:09.319448+0000 mon.a (mon.0) 630 : audit [INF] from='client.? 172.21.15.99:0/1908420454' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:10.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:10 smithi099 bash[87147]: cluster 2024-09-13T14:36:09.324781+0000 mon.a (mon.0) 631 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-13T14:36:10.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:10 smithi099 bash[87147]: cluster 2024-09-13T14:36:09.324781+0000 mon.a (mon.0) 631 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-13T14:36:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:11 smithi177 bash[68594]: cluster 2024-09-13T14:36:10.012598+0000 mgr.y (mgr.26252) 200 : cluster [DBG] pgmap v366: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:11 smithi177 bash[68594]: cluster 2024-09-13T14:36:10.012598+0000 mgr.y (mgr.26252) 200 : cluster [DBG] pgmap v366: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:11 smithi177 bash[68594]: cluster 2024-09-13T14:36:10.341725+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-13T14:36:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:11 smithi177 bash[68594]: cluster 2024-09-13T14:36:10.341725+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-13T14:36:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:11 smithi177 bash[68594]: cluster 2024-09-13T14:36:11.021545+0000 mon.a (mon.0) 633 : cluster [WRN] Health check update: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:11.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:11 smithi177 bash[68594]: cluster 2024-09-13T14:36:11.021545+0000 mon.a (mon.0) 633 : cluster [WRN] Health check update: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[84576]: cluster 2024-09-13T14:36:10.012598+0000 mgr.y (mgr.26252) 200 : cluster [DBG] pgmap v366: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[84576]: cluster 2024-09-13T14:36:10.012598+0000 mgr.y (mgr.26252) 200 : cluster [DBG] pgmap v366: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[84576]: cluster 2024-09-13T14:36:10.341725+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-13T14:36:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[84576]: cluster 2024-09-13T14:36:10.341725+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-13T14:36:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[84576]: cluster 2024-09-13T14:36:11.021545+0000 mon.a (mon.0) 633 : cluster [WRN] Health check update: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[84576]: cluster 2024-09-13T14:36:11.021545+0000 mon.a (mon.0) 633 : cluster [WRN] Health check update: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[87147]: cluster 2024-09-13T14:36:10.012598+0000 mgr.y (mgr.26252) 200 : cluster [DBG] pgmap v366: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[87147]: cluster 2024-09-13T14:36:10.012598+0000 mgr.y (mgr.26252) 200 : cluster [DBG] pgmap v366: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[87147]: cluster 2024-09-13T14:36:10.341725+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-13T14:36:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[87147]: cluster 2024-09-13T14:36:10.341725+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-13T14:36:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[87147]: cluster 2024-09-13T14:36:11.021545+0000 mon.a (mon.0) 633 : cluster [WRN] Health check update: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:11 smithi099 bash[87147]: cluster 2024-09-13T14:36:11.021545+0000 mon.a (mon.0) 633 : cluster [WRN] Health check update: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:12.450 INFO:tasks.workunit.client.0.smithi099.stdout:image: image22 2024-09-13T14:36:12.451 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image22 2024-09-13T14:36:12.451 INFO:tasks.workunit.client.0.smithi099.stdout:found image22 2024-09-13T14:36:12.451 INFO:tasks.workunit.client.0.smithi099.stdout:erased image22 2024-09-13T14:36:12.451 INFO:tasks.workunit.client.0.smithi099.stdout:image: image22 2024-09-13T14:36:12.451 INFO:tasks.workunit.client.0.smithi099.stdout:image: image23 2024-09-13T14:36:12.452 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image22 2024-09-13T14:36:12.452 INFO:tasks.workunit.client.0.smithi099.stdout:found image22 2024-09-13T14:36:12.452 INFO:tasks.workunit.client.0.smithi099.stdout:erased image22 2024-09-13T14:36:12.452 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image23 2024-09-13T14:36:12.452 INFO:tasks.workunit.client.0.smithi099.stdout:found image23 2024-09-13T14:36:12.453 INFO:tasks.workunit.client.0.smithi099.stdout:erased image23 2024-09-13T14:36:12.453 INFO:tasks.workunit.client.0.smithi099.stdout:100.00% done 2024-09-13T14:36:12.453 INFO:tasks.workunit.client.0.smithi099.stdout:image: image22 2024-09-13T14:36:12.453 INFO:tasks.workunit.client.0.smithi099.stdout:image: image23 2024-09-13T14:36:12.453 INFO:tasks.workunit.client.0.smithi099.stdout:image: image24 2024-09-13T14:36:12.453 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image22 2024-09-13T14:36:12.454 INFO:tasks.workunit.client.0.smithi099.stdout:found image22 2024-09-13T14:36:12.454 INFO:tasks.workunit.client.0.smithi099.stdout:erased image22 2024-09-13T14:36:12.454 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image23 2024-09-13T14:36:12.454 INFO:tasks.workunit.client.0.smithi099.stdout:found image23 2024-09-13T14:36:12.454 INFO:tasks.workunit.client.0.smithi099.stdout:erased image23 2024-09-13T14:36:12.455 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image24 2024-09-13T14:36:12.455 INFO:tasks.workunit.client.0.smithi099.stdout:found image24 2024-09-13T14:36:12.455 INFO:tasks.workunit.client.0.smithi099.stdout:erased image24 2024-09-13T14:36:12.455 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCopy (4948 ms) 2024-09-13T14:36:12.455 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCopyPP 2024-09-13T14:36:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:12 smithi177 bash[68594]: cluster 2024-09-13T14:36:11.343593+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-13T14:36:12.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:12 smithi177 bash[68594]: cluster 2024-09-13T14:36:11.343593+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-13T14:36:12.770 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:12 smithi099 bash[84576]: cluster 2024-09-13T14:36:11.343593+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-13T14:36:12.770 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:12 smithi099 bash[84576]: cluster 2024-09-13T14:36:11.343593+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-13T14:36:12.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:12 smithi099 bash[87147]: cluster 2024-09-13T14:36:11.343593+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-13T14:36:12.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:12 smithi099 bash[87147]: cluster 2024-09-13T14:36:11.343593+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-13T14:36:13.281 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:13.635 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:13 smithi099 bash[87147]: cluster 2024-09-13T14:36:12.013469+0000 mgr.y (mgr.26252) 201 : cluster [DBG] pgmap v369: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:13.635 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:13 smithi099 bash[87147]: cluster 2024-09-13T14:36:12.013469+0000 mgr.y (mgr.26252) 201 : cluster [DBG] pgmap v369: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:13.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:13 smithi099 bash[84576]: cluster 2024-09-13T14:36:12.013469+0000 mgr.y (mgr.26252) 201 : cluster [DBG] pgmap v369: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:13.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:13 smithi099 bash[84576]: cluster 2024-09-13T14:36:12.013469+0000 mgr.y (mgr.26252) 201 : cluster [DBG] pgmap v369: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:13.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:13 smithi177 bash[68594]: cluster 2024-09-13T14:36:12.013469+0000 mgr.y (mgr.26252) 201 : cluster [DBG] pgmap v369: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:13.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:13 smithi177 bash[68594]: cluster 2024-09-13T14:36:12.013469+0000 mgr.y (mgr.26252) 201 : cluster [DBG] pgmap v369: 201 pgs: 29 creating+peering, 36 peering, 6 unknown, 11 active+undersized, 5 active+undersized+degraded, 114 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 36 op/s; 11/252 objects degraded (4.365%) 2024-09-13T14:36:14.379 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:14.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[84576]: cluster 2024-09-13T14:36:13.369235+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-13T14:36:14.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[84576]: cluster 2024-09-13T14:36:13.369235+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-13T14:36:14.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[84576]: audit 2024-09-13T14:36:13.387754+0000 mon.a (mon.0) 636 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:14.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[84576]: audit 2024-09-13T14:36:13.387754+0000 mon.a (mon.0) 636 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:14.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[84576]: audit 2024-09-13T14:36:14.360932+0000 mon.a (mon.0) 637 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:14.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[84576]: audit 2024-09-13T14:36:14.360932+0000 mon.a (mon.0) 637 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:14.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[84576]: cluster 2024-09-13T14:36:14.370666+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-13T14:36:14.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[84576]: cluster 2024-09-13T14:36:14.370666+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-13T14:36:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[87147]: cluster 2024-09-13T14:36:13.369235+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-13T14:36:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[87147]: cluster 2024-09-13T14:36:13.369235+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-13T14:36:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[87147]: audit 2024-09-13T14:36:13.387754+0000 mon.a (mon.0) 636 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[87147]: audit 2024-09-13T14:36:13.387754+0000 mon.a (mon.0) 636 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[87147]: audit 2024-09-13T14:36:14.360932+0000 mon.a (mon.0) 637 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:14.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[87147]: audit 2024-09-13T14:36:14.360932+0000 mon.a (mon.0) 637 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:14.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[87147]: cluster 2024-09-13T14:36:14.370666+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-13T14:36:14.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:14 smithi099 bash[87147]: cluster 2024-09-13T14:36:14.370666+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-13T14:36:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:14 smithi177 bash[68594]: cluster 2024-09-13T14:36:13.369235+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-13T14:36:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:14 smithi177 bash[68594]: cluster 2024-09-13T14:36:13.369235+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-13T14:36:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:14 smithi177 bash[68594]: audit 2024-09-13T14:36:13.387754+0000 mon.a (mon.0) 636 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:14 smithi177 bash[68594]: audit 2024-09-13T14:36:13.387754+0000 mon.a (mon.0) 636 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:14 smithi177 bash[68594]: audit 2024-09-13T14:36:14.360932+0000 mon.a (mon.0) 637 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:14 smithi177 bash[68594]: audit 2024-09-13T14:36:14.360932+0000 mon.a (mon.0) 637 : audit [INF] from='client.? 172.21.15.99:0/2881507310' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:14 smithi177 bash[68594]: cluster 2024-09-13T14:36:14.370666+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-13T14:36:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:14 smithi177 bash[68594]: cluster 2024-09-13T14:36:14.370666+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-13T14:36:15.672 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[84576]: cluster 2024-09-13T14:36:14.015805+0000 mgr.y (mgr.26252) 202 : cluster [DBG] pgmap v371: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 292 KiB/s rd, 19 KiB/s wr, 341 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:15.672 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[84576]: cluster 2024-09-13T14:36:14.015805+0000 mgr.y (mgr.26252) 202 : cluster [DBG] pgmap v371: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 292 KiB/s rd, 19 KiB/s wr, 341 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:15.672 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[84576]: cluster 2024-09-13T14:36:14.382617+0000 mon.a (mon.0) 639 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:15.672 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[84576]: cluster 2024-09-13T14:36:14.382617+0000 mon.a (mon.0) 639 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:15.672 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[84576]: cluster 2024-09-13T14:36:14.382695+0000 mon.a (mon.0) 640 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded) 2024-09-13T14:36:15.673 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[84576]: cluster 2024-09-13T14:36:14.382695+0000 mon.a (mon.0) 640 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded) 2024-09-13T14:36:15.673 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[84576]: cluster 2024-09-13T14:36:15.397594+0000 mon.a (mon.0) 641 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-13T14:36:15.673 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[84576]: cluster 2024-09-13T14:36:15.397594+0000 mon.a (mon.0) 641 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-13T14:36:15.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[87147]: cluster 2024-09-13T14:36:14.015805+0000 mgr.y (mgr.26252) 202 : cluster [DBG] pgmap v371: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 292 KiB/s rd, 19 KiB/s wr, 341 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:15.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[87147]: cluster 2024-09-13T14:36:14.015805+0000 mgr.y (mgr.26252) 202 : cluster [DBG] pgmap v371: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 292 KiB/s rd, 19 KiB/s wr, 341 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:15.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[87147]: cluster 2024-09-13T14:36:14.382617+0000 mon.a (mon.0) 639 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:15.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[87147]: cluster 2024-09-13T14:36:14.382617+0000 mon.a (mon.0) 639 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:15.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[87147]: cluster 2024-09-13T14:36:14.382695+0000 mon.a (mon.0) 640 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded) 2024-09-13T14:36:15.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[87147]: cluster 2024-09-13T14:36:14.382695+0000 mon.a (mon.0) 640 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded) 2024-09-13T14:36:15.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[87147]: cluster 2024-09-13T14:36:15.397594+0000 mon.a (mon.0) 641 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-13T14:36:15.674 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:15 smithi099 bash[87147]: cluster 2024-09-13T14:36:15.397594+0000 mon.a (mon.0) 641 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-13T14:36:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:15 smithi177 bash[68594]: cluster 2024-09-13T14:36:14.015805+0000 mgr.y (mgr.26252) 202 : cluster [DBG] pgmap v371: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 292 KiB/s rd, 19 KiB/s wr, 341 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:15 smithi177 bash[68594]: cluster 2024-09-13T14:36:14.015805+0000 mgr.y (mgr.26252) 202 : cluster [DBG] pgmap v371: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 292 KiB/s rd, 19 KiB/s wr, 341 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:15 smithi177 bash[68594]: cluster 2024-09-13T14:36:14.382617+0000 mon.a (mon.0) 639 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:15 smithi177 bash[68594]: cluster 2024-09-13T14:36:14.382617+0000 mon.a (mon.0) 639 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:15 smithi177 bash[68594]: cluster 2024-09-13T14:36:14.382695+0000 mon.a (mon.0) 640 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded) 2024-09-13T14:36:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:15 smithi177 bash[68594]: cluster 2024-09-13T14:36:14.382695+0000 mon.a (mon.0) 640 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/252 objects degraded (4.365%), 5 pgs degraded) 2024-09-13T14:36:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:15 smithi177 bash[68594]: cluster 2024-09-13T14:36:15.397594+0000 mon.a (mon.0) 641 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-13T14:36:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:15 smithi177 bash[68594]: cluster 2024-09-13T14:36:15.397594+0000 mon.a (mon.0) 641 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-13T14:36:16.371 INFO:tasks.workunit.client.0.smithi099.stdout:num images is: 1 2024-09-13T14:36:16.371 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:36:16.372 INFO:tasks.workunit.client.0.smithi099.stdout:image: image25 2024-09-13T14:36:16.372 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image25 2024-09-13T14:36:16.400 INFO:tasks.workunit.client.0.smithi099.stdout:num images is: 2 2024-09-13T14:36:16.400 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:36:16.400 INFO:tasks.workunit.client.0.smithi099.stdout:image: image25 2024-09-13T14:36:16.401 INFO:tasks.workunit.client.0.smithi099.stdout:image: image26 2024-09-13T14:36:16.401 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image25 2024-09-13T14:36:16.401 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image26 2024-09-13T14:36:16.466 INFO:tasks.workunit.client.0.smithi099.stdout:100.00% done 2024-09-13T14:36:16.466 INFO:tasks.workunit.client.0.smithi099.stdout:num images is: 3 2024-09-13T14:36:16.466 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 3 2024-09-13T14:36:16.467 INFO:tasks.workunit.client.0.smithi099.stdout:image: image25 2024-09-13T14:36:16.467 INFO:tasks.workunit.client.0.smithi099.stdout:image: image26 2024-09-13T14:36:16.467 INFO:tasks.workunit.client.0.smithi099.stdout:image: image27 2024-09-13T14:36:16.467 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image25 2024-09-13T14:36:16.467 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image26 2024-09-13T14:36:16.467 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image27 2024-09-13T14:36:16.521 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCopyPP (4071 ms) 2024-09-13T14:36:16.521 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestDeepCopy 2024-09-13T14:36:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:36:17] "GET /metrics HTTP/1.1" 200 38922 "" "Prometheus/2.51.0" 2024-09-13T14:36:17.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[84576]: cluster 2024-09-13T14:36:16.016766+0000 mgr.y (mgr.26252) 203 : cluster [DBG] pgmap v374: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 293 KiB/s rd, 19 KiB/s wr, 342 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:17.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[84576]: cluster 2024-09-13T14:36:16.016766+0000 mgr.y (mgr.26252) 203 : cluster [DBG] pgmap v374: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 293 KiB/s rd, 19 KiB/s wr, 342 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:17.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[84576]: cluster 2024-09-13T14:36:16.033283+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[84576]: cluster 2024-09-13T14:36:16.033283+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[84576]: cluster 2024-09-13T14:36:16.404146+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[84576]: cluster 2024-09-13T14:36:16.404146+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[84576]: cluster 2024-09-13T14:36:16.404175+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[84576]: cluster 2024-09-13T14:36:16.404175+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[87147]: cluster 2024-09-13T14:36:16.016766+0000 mgr.y (mgr.26252) 203 : cluster [DBG] pgmap v374: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 293 KiB/s rd, 19 KiB/s wr, 342 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[87147]: cluster 2024-09-13T14:36:16.016766+0000 mgr.y (mgr.26252) 203 : cluster [DBG] pgmap v374: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 293 KiB/s rd, 19 KiB/s wr, 342 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[87147]: cluster 2024-09-13T14:36:16.033283+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[87147]: cluster 2024-09-13T14:36:16.033283+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[87147]: cluster 2024-09-13T14:36:16.404146+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[87147]: cluster 2024-09-13T14:36:16.404146+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[87147]: cluster 2024-09-13T14:36:16.404175+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:17 smithi099 bash[87147]: cluster 2024-09-13T14:36:16.404175+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:17.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:17 smithi177 bash[68594]: cluster 2024-09-13T14:36:16.016766+0000 mgr.y (mgr.26252) 203 : cluster [DBG] pgmap v374: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 293 KiB/s rd, 19 KiB/s wr, 342 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:17.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:17 smithi177 bash[68594]: cluster 2024-09-13T14:36:16.016766+0000 mgr.y (mgr.26252) 203 : cluster [DBG] pgmap v374: 233 pgs: 31 creating+peering, 1 unknown, 201 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 293 KiB/s rd, 19 KiB/s wr, 342 op/s; 103 B/s, 14 keys/s, 5 objects/s recovering 2024-09-13T14:36:17.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:17 smithi177 bash[68594]: cluster 2024-09-13T14:36:16.033283+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-13T14:36:17.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:17 smithi177 bash[68594]: cluster 2024-09-13T14:36:16.033283+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-13T14:36:17.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:17 smithi177 bash[68594]: cluster 2024-09-13T14:36:16.404146+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:17.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:17 smithi177 bash[68594]: cluster 2024-09-13T14:36:16.404146+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:17.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:17 smithi177 bash[68594]: cluster 2024-09-13T14:36:16.404175+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:17.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:17 smithi177 bash[68594]: cluster 2024-09-13T14:36:16.404175+0000 mon.a (mon.0) 644 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:18.442 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[87147]: cluster 2024-09-13T14:36:17.414853+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-13T14:36:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[87147]: cluster 2024-09-13T14:36:17.414853+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-13T14:36:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[87147]: audit 2024-09-13T14:36:17.435132+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.99:0/1716724057' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[87147]: audit 2024-09-13T14:36:17.435132+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.99:0/1716724057' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[87147]: audit 2024-09-13T14:36:17.436999+0000 mon.a (mon.0) 646 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[87147]: audit 2024-09-13T14:36:17.436999+0000 mon.a (mon.0) 646 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[87147]: audit 2024-09-13T14:36:18.253155+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[87147]: audit 2024-09-13T14:36:18.253155+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[87147]: audit 2024-09-13T14:36:18.255298+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[87147]: audit 2024-09-13T14:36:18.255298+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[84576]: cluster 2024-09-13T14:36:17.414853+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[84576]: cluster 2024-09-13T14:36:17.414853+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[84576]: audit 2024-09-13T14:36:17.435132+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.99:0/1716724057' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[84576]: audit 2024-09-13T14:36:17.435132+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.99:0/1716724057' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[84576]: audit 2024-09-13T14:36:17.436999+0000 mon.a (mon.0) 646 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[84576]: audit 2024-09-13T14:36:17.436999+0000 mon.a (mon.0) 646 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[84576]: audit 2024-09-13T14:36:18.253155+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[84576]: audit 2024-09-13T14:36:18.253155+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:18.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[84576]: audit 2024-09-13T14:36:18.255298+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:18.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[84576]: audit 2024-09-13T14:36:18.255298+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:18.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:18 smithi177 bash[68594]: cluster 2024-09-13T14:36:17.414853+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-13T14:36:18.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:18 smithi177 bash[68594]: cluster 2024-09-13T14:36:17.414853+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-13T14:36:18.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:18 smithi177 bash[68594]: audit 2024-09-13T14:36:17.435132+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.99:0/1716724057' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:18 smithi177 bash[68594]: audit 2024-09-13T14:36:17.435132+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.99:0/1716724057' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:18 smithi177 bash[68594]: audit 2024-09-13T14:36:17.436999+0000 mon.a (mon.0) 646 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:18 smithi177 bash[68594]: audit 2024-09-13T14:36:17.436999+0000 mon.a (mon.0) 646 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:18.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:18 smithi177 bash[68594]: audit 2024-09-13T14:36:18.253155+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:18.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:18 smithi177 bash[68594]: audit 2024-09-13T14:36:18.253155+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:18.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:18 smithi177 bash[68594]: audit 2024-09-13T14:36:18.255298+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:18.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:18 smithi177 bash[68594]: audit 2024-09-13T14:36:18.255298+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:19.262 INFO:teuthology.orchestra.run.smithi099.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:19.274 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[76324]: debug 2024-09-13T14:36:18.823+0000 7fd55bc0f640 -1 mgr.server reply reply (11) Resource temporarily unavailable 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:36:19.274 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:18 smithi099 bash[76324]: debug 2024-09-13T14:36:18.823+0000 7fd55bc0f640 -1 mgr.server reply reply (16) Device or resource busy 32 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:36:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: cluster 2024-09-13T14:36:18.017647+0000 mgr.y (mgr.26252) 204 : cluster [DBG] pgmap v377: 265 pgs: 32 unknown, 31 creating+peering, 202 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 1023 B/s wr, 32 op/s 2024-09-13T14:36:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: cluster 2024-09-13T14:36:18.017647+0000 mgr.y (mgr.26252) 204 : cluster [DBG] pgmap v377: 265 pgs: 32 unknown, 31 creating+peering, 202 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 1023 B/s wr, 32 op/s 2024-09-13T14:36:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: cluster 2024-09-13T14:36:18.409483+0000 mon.a (mon.0) 649 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: cluster 2024-09-13T14:36:18.409483+0000 mon.a (mon.0) 649 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.419502+0000 mon.a (mon.0) 650 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.419502+0000 mon.a (mon.0) 650 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: cluster 2024-09-13T14:36:18.428470+0000 mon.a (mon.0) 651 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-13T14:36:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: cluster 2024-09-13T14:36:18.428470+0000 mon.a (mon.0) 651 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.686235+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.686235+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.694066+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.694066+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.695817+0000 mon.a (mon.0) 654 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.695817+0000 mon.a (mon.0) 654 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.697098+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.697098+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.707124+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.707124+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.815778+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: cluster 2024-09-13T14:36:18.017647+0000 mgr.y (mgr.26252) 204 : cluster [DBG] pgmap v377: 265 pgs: 32 unknown, 31 creating+peering, 202 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 1023 B/s wr, 32 op/s 2024-09-13T14:36:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: cluster 2024-09-13T14:36:18.017647+0000 mgr.y (mgr.26252) 204 : cluster [DBG] pgmap v377: 265 pgs: 32 unknown, 31 creating+peering, 202 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 1023 B/s wr, 32 op/s 2024-09-13T14:36:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: cluster 2024-09-13T14:36:18.409483+0000 mon.a (mon.0) 649 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: cluster 2024-09-13T14:36:18.409483+0000 mon.a (mon.0) 649 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.419502+0000 mon.a (mon.0) 650 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.419502+0000 mon.a (mon.0) 650 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: cluster 2024-09-13T14:36:18.428470+0000 mon.a (mon.0) 651 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-13T14:36:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: cluster 2024-09-13T14:36:18.428470+0000 mon.a (mon.0) 651 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-13T14:36:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.686235+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.686235+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.694066+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.694066+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.695817+0000 mon.a (mon.0) 654 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.695817+0000 mon.a (mon.0) 654 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.697098+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.697098+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.707124+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.707124+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.815778+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.815778+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.818872+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.818872+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.821583+0000 mon.a (mon.0) 659 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.821583+0000 mon.a (mon.0) 659 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.823494+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.823494+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.825808+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:18.825808+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:19.264738+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[84576]: audit 2024-09-13T14:36:19.264738+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.815778+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.818872+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.818872+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.821583+0000 mon.a (mon.0) 659 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.821583+0000 mon.a (mon.0) 659 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.823494+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.823494+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.825808+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:18.825808+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:19.264738+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:19 smithi099 bash[87147]: audit 2024-09-13T14:36:19.264738+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: cluster 2024-09-13T14:36:18.017647+0000 mgr.y (mgr.26252) 204 : cluster [DBG] pgmap v377: 265 pgs: 32 unknown, 31 creating+peering, 202 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 1023 B/s wr, 32 op/s 2024-09-13T14:36:19.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: cluster 2024-09-13T14:36:18.017647+0000 mgr.y (mgr.26252) 204 : cluster [DBG] pgmap v377: 265 pgs: 32 unknown, 31 creating+peering, 202 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 1023 B/s wr, 32 op/s 2024-09-13T14:36:19.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: cluster 2024-09-13T14:36:18.409483+0000 mon.a (mon.0) 649 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:19.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: cluster 2024-09-13T14:36:18.409483+0000 mon.a (mon.0) 649 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:36:19.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.419502+0000 mon.a (mon.0) 650 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:19.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.419502+0000 mon.a (mon.0) 650 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:19.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: cluster 2024-09-13T14:36:18.428470+0000 mon.a (mon.0) 651 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-13T14:36:19.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: cluster 2024-09-13T14:36:18.428470+0000 mon.a (mon.0) 651 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-13T14:36:19.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.686235+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.686235+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.694066+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.694066+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.695817+0000 mon.a (mon.0) 654 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.695817+0000 mon.a (mon.0) 654 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.697098+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.697098+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.707124+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.707124+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.815778+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.815778+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.818872+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.818872+0000 mon.a (mon.0) 658 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.821583+0000 mon.a (mon.0) 659 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.821583+0000 mon.a (mon.0) 659 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.823494+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.823494+0000 mon.a (mon.0) 660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:19.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.825808+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:36:19.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:18.825808+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:36:19.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:19.264738+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:19.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:19 smithi177 bash[68594]: audit 2024-09-13T14:36:19.264738+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:20.289 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:36:20.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[84576]: audit 2024-09-13T14:36:18.826171+0000 mgr.y (mgr.26252) 205 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:36:20.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[84576]: audit 2024-09-13T14:36:18.826171+0000 mgr.y (mgr.26252) 205 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:36:20.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[84576]: cephadm 2024-09-13T14:36:18.827737+0000 mgr.y (mgr.26252) 206 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:36:20.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[84576]: cephadm 2024-09-13T14:36:18.827737+0000 mgr.y (mgr.26252) 206 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:36:20.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[84576]: audit 2024-09-13T14:36:19.259136+0000 mgr.y (mgr.26252) 207 : audit [DBG] from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[84576]: audit 2024-09-13T14:36:19.259136+0000 mgr.y (mgr.26252) 207 : audit [DBG] from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[84576]: cephadm 2024-09-13T14:36:19.259603+0000 mgr.y (mgr.26252) 208 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[84576]: cephadm 2024-09-13T14:36:19.259603+0000 mgr.y (mgr.26252) 208 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[84576]: cluster 2024-09-13T14:36:19.426104+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[84576]: cluster 2024-09-13T14:36:19.426104+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[87147]: audit 2024-09-13T14:36:18.826171+0000 mgr.y (mgr.26252) 205 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[87147]: audit 2024-09-13T14:36:18.826171+0000 mgr.y (mgr.26252) 205 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[87147]: cephadm 2024-09-13T14:36:18.827737+0000 mgr.y (mgr.26252) 206 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[87147]: cephadm 2024-09-13T14:36:18.827737+0000 mgr.y (mgr.26252) 206 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[87147]: audit 2024-09-13T14:36:19.259136+0000 mgr.y (mgr.26252) 207 : audit [DBG] from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:20.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[87147]: audit 2024-09-13T14:36:19.259136+0000 mgr.y (mgr.26252) 207 : audit [DBG] from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:20.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[87147]: cephadm 2024-09-13T14:36:19.259603+0000 mgr.y (mgr.26252) 208 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:20.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[87147]: cephadm 2024-09-13T14:36:19.259603+0000 mgr.y (mgr.26252) 208 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:20.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[87147]: cluster 2024-09-13T14:36:19.426104+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-13T14:36:20.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:20 smithi099 bash[87147]: cluster 2024-09-13T14:36:19.426104+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-13T14:36:20.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[61439]: ts=2024-09-13T14:36:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[68594]: audit 2024-09-13T14:36:18.826171+0000 mgr.y (mgr.26252) 205 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:36:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[68594]: audit 2024-09-13T14:36:18.826171+0000 mgr.y (mgr.26252) 205 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:36:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[68594]: cephadm 2024-09-13T14:36:18.827737+0000 mgr.y (mgr.26252) 206 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:36:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[68594]: cephadm 2024-09-13T14:36:18.827737+0000 mgr.y (mgr.26252) 206 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:36:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[68594]: audit 2024-09-13T14:36:19.259136+0000 mgr.y (mgr.26252) 207 : audit [DBG] from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[68594]: audit 2024-09-13T14:36:19.259136+0000 mgr.y (mgr.26252) 207 : audit [DBG] from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[68594]: cephadm 2024-09-13T14:36:19.259603+0000 mgr.y (mgr.26252) 208 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:20.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[68594]: cephadm 2024-09-13T14:36:19.259603+0000 mgr.y (mgr.26252) 208 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:36:20.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[68594]: cluster 2024-09-13T14:36:19.426104+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-13T14:36:20.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:20 smithi177 bash[68594]: cluster 2024-09-13T14:36:19.426104+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-13T14:36:21.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[84576]: cluster 2024-09-13T14:36:20.018514+0000 mgr.y (mgr.26252) 209 : cluster [DBG] pgmap v380: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T14:36:21.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[84576]: cluster 2024-09-13T14:36:20.018514+0000 mgr.y (mgr.26252) 209 : cluster [DBG] pgmap v380: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T14:36:21.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[84576]: cluster 2024-09-13T14:36:20.432987+0000 mon.a (mon.0) 664 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:21.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[84576]: cluster 2024-09-13T14:36:20.432987+0000 mon.a (mon.0) 664 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[84576]: cluster 2024-09-13T14:36:20.433039+0000 mon.a (mon.0) 665 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[84576]: cluster 2024-09-13T14:36:20.433039+0000 mon.a (mon.0) 665 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[84576]: cluster 2024-09-13T14:36:20.450407+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-13T14:36:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[84576]: cluster 2024-09-13T14:36:20.450407+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-13T14:36:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[84576]: cluster 2024-09-13T14:36:21.038240+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-13T14:36:21.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[84576]: cluster 2024-09-13T14:36:21.038240+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-13T14:36:21.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[87147]: cluster 2024-09-13T14:36:20.018514+0000 mgr.y (mgr.26252) 209 : cluster [DBG] pgmap v380: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T14:36:21.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[87147]: cluster 2024-09-13T14:36:20.018514+0000 mgr.y (mgr.26252) 209 : cluster [DBG] pgmap v380: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T14:36:21.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[87147]: cluster 2024-09-13T14:36:20.432987+0000 mon.a (mon.0) 664 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:21.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[87147]: cluster 2024-09-13T14:36:20.432987+0000 mon.a (mon.0) 664 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:21.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[87147]: cluster 2024-09-13T14:36:20.433039+0000 mon.a (mon.0) 665 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:21.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[87147]: cluster 2024-09-13T14:36:20.433039+0000 mon.a (mon.0) 665 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:21.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[87147]: cluster 2024-09-13T14:36:20.450407+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-13T14:36:21.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[87147]: cluster 2024-09-13T14:36:20.450407+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-13T14:36:21.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[87147]: cluster 2024-09-13T14:36:21.038240+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-13T14:36:21.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:21 smithi099 bash[87147]: cluster 2024-09-13T14:36:21.038240+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-13T14:36:21.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:21 smithi177 bash[68594]: cluster 2024-09-13T14:36:20.018514+0000 mgr.y (mgr.26252) 209 : cluster [DBG] pgmap v380: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T14:36:21.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:21 smithi177 bash[68594]: cluster 2024-09-13T14:36:20.018514+0000 mgr.y (mgr.26252) 209 : cluster [DBG] pgmap v380: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T14:36:21.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:21 smithi177 bash[68594]: cluster 2024-09-13T14:36:20.432987+0000 mon.a (mon.0) 664 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:21.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:21 smithi177 bash[68594]: cluster 2024-09-13T14:36:20.432987+0000 mon.a (mon.0) 664 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:36:21.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:21 smithi177 bash[68594]: cluster 2024-09-13T14:36:20.433039+0000 mon.a (mon.0) 665 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:21.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:21 smithi177 bash[68594]: cluster 2024-09-13T14:36:20.433039+0000 mon.a (mon.0) 665 : cluster [INF] Cluster is now healthy 2024-09-13T14:36:21.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:21 smithi177 bash[68594]: cluster 2024-09-13T14:36:20.450407+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-13T14:36:21.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:21 smithi177 bash[68594]: cluster 2024-09-13T14:36:20.450407+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-13T14:36:21.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:21 smithi177 bash[68594]: cluster 2024-09-13T14:36:21.038240+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-13T14:36:21.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:21 smithi177 bash[68594]: cluster 2024-09-13T14:36:21.038240+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-13T14:36:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:23 smithi099 bash[84576]: cluster 2024-09-13T14:36:22.019380+0000 mgr.y (mgr.26252) 210 : cluster [DBG] pgmap v383: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 21 KiB/s wr, 365 op/s 2024-09-13T14:36:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:23 smithi099 bash[84576]: cluster 2024-09-13T14:36:22.019380+0000 mgr.y (mgr.26252) 210 : cluster [DBG] pgmap v383: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 21 KiB/s wr, 365 op/s 2024-09-13T14:36:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:23 smithi099 bash[84576]: cluster 2024-09-13T14:36:22.459236+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-13T14:36:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:23 smithi099 bash[84576]: cluster 2024-09-13T14:36:22.459236+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-13T14:36:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:23 smithi099 bash[87147]: cluster 2024-09-13T14:36:22.019380+0000 mgr.y (mgr.26252) 210 : cluster [DBG] pgmap v383: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 21 KiB/s wr, 365 op/s 2024-09-13T14:36:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:23 smithi099 bash[87147]: cluster 2024-09-13T14:36:22.019380+0000 mgr.y (mgr.26252) 210 : cluster [DBG] pgmap v383: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 21 KiB/s wr, 365 op/s 2024-09-13T14:36:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:23 smithi099 bash[87147]: cluster 2024-09-13T14:36:22.459236+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-13T14:36:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:23 smithi099 bash[87147]: cluster 2024-09-13T14:36:22.459236+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-13T14:36:23.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:23 smithi177 bash[68594]: cluster 2024-09-13T14:36:22.019380+0000 mgr.y (mgr.26252) 210 : cluster [DBG] pgmap v383: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 21 KiB/s wr, 365 op/s 2024-09-13T14:36:23.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:23 smithi177 bash[68594]: cluster 2024-09-13T14:36:22.019380+0000 mgr.y (mgr.26252) 210 : cluster [DBG] pgmap v383: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 21 KiB/s wr, 365 op/s 2024-09-13T14:36:23.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:23 smithi177 bash[68594]: cluster 2024-09-13T14:36:22.459236+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-13T14:36:23.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:23 smithi177 bash[68594]: cluster 2024-09-13T14:36:22.459236+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-13T14:36:24.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:24 smithi099 bash[84576]: cluster 2024-09-13T14:36:23.460923+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-13T14:36:24.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:24 smithi099 bash[84576]: cluster 2024-09-13T14:36:23.460923+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-13T14:36:24.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:24 smithi099 bash[87147]: cluster 2024-09-13T14:36:23.460923+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-13T14:36:24.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:24 smithi099 bash[87147]: cluster 2024-09-13T14:36:23.460923+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-13T14:36:24.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:24 smithi177 bash[68594]: cluster 2024-09-13T14:36:23.460923+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-13T14:36:24.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:24 smithi177 bash[68594]: cluster 2024-09-13T14:36:23.460923+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-13T14:36:25.100 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:36:25.657 INFO:tasks.workunit.client.0.smithi099.stdout:image: image28 2024-09-13T14:36:25.657 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image28 2024-09-13T14:36:25.657 INFO:tasks.workunit.client.0.smithi099.stdout:found image28 2024-09-13T14:36:25.657 INFO:tasks.workunit.client.0.smithi099.stdout:erased image28 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:image: image28 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:image: image29 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image28 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:found image28 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:erased image28 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image29 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:found image29 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:erased image29 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:100.00% done 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:image: image28 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:image: image29 2024-09-13T14:36:25.658 INFO:tasks.workunit.client.0.smithi099.stdout:image: image30 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image28 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:found image28 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:erased image28 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image29 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:found image29 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:erased image29 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image30 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:found image30 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:erased image30 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:image: image28 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:image: image29 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:image: image30 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:image: image31 2024-09-13T14:36:25.659 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image28 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:found image28 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:erased image28 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image29 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:found image29 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:erased image29 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image30 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:found image30 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:erased image30 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image31 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:found image31 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:erased image31 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:image: image28 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:image: image29 2024-09-13T14:36:25.660 INFO:tasks.workunit.client.0.smithi099.stdout:image: image30 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:image: image31 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:image: image32 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image28 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:found image28 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:erased image28 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image29 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:found image29 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:erased image29 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image30 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:found image30 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:erased image30 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image31 2024-09-13T14:36:25.661 INFO:tasks.workunit.client.0.smithi099.stdout:found image31 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:erased image31 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image32 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:found image32 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:erased image32 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:100.00% done 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:image: image28 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:image: image29 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:image: image30 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:image: image31 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:image: image32 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:image: image33 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image28 2024-09-13T14:36:25.662 INFO:tasks.workunit.client.0.smithi099.stdout:found image28 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:erased image28 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image29 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:found image29 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:erased image29 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image30 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:found image30 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:erased image30 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image31 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:found image31 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:erased image31 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image32 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:found image32 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:erased image32 2024-09-13T14:36:25.663 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image33 2024-09-13T14:36:25.664 INFO:tasks.workunit.client.0.smithi099.stdout:found image33 2024-09-13T14:36:25.664 INFO:tasks.workunit.client.0.smithi099.stdout:erased image33 2024-09-13T14:36:25.664 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestDeepCopy (9136 ms) 2024-09-13T14:36:25.664 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestDeepCopyPP 2024-09-13T14:36:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:25 smithi099 bash[84576]: cluster 2024-09-13T14:36:24.021674+0000 mgr.y (mgr.26252) 211 : cluster [DBG] pgmap v386: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:25 smithi099 bash[84576]: cluster 2024-09-13T14:36:24.021674+0000 mgr.y (mgr.26252) 211 : cluster [DBG] pgmap v386: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:25 smithi099 bash[84576]: cluster 2024-09-13T14:36:24.488484+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-13T14:36:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:25 smithi099 bash[84576]: cluster 2024-09-13T14:36:24.488484+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-13T14:36:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:25 smithi099 bash[87147]: cluster 2024-09-13T14:36:24.021674+0000 mgr.y (mgr.26252) 211 : cluster [DBG] pgmap v386: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:25 smithi099 bash[87147]: cluster 2024-09-13T14:36:24.021674+0000 mgr.y (mgr.26252) 211 : cluster [DBG] pgmap v386: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:25 smithi099 bash[87147]: cluster 2024-09-13T14:36:24.488484+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-13T14:36:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:25 smithi099 bash[87147]: cluster 2024-09-13T14:36:24.488484+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-13T14:36:25.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:25 smithi177 bash[68594]: cluster 2024-09-13T14:36:24.021674+0000 mgr.y (mgr.26252) 211 : cluster [DBG] pgmap v386: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:25.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:25 smithi177 bash[68594]: cluster 2024-09-13T14:36:24.021674+0000 mgr.y (mgr.26252) 211 : cluster [DBG] pgmap v386: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:25.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:25 smithi177 bash[68594]: cluster 2024-09-13T14:36:24.488484+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-13T14:36:25.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:25 smithi177 bash[68594]: cluster 2024-09-13T14:36:24.488484+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-13T14:36:26.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[84576]: cluster 2024-09-13T14:36:25.493812+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-13T14:36:26.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[84576]: cluster 2024-09-13T14:36:25.493812+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-13T14:36:26.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[84576]: cluster 2024-09-13T14:36:26.043791+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-09-13T14:36:26.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[84576]: cluster 2024-09-13T14:36:26.043791+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-09-13T14:36:26.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[84576]: audit 2024-09-13T14:36:26.049740+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.99:0/2443544103' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[84576]: audit 2024-09-13T14:36:26.049740+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.99:0/2443544103' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[84576]: audit 2024-09-13T14:36:26.053645+0000 mon.a (mon.0) 673 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[84576]: audit 2024-09-13T14:36:26.053645+0000 mon.a (mon.0) 673 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[87147]: cluster 2024-09-13T14:36:25.493812+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-13T14:36:26.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[87147]: cluster 2024-09-13T14:36:25.493812+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-13T14:36:26.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[87147]: cluster 2024-09-13T14:36:26.043791+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-09-13T14:36:26.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[87147]: cluster 2024-09-13T14:36:26.043791+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-09-13T14:36:26.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[87147]: audit 2024-09-13T14:36:26.049740+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.99:0/2443544103' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[87147]: audit 2024-09-13T14:36:26.049740+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.99:0/2443544103' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[87147]: audit 2024-09-13T14:36:26.053645+0000 mon.a (mon.0) 673 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:26 smithi099 bash[87147]: audit 2024-09-13T14:36:26.053645+0000 mon.a (mon.0) 673 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:26 smithi177 bash[68594]: cluster 2024-09-13T14:36:25.493812+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-13T14:36:26.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:26 smithi177 bash[68594]: cluster 2024-09-13T14:36:25.493812+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-13T14:36:26.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:26 smithi177 bash[68594]: cluster 2024-09-13T14:36:26.043791+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-09-13T14:36:26.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:26 smithi177 bash[68594]: cluster 2024-09-13T14:36:26.043791+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-09-13T14:36:26.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:26 smithi177 bash[68594]: audit 2024-09-13T14:36:26.049740+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.99:0/2443544103' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:26 smithi177 bash[68594]: audit 2024-09-13T14:36:26.049740+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.99:0/2443544103' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:26 smithi177 bash[68594]: audit 2024-09-13T14:36:26.053645+0000 mon.a (mon.0) 673 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:26.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:26 smithi177 bash[68594]: audit 2024-09-13T14:36:26.053645+0000 mon.a (mon.0) 673 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:27.051 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:27.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:36:27] "GET /metrics HTTP/1.1" 200 48539 "" "Prometheus/2.51.0" 2024-09-13T14:36:27.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[87147]: cluster 2024-09-13T14:36:26.022503+0000 mgr.y (mgr.26252) 212 : cluster [DBG] pgmap v389: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:27.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[87147]: cluster 2024-09-13T14:36:26.022503+0000 mgr.y (mgr.26252) 212 : cluster [DBG] pgmap v389: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[87147]: audit 2024-09-13T14:36:27.036092+0000 mon.a (mon.0) 674 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[87147]: audit 2024-09-13T14:36:27.036092+0000 mon.a (mon.0) 674 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[87147]: cluster 2024-09-13T14:36:27.044688+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-09-13T14:36:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[87147]: cluster 2024-09-13T14:36:27.044688+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-09-13T14:36:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[84576]: cluster 2024-09-13T14:36:26.022503+0000 mgr.y (mgr.26252) 212 : cluster [DBG] pgmap v389: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[84576]: cluster 2024-09-13T14:36:26.022503+0000 mgr.y (mgr.26252) 212 : cluster [DBG] pgmap v389: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[84576]: audit 2024-09-13T14:36:27.036092+0000 mon.a (mon.0) 674 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[84576]: audit 2024-09-13T14:36:27.036092+0000 mon.a (mon.0) 674 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[84576]: cluster 2024-09-13T14:36:27.044688+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-09-13T14:36:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:27 smithi099 bash[84576]: cluster 2024-09-13T14:36:27.044688+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-09-13T14:36:27.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:27 smithi177 bash[68594]: cluster 2024-09-13T14:36:26.022503+0000 mgr.y (mgr.26252) 212 : cluster [DBG] pgmap v389: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:27.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:27 smithi177 bash[68594]: cluster 2024-09-13T14:36:26.022503+0000 mgr.y (mgr.26252) 212 : cluster [DBG] pgmap v389: 265 pgs: 265 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 416 KiB/s rd, 26 KiB/s wr, 493 op/s 2024-09-13T14:36:27.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:27 smithi177 bash[68594]: audit 2024-09-13T14:36:27.036092+0000 mon.a (mon.0) 674 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:27.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:27 smithi177 bash[68594]: audit 2024-09-13T14:36:27.036092+0000 mon.a (mon.0) 674 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:27.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:27 smithi177 bash[68594]: cluster 2024-09-13T14:36:27.044688+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-09-13T14:36:27.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:27 smithi177 bash[68594]: cluster 2024-09-13T14:36:27.044688+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-09-13T14:36:29.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[84576]: cluster 2024-09-13T14:36:28.023646+0000 mgr.y (mgr.26252) 213 : cluster [DBG] pgmap v392: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.2 KiB/s wr, 36 op/s 2024-09-13T14:36:29.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[84576]: cluster 2024-09-13T14:36:28.023646+0000 mgr.y (mgr.26252) 213 : cluster [DBG] pgmap v392: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.2 KiB/s wr, 36 op/s 2024-09-13T14:36:29.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[84576]: cluster 2024-09-13T14:36:28.048677+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-09-13T14:36:29.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[84576]: cluster 2024-09-13T14:36:28.048677+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-09-13T14:36:29.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[84576]: audit 2024-09-13T14:36:28.342206+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:29.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[84576]: audit 2024-09-13T14:36:28.342206+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:29.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[87147]: cluster 2024-09-13T14:36:28.023646+0000 mgr.y (mgr.26252) 213 : cluster [DBG] pgmap v392: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.2 KiB/s wr, 36 op/s 2024-09-13T14:36:29.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[87147]: cluster 2024-09-13T14:36:28.023646+0000 mgr.y (mgr.26252) 213 : cluster [DBG] pgmap v392: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.2 KiB/s wr, 36 op/s 2024-09-13T14:36:29.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[87147]: cluster 2024-09-13T14:36:28.048677+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-09-13T14:36:29.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[87147]: cluster 2024-09-13T14:36:28.048677+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-09-13T14:36:29.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[87147]: audit 2024-09-13T14:36:28.342206+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:29.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:29 smithi099 bash[87147]: audit 2024-09-13T14:36:28.342206+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:29.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:29 smithi177 bash[68594]: cluster 2024-09-13T14:36:28.023646+0000 mgr.y (mgr.26252) 213 : cluster [DBG] pgmap v392: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.2 KiB/s wr, 36 op/s 2024-09-13T14:36:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:29 smithi177 bash[68594]: cluster 2024-09-13T14:36:28.023646+0000 mgr.y (mgr.26252) 213 : cluster [DBG] pgmap v392: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.2 KiB/s wr, 36 op/s 2024-09-13T14:36:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:29 smithi177 bash[68594]: cluster 2024-09-13T14:36:28.048677+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-09-13T14:36:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:29 smithi177 bash[68594]: cluster 2024-09-13T14:36:28.048677+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-09-13T14:36:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:29 smithi177 bash[68594]: audit 2024-09-13T14:36:28.342206+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:29.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:29 smithi177 bash[68594]: audit 2024-09-13T14:36:28.342206+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:29.373 INFO:tasks.workunit.client.0.smithi099.stdout:num images is: 1 2024-09-13T14:36:29.373 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:36:29.374 INFO:tasks.workunit.client.0.smithi099.stdout:image: image34 2024-09-13T14:36:29.374 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image34 2024-09-13T14:36:29.432 INFO:tasks.workunit.client.0.smithi099.stdout:num images is: 2 2024-09-13T14:36:29.432 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:36:29.432 INFO:tasks.workunit.client.0.smithi099.stdout:image: image34 2024-09-13T14:36:29.433 INFO:tasks.workunit.client.0.smithi099.stdout:image: image35 2024-09-13T14:36:29.433 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image34 2024-09-13T14:36:29.433 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image35 2024-09-13T14:36:29.600 INFO:tasks.workunit.client.0.smithi099.stdout:100.00% done 2024-09-13T14:36:29.600 INFO:tasks.workunit.client.0.smithi099.stdout:num images is: 3 2024-09-13T14:36:29.600 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 3 2024-09-13T14:36:29.600 INFO:tasks.workunit.client.0.smithi099.stdout:image: image34 2024-09-13T14:36:29.600 INFO:tasks.workunit.client.0.smithi099.stdout:image: image35 2024-09-13T14:36:29.600 INFO:tasks.workunit.client.0.smithi099.stdout:image: image36 2024-09-13T14:36:29.600 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image34 2024-09-13T14:36:29.600 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image35 2024-09-13T14:36:29.600 INFO:tasks.workunit.client.0.smithi099.stdout:expected = image36 2024-09-13T14:36:29.682 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestDeepCopyPP (4024 ms) 2024-09-13T14:36:29.682 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnap 2024-09-13T14:36:29.682 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:30.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:30 smithi099 bash[84576]: cluster 2024-09-13T14:36:29.051567+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-09-13T14:36:30.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:30 smithi099 bash[84576]: cluster 2024-09-13T14:36:29.051567+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-09-13T14:36:30.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:30 smithi099 bash[87147]: cluster 2024-09-13T14:36:29.051567+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-09-13T14:36:30.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:30 smithi099 bash[87147]: cluster 2024-09-13T14:36:29.051567+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-09-13T14:36:30.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:30 smithi177 bash[68594]: cluster 2024-09-13T14:36:29.051567+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-09-13T14:36:30.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:30 smithi177 bash[68594]: cluster 2024-09-13T14:36:29.051567+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-09-13T14:36:30.356 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:36:30 smithi177 bash[61439]: ts=2024-09-13T14:36:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:31.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:31 smithi177 bash[68594]: cluster 2024-09-13T14:36:30.024588+0000 mgr.y (mgr.26252) 214 : cluster [DBG] pgmap v395: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 6.7 KiB/s wr, 157 op/s 2024-09-13T14:36:31.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:31 smithi177 bash[68594]: cluster 2024-09-13T14:36:30.024588+0000 mgr.y (mgr.26252) 214 : cluster [DBG] pgmap v395: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 6.7 KiB/s wr, 157 op/s 2024-09-13T14:36:31.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:31 smithi177 bash[68594]: cluster 2024-09-13T14:36:30.063437+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-09-13T14:36:31.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:31 smithi177 bash[68594]: cluster 2024-09-13T14:36:30.063437+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-09-13T14:36:31.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:31 smithi099 bash[84576]: cluster 2024-09-13T14:36:30.024588+0000 mgr.y (mgr.26252) 214 : cluster [DBG] pgmap v395: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 6.7 KiB/s wr, 157 op/s 2024-09-13T14:36:31.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:31 smithi099 bash[84576]: cluster 2024-09-13T14:36:30.024588+0000 mgr.y (mgr.26252) 214 : cluster [DBG] pgmap v395: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 6.7 KiB/s wr, 157 op/s 2024-09-13T14:36:31.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:31 smithi099 bash[84576]: cluster 2024-09-13T14:36:30.063437+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-09-13T14:36:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:31 smithi099 bash[84576]: cluster 2024-09-13T14:36:30.063437+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-09-13T14:36:31.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:31 smithi099 bash[87147]: cluster 2024-09-13T14:36:30.024588+0000 mgr.y (mgr.26252) 214 : cluster [DBG] pgmap v395: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 6.7 KiB/s wr, 157 op/s 2024-09-13T14:36:31.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:31 smithi099 bash[87147]: cluster 2024-09-13T14:36:30.024588+0000 mgr.y (mgr.26252) 214 : cluster [DBG] pgmap v395: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 6.7 KiB/s wr, 157 op/s 2024-09-13T14:36:31.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:31 smithi099 bash[87147]: cluster 2024-09-13T14:36:30.063437+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-09-13T14:36:31.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:31 smithi099 bash[87147]: cluster 2024-09-13T14:36:30.063437+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-09-13T14:36:32.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:32 smithi177 bash[68594]: cluster 2024-09-13T14:36:31.066396+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-09-13T14:36:32.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:32 smithi177 bash[68594]: cluster 2024-09-13T14:36:31.066396+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-09-13T14:36:32.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:32 smithi099 bash[84576]: cluster 2024-09-13T14:36:31.066396+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-09-13T14:36:32.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:32 smithi099 bash[84576]: cluster 2024-09-13T14:36:31.066396+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-09-13T14:36:32.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:32 smithi099 bash[87147]: cluster 2024-09-13T14:36:31.066396+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-09-13T14:36:32.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:32 smithi099 bash[87147]: cluster 2024-09-13T14:36:31.066396+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-09-13T14:36:33.152 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 1 2024-09-13T14:36:33.152 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:36:33.152 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap1 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:found snap1 with size 2097152 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 2 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap1 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap2 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:found snap1 with size 2097152 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:found snap2 with size 4194304 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 1 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap2 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:found snap2 with size 4194304 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 0 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 0 2024-09-13T14:36:33.153 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnap (3471 ms) 2024-09-13T14:36:33.154 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestGetSnapShotTimeStamp 2024-09-13T14:36:33.154 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:33.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:33 smithi177 bash[68594]: cluster 2024-09-13T14:36:32.025862+0000 mgr.y (mgr.26252) 215 : cluster [DBG] pgmap v398: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 4.5 KiB/s wr, 121 op/s 2024-09-13T14:36:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:33 smithi177 bash[68594]: cluster 2024-09-13T14:36:32.025862+0000 mgr.y (mgr.26252) 215 : cluster [DBG] pgmap v398: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 4.5 KiB/s wr, 121 op/s 2024-09-13T14:36:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:33 smithi177 bash[68594]: cluster 2024-09-13T14:36:32.096346+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-09-13T14:36:33.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:33 smithi177 bash[68594]: cluster 2024-09-13T14:36:32.096346+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-09-13T14:36:33.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:33 smithi099 bash[84576]: cluster 2024-09-13T14:36:32.025862+0000 mgr.y (mgr.26252) 215 : cluster [DBG] pgmap v398: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 4.5 KiB/s wr, 121 op/s 2024-09-13T14:36:33.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:33 smithi099 bash[84576]: cluster 2024-09-13T14:36:32.025862+0000 mgr.y (mgr.26252) 215 : cluster [DBG] pgmap v398: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 4.5 KiB/s wr, 121 op/s 2024-09-13T14:36:33.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:33 smithi099 bash[84576]: cluster 2024-09-13T14:36:32.096346+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-09-13T14:36:33.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:33 smithi099 bash[84576]: cluster 2024-09-13T14:36:32.096346+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-09-13T14:36:33.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:33 smithi099 bash[87147]: cluster 2024-09-13T14:36:32.025862+0000 mgr.y (mgr.26252) 215 : cluster [DBG] pgmap v398: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 4.5 KiB/s wr, 121 op/s 2024-09-13T14:36:33.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:33 smithi099 bash[87147]: cluster 2024-09-13T14:36:32.025862+0000 mgr.y (mgr.26252) 215 : cluster [DBG] pgmap v398: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 4.5 KiB/s wr, 121 op/s 2024-09-13T14:36:33.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:33 smithi099 bash[87147]: cluster 2024-09-13T14:36:32.096346+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-09-13T14:36:33.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:33 smithi099 bash[87147]: cluster 2024-09-13T14:36:32.096346+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-09-13T14:36:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: cluster 2024-09-13T14:36:33.106927+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-09-13T14:36:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: cluster 2024-09-13T14:36:33.106927+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-09-13T14:36:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.247241+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.247241+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.831917+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.831917+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.835868+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.835868+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.837751+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.837751+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.848696+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.848696+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.994596+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.994596+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.997829+0000 mon.a (mon.0) 689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.997829+0000 mon.a (mon.0) 689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.999520+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:33.999520+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:34.009753+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[84576]: audit 2024-09-13T14:36:34.009753+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: cluster 2024-09-13T14:36:33.106927+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: cluster 2024-09-13T14:36:33.106927+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.247241+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.247241+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.831917+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.831917+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.835868+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.835868+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.837751+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.837751+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.848696+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.848696+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.994596+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.994596+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.997829+0000 mon.a (mon.0) 689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.997829+0000 mon.a (mon.0) 689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.999520+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:33.999520+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:34.009753+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:34 smithi099 bash[87147]: audit 2024-09-13T14:36:34.009753+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: cluster 2024-09-13T14:36:33.106927+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-09-13T14:36:34.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: cluster 2024-09-13T14:36:33.106927+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-09-13T14:36:34.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.247241+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:34.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.247241+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:34.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.831917+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.831917+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.835868+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.835868+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.837751+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.837751+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.848696+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.848696+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.994596+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.994596+0000 mon.a (mon.0) 688 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.997829+0000 mon.a (mon.0) 689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.997829+0000 mon.a (mon.0) 689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.999520+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:33.999520+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:34.009753+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:34.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:34 smithi177 bash[68594]: audit 2024-09-13T14:36:34.009753+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:35.177 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestGetSnapShotTimeStamp (2024 ms) 2024-09-13T14:36:35.177 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnapPP 2024-09-13T14:36:35.178 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:35 smithi099 bash[84576]: cluster 2024-09-13T14:36:34.027495+0000 mgr.y (mgr.26252) 216 : cluster [DBG] pgmap v401: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:35 smithi099 bash[84576]: cluster 2024-09-13T14:36:34.027495+0000 mgr.y (mgr.26252) 216 : cluster [DBG] pgmap v401: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:35 smithi099 bash[84576]: cluster 2024-09-13T14:36:34.105105+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-09-13T14:36:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:35 smithi099 bash[84576]: cluster 2024-09-13T14:36:34.105105+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-09-13T14:36:35.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:35 smithi099 bash[87147]: cluster 2024-09-13T14:36:34.027495+0000 mgr.y (mgr.26252) 216 : cluster [DBG] pgmap v401: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:35.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:35 smithi099 bash[87147]: cluster 2024-09-13T14:36:34.027495+0000 mgr.y (mgr.26252) 216 : cluster [DBG] pgmap v401: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:35.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:35 smithi099 bash[87147]: cluster 2024-09-13T14:36:34.105105+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-09-13T14:36:35.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:35 smithi099 bash[87147]: cluster 2024-09-13T14:36:34.105105+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-09-13T14:36:35.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:35 smithi177 bash[68594]: cluster 2024-09-13T14:36:34.027495+0000 mgr.y (mgr.26252) 216 : cluster [DBG] pgmap v401: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:35.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:35 smithi177 bash[68594]: cluster 2024-09-13T14:36:34.027495+0000 mgr.y (mgr.26252) 216 : cluster [DBG] pgmap v401: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:35.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:35 smithi177 bash[68594]: cluster 2024-09-13T14:36:34.105105+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-09-13T14:36:35.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:35 smithi177 bash[68594]: cluster 2024-09-13T14:36:34.105105+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-09-13T14:36:36.088 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 1 2024-09-13T14:36:36.089 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:36:36.089 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap1 2024-09-13T14:36:36.089 INFO:tasks.workunit.client.0.smithi099.stdout:found snap1 with size 2097152 2024-09-13T14:36:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:36 smithi099 bash[84576]: cluster 2024-09-13T14:36:35.135102+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-09-13T14:36:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:36 smithi099 bash[84576]: cluster 2024-09-13T14:36:35.135102+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-09-13T14:36:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:36 smithi099 bash[84576]: cluster 2024-09-13T14:36:36.044441+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-09-13T14:36:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:36 smithi099 bash[84576]: cluster 2024-09-13T14:36:36.044441+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-09-13T14:36:36.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:36 smithi099 bash[87147]: cluster 2024-09-13T14:36:35.135102+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-09-13T14:36:36.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:36 smithi099 bash[87147]: cluster 2024-09-13T14:36:35.135102+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-09-13T14:36:36.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:36 smithi099 bash[87147]: cluster 2024-09-13T14:36:36.044441+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-09-13T14:36:36.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:36 smithi099 bash[87147]: cluster 2024-09-13T14:36:36.044441+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-09-13T14:36:36.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:36 smithi177 bash[68594]: cluster 2024-09-13T14:36:35.135102+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-09-13T14:36:36.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:36 smithi177 bash[68594]: cluster 2024-09-13T14:36:35.135102+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-09-13T14:36:36.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:36 smithi177 bash[68594]: cluster 2024-09-13T14:36:36.044441+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-09-13T14:36:36.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:36 smithi177 bash[68594]: cluster 2024-09-13T14:36:36.044441+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-09-13T14:36:37.097 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 2 2024-09-13T14:36:37.098 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:36:37.098 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap1 2024-09-13T14:36:37.098 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap2 2024-09-13T14:36:37.098 INFO:tasks.workunit.client.0.smithi099.stdout:found snap1 with size 2097152 2024-09-13T14:36:37.098 INFO:tasks.workunit.client.0.smithi099.stdout:found snap2 with size 4194304 2024-09-13T14:36:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:36:37] "GET /metrics HTTP/1.1" 200 50987 "" "Prometheus/2.51.0" 2024-09-13T14:36:37.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:37 smithi099 bash[84576]: cluster 2024-09-13T14:36:36.028328+0000 mgr.y (mgr.26252) 217 : cluster [DBG] pgmap v404: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:37.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:37 smithi099 bash[84576]: cluster 2024-09-13T14:36:36.028328+0000 mgr.y (mgr.26252) 217 : cluster [DBG] pgmap v404: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:37.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:37 smithi099 bash[84576]: cluster 2024-09-13T14:36:37.048063+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-09-13T14:36:37.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:37 smithi099 bash[84576]: cluster 2024-09-13T14:36:37.048063+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-09-13T14:36:37.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:37 smithi099 bash[87147]: cluster 2024-09-13T14:36:36.028328+0000 mgr.y (mgr.26252) 217 : cluster [DBG] pgmap v404: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:37.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:37 smithi099 bash[87147]: cluster 2024-09-13T14:36:36.028328+0000 mgr.y (mgr.26252) 217 : cluster [DBG] pgmap v404: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:37.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:37 smithi099 bash[87147]: cluster 2024-09-13T14:36:37.048063+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-09-13T14:36:37.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:37 smithi099 bash[87147]: cluster 2024-09-13T14:36:37.048063+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-09-13T14:36:37.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:37 smithi177 bash[68594]: cluster 2024-09-13T14:36:36.028328+0000 mgr.y (mgr.26252) 217 : cluster [DBG] pgmap v404: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:37.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:37 smithi177 bash[68594]: cluster 2024-09-13T14:36:36.028328+0000 mgr.y (mgr.26252) 217 : cluster [DBG] pgmap v404: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 28 KiB/s wr, 478 op/s 2024-09-13T14:36:37.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:37 smithi177 bash[68594]: cluster 2024-09-13T14:36:37.048063+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-09-13T14:36:37.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:37 smithi177 bash[68594]: cluster 2024-09-13T14:36:37.048063+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-09-13T14:36:38.081 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 1 2024-09-13T14:36:38.081 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:36:38.081 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap2 2024-09-13T14:36:38.081 INFO:tasks.workunit.client.0.smithi099.stdout:found snap2 with size 4194304 2024-09-13T14:36:39.096 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 0 2024-09-13T14:36:39.096 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 0 2024-09-13T14:36:39.120 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnapPP (3942 ms) 2024-09-13T14:36:39.121 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestGetNameIdSnapPP 2024-09-13T14:36:39.121 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[84576]: cluster 2024-09-13T14:36:38.029567+0000 mgr.y (mgr.26252) 218 : cluster [DBG] pgmap v407: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[84576]: cluster 2024-09-13T14:36:38.029567+0000 mgr.y (mgr.26252) 218 : cluster [DBG] pgmap v407: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[84576]: cluster 2024-09-13T14:36:38.049932+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-09-13T14:36:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[84576]: cluster 2024-09-13T14:36:38.049932+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-09-13T14:36:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[84576]: audit 2024-09-13T14:36:38.400396+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[84576]: audit 2024-09-13T14:36:38.400396+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:39.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[87147]: cluster 2024-09-13T14:36:38.029567+0000 mgr.y (mgr.26252) 218 : cluster [DBG] pgmap v407: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:39.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[87147]: cluster 2024-09-13T14:36:38.029567+0000 mgr.y (mgr.26252) 218 : cluster [DBG] pgmap v407: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:39.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[87147]: cluster 2024-09-13T14:36:38.049932+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-09-13T14:36:39.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[87147]: cluster 2024-09-13T14:36:38.049932+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-09-13T14:36:39.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[87147]: audit 2024-09-13T14:36:38.400396+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:39.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:39 smithi099 bash[87147]: audit 2024-09-13T14:36:38.400396+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:39 smithi177 bash[68594]: cluster 2024-09-13T14:36:38.029567+0000 mgr.y (mgr.26252) 218 : cluster [DBG] pgmap v407: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:39 smithi177 bash[68594]: cluster 2024-09-13T14:36:38.029567+0000 mgr.y (mgr.26252) 218 : cluster [DBG] pgmap v407: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:36:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:39 smithi177 bash[68594]: cluster 2024-09-13T14:36:38.049932+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-09-13T14:36:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:39 smithi177 bash[68594]: cluster 2024-09-13T14:36:38.049932+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-09-13T14:36:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:39 smithi177 bash[68594]: audit 2024-09-13T14:36:38.400396+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:39 smithi177 bash[68594]: audit 2024-09-13T14:36:38.400396+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:36:40.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:40 smithi099 bash[84576]: cluster 2024-09-13T14:36:39.060871+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-13T14:36:40.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:40 smithi099 bash[84576]: cluster 2024-09-13T14:36:39.060871+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-13T14:36:40.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:40 smithi099 bash[87147]: cluster 2024-09-13T14:36:39.060871+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-13T14:36:40.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:40 smithi099 bash[87147]: cluster 2024-09-13T14:36:39.060871+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-13T14:36:40.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:40 smithi177 bash[68594]: cluster 2024-09-13T14:36:39.060871+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-13T14:36:40.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:40 smithi177 bash[68594]: cluster 2024-09-13T14:36:39.060871+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-13T14:36:40.356 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:36:40 smithi177 bash[61439]: ts=2024-09-13T14:36:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:41.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[84576]: cluster 2024-09-13T14:36:40.030431+0000 mgr.y (mgr.26252) 219 : cluster [DBG] pgmap v410: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[84576]: cluster 2024-09-13T14:36:40.030431+0000 mgr.y (mgr.26252) 219 : cluster [DBG] pgmap v410: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[84576]: cluster 2024-09-13T14:36:40.075383+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[84576]: cluster 2024-09-13T14:36:40.075383+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[84576]: cluster 2024-09-13T14:36:41.046811+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[84576]: cluster 2024-09-13T14:36:41.046811+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[87147]: cluster 2024-09-13T14:36:40.030431+0000 mgr.y (mgr.26252) 219 : cluster [DBG] pgmap v410: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[87147]: cluster 2024-09-13T14:36:40.030431+0000 mgr.y (mgr.26252) 219 : cluster [DBG] pgmap v410: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[87147]: cluster 2024-09-13T14:36:40.075383+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-13T14:36:41.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[87147]: cluster 2024-09-13T14:36:40.075383+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-13T14:36:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[87147]: cluster 2024-09-13T14:36:41.046811+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-13T14:36:41.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:41 smithi099 bash[87147]: cluster 2024-09-13T14:36:41.046811+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-13T14:36:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:41 smithi177 bash[68594]: cluster 2024-09-13T14:36:40.030431+0000 mgr.y (mgr.26252) 219 : cluster [DBG] pgmap v410: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:41 smithi177 bash[68594]: cluster 2024-09-13T14:36:40.030431+0000 mgr.y (mgr.26252) 219 : cluster [DBG] pgmap v410: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:41 smithi177 bash[68594]: cluster 2024-09-13T14:36:40.075383+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-13T14:36:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:41 smithi177 bash[68594]: cluster 2024-09-13T14:36:40.075383+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-13T14:36:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:41 smithi177 bash[68594]: cluster 2024-09-13T14:36:41.046811+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-13T14:36:41.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:41 smithi177 bash[68594]: cluster 2024-09-13T14:36:41.046811+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-13T14:36:43.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:43 smithi177 bash[68594]: cluster 2024-09-13T14:36:42.031369+0000 mgr.y (mgr.26252) 220 : cluster [DBG] pgmap v413: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:43.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:43 smithi177 bash[68594]: cluster 2024-09-13T14:36:42.031369+0000 mgr.y (mgr.26252) 220 : cluster [DBG] pgmap v413: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:43.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:43 smithi177 bash[68594]: cluster 2024-09-13T14:36:42.083598+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-13T14:36:43.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:43 smithi177 bash[68594]: cluster 2024-09-13T14:36:42.083598+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-13T14:36:43.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:43 smithi099 bash[84576]: cluster 2024-09-13T14:36:42.031369+0000 mgr.y (mgr.26252) 220 : cluster [DBG] pgmap v413: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:43.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:43 smithi099 bash[84576]: cluster 2024-09-13T14:36:42.031369+0000 mgr.y (mgr.26252) 220 : cluster [DBG] pgmap v413: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:43 smithi099 bash[84576]: cluster 2024-09-13T14:36:42.083598+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-13T14:36:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:43 smithi099 bash[84576]: cluster 2024-09-13T14:36:42.083598+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-13T14:36:43.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:43 smithi099 bash[87147]: cluster 2024-09-13T14:36:42.031369+0000 mgr.y (mgr.26252) 220 : cluster [DBG] pgmap v413: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:43.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:43 smithi099 bash[87147]: cluster 2024-09-13T14:36:42.031369+0000 mgr.y (mgr.26252) 220 : cluster [DBG] pgmap v413: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 74 op/s 2024-09-13T14:36:43.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:43 smithi099 bash[87147]: cluster 2024-09-13T14:36:42.083598+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-13T14:36:43.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:43 smithi099 bash[87147]: cluster 2024-09-13T14:36:42.083598+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-13T14:36:44.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:44 smithi177 bash[68594]: cluster 2024-09-13T14:36:43.096759+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-13T14:36:44.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:44 smithi177 bash[68594]: cluster 2024-09-13T14:36:43.096759+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-13T14:36:44.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:44 smithi099 bash[84576]: cluster 2024-09-13T14:36:43.096759+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-13T14:36:44.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:44 smithi099 bash[84576]: cluster 2024-09-13T14:36:43.096759+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-13T14:36:44.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:44 smithi099 bash[87147]: cluster 2024-09-13T14:36:43.096759+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-13T14:36:44.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:44 smithi099 bash[87147]: cluster 2024-09-13T14:36:43.096759+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-13T14:36:45.143 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 0 2024-09-13T14:36:45.143 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 0 2024-09-13T14:36:45.166 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestGetNameIdSnapPP (6047 ms) 2024-09-13T14:36:45.166 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCreateLsRenameSnapPP 2024-09-13T14:36:45.166 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:45 smithi177 bash[68594]: cluster 2024-09-13T14:36:44.033905+0000 mgr.y (mgr.26252) 221 : cluster [DBG] pgmap v416: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:45 smithi177 bash[68594]: cluster 2024-09-13T14:36:44.033905+0000 mgr.y (mgr.26252) 221 : cluster [DBG] pgmap v416: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:45 smithi177 bash[68594]: cluster 2024-09-13T14:36:44.105515+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-13T14:36:45.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:45 smithi177 bash[68594]: cluster 2024-09-13T14:36:44.105515+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-13T14:36:45.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:45 smithi099 bash[84576]: cluster 2024-09-13T14:36:44.033905+0000 mgr.y (mgr.26252) 221 : cluster [DBG] pgmap v416: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:45.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:45 smithi099 bash[84576]: cluster 2024-09-13T14:36:44.033905+0000 mgr.y (mgr.26252) 221 : cluster [DBG] pgmap v416: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:45.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:45 smithi099 bash[84576]: cluster 2024-09-13T14:36:44.105515+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-13T14:36:45.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:45 smithi099 bash[84576]: cluster 2024-09-13T14:36:44.105515+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-13T14:36:45.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:45 smithi099 bash[87147]: cluster 2024-09-13T14:36:44.033905+0000 mgr.y (mgr.26252) 221 : cluster [DBG] pgmap v416: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:45.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:45 smithi099 bash[87147]: cluster 2024-09-13T14:36:44.033905+0000 mgr.y (mgr.26252) 221 : cluster [DBG] pgmap v416: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:45.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:45 smithi099 bash[87147]: cluster 2024-09-13T14:36:44.105515+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-13T14:36:45.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:45 smithi099 bash[87147]: cluster 2024-09-13T14:36:44.105515+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-13T14:36:46.101 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 1 2024-09-13T14:36:46.101 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:36:46.102 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap1 2024-09-13T14:36:46.102 INFO:tasks.workunit.client.0.smithi099.stdout:found snap1 with size 2097152 2024-09-13T14:36:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:46 smithi099 bash[84576]: cluster 2024-09-13T14:36:45.115540+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-13T14:36:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:46 smithi099 bash[84576]: cluster 2024-09-13T14:36:45.115540+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-13T14:36:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:46 smithi099 bash[84576]: cluster 2024-09-13T14:36:46.047800+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-13T14:36:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:46 smithi099 bash[84576]: cluster 2024-09-13T14:36:46.047800+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-13T14:36:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:46 smithi099 bash[87147]: cluster 2024-09-13T14:36:45.115540+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-13T14:36:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:46 smithi099 bash[87147]: cluster 2024-09-13T14:36:45.115540+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-13T14:36:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:46 smithi099 bash[87147]: cluster 2024-09-13T14:36:46.047800+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-13T14:36:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:46 smithi099 bash[87147]: cluster 2024-09-13T14:36:46.047800+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-13T14:36:46.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:46 smithi177 bash[68594]: cluster 2024-09-13T14:36:45.115540+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-13T14:36:46.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:46 smithi177 bash[68594]: cluster 2024-09-13T14:36:45.115540+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-13T14:36:46.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:46 smithi177 bash[68594]: cluster 2024-09-13T14:36:46.047800+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-13T14:36:46.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:46 smithi177 bash[68594]: cluster 2024-09-13T14:36:46.047800+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-13T14:36:47.162 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 2 2024-09-13T14:36:47.162 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:36:47.163 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap1 2024-09-13T14:36:47.163 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap2 2024-09-13T14:36:47.163 INFO:tasks.workunit.client.0.smithi099.stdout:found snap1 with size 2097152 2024-09-13T14:36:47.163 INFO:tasks.workunit.client.0.smithi099.stdout:found snap2 with size 4194304 2024-09-13T14:36:47.172 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 2 2024-09-13T14:36:47.172 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:36:47.172 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap1-rename 2024-09-13T14:36:47.172 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap2 2024-09-13T14:36:47.172 INFO:tasks.workunit.client.0.smithi099.stdout:found snap1-rename with size 2097152 2024-09-13T14:36:47.172 INFO:tasks.workunit.client.0.smithi099.stdout:found snap2 with size 4194304 2024-09-13T14:36:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:36:47] "GET /metrics HTTP/1.1" 200 50987 "" "Prometheus/2.51.0" 2024-09-13T14:36:47.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:47 smithi099 bash[84576]: cluster 2024-09-13T14:36:46.034774+0000 mgr.y (mgr.26252) 222 : cluster [DBG] pgmap v419: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:47 smithi099 bash[84576]: cluster 2024-09-13T14:36:46.034774+0000 mgr.y (mgr.26252) 222 : cluster [DBG] pgmap v419: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:47 smithi099 bash[84576]: cluster 2024-09-13T14:36:47.119795+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-13T14:36:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:47 smithi099 bash[84576]: cluster 2024-09-13T14:36:47.119795+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-13T14:36:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:47 smithi099 bash[87147]: cluster 2024-09-13T14:36:46.034774+0000 mgr.y (mgr.26252) 222 : cluster [DBG] pgmap v419: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:47 smithi099 bash[87147]: cluster 2024-09-13T14:36:46.034774+0000 mgr.y (mgr.26252) 222 : cluster [DBG] pgmap v419: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:47 smithi099 bash[87147]: cluster 2024-09-13T14:36:47.119795+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-13T14:36:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:47 smithi099 bash[87147]: cluster 2024-09-13T14:36:47.119795+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-13T14:36:47.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:47 smithi177 bash[68594]: cluster 2024-09-13T14:36:46.034774+0000 mgr.y (mgr.26252) 222 : cluster [DBG] pgmap v419: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:47.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:47 smithi177 bash[68594]: cluster 2024-09-13T14:36:46.034774+0000 mgr.y (mgr.26252) 222 : cluster [DBG] pgmap v419: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-13T14:36:47.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:47 smithi177 bash[68594]: cluster 2024-09-13T14:36:47.119795+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-13T14:36:47.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:47 smithi177 bash[68594]: cluster 2024-09-13T14:36:47.119795+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-13T14:36:48.200 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 1 2024-09-13T14:36:48.200 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:36:48.200 INFO:tasks.workunit.client.0.smithi099.stdout:snap: snap2-rename 2024-09-13T14:36:48.201 INFO:tasks.workunit.client.0.smithi099.stdout:found snap2-rename with size 4194304 2024-09-13T14:36:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.991463+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-09-13T14:36:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.991463+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-09-13T14:36:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.992112+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.992112+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.992491+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.992491+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.992854+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.992854+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.993214+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.993214+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.993566+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.993566+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.993941+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-13T14:36:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[84576]: audit 2024-09-13T14:36:47.993941+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-13T14:36:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.991463+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-09-13T14:36:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.991463+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-09-13T14:36:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.992112+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.992112+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.992491+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.992491+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.992854+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.992854+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.993214+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.993214+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.993566+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.993566+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.993941+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-13T14:36:48.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:48 smithi099 bash[87147]: audit 2024-09-13T14:36:47.993941+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-13T14:36:48.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.991463+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-09-13T14:36:48.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.991463+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-09-13T14:36:48.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.992112+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.992112+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.992491+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.992491+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.992854+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.992854+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-09-13T14:36:48.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.993214+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.993214+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.993566+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.993566+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]} : dispatch 2024-09-13T14:36:48.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.993941+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-13T14:36:48.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:48 smithi177 bash[68594]: audit 2024-09-13T14:36:47.993941+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]} : dispatch 2024-09-13T14:36:49.206 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 0 2024-09-13T14:36:49.207 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 0 2024-09-13T14:36:49.214 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCreateLsRenameSnapPP (4049 ms) 2024-09-13T14:36:49.215 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ConcurrentCreatesUnvalidatedPool 2024-09-13T14:36:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: cluster 2024-09-13T14:36:48.035875+0000 mgr.y (mgr.26252) 223 : cluster [DBG] pgmap v422: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: cluster 2024-09-13T14:36:48.035875+0000 mgr.y (mgr.26252) 223 : cluster [DBG] pgmap v422: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.137391+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.137391+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.137543+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.137543+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.137662+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.137662+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.137770+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.137770+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.137904+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.137904+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.138091+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.138091+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]': finished 2024-09-13T14:36:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.138246+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]}]': finished 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: cluster 2024-09-13T14:36:48.035875+0000 mgr.y (mgr.26252) 223 : cluster [DBG] pgmap v422: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: cluster 2024-09-13T14:36:48.035875+0000 mgr.y (mgr.26252) 223 : cluster [DBG] pgmap v422: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.137391+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.137391+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.137543+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]}]': finished 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.137543+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]}]': finished 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.137662+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]': finished 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.137662+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]': finished 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.137770+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.137770+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-09-13T14:36:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.137904+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-09-13T14:36:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.137904+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-09-13T14:36:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.138091+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]': finished 2024-09-13T14:36:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.138091+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]': finished 2024-09-13T14:36:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.138246+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]}]': finished 2024-09-13T14:36:49.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.138246+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]}]': finished 2024-09-13T14:36:49.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: cluster 2024-09-13T14:36:48.149331+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-13T14:36:49.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: cluster 2024-09-13T14:36:48.149331+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-13T14:36:49.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.248315+0000 mon.a (mon.0) 722 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:49.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[84576]: audit 2024-09-13T14:36:48.248315+0000 mon.a (mon.0) 722 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.138246+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]}]': finished 2024-09-13T14:36:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: cluster 2024-09-13T14:36:48.149331+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-13T14:36:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: cluster 2024-09-13T14:36:48.149331+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-13T14:36:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.248315+0000 mon.a (mon.0) 722 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:49 smithi099 bash[87147]: audit 2024-09-13T14:36:48.248315+0000 mon.a (mon.0) 722 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:49.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: cluster 2024-09-13T14:36:48.035875+0000 mgr.y (mgr.26252) 223 : cluster [DBG] pgmap v422: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:49.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: cluster 2024-09-13T14:36:48.035875+0000 mgr.y (mgr.26252) 223 : cluster [DBG] pgmap v422: 297 pgs: 297 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-13T14:36:49.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.137391+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-09-13T14:36:49.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.137391+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-09-13T14:36:49.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.137543+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.137543+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 5]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.137662+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.137662+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.137770+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.137770+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.137904+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.137904+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.138091+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.138091+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.138246+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.138246+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1a", "id": [7, 2]}]': finished 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: cluster 2024-09-13T14:36:48.149331+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-13T14:36:49.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: cluster 2024-09-13T14:36:48.149331+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-13T14:36:49.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.248315+0000 mon.a (mon.0) 722 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:49.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:49 smithi177 bash[68594]: audit 2024-09-13T14:36:48.248315+0000 mon.a (mon.0) 722 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:50.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:50 smithi099 bash[87147]: cluster 2024-09-13T14:36:49.147517+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-13T14:36:50.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:50 smithi099 bash[87147]: cluster 2024-09-13T14:36:49.147517+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-13T14:36:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:50 smithi099 bash[84576]: cluster 2024-09-13T14:36:49.147517+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-13T14:36:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:50 smithi099 bash[84576]: cluster 2024-09-13T14:36:49.147517+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-13T14:36:50.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:50 smithi177 bash[68594]: cluster 2024-09-13T14:36:49.147517+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-13T14:36:50.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:50 smithi177 bash[68594]: cluster 2024-09-13T14:36:49.147517+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-13T14:36:50.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:36:50 smithi177 bash[61439]: ts=2024-09-13T14:36:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:36:51.067 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:51.067 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:51.067 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:51.067 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[84576]: cluster 2024-09-13T14:36:50.036710+0000 mgr.y (mgr.26252) 224 : cluster [DBG] pgmap v425: 297 pgs: 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 87 op/s 2024-09-13T14:36:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[84576]: cluster 2024-09-13T14:36:50.036710+0000 mgr.y (mgr.26252) 224 : cluster [DBG] pgmap v425: 297 pgs: 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 87 op/s 2024-09-13T14:36:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[84576]: cluster 2024-09-13T14:36:50.176320+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-13T14:36:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[84576]: cluster 2024-09-13T14:36:50.176320+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-13T14:36:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[84576]: audit 2024-09-13T14:36:50.180408+0000 mon.a (mon.0) 725 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[84576]: audit 2024-09-13T14:36:50.180408+0000 mon.a (mon.0) 725 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[84576]: audit 2024-09-13T14:36:51.044690+0000 mon.a (mon.0) 726 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[84576]: audit 2024-09-13T14:36:51.044690+0000 mon.a (mon.0) 726 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[84576]: cluster 2024-09-13T14:36:51.054172+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[84576]: cluster 2024-09-13T14:36:51.054172+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[87147]: cluster 2024-09-13T14:36:50.036710+0000 mgr.y (mgr.26252) 224 : cluster [DBG] pgmap v425: 297 pgs: 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 87 op/s 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[87147]: cluster 2024-09-13T14:36:50.036710+0000 mgr.y (mgr.26252) 224 : cluster [DBG] pgmap v425: 297 pgs: 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 87 op/s 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[87147]: cluster 2024-09-13T14:36:50.176320+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[87147]: cluster 2024-09-13T14:36:50.176320+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[87147]: audit 2024-09-13T14:36:50.180408+0000 mon.a (mon.0) 725 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[87147]: audit 2024-09-13T14:36:50.180408+0000 mon.a (mon.0) 725 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[87147]: audit 2024-09-13T14:36:51.044690+0000 mon.a (mon.0) 726 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[87147]: audit 2024-09-13T14:36:51.044690+0000 mon.a (mon.0) 726 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[87147]: cluster 2024-09-13T14:36:51.054172+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-13T14:36:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:51 smithi099 bash[87147]: cluster 2024-09-13T14:36:51.054172+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-13T14:36:51.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:51 smithi177 bash[68594]: cluster 2024-09-13T14:36:50.036710+0000 mgr.y (mgr.26252) 224 : cluster [DBG] pgmap v425: 297 pgs: 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 87 op/s 2024-09-13T14:36:51.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:51 smithi177 bash[68594]: cluster 2024-09-13T14:36:50.036710+0000 mgr.y (mgr.26252) 224 : cluster [DBG] pgmap v425: 297 pgs: 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 87 op/s 2024-09-13T14:36:51.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:51 smithi177 bash[68594]: cluster 2024-09-13T14:36:50.176320+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-13T14:36:51.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:51 smithi177 bash[68594]: cluster 2024-09-13T14:36:50.176320+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-13T14:36:51.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:51 smithi177 bash[68594]: audit 2024-09-13T14:36:50.180408+0000 mon.a (mon.0) 725 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:51 smithi177 bash[68594]: audit 2024-09-13T14:36:50.180408+0000 mon.a (mon.0) 725 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:51.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:51 smithi177 bash[68594]: audit 2024-09-13T14:36:51.044690+0000 mon.a (mon.0) 726 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:51.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:51 smithi177 bash[68594]: audit 2024-09-13T14:36:51.044690+0000 mon.a (mon.0) 726 : audit [INF] from='client.? 172.21.15.99:0/4064769590' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:51.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:51 smithi177 bash[68594]: cluster 2024-09-13T14:36:51.054172+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-13T14:36:51.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:51 smithi177 bash[68594]: cluster 2024-09-13T14:36:51.054172+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-13T14:36:53.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:53 smithi099 bash[84576]: cluster 2024-09-13T14:36:52.037589+0000 mgr.y (mgr.26252) 225 : cluster [DBG] pgmap v428: 329 pgs: 32 unknown, 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 86 op/s 2024-09-13T14:36:53.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:53 smithi099 bash[84576]: cluster 2024-09-13T14:36:52.037589+0000 mgr.y (mgr.26252) 225 : cluster [DBG] pgmap v428: 329 pgs: 32 unknown, 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 86 op/s 2024-09-13T14:36:53.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:53 smithi099 bash[84576]: cluster 2024-09-13T14:36:52.055174+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-13T14:36:53.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:53 smithi099 bash[84576]: cluster 2024-09-13T14:36:52.055174+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-13T14:36:53.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:53 smithi099 bash[87147]: cluster 2024-09-13T14:36:52.037589+0000 mgr.y (mgr.26252) 225 : cluster [DBG] pgmap v428: 329 pgs: 32 unknown, 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 86 op/s 2024-09-13T14:36:53.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:53 smithi099 bash[87147]: cluster 2024-09-13T14:36:52.037589+0000 mgr.y (mgr.26252) 225 : cluster [DBG] pgmap v428: 329 pgs: 32 unknown, 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 86 op/s 2024-09-13T14:36:53.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:53 smithi099 bash[87147]: cluster 2024-09-13T14:36:52.055174+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-13T14:36:53.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:53 smithi099 bash[87147]: cluster 2024-09-13T14:36:52.055174+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-13T14:36:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:53 smithi177 bash[68594]: cluster 2024-09-13T14:36:52.037589+0000 mgr.y (mgr.26252) 225 : cluster [DBG] pgmap v428: 329 pgs: 32 unknown, 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 86 op/s 2024-09-13T14:36:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:53 smithi177 bash[68594]: cluster 2024-09-13T14:36:52.037589+0000 mgr.y (mgr.26252) 225 : cluster [DBG] pgmap v428: 329 pgs: 32 unknown, 5 peering, 292 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.2 KiB/s wr, 86 op/s 2024-09-13T14:36:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:53 smithi177 bash[68594]: cluster 2024-09-13T14:36:52.055174+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-13T14:36:53.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:53 smithi177 bash[68594]: cluster 2024-09-13T14:36:52.055174+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-13T14:36:53.685 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ConcurrentCreatesUnvalidatedPool (4470 ms) 2024-09-13T14:36:53.685 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.RemoveFullTry 2024-09-13T14:36:54.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:54 smithi177 bash[68594]: cluster 2024-09-13T14:36:53.069814+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-13T14:36:54.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:54 smithi177 bash[68594]: cluster 2024-09-13T14:36:53.069814+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-13T14:36:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:54 smithi099 bash[84576]: cluster 2024-09-13T14:36:53.069814+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-13T14:36:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:54 smithi099 bash[84576]: cluster 2024-09-13T14:36:53.069814+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-13T14:36:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:54 smithi099 bash[87147]: cluster 2024-09-13T14:36:53.069814+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-13T14:36:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:54 smithi099 bash[87147]: cluster 2024-09-13T14:36:53.069814+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-13T14:36:55.146 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:55 smithi177 bash[68594]: cluster 2024-09-13T14:36:54.040257+0000 mgr.y (mgr.26252) 226 : cluster [DBG] pgmap v431: 329 pgs: 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:55 smithi177 bash[68594]: cluster 2024-09-13T14:36:54.040257+0000 mgr.y (mgr.26252) 226 : cluster [DBG] pgmap v431: 329 pgs: 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:55 smithi177 bash[68594]: cluster 2024-09-13T14:36:54.083830+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-13T14:36:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:55 smithi177 bash[68594]: cluster 2024-09-13T14:36:54.083830+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-13T14:36:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:55 smithi177 bash[68594]: audit 2024-09-13T14:36:54.094057+0000 mon.a (mon.0) 731 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:55.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:55 smithi177 bash[68594]: audit 2024-09-13T14:36:54.094057+0000 mon.a (mon.0) 731 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:55.565 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[76324]: [13/Sep/2024:14:36:55] ENGINE Bus STOPPING 2024-09-13T14:36:55.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[84576]: cluster 2024-09-13T14:36:54.040257+0000 mgr.y (mgr.26252) 226 : cluster [DBG] pgmap v431: 329 pgs: 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:55.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[84576]: cluster 2024-09-13T14:36:54.040257+0000 mgr.y (mgr.26252) 226 : cluster [DBG] pgmap v431: 329 pgs: 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:55.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[84576]: cluster 2024-09-13T14:36:54.083830+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-13T14:36:55.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[84576]: cluster 2024-09-13T14:36:54.083830+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-13T14:36:55.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[84576]: audit 2024-09-13T14:36:54.094057+0000 mon.a (mon.0) 731 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:55.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[84576]: audit 2024-09-13T14:36:54.094057+0000 mon.a (mon.0) 731 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:55.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[87147]: cluster 2024-09-13T14:36:54.040257+0000 mgr.y (mgr.26252) 226 : cluster [DBG] pgmap v431: 329 pgs: 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:55.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[87147]: cluster 2024-09-13T14:36:54.040257+0000 mgr.y (mgr.26252) 226 : cluster [DBG] pgmap v431: 329 pgs: 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:55.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[87147]: cluster 2024-09-13T14:36:54.083830+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-13T14:36:55.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[87147]: cluster 2024-09-13T14:36:54.083830+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-13T14:36:55.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[87147]: audit 2024-09-13T14:36:54.094057+0000 mon.a (mon.0) 731 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:55.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[87147]: audit 2024-09-13T14:36:54.094057+0000 mon.a (mon.0) 731 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:36:55.821 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[76324]: [13/Sep/2024:14:36:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:36:55.821 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[76324]: [13/Sep/2024:14:36:55] ENGINE Bus STOPPED 2024-09-13T14:36:55.822 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[76324]: [13/Sep/2024:14:36:55] ENGINE Bus STARTING 2024-09-13T14:36:55.822 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[76324]: [13/Sep/2024:14:36:55] ENGINE Serving on http://:::9283 2024-09-13T14:36:55.822 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:55 smithi099 bash[76324]: [13/Sep/2024:14:36:55] ENGINE Bus STARTED 2024-09-13T14:36:56.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: audit 2024-09-13T14:36:55.078554+0000 mon.a (mon.0) 732 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:56.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: audit 2024-09-13T14:36:55.078554+0000 mon.a (mon.0) 732 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:56.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: cluster 2024-09-13T14:36:55.088369+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-13T14:36:56.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: cluster 2024-09-13T14:36:55.088369+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-13T14:36:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: audit 2024-09-13T14:36:55.149231+0000 mon.a (mon.0) 734 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:36:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: audit 2024-09-13T14:36:55.149231+0000 mon.a (mon.0) 734 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:36:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: audit 2024-09-13T14:36:55.168996+0000 mon.a (mon.0) 735 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: audit 2024-09-13T14:36:55.168996+0000 mon.a (mon.0) 735 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: audit 2024-09-13T14:36:55.568582+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:36:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: audit 2024-09-13T14:36:55.568582+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:36:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: audit 2024-09-13T14:36:55.570291+0000 mgr.y (mgr.26252) 227 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:36:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: audit 2024-09-13T14:36:55.570291+0000 mgr.y (mgr.26252) 227 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:36:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: cluster 2024-09-13T14:36:56.052958+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-13T14:36:56.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:56 smithi177 bash[68594]: cluster 2024-09-13T14:36:56.052958+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-13T14:36:56.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: audit 2024-09-13T14:36:55.078554+0000 mon.a (mon.0) 732 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:56.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: audit 2024-09-13T14:36:55.078554+0000 mon.a (mon.0) 732 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:56.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: cluster 2024-09-13T14:36:55.088369+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: cluster 2024-09-13T14:36:55.088369+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: audit 2024-09-13T14:36:55.149231+0000 mon.a (mon.0) 734 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: audit 2024-09-13T14:36:55.149231+0000 mon.a (mon.0) 734 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: audit 2024-09-13T14:36:55.168996+0000 mon.a (mon.0) 735 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: audit 2024-09-13T14:36:55.168996+0000 mon.a (mon.0) 735 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: audit 2024-09-13T14:36:55.568582+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: audit 2024-09-13T14:36:55.568582+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: audit 2024-09-13T14:36:55.570291+0000 mgr.y (mgr.26252) 227 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: audit 2024-09-13T14:36:55.570291+0000 mgr.y (mgr.26252) 227 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: cluster 2024-09-13T14:36:56.052958+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-13T14:36:56.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[84576]: cluster 2024-09-13T14:36:56.052958+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: audit 2024-09-13T14:36:55.078554+0000 mon.a (mon.0) 732 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: audit 2024-09-13T14:36:55.078554+0000 mon.a (mon.0) 732 : audit [INF] from='client.? 172.21.15.99:0/880586784' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: cluster 2024-09-13T14:36:55.088369+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: cluster 2024-09-13T14:36:55.088369+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: audit 2024-09-13T14:36:55.149231+0000 mon.a (mon.0) 734 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: audit 2024-09-13T14:36:55.149231+0000 mon.a (mon.0) 734 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: audit 2024-09-13T14:36:55.168996+0000 mon.a (mon.0) 735 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: audit 2024-09-13T14:36:55.168996+0000 mon.a (mon.0) 735 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: audit 2024-09-13T14:36:55.568582+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: audit 2024-09-13T14:36:55.568582+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: audit 2024-09-13T14:36:55.570291+0000 mgr.y (mgr.26252) 227 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: audit 2024-09-13T14:36:55.570291+0000 mgr.y (mgr.26252) 227 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:36:56.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: cluster 2024-09-13T14:36:56.052958+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-13T14:36:56.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:56 smithi099 bash[87147]: cluster 2024-09-13T14:36:56.052958+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-13T14:36:57.250 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:36:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:36:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:36:57] "GET /metrics HTTP/1.1" 200 51003 "" "Prometheus/2.51.0" 2024-09-13T14:36:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:57 smithi099 bash[84576]: cluster 2024-09-13T14:36:56.041251+0000 mgr.y (mgr.26252) 228 : cluster [DBG] pgmap v434: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:57 smithi099 bash[84576]: cluster 2024-09-13T14:36:56.041251+0000 mgr.y (mgr.26252) 228 : cluster [DBG] pgmap v434: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:57 smithi099 bash[84576]: cluster 2024-09-13T14:36:57.060046+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e623: 8 total, 8 up, 8 in 2024-09-13T14:36:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:57 smithi099 bash[84576]: cluster 2024-09-13T14:36:57.060046+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e623: 8 total, 8 up, 8 in 2024-09-13T14:36:57.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:57 smithi099 bash[87147]: cluster 2024-09-13T14:36:56.041251+0000 mgr.y (mgr.26252) 228 : cluster [DBG] pgmap v434: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:57.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:57 smithi099 bash[87147]: cluster 2024-09-13T14:36:56.041251+0000 mgr.y (mgr.26252) 228 : cluster [DBG] pgmap v434: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:57.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:57 smithi099 bash[87147]: cluster 2024-09-13T14:36:57.060046+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e623: 8 total, 8 up, 8 in 2024-09-13T14:36:57.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:57 smithi099 bash[87147]: cluster 2024-09-13T14:36:57.060046+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e623: 8 total, 8 up, 8 in 2024-09-13T14:36:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:57 smithi177 bash[68594]: cluster 2024-09-13T14:36:56.041251+0000 mgr.y (mgr.26252) 228 : cluster [DBG] pgmap v434: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:57 smithi177 bash[68594]: cluster 2024-09-13T14:36:56.041251+0000 mgr.y (mgr.26252) 228 : cluster [DBG] pgmap v434: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.2 KiB/s wr, 118 op/s; 4 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:36:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:57 smithi177 bash[68594]: cluster 2024-09-13T14:36:57.060046+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e623: 8 total, 8 up, 8 in 2024-09-13T14:36:57.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:57 smithi177 bash[68594]: cluster 2024-09-13T14:36:57.060046+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e623: 8 total, 8 up, 8 in 2024-09-13T14:36:58.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:58 smithi099 bash[87147]: audit 2024-09-13T14:36:57.275871+0000 mon.a (mon.0) 739 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:36:58.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:58 smithi099 bash[87147]: audit 2024-09-13T14:36:57.275871+0000 mon.a (mon.0) 739 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:36:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:58 smithi099 bash[84576]: audit 2024-09-13T14:36:57.275871+0000 mon.a (mon.0) 739 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:36:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:58 smithi099 bash[84576]: audit 2024-09-13T14:36:57.275871+0000 mon.a (mon.0) 739 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:36:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:58 smithi177 bash[68594]: audit 2024-09-13T14:36:57.275871+0000 mon.a (mon.0) 739 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:36:58.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:58 smithi177 bash[68594]: audit 2024-09-13T14:36:57.275871+0000 mon.a (mon.0) 739 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:36:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[84576]: cluster 2024-09-13T14:36:58.042585+0000 mgr.y (mgr.26252) 229 : cluster [DBG] pgmap v437: 361 pgs: 30 unknown, 331 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 1.5 KiB/s wr, 33 op/s 2024-09-13T14:36:59.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[84576]: cluster 2024-09-13T14:36:58.042585+0000 mgr.y (mgr.26252) 229 : cluster [DBG] pgmap v437: 361 pgs: 30 unknown, 331 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 1.5 KiB/s wr, 33 op/s 2024-09-13T14:36:59.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[84576]: audit 2024-09-13T14:36:58.162817+0000 mon.a (mon.0) 740 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:36:59.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[84576]: audit 2024-09-13T14:36:58.162817+0000 mon.a (mon.0) 740 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:36:59.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[84576]: cluster 2024-09-13T14:36:58.170214+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e624: 8 total, 8 up, 8 in 2024-09-13T14:36:59.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[84576]: cluster 2024-09-13T14:36:58.170214+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e624: 8 total, 8 up, 8 in 2024-09-13T14:36:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[87147]: cluster 2024-09-13T14:36:58.042585+0000 mgr.y (mgr.26252) 229 : cluster [DBG] pgmap v437: 361 pgs: 30 unknown, 331 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 1.5 KiB/s wr, 33 op/s 2024-09-13T14:36:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[87147]: cluster 2024-09-13T14:36:58.042585+0000 mgr.y (mgr.26252) 229 : cluster [DBG] pgmap v437: 361 pgs: 30 unknown, 331 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 1.5 KiB/s wr, 33 op/s 2024-09-13T14:36:59.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[87147]: audit 2024-09-13T14:36:58.162817+0000 mon.a (mon.0) 740 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:36:59.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[87147]: audit 2024-09-13T14:36:58.162817+0000 mon.a (mon.0) 740 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:36:59.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[87147]: cluster 2024-09-13T14:36:58.170214+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e624: 8 total, 8 up, 8 in 2024-09-13T14:36:59.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:36:59 smithi099 bash[87147]: cluster 2024-09-13T14:36:58.170214+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e624: 8 total, 8 up, 8 in 2024-09-13T14:36:59.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:59 smithi177 bash[68594]: cluster 2024-09-13T14:36:58.042585+0000 mgr.y (mgr.26252) 229 : cluster [DBG] pgmap v437: 361 pgs: 30 unknown, 331 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 1.5 KiB/s wr, 33 op/s 2024-09-13T14:36:59.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:59 smithi177 bash[68594]: cluster 2024-09-13T14:36:58.042585+0000 mgr.y (mgr.26252) 229 : cluster [DBG] pgmap v437: 361 pgs: 30 unknown, 331 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 1.5 KiB/s wr, 33 op/s 2024-09-13T14:36:59.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:59 smithi177 bash[68594]: audit 2024-09-13T14:36:58.162817+0000 mon.a (mon.0) 740 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:36:59.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:59 smithi177 bash[68594]: audit 2024-09-13T14:36:58.162817+0000 mon.a (mon.0) 740 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:36:59.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:59 smithi177 bash[68594]: cluster 2024-09-13T14:36:58.170214+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e624: 8 total, 8 up, 8 in 2024-09-13T14:36:59.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:36:59 smithi177 bash[68594]: cluster 2024-09-13T14:36:58.170214+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e624: 8 total, 8 up, 8 in 2024-09-13T14:37:00.605 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:37:00 smithi177 bash[61439]: ts=2024-09-13T14:37:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:01.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:01 smithi099 bash[84576]: cluster 2024-09-13T14:37:00.045566+0000 mgr.y (mgr.26252) 230 : cluster [DBG] pgmap v439: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 4.2 KiB/s wr, 67 op/s 2024-09-13T14:37:01.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:01 smithi099 bash[84576]: cluster 2024-09-13T14:37:00.045566+0000 mgr.y (mgr.26252) 230 : cluster [DBG] pgmap v439: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 4.2 KiB/s wr, 67 op/s 2024-09-13T14:37:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:01 smithi099 bash[84576]: cluster 2024-09-13T14:37:01.061862+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e625: 8 total, 8 up, 8 in 2024-09-13T14:37:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:01 smithi099 bash[84576]: cluster 2024-09-13T14:37:01.061862+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e625: 8 total, 8 up, 8 in 2024-09-13T14:37:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:01 smithi099 bash[87147]: cluster 2024-09-13T14:37:00.045566+0000 mgr.y (mgr.26252) 230 : cluster [DBG] pgmap v439: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 4.2 KiB/s wr, 67 op/s 2024-09-13T14:37:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:01 smithi099 bash[87147]: cluster 2024-09-13T14:37:00.045566+0000 mgr.y (mgr.26252) 230 : cluster [DBG] pgmap v439: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 4.2 KiB/s wr, 67 op/s 2024-09-13T14:37:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:01 smithi099 bash[87147]: cluster 2024-09-13T14:37:01.061862+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e625: 8 total, 8 up, 8 in 2024-09-13T14:37:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:01 smithi099 bash[87147]: cluster 2024-09-13T14:37:01.061862+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e625: 8 total, 8 up, 8 in 2024-09-13T14:37:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:01 smithi177 bash[68594]: cluster 2024-09-13T14:37:00.045566+0000 mgr.y (mgr.26252) 230 : cluster [DBG] pgmap v439: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 4.2 KiB/s wr, 67 op/s 2024-09-13T14:37:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:01 smithi177 bash[68594]: cluster 2024-09-13T14:37:00.045566+0000 mgr.y (mgr.26252) 230 : cluster [DBG] pgmap v439: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 4.2 KiB/s wr, 67 op/s 2024-09-13T14:37:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:01 smithi177 bash[68594]: cluster 2024-09-13T14:37:01.061862+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e625: 8 total, 8 up, 8 in 2024-09-13T14:37:01.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:01 smithi177 bash[68594]: cluster 2024-09-13T14:37:01.061862+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e625: 8 total, 8 up, 8 in 2024-09-13T14:37:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[84576]: audit 2024-09-13T14:37:01.193387+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[84576]: audit 2024-09-13T14:37:01.193387+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[84576]: audit 2024-09-13T14:37:01.205731+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[84576]: audit 2024-09-13T14:37:01.205731+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[84576]: audit 2024-09-13T14:37:01.865555+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[84576]: audit 2024-09-13T14:37:01.865555+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[84576]: audit 2024-09-13T14:37:01.867506+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[84576]: audit 2024-09-13T14:37:01.867506+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[84576]: audit 2024-09-13T14:37:01.877316+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[84576]: audit 2024-09-13T14:37:01.877316+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[87147]: audit 2024-09-13T14:37:01.193387+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[87147]: audit 2024-09-13T14:37:01.193387+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[87147]: audit 2024-09-13T14:37:01.205731+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[87147]: audit 2024-09-13T14:37:01.205731+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[87147]: audit 2024-09-13T14:37:01.865555+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[87147]: audit 2024-09-13T14:37:01.865555+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[87147]: audit 2024-09-13T14:37:01.867506+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[87147]: audit 2024-09-13T14:37:01.867506+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:02.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[87147]: audit 2024-09-13T14:37:01.877316+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:02 smithi099 bash[87147]: audit 2024-09-13T14:37:01.877316+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:02 smithi177 bash[68594]: audit 2024-09-13T14:37:01.193387+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:02 smithi177 bash[68594]: audit 2024-09-13T14:37:01.193387+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:02 smithi177 bash[68594]: audit 2024-09-13T14:37:01.205731+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:02 smithi177 bash[68594]: audit 2024-09-13T14:37:01.205731+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:02 smithi177 bash[68594]: audit 2024-09-13T14:37:01.865555+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:02 smithi177 bash[68594]: audit 2024-09-13T14:37:01.865555+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:02 smithi177 bash[68594]: audit 2024-09-13T14:37:01.867506+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:02 smithi177 bash[68594]: audit 2024-09-13T14:37:01.867506+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:02.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:02 smithi177 bash[68594]: audit 2024-09-13T14:37:01.877316+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:02.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:02 smithi177 bash[68594]: audit 2024-09-13T14:37:01.877316+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:03 smithi099 bash[84576]: cluster 2024-09-13T14:37:02.046512+0000 mgr.y (mgr.26252) 231 : cluster [DBG] pgmap v441: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 3.5 KiB/s wr, 56 op/s 2024-09-13T14:37:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:03 smithi099 bash[84576]: cluster 2024-09-13T14:37:02.046512+0000 mgr.y (mgr.26252) 231 : cluster [DBG] pgmap v441: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 3.5 KiB/s wr, 56 op/s 2024-09-13T14:37:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:03 smithi099 bash[87147]: cluster 2024-09-13T14:37:02.046512+0000 mgr.y (mgr.26252) 231 : cluster [DBG] pgmap v441: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 3.5 KiB/s wr, 56 op/s 2024-09-13T14:37:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:03 smithi099 bash[87147]: cluster 2024-09-13T14:37:02.046512+0000 mgr.y (mgr.26252) 231 : cluster [DBG] pgmap v441: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 3.5 KiB/s wr, 56 op/s 2024-09-13T14:37:03.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:03 smithi177 bash[68594]: cluster 2024-09-13T14:37:02.046512+0000 mgr.y (mgr.26252) 231 : cluster [DBG] pgmap v441: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 3.5 KiB/s wr, 56 op/s 2024-09-13T14:37:03.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:03 smithi177 bash[68594]: cluster 2024-09-13T14:37:02.046512+0000 mgr.y (mgr.26252) 231 : cluster [DBG] pgmap v441: 361 pgs: 361 active+clean; 584 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 3.5 KiB/s wr, 56 op/s 2024-09-13T14:37:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:04 smithi099 bash[84576]: audit 2024-09-13T14:37:03.248354+0000 mon.a (mon.0) 748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:04 smithi099 bash[84576]: audit 2024-09-13T14:37:03.248354+0000 mon.a (mon.0) 748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:04 smithi099 bash[84576]: audit 2024-09-13T14:37:03.486341+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:04 smithi099 bash[84576]: audit 2024-09-13T14:37:03.486341+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:04 smithi099 bash[87147]: audit 2024-09-13T14:37:03.248354+0000 mon.a (mon.0) 748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:04 smithi099 bash[87147]: audit 2024-09-13T14:37:03.248354+0000 mon.a (mon.0) 748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:04 smithi099 bash[87147]: audit 2024-09-13T14:37:03.486341+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:04.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:04 smithi099 bash[87147]: audit 2024-09-13T14:37:03.486341+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:04.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:04 smithi177 bash[68594]: audit 2024-09-13T14:37:03.248354+0000 mon.a (mon.0) 748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:04.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:04 smithi177 bash[68594]: audit 2024-09-13T14:37:03.248354+0000 mon.a (mon.0) 748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:04.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:04 smithi177 bash[68594]: audit 2024-09-13T14:37:03.486341+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:04.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:04 smithi177 bash[68594]: audit 2024-09-13T14:37:03.486341+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:05.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:05 smithi099 bash[84576]: cluster 2024-09-13T14:37:04.049542+0000 mgr.y (mgr.26252) 232 : cluster [DBG] pgmap v442: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 882 KiB/s wr, 50 op/s 2024-09-13T14:37:05.850 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:05 smithi099 bash[84576]: cluster 2024-09-13T14:37:04.049542+0000 mgr.y (mgr.26252) 232 : cluster [DBG] pgmap v442: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 882 KiB/s wr, 50 op/s 2024-09-13T14:37:05.850 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:05 smithi099 bash[87147]: cluster 2024-09-13T14:37:04.049542+0000 mgr.y (mgr.26252) 232 : cluster [DBG] pgmap v442: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 882 KiB/s wr, 50 op/s 2024-09-13T14:37:05.850 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:05 smithi099 bash[87147]: cluster 2024-09-13T14:37:04.049542+0000 mgr.y (mgr.26252) 232 : cluster [DBG] pgmap v442: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 882 KiB/s wr, 50 op/s 2024-09-13T14:37:05.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:05 smithi177 bash[68594]: cluster 2024-09-13T14:37:04.049542+0000 mgr.y (mgr.26252) 232 : cluster [DBG] pgmap v442: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 882 KiB/s wr, 50 op/s 2024-09-13T14:37:05.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:05 smithi177 bash[68594]: cluster 2024-09-13T14:37:04.049542+0000 mgr.y (mgr.26252) 232 : cluster [DBG] pgmap v442: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 882 KiB/s wr, 50 op/s 2024-09-13T14:37:07.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:37:07] "GET /metrics HTTP/1.1" 200 55823 "" "Prometheus/2.51.0" 2024-09-13T14:37:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:07 smithi099 bash[84576]: cluster 2024-09-13T14:37:06.050437+0000 mgr.y (mgr.26252) 233 : cluster [DBG] pgmap v443: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 769 KiB/s wr, 26 op/s 2024-09-13T14:37:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:07 smithi099 bash[84576]: cluster 2024-09-13T14:37:06.050437+0000 mgr.y (mgr.26252) 233 : cluster [DBG] pgmap v443: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 769 KiB/s wr, 26 op/s 2024-09-13T14:37:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:07 smithi099 bash[87147]: cluster 2024-09-13T14:37:06.050437+0000 mgr.y (mgr.26252) 233 : cluster [DBG] pgmap v443: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 769 KiB/s wr, 26 op/s 2024-09-13T14:37:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:07 smithi099 bash[87147]: cluster 2024-09-13T14:37:06.050437+0000 mgr.y (mgr.26252) 233 : cluster [DBG] pgmap v443: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 769 KiB/s wr, 26 op/s 2024-09-13T14:37:07.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:07 smithi177 bash[68594]: cluster 2024-09-13T14:37:06.050437+0000 mgr.y (mgr.26252) 233 : cluster [DBG] pgmap v443: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 769 KiB/s wr, 26 op/s 2024-09-13T14:37:07.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:07 smithi177 bash[68594]: cluster 2024-09-13T14:37:06.050437+0000 mgr.y (mgr.26252) 233 : cluster [DBG] pgmap v443: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 769 KiB/s wr, 26 op/s 2024-09-13T14:37:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:09 smithi099 bash[84576]: cluster 2024-09-13T14:37:08.051723+0000 mgr.y (mgr.26252) 234 : cluster [DBG] pgmap v444: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 623 KiB/s wr, 21 op/s 2024-09-13T14:37:09.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:09 smithi099 bash[84576]: cluster 2024-09-13T14:37:08.051723+0000 mgr.y (mgr.26252) 234 : cluster [DBG] pgmap v444: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 623 KiB/s wr, 21 op/s 2024-09-13T14:37:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:09 smithi099 bash[87147]: cluster 2024-09-13T14:37:08.051723+0000 mgr.y (mgr.26252) 234 : cluster [DBG] pgmap v444: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 623 KiB/s wr, 21 op/s 2024-09-13T14:37:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:09 smithi099 bash[87147]: cluster 2024-09-13T14:37:08.051723+0000 mgr.y (mgr.26252) 234 : cluster [DBG] pgmap v444: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 623 KiB/s wr, 21 op/s 2024-09-13T14:37:09.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:09 smithi177 bash[68594]: cluster 2024-09-13T14:37:08.051723+0000 mgr.y (mgr.26252) 234 : cluster [DBG] pgmap v444: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 623 KiB/s wr, 21 op/s 2024-09-13T14:37:09.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:09 smithi177 bash[68594]: cluster 2024-09-13T14:37:08.051723+0000 mgr.y (mgr.26252) 234 : cluster [DBG] pgmap v444: 361 pgs: 361 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 623 KiB/s wr, 21 op/s 2024-09-13T14:37:10.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:37:10 smithi177 bash[61439]: ts=2024-09-13T14:37:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:11.519 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.RemoveFullTry (17834 ms) 2024-09-13T14:37:11.519 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.RemoveFullTryDataPool 2024-09-13T14:37:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[84576]: cluster 2024-09-13T14:37:10.054574+0000 mgr.y (mgr.26252) 235 : cluster [DBG] pgmap v445: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[84576]: cluster 2024-09-13T14:37:10.054574+0000 mgr.y (mgr.26252) 235 : cluster [DBG] pgmap v445: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[84576]: cluster 2024-09-13T14:37:11.049967+0000 mon.a (mon.0) 750 : cluster [WRN] pool 'test-librbd-smithi099-69110-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[84576]: cluster 2024-09-13T14:37:11.049967+0000 mon.a (mon.0) 750 : cluster [WRN] pool 'test-librbd-smithi099-69110-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[84576]: cluster 2024-09-13T14:37:11.050576+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[84576]: cluster 2024-09-13T14:37:11.050576+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[84576]: cluster 2024-09-13T14:37:11.060493+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e626: 8 total, 8 up, 8 in 2024-09-13T14:37:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[84576]: cluster 2024-09-13T14:37:11.060493+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e626: 8 total, 8 up, 8 in 2024-09-13T14:37:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[87147]: cluster 2024-09-13T14:37:10.054574+0000 mgr.y (mgr.26252) 235 : cluster [DBG] pgmap v445: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[87147]: cluster 2024-09-13T14:37:10.054574+0000 mgr.y (mgr.26252) 235 : cluster [DBG] pgmap v445: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[87147]: cluster 2024-09-13T14:37:11.049967+0000 mon.a (mon.0) 750 : cluster [WRN] pool 'test-librbd-smithi099-69110-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[87147]: cluster 2024-09-13T14:37:11.049967+0000 mon.a (mon.0) 750 : cluster [WRN] pool 'test-librbd-smithi099-69110-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[87147]: cluster 2024-09-13T14:37:11.050576+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[87147]: cluster 2024-09-13T14:37:11.050576+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[87147]: cluster 2024-09-13T14:37:11.060493+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e626: 8 total, 8 up, 8 in 2024-09-13T14:37:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:11 smithi099 bash[87147]: cluster 2024-09-13T14:37:11.060493+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e626: 8 total, 8 up, 8 in 2024-09-13T14:37:11.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:11 smithi177 bash[68594]: cluster 2024-09-13T14:37:10.054574+0000 mgr.y (mgr.26252) 235 : cluster [DBG] pgmap v445: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:11.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:11 smithi177 bash[68594]: cluster 2024-09-13T14:37:10.054574+0000 mgr.y (mgr.26252) 235 : cluster [DBG] pgmap v445: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:11.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:11 smithi177 bash[68594]: cluster 2024-09-13T14:37:11.049967+0000 mon.a (mon.0) 750 : cluster [WRN] pool 'test-librbd-smithi099-69110-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:11.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:11 smithi177 bash[68594]: cluster 2024-09-13T14:37:11.049967+0000 mon.a (mon.0) 750 : cluster [WRN] pool 'test-librbd-smithi099-69110-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:11.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:11 smithi177 bash[68594]: cluster 2024-09-13T14:37:11.050576+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:11.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:11 smithi177 bash[68594]: cluster 2024-09-13T14:37:11.050576+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:11.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:11 smithi177 bash[68594]: cluster 2024-09-13T14:37:11.060493+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e626: 8 total, 8 up, 8 in 2024-09-13T14:37:11.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:11 smithi177 bash[68594]: cluster 2024-09-13T14:37:11.060493+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e626: 8 total, 8 up, 8 in 2024-09-13T14:37:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[84576]: cluster 2024-09-13T14:37:12.055479+0000 mgr.y (mgr.26252) 236 : cluster [DBG] pgmap v447: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[84576]: cluster 2024-09-13T14:37:12.055479+0000 mgr.y (mgr.26252) 236 : cluster [DBG] pgmap v447: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[84576]: cluster 2024-09-13T14:37:12.519767+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2024-09-13T14:37:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[84576]: cluster 2024-09-13T14:37:12.519767+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2024-09-13T14:37:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[84576]: audit 2024-09-13T14:37:12.524335+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.99:0/2403632968' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[84576]: audit 2024-09-13T14:37:12.524335+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.99:0/2403632968' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[84576]: audit 2024-09-13T14:37:12.525025+0000 mon.a (mon.0) 754 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[84576]: audit 2024-09-13T14:37:12.525025+0000 mon.a (mon.0) 754 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[87147]: cluster 2024-09-13T14:37:12.055479+0000 mgr.y (mgr.26252) 236 : cluster [DBG] pgmap v447: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:13.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[87147]: cluster 2024-09-13T14:37:12.055479+0000 mgr.y (mgr.26252) 236 : cluster [DBG] pgmap v447: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:13.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[87147]: cluster 2024-09-13T14:37:12.519767+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2024-09-13T14:37:13.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[87147]: cluster 2024-09-13T14:37:12.519767+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2024-09-13T14:37:13.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[87147]: audit 2024-09-13T14:37:12.524335+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.99:0/2403632968' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[87147]: audit 2024-09-13T14:37:12.524335+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.99:0/2403632968' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[87147]: audit 2024-09-13T14:37:12.525025+0000 mon.a (mon.0) 754 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:13 smithi099 bash[87147]: audit 2024-09-13T14:37:12.525025+0000 mon.a (mon.0) 754 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:13 smithi177 bash[68594]: cluster 2024-09-13T14:37:12.055479+0000 mgr.y (mgr.26252) 236 : cluster [DBG] pgmap v447: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:13.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:13 smithi177 bash[68594]: cluster 2024-09-13T14:37:12.055479+0000 mgr.y (mgr.26252) 236 : cluster [DBG] pgmap v447: 361 pgs: 361 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-09-13T14:37:13.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:13 smithi177 bash[68594]: cluster 2024-09-13T14:37:12.519767+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2024-09-13T14:37:13.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:13 smithi177 bash[68594]: cluster 2024-09-13T14:37:12.519767+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2024-09-13T14:37:13.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:13 smithi177 bash[68594]: audit 2024-09-13T14:37:12.524335+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.99:0/2403632968' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:13 smithi177 bash[68594]: audit 2024-09-13T14:37:12.524335+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.99:0/2403632968' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:13 smithi177 bash[68594]: audit 2024-09-13T14:37:12.525025+0000 mon.a (mon.0) 754 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:13.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:13 smithi177 bash[68594]: audit 2024-09-13T14:37:12.525025+0000 mon.a (mon.0) 754 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:14.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:14 smithi099 bash[84576]: audit 2024-09-13T14:37:13.526298+0000 mon.a (mon.0) 755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:14.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:14 smithi099 bash[84576]: audit 2024-09-13T14:37:13.526298+0000 mon.a (mon.0) 755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:14.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:14 smithi099 bash[84576]: cluster 2024-09-13T14:37:13.536100+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-09-13T14:37:14.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:14 smithi099 bash[84576]: cluster 2024-09-13T14:37:13.536100+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-09-13T14:37:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:14 smithi099 bash[87147]: audit 2024-09-13T14:37:13.526298+0000 mon.a (mon.0) 755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:14 smithi099 bash[87147]: audit 2024-09-13T14:37:13.526298+0000 mon.a (mon.0) 755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:14 smithi099 bash[87147]: cluster 2024-09-13T14:37:13.536100+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-09-13T14:37:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:14 smithi099 bash[87147]: cluster 2024-09-13T14:37:13.536100+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-09-13T14:37:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:14 smithi177 bash[68594]: audit 2024-09-13T14:37:13.526298+0000 mon.a (mon.0) 755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:14 smithi177 bash[68594]: audit 2024-09-13T14:37:13.526298+0000 mon.a (mon.0) 755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:14 smithi177 bash[68594]: cluster 2024-09-13T14:37:13.536100+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-09-13T14:37:14.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:14 smithi177 bash[68594]: cluster 2024-09-13T14:37:13.536100+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-09-13T14:37:15.605 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:37:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:15 smithi177 bash[68594]: cluster 2024-09-13T14:37:14.058369+0000 mgr.y (mgr.26252) 237 : cluster [DBG] pgmap v450: 393 pgs: 6 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 MiB/s wr, 29 op/s 2024-09-13T14:37:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:15 smithi177 bash[68594]: cluster 2024-09-13T14:37:14.058369+0000 mgr.y (mgr.26252) 237 : cluster [DBG] pgmap v450: 393 pgs: 6 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 MiB/s wr, 29 op/s 2024-09-13T14:37:15.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:15 smithi177 bash[68594]: cluster 2024-09-13T14:37:14.549220+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-13T14:37:15.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:15 smithi177 bash[68594]: cluster 2024-09-13T14:37:14.549220+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-13T14:37:15.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:15 smithi177 bash[68594]: audit 2024-09-13T14:37:14.550589+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.99:0/3416426822' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:15.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:15 smithi177 bash[68594]: audit 2024-09-13T14:37:14.550589+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.99:0/3416426822' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:15.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:15 smithi177 bash[68594]: audit 2024-09-13T14:37:14.554933+0000 mon.a (mon.0) 758 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:15.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:15 smithi177 bash[68594]: audit 2024-09-13T14:37:14.554933+0000 mon.a (mon.0) 758 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:16.062 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[76324]: [13/Sep/2024:14:37:15] ENGINE Bus STOPPING 2024-09-13T14:37:16.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[87147]: cluster 2024-09-13T14:37:14.058369+0000 mgr.y (mgr.26252) 237 : cluster [DBG] pgmap v450: 393 pgs: 6 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 MiB/s wr, 29 op/s 2024-09-13T14:37:16.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[87147]: cluster 2024-09-13T14:37:14.058369+0000 mgr.y (mgr.26252) 237 : cluster [DBG] pgmap v450: 393 pgs: 6 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 MiB/s wr, 29 op/s 2024-09-13T14:37:16.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[87147]: cluster 2024-09-13T14:37:14.549220+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-13T14:37:16.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[87147]: cluster 2024-09-13T14:37:14.549220+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-13T14:37:16.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[87147]: audit 2024-09-13T14:37:14.550589+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.99:0/3416426822' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:16.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[87147]: audit 2024-09-13T14:37:14.550589+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.99:0/3416426822' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:16.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[87147]: audit 2024-09-13T14:37:14.554933+0000 mon.a (mon.0) 758 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:16.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[87147]: audit 2024-09-13T14:37:14.554933+0000 mon.a (mon.0) 758 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:16.063 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[84576]: cluster 2024-09-13T14:37:14.058369+0000 mgr.y (mgr.26252) 237 : cluster [DBG] pgmap v450: 393 pgs: 6 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 MiB/s wr, 29 op/s 2024-09-13T14:37:16.063 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[84576]: cluster 2024-09-13T14:37:14.058369+0000 mgr.y (mgr.26252) 237 : cluster [DBG] pgmap v450: 393 pgs: 6 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 MiB/s wr, 29 op/s 2024-09-13T14:37:16.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[84576]: cluster 2024-09-13T14:37:14.549220+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-13T14:37:16.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[84576]: cluster 2024-09-13T14:37:14.549220+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-13T14:37:16.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[84576]: audit 2024-09-13T14:37:14.550589+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.99:0/3416426822' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:16.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[84576]: audit 2024-09-13T14:37:14.550589+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.99:0/3416426822' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:16.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[84576]: audit 2024-09-13T14:37:14.554933+0000 mon.a (mon.0) 758 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:16.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:15 smithi099 bash[84576]: audit 2024-09-13T14:37:14.554933+0000 mon.a (mon.0) 758 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:37:16.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[76324]: [13/Sep/2024:14:37:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:37:16.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[76324]: [13/Sep/2024:14:37:16] ENGINE Bus STOPPED 2024-09-13T14:37:16.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[76324]: [13/Sep/2024:14:37:16] ENGINE Bus STARTING 2024-09-13T14:37:16.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[76324]: [13/Sep/2024:14:37:16] ENGINE Serving on http://:::9283 2024-09-13T14:37:16.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[76324]: [13/Sep/2024:14:37:16] ENGINE Bus STARTED 2024-09-13T14:37:16.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: audit 2024-09-13T14:37:15.544081+0000 mon.a (mon.0) 759 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:16.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: audit 2024-09-13T14:37:15.544081+0000 mon.a (mon.0) 759 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:16.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: cluster 2024-09-13T14:37:15.555539+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-13T14:37:16.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: cluster 2024-09-13T14:37:15.555539+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-13T14:37:16.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: audit 2024-09-13T14:37:15.608944+0000 mon.a (mon.0) 761 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:37:16.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: audit 2024-09-13T14:37:15.608944+0000 mon.a (mon.0) 761 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:37:16.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: audit 2024-09-13T14:37:15.624593+0000 mon.a (mon.0) 762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:16.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: audit 2024-09-13T14:37:15.624593+0000 mon.a (mon.0) 762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: cluster 2024-09-13T14:37:16.052262+0000 mon.a (mon.0) 763 : cluster [INF] pool 'test-librbd-smithi099-69110-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: cluster 2024-09-13T14:37:16.052262+0000 mon.a (mon.0) 763 : cluster [INF] pool 'test-librbd-smithi099-69110-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: cluster 2024-09-13T14:37:16.053095+0000 mon.a (mon.0) 764 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: cluster 2024-09-13T14:37:16.053095+0000 mon.a (mon.0) 764 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: cluster 2024-09-13T14:37:16.053150+0000 mon.a (mon.0) 765 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: cluster 2024-09-13T14:37:16.053150+0000 mon.a (mon.0) 765 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: cluster 2024-09-13T14:37:16.074833+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-13T14:37:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: cluster 2024-09-13T14:37:16.074833+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-13T14:37:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: audit 2024-09-13T14:37:16.076462+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:37:16.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:16 smithi177 bash[68594]: audit 2024-09-13T14:37:16.076462+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:37:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: audit 2024-09-13T14:37:15.544081+0000 mon.a (mon.0) 759 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: audit 2024-09-13T14:37:15.544081+0000 mon.a (mon.0) 759 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: cluster 2024-09-13T14:37:15.555539+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-13T14:37:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: cluster 2024-09-13T14:37:15.555539+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-13T14:37:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: audit 2024-09-13T14:37:15.608944+0000 mon.a (mon.0) 761 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:37:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: audit 2024-09-13T14:37:15.608944+0000 mon.a (mon.0) 761 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: audit 2024-09-13T14:37:15.624593+0000 mon.a (mon.0) 762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: audit 2024-09-13T14:37:15.624593+0000 mon.a (mon.0) 762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: cluster 2024-09-13T14:37:16.052262+0000 mon.a (mon.0) 763 : cluster [INF] pool 'test-librbd-smithi099-69110-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: cluster 2024-09-13T14:37:16.052262+0000 mon.a (mon.0) 763 : cluster [INF] pool 'test-librbd-smithi099-69110-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: cluster 2024-09-13T14:37:16.053095+0000 mon.a (mon.0) 764 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: cluster 2024-09-13T14:37:16.053095+0000 mon.a (mon.0) 764 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: cluster 2024-09-13T14:37:16.053150+0000 mon.a (mon.0) 765 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: cluster 2024-09-13T14:37:16.053150+0000 mon.a (mon.0) 765 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: cluster 2024-09-13T14:37:16.074833+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: cluster 2024-09-13T14:37:16.074833+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: audit 2024-09-13T14:37:16.076462+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:37:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[84576]: audit 2024-09-13T14:37:16.076462+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: audit 2024-09-13T14:37:15.544081+0000 mon.a (mon.0) 759 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: audit 2024-09-13T14:37:15.544081+0000 mon.a (mon.0) 759 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: cluster 2024-09-13T14:37:15.555539+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: cluster 2024-09-13T14:37:15.555539+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: audit 2024-09-13T14:37:15.608944+0000 mon.a (mon.0) 761 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: audit 2024-09-13T14:37:15.608944+0000 mon.a (mon.0) 761 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: audit 2024-09-13T14:37:15.624593+0000 mon.a (mon.0) 762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: audit 2024-09-13T14:37:15.624593+0000 mon.a (mon.0) 762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: cluster 2024-09-13T14:37:16.052262+0000 mon.a (mon.0) 763 : cluster [INF] pool 'test-librbd-smithi099-69110-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: cluster 2024-09-13T14:37:16.052262+0000 mon.a (mon.0) 763 : cluster [INF] pool 'test-librbd-smithi099-69110-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: cluster 2024-09-13T14:37:16.053095+0000 mon.a (mon.0) 764 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: cluster 2024-09-13T14:37:16.053095+0000 mon.a (mon.0) 764 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: cluster 2024-09-13T14:37:16.053150+0000 mon.a (mon.0) 765 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: cluster 2024-09-13T14:37:16.053150+0000 mon.a (mon.0) 765 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: cluster 2024-09-13T14:37:16.074833+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-13T14:37:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: cluster 2024-09-13T14:37:16.074833+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-13T14:37:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: audit 2024-09-13T14:37:16.076462+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:37:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:16 smithi099 bash[87147]: audit 2024-09-13T14:37:16.076462+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:37:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:37:17] "GET /metrics HTTP/1.1" 200 55823 "" "Prometheus/2.51.0" 2024-09-13T14:37:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[84576]: cluster 2024-09-13T14:37:16.059014+0000 mgr.y (mgr.26252) 238 : cluster [DBG] pgmap v453: 425 pgs: 38 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-13T14:37:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[84576]: cluster 2024-09-13T14:37:16.059014+0000 mgr.y (mgr.26252) 238 : cluster [DBG] pgmap v453: 425 pgs: 38 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-13T14:37:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[84576]: audit 2024-09-13T14:37:16.078251+0000 mgr.y (mgr.26252) 239 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:37:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[84576]: audit 2024-09-13T14:37:16.078251+0000 mgr.y (mgr.26252) 239 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:37:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[84576]: cluster 2024-09-13T14:37:17.093537+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-13T14:37:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[84576]: cluster 2024-09-13T14:37:17.093537+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-13T14:37:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[84576]: audit 2024-09-13T14:37:17.139568+0000 mon.a (mon.0) 769 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:37:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[84576]: audit 2024-09-13T14:37:17.139568+0000 mon.a (mon.0) 769 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:37:18.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[87147]: cluster 2024-09-13T14:37:16.059014+0000 mgr.y (mgr.26252) 238 : cluster [DBG] pgmap v453: 425 pgs: 38 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-13T14:37:18.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[87147]: cluster 2024-09-13T14:37:16.059014+0000 mgr.y (mgr.26252) 238 : cluster [DBG] pgmap v453: 425 pgs: 38 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-13T14:37:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[87147]: audit 2024-09-13T14:37:16.078251+0000 mgr.y (mgr.26252) 239 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:37:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[87147]: audit 2024-09-13T14:37:16.078251+0000 mgr.y (mgr.26252) 239 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:37:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[87147]: cluster 2024-09-13T14:37:17.093537+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-13T14:37:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[87147]: cluster 2024-09-13T14:37:17.093537+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-13T14:37:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[87147]: audit 2024-09-13T14:37:17.139568+0000 mon.a (mon.0) 769 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:37:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:17 smithi099 bash[87147]: audit 2024-09-13T14:37:17.139568+0000 mon.a (mon.0) 769 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:37:18.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:17 smithi177 bash[68594]: cluster 2024-09-13T14:37:16.059014+0000 mgr.y (mgr.26252) 238 : cluster [DBG] pgmap v453: 425 pgs: 38 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-13T14:37:18.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:17 smithi177 bash[68594]: cluster 2024-09-13T14:37:16.059014+0000 mgr.y (mgr.26252) 238 : cluster [DBG] pgmap v453: 425 pgs: 38 unknown, 387 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-13T14:37:18.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:17 smithi177 bash[68594]: audit 2024-09-13T14:37:16.078251+0000 mgr.y (mgr.26252) 239 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:37:18.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:17 smithi177 bash[68594]: audit 2024-09-13T14:37:16.078251+0000 mgr.y (mgr.26252) 239 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:37:18.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:17 smithi177 bash[68594]: cluster 2024-09-13T14:37:17.093537+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-13T14:37:18.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:17 smithi177 bash[68594]: cluster 2024-09-13T14:37:17.093537+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-13T14:37:18.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:17 smithi177 bash[68594]: audit 2024-09-13T14:37:17.139568+0000 mon.a (mon.0) 769 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:37:18.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:17 smithi177 bash[68594]: audit 2024-09-13T14:37:17.139568+0000 mon.a (mon.0) 769 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-13T14:37:19.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:19 smithi177 bash[68594]: cluster 2024-09-13T14:37:18.060419+0000 mgr.y (mgr.26252) 240 : cluster [DBG] pgmap v456: 425 pgs: 27 unknown, 398 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 1 op/s 2024-09-13T14:37:19.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:19 smithi177 bash[68594]: cluster 2024-09-13T14:37:18.060419+0000 mgr.y (mgr.26252) 240 : cluster [DBG] pgmap v456: 425 pgs: 27 unknown, 398 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 1 op/s 2024-09-13T14:37:19.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:19 smithi177 bash[68594]: audit 2024-09-13T14:37:18.086822+0000 mon.a (mon.0) 770 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:37:19.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:19 smithi177 bash[68594]: audit 2024-09-13T14:37:18.086822+0000 mon.a (mon.0) 770 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:37:19.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:19 smithi177 bash[68594]: cluster 2024-09-13T14:37:18.094544+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-13T14:37:19.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:19 smithi177 bash[68594]: cluster 2024-09-13T14:37:18.094544+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-13T14:37:19.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:19 smithi177 bash[68594]: audit 2024-09-13T14:37:18.248604+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:19.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:19 smithi177 bash[68594]: audit 2024-09-13T14:37:18.248604+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[84576]: cluster 2024-09-13T14:37:18.060419+0000 mgr.y (mgr.26252) 240 : cluster [DBG] pgmap v456: 425 pgs: 27 unknown, 398 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 1 op/s 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[84576]: cluster 2024-09-13T14:37:18.060419+0000 mgr.y (mgr.26252) 240 : cluster [DBG] pgmap v456: 425 pgs: 27 unknown, 398 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 1 op/s 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[84576]: audit 2024-09-13T14:37:18.086822+0000 mon.a (mon.0) 770 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[84576]: audit 2024-09-13T14:37:18.086822+0000 mon.a (mon.0) 770 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[84576]: cluster 2024-09-13T14:37:18.094544+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[84576]: cluster 2024-09-13T14:37:18.094544+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[84576]: audit 2024-09-13T14:37:18.248604+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[84576]: audit 2024-09-13T14:37:18.248604+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:19.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[87147]: cluster 2024-09-13T14:37:18.060419+0000 mgr.y (mgr.26252) 240 : cluster [DBG] pgmap v456: 425 pgs: 27 unknown, 398 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 1 op/s 2024-09-13T14:37:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[87147]: cluster 2024-09-13T14:37:18.060419+0000 mgr.y (mgr.26252) 240 : cluster [DBG] pgmap v456: 425 pgs: 27 unknown, 398 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 1 op/s 2024-09-13T14:37:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[87147]: audit 2024-09-13T14:37:18.086822+0000 mon.a (mon.0) 770 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:37:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[87147]: audit 2024-09-13T14:37:18.086822+0000 mon.a (mon.0) 770 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi099-69110-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T14:37:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[87147]: cluster 2024-09-13T14:37:18.094544+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-13T14:37:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[87147]: cluster 2024-09-13T14:37:18.094544+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-13T14:37:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[87147]: audit 2024-09-13T14:37:18.248604+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:19 smithi099 bash[87147]: audit 2024-09-13T14:37:18.248604+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:20.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:37:20 smithi177 bash[61439]: ts=2024-09-13T14:37:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:21.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:21 smithi177 bash[68594]: cluster 2024-09-13T14:37:20.063874+0000 mgr.y (mgr.26252) 241 : cluster [DBG] pgmap v458: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 229 KiB/s wr, 6 op/s 2024-09-13T14:37:21.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:21 smithi177 bash[68594]: cluster 2024-09-13T14:37:20.063874+0000 mgr.y (mgr.26252) 241 : cluster [DBG] pgmap v458: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 229 KiB/s wr, 6 op/s 2024-09-13T14:37:21.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:21 smithi177 bash[68594]: cluster 2024-09-13T14:37:21.069779+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-13T14:37:21.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:21 smithi177 bash[68594]: cluster 2024-09-13T14:37:21.069779+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-13T14:37:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:21 smithi099 bash[84576]: cluster 2024-09-13T14:37:20.063874+0000 mgr.y (mgr.26252) 241 : cluster [DBG] pgmap v458: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 229 KiB/s wr, 6 op/s 2024-09-13T14:37:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:21 smithi099 bash[84576]: cluster 2024-09-13T14:37:20.063874+0000 mgr.y (mgr.26252) 241 : cluster [DBG] pgmap v458: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 229 KiB/s wr, 6 op/s 2024-09-13T14:37:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:21 smithi099 bash[84576]: cluster 2024-09-13T14:37:21.069779+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-13T14:37:21.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:21 smithi099 bash[84576]: cluster 2024-09-13T14:37:21.069779+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-13T14:37:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:21 smithi099 bash[87147]: cluster 2024-09-13T14:37:20.063874+0000 mgr.y (mgr.26252) 241 : cluster [DBG] pgmap v458: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 229 KiB/s wr, 6 op/s 2024-09-13T14:37:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:21 smithi099 bash[87147]: cluster 2024-09-13T14:37:20.063874+0000 mgr.y (mgr.26252) 241 : cluster [DBG] pgmap v458: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 229 KiB/s wr, 6 op/s 2024-09-13T14:37:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:21 smithi099 bash[87147]: cluster 2024-09-13T14:37:21.069779+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-13T14:37:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:21 smithi099 bash[87147]: cluster 2024-09-13T14:37:21.069779+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-13T14:37:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[84576]: audit 2024-09-13T14:37:21.419844+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[84576]: audit 2024-09-13T14:37:21.419844+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[84576]: audit 2024-09-13T14:37:21.425580+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[84576]: audit 2024-09-13T14:37:21.425580+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[84576]: audit 2024-09-13T14:37:22.019328+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[84576]: audit 2024-09-13T14:37:22.019328+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[84576]: audit 2024-09-13T14:37:22.020282+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[84576]: audit 2024-09-13T14:37:22.020282+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[84576]: audit 2024-09-13T14:37:22.025686+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[84576]: audit 2024-09-13T14:37:22.025686+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[87147]: audit 2024-09-13T14:37:21.419844+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[87147]: audit 2024-09-13T14:37:21.419844+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[87147]: audit 2024-09-13T14:37:21.425580+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[87147]: audit 2024-09-13T14:37:21.425580+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[87147]: audit 2024-09-13T14:37:22.019328+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[87147]: audit 2024-09-13T14:37:22.019328+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[87147]: audit 2024-09-13T14:37:22.020282+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[87147]: audit 2024-09-13T14:37:22.020282+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[87147]: audit 2024-09-13T14:37:22.025686+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:22 smithi099 bash[87147]: audit 2024-09-13T14:37:22.025686+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:22 smithi177 bash[68594]: audit 2024-09-13T14:37:21.419844+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:22 smithi177 bash[68594]: audit 2024-09-13T14:37:21.419844+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:22 smithi177 bash[68594]: audit 2024-09-13T14:37:21.425580+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:22 smithi177 bash[68594]: audit 2024-09-13T14:37:21.425580+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:22 smithi177 bash[68594]: audit 2024-09-13T14:37:22.019328+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:22.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:22 smithi177 bash[68594]: audit 2024-09-13T14:37:22.019328+0000 mon.a (mon.0) 776 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:22.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:22 smithi177 bash[68594]: audit 2024-09-13T14:37:22.020282+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:22.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:22 smithi177 bash[68594]: audit 2024-09-13T14:37:22.020282+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:22.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:22 smithi177 bash[68594]: audit 2024-09-13T14:37:22.025686+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:22.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:22 smithi177 bash[68594]: audit 2024-09-13T14:37:22.025686+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:23 smithi099 bash[84576]: cluster 2024-09-13T14:37:22.064798+0000 mgr.y (mgr.26252) 242 : cluster [DBG] pgmap v460: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s rd, 172 KiB/s wr, 5 op/s 2024-09-13T14:37:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:23 smithi099 bash[84576]: cluster 2024-09-13T14:37:22.064798+0000 mgr.y (mgr.26252) 242 : cluster [DBG] pgmap v460: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s rd, 172 KiB/s wr, 5 op/s 2024-09-13T14:37:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:23 smithi099 bash[87147]: cluster 2024-09-13T14:37:22.064798+0000 mgr.y (mgr.26252) 242 : cluster [DBG] pgmap v460: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s rd, 172 KiB/s wr, 5 op/s 2024-09-13T14:37:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:23 smithi099 bash[87147]: cluster 2024-09-13T14:37:22.064798+0000 mgr.y (mgr.26252) 242 : cluster [DBG] pgmap v460: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s rd, 172 KiB/s wr, 5 op/s 2024-09-13T14:37:23.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:23 smithi177 bash[68594]: cluster 2024-09-13T14:37:22.064798+0000 mgr.y (mgr.26252) 242 : cluster [DBG] pgmap v460: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s rd, 172 KiB/s wr, 5 op/s 2024-09-13T14:37:23.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:23 smithi177 bash[68594]: cluster 2024-09-13T14:37:22.064798+0000 mgr.y (mgr.26252) 242 : cluster [DBG] pgmap v460: 425 pgs: 425 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s rd, 172 KiB/s wr, 5 op/s 2024-09-13T14:37:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:25 smithi099 bash[84576]: cluster 2024-09-13T14:37:24.068525+0000 mgr.y (mgr.26252) 243 : cluster [DBG] pgmap v461: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 735 KiB/s wr, 10 op/s 2024-09-13T14:37:25.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:25 smithi099 bash[84576]: cluster 2024-09-13T14:37:24.068525+0000 mgr.y (mgr.26252) 243 : cluster [DBG] pgmap v461: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 735 KiB/s wr, 10 op/s 2024-09-13T14:37:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:25 smithi099 bash[87147]: cluster 2024-09-13T14:37:24.068525+0000 mgr.y (mgr.26252) 243 : cluster [DBG] pgmap v461: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 735 KiB/s wr, 10 op/s 2024-09-13T14:37:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:25 smithi099 bash[87147]: cluster 2024-09-13T14:37:24.068525+0000 mgr.y (mgr.26252) 243 : cluster [DBG] pgmap v461: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 735 KiB/s wr, 10 op/s 2024-09-13T14:37:25.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:25 smithi177 bash[68594]: cluster 2024-09-13T14:37:24.068525+0000 mgr.y (mgr.26252) 243 : cluster [DBG] pgmap v461: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 735 KiB/s wr, 10 op/s 2024-09-13T14:37:25.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:25 smithi177 bash[68594]: cluster 2024-09-13T14:37:24.068525+0000 mgr.y (mgr.26252) 243 : cluster [DBG] pgmap v461: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 735 KiB/s wr, 10 op/s 2024-09-13T14:37:27.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:37:27] "GET /metrics HTTP/1.1" 200 60582 "" "Prometheus/2.51.0" 2024-09-13T14:37:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:27 smithi099 bash[84576]: cluster 2024-09-13T14:37:26.069446+0000 mgr.y (mgr.26252) 244 : cluster [DBG] pgmap v462: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 640 KiB/s wr, 7 op/s 2024-09-13T14:37:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:27 smithi099 bash[84576]: cluster 2024-09-13T14:37:26.069446+0000 mgr.y (mgr.26252) 244 : cluster [DBG] pgmap v462: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 640 KiB/s wr, 7 op/s 2024-09-13T14:37:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:27 smithi099 bash[87147]: cluster 2024-09-13T14:37:26.069446+0000 mgr.y (mgr.26252) 244 : cluster [DBG] pgmap v462: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 640 KiB/s wr, 7 op/s 2024-09-13T14:37:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:27 smithi099 bash[87147]: cluster 2024-09-13T14:37:26.069446+0000 mgr.y (mgr.26252) 244 : cluster [DBG] pgmap v462: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 640 KiB/s wr, 7 op/s 2024-09-13T14:37:27.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:27 smithi177 bash[68594]: cluster 2024-09-13T14:37:26.069446+0000 mgr.y (mgr.26252) 244 : cluster [DBG] pgmap v462: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 640 KiB/s wr, 7 op/s 2024-09-13T14:37:27.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:27 smithi177 bash[68594]: cluster 2024-09-13T14:37:26.069446+0000 mgr.y (mgr.26252) 244 : cluster [DBG] pgmap v462: 425 pgs: 425 active+clean; 5.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 640 KiB/s wr, 7 op/s 2024-09-13T14:37:29.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:29 smithi099 bash[87147]: cluster 2024-09-13T14:37:28.070863+0000 mgr.y (mgr.26252) 245 : cluster [DBG] pgmap v463: 425 pgs: 425 active+clean; 8.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 822 KiB/s wr, 6 op/s 2024-09-13T14:37:29.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:29 smithi099 bash[87147]: cluster 2024-09-13T14:37:28.070863+0000 mgr.y (mgr.26252) 245 : cluster [DBG] pgmap v463: 425 pgs: 425 active+clean; 8.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 822 KiB/s wr, 6 op/s 2024-09-13T14:37:29.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:29 smithi099 bash[87147]: audit 2024-09-13T14:37:28.537779+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:29.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:29 smithi099 bash[87147]: audit 2024-09-13T14:37:28.537779+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:29 smithi099 bash[84576]: cluster 2024-09-13T14:37:28.070863+0000 mgr.y (mgr.26252) 245 : cluster [DBG] pgmap v463: 425 pgs: 425 active+clean; 8.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 822 KiB/s wr, 6 op/s 2024-09-13T14:37:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:29 smithi099 bash[84576]: cluster 2024-09-13T14:37:28.070863+0000 mgr.y (mgr.26252) 245 : cluster [DBG] pgmap v463: 425 pgs: 425 active+clean; 8.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 822 KiB/s wr, 6 op/s 2024-09-13T14:37:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:29 smithi099 bash[84576]: audit 2024-09-13T14:37:28.537779+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:29 smithi099 bash[84576]: audit 2024-09-13T14:37:28.537779+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:29.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:29 smithi177 bash[68594]: cluster 2024-09-13T14:37:28.070863+0000 mgr.y (mgr.26252) 245 : cluster [DBG] pgmap v463: 425 pgs: 425 active+clean; 8.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 822 KiB/s wr, 6 op/s 2024-09-13T14:37:29.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:29 smithi177 bash[68594]: cluster 2024-09-13T14:37:28.070863+0000 mgr.y (mgr.26252) 245 : cluster [DBG] pgmap v463: 425 pgs: 425 active+clean; 8.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 822 KiB/s wr, 6 op/s 2024-09-13T14:37:29.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:29 smithi177 bash[68594]: audit 2024-09-13T14:37:28.537779+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:29.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:29 smithi177 bash[68594]: audit 2024-09-13T14:37:28.537779+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:30.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade resume' 2024-09-13T14:37:30.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:37:30 smithi177 bash[61439]: ts=2024-09-13T14:37:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:31.426 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.RemoveFullTryDataPool (19908 ms) 2024-09-13T14:37:31.427 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestIO 2024-09-13T14:37:31.427 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:37:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[84576]: cluster 2024-09-13T14:37:30.074320+0000 mgr.y (mgr.26252) 246 : cluster [DBG] pgmap v464: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[84576]: cluster 2024-09-13T14:37:30.074320+0000 mgr.y (mgr.26252) 246 : cluster [DBG] pgmap v464: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[84576]: cluster 2024-09-13T14:37:31.066079+0000 mon.a (mon.0) 780 : cluster [WRN] pool 'test-librbd-smithi099-69110-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[84576]: cluster 2024-09-13T14:37:31.066079+0000 mon.a (mon.0) 780 : cluster [WRN] pool 'test-librbd-smithi099-69110-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[84576]: cluster 2024-09-13T14:37:31.066641+0000 mon.a (mon.0) 781 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[84576]: cluster 2024-09-13T14:37:31.066641+0000 mon.a (mon.0) 781 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[84576]: cluster 2024-09-13T14:37:31.076699+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-13T14:37:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[84576]: cluster 2024-09-13T14:37:31.076699+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-13T14:37:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[87147]: cluster 2024-09-13T14:37:30.074320+0000 mgr.y (mgr.26252) 246 : cluster [DBG] pgmap v464: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[87147]: cluster 2024-09-13T14:37:30.074320+0000 mgr.y (mgr.26252) 246 : cluster [DBG] pgmap v464: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[87147]: cluster 2024-09-13T14:37:31.066079+0000 mon.a (mon.0) 780 : cluster [WRN] pool 'test-librbd-smithi099-69110-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[87147]: cluster 2024-09-13T14:37:31.066079+0000 mon.a (mon.0) 780 : cluster [WRN] pool 'test-librbd-smithi099-69110-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[87147]: cluster 2024-09-13T14:37:31.066641+0000 mon.a (mon.0) 781 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[87147]: cluster 2024-09-13T14:37:31.066641+0000 mon.a (mon.0) 781 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[87147]: cluster 2024-09-13T14:37:31.076699+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-13T14:37:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:31 smithi099 bash[87147]: cluster 2024-09-13T14:37:31.076699+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-13T14:37:31.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:31 smithi177 bash[68594]: cluster 2024-09-13T14:37:30.074320+0000 mgr.y (mgr.26252) 246 : cluster [DBG] pgmap v464: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:31.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:31 smithi177 bash[68594]: cluster 2024-09-13T14:37:30.074320+0000 mgr.y (mgr.26252) 246 : cluster [DBG] pgmap v464: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:31.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:31 smithi177 bash[68594]: cluster 2024-09-13T14:37:31.066079+0000 mon.a (mon.0) 780 : cluster [WRN] pool 'test-librbd-smithi099-69110-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:31.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:31 smithi177 bash[68594]: cluster 2024-09-13T14:37:31.066079+0000 mon.a (mon.0) 780 : cluster [WRN] pool 'test-librbd-smithi099-69110-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T14:37:31.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:31 smithi177 bash[68594]: cluster 2024-09-13T14:37:31.066641+0000 mon.a (mon.0) 781 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:31.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:31 smithi177 bash[68594]: cluster 2024-09-13T14:37:31.066641+0000 mon.a (mon.0) 781 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T14:37:31.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:31 smithi177 bash[68594]: cluster 2024-09-13T14:37:31.076699+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-13T14:37:31.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:31 smithi177 bash[68594]: cluster 2024-09-13T14:37:31.076699+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[84576]: cluster 2024-09-13T14:37:32.075263+0000 mgr.y (mgr.26252) 247 : cluster [DBG] pgmap v466: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[84576]: cluster 2024-09-13T14:37:32.075263+0000 mgr.y (mgr.26252) 247 : cluster [DBG] pgmap v466: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[84576]: audit 2024-09-13T14:37:33.255441+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[84576]: audit 2024-09-13T14:37:33.255441+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[84576]: audit 2024-09-13T14:37:33.257846+0000 mon.a (mon.0) 784 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[84576]: audit 2024-09-13T14:37:33.257846+0000 mon.a (mon.0) 784 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:33.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[87147]: cluster 2024-09-13T14:37:32.075263+0000 mgr.y (mgr.26252) 247 : cluster [DBG] pgmap v466: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[87147]: cluster 2024-09-13T14:37:32.075263+0000 mgr.y (mgr.26252) 247 : cluster [DBG] pgmap v466: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[87147]: audit 2024-09-13T14:37:33.255441+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[87147]: audit 2024-09-13T14:37:33.255441+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[87147]: audit 2024-09-13T14:37:33.257846+0000 mon.a (mon.0) 784 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:33 smithi099 bash[87147]: audit 2024-09-13T14:37:33.257846+0000 mon.a (mon.0) 784 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:33.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:33 smithi177 bash[68594]: cluster 2024-09-13T14:37:32.075263+0000 mgr.y (mgr.26252) 247 : cluster [DBG] pgmap v466: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:33.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:33 smithi177 bash[68594]: cluster 2024-09-13T14:37:32.075263+0000 mgr.y (mgr.26252) 247 : cluster [DBG] pgmap v466: 425 pgs: 425 active+clean; 12 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 KiB/s wr, 5 op/s 2024-09-13T14:37:33.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:33 smithi177 bash[68594]: audit 2024-09-13T14:37:33.255441+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:33.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:33 smithi177 bash[68594]: audit 2024-09-13T14:37:33.255441+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:33.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:33 smithi177 bash[68594]: audit 2024-09-13T14:37:33.257846+0000 mon.a (mon.0) 784 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:33.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:33 smithi177 bash[68594]: audit 2024-09-13T14:37:33.257846+0000 mon.a (mon.0) 784 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:34.851 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:35 smithi099 bash[84576]: cluster 2024-09-13T14:37:34.079089+0000 mgr.y (mgr.26252) 248 : cluster [DBG] pgmap v467: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 886 KiB/s wr, 244 op/s 2024-09-13T14:37:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:35 smithi099 bash[84576]: cluster 2024-09-13T14:37:34.079089+0000 mgr.y (mgr.26252) 248 : cluster [DBG] pgmap v467: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 886 KiB/s wr, 244 op/s 2024-09-13T14:37:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:35 smithi099 bash[87147]: cluster 2024-09-13T14:37:34.079089+0000 mgr.y (mgr.26252) 248 : cluster [DBG] pgmap v467: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 886 KiB/s wr, 244 op/s 2024-09-13T14:37:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:35 smithi099 bash[87147]: cluster 2024-09-13T14:37:34.079089+0000 mgr.y (mgr.26252) 248 : cluster [DBG] pgmap v467: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 886 KiB/s wr, 244 op/s 2024-09-13T14:37:35.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:35 smithi177 bash[68594]: cluster 2024-09-13T14:37:34.079089+0000 mgr.y (mgr.26252) 248 : cluster [DBG] pgmap v467: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 886 KiB/s wr, 244 op/s 2024-09-13T14:37:35.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:35 smithi177 bash[68594]: cluster 2024-09-13T14:37:34.079089+0000 mgr.y (mgr.26252) 248 : cluster [DBG] pgmap v467: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 886 KiB/s wr, 244 op/s 2024-09-13T14:37:36.463 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:36.502 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:36.502 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:36.502 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:36.502 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:36.502 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.502 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:37:36.502 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.502 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.502 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.503 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:37:36.504 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.505 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:37:36.506 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:37:36.507 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:37:36.508 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 512~512 = 512 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:aio discard: 1536~512 = 0 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.509 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.510 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.511 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:37:36.512 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:37:36.513 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.514 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.515 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.516 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.517 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:36.518 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:36.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[84576]: cluster 2024-09-13T14:37:36.068335+0000 mon.a (mon.0) 785 : cluster [INF] pool 'test-librbd-smithi099-69110-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:36.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[84576]: cluster 2024-09-13T14:37:36.068335+0000 mon.a (mon.0) 785 : cluster [INF] pool 'test-librbd-smithi099-69110-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[84576]: cluster 2024-09-13T14:37:36.068924+0000 mon.a (mon.0) 786 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[84576]: cluster 2024-09-13T14:37:36.068924+0000 mon.a (mon.0) 786 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[84576]: cluster 2024-09-13T14:37:36.068964+0000 mon.a (mon.0) 787 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[84576]: cluster 2024-09-13T14:37:36.068964+0000 mon.a (mon.0) 787 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[84576]: cluster 2024-09-13T14:37:36.083342+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[84576]: cluster 2024-09-13T14:37:36.083342+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[87147]: cluster 2024-09-13T14:37:36.068335+0000 mon.a (mon.0) 785 : cluster [INF] pool 'test-librbd-smithi099-69110-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[87147]: cluster 2024-09-13T14:37:36.068335+0000 mon.a (mon.0) 785 : cluster [INF] pool 'test-librbd-smithi099-69110-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[87147]: cluster 2024-09-13T14:37:36.068924+0000 mon.a (mon.0) 786 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[87147]: cluster 2024-09-13T14:37:36.068924+0000 mon.a (mon.0) 786 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[87147]: cluster 2024-09-13T14:37:36.068964+0000 mon.a (mon.0) 787 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[87147]: cluster 2024-09-13T14:37:36.068964+0000 mon.a (mon.0) 787 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:36.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[87147]: cluster 2024-09-13T14:37:36.083342+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-13T14:37:36.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:36 smithi099 bash[87147]: cluster 2024-09-13T14:37:36.083342+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-13T14:37:36.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:36 smithi177 bash[68594]: cluster 2024-09-13T14:37:36.068335+0000 mon.a (mon.0) 785 : cluster [INF] pool 'test-librbd-smithi099-69110-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:36.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:36 smithi177 bash[68594]: cluster 2024-09-13T14:37:36.068335+0000 mon.a (mon.0) 785 : cluster [INF] pool 'test-librbd-smithi099-69110-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T14:37:36.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:36 smithi177 bash[68594]: cluster 2024-09-13T14:37:36.068924+0000 mon.a (mon.0) 786 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:36.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:36 smithi177 bash[68594]: cluster 2024-09-13T14:37:36.068924+0000 mon.a (mon.0) 786 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T14:37:36.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:36 smithi177 bash[68594]: cluster 2024-09-13T14:37:36.068964+0000 mon.a (mon.0) 787 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:36.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:36 smithi177 bash[68594]: cluster 2024-09-13T14:37:36.068964+0000 mon.a (mon.0) 787 : cluster [INF] Cluster is now healthy 2024-09-13T14:37:36.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:36 smithi177 bash[68594]: cluster 2024-09-13T14:37:36.083342+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-13T14:37:36.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:36 smithi177 bash[68594]: cluster 2024-09-13T14:37:36.083342+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-13T14:37:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:37:37] "GET /metrics HTTP/1.1" 200 60700 "" "Prometheus/2.51.0" 2024-09-13T14:37:37.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:37 smithi099 bash[87147]: cluster 2024-09-13T14:37:36.080285+0000 mgr.y (mgr.26252) 249 : cluster [DBG] pgmap v469: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 724 KiB/s wr, 305 op/s 2024-09-13T14:37:37.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:37 smithi099 bash[87147]: cluster 2024-09-13T14:37:36.080285+0000 mgr.y (mgr.26252) 249 : cluster [DBG] pgmap v469: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 724 KiB/s wr, 305 op/s 2024-09-13T14:37:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:37 smithi099 bash[84576]: cluster 2024-09-13T14:37:36.080285+0000 mgr.y (mgr.26252) 249 : cluster [DBG] pgmap v469: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 724 KiB/s wr, 305 op/s 2024-09-13T14:37:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:37 smithi099 bash[84576]: cluster 2024-09-13T14:37:36.080285+0000 mgr.y (mgr.26252) 249 : cluster [DBG] pgmap v469: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 724 KiB/s wr, 305 op/s 2024-09-13T14:37:37.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:37 smithi177 bash[68594]: cluster 2024-09-13T14:37:36.080285+0000 mgr.y (mgr.26252) 249 : cluster [DBG] pgmap v469: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 724 KiB/s wr, 305 op/s 2024-09-13T14:37:37.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:37 smithi177 bash[68594]: cluster 2024-09-13T14:37:36.080285+0000 mgr.y (mgr.26252) 249 : cluster [DBG] pgmap v469: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 724 KiB/s wr, 305 op/s 2024-09-13T14:37:39.577 INFO:teuthology.orchestra.run.smithi099.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:39 smithi099 bash[84576]: cluster 2024-09-13T14:37:38.083740+0000 mgr.y (mgr.26252) 250 : cluster [DBG] pgmap v470: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 340 KiB/s wr, 303 op/s 2024-09-13T14:37:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:39 smithi099 bash[84576]: cluster 2024-09-13T14:37:38.083740+0000 mgr.y (mgr.26252) 250 : cluster [DBG] pgmap v470: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 340 KiB/s wr, 303 op/s 2024-09-13T14:37:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:39 smithi099 bash[87147]: cluster 2024-09-13T14:37:38.083740+0000 mgr.y (mgr.26252) 250 : cluster [DBG] pgmap v470: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 340 KiB/s wr, 303 op/s 2024-09-13T14:37:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:39 smithi099 bash[87147]: cluster 2024-09-13T14:37:38.083740+0000 mgr.y (mgr.26252) 250 : cluster [DBG] pgmap v470: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 340 KiB/s wr, 303 op/s 2024-09-13T14:37:39.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:39 smithi177 bash[68594]: cluster 2024-09-13T14:37:38.083740+0000 mgr.y (mgr.26252) 250 : cluster [DBG] pgmap v470: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 340 KiB/s wr, 303 op/s 2024-09-13T14:37:39.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:39 smithi177 bash[68594]: cluster 2024-09-13T14:37:38.083740+0000 mgr.y (mgr.26252) 250 : cluster [DBG] pgmap v470: 425 pgs: 425 active+clean; 745 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 340 KiB/s wr, 303 op/s 2024-09-13T14:37:39.859 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.859 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.860 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.861 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.862 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.863 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.864 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:39.865 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.866 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.867 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestIO (8433 ms) 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS1 2024-09-13T14:37:39.868 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:37:40.578 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[61439]: ts=2024-09-13T14:37:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:40.801 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for half of osds to upgrade' 2024-09-13T14:37:40.802 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.573182+0000 mgr.y (mgr.26252) 251 : audit [DBG] from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:40.802 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.573182+0000 mgr.y (mgr.26252) 251 : audit [DBG] from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:40.802 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: cephadm 2024-09-13T14:37:39.573938+0000 mgr.y (mgr.26252) 252 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: cephadm 2024-09-13T14:37:39.573938+0000 mgr.y (mgr.26252) 252 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.579820+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.579820+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.580809+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.580809+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.583184+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.583184+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.584266+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.584266+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.592796+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.592796+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.680538+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.680538+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.682679+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.682679+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.684344+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.684344+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.685494+0000 mon.a (mon.0) 797 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.685494+0000 mon.a (mon.0) 797 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.686917+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.686917+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:37:40.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.687047+0000 mgr.y (mgr.26252) 253 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:37:40.805 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: audit 2024-09-13T14:37:39.687047+0000 mgr.y (mgr.26252) 253 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:37:40.805 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: cephadm 2024-09-13T14:37:39.688590+0000 mgr.y (mgr.26252) 254 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-13T14:37:40.805 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[87147]: cephadm 2024-09-13T14:37:39.688590+0000 mgr.y (mgr.26252) 254 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-13T14:37:40.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.573182+0000 mgr.y (mgr.26252) 251 : audit [DBG] from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:40.805 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.573182+0000 mgr.y (mgr.26252) 251 : audit [DBG] from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: cephadm 2024-09-13T14:37:39.573938+0000 mgr.y (mgr.26252) 252 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: cephadm 2024-09-13T14:37:39.573938+0000 mgr.y (mgr.26252) 252 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.579820+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.579820+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.580809+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.580809+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.583184+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.583184+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.584266+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.584266+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:40.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.592796+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.592796+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.680538+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.680538+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.682679+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.682679+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.684344+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.684344+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.685494+0000 mon.a (mon.0) 797 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.685494+0000 mon.a (mon.0) 797 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.686917+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.686917+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:37:40.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.687047+0000 mgr.y (mgr.26252) 253 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:37:40.808 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: audit 2024-09-13T14:37:39.687047+0000 mgr.y (mgr.26252) 253 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:37:40.808 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: cephadm 2024-09-13T14:37:39.688590+0000 mgr.y (mgr.26252) 254 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-13T14:37:40.808 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:40 smithi099 bash[84576]: cephadm 2024-09-13T14:37:39.688590+0000 mgr.y (mgr.26252) 254 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-13T14:37:40.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.573182+0000 mgr.y (mgr.26252) 251 : audit [DBG] from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:40.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.573182+0000 mgr.y (mgr.26252) 251 : audit [DBG] from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:40.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: cephadm 2024-09-13T14:37:39.573938+0000 mgr.y (mgr.26252) 252 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:40.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: cephadm 2024-09-13T14:37:39.573938+0000 mgr.y (mgr.26252) 252 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:37:40.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.579820+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.579820+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.580809+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.580809+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.583184+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.583184+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.584266+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.584266+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.592796+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.592796+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.680538+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.680538+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.682679+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.682679+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.684344+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.684344+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.685494+0000 mon.a (mon.0) 797 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.685494+0000 mon.a (mon.0) 797 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.686917+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.686917+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.687047+0000 mgr.y (mgr.26252) 253 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: audit 2024-09-13T14:37:39.687047+0000 mgr.y (mgr.26252) 253 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:37:40.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: cephadm 2024-09-13T14:37:39.688590+0000 mgr.y (mgr.26252) 254 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-13T14:37:40.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:40 smithi177 bash[68594]: cephadm 2024-09-13T14:37:39.688590+0000 mgr.y (mgr.26252) 254 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-13T14:37:41.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:41 smithi177 bash[68594]: cluster 2024-09-13T14:37:40.085616+0000 mgr.y (mgr.26252) 255 : cluster [DBG] pgmap v471: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 434 KiB/s rd, 439 KiB/s wr, 820 op/s 2024-09-13T14:37:41.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:41 smithi177 bash[68594]: cluster 2024-09-13T14:37:40.085616+0000 mgr.y (mgr.26252) 255 : cluster [DBG] pgmap v471: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 434 KiB/s rd, 439 KiB/s wr, 820 op/s 2024-09-13T14:37:42.012 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:41 smithi099 bash[84576]: cluster 2024-09-13T14:37:40.085616+0000 mgr.y (mgr.26252) 255 : cluster [DBG] pgmap v471: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 434 KiB/s rd, 439 KiB/s wr, 820 op/s 2024-09-13T14:37:42.012 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:41 smithi099 bash[84576]: cluster 2024-09-13T14:37:40.085616+0000 mgr.y (mgr.26252) 255 : cluster [DBG] pgmap v471: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 434 KiB/s rd, 439 KiB/s wr, 820 op/s 2024-09-13T14:37:42.013 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:41 smithi099 bash[87147]: cluster 2024-09-13T14:37:40.085616+0000 mgr.y (mgr.26252) 255 : cluster [DBG] pgmap v471: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 434 KiB/s rd, 439 KiB/s wr, 820 op/s 2024-09-13T14:37:42.013 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:41 smithi099 bash[87147]: cluster 2024-09-13T14:37:40.085616+0000 mgr.y (mgr.26252) 255 : cluster [DBG] pgmap v471: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 434 KiB/s rd, 439 KiB/s wr, 820 op/s 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: cluster 2024-09-13T14:37:42.086494+0000 mgr.y (mgr.26252) 256 : cluster [DBG] pgmap v472: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 395 KiB/s wr, 739 op/s 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: cluster 2024-09-13T14:37:42.086494+0000 mgr.y (mgr.26252) 256 : cluster [DBG] pgmap v472: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 395 KiB/s wr, 739 op/s 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: cephadm 2024-09-13T14:37:42.554281+0000 mgr.y (mgr.26252) 257 : cephadm [INF] Upgrade: Updating osd.1 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: cephadm 2024-09-13T14:37:42.554281+0000 mgr.y (mgr.26252) 257 : cephadm [INF] Upgrade: Updating osd.1 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: audit 2024-09-13T14:37:42.558553+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: audit 2024-09-13T14:37:42.558553+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: audit 2024-09-13T14:37:42.560357+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:37:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: audit 2024-09-13T14:37:42.560357+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: audit 2024-09-13T14:37:42.561072+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: audit 2024-09-13T14:37:42.561072+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: cephadm 2024-09-13T14:37:42.565575+0000 mgr.y (mgr.26252) 258 : cephadm [INF] Deploying daemon osd.1 on smithi099 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[84576]: cephadm 2024-09-13T14:37:42.565575+0000 mgr.y (mgr.26252) 258 : cephadm [INF] Deploying daemon osd.1 on smithi099 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: cluster 2024-09-13T14:37:42.086494+0000 mgr.y (mgr.26252) 256 : cluster [DBG] pgmap v472: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 395 KiB/s wr, 739 op/s 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: cluster 2024-09-13T14:37:42.086494+0000 mgr.y (mgr.26252) 256 : cluster [DBG] pgmap v472: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 395 KiB/s wr, 739 op/s 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: cephadm 2024-09-13T14:37:42.554281+0000 mgr.y (mgr.26252) 257 : cephadm [INF] Upgrade: Updating osd.1 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: cephadm 2024-09-13T14:37:42.554281+0000 mgr.y (mgr.26252) 257 : cephadm [INF] Upgrade: Updating osd.1 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: audit 2024-09-13T14:37:42.558553+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: audit 2024-09-13T14:37:42.558553+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: audit 2024-09-13T14:37:42.560357+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:37:43.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: audit 2024-09-13T14:37:42.560357+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:37:43.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: audit 2024-09-13T14:37:42.561072+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:43.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: audit 2024-09-13T14:37:42.561072+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:43.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: cephadm 2024-09-13T14:37:42.565575+0000 mgr.y (mgr.26252) 258 : cephadm [INF] Deploying daemon osd.1 on smithi099 2024-09-13T14:37:43.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:43 smithi099 bash[87147]: cephadm 2024-09-13T14:37:42.565575+0000 mgr.y (mgr.26252) 258 : cephadm [INF] Deploying daemon osd.1 on smithi099 2024-09-13T14:37:43.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: cluster 2024-09-13T14:37:42.086494+0000 mgr.y (mgr.26252) 256 : cluster [DBG] pgmap v472: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 395 KiB/s wr, 739 op/s 2024-09-13T14:37:43.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: cluster 2024-09-13T14:37:42.086494+0000 mgr.y (mgr.26252) 256 : cluster [DBG] pgmap v472: 425 pgs: 425 active+clean; 782 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 391 KiB/s rd, 395 KiB/s wr, 739 op/s 2024-09-13T14:37:43.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: cephadm 2024-09-13T14:37:42.554281+0000 mgr.y (mgr.26252) 257 : cephadm [INF] Upgrade: Updating osd.1 2024-09-13T14:37:43.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: cephadm 2024-09-13T14:37:42.554281+0000 mgr.y (mgr.26252) 257 : cephadm [INF] Upgrade: Updating osd.1 2024-09-13T14:37:43.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: audit 2024-09-13T14:37:42.558553+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:43.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: audit 2024-09-13T14:37:42.558553+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:43.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: audit 2024-09-13T14:37:42.560357+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:37:43.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: audit 2024-09-13T14:37:42.560357+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:37:43.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: audit 2024-09-13T14:37:42.561072+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:43.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: audit 2024-09-13T14:37:42.561072+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:43.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: cephadm 2024-09-13T14:37:42.565575+0000 mgr.y (mgr.26252) 258 : cephadm [INF] Deploying daemon osd.1 on smithi099 2024-09-13T14:37:43.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:43 smithi177 bash[68594]: cephadm 2024-09-13T14:37:42.565575+0000 mgr.y (mgr.26252) 258 : cephadm [INF] Deploying daemon osd.1 on smithi099 2024-09-13T14:37:45.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:45 smithi099 bash[84576]: cluster 2024-09-13T14:37:44.088099+0000 mgr.y (mgr.26252) 259 : cluster [DBG] pgmap v473: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:45.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:45 smithi099 bash[84576]: cluster 2024-09-13T14:37:44.088099+0000 mgr.y (mgr.26252) 259 : cluster [DBG] pgmap v473: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:45.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:45 smithi099 bash[87147]: cluster 2024-09-13T14:37:44.088099+0000 mgr.y (mgr.26252) 259 : cluster [DBG] pgmap v473: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:45.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:45 smithi099 bash[87147]: cluster 2024-09-13T14:37:44.088099+0000 mgr.y (mgr.26252) 259 : cluster [DBG] pgmap v473: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:45 smithi177 bash[68594]: cluster 2024-09-13T14:37:44.088099+0000 mgr.y (mgr.26252) 259 : cluster [DBG] pgmap v473: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:45.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:45 smithi177 bash[68594]: cluster 2024-09-13T14:37:44.088099+0000 mgr.y (mgr.26252) 259 : cluster [DBG] pgmap v473: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:46.081 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:37:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:37:47] "GET /metrics HTTP/1.1" 200 60700 "" "Prometheus/2.51.0" 2024-09-13T14:37:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:47 smithi099 bash[84576]: cluster 2024-09-13T14:37:46.088728+0000 mgr.y (mgr.26252) 260 : cluster [DBG] pgmap v474: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:47 smithi099 bash[84576]: cluster 2024-09-13T14:37:46.088728+0000 mgr.y (mgr.26252) 260 : cluster [DBG] pgmap v474: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:47 smithi099 bash[87147]: cluster 2024-09-13T14:37:46.088728+0000 mgr.y (mgr.26252) 260 : cluster [DBG] pgmap v474: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:47 smithi099 bash[87147]: cluster 2024-09-13T14:37:46.088728+0000 mgr.y (mgr.26252) 260 : cluster [DBG] pgmap v474: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:47.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:47 smithi177 bash[68594]: cluster 2024-09-13T14:37:46.088728+0000 mgr.y (mgr.26252) 260 : cluster [DBG] pgmap v474: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:47.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:47 smithi177 bash[68594]: cluster 2024-09-13T14:37:46.088728+0000 mgr.y (mgr.26252) 260 : cluster [DBG] pgmap v474: 425 pgs: 425 active+clean; 2.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 299 KiB/s rd, 144 KiB/s wr, 622 op/s 2024-09-13T14:37:47.959 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:47.959 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:47.959 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:47.959 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:47.959 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:47.959 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.959 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.960 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:37:47.961 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.962 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.963 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:47.964 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.965 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:37:47.965 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:47.965 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:37:47.965 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.965 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:37:47.965 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:37:47.966 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 512~512 = 512 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:aio discard: 1536~512 = 0 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.967 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.968 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:37:47.969 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.970 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.971 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:47.972 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.973 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:47.974 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:47.975 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:47.976 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:47.976 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:47.976 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:48.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:48 smithi177 bash[68594]: audit 2024-09-13T14:37:48.214708+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]} : dispatch 2024-09-13T14:37:48.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:48 smithi177 bash[68594]: audit 2024-09-13T14:37:48.214708+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]} : dispatch 2024-09-13T14:37:48.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:48 smithi177 bash[68594]: audit 2024-09-13T14:37:48.215010+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]} : dispatch 2024-09-13T14:37:48.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:48 smithi177 bash[68594]: audit 2024-09-13T14:37:48.215010+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]} : dispatch 2024-09-13T14:37:48.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:48 smithi177 bash[68594]: audit 2024-09-13T14:37:48.215269+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-13T14:37:48.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:48 smithi177 bash[68594]: audit 2024-09-13T14:37:48.215269+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-13T14:37:48.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:48 smithi177 bash[68594]: audit 2024-09-13T14:37:48.255407+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:48.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:48 smithi177 bash[68594]: audit 2024-09-13T14:37:48.255407+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:48.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:48 smithi177 bash[68594]: audit 2024-09-13T14:37:48.256863+0000 mon.a (mon.0) 806 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:48.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:48 smithi177 bash[68594]: audit 2024-09-13T14:37:48.256863+0000 mon.a (mon.0) 806 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:49.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:48 smithi099 bash[84576]: audit 2024-09-13T14:37:48.214708+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]} : dispatch 2024-09-13T14:37:49.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:48 smithi099 bash[84576]: audit 2024-09-13T14:37:48.214708+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]} : dispatch 2024-09-13T14:37:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.215010+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]} : dispatch 2024-09-13T14:37:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.215010+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]} : dispatch 2024-09-13T14:37:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.215269+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-13T14:37:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.215269+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-13T14:37:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.255407+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.255407+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.256863+0000 mon.a (mon.0) 806 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.256863+0000 mon.a (mon.0) 806 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.214708+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]} : dispatch 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.214708+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]} : dispatch 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.215010+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]} : dispatch 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.215010+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]} : dispatch 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.215269+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.215269+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.255407+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.255407+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.256863+0000 mon.a (mon.0) 806 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.256863+0000 mon.a (mon.0) 806 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: cluster 2024-09-13T14:37:48.089528+0000 mgr.y (mgr.26252) 261 : cluster [DBG] pgmap v475: 425 pgs: 425 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 461 KiB/s wr, 519 op/s 2024-09-13T14:37:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: cluster 2024-09-13T14:37:48.089528+0000 mgr.y (mgr.26252) 261 : cluster [DBG] pgmap v475: 425 pgs: 425 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 461 KiB/s wr, 519 op/s 2024-09-13T14:37:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.576615+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]': finished 2024-09-13T14:37:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.576615+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]': finished 2024-09-13T14:37:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.576704+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-09-13T14:37:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.576704+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-09-13T14:37:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.576766+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]}]': finished 2024-09-13T14:37:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: audit 2024-09-13T14:37:48.576766+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]}]': finished 2024-09-13T14:37:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: cluster 2024-09-13T14:37:48.582336+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2024-09-13T14:37:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[84576]: cluster 2024-09-13T14:37:48.582336+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2024-09-13T14:37:49.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: cluster 2024-09-13T14:37:48.089528+0000 mgr.y (mgr.26252) 261 : cluster [DBG] pgmap v475: 425 pgs: 425 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 461 KiB/s wr, 519 op/s 2024-09-13T14:37:49.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: cluster 2024-09-13T14:37:48.089528+0000 mgr.y (mgr.26252) 261 : cluster [DBG] pgmap v475: 425 pgs: 425 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 461 KiB/s wr, 519 op/s 2024-09-13T14:37:49.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.576615+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]': finished 2024-09-13T14:37:49.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.576615+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]': finished 2024-09-13T14:37:49.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.576704+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-09-13T14:37:49.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.576704+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-09-13T14:37:49.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.576766+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]}]': finished 2024-09-13T14:37:49.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: audit 2024-09-13T14:37:48.576766+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]}]': finished 2024-09-13T14:37:49.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: cluster 2024-09-13T14:37:48.582336+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2024-09-13T14:37:49.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:49 smithi099 bash[87147]: cluster 2024-09-13T14:37:48.582336+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2024-09-13T14:37:49.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:49 smithi177 bash[68594]: cluster 2024-09-13T14:37:48.089528+0000 mgr.y (mgr.26252) 261 : cluster [DBG] pgmap v475: 425 pgs: 425 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 461 KiB/s wr, 519 op/s 2024-09-13T14:37:49.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:49 smithi177 bash[68594]: cluster 2024-09-13T14:37:48.089528+0000 mgr.y (mgr.26252) 261 : cluster [DBG] pgmap v475: 425 pgs: 425 active+clean; 6.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 461 KiB/s wr, 519 op/s 2024-09-13T14:37:49.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:49 smithi177 bash[68594]: audit 2024-09-13T14:37:48.576615+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]': finished 2024-09-13T14:37:49.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:49 smithi177 bash[68594]: audit 2024-09-13T14:37:48.576615+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]': finished 2024-09-13T14:37:49.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:49 smithi177 bash[68594]: audit 2024-09-13T14:37:48.576704+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-09-13T14:37:49.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:49 smithi177 bash[68594]: audit 2024-09-13T14:37:48.576704+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-09-13T14:37:49.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:49 smithi177 bash[68594]: audit 2024-09-13T14:37:48.576766+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]}]': finished 2024-09-13T14:37:49.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:49 smithi177 bash[68594]: audit 2024-09-13T14:37:48.576766+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]}]': finished 2024-09-13T14:37:49.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:49 smithi177 bash[68594]: cluster 2024-09-13T14:37:48.582336+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2024-09-13T14:37:49.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:49 smithi177 bash[68594]: cluster 2024-09-13T14:37:48.582336+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2024-09-13T14:37:50.571 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:37:50 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:50.572 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:37:50 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:50.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:50 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:50.572 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:50 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:50.573 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:50 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:50.573 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:37:50 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:50.573 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:37:50 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:50.574 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:37:50 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:50 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:50.594 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:37:50 smithi177 bash[61439]: ts=2024-09-13T14:37:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:37:50.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:50 smithi177 bash[68594]: cluster 2024-09-13T14:37:49.583451+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2024-09-13T14:37:50.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:50 smithi177 bash[68594]: cluster 2024-09-13T14:37:49.583451+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2024-09-13T14:37:51.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:50 smithi099 bash[84576]: cluster 2024-09-13T14:37:49.583451+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2024-09-13T14:37:51.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:50 smithi099 bash[84576]: cluster 2024-09-13T14:37:49.583451+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2024-09-13T14:37:51.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:50 smithi099 bash[87147]: cluster 2024-09-13T14:37:49.583451+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2024-09-13T14:37:51.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:50 smithi099 bash[87147]: cluster 2024-09-13T14:37:49.583451+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2024-09-13T14:37:51.072 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:50 smithi099 systemd[1]: Stopping Ceph osd.1 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:37:51.572 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[40584]: debug 2024-09-13T14:37:51.156+0000 7f9a03add640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T14:37:51.572 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[40584]: debug 2024-09-13T14:37:51.156+0000 7f9a03add640 -1 osd.1 638 *** Got signal Terminated *** 2024-09-13T14:37:51.572 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[40584]: debug 2024-09-13T14:37:51.156+0000 7f9a03add640 -1 osd.1 638 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:37:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[84576]: cluster 2024-09-13T14:37:50.090705+0000 mgr.y (mgr.26252) 262 : cluster [DBG] pgmap v478: 425 pgs: 3 peering, 422 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 778 KiB/s rd, 905 KiB/s wr, 888 op/s 2024-09-13T14:37:52.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[84576]: cluster 2024-09-13T14:37:50.090705+0000 mgr.y (mgr.26252) 262 : cluster [DBG] pgmap v478: 425 pgs: 3 peering, 422 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 778 KiB/s rd, 905 KiB/s wr, 888 op/s 2024-09-13T14:37:52.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[84576]: cluster 2024-09-13T14:37:51.159430+0000 mon.a (mon.0) 812 : cluster [INF] osd.1 marked itself down and dead 2024-09-13T14:37:52.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[84576]: cluster 2024-09-13T14:37:51.159430+0000 mon.a (mon.0) 812 : cluster [INF] osd.1 marked itself down and dead 2024-09-13T14:37:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[87147]: cluster 2024-09-13T14:37:50.090705+0000 mgr.y (mgr.26252) 262 : cluster [DBG] pgmap v478: 425 pgs: 3 peering, 422 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 778 KiB/s rd, 905 KiB/s wr, 888 op/s 2024-09-13T14:37:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[87147]: cluster 2024-09-13T14:37:50.090705+0000 mgr.y (mgr.26252) 262 : cluster [DBG] pgmap v478: 425 pgs: 3 peering, 422 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 778 KiB/s rd, 905 KiB/s wr, 888 op/s 2024-09-13T14:37:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[87147]: cluster 2024-09-13T14:37:51.159430+0000 mon.a (mon.0) 812 : cluster [INF] osd.1 marked itself down and dead 2024-09-13T14:37:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:51 smithi099 bash[87147]: cluster 2024-09-13T14:37:51.159430+0000 mon.a (mon.0) 812 : cluster [INF] osd.1 marked itself down and dead 2024-09-13T14:37:52.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:51 smithi177 bash[68594]: cluster 2024-09-13T14:37:50.090705+0000 mgr.y (mgr.26252) 262 : cluster [DBG] pgmap v478: 425 pgs: 3 peering, 422 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 778 KiB/s rd, 905 KiB/s wr, 888 op/s 2024-09-13T14:37:52.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:51 smithi177 bash[68594]: cluster 2024-09-13T14:37:50.090705+0000 mgr.y (mgr.26252) 262 : cluster [DBG] pgmap v478: 425 pgs: 3 peering, 422 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 778 KiB/s rd, 905 KiB/s wr, 888 op/s 2024-09-13T14:37:52.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:51 smithi177 bash[68594]: cluster 2024-09-13T14:37:51.159430+0000 mon.a (mon.0) 812 : cluster [INF] osd.1 marked itself down and dead 2024-09-13T14:37:52.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:51 smithi177 bash[68594]: cluster 2024-09-13T14:37:51.159430+0000 mon.a (mon.0) 812 : cluster [INF] osd.1 marked itself down and dead 2024-09-13T14:37:52.222 INFO:teuthology.orchestra.run.smithi099.stdout:wait for half of osds to upgrade 2024-09-13T14:37:52.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[84576]: cluster 2024-09-13T14:37:51.597667+0000 mon.a (mon.0) 813 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:37:52.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[84576]: cluster 2024-09-13T14:37:51.597667+0000 mon.a (mon.0) 813 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:37:52.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[84576]: cluster 2024-09-13T14:37:51.605759+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-09-13T14:37:52.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[84576]: cluster 2024-09-13T14:37:51.605759+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-09-13T14:37:52.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[84576]: cluster 2024-09-13T14:37:52.605419+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-09-13T14:37:52.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[84576]: cluster 2024-09-13T14:37:52.605419+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-09-13T14:37:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[87147]: cluster 2024-09-13T14:37:51.597667+0000 mon.a (mon.0) 813 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:37:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[87147]: cluster 2024-09-13T14:37:51.597667+0000 mon.a (mon.0) 813 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:37:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[87147]: cluster 2024-09-13T14:37:51.605759+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-09-13T14:37:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[87147]: cluster 2024-09-13T14:37:51.605759+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-09-13T14:37:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[87147]: cluster 2024-09-13T14:37:52.605419+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-09-13T14:37:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:52 smithi099 bash[87147]: cluster 2024-09-13T14:37:52.605419+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-09-13T14:37:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:52 smithi177 bash[68594]: cluster 2024-09-13T14:37:51.597667+0000 mon.a (mon.0) 813 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:37:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:52 smithi177 bash[68594]: cluster 2024-09-13T14:37:51.597667+0000 mon.a (mon.0) 813 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:37:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:52 smithi177 bash[68594]: cluster 2024-09-13T14:37:51.605759+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-09-13T14:37:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:52 smithi177 bash[68594]: cluster 2024-09-13T14:37:51.605759+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-09-13T14:37:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:52 smithi177 bash[68594]: cluster 2024-09-13T14:37:52.605419+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-09-13T14:37:53.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:52 smithi177 bash[68594]: cluster 2024-09-13T14:37:52.605419+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.499 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.500 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.501 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:53.502 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.503 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.504 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.505 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.506 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:37:53.507 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:53.508 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:53.508 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:37:53.508 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:37:53.508 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 4 2024-09-13T14:37:53.508 INFO:tasks.workunit.client.0.smithi099.stdout:read: 4 2024-09-13T14:37:53.508 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS1 (13638 ms) 2024-09-13T14:37:53.508 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS2 2024-09-13T14:37:53.508 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:37:53.810 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:53 smithi099 bash[99464]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-1 2024-09-13T14:37:53.811 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:53 smithi099 bash[84576]: cluster 2024-09-13T14:37:52.091661+0000 mgr.y (mgr.26252) 263 : cluster [DBG] pgmap v480: 425 pgs: 61 stale+active+clean, 3 peering, 361 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 962 KiB/s rd, 1.1 MiB/s wr, 973 op/s 2024-09-13T14:37:53.811 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:53 smithi099 bash[84576]: cluster 2024-09-13T14:37:52.091661+0000 mgr.y (mgr.26252) 263 : cluster [DBG] pgmap v480: 425 pgs: 61 stale+active+clean, 3 peering, 361 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 962 KiB/s rd, 1.1 MiB/s wr, 973 op/s 2024-09-13T14:37:53.811 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:53 smithi099 bash[87147]: cluster 2024-09-13T14:37:52.091661+0000 mgr.y (mgr.26252) 263 : cluster [DBG] pgmap v480: 425 pgs: 61 stale+active+clean, 3 peering, 361 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 962 KiB/s rd, 1.1 MiB/s wr, 973 op/s 2024-09-13T14:37:53.811 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:53 smithi099 bash[87147]: cluster 2024-09-13T14:37:52.091661+0000 mgr.y (mgr.26252) 263 : cluster [DBG] pgmap v480: 425 pgs: 61 stale+active+clean, 3 peering, 361 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 962 KiB/s rd, 1.1 MiB/s wr, 973 op/s 2024-09-13T14:37:53.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-13T14:37:54.071 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:53 smithi099 bash[99590]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-1 2024-09-13T14:37:54.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:53 smithi177 bash[68594]: cluster 2024-09-13T14:37:52.091661+0000 mgr.y (mgr.26252) 263 : cluster [DBG] pgmap v480: 425 pgs: 61 stale+active+clean, 3 peering, 361 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 962 KiB/s rd, 1.1 MiB/s wr, 973 op/s 2024-09-13T14:37:54.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:53 smithi177 bash[68594]: cluster 2024-09-13T14:37:52.091661+0000 mgr.y (mgr.26252) 263 : cluster [DBG] pgmap v480: 425 pgs: 61 stale+active+clean, 3 peering, 361 active+clean; 6.8 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 962 KiB/s rd, 1.1 MiB/s wr, 973 op/s 2024-09-13T14:37:55.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:54 smithi099 bash[84576]: cluster 2024-09-13T14:37:54.653694+0000 mon.a (mon.0) 816 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-13T14:37:55.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:54 smithi099 bash[84576]: cluster 2024-09-13T14:37:54.653694+0000 mon.a (mon.0) 816 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-13T14:37:55.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:54 smithi099 bash[84576]: cluster 2024-09-13T14:37:54.653723+0000 mon.a (mon.0) 817 : cluster [WRN] Health check failed: Degraded data redundancy: 52/549 objects degraded (9.472%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:55.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:54 smithi099 bash[84576]: cluster 2024-09-13T14:37:54.653723+0000 mon.a (mon.0) 817 : cluster [WRN] Health check failed: Degraded data redundancy: 52/549 objects degraded (9.472%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:55.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:54 smithi099 bash[87147]: cluster 2024-09-13T14:37:54.653694+0000 mon.a (mon.0) 816 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-13T14:37:55.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:54 smithi099 bash[87147]: cluster 2024-09-13T14:37:54.653694+0000 mon.a (mon.0) 816 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-13T14:37:55.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:54 smithi099 bash[87147]: cluster 2024-09-13T14:37:54.653723+0000 mon.a (mon.0) 817 : cluster [WRN] Health check failed: Degraded data redundancy: 52/549 objects degraded (9.472%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:55.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:54 smithi099 bash[87147]: cluster 2024-09-13T14:37:54.653723+0000 mon.a (mon.0) 817 : cluster [WRN] Health check failed: Degraded data redundancy: 52/549 objects degraded (9.472%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:55.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:54 smithi177 bash[68594]: cluster 2024-09-13T14:37:54.653694+0000 mon.a (mon.0) 816 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-13T14:37:55.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:54 smithi177 bash[68594]: cluster 2024-09-13T14:37:54.653694+0000 mon.a (mon.0) 816 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-13T14:37:55.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:54 smithi177 bash[68594]: cluster 2024-09-13T14:37:54.653723+0000 mon.a (mon.0) 817 : cluster [WRN] Health check failed: Degraded data redundancy: 52/549 objects degraded (9.472%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:55.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:54 smithi177 bash[68594]: cluster 2024-09-13T14:37:54.653723+0000 mon.a (mon.0) 817 : cluster [WRN] Health check failed: Degraded data redundancy: 52/549 objects degraded (9.472%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:56.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:55 smithi099 bash[84576]: cluster 2024-09-13T14:37:54.093345+0000 mgr.y (mgr.26252) 264 : cluster [DBG] pgmap v482: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 764 KiB/s wr, 1.53k op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:56.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:55 smithi099 bash[84576]: cluster 2024-09-13T14:37:54.093345+0000 mgr.y (mgr.26252) 264 : cluster [DBG] pgmap v482: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 764 KiB/s wr, 1.53k op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:56.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:55 smithi099 bash[87147]: cluster 2024-09-13T14:37:54.093345+0000 mgr.y (mgr.26252) 264 : cluster [DBG] pgmap v482: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 764 KiB/s wr, 1.53k op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:56.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:55 smithi099 bash[87147]: cluster 2024-09-13T14:37:54.093345+0000 mgr.y (mgr.26252) 264 : cluster [DBG] pgmap v482: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 764 KiB/s wr, 1.53k op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:56.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:55 smithi177 bash[68594]: cluster 2024-09-13T14:37:54.093345+0000 mgr.y (mgr.26252) 264 : cluster [DBG] pgmap v482: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 764 KiB/s wr, 1.53k op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:56.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:55 smithi177 bash[68594]: cluster 2024-09-13T14:37:54.093345+0000 mgr.y (mgr.26252) 264 : cluster [DBG] pgmap v482: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 764 KiB/s wr, 1.53k op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:56.696 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:56.696 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:56.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:56.697 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:56.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:56.697 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T14:37:56.697 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: Stopped Ceph osd.1 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:37:56.697 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:56.697 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: Started Ceph osd.1 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:37:56.698 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:56.698 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:56.698 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:37:56 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:37:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:37:57] "GET /metrics HTTP/1.1" 200 60673 "" "Prometheus/2.51.0" 2024-09-13T14:37:58.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[84576]: cluster 2024-09-13T14:37:56.094296+0000 mgr.y (mgr.26252) 265 : cluster [DBG] pgmap v483: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 196 KiB/s wr, 396 op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:58.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[84576]: cluster 2024-09-13T14:37:56.094296+0000 mgr.y (mgr.26252) 265 : cluster [DBG] pgmap v483: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 196 KiB/s wr, 396 op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:58.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[84576]: audit 2024-09-13T14:37:56.766565+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[84576]: audit 2024-09-13T14:37:56.766565+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[84576]: audit 2024-09-13T14:37:56.771784+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[84576]: audit 2024-09-13T14:37:56.771784+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[84576]: audit 2024-09-13T14:37:56.772874+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:58.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[84576]: audit 2024-09-13T14:37:56.772874+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:58.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[87147]: cluster 2024-09-13T14:37:56.094296+0000 mgr.y (mgr.26252) 265 : cluster [DBG] pgmap v483: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 196 KiB/s wr, 396 op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:58.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[87147]: cluster 2024-09-13T14:37:56.094296+0000 mgr.y (mgr.26252) 265 : cluster [DBG] pgmap v483: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 196 KiB/s wr, 396 op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:58.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[87147]: audit 2024-09-13T14:37:56.766565+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[87147]: audit 2024-09-13T14:37:56.766565+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[87147]: audit 2024-09-13T14:37:56.771784+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[87147]: audit 2024-09-13T14:37:56.771784+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[87147]: audit 2024-09-13T14:37:56.772874+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:58.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:57 smithi099 bash[87147]: audit 2024-09-13T14:37:56.772874+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:58.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:57 smithi177 bash[68594]: cluster 2024-09-13T14:37:56.094296+0000 mgr.y (mgr.26252) 265 : cluster [DBG] pgmap v483: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 196 KiB/s wr, 396 op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:58.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:57 smithi177 bash[68594]: cluster 2024-09-13T14:37:56.094296+0000 mgr.y (mgr.26252) 265 : cluster [DBG] pgmap v483: 425 pgs: 103 active+undersized, 21 peering, 34 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 196 KiB/s wr, 396 op/s; 52/549 objects degraded (9.472%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:58.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:57 smithi177 bash[68594]: audit 2024-09-13T14:37:56.766565+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:57 smithi177 bash[68594]: audit 2024-09-13T14:37:56.766565+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:57 smithi177 bash[68594]: audit 2024-09-13T14:37:56.771784+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:57 smithi177 bash[68594]: audit 2024-09-13T14:37:56.771784+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:37:58.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:57 smithi177 bash[68594]: audit 2024-09-13T14:37:56.772874+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:58.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:57 smithi177 bash[68594]: audit 2024-09-13T14:37:56.772874+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:58.363 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:58.363 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:58.761 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:37:58.762 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:58.762 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:37:58.762 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:37:58.762 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:37:58.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[84576]: cluster 2024-09-13T14:37:58.095486+0000 mgr.y (mgr.26252) 266 : cluster [DBG] pgmap v484: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 160 KiB/s wr, 323 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:58.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[84576]: cluster 2024-09-13T14:37:58.095486+0000 mgr.y (mgr.26252) 266 : cluster [DBG] pgmap v484: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 160 KiB/s wr, 323 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:58.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[84576]: cluster 2024-09-13T14:37:58.711092+0000 mon.a (mon.0) 821 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-13T14:37:58.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[84576]: cluster 2024-09-13T14:37:58.711092+0000 mon.a (mon.0) 821 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-13T14:37:59.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[87147]: cluster 2024-09-13T14:37:58.095486+0000 mgr.y (mgr.26252) 266 : cluster [DBG] pgmap v484: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 160 KiB/s wr, 323 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:59.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[87147]: cluster 2024-09-13T14:37:58.095486+0000 mgr.y (mgr.26252) 266 : cluster [DBG] pgmap v484: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 160 KiB/s wr, 323 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:59.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[87147]: cluster 2024-09-13T14:37:58.711092+0000 mon.a (mon.0) 821 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-13T14:37:59.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[87147]: cluster 2024-09-13T14:37:58.711092+0000 mon.a (mon.0) 821 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-13T14:37:59.072 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:37:59.072 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:37:59.072 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:37:59.072 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:37:59.072 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:37:58 smithi099 bash[100141]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:37:59.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:58 smithi177 bash[68594]: cluster 2024-09-13T14:37:58.095486+0000 mgr.y (mgr.26252) 266 : cluster [DBG] pgmap v484: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 160 KiB/s wr, 323 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:59.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:58 smithi177 bash[68594]: cluster 2024-09-13T14:37:58.095486+0000 mgr.y (mgr.26252) 266 : cluster [DBG] pgmap v484: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 160 KiB/s wr, 323 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:37:59.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:58 smithi177 bash[68594]: cluster 2024-09-13T14:37:58.711092+0000 mon.a (mon.0) 821 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-13T14:37:59.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:37:58 smithi177 bash[68594]: cluster 2024-09-13T14:37:58.711092+0000 mon.a (mon.0) 821 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 8 pgs peering) 2024-09-13T14:38:00.206 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:38:00.607 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:38:00 smithi177 bash[61439]: ts=2024-09-13T14:38:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:01.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:01 smithi099 bash[84576]: cluster 2024-09-13T14:38:00.099588+0000 mgr.y (mgr.26252) 267 : cluster [DBG] pgmap v485: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 303 KiB/s rd, 150 KiB/s wr, 304 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:01 smithi099 bash[84576]: cluster 2024-09-13T14:38:00.099588+0000 mgr.y (mgr.26252) 267 : cluster [DBG] pgmap v485: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 303 KiB/s rd, 150 KiB/s wr, 304 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:01 smithi099 bash[84576]: cluster 2024-09-13T14:38:01.074689+0000 mon.a (mon.0) 822 : cluster [WRN] Health check update: Degraded data redundancy: 70/549 objects degraded (12.750%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:01 smithi099 bash[84576]: cluster 2024-09-13T14:38:01.074689+0000 mon.a (mon.0) 822 : cluster [WRN] Health check update: Degraded data redundancy: 70/549 objects degraded (12.750%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:01 smithi099 bash[87147]: cluster 2024-09-13T14:38:00.099588+0000 mgr.y (mgr.26252) 267 : cluster [DBG] pgmap v485: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 303 KiB/s rd, 150 KiB/s wr, 304 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:01 smithi099 bash[87147]: cluster 2024-09-13T14:38:00.099588+0000 mgr.y (mgr.26252) 267 : cluster [DBG] pgmap v485: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 303 KiB/s rd, 150 KiB/s wr, 304 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:01 smithi099 bash[87147]: cluster 2024-09-13T14:38:01.074689+0000 mon.a (mon.0) 822 : cluster [WRN] Health check update: Degraded data redundancy: 70/549 objects degraded (12.750%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:01 smithi099 bash[87147]: cluster 2024-09-13T14:38:01.074689+0000 mon.a (mon.0) 822 : cluster [WRN] Health check update: Degraded data redundancy: 70/549 objects degraded (12.750%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:01.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:01 smithi177 bash[68594]: cluster 2024-09-13T14:38:00.099588+0000 mgr.y (mgr.26252) 267 : cluster [DBG] pgmap v485: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 303 KiB/s rd, 150 KiB/s wr, 304 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:01.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:01 smithi177 bash[68594]: cluster 2024-09-13T14:38:00.099588+0000 mgr.y (mgr.26252) 267 : cluster [DBG] pgmap v485: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 303 KiB/s rd, 150 KiB/s wr, 304 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:01.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:01 smithi177 bash[68594]: cluster 2024-09-13T14:38:01.074689+0000 mon.a (mon.0) 822 : cluster [WRN] Health check update: Degraded data redundancy: 70/549 objects degraded (12.750%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:01.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:01 smithi177 bash[68594]: cluster 2024-09-13T14:38:01.074689+0000 mon.a (mon.0) 822 : cluster [WRN] Health check update: Degraded data redundancy: 70/549 objects degraded (12.750%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:02.413 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:38:02 smithi099 bash[100499]: debug 2024-09-13T14:38:02.188+0000 7f69961f0640 1 -- 172.21.15.99:0/1756769378 <== mon.2 v2:172.21.15.99:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558cd45b6960 con 0x558cd45d7800 2024-09-13T14:38:03.104 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:03.104 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:03.105 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.106 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:03.107 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.108 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.109 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:03.110 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 512~512 = 512 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:aio discard: 1536~512 = 0 2024-09-13T14:38:03.111 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.112 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:38:03.113 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:38:03.114 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.115 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.116 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:03.117 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.118 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.119 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:03.120 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:03.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[87147]: cluster 2024-09-13T14:38:02.101022+0000 mgr.y (mgr.26252) 268 : cluster [DBG] pgmap v486: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 257 KiB/s rd, 128 KiB/s wr, 258 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:03.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[87147]: cluster 2024-09-13T14:38:02.101022+0000 mgr.y (mgr.26252) 268 : cluster [DBG] pgmap v486: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 257 KiB/s rd, 128 KiB/s wr, 258 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[87147]: audit 2024-09-13T14:38:03.006272+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[87147]: audit 2024-09-13T14:38:03.006272+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[87147]: audit 2024-09-13T14:38:03.012202+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[87147]: audit 2024-09-13T14:38:03.012202+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[84576]: cluster 2024-09-13T14:38:02.101022+0000 mgr.y (mgr.26252) 268 : cluster [DBG] pgmap v486: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 257 KiB/s rd, 128 KiB/s wr, 258 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[84576]: cluster 2024-09-13T14:38:02.101022+0000 mgr.y (mgr.26252) 268 : cluster [DBG] pgmap v486: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 257 KiB/s rd, 128 KiB/s wr, 258 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[84576]: audit 2024-09-13T14:38:03.006272+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[84576]: audit 2024-09-13T14:38:03.006272+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[84576]: audit 2024-09-13T14:38:03.012202+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:03 smithi099 bash[84576]: audit 2024-09-13T14:38:03.012202+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:03 smithi177 bash[68594]: cluster 2024-09-13T14:38:02.101022+0000 mgr.y (mgr.26252) 268 : cluster [DBG] pgmap v486: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 257 KiB/s rd, 128 KiB/s wr, 258 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:03.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:03 smithi177 bash[68594]: cluster 2024-09-13T14:38:02.101022+0000 mgr.y (mgr.26252) 268 : cluster [DBG] pgmap v486: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 11 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 257 KiB/s rd, 128 KiB/s wr, 258 op/s; 70/549 objects degraded (12.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:03.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:03 smithi177 bash[68594]: audit 2024-09-13T14:38:03.006272+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:03 smithi177 bash[68594]: audit 2024-09-13T14:38:03.006272+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:03 smithi177 bash[68594]: audit 2024-09-13T14:38:03.012202+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:03.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:03 smithi177 bash[68594]: audit 2024-09-13T14:38:03.012202+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.571 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[100499]: debug 2024-09-13T14:38:04.288+0000 7f6998a57640 -1 Falling back to public interface 2024-09-13T14:38:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[84576]: audit 2024-09-13T14:38:03.253147+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[84576]: audit 2024-09-13T14:38:03.253147+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[84576]: audit 2024-09-13T14:38:03.254494+0000 mon.a (mon.0) 826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[84576]: audit 2024-09-13T14:38:03.254494+0000 mon.a (mon.0) 826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[84576]: audit 2024-09-13T14:38:03.579371+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[84576]: audit 2024-09-13T14:38:03.579371+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[84576]: audit 2024-09-13T14:38:04.216067+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[84576]: audit 2024-09-13T14:38:04.216067+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[84576]: audit 2024-09-13T14:38:04.221728+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[84576]: audit 2024-09-13T14:38:04.221728+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[87147]: audit 2024-09-13T14:38:03.253147+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[87147]: audit 2024-09-13T14:38:03.253147+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[87147]: audit 2024-09-13T14:38:03.254494+0000 mon.a (mon.0) 826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:04.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[87147]: audit 2024-09-13T14:38:03.254494+0000 mon.a (mon.0) 826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:04.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[87147]: audit 2024-09-13T14:38:03.579371+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[87147]: audit 2024-09-13T14:38:03.579371+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[87147]: audit 2024-09-13T14:38:04.216067+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[87147]: audit 2024-09-13T14:38:04.216067+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[87147]: audit 2024-09-13T14:38:04.221728+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:04 smithi099 bash[87147]: audit 2024-09-13T14:38:04.221728+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:04 smithi177 bash[68594]: audit 2024-09-13T14:38:03.253147+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:04 smithi177 bash[68594]: audit 2024-09-13T14:38:03.253147+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:04 smithi177 bash[68594]: audit 2024-09-13T14:38:03.254494+0000 mon.a (mon.0) 826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:04.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:04 smithi177 bash[68594]: audit 2024-09-13T14:38:03.254494+0000 mon.a (mon.0) 826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:04.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:04 smithi177 bash[68594]: audit 2024-09-13T14:38:03.579371+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:04 smithi177 bash[68594]: audit 2024-09-13T14:38:03.579371+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:04 smithi177 bash[68594]: audit 2024-09-13T14:38:04.216067+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:04 smithi177 bash[68594]: audit 2024-09-13T14:38:04.216067+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:04 smithi177 bash[68594]: audit 2024-09-13T14:38:04.221728+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:04.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:04 smithi177 bash[68594]: audit 2024-09-13T14:38:04.221728+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:05.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:05 smithi177 bash[68594]: cluster 2024-09-13T14:38:04.103244+0000 mgr.y (mgr.26252) 269 : cluster [DBG] pgmap v487: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.7 MiB/s wr, 548 op/s; 72/564 objects degraded (12.766%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:05.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:05 smithi177 bash[68594]: cluster 2024-09-13T14:38:04.103244+0000 mgr.y (mgr.26252) 269 : cluster [DBG] pgmap v487: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.7 MiB/s wr, 548 op/s; 72/564 objects degraded (12.766%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:05.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:05 smithi177 bash[68594]: audit 2024-09-13T14:38:05.566696+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.99:0/3070955922' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:05.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:05 smithi177 bash[68594]: audit 2024-09-13T14:38:05.566696+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.99:0/3070955922' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:05.913 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:06.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:05 smithi099 bash[87147]: cluster 2024-09-13T14:38:04.103244+0000 mgr.y (mgr.26252) 269 : cluster [DBG] pgmap v487: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.7 MiB/s wr, 548 op/s; 72/564 objects degraded (12.766%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:06.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:05 smithi099 bash[87147]: cluster 2024-09-13T14:38:04.103244+0000 mgr.y (mgr.26252) 269 : cluster [DBG] pgmap v487: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.7 MiB/s wr, 548 op/s; 72/564 objects degraded (12.766%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:06.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:05 smithi099 bash[87147]: audit 2024-09-13T14:38:05.566696+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.99:0/3070955922' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:06.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:05 smithi099 bash[87147]: audit 2024-09-13T14:38:05.566696+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.99:0/3070955922' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:06.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:05 smithi099 bash[84576]: cluster 2024-09-13T14:38:04.103244+0000 mgr.y (mgr.26252) 269 : cluster [DBG] pgmap v487: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.7 MiB/s wr, 548 op/s; 72/564 objects degraded (12.766%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:06.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:05 smithi099 bash[84576]: cluster 2024-09-13T14:38:04.103244+0000 mgr.y (mgr.26252) 269 : cluster [DBG] pgmap v487: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.7 MiB/s wr, 548 op/s; 72/564 objects degraded (12.766%); 0 B/s, 0 objects/s recovering 2024-09-13T14:38:06.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:05 smithi099 bash[84576]: audit 2024-09-13T14:38:05.566696+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.99:0/3070955922' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:06.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:05 smithi099 bash[84576]: audit 2024-09-13T14:38:05.566696+0000 mon.a (mon.0) 830 : audit [DBG] from='client.? 172.21.15.99:0/3070955922' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:06.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:06 smithi177 bash[68594]: cluster 2024-09-13T14:38:06.076035+0000 mon.a (mon.0) 831 : cluster [WRN] Health check update: Degraded data redundancy: 72/564 objects degraded (12.766%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:06.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:06 smithi177 bash[68594]: cluster 2024-09-13T14:38:06.076035+0000 mon.a (mon.0) 831 : cluster [WRN] Health check update: Degraded data redundancy: 72/564 objects degraded (12.766%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:07.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:06 smithi099 bash[84576]: cluster 2024-09-13T14:38:06.076035+0000 mon.a (mon.0) 831 : cluster [WRN] Health check update: Degraded data redundancy: 72/564 objects degraded (12.766%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:07.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:06 smithi099 bash[84576]: cluster 2024-09-13T14:38:06.076035+0000 mon.a (mon.0) 831 : cluster [WRN] Health check update: Degraded data redundancy: 72/564 objects degraded (12.766%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:07.045 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:38:07] "GET /metrics HTTP/1.1" 200 60690 "" "Prometheus/2.51.0" 2024-09-13T14:38:07.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:06 smithi099 bash[87147]: cluster 2024-09-13T14:38:06.076035+0000 mon.a (mon.0) 831 : cluster [WRN] Health check update: Degraded data redundancy: 72/564 objects degraded (12.766%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:07.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:06 smithi099 bash[87147]: cluster 2024-09-13T14:38:06.076035+0000 mon.a (mon.0) 831 : cluster [WRN] Health check update: Degraded data redundancy: 72/564 objects degraded (12.766%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:07.531 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.531 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.531 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.531 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.531 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.531 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.531 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:07.532 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.533 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.534 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.535 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.536 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.537 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.538 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:07.539 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 4 2024-09-13T14:38:07.540 INFO:tasks.workunit.client.0.smithi099.stdout:read: 4 2024-09-13T14:38:07.541 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS2 (14032 ms) 2024-09-13T14:38:07.541 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestIOWithIOHint 2024-09-13T14:38:07.541 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:07.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:07 smithi177 bash[68594]: audit 2024-09-13T14:38:05.903295+0000 mgr.y (mgr.26252) 270 : audit [DBG] from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:07.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:07 smithi177 bash[68594]: audit 2024-09-13T14:38:05.903295+0000 mgr.y (mgr.26252) 270 : audit [DBG] from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:07.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:07 smithi177 bash[68594]: cluster 2024-09-13T14:38:06.104106+0000 mgr.y (mgr.26252) 271 : cluster [DBG] pgmap v488: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:07.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:07 smithi177 bash[68594]: cluster 2024-09-13T14:38:06.104106+0000 mgr.y (mgr.26252) 271 : cluster [DBG] pgmap v488: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:07.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:07 smithi177 bash[68594]: audit 2024-09-13T14:38:06.258348+0000 mgr.y (mgr.26252) 272 : audit [DBG] from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:07.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:07 smithi177 bash[68594]: audit 2024-09-13T14:38:06.258348+0000 mgr.y (mgr.26252) 272 : audit [DBG] from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:08.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[87147]: audit 2024-09-13T14:38:05.903295+0000 mgr.y (mgr.26252) 270 : audit [DBG] from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:08.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[87147]: audit 2024-09-13T14:38:05.903295+0000 mgr.y (mgr.26252) 270 : audit [DBG] from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:08.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[87147]: cluster 2024-09-13T14:38:06.104106+0000 mgr.y (mgr.26252) 271 : cluster [DBG] pgmap v488: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:08.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[87147]: cluster 2024-09-13T14:38:06.104106+0000 mgr.y (mgr.26252) 271 : cluster [DBG] pgmap v488: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:08.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[87147]: audit 2024-09-13T14:38:06.258348+0000 mgr.y (mgr.26252) 272 : audit [DBG] from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:08.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[87147]: audit 2024-09-13T14:38:06.258348+0000 mgr.y (mgr.26252) 272 : audit [DBG] from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:08.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[84576]: audit 2024-09-13T14:38:05.903295+0000 mgr.y (mgr.26252) 270 : audit [DBG] from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:08.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[84576]: audit 2024-09-13T14:38:05.903295+0000 mgr.y (mgr.26252) 270 : audit [DBG] from='client.44285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:08.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[84576]: cluster 2024-09-13T14:38:06.104106+0000 mgr.y (mgr.26252) 271 : cluster [DBG] pgmap v488: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:08.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[84576]: cluster 2024-09-13T14:38:06.104106+0000 mgr.y (mgr.26252) 271 : cluster [DBG] pgmap v488: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:08.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[84576]: audit 2024-09-13T14:38:06.258348+0000 mgr.y (mgr.26252) 272 : audit [DBG] from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:08.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:07 smithi099 bash[84576]: audit 2024-09-13T14:38:06.258348+0000 mgr.y (mgr.26252) 272 : audit [DBG] from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:08.821 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:38:08 smithi099 bash[100499]: debug 2024-09-13T14:38:08.436+0000 7f6998a57640 -1 osd.1 638 log_to_monitors true 2024-09-13T14:38:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:08 smithi099 bash[84576]: audit 2024-09-13T14:38:08.444289+0000 mon.a (mon.0) 832 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:38:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:08 smithi099 bash[84576]: audit 2024-09-13T14:38:08.444289+0000 mon.a (mon.0) 832 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:38:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:08 smithi099 bash[87147]: audit 2024-09-13T14:38:08.444289+0000 mon.a (mon.0) 832 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:38:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:08 smithi099 bash[87147]: audit 2024-09-13T14:38:08.444289+0000 mon.a (mon.0) 832 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:38:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:08 smithi177 bash[68594]: audit 2024-09-13T14:38:08.444289+0000 mon.a (mon.0) 832 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:38:08.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:08 smithi177 bash[68594]: audit 2024-09-13T14:38:08.444289+0000 mon.a (mon.0) 832 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:38:08.953 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:09.385 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[100499]: debug 2024-09-13T14:38:09.132+0000 7f698c7f9640 -1 osd.1 638 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:38:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: cluster 2024-09-13T14:38:08.104987+0000 mgr.y (mgr.26252) 273 : cluster [DBG] pgmap v489: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: cluster 2024-09-13T14:38:08.104987+0000 mgr.y (mgr.26252) 273 : cluster [DBG] pgmap v489: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: audit 2024-09-13T14:38:08.597115+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.99:0/2551775017' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: audit 2024-09-13T14:38:08.597115+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.99:0/2551775017' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: audit 2024-09-13T14:38:08.599572+0000 mon.a (mon.0) 833 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: audit 2024-09-13T14:38:08.599572+0000 mon.a (mon.0) 833 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: cluster 2024-09-13T14:38:08.603894+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: cluster 2024-09-13T14:38:08.603894+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: audit 2024-09-13T14:38:08.604303+0000 mon.a (mon.0) 835 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: audit 2024-09-13T14:38:08.604303+0000 mon.a (mon.0) 835 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: audit 2024-09-13T14:38:09.115463+0000 mon.a (mon.0) 836 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[87147]: audit 2024-09-13T14:38:09.115463+0000 mon.a (mon.0) 836 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: cluster 2024-09-13T14:38:08.104987+0000 mgr.y (mgr.26252) 273 : cluster [DBG] pgmap v489: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: cluster 2024-09-13T14:38:08.104987+0000 mgr.y (mgr.26252) 273 : cluster [DBG] pgmap v489: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:09.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: audit 2024-09-13T14:38:08.597115+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.99:0/2551775017' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: audit 2024-09-13T14:38:08.597115+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.99:0/2551775017' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: audit 2024-09-13T14:38:08.599572+0000 mon.a (mon.0) 833 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:38:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: audit 2024-09-13T14:38:08.599572+0000 mon.a (mon.0) 833 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:38:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: cluster 2024-09-13T14:38:08.603894+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-09-13T14:38:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: cluster 2024-09-13T14:38:08.603894+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-09-13T14:38:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: audit 2024-09-13T14:38:08.604303+0000 mon.a (mon.0) 835 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: audit 2024-09-13T14:38:08.604303+0000 mon.a (mon.0) 835 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: audit 2024-09-13T14:38:09.115463+0000 mon.a (mon.0) 836 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' 2024-09-13T14:38:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:09 smithi099 bash[84576]: audit 2024-09-13T14:38:09.115463+0000 mon.a (mon.0) 836 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' 2024-09-13T14:38:09.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: cluster 2024-09-13T14:38:08.104987+0000 mgr.y (mgr.26252) 273 : cluster [DBG] pgmap v489: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:09.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: cluster 2024-09-13T14:38:08.104987+0000 mgr.y (mgr.26252) 273 : cluster [DBG] pgmap v489: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 27 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 521 op/s; 72/564 objects degraded (12.766%) 2024-09-13T14:38:09.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: audit 2024-09-13T14:38:08.597115+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.99:0/2551775017' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: audit 2024-09-13T14:38:08.597115+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.99:0/2551775017' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: audit 2024-09-13T14:38:08.599572+0000 mon.a (mon.0) 833 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:38:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: audit 2024-09-13T14:38:08.599572+0000 mon.a (mon.0) 833 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:38:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: cluster 2024-09-13T14:38:08.603894+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-09-13T14:38:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: cluster 2024-09-13T14:38:08.603894+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-09-13T14:38:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: audit 2024-09-13T14:38:08.604303+0000 mon.a (mon.0) 835 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: audit 2024-09-13T14:38:08.604303+0000 mon.a (mon.0) 835 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: audit 2024-09-13T14:38:09.115463+0000 mon.a (mon.0) 836 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' 2024-09-13T14:38:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:09 smithi177 bash[68594]: audit 2024-09-13T14:38:09.115463+0000 mon.a (mon.0) 836 : audit [INF] from='osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824]' entity='osd.1' 2024-09-13T14:38:10.606 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[61439]: ts=2024-09-13T14:38:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: audit 2024-09-13T14:38:08.942093+0000 mgr.y (mgr.26252) 274 : audit [DBG] from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: audit 2024-09-13T14:38:08.942093+0000 mgr.y (mgr.26252) 274 : audit [DBG] from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: audit 2024-09-13T14:38:09.285864+0000 mgr.y (mgr.26252) 275 : audit [DBG] from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: audit 2024-09-13T14:38:09.285864+0000 mgr.y (mgr.26252) 275 : audit [DBG] from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: cluster 2024-09-13T14:38:10.114805+0000 mon.a (mon.0) 837 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: cluster 2024-09-13T14:38:10.114805+0000 mon.a (mon.0) 837 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: cluster 2024-09-13T14:38:10.126307+0000 mon.a (mon.0) 838 : cluster [INF] osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824] boot 2024-09-13T14:38:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: cluster 2024-09-13T14:38:10.126307+0000 mon.a (mon.0) 838 : cluster [INF] osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824] boot 2024-09-13T14:38:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: cluster 2024-09-13T14:38:10.126502+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2024-09-13T14:38:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: cluster 2024-09-13T14:38:10.126502+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2024-09-13T14:38:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: audit 2024-09-13T14:38:10.131847+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:38:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:10 smithi177 bash[68594]: audit 2024-09-13T14:38:10.131847+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:38:11.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: audit 2024-09-13T14:38:08.942093+0000 mgr.y (mgr.26252) 274 : audit [DBG] from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: audit 2024-09-13T14:38:08.942093+0000 mgr.y (mgr.26252) 274 : audit [DBG] from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: audit 2024-09-13T14:38:09.285864+0000 mgr.y (mgr.26252) 275 : audit [DBG] from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: audit 2024-09-13T14:38:09.285864+0000 mgr.y (mgr.26252) 275 : audit [DBG] from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: cluster 2024-09-13T14:38:10.114805+0000 mon.a (mon.0) 837 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: cluster 2024-09-13T14:38:10.114805+0000 mon.a (mon.0) 837 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: cluster 2024-09-13T14:38:10.126307+0000 mon.a (mon.0) 838 : cluster [INF] osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824] boot 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: cluster 2024-09-13T14:38:10.126307+0000 mon.a (mon.0) 838 : cluster [INF] osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824] boot 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: cluster 2024-09-13T14:38:10.126502+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: cluster 2024-09-13T14:38:10.126502+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: audit 2024-09-13T14:38:10.131847+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[87147]: audit 2024-09-13T14:38:10.131847+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: audit 2024-09-13T14:38:08.942093+0000 mgr.y (mgr.26252) 274 : audit [DBG] from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: audit 2024-09-13T14:38:08.942093+0000 mgr.y (mgr.26252) 274 : audit [DBG] from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: audit 2024-09-13T14:38:09.285864+0000 mgr.y (mgr.26252) 275 : audit [DBG] from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: audit 2024-09-13T14:38:09.285864+0000 mgr.y (mgr.26252) 275 : audit [DBG] from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: cluster 2024-09-13T14:38:10.114805+0000 mon.a (mon.0) 837 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:11.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: cluster 2024-09-13T14:38:10.114805+0000 mon.a (mon.0) 837 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:11.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: cluster 2024-09-13T14:38:10.126307+0000 mon.a (mon.0) 838 : cluster [INF] osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824] boot 2024-09-13T14:38:11.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: cluster 2024-09-13T14:38:10.126307+0000 mon.a (mon.0) 838 : cluster [INF] osd.1 [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824] boot 2024-09-13T14:38:11.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: cluster 2024-09-13T14:38:10.126502+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2024-09-13T14:38:11.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: cluster 2024-09-13T14:38:10.126502+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2024-09-13T14:38:11.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: audit 2024-09-13T14:38:10.131847+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:38:11.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:10 smithi099 bash[84576]: audit 2024-09-13T14:38:10.131847+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:38:11.459 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:11.459 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.460 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:11.461 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.462 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:11.463 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:11.464 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:11.465 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 512~512 = 512 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:aio discard: 1536~512 = 0 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:11.466 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.467 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.468 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.469 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.470 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.471 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:11.472 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:11.473 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.474 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.475 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:11.475 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:11.475 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:11.475 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:11.475 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:11.475 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:11.475 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:11.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[87147]: cluster 2024-09-13T14:38:10.108870+0000 mgr.y (mgr.26252) 276 : cluster [DBG] pgmap v491: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.04k op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[87147]: cluster 2024-09-13T14:38:10.108870+0000 mgr.y (mgr.26252) 276 : cluster [DBG] pgmap v491: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.04k op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[87147]: cluster 2024-09-13T14:38:11.077133+0000 mon.a (mon.0) 841 : cluster [WRN] Health check update: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[87147]: cluster 2024-09-13T14:38:11.077133+0000 mon.a (mon.0) 841 : cluster [WRN] Health check update: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[87147]: cluster 2024-09-13T14:38:11.145769+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-13T14:38:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[87147]: cluster 2024-09-13T14:38:11.145769+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-13T14:38:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[84576]: cluster 2024-09-13T14:38:10.108870+0000 mgr.y (mgr.26252) 276 : cluster [DBG] pgmap v491: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.04k op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[84576]: cluster 2024-09-13T14:38:10.108870+0000 mgr.y (mgr.26252) 276 : cluster [DBG] pgmap v491: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.04k op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[84576]: cluster 2024-09-13T14:38:11.077133+0000 mon.a (mon.0) 841 : cluster [WRN] Health check update: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[84576]: cluster 2024-09-13T14:38:11.077133+0000 mon.a (mon.0) 841 : cluster [WRN] Health check update: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[84576]: cluster 2024-09-13T14:38:11.145769+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-13T14:38:11.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:11 smithi099 bash[84576]: cluster 2024-09-13T14:38:11.145769+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-13T14:38:11.957 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:12.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:11 smithi177 bash[68594]: cluster 2024-09-13T14:38:10.108870+0000 mgr.y (mgr.26252) 276 : cluster [DBG] pgmap v491: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.04k op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:12.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:11 smithi177 bash[68594]: cluster 2024-09-13T14:38:10.108870+0000 mgr.y (mgr.26252) 276 : cluster [DBG] pgmap v491: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.04k op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:12.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:11 smithi177 bash[68594]: cluster 2024-09-13T14:38:11.077133+0000 mon.a (mon.0) 841 : cluster [WRN] Health check update: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:12.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:11 smithi177 bash[68594]: cluster 2024-09-13T14:38:11.077133+0000 mon.a (mon.0) 841 : cluster [WRN] Health check update: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:12.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:11 smithi177 bash[68594]: cluster 2024-09-13T14:38:11.145769+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-13T14:38:12.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:11 smithi177 bash[68594]: cluster 2024-09-13T14:38:11.145769+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-13T14:38:12.976 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:12 smithi099 bash[84576]: audit 2024-09-13T14:38:11.615280+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.99:0/3970519449' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:12.976 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:12 smithi099 bash[84576]: audit 2024-09-13T14:38:11.615280+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.99:0/3970519449' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:12.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:12 smithi099 bash[87147]: audit 2024-09-13T14:38:11.615280+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.99:0/3970519449' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:12.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:12 smithi099 bash[87147]: audit 2024-09-13T14:38:11.615280+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.99:0/3970519449' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:13.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:12 smithi177 bash[68594]: audit 2024-09-13T14:38:11.615280+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.99:0/3970519449' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:13.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:12 smithi177 bash[68594]: audit 2024-09-13T14:38:11.615280+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.99:0/3970519449' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:14.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[84576]: audit 2024-09-13T14:38:11.945075+0000 mgr.y (mgr.26252) 277 : audit [DBG] from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[84576]: audit 2024-09-13T14:38:11.945075+0000 mgr.y (mgr.26252) 277 : audit [DBG] from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[84576]: cluster 2024-09-13T14:38:12.109597+0000 mgr.y (mgr.26252) 278 : cluster [DBG] pgmap v494: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 277 KiB/s wr, 690 op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:14.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[84576]: cluster 2024-09-13T14:38:12.109597+0000 mgr.y (mgr.26252) 278 : cluster [DBG] pgmap v494: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 277 KiB/s wr, 690 op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:14.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[84576]: audit 2024-09-13T14:38:12.306817+0000 mgr.y (mgr.26252) 279 : audit [DBG] from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[84576]: audit 2024-09-13T14:38:12.306817+0000 mgr.y (mgr.26252) 279 : audit [DBG] from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[87147]: audit 2024-09-13T14:38:11.945075+0000 mgr.y (mgr.26252) 277 : audit [DBG] from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[87147]: audit 2024-09-13T14:38:11.945075+0000 mgr.y (mgr.26252) 277 : audit [DBG] from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[87147]: cluster 2024-09-13T14:38:12.109597+0000 mgr.y (mgr.26252) 278 : cluster [DBG] pgmap v494: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 277 KiB/s wr, 690 op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:14.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[87147]: cluster 2024-09-13T14:38:12.109597+0000 mgr.y (mgr.26252) 278 : cluster [DBG] pgmap v494: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 277 KiB/s wr, 690 op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:14.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[87147]: audit 2024-09-13T14:38:12.306817+0000 mgr.y (mgr.26252) 279 : audit [DBG] from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:13 smithi099 bash[87147]: audit 2024-09-13T14:38:12.306817+0000 mgr.y (mgr.26252) 279 : audit [DBG] from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:13 smithi177 bash[68594]: audit 2024-09-13T14:38:11.945075+0000 mgr.y (mgr.26252) 277 : audit [DBG] from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:13 smithi177 bash[68594]: audit 2024-09-13T14:38:11.945075+0000 mgr.y (mgr.26252) 277 : audit [DBG] from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:13 smithi177 bash[68594]: cluster 2024-09-13T14:38:12.109597+0000 mgr.y (mgr.26252) 278 : cluster [DBG] pgmap v494: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 277 KiB/s wr, 690 op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:14.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:13 smithi177 bash[68594]: cluster 2024-09-13T14:38:12.109597+0000 mgr.y (mgr.26252) 278 : cluster [DBG] pgmap v494: 425 pgs: 116 active+undersized, 42 active+undersized+degraded, 267 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 277 KiB/s wr, 690 op/s; 75/579 objects degraded (12.953%) 2024-09-13T14:38:14.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:13 smithi177 bash[68594]: audit 2024-09-13T14:38:12.306817+0000 mgr.y (mgr.26252) 279 : audit [DBG] from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:13 smithi177 bash[68594]: audit 2024-09-13T14:38:12.306817+0000 mgr.y (mgr.26252) 279 : audit [DBG] from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:14.763 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.763 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.763 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.763 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.763 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.763 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.763 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.763 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.764 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.765 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.766 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.767 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.768 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:14.769 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.770 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:14.771 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.772 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.772 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:14.772 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:14.772 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:14.772 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:14.772 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:14.772 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:14.772 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:14.772 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestIOWithIOHint (7232 ms) 2024-09-13T14:38:14.773 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestDataPoolIO 2024-09-13T14:38:14.967 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:15.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[87147]: cluster 2024-09-13T14:38:14.622223+0000 mon.a (mon.0) 844 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded) 2024-09-13T14:38:15.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[87147]: cluster 2024-09-13T14:38:14.622223+0000 mon.a (mon.0) 844 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded) 2024-09-13T14:38:15.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[87147]: cluster 2024-09-13T14:38:14.622257+0000 mon.a (mon.0) 845 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:15.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[87147]: cluster 2024-09-13T14:38:14.622257+0000 mon.a (mon.0) 845 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:15.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[87147]: audit 2024-09-13T14:38:14.634713+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.99:0/4022387038' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:15.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[87147]: audit 2024-09-13T14:38:14.634713+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.99:0/4022387038' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:15.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[84576]: cluster 2024-09-13T14:38:14.622223+0000 mon.a (mon.0) 844 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded) 2024-09-13T14:38:15.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[84576]: cluster 2024-09-13T14:38:14.622223+0000 mon.a (mon.0) 844 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded) 2024-09-13T14:38:15.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[84576]: cluster 2024-09-13T14:38:14.622257+0000 mon.a (mon.0) 845 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:15.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[84576]: cluster 2024-09-13T14:38:14.622257+0000 mon.a (mon.0) 845 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:15.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[84576]: audit 2024-09-13T14:38:14.634713+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.99:0/4022387038' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:15.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:14 smithi099 bash[84576]: audit 2024-09-13T14:38:14.634713+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.99:0/4022387038' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:14 smithi177 bash[68594]: cluster 2024-09-13T14:38:14.622223+0000 mon.a (mon.0) 844 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded) 2024-09-13T14:38:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:14 smithi177 bash[68594]: cluster 2024-09-13T14:38:14.622223+0000 mon.a (mon.0) 844 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/579 objects degraded (12.953%), 42 pgs degraded) 2024-09-13T14:38:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:14 smithi177 bash[68594]: cluster 2024-09-13T14:38:14.622257+0000 mon.a (mon.0) 845 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:15.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:14 smithi177 bash[68594]: cluster 2024-09-13T14:38:14.622257+0000 mon.a (mon.0) 845 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:14 smithi177 bash[68594]: audit 2024-09-13T14:38:14.634713+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.99:0/4022387038' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:14 smithi177 bash[68594]: audit 2024-09-13T14:38:14.634713+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.99:0/4022387038' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:16.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:15 smithi099 bash[87147]: cluster 2024-09-13T14:38:14.112063+0000 mgr.y (mgr.26252) 280 : cluster [DBG] pgmap v495: 425 pgs: 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 509 KiB/s wr, 1.64k op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:16.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:15 smithi099 bash[87147]: cluster 2024-09-13T14:38:14.112063+0000 mgr.y (mgr.26252) 280 : cluster [DBG] pgmap v495: 425 pgs: 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 509 KiB/s wr, 1.64k op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:16.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:15 smithi099 bash[84576]: cluster 2024-09-13T14:38:14.112063+0000 mgr.y (mgr.26252) 280 : cluster [DBG] pgmap v495: 425 pgs: 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 509 KiB/s wr, 1.64k op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:16.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:15 smithi099 bash[84576]: cluster 2024-09-13T14:38:14.112063+0000 mgr.y (mgr.26252) 280 : cluster [DBG] pgmap v495: 425 pgs: 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 509 KiB/s wr, 1.64k op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:16.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:15 smithi177 bash[68594]: cluster 2024-09-13T14:38:14.112063+0000 mgr.y (mgr.26252) 280 : cluster [DBG] pgmap v495: 425 pgs: 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 509 KiB/s wr, 1.64k op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:16.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:15 smithi177 bash[68594]: cluster 2024-09-13T14:38:14.112063+0000 mgr.y (mgr.26252) 280 : cluster [DBG] pgmap v495: 425 pgs: 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 509 KiB/s wr, 1.64k op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:16.571 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[76324]: debug 2024-09-13T14:38:16.300+0000 7fd55bc0f640 -1 mgr.server reply reply (11) Resource temporarily unavailable 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:38:16.572 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[76324]: debug 2024-09-13T14:38:16.300+0000 7fd55bc0f640 -1 mgr.server reply reply (16) Device or resource busy 32 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2024-09-13T14:38:16.700 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:17.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:14.958186+0000 mgr.y (mgr.26252) 281 : audit [DBG] from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:14.958186+0000 mgr.y (mgr.26252) 281 : audit [DBG] from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:15.268950+0000 mgr.y (mgr.26252) 282 : audit [DBG] from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:15.268950+0000 mgr.y (mgr.26252) 282 : audit [DBG] from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: cluster 2024-09-13T14:38:15.693746+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-13T14:38:17.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: cluster 2024-09-13T14:38:15.693746+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-13T14:38:17.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:15.698748+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.99:0/922802798' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:15.698748+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.99:0/922802798' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:15.711392+0000 mon.a (mon.0) 848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:15.711392+0000 mon.a (mon.0) 848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.112005+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.112005+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.123058+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.123058+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.124933+0000 mon.a (mon.0) 851 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.124933+0000 mon.a (mon.0) 851 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.126514+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.126514+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:14.958186+0000 mgr.y (mgr.26252) 281 : audit [DBG] from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:14.958186+0000 mgr.y (mgr.26252) 281 : audit [DBG] from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:15.268950+0000 mgr.y (mgr.26252) 282 : audit [DBG] from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:15.268950+0000 mgr.y (mgr.26252) 282 : audit [DBG] from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: cluster 2024-09-13T14:38:15.693746+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: cluster 2024-09-13T14:38:15.693746+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:15.698748+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.99:0/922802798' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:15.698748+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.99:0/922802798' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:15.711392+0000 mon.a (mon.0) 848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:15.711392+0000 mon.a (mon.0) 848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.112005+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.112005+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.123058+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.123058+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.124933+0000 mon.a (mon.0) 851 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.124933+0000 mon.a (mon.0) 851 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.126514+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:17.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.126514+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:17.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.136753+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.136753+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.291375+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.291375+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.294387+0000 mon.a (mon.0) 855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.294387+0000 mon.a (mon.0) 855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.296767+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.296767+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.298693+0000 mon.a (mon.0) 857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.298693+0000 mon.a (mon.0) 857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.301165+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[84576]: audit 2024-09-13T14:38:16.301165+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.136753+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.136753+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.291375+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:17.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.291375+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:17.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.294387+0000 mon.a (mon.0) 855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.294387+0000 mon.a (mon.0) 855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.296767+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.296767+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.298693+0000 mon.a (mon.0) 857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.298693+0000 mon.a (mon.0) 857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.301165+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:17.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:16 smithi099 bash[87147]: audit 2024-09-13T14:38:16.301165+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:14.958186+0000 mgr.y (mgr.26252) 281 : audit [DBG] from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:14.958186+0000 mgr.y (mgr.26252) 281 : audit [DBG] from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:15.268950+0000 mgr.y (mgr.26252) 282 : audit [DBG] from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:15.268950+0000 mgr.y (mgr.26252) 282 : audit [DBG] from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: cluster 2024-09-13T14:38:15.693746+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-13T14:38:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: cluster 2024-09-13T14:38:15.693746+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-13T14:38:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:15.698748+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.99:0/922802798' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:15.698748+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.99:0/922802798' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:15.711392+0000 mon.a (mon.0) 848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:15.711392+0000 mon.a (mon.0) 848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.112005+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.112005+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.123058+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.123058+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.124933+0000 mon.a (mon.0) 851 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.124933+0000 mon.a (mon.0) 851 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.126514+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.126514+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.136753+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.136753+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.291375+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.291375+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.294387+0000 mon.a (mon.0) 855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.294387+0000 mon.a (mon.0) 855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.296767+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.296767+0000 mon.a (mon.0) 856 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.298693+0000 mon.a (mon.0) 857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.298693+0000 mon.a (mon.0) 857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.301165+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:16 smithi177 bash[68594]: audit 2024-09-13T14:38:16.301165+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:38:17] "GET /metrics HTTP/1.1" 200 60690 "" "Prometheus/2.51.0" 2024-09-13T14:38:17.919 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: cluster 2024-09-13T14:38:16.113152+0000 mgr.y (mgr.26252) 283 : cluster [DBG] pgmap v497: 457 pgs: 32 unknown, 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 951 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: cluster 2024-09-13T14:38:16.113152+0000 mgr.y (mgr.26252) 283 : cluster [DBG] pgmap v497: 457 pgs: 32 unknown, 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 951 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: audit 2024-09-13T14:38:16.301460+0000 mgr.y (mgr.26252) 284 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: audit 2024-09-13T14:38:16.301460+0000 mgr.y (mgr.26252) 284 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: cephadm 2024-09-13T14:38:16.303714+0000 mgr.y (mgr.26252) 285 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:38:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: cephadm 2024-09-13T14:38:16.303714+0000 mgr.y (mgr.26252) 285 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:38:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: audit 2024-09-13T14:38:16.689141+0000 mon.a (mon.0) 859 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: audit 2024-09-13T14:38:16.689141+0000 mon.a (mon.0) 859 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: cluster 2024-09-13T14:38:16.698130+0000 mon.a (mon.0) 860 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: cluster 2024-09-13T14:38:16.698130+0000 mon.a (mon.0) 860 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: cluster 2024-09-13T14:38:17.107657+0000 mon.a (mon.0) 861 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: cluster 2024-09-13T14:38:17.107657+0000 mon.a (mon.0) 861 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: audit 2024-09-13T14:38:17.578621+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.99:0/705633051' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: audit 2024-09-13T14:38:17.578621+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.99:0/705633051' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: cluster 2024-09-13T14:38:17.694578+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[84576]: cluster 2024-09-13T14:38:17.694578+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: cluster 2024-09-13T14:38:16.113152+0000 mgr.y (mgr.26252) 283 : cluster [DBG] pgmap v497: 457 pgs: 32 unknown, 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 951 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: cluster 2024-09-13T14:38:16.113152+0000 mgr.y (mgr.26252) 283 : cluster [DBG] pgmap v497: 457 pgs: 32 unknown, 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 951 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: audit 2024-09-13T14:38:16.301460+0000 mgr.y (mgr.26252) 284 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: audit 2024-09-13T14:38:16.301460+0000 mgr.y (mgr.26252) 284 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: cephadm 2024-09-13T14:38:16.303714+0000 mgr.y (mgr.26252) 285 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: cephadm 2024-09-13T14:38:16.303714+0000 mgr.y (mgr.26252) 285 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: audit 2024-09-13T14:38:16.689141+0000 mon.a (mon.0) 859 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: audit 2024-09-13T14:38:16.689141+0000 mon.a (mon.0) 859 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: cluster 2024-09-13T14:38:16.698130+0000 mon.a (mon.0) 860 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: cluster 2024-09-13T14:38:16.698130+0000 mon.a (mon.0) 860 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: cluster 2024-09-13T14:38:17.107657+0000 mon.a (mon.0) 861 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: cluster 2024-09-13T14:38:17.107657+0000 mon.a (mon.0) 861 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: audit 2024-09-13T14:38:17.578621+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.99:0/705633051' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: audit 2024-09-13T14:38:17.578621+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.99:0/705633051' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: cluster 2024-09-13T14:38:17.694578+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-09-13T14:38:18.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:17 smithi099 bash[87147]: cluster 2024-09-13T14:38:17.694578+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-09-13T14:38:18.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: cluster 2024-09-13T14:38:16.113152+0000 mgr.y (mgr.26252) 283 : cluster [DBG] pgmap v497: 457 pgs: 32 unknown, 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 951 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: cluster 2024-09-13T14:38:16.113152+0000 mgr.y (mgr.26252) 283 : cluster [DBG] pgmap v497: 457 pgs: 32 unknown, 425 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 951 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: audit 2024-09-13T14:38:16.301460+0000 mgr.y (mgr.26252) 284 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: audit 2024-09-13T14:38:16.301460+0000 mgr.y (mgr.26252) 284 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: cephadm 2024-09-13T14:38:16.303714+0000 mgr.y (mgr.26252) 285 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: cephadm 2024-09-13T14:38:16.303714+0000 mgr.y (mgr.26252) 285 : cephadm [INF] Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: audit 2024-09-13T14:38:16.689141+0000 mon.a (mon.0) 859 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: audit 2024-09-13T14:38:16.689141+0000 mon.a (mon.0) 859 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: cluster 2024-09-13T14:38:16.698130+0000 mon.a (mon.0) 860 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: cluster 2024-09-13T14:38:16.698130+0000 mon.a (mon.0) 860 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: cluster 2024-09-13T14:38:17.107657+0000 mon.a (mon.0) 861 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:38:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: cluster 2024-09-13T14:38:17.107657+0000 mon.a (mon.0) 861 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:38:18.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: audit 2024-09-13T14:38:17.578621+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.99:0/705633051' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:18.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: audit 2024-09-13T14:38:17.578621+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.99:0/705633051' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:18.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: cluster 2024-09-13T14:38:17.694578+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-09-13T14:38:18.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:17 smithi177 bash[68594]: cluster 2024-09-13T14:38:17.694578+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-09-13T14:38:18.842 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:18.843 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:18.844 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:18.845 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:18.846 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 512~512 = 512 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:aio discard: 1536~512 = 0 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestDataPoolIO (4079 ms) 2024-09-13T14:38:18.847 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatch 2024-09-13T14:38:18.848 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:18.908 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatch (66 ms) 2024-09-13T14:38:18.908 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatch 2024-09-13T14:38:18.908 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:18.980 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatch (72 ms) 2024-09-13T14:38:18.981 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccess 2024-09-13T14:38:18.981 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:19.054 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccess (74 ms) 2024-09-13T14:38:19.054 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccess 2024-09-13T14:38:19.054 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:19.126 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccess (72 ms) 2024-09-13T14:38:19.126 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned 2024-09-13T14:38:19.126 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:19.261 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned (135 ms) 2024-09-13T14:38:19.261 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned 2024-09-13T14:38:19.261 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:19.380 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned (118 ms) 2024-09-13T14:38:19.380 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLarge 2024-09-13T14:38:19.380 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:19.519 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLarge (139 ms) 2024-09-13T14:38:19.520 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLarge 2024-09-13T14:38:19.520 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: audit 2024-09-13T14:38:17.908382+0000 mgr.y (mgr.26252) 286 : audit [DBG] from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: audit 2024-09-13T14:38:17.908382+0000 mgr.y (mgr.26252) 286 : audit [DBG] from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: cluster 2024-09-13T14:38:18.114113+0000 mgr.y (mgr.26252) 287 : cluster [DBG] pgmap v500: 457 pgs: 31 unknown, 426 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 952 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: cluster 2024-09-13T14:38:18.114113+0000 mgr.y (mgr.26252) 287 : cluster [DBG] pgmap v500: 457 pgs: 31 unknown, 426 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 952 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: audit 2024-09-13T14:38:18.224510+0000 mgr.y (mgr.26252) 288 : audit [DBG] from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: audit 2024-09-13T14:38:18.224510+0000 mgr.y (mgr.26252) 288 : audit [DBG] from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: audit 2024-09-13T14:38:18.252972+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: audit 2024-09-13T14:38:18.252972+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: audit 2024-09-13T14:38:18.254217+0000 mon.a (mon.0) 864 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: audit 2024-09-13T14:38:18.254217+0000 mon.a (mon.0) 864 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: cluster 2024-09-13T14:38:18.701912+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: cluster 2024-09-13T14:38:18.701912+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: cluster 2024-09-13T14:38:18.722235+0000 mon.a (mon.0) 866 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:38:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: cluster 2024-09-13T14:38:18.722235+0000 mon.a (mon.0) 866 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: cluster 2024-09-13T14:38:18.722311+0000 mon.a (mon.0) 867 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[84576]: cluster 2024-09-13T14:38:18.722311+0000 mon.a (mon.0) 867 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: audit 2024-09-13T14:38:17.908382+0000 mgr.y (mgr.26252) 286 : audit [DBG] from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: audit 2024-09-13T14:38:17.908382+0000 mgr.y (mgr.26252) 286 : audit [DBG] from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: cluster 2024-09-13T14:38:18.114113+0000 mgr.y (mgr.26252) 287 : cluster [DBG] pgmap v500: 457 pgs: 31 unknown, 426 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 952 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: cluster 2024-09-13T14:38:18.114113+0000 mgr.y (mgr.26252) 287 : cluster [DBG] pgmap v500: 457 pgs: 31 unknown, 426 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 952 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: audit 2024-09-13T14:38:18.224510+0000 mgr.y (mgr.26252) 288 : audit [DBG] from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: audit 2024-09-13T14:38:18.224510+0000 mgr.y (mgr.26252) 288 : audit [DBG] from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: audit 2024-09-13T14:38:18.252972+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: audit 2024-09-13T14:38:18.252972+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: audit 2024-09-13T14:38:18.254217+0000 mon.a (mon.0) 864 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: audit 2024-09-13T14:38:18.254217+0000 mon.a (mon.0) 864 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: cluster 2024-09-13T14:38:18.701912+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2024-09-13T14:38:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: cluster 2024-09-13T14:38:18.701912+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2024-09-13T14:38:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: cluster 2024-09-13T14:38:18.722235+0000 mon.a (mon.0) 866 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:38:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: cluster 2024-09-13T14:38:18.722235+0000 mon.a (mon.0) 866 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:38:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: cluster 2024-09-13T14:38:18.722311+0000 mon.a (mon.0) 867 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:19 smithi099 bash[87147]: cluster 2024-09-13T14:38:18.722311+0000 mon.a (mon.0) 867 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:19.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: audit 2024-09-13T14:38:17.908382+0000 mgr.y (mgr.26252) 286 : audit [DBG] from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: audit 2024-09-13T14:38:17.908382+0000 mgr.y (mgr.26252) 286 : audit [DBG] from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: cluster 2024-09-13T14:38:18.114113+0000 mgr.y (mgr.26252) 287 : cluster [DBG] pgmap v500: 457 pgs: 31 unknown, 426 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 952 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: cluster 2024-09-13T14:38:18.114113+0000 mgr.y (mgr.26252) 287 : cluster [DBG] pgmap v500: 457 pgs: 31 unknown, 426 active+clean; 31 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 464 KiB/s rd, 232 KiB/s wr, 952 op/s; 1.3 MiB/s, 10 keys/s, 1 objects/s recovering 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: audit 2024-09-13T14:38:18.224510+0000 mgr.y (mgr.26252) 288 : audit [DBG] from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: audit 2024-09-13T14:38:18.224510+0000 mgr.y (mgr.26252) 288 : audit [DBG] from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: audit 2024-09-13T14:38:18.252972+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: audit 2024-09-13T14:38:18.252972+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: audit 2024-09-13T14:38:18.254217+0000 mon.a (mon.0) 864 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: audit 2024-09-13T14:38:18.254217+0000 mon.a (mon.0) 864 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: cluster 2024-09-13T14:38:18.701912+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2024-09-13T14:38:19.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: cluster 2024-09-13T14:38:18.701912+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2024-09-13T14:38:19.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: cluster 2024-09-13T14:38:18.722235+0000 mon.a (mon.0) 866 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:38:19.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: cluster 2024-09-13T14:38:18.722235+0000 mon.a (mon.0) 866 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:38:19.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: cluster 2024-09-13T14:38:18.722311+0000 mon.a (mon.0) 867 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:19.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:19 smithi177 bash[68594]: cluster 2024-09-13T14:38:18.722311+0000 mon.a (mon.0) 867 : cluster [INF] Cluster is now healthy 2024-09-13T14:38:19.650 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLarge (131 ms) 2024-09-13T14:38:19.650 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess 2024-09-13T14:38:19.650 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:20.207 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess (556 ms) 2024-09-13T14:38:20.207 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess 2024-09-13T14:38:20.207 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:20.526 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess (319 ms) 2024-09-13T14:38:20.527 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers 2024-09-13T14:38:20.527 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:20.577 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers (50 ms) 2024-09-13T14:38:20.577 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVMismatch 2024-09-13T14:38:20.577 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:20.607 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:38:20 smithi177 bash[61439]: ts=2024-09-13T14:38:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:20.615 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVMismatch (39 ms) 2024-09-13T14:38:20.616 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVSuccess 2024-09-13T14:38:20.616 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:20.666 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVSuccess (50 ms) 2024-09-13T14:38:20.666 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestScatterGatherIO 2024-09-13T14:38:20.666 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:20.711 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestScatterGatherIO (45 ms) 2024-09-13T14:38:20.711 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestEmptyDiscard 2024-09-13T14:38:20.711 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:20.762 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:20.762 INFO:tasks.workunit.client.0.smithi099.stdout:aio discard: 0~1048576 = 0 2024-09-13T14:38:20.762 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:20.762 INFO:tasks.workunit.client.0.smithi099.stdout:aio discard: 0~4194304 = 0 2024-09-13T14:38:20.762 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:20.762 INFO:tasks.workunit.client.0.smithi099.stdout:aio discard: 3145728~1048576 = 0 2024-09-13T14:38:20.762 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestEmptyDiscard (50 ms) 2024-09-13T14:38:20.762 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestFUA 2024-09-13T14:38:20.763 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:20.877 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:20.988 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:20.988 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:20.988 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:20.988 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:20.988 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:20.988 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:20.988 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:20.988 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:20.989 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:20.990 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:20.991 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:20.992 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:20.992 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:20.992 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:20.992 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:20.992 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:20.992 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:20.992 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:20.992 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestFUA (227 ms) 2024-09-13T14:38:20.992 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestIOPP 2024-09-13T14:38:20.992 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:21.021 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:21.021 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:21.021 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:21.021 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:21.021 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:21.021 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.021 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:21.022 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:21.023 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.023 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:21.023 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.023 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:21.023 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:21.023 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.023 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:21.023 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.056 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:21.056 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:start compare and write 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:compare and wrote: 512 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.057 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:21.058 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.058 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:21.058 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:21.058 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.058 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:21.059 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.061 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:21.061 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:21.061 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.061 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:21.061 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.066 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:21.066 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:21.066 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.066 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:21.067 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.069 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:21.069 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:21.069 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.069 INFO:tasks.workunit.client.0.smithi099.stdout:started aio compare and write 2024-09-13T14:38:21.069 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.075 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:21.075 INFO:tasks.workunit.client.0.smithi099.stdout:finished aio compare and write 2024-09-13T14:38:21.075 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:21.075 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:21.075 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:21.075 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:21.075 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:finished read 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:finished read 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:21.076 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:21.077 INFO:tasks.workunit.client.0.smithi099.stdout:finished read 2024-09-13T14:38:21.077 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.077 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:21.077 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:21.077 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:21.077 INFO:tasks.workunit.client.0.smithi099.stdout:finished read 2024-09-13T14:38:21.077 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:21.077 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:21.078 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:21.078 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:21.078 INFO:tasks.workunit.client.0.smithi099.stdout:finished read 2024-09-13T14:38:21.078 INFO:tasks.workunit.client.0.smithi099.stdout:discard: 512~512 2024-09-13T14:38:21.078 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:21.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:21 smithi099 bash[84576]: cluster 2024-09-13T14:38:20.117640+0000 mgr.y (mgr.26252) 289 : cluster [DBG] pgmap v502: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 54 KiB/s wr, 265 op/s 2024-09-13T14:38:21.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:21 smithi099 bash[84576]: cluster 2024-09-13T14:38:20.117640+0000 mgr.y (mgr.26252) 289 : cluster [DBG] pgmap v502: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 54 KiB/s wr, 265 op/s 2024-09-13T14:38:21.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:21 smithi099 bash[84576]: audit 2024-09-13T14:38:20.534679+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.99:0/2514166232' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:21.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:21 smithi099 bash[84576]: audit 2024-09-13T14:38:20.534679+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.99:0/2514166232' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:21 smithi099 bash[87147]: cluster 2024-09-13T14:38:20.117640+0000 mgr.y (mgr.26252) 289 : cluster [DBG] pgmap v502: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 54 KiB/s wr, 265 op/s 2024-09-13T14:38:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:21 smithi099 bash[87147]: cluster 2024-09-13T14:38:20.117640+0000 mgr.y (mgr.26252) 289 : cluster [DBG] pgmap v502: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 54 KiB/s wr, 265 op/s 2024-09-13T14:38:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:21 smithi099 bash[87147]: audit 2024-09-13T14:38:20.534679+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.99:0/2514166232' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:21 smithi099 bash[87147]: audit 2024-09-13T14:38:20.534679+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.99:0/2514166232' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:21.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:21 smithi177 bash[68594]: cluster 2024-09-13T14:38:20.117640+0000 mgr.y (mgr.26252) 289 : cluster [DBG] pgmap v502: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 54 KiB/s wr, 265 op/s 2024-09-13T14:38:21.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:21 smithi177 bash[68594]: cluster 2024-09-13T14:38:20.117640+0000 mgr.y (mgr.26252) 289 : cluster [DBG] pgmap v502: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 54 KiB/s wr, 265 op/s 2024-09-13T14:38:21.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:21 smithi177 bash[68594]: audit 2024-09-13T14:38:20.534679+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.99:0/2514166232' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:21.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:21 smithi177 bash[68594]: audit 2024-09-13T14:38:20.534679+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.99:0/2514166232' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:22.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:22 smithi099 bash[87147]: audit 2024-09-13T14:38:20.867015+0000 mgr.y (mgr.26252) 290 : audit [DBG] from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:22 smithi099 bash[87147]: audit 2024-09-13T14:38:20.867015+0000 mgr.y (mgr.26252) 290 : audit [DBG] from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:22 smithi099 bash[87147]: audit 2024-09-13T14:38:21.193202+0000 mgr.y (mgr.26252) 291 : audit [DBG] from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:22 smithi099 bash[87147]: audit 2024-09-13T14:38:21.193202+0000 mgr.y (mgr.26252) 291 : audit [DBG] from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:22 smithi099 bash[84576]: audit 2024-09-13T14:38:20.867015+0000 mgr.y (mgr.26252) 290 : audit [DBG] from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:22 smithi099 bash[84576]: audit 2024-09-13T14:38:20.867015+0000 mgr.y (mgr.26252) 290 : audit [DBG] from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:22 smithi099 bash[84576]: audit 2024-09-13T14:38:21.193202+0000 mgr.y (mgr.26252) 291 : audit [DBG] from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:22 smithi099 bash[84576]: audit 2024-09-13T14:38:21.193202+0000 mgr.y (mgr.26252) 291 : audit [DBG] from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:22 smithi177 bash[68594]: audit 2024-09-13T14:38:20.867015+0000 mgr.y (mgr.26252) 290 : audit [DBG] from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:22 smithi177 bash[68594]: audit 2024-09-13T14:38:20.867015+0000 mgr.y (mgr.26252) 290 : audit [DBG] from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:22 smithi177 bash[68594]: audit 2024-09-13T14:38:21.193202+0000 mgr.y (mgr.26252) 291 : audit [DBG] from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:22.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:22 smithi177 bash[68594]: audit 2024-09-13T14:38:21.193202+0000 mgr.y (mgr.26252) 291 : audit [DBG] from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:23.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:23 smithi099 bash[87147]: cluster 2024-09-13T14:38:22.118659+0000 mgr.y (mgr.26252) 292 : cluster [DBG] pgmap v503: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 40 KiB/s wr, 195 op/s 2024-09-13T14:38:23.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:23 smithi099 bash[87147]: cluster 2024-09-13T14:38:22.118659+0000 mgr.y (mgr.26252) 292 : cluster [DBG] pgmap v503: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 40 KiB/s wr, 195 op/s 2024-09-13T14:38:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:23 smithi099 bash[84576]: cluster 2024-09-13T14:38:22.118659+0000 mgr.y (mgr.26252) 292 : cluster [DBG] pgmap v503: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 40 KiB/s wr, 195 op/s 2024-09-13T14:38:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:23 smithi099 bash[84576]: cluster 2024-09-13T14:38:22.118659+0000 mgr.y (mgr.26252) 292 : cluster [DBG] pgmap v503: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 40 KiB/s wr, 195 op/s 2024-09-13T14:38:23.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:23 smithi177 bash[68594]: cluster 2024-09-13T14:38:22.118659+0000 mgr.y (mgr.26252) 292 : cluster [DBG] pgmap v503: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 40 KiB/s wr, 195 op/s 2024-09-13T14:38:23.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:23 smithi177 bash[68594]: cluster 2024-09-13T14:38:22.118659+0000 mgr.y (mgr.26252) 292 : cluster [DBG] pgmap v503: 457 pgs: 457 active+clean; 35 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 40 KiB/s wr, 195 op/s 2024-09-13T14:38:23.828 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:24 smithi099 bash[84576]: audit 2024-09-13T14:38:23.507721+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.99:0/4294086409' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:24 smithi099 bash[84576]: audit 2024-09-13T14:38:23.507721+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.99:0/4294086409' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:24.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:24 smithi099 bash[87147]: audit 2024-09-13T14:38:23.507721+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.99:0/4294086409' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:24.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:24 smithi099 bash[87147]: audit 2024-09-13T14:38:23.507721+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.99:0/4294086409' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:24.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:24 smithi177 bash[68594]: audit 2024-09-13T14:38:23.507721+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.99:0/4294086409' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:24.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:24 smithi177 bash[68594]: audit 2024-09-13T14:38:23.507721+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.99:0/4294086409' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:24.736 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:24.736 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:24.736 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:24.736 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:24.736 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:24.736 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.737 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.738 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:38:24.739 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.740 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.740 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.740 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.740 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:38:24.741 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.741 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.741 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:38:24.741 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.741 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.742 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:38:24.742 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.742 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.742 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:38:24.742 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.742 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.743 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.743 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.743 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:38:24.743 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.743 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.743 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:38:24.743 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.744 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.744 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:38:24.744 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.744 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.744 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:38:24.745 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.745 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.745 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.745 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.745 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:24.745 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:24.745 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:24.746 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:24.746 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:24.746 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.746 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.746 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:24.746 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:24.747 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:24.747 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:24.747 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:24.747 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.747 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.747 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:24.748 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:24.748 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:24.766 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:24.766 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:24.766 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.766 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.766 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:24.766 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:24.766 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:24.794 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:24.794 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:24.794 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.794 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.794 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:24.794 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:24.794 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.795 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:24.795 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:24.795 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:24.795 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:24.795 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:24.795 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:24.882 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:24.883 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:24.883 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.883 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.883 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:24.883 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:24.883 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:24.966 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:24.966 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:24.966 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:24.966 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:24.966 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:24.966 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:24.966 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:25.089 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:25.090 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:25.090 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:25.090 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:25.090 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:25.090 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:25.090 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:25.210 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:25.211 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:25.211 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:25.211 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:25.211 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:25.211 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:25.212 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:25.212 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:25.212 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:25.212 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:25.212 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:25.212 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:25.213 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:25.376 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:25.376 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:25.377 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:25.377 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:25.377 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:25.377 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:25.377 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:25.542 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:25.542 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:25.543 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:25.543 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:25.543 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:25.543 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:25.543 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[84576]: audit 2024-09-13T14:38:23.817440+0000 mgr.y (mgr.26252) 293 : audit [DBG] from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[84576]: audit 2024-09-13T14:38:23.817440+0000 mgr.y (mgr.26252) 293 : audit [DBG] from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[84576]: cluster 2024-09-13T14:38:24.122680+0000 mgr.y (mgr.26252) 294 : cluster [DBG] pgmap v504: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 7.7 MiB/s wr, 778 op/s 2024-09-13T14:38:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[84576]: cluster 2024-09-13T14:38:24.122680+0000 mgr.y (mgr.26252) 294 : cluster [DBG] pgmap v504: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 7.7 MiB/s wr, 778 op/s 2024-09-13T14:38:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[84576]: audit 2024-09-13T14:38:24.137186+0000 mgr.y (mgr.26252) 295 : audit [DBG] from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[84576]: audit 2024-09-13T14:38:24.137186+0000 mgr.y (mgr.26252) 295 : audit [DBG] from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[87147]: audit 2024-09-13T14:38:23.817440+0000 mgr.y (mgr.26252) 293 : audit [DBG] from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[87147]: audit 2024-09-13T14:38:23.817440+0000 mgr.y (mgr.26252) 293 : audit [DBG] from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[87147]: cluster 2024-09-13T14:38:24.122680+0000 mgr.y (mgr.26252) 294 : cluster [DBG] pgmap v504: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 7.7 MiB/s wr, 778 op/s 2024-09-13T14:38:25.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[87147]: cluster 2024-09-13T14:38:24.122680+0000 mgr.y (mgr.26252) 294 : cluster [DBG] pgmap v504: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 7.7 MiB/s wr, 778 op/s 2024-09-13T14:38:25.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[87147]: audit 2024-09-13T14:38:24.137186+0000 mgr.y (mgr.26252) 295 : audit [DBG] from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:25 smithi099 bash[87147]: audit 2024-09-13T14:38:24.137186+0000 mgr.y (mgr.26252) 295 : audit [DBG] from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:25 smithi177 bash[68594]: audit 2024-09-13T14:38:23.817440+0000 mgr.y (mgr.26252) 293 : audit [DBG] from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:25 smithi177 bash[68594]: audit 2024-09-13T14:38:23.817440+0000 mgr.y (mgr.26252) 293 : audit [DBG] from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:25 smithi177 bash[68594]: cluster 2024-09-13T14:38:24.122680+0000 mgr.y (mgr.26252) 294 : cluster [DBG] pgmap v504: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 7.7 MiB/s wr, 778 op/s 2024-09-13T14:38:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:25 smithi177 bash[68594]: cluster 2024-09-13T14:38:24.122680+0000 mgr.y (mgr.26252) 294 : cluster [DBG] pgmap v504: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 MiB/s rd, 7.7 MiB/s wr, 778 op/s 2024-09-13T14:38:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:25 smithi177 bash[68594]: audit 2024-09-13T14:38:24.137186+0000 mgr.y (mgr.26252) 295 : audit [DBG] from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:25 smithi177 bash[68594]: audit 2024-09-13T14:38:24.137186+0000 mgr.y (mgr.26252) 295 : audit [DBG] from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:25.760 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:25.761 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:25.761 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:25.761 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:25.761 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:25.761 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:25.761 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:25.968 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:25.968 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:25.969 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:26.244 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:26.244 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:26.244 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:26.244 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:26.244 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:26.244 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:26.244 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:26.437 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:26.437 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:26.437 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:26.437 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:26.437 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:26.437 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:26.437 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:26.669 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:26.670 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:26.670 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:26.670 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:26.670 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:26.670 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:26.670 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:26.790 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:26.888 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:26.889 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:27.191 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:27.191 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:27.191 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:27.191 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:27.191 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:27.191 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:27.191 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:27.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:38:27] "GET /metrics HTTP/1.1" 200 63085 "" "Prometheus/2.51.0" 2024-09-13T14:38:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[84576]: cluster 2024-09-13T14:38:26.093366+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2024-09-13T14:38:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[84576]: cluster 2024-09-13T14:38:26.093366+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2024-09-13T14:38:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[84576]: cluster 2024-09-13T14:38:26.123757+0000 mgr.y (mgr.26252) 296 : cluster [DBG] pgmap v506: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.2 MiB/s wr, 721 op/s 2024-09-13T14:38:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[84576]: cluster 2024-09-13T14:38:26.123757+0000 mgr.y (mgr.26252) 296 : cluster [DBG] pgmap v506: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.2 MiB/s wr, 721 op/s 2024-09-13T14:38:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[84576]: audit 2024-09-13T14:38:26.459577+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.99:0/3401964185' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[84576]: audit 2024-09-13T14:38:26.459577+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.99:0/3401964185' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[87147]: cluster 2024-09-13T14:38:26.093366+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2024-09-13T14:38:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[87147]: cluster 2024-09-13T14:38:26.093366+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2024-09-13T14:38:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[87147]: cluster 2024-09-13T14:38:26.123757+0000 mgr.y (mgr.26252) 296 : cluster [DBG] pgmap v506: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.2 MiB/s wr, 721 op/s 2024-09-13T14:38:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[87147]: cluster 2024-09-13T14:38:26.123757+0000 mgr.y (mgr.26252) 296 : cluster [DBG] pgmap v506: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.2 MiB/s wr, 721 op/s 2024-09-13T14:38:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[87147]: audit 2024-09-13T14:38:26.459577+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.99:0/3401964185' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:27 smithi099 bash[87147]: audit 2024-09-13T14:38:26.459577+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.99:0/3401964185' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:27.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:27 smithi177 bash[68594]: cluster 2024-09-13T14:38:26.093366+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2024-09-13T14:38:27.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:27 smithi177 bash[68594]: cluster 2024-09-13T14:38:26.093366+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2024-09-13T14:38:27.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:27 smithi177 bash[68594]: cluster 2024-09-13T14:38:26.123757+0000 mgr.y (mgr.26252) 296 : cluster [DBG] pgmap v506: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.2 MiB/s wr, 721 op/s 2024-09-13T14:38:27.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:27 smithi177 bash[68594]: cluster 2024-09-13T14:38:26.123757+0000 mgr.y (mgr.26252) 296 : cluster [DBG] pgmap v506: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.2 MiB/s wr, 721 op/s 2024-09-13T14:38:27.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:27 smithi177 bash[68594]: audit 2024-09-13T14:38:26.459577+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.99:0/3401964185' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:27.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:27 smithi177 bash[68594]: audit 2024-09-13T14:38:26.459577+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.99:0/3401964185' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:27.522 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:27.522 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:27.522 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:27.522 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:27.522 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:27.523 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:27.523 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:27.933 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:27.934 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:27.934 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:27.934 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:27.934 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:27.934 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:27.934 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:28.303 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:28.303 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:28.303 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:28.303 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:28.303 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:28.304 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:28.304 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:28.304 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:28.304 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:28.304 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:28.304 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestIOPP (7315 ms) 2024-09-13T14:38:28.304 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP 2024-09-13T14:38:28.304 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:28.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:28 smithi177 bash[68594]: audit 2024-09-13T14:38:26.779745+0000 mgr.y (mgr.26252) 297 : audit [DBG] from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:28 smithi177 bash[68594]: audit 2024-09-13T14:38:26.779745+0000 mgr.y (mgr.26252) 297 : audit [DBG] from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:28 smithi177 bash[68594]: audit 2024-09-13T14:38:27.100024+0000 mgr.y (mgr.26252) 298 : audit [DBG] from='client.44423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:28 smithi177 bash[68594]: audit 2024-09-13T14:38:27.100024+0000 mgr.y (mgr.26252) 298 : audit [DBG] from='client.44423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.365 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP (62 ms) 2024-09-13T14:38:28.365 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP 2024-09-13T14:38:28.365 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:28.426 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP (61 ms) 2024-09-13T14:38:28.426 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP 2024-09-13T14:38:28.426 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:28.490 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP (64 ms) 2024-09-13T14:38:28.490 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP 2024-09-13T14:38:28.490 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:28.551 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP (61 ms) 2024-09-13T14:38:28.551 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchPP 2024-09-13T14:38:28.551 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:28 smithi099 bash[84576]: audit 2024-09-13T14:38:26.779745+0000 mgr.y (mgr.26252) 297 : audit [DBG] from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:28 smithi099 bash[84576]: audit 2024-09-13T14:38:26.779745+0000 mgr.y (mgr.26252) 297 : audit [DBG] from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:28 smithi099 bash[84576]: audit 2024-09-13T14:38:27.100024+0000 mgr.y (mgr.26252) 298 : audit [DBG] from='client.44423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:28 smithi099 bash[84576]: audit 2024-09-13T14:38:27.100024+0000 mgr.y (mgr.26252) 298 : audit [DBG] from='client.44423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:28 smithi099 bash[87147]: audit 2024-09-13T14:38:26.779745+0000 mgr.y (mgr.26252) 297 : audit [DBG] from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:28 smithi099 bash[87147]: audit 2024-09-13T14:38:26.779745+0000 mgr.y (mgr.26252) 297 : audit [DBG] from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:28 smithi099 bash[87147]: audit 2024-09-13T14:38:27.100024+0000 mgr.y (mgr.26252) 298 : audit [DBG] from='client.44423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:28 smithi099 bash[87147]: audit 2024-09-13T14:38:27.100024+0000 mgr.y (mgr.26252) 298 : audit [DBG] from='client.44423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:28.616 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchPP (65 ms) 2024-09-13T14:38:28.616 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchPP 2024-09-13T14:38:28.616 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:28.677 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:28.687 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchPP (71 ms) 2024-09-13T14:38:28.687 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP 2024-09-13T14:38:28.687 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:28.757 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP (70 ms) 2024-09-13T14:38:28.757 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP 2024-09-13T14:38:28.757 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:28.816 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:28.826 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP (69 ms) 2024-09-13T14:38:28.826 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessPP 2024-09-13T14:38:28.826 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:28.895 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessPP (69 ms) 2024-09-13T14:38:28.895 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessPP 2024-09-13T14:38:28.895 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:28.953 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:28.962 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessPP (67 ms) 2024-09-13T14:38:28.962 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP 2024-09-13T14:38:28.962 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:29.034 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP (72 ms) 2024-09-13T14:38:29.034 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP 2024-09-13T14:38:29.034 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:29.093 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:29.103 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP (68 ms) 2024-09-13T14:38:29.103 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP 2024-09-13T14:38:29.103 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:29.210 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP (108 ms) 2024-09-13T14:38:29.210 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP 2024-09-13T14:38:29.210 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:29.284 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:29.318 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP (108 ms) 2024-09-13T14:38:29.318 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLargePP 2024-09-13T14:38:29.318 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:29.443 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLargePP (125 ms) 2024-09-13T14:38:29.443 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLargePP 2024-09-13T14:38:29.443 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:29.505 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:29.545 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLargePP (102 ms) 2024-09-13T14:38:29.545 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP 2024-09-13T14:38:29.546 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:29.774 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:30.052 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP (507 ms) 2024-09-13T14:38:30.052 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP 2024-09-13T14:38:30.052 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:30.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[84576]: cluster 2024-09-13T14:38:28.125235+0000 mgr.y (mgr.26252) 299 : cluster [DBG] pgmap v507: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-09-13T14:38:30.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[84576]: cluster 2024-09-13T14:38:28.125235+0000 mgr.y (mgr.26252) 299 : cluster [DBG] pgmap v507: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-09-13T14:38:30.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[84576]: audit 2024-09-13T14:38:28.611494+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:30.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[84576]: audit 2024-09-13T14:38:28.611494+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:30.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[84576]: audit 2024-09-13T14:38:29.430194+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.99:0/511731588' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:30.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[84576]: audit 2024-09-13T14:38:29.430194+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.99:0/511731588' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:30.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[87147]: cluster 2024-09-13T14:38:28.125235+0000 mgr.y (mgr.26252) 299 : cluster [DBG] pgmap v507: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-09-13T14:38:30.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[87147]: cluster 2024-09-13T14:38:28.125235+0000 mgr.y (mgr.26252) 299 : cluster [DBG] pgmap v507: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-09-13T14:38:30.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[87147]: audit 2024-09-13T14:38:28.611494+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:30.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[87147]: audit 2024-09-13T14:38:28.611494+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:30.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[87147]: audit 2024-09-13T14:38:29.430194+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.99:0/511731588' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:30.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:29 smithi099 bash[87147]: audit 2024-09-13T14:38:29.430194+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.99:0/511731588' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:30.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:29 smithi177 bash[68594]: cluster 2024-09-13T14:38:28.125235+0000 mgr.y (mgr.26252) 299 : cluster [DBG] pgmap v507: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-09-13T14:38:30.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:29 smithi177 bash[68594]: cluster 2024-09-13T14:38:28.125235+0000 mgr.y (mgr.26252) 299 : cluster [DBG] pgmap v507: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-09-13T14:38:30.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:29 smithi177 bash[68594]: audit 2024-09-13T14:38:28.611494+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:30.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:29 smithi177 bash[68594]: audit 2024-09-13T14:38:28.611494+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:30.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:29 smithi177 bash[68594]: audit 2024-09-13T14:38:29.430194+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.99:0/511731588' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:30.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:29 smithi177 bash[68594]: audit 2024-09-13T14:38:29.430194+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.99:0/511731588' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:30.458 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:30.494 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP (442 ms) 2024-09-13T14:38:30.494 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestIOPPWithIOHint 2024-09-13T14:38:30.494 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:30.564 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:30.564 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:30.564 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:30.564 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:30.564 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:38:30.564 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:30.564 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:30.564 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:30.566 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:30.566 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:30.566 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:30.566 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:30.566 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:30.568 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:30.569 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:30.569 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:30.569 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:30.569 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:30.571 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:30.571 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:30.571 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:30.571 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:30.571 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:30.576 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:30.576 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:30.576 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:30.577 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:38:30.577 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:30.578 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:30.578 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:38:30.578 INFO:tasks.workunit.client.0.smithi099.stdout:read: 512 2024-09-13T14:38:30.578 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:30.578 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:30.578 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:30.579 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:30.579 INFO:tasks.workunit.client.0.smithi099.stdout:finished read 2024-09-13T14:38:30.579 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:30.579 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:30.579 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:30.580 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:30.580 INFO:tasks.workunit.client.0.smithi099.stdout:finished read 2024-09-13T14:38:30.580 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:30.580 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:30.580 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:30.581 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:30.581 INFO:tasks.workunit.client.0.smithi099.stdout:finished read 2024-09-13T14:38:30.581 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:30.581 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:30.581 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:30.582 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:30.582 INFO:tasks.workunit.client.0.smithi099.stdout:finished read 2024-09-13T14:38:30.582 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:30.582 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:38:30.582 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:38:30.607 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:38:30 smithi177 bash[61439]: ts=2024-09-13T14:38:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:31.980 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[87147]: audit 2024-09-13T14:38:29.763196+0000 mgr.y (mgr.26252) 300 : audit [DBG] from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:31.980 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[87147]: audit 2024-09-13T14:38:29.763196+0000 mgr.y (mgr.26252) 300 : audit [DBG] from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:31.980 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[87147]: audit 2024-09-13T14:38:30.085941+0000 mgr.y (mgr.26252) 301 : audit [DBG] from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:31.980 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[87147]: audit 2024-09-13T14:38:30.085941+0000 mgr.y (mgr.26252) 301 : audit [DBG] from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:31.980 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[87147]: cluster 2024-09-13T14:38:30.128639+0000 mgr.y (mgr.26252) 302 : cluster [DBG] pgmap v508: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:31.980 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[87147]: cluster 2024-09-13T14:38:30.128639+0000 mgr.y (mgr.26252) 302 : cluster [DBG] pgmap v508: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:31.980 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[87147]: audit 2024-09-13T14:38:31.318638+0000 mon.a (mon.0) 872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:31.981 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[87147]: audit 2024-09-13T14:38:31.318638+0000 mon.a (mon.0) 872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:31.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[84576]: audit 2024-09-13T14:38:29.763196+0000 mgr.y (mgr.26252) 300 : audit [DBG] from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:31.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[84576]: audit 2024-09-13T14:38:29.763196+0000 mgr.y (mgr.26252) 300 : audit [DBG] from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:31.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[84576]: audit 2024-09-13T14:38:30.085941+0000 mgr.y (mgr.26252) 301 : audit [DBG] from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:31.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[84576]: audit 2024-09-13T14:38:30.085941+0000 mgr.y (mgr.26252) 301 : audit [DBG] from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:31.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[84576]: cluster 2024-09-13T14:38:30.128639+0000 mgr.y (mgr.26252) 302 : cluster [DBG] pgmap v508: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:31.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[84576]: cluster 2024-09-13T14:38:30.128639+0000 mgr.y (mgr.26252) 302 : cluster [DBG] pgmap v508: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:31.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[84576]: audit 2024-09-13T14:38:31.318638+0000 mon.a (mon.0) 872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:31.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:31 smithi099 bash[84576]: audit 2024-09-13T14:38:31.318638+0000 mon.a (mon.0) 872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:32.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:31 smithi177 bash[68594]: audit 2024-09-13T14:38:29.763196+0000 mgr.y (mgr.26252) 300 : audit [DBG] from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:32.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:31 smithi177 bash[68594]: audit 2024-09-13T14:38:29.763196+0000 mgr.y (mgr.26252) 300 : audit [DBG] from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:32.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:31 smithi177 bash[68594]: audit 2024-09-13T14:38:30.085941+0000 mgr.y (mgr.26252) 301 : audit [DBG] from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:32.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:31 smithi177 bash[68594]: audit 2024-09-13T14:38:30.085941+0000 mgr.y (mgr.26252) 301 : audit [DBG] from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:32.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:31 smithi177 bash[68594]: cluster 2024-09-13T14:38:30.128639+0000 mgr.y (mgr.26252) 302 : cluster [DBG] pgmap v508: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:32.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:31 smithi177 bash[68594]: cluster 2024-09-13T14:38:30.128639+0000 mgr.y (mgr.26252) 302 : cluster [DBG] pgmap v508: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:32.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:31 smithi177 bash[68594]: audit 2024-09-13T14:38:31.318638+0000 mon.a (mon.0) 872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:32.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:31 smithi177 bash[68594]: audit 2024-09-13T14:38:31.318638+0000 mon.a (mon.0) 872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:32.310 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:38:32.310 INFO:tasks.workunit.client.0.smithi099.stdout:finished read 2024-09-13T14:38:32.310 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:38:32.310 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.310 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 0 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 16384 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:38:32.311 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 49152 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 65536 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:38:32.312 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 98304 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 114688 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:38:32.313 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 147456 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 163840 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.314 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 196608 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 212992 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.315 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.316 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.317 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.317 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.317 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.317 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.317 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.317 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.323 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.323 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.323 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.323 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.323 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.324 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.324 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.332 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.332 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.333 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.358 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.358 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.359 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.359 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.359 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.359 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.359 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.386 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.386 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.386 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.386 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.386 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.386 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.386 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.416 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.417 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.417 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.417 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.417 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.417 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.417 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.448 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.448 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.448 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.448 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.448 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.449 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.449 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.449 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.449 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.449 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.449 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.449 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.449 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.498 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.498 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.498 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.498 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.498 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.498 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.498 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.541 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.541 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.541 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.541 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.541 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.541 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.541 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.602 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.602 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.603 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.603 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.603 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.603 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.603 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.664 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.664 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.665 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.763 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.764 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.764 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.764 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.764 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.764 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.764 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.822 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:32.829 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.829 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.829 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.829 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.829 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.829 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.829 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.917 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.918 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.918 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.918 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.918 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.918 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.918 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:32.984 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:32.985 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:32.985 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:33.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[84576]: audit 2024-09-13T14:38:31.319071+0000 mgr.y (mgr.26252) 303 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[84576]: audit 2024-09-13T14:38:31.319071+0000 mgr.y (mgr.26252) 303 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[84576]: cephadm 2024-09-13T14:38:31.323403+0000 mgr.y (mgr.26252) 304 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[84576]: cephadm 2024-09-13T14:38:31.323403+0000 mgr.y (mgr.26252) 304 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[84576]: audit 2024-09-13T14:38:32.440382+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.99:0/1459730339' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[84576]: audit 2024-09-13T14:38:32.440382+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.99:0/1459730339' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[87147]: audit 2024-09-13T14:38:31.319071+0000 mgr.y (mgr.26252) 303 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[87147]: audit 2024-09-13T14:38:31.319071+0000 mgr.y (mgr.26252) 303 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[87147]: cephadm 2024-09-13T14:38:31.323403+0000 mgr.y (mgr.26252) 304 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-13T14:38:33.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[87147]: cephadm 2024-09-13T14:38:31.323403+0000 mgr.y (mgr.26252) 304 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-13T14:38:33.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[87147]: audit 2024-09-13T14:38:32.440382+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.99:0/1459730339' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:33.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:32 smithi099 bash[87147]: audit 2024-09-13T14:38:32.440382+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.99:0/1459730339' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:33.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:32 smithi177 bash[68594]: audit 2024-09-13T14:38:31.319071+0000 mgr.y (mgr.26252) 303 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:33.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:32 smithi177 bash[68594]: audit 2024-09-13T14:38:31.319071+0000 mgr.y (mgr.26252) 303 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:33.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:32 smithi177 bash[68594]: cephadm 2024-09-13T14:38:31.323403+0000 mgr.y (mgr.26252) 304 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-13T14:38:33.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:32 smithi177 bash[68594]: cephadm 2024-09-13T14:38:31.323403+0000 mgr.y (mgr.26252) 304 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-13T14:38:33.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:32 smithi177 bash[68594]: audit 2024-09-13T14:38:32.440382+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.99:0/1459730339' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:33.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:32 smithi177 bash[68594]: audit 2024-09-13T14:38:32.440382+0000 mon.a (mon.0) 873 : audit [DBG] from='client.? 172.21.15.99:0/1459730339' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:33.109 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:33.109 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:33.109 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:33.109 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:33.109 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:33.109 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:33.109 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:33.246 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:33.247 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:33.247 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:33.247 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:33.247 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:33.247 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:33.247 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:33.424 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:33.424 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:33.424 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:33.425 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:33.425 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:33.425 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:33.425 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:38:33.617 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:38:33.617 INFO:tasks.workunit.client.0.smithi099.stdout:finished writesame 2024-09-13T14:38:33.617 INFO:tasks.workunit.client.0.smithi099.stdout:to verify the data 2024-09-13T14:38:33.617 INFO:tasks.workunit.client.0.smithi099.stdout:verified 2024-09-13T14:38:33.618 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:33.618 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:33.618 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:33.618 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:38:33.618 INFO:tasks.workunit.client.0.smithi099.stdout:started writesame 2024-09-13T14:38:33.618 INFO:tasks.workunit.client.0.smithi099.stdout:expected fail, finished writesame 2024-09-13T14:38:33.618 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestIOPPWithIOHint (3123 ms) 2024-09-13T14:38:33.618 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestIOToSnapshot 2024-09-13T14:38:33.618 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[84576]: cluster 2024-09-13T14:38:32.129184+0000 mgr.y (mgr.26252) 305 : cluster [DBG] pgmap v509: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[84576]: cluster 2024-09-13T14:38:32.129184+0000 mgr.y (mgr.26252) 305 : cluster [DBG] pgmap v509: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[84576]: audit 2024-09-13T14:38:33.256587+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[84576]: audit 2024-09-13T14:38:33.256587+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[84576]: audit 2024-09-13T14:38:33.258470+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[84576]: audit 2024-09-13T14:38:33.258470+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:34.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[87147]: cluster 2024-09-13T14:38:32.129184+0000 mgr.y (mgr.26252) 305 : cluster [DBG] pgmap v509: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:34.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[87147]: cluster 2024-09-13T14:38:32.129184+0000 mgr.y (mgr.26252) 305 : cluster [DBG] pgmap v509: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:34.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[87147]: audit 2024-09-13T14:38:33.256587+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:34.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[87147]: audit 2024-09-13T14:38:33.256587+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:34.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[87147]: audit 2024-09-13T14:38:33.258470+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:34.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:33 smithi099 bash[87147]: audit 2024-09-13T14:38:33.258470+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:33 smithi177 bash[68594]: cluster 2024-09-13T14:38:32.129184+0000 mgr.y (mgr.26252) 305 : cluster [DBG] pgmap v509: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:33 smithi177 bash[68594]: cluster 2024-09-13T14:38:32.129184+0000 mgr.y (mgr.26252) 305 : cluster [DBG] pgmap v509: 457 pgs: 457 active+clean; 83 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.01k op/s 2024-09-13T14:38:34.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:33 smithi177 bash[68594]: audit 2024-09-13T14:38:33.256587+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:34.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:33 smithi177 bash[68594]: audit 2024-09-13T14:38:33.256587+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:34.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:33 smithi177 bash[68594]: audit 2024-09-13T14:38:33.258470+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:34.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:33 smithi177 bash[68594]: audit 2024-09-13T14:38:33.258470+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:34.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:34 smithi099 bash[84576]: audit 2024-09-13T14:38:32.812876+0000 mgr.y (mgr.26252) 306 : audit [DBG] from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:34.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:34 smithi099 bash[84576]: audit 2024-09-13T14:38:32.812876+0000 mgr.y (mgr.26252) 306 : audit [DBG] from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:34.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:34 smithi099 bash[84576]: audit 2024-09-13T14:38:33.137229+0000 mgr.y (mgr.26252) 307 : audit [DBG] from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:34.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:34 smithi099 bash[84576]: audit 2024-09-13T14:38:33.137229+0000 mgr.y (mgr.26252) 307 : audit [DBG] from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:34.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:34 smithi099 bash[87147]: audit 2024-09-13T14:38:32.812876+0000 mgr.y (mgr.26252) 306 : audit [DBG] from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:34.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:34 smithi099 bash[87147]: audit 2024-09-13T14:38:32.812876+0000 mgr.y (mgr.26252) 306 : audit [DBG] from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:34.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:34 smithi099 bash[87147]: audit 2024-09-13T14:38:33.137229+0000 mgr.y (mgr.26252) 307 : audit [DBG] from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:34.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:34 smithi099 bash[87147]: audit 2024-09-13T14:38:33.137229+0000 mgr.y (mgr.26252) 307 : audit [DBG] from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:35.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:34 smithi177 bash[68594]: audit 2024-09-13T14:38:32.812876+0000 mgr.y (mgr.26252) 306 : audit [DBG] from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:35.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:34 smithi177 bash[68594]: audit 2024-09-13T14:38:32.812876+0000 mgr.y (mgr.26252) 306 : audit [DBG] from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:35.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:34 smithi177 bash[68594]: audit 2024-09-13T14:38:33.137229+0000 mgr.y (mgr.26252) 307 : audit [DBG] from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:35.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:34 smithi177 bash[68594]: audit 2024-09-13T14:38:33.137229+0000 mgr.y (mgr.26252) 307 : audit [DBG] from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:35.703 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 0 2024-09-13T14:38:35.703 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 0 2024-09-13T14:38:35.703 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 1 2024-09-13T14:38:35.703 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:38:35.703 INFO:tasks.workunit.client.0.smithi099.stdout:snap: orig 2024-09-13T14:38:35.703 INFO:tasks.workunit.client.0.smithi099.stdout:found orig with size 2097152 2024-09-13T14:38:35.703 INFO:tasks.workunit.client.0.smithi099.stdout:read: 80 2024-09-13T14:38:35.703 INFO:tasks.workunit.client.0.smithi099.stdout:write test data! 2024-09-13T14:38:35.703 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 80 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 2 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:snap: orig 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:snap: written 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:found orig with size 2097152 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:found written with size 2097152 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:read: 80 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:read: 80 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:read: 80 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:write to snapshot returned -30 2024-09-13T14:38:35.704 INFO:tasks.workunit.client.0.smithi099.stdout:Read-only file system 2024-09-13T14:38:35.749 INFO:tasks.workunit.client.0.smithi099.stdout:read: 80 2024-09-13T14:38:35.749 INFO:tasks.workunit.client.0.smithi099.stdout:read: 80 2024-09-13T14:38:35.749 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 80 2024-09-13T14:38:35.749 INFO:tasks.workunit.client.0.smithi099.stdout:opening testimg@orig 2024-09-13T14:38:35.750 INFO:tasks.workunit.client.0.smithi099.stdout:read: 80 2024-09-13T14:38:35.750 INFO:tasks.workunit.client.0.smithi099.stdout:write to snapshot returned -30 2024-09-13T14:38:35.750 INFO:tasks.workunit.client.0.smithi099.stdout:Read-only file system 2024-09-13T14:38:35.800 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: cluster 2024-09-13T14:38:34.131763+0000 mgr.y (mgr.26252) 308 : cluster [DBG] pgmap v510: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.06k op/s 2024-09-13T14:38:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: cluster 2024-09-13T14:38:34.131763+0000 mgr.y (mgr.26252) 308 : cluster [DBG] pgmap v510: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.06k op/s 2024-09-13T14:38:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: cluster 2024-09-13T14:38:34.645378+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-13T14:38:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: cluster 2024-09-13T14:38:34.645378+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-13T14:38:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: audit 2024-09-13T14:38:35.170096+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: audit 2024-09-13T14:38:35.170096+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: audit 2024-09-13T14:38:35.173172+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: audit 2024-09-13T14:38:35.173172+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: audit 2024-09-13T14:38:35.174126+0000 mon.a (mon.0) 879 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: audit 2024-09-13T14:38:35.174126+0000 mon.a (mon.0) 879 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: audit 2024-09-13T14:38:35.452704+0000 mon.a (mon.0) 880 : audit [DBG] from='client.? 172.21.15.99:0/2290386350' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[87147]: audit 2024-09-13T14:38:35.452704+0000 mon.a (mon.0) 880 : audit [DBG] from='client.? 172.21.15.99:0/2290386350' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: cluster 2024-09-13T14:38:34.131763+0000 mgr.y (mgr.26252) 308 : cluster [DBG] pgmap v510: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.06k op/s 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: cluster 2024-09-13T14:38:34.131763+0000 mgr.y (mgr.26252) 308 : cluster [DBG] pgmap v510: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.06k op/s 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: cluster 2024-09-13T14:38:34.645378+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: cluster 2024-09-13T14:38:34.645378+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: audit 2024-09-13T14:38:35.170096+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: audit 2024-09-13T14:38:35.170096+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: audit 2024-09-13T14:38:35.173172+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:38:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: audit 2024-09-13T14:38:35.173172+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:38:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: audit 2024-09-13T14:38:35.174126+0000 mon.a (mon.0) 879 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: audit 2024-09-13T14:38:35.174126+0000 mon.a (mon.0) 879 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: audit 2024-09-13T14:38:35.452704+0000 mon.a (mon.0) 880 : audit [DBG] from='client.? 172.21.15.99:0/2290386350' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:35 smithi099 bash[84576]: audit 2024-09-13T14:38:35.452704+0000 mon.a (mon.0) 880 : audit [DBG] from='client.? 172.21.15.99:0/2290386350' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:36.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: cluster 2024-09-13T14:38:34.131763+0000 mgr.y (mgr.26252) 308 : cluster [DBG] pgmap v510: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.06k op/s 2024-09-13T14:38:36.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: cluster 2024-09-13T14:38:34.131763+0000 mgr.y (mgr.26252) 308 : cluster [DBG] pgmap v510: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.06k op/s 2024-09-13T14:38:36.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: cluster 2024-09-13T14:38:34.645378+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-13T14:38:36.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: cluster 2024-09-13T14:38:34.645378+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-13T14:38:36.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: audit 2024-09-13T14:38:35.170096+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:36.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: audit 2024-09-13T14:38:35.170096+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:36.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: audit 2024-09-13T14:38:35.173172+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:38:36.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: audit 2024-09-13T14:38:35.173172+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:38:36.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: audit 2024-09-13T14:38:35.174126+0000 mon.a (mon.0) 879 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:36.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: audit 2024-09-13T14:38:35.174126+0000 mon.a (mon.0) 879 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:36.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: audit 2024-09-13T14:38:35.452704+0000 mon.a (mon.0) 880 : audit [DBG] from='client.? 172.21.15.99:0/2290386350' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:36.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:35 smithi177 bash[68594]: audit 2024-09-13T14:38:35.452704+0000 mon.a (mon.0) 880 : audit [DBG] from='client.? 172.21.15.99:0/2290386350' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:37.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[87147]: cephadm 2024-09-13T14:38:35.164839+0000 mgr.y (mgr.26252) 309 : cephadm [INF] Upgrade: Updating osd.2 2024-09-13T14:38:37.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[87147]: cephadm 2024-09-13T14:38:35.164839+0000 mgr.y (mgr.26252) 309 : cephadm [INF] Upgrade: Updating osd.2 2024-09-13T14:38:37.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[87147]: cephadm 2024-09-13T14:38:35.180237+0000 mgr.y (mgr.26252) 310 : cephadm [INF] Deploying daemon osd.2 on smithi099 2024-09-13T14:38:37.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[87147]: cephadm 2024-09-13T14:38:35.180237+0000 mgr.y (mgr.26252) 310 : cephadm [INF] Deploying daemon osd.2 on smithi099 2024-09-13T14:38:37.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[87147]: cluster 2024-09-13T14:38:35.647737+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-13T14:38:37.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[87147]: cluster 2024-09-13T14:38:35.647737+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-13T14:38:37.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[84576]: cephadm 2024-09-13T14:38:35.164839+0000 mgr.y (mgr.26252) 309 : cephadm [INF] Upgrade: Updating osd.2 2024-09-13T14:38:37.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[84576]: cephadm 2024-09-13T14:38:35.164839+0000 mgr.y (mgr.26252) 309 : cephadm [INF] Upgrade: Updating osd.2 2024-09-13T14:38:37.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[84576]: cephadm 2024-09-13T14:38:35.180237+0000 mgr.y (mgr.26252) 310 : cephadm [INF] Deploying daemon osd.2 on smithi099 2024-09-13T14:38:37.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[84576]: cephadm 2024-09-13T14:38:35.180237+0000 mgr.y (mgr.26252) 310 : cephadm [INF] Deploying daemon osd.2 on smithi099 2024-09-13T14:38:37.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[84576]: cluster 2024-09-13T14:38:35.647737+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-13T14:38:37.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:36 smithi099 bash[84576]: cluster 2024-09-13T14:38:35.647737+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-13T14:38:37.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:36 smithi177 bash[68594]: cephadm 2024-09-13T14:38:35.164839+0000 mgr.y (mgr.26252) 309 : cephadm [INF] Upgrade: Updating osd.2 2024-09-13T14:38:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:36 smithi177 bash[68594]: cephadm 2024-09-13T14:38:35.164839+0000 mgr.y (mgr.26252) 309 : cephadm [INF] Upgrade: Updating osd.2 2024-09-13T14:38:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:36 smithi177 bash[68594]: cephadm 2024-09-13T14:38:35.180237+0000 mgr.y (mgr.26252) 310 : cephadm [INF] Deploying daemon osd.2 on smithi099 2024-09-13T14:38:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:36 smithi177 bash[68594]: cephadm 2024-09-13T14:38:35.180237+0000 mgr.y (mgr.26252) 310 : cephadm [INF] Deploying daemon osd.2 on smithi099 2024-09-13T14:38:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:36 smithi177 bash[68594]: cluster 2024-09-13T14:38:35.647737+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-13T14:38:37.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:36 smithi177 bash[68594]: cluster 2024-09-13T14:38:35.647737+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-13T14:38:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:38:37] "GET /metrics HTTP/1.1" 200 63108 "" "Prometheus/2.51.0" 2024-09-13T14:38:37.737 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 2 2024-09-13T14:38:37.737 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:38:37.737 INFO:tasks.workunit.client.0.smithi099.stdout:snap: orig 2024-09-13T14:38:37.737 INFO:tasks.workunit.client.0.smithi099.stdout:snap: written 2024-09-13T14:38:37.737 INFO:tasks.workunit.client.0.smithi099.stdout:found orig with size 2097152 2024-09-13T14:38:37.737 INFO:tasks.workunit.client.0.smithi099.stdout:found written with size 2097152 2024-09-13T14:38:37.737 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 1 2024-09-13T14:38:37.737 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:38:37.737 INFO:tasks.workunit.client.0.smithi099.stdout:snap: orig 2024-09-13T14:38:37.737 INFO:tasks.workunit.client.0.smithi099.stdout:found orig with size 2097152 2024-09-13T14:38:37.738 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 0 2024-09-13T14:38:37.738 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 0 2024-09-13T14:38:37.738 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestIOToSnapshot (4120 ms) 2024-09-13T14:38:37.738 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestSnapshotDeletedIo 2024-09-13T14:38:37.738 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[84576]: audit 2024-09-13T14:38:35.791175+0000 mgr.y (mgr.26252) 311 : audit [DBG] from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[84576]: audit 2024-09-13T14:38:35.791175+0000 mgr.y (mgr.26252) 311 : audit [DBG] from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[84576]: audit 2024-09-13T14:38:36.098637+0000 mgr.y (mgr.26252) 312 : audit [DBG] from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[84576]: audit 2024-09-13T14:38:36.098637+0000 mgr.y (mgr.26252) 312 : audit [DBG] from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[84576]: cluster 2024-09-13T14:38:36.132397+0000 mgr.y (mgr.26252) 313 : cluster [DBG] pgmap v513: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.33k op/s 2024-09-13T14:38:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[84576]: cluster 2024-09-13T14:38:36.132397+0000 mgr.y (mgr.26252) 313 : cluster [DBG] pgmap v513: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.33k op/s 2024-09-13T14:38:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[84576]: cluster 2024-09-13T14:38:36.672824+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-13T14:38:37.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[84576]: cluster 2024-09-13T14:38:36.672824+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-13T14:38:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[87147]: audit 2024-09-13T14:38:35.791175+0000 mgr.y (mgr.26252) 311 : audit [DBG] from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[87147]: audit 2024-09-13T14:38:35.791175+0000 mgr.y (mgr.26252) 311 : audit [DBG] from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[87147]: audit 2024-09-13T14:38:36.098637+0000 mgr.y (mgr.26252) 312 : audit [DBG] from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[87147]: audit 2024-09-13T14:38:36.098637+0000 mgr.y (mgr.26252) 312 : audit [DBG] from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[87147]: cluster 2024-09-13T14:38:36.132397+0000 mgr.y (mgr.26252) 313 : cluster [DBG] pgmap v513: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.33k op/s 2024-09-13T14:38:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[87147]: cluster 2024-09-13T14:38:36.132397+0000 mgr.y (mgr.26252) 313 : cluster [DBG] pgmap v513: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.33k op/s 2024-09-13T14:38:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[87147]: cluster 2024-09-13T14:38:36.672824+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-13T14:38:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:37 smithi099 bash[87147]: cluster 2024-09-13T14:38:36.672824+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-13T14:38:38.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:37 smithi177 bash[68594]: audit 2024-09-13T14:38:35.791175+0000 mgr.y (mgr.26252) 311 : audit [DBG] from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:38.176 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:37 smithi177 bash[68594]: audit 2024-09-13T14:38:35.791175+0000 mgr.y (mgr.26252) 311 : audit [DBG] from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:38.176 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:37 smithi177 bash[68594]: audit 2024-09-13T14:38:36.098637+0000 mgr.y (mgr.26252) 312 : audit [DBG] from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:38.177 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:37 smithi177 bash[68594]: audit 2024-09-13T14:38:36.098637+0000 mgr.y (mgr.26252) 312 : audit [DBG] from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:38.177 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:37 smithi177 bash[68594]: cluster 2024-09-13T14:38:36.132397+0000 mgr.y (mgr.26252) 313 : cluster [DBG] pgmap v513: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.33k op/s 2024-09-13T14:38:38.177 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:37 smithi177 bash[68594]: cluster 2024-09-13T14:38:36.132397+0000 mgr.y (mgr.26252) 313 : cluster [DBG] pgmap v513: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.33k op/s 2024-09-13T14:38:38.177 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:37 smithi177 bash[68594]: cluster 2024-09-13T14:38:36.672824+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-13T14:38:38.177 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:37 smithi177 bash[68594]: cluster 2024-09-13T14:38:36.672824+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-13T14:38:38.731 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:39.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:38 smithi099 bash[84576]: cluster 2024-09-13T14:38:37.673205+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-13T14:38:39.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:38 smithi099 bash[84576]: cluster 2024-09-13T14:38:37.673205+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-13T14:38:39.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:38 smithi099 bash[84576]: audit 2024-09-13T14:38:38.391491+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.99:0/1562467370' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:39.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:38 smithi099 bash[84576]: audit 2024-09-13T14:38:38.391491+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.99:0/1562467370' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:38 smithi099 bash[87147]: cluster 2024-09-13T14:38:37.673205+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-13T14:38:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:38 smithi099 bash[87147]: cluster 2024-09-13T14:38:37.673205+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-13T14:38:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:38 smithi099 bash[87147]: audit 2024-09-13T14:38:38.391491+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.99:0/1562467370' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:38 smithi099 bash[87147]: audit 2024-09-13T14:38:38.391491+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.99:0/1562467370' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:39.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:38 smithi177 bash[68594]: cluster 2024-09-13T14:38:37.673205+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-13T14:38:39.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:38 smithi177 bash[68594]: cluster 2024-09-13T14:38:37.673205+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-13T14:38:39.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:38 smithi177 bash[68594]: audit 2024-09-13T14:38:38.391491+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.99:0/1562467370' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:39.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:38 smithi177 bash[68594]: audit 2024-09-13T14:38:38.391491+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.99:0/1562467370' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:39.722 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestSnapshotDeletedIo (1985 ms) 2024-09-13T14:38:39.722 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestClone 2024-09-13T14:38:39.723 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:39.851 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[84576]: cluster 2024-09-13T14:38:38.132942+0000 mgr.y (mgr.26252) 314 : cluster [DBG] pgmap v516: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 244 KiB/s wr, 999 op/s 2024-09-13T14:38:39.851 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[84576]: cluster 2024-09-13T14:38:38.132942+0000 mgr.y (mgr.26252) 314 : cluster [DBG] pgmap v516: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 244 KiB/s wr, 999 op/s 2024-09-13T14:38:39.851 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[84576]: cluster 2024-09-13T14:38:38.700077+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-13T14:38:39.851 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[84576]: cluster 2024-09-13T14:38:38.700077+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-13T14:38:39.851 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[84576]: audit 2024-09-13T14:38:38.715421+0000 mgr.y (mgr.26252) 315 : audit [DBG] from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:39.851 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[84576]: audit 2024-09-13T14:38:38.715421+0000 mgr.y (mgr.26252) 315 : audit [DBG] from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:39.852 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[87147]: cluster 2024-09-13T14:38:38.132942+0000 mgr.y (mgr.26252) 314 : cluster [DBG] pgmap v516: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 244 KiB/s wr, 999 op/s 2024-09-13T14:38:39.852 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[87147]: cluster 2024-09-13T14:38:38.132942+0000 mgr.y (mgr.26252) 314 : cluster [DBG] pgmap v516: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 244 KiB/s wr, 999 op/s 2024-09-13T14:38:39.852 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[87147]: cluster 2024-09-13T14:38:38.700077+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-13T14:38:39.852 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[87147]: cluster 2024-09-13T14:38:38.700077+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-13T14:38:39.852 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[87147]: audit 2024-09-13T14:38:38.715421+0000 mgr.y (mgr.26252) 315 : audit [DBG] from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:39.852 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:39 smithi099 bash[87147]: audit 2024-09-13T14:38:38.715421+0000 mgr.y (mgr.26252) 315 : audit [DBG] from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:40.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:39 smithi177 bash[68594]: cluster 2024-09-13T14:38:38.132942+0000 mgr.y (mgr.26252) 314 : cluster [DBG] pgmap v516: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 244 KiB/s wr, 999 op/s 2024-09-13T14:38:40.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:39 smithi177 bash[68594]: cluster 2024-09-13T14:38:38.132942+0000 mgr.y (mgr.26252) 314 : cluster [DBG] pgmap v516: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 244 KiB/s wr, 999 op/s 2024-09-13T14:38:40.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:39 smithi177 bash[68594]: cluster 2024-09-13T14:38:38.700077+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-13T14:38:40.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:39 smithi177 bash[68594]: cluster 2024-09-13T14:38:38.700077+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-13T14:38:40.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:39 smithi177 bash[68594]: audit 2024-09-13T14:38:38.715421+0000 mgr.y (mgr.26252) 315 : audit [DBG] from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:40.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:39 smithi177 bash[68594]: audit 2024-09-13T14:38:38.715421+0000 mgr.y (mgr.26252) 315 : audit [DBG] from='client.44489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:40.607 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:38:40 smithi177 bash[61439]: ts=2024-09-13T14:38:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:41.013 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:40 smithi099 bash[84576]: audit 2024-09-13T14:38:39.034305+0000 mgr.y (mgr.26252) 316 : audit [DBG] from='client.34605 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.013 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:40 smithi099 bash[84576]: audit 2024-09-13T14:38:39.034305+0000 mgr.y (mgr.26252) 316 : audit [DBG] from='client.34605 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.013 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:40 smithi099 bash[84576]: cluster 2024-09-13T14:38:39.697914+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-13T14:38:41.013 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:40 smithi099 bash[84576]: cluster 2024-09-13T14:38:39.697914+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-13T14:38:41.013 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:40 smithi099 bash[87147]: audit 2024-09-13T14:38:39.034305+0000 mgr.y (mgr.26252) 316 : audit [DBG] from='client.34605 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.014 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:40 smithi099 bash[87147]: audit 2024-09-13T14:38:39.034305+0000 mgr.y (mgr.26252) 316 : audit [DBG] from='client.34605 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.014 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:40 smithi099 bash[87147]: cluster 2024-09-13T14:38:39.697914+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-13T14:38:41.014 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:40 smithi099 bash[87147]: cluster 2024-09-13T14:38:39.697914+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-13T14:38:41.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:40 smithi177 bash[68594]: audit 2024-09-13T14:38:39.034305+0000 mgr.y (mgr.26252) 316 : audit [DBG] from='client.34605 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:40 smithi177 bash[68594]: audit 2024-09-13T14:38:39.034305+0000 mgr.y (mgr.26252) 316 : audit [DBG] from='client.34605 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:40 smithi177 bash[68594]: cluster 2024-09-13T14:38:39.697914+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-13T14:38:41.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:40 smithi177 bash[68594]: cluster 2024-09-13T14:38:39.697914+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-13T14:38:41.286 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:38:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:41.286 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:38:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:41.287 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:38:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:41.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:41.287 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:38:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:41.288 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:38:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:41.288 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:41.288 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:41 smithi099 systemd[1]: Stopping Ceph osd.2 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:38:41.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:41.289 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:38:41 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:41.572 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[46182]: debug 2024-09-13T14:38:41.283+0000 7feb59a40640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T14:38:41.572 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[46182]: debug 2024-09-13T14:38:41.283+0000 7feb59a40640 -1 osd.2 656 *** Got signal Terminated *** 2024-09-13T14:38:41.572 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[46182]: debug 2024-09-13T14:38:41.283+0000 7feb59a40640 -1 osd.2 656 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:38:41.684 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[84576]: cluster 2024-09-13T14:38:40.133979+0000 mgr.y (mgr.26252) 317 : cluster [DBG] pgmap v519: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 556 KiB/s rd, 252 KiB/s wr, 1.09k op/s 2024-09-13T14:38:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[84576]: cluster 2024-09-13T14:38:40.133979+0000 mgr.y (mgr.26252) 317 : cluster [DBG] pgmap v519: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 556 KiB/s rd, 252 KiB/s wr, 1.09k op/s 2024-09-13T14:38:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[84576]: cluster 2024-09-13T14:38:40.718440+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-13T14:38:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[84576]: cluster 2024-09-13T14:38:40.718440+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-13T14:38:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[84576]: cluster 2024-09-13T14:38:41.095499+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-13T14:38:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[84576]: cluster 2024-09-13T14:38:41.095499+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-13T14:38:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[84576]: cluster 2024-09-13T14:38:41.289334+0000 mon.a (mon.0) 888 : cluster [INF] osd.2 marked itself down and dead 2024-09-13T14:38:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[84576]: cluster 2024-09-13T14:38:41.289334+0000 mon.a (mon.0) 888 : cluster [INF] osd.2 marked itself down and dead 2024-09-13T14:38:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[84576]: audit 2024-09-13T14:38:41.355577+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.99:0/3079130168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[84576]: audit 2024-09-13T14:38:41.355577+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.99:0/3079130168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[87147]: cluster 2024-09-13T14:38:40.133979+0000 mgr.y (mgr.26252) 317 : cluster [DBG] pgmap v519: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 556 KiB/s rd, 252 KiB/s wr, 1.09k op/s 2024-09-13T14:38:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[87147]: cluster 2024-09-13T14:38:40.133979+0000 mgr.y (mgr.26252) 317 : cluster [DBG] pgmap v519: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 556 KiB/s rd, 252 KiB/s wr, 1.09k op/s 2024-09-13T14:38:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[87147]: cluster 2024-09-13T14:38:40.718440+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-13T14:38:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[87147]: cluster 2024-09-13T14:38:40.718440+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-13T14:38:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[87147]: cluster 2024-09-13T14:38:41.095499+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-13T14:38:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[87147]: cluster 2024-09-13T14:38:41.095499+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-13T14:38:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[87147]: cluster 2024-09-13T14:38:41.289334+0000 mon.a (mon.0) 888 : cluster [INF] osd.2 marked itself down and dead 2024-09-13T14:38:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[87147]: cluster 2024-09-13T14:38:41.289334+0000 mon.a (mon.0) 888 : cluster [INF] osd.2 marked itself down and dead 2024-09-13T14:38:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[87147]: audit 2024-09-13T14:38:41.355577+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.99:0/3079130168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:41 smithi099 bash[87147]: audit 2024-09-13T14:38:41.355577+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.99:0/3079130168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:42.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:41 smithi177 bash[68594]: cluster 2024-09-13T14:38:40.133979+0000 mgr.y (mgr.26252) 317 : cluster [DBG] pgmap v519: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 556 KiB/s rd, 252 KiB/s wr, 1.09k op/s 2024-09-13T14:38:42.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:41 smithi177 bash[68594]: cluster 2024-09-13T14:38:40.133979+0000 mgr.y (mgr.26252) 317 : cluster [DBG] pgmap v519: 457 pgs: 457 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 556 KiB/s rd, 252 KiB/s wr, 1.09k op/s 2024-09-13T14:38:42.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:41 smithi177 bash[68594]: cluster 2024-09-13T14:38:40.718440+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-13T14:38:42.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:41 smithi177 bash[68594]: cluster 2024-09-13T14:38:40.718440+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-13T14:38:42.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:41 smithi177 bash[68594]: cluster 2024-09-13T14:38:41.095499+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-13T14:38:42.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:41 smithi177 bash[68594]: cluster 2024-09-13T14:38:41.095499+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-13T14:38:42.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:41 smithi177 bash[68594]: cluster 2024-09-13T14:38:41.289334+0000 mon.a (mon.0) 888 : cluster [INF] osd.2 marked itself down and dead 2024-09-13T14:38:42.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:41 smithi177 bash[68594]: cluster 2024-09-13T14:38:41.289334+0000 mon.a (mon.0) 888 : cluster [INF] osd.2 marked itself down and dead 2024-09-13T14:38:42.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:41 smithi177 bash[68594]: audit 2024-09-13T14:38:41.355577+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.99:0/3079130168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:42.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:41 smithi177 bash[68594]: audit 2024-09-13T14:38:41.355577+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.99:0/3079130168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:43.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:42 smithi177 bash[68594]: audit 2024-09-13T14:38:41.674005+0000 mgr.y (mgr.26252) 318 : audit [DBG] from='client.44507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:43.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:42 smithi177 bash[68594]: audit 2024-09-13T14:38:41.674005+0000 mgr.y (mgr.26252) 318 : audit [DBG] from='client.44507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:43.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:42 smithi177 bash[68594]: cluster 2024-09-13T14:38:42.091017+0000 mon.a (mon.0) 889 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:38:43.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:42 smithi177 bash[68594]: cluster 2024-09-13T14:38:42.091017+0000 mon.a (mon.0) 889 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:38:43.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:42 smithi177 bash[68594]: cluster 2024-09-13T14:38:42.102903+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e657: 8 total, 7 up, 8 in 2024-09-13T14:38:43.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:42 smithi177 bash[68594]: cluster 2024-09-13T14:38:42.102903+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e657: 8 total, 7 up, 8 in 2024-09-13T14:38:43.209 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[84576]: audit 2024-09-13T14:38:41.674005+0000 mgr.y (mgr.26252) 318 : audit [DBG] from='client.44507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:43.210 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[84576]: audit 2024-09-13T14:38:41.674005+0000 mgr.y (mgr.26252) 318 : audit [DBG] from='client.44507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:43.210 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[84576]: cluster 2024-09-13T14:38:42.091017+0000 mon.a (mon.0) 889 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:38:43.210 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[84576]: cluster 2024-09-13T14:38:42.091017+0000 mon.a (mon.0) 889 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:38:43.210 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[84576]: cluster 2024-09-13T14:38:42.102903+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e657: 8 total, 7 up, 8 in 2024-09-13T14:38:43.210 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[84576]: cluster 2024-09-13T14:38:42.102903+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e657: 8 total, 7 up, 8 in 2024-09-13T14:38:43.210 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[87147]: audit 2024-09-13T14:38:41.674005+0000 mgr.y (mgr.26252) 318 : audit [DBG] from='client.44507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:43.210 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[87147]: audit 2024-09-13T14:38:41.674005+0000 mgr.y (mgr.26252) 318 : audit [DBG] from='client.44507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:43.210 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[87147]: cluster 2024-09-13T14:38:42.091017+0000 mon.a (mon.0) 889 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:38:43.210 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[87147]: cluster 2024-09-13T14:38:42.091017+0000 mon.a (mon.0) 889 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:38:43.210 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[87147]: cluster 2024-09-13T14:38:42.102903+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e657: 8 total, 7 up, 8 in 2024-09-13T14:38:43.211 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:42 smithi099 bash[87147]: cluster 2024-09-13T14:38:42.102903+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e657: 8 total, 7 up, 8 in 2024-09-13T14:38:43.289 INFO:tasks.workunit.client.0.smithi099.stdout:made parent image "parent" 2024-09-13T14:38:43.289 INFO:tasks.workunit.client.0.smithi099.stdout:parent has no parent info 2024-09-13T14:38:43.289 INFO:tasks.workunit.client.0.smithi099.stdout:made snapshot "parent@parent_snap" 2024-09-13T14:38:43.289 INFO:tasks.workunit.client.0.smithi099.stdout:can't unprotect an unprotected snap 2024-09-13T14:38:43.289 INFO:tasks.workunit.client.0.smithi099.stdout:can't protect a protected snap 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:made and opened clone "child" 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:read: 8 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:read: 8 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:read: 8 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:sizes and overlaps are good between parent and child 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:child image successfully cloned all image-meta pairs 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:sized down clone, changed overlap 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:parent info: size 4194304 obj_size 4194304 parent_pool 18446744073709551615 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:sized up clone, changed size but not overlap or parent's size 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:can't remove parent while child still exists 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:can't remove parent while still protected 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:removed parent snap after unprotecting 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestClone (3567 ms) 2024-09-13T14:38:43.290 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestClone2 2024-09-13T14:38:43.291 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:44.071 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[105191]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-2 2024-09-13T14:38:44.072 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[105307]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-2 2024-09-13T14:38:44.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[87147]: audit 2024-09-13T14:38:41.989069+0000 mgr.y (mgr.26252) 319 : audit [DBG] from='client.44513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:44.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[87147]: audit 2024-09-13T14:38:41.989069+0000 mgr.y (mgr.26252) 319 : audit [DBG] from='client.44513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:44.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[87147]: cluster 2024-09-13T14:38:42.134938+0000 mgr.y (mgr.26252) 320 : cluster [DBG] pgmap v523: 457 pgs: 46 stale+active+clean, 411 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 8.7 KiB/s wr, 108 op/s 2024-09-13T14:38:44.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[87147]: cluster 2024-09-13T14:38:42.134938+0000 mgr.y (mgr.26252) 320 : cluster [DBG] pgmap v523: 457 pgs: 46 stale+active+clean, 411 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 8.7 KiB/s wr, 108 op/s 2024-09-13T14:38:44.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[87147]: cluster 2024-09-13T14:38:43.109425+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e658: 8 total, 7 up, 8 in 2024-09-13T14:38:44.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[87147]: cluster 2024-09-13T14:38:43.109425+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e658: 8 total, 7 up, 8 in 2024-09-13T14:38:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[84576]: audit 2024-09-13T14:38:41.989069+0000 mgr.y (mgr.26252) 319 : audit [DBG] from='client.44513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[84576]: audit 2024-09-13T14:38:41.989069+0000 mgr.y (mgr.26252) 319 : audit [DBG] from='client.44513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[84576]: cluster 2024-09-13T14:38:42.134938+0000 mgr.y (mgr.26252) 320 : cluster [DBG] pgmap v523: 457 pgs: 46 stale+active+clean, 411 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 8.7 KiB/s wr, 108 op/s 2024-09-13T14:38:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[84576]: cluster 2024-09-13T14:38:42.134938+0000 mgr.y (mgr.26252) 320 : cluster [DBG] pgmap v523: 457 pgs: 46 stale+active+clean, 411 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 8.7 KiB/s wr, 108 op/s 2024-09-13T14:38:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[84576]: cluster 2024-09-13T14:38:43.109425+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e658: 8 total, 7 up, 8 in 2024-09-13T14:38:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:43 smithi099 bash[84576]: cluster 2024-09-13T14:38:43.109425+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e658: 8 total, 7 up, 8 in 2024-09-13T14:38:44.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:43 smithi177 bash[68594]: audit 2024-09-13T14:38:41.989069+0000 mgr.y (mgr.26252) 319 : audit [DBG] from='client.44513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:43 smithi177 bash[68594]: audit 2024-09-13T14:38:41.989069+0000 mgr.y (mgr.26252) 319 : audit [DBG] from='client.44513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:43 smithi177 bash[68594]: cluster 2024-09-13T14:38:42.134938+0000 mgr.y (mgr.26252) 320 : cluster [DBG] pgmap v523: 457 pgs: 46 stale+active+clean, 411 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 8.7 KiB/s wr, 108 op/s 2024-09-13T14:38:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:43 smithi177 bash[68594]: cluster 2024-09-13T14:38:42.134938+0000 mgr.y (mgr.26252) 320 : cluster [DBG] pgmap v523: 457 pgs: 46 stale+active+clean, 411 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 8.7 KiB/s wr, 108 op/s 2024-09-13T14:38:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:43 smithi177 bash[68594]: cluster 2024-09-13T14:38:43.109425+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e658: 8 total, 7 up, 8 in 2024-09-13T14:38:44.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:43 smithi177 bash[68594]: cluster 2024-09-13T14:38:43.109425+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e658: 8 total, 7 up, 8 in 2024-09-13T14:38:44.630 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:44.976 INFO:tasks.workunit.client.0.smithi099.stdout:made parent image "parent" 2024-09-13T14:38:44.976 INFO:tasks.workunit.client.0.smithi099.stdout:parent has no parent info 2024-09-13T14:38:44.976 INFO:tasks.workunit.client.0.smithi099.stdout:made snapshot "parent@parent_snap" 2024-09-13T14:38:44.976 INFO:tasks.workunit.client.0.smithi099.stdout:made and opened clone "child" 2024-09-13T14:38:44.976 INFO:tasks.workunit.client.0.smithi099.stdout:child image successfully cloned all image-meta pairs 2024-09-13T14:38:44.976 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestClone2 (1687 ms) 2024-09-13T14:38:44.976 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ListChildren 2024-09-13T14:38:45.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:44 smithi099 bash[84576]: audit 2024-09-13T14:38:44.300193+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.99:0/3896093168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:45.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:44 smithi099 bash[84576]: audit 2024-09-13T14:38:44.300193+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.99:0/3896093168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:45.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:44 smithi099 bash[87147]: audit 2024-09-13T14:38:44.300193+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.99:0/3896093168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:45.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:44 smithi099 bash[87147]: audit 2024-09-13T14:38:44.300193+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.99:0/3896093168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:45.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:44 smithi177 bash[68594]: audit 2024-09-13T14:38:44.300193+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.99:0/3896093168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:45.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:44 smithi177 bash[68594]: audit 2024-09-13T14:38:44.300193+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.99:0/3896093168' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:45.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[84576]: cluster 2024-09-13T14:38:44.137300+0000 mgr.y (mgr.26252) 321 : cluster [DBG] pgmap v525: 457 pgs: 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:45.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[84576]: cluster 2024-09-13T14:38:44.137300+0000 mgr.y (mgr.26252) 321 : cluster [DBG] pgmap v525: 457 pgs: 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:45.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[84576]: audit 2024-09-13T14:38:44.618858+0000 mgr.y (mgr.26252) 322 : audit [DBG] from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:45.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[84576]: audit 2024-09-13T14:38:44.618858+0000 mgr.y (mgr.26252) 322 : audit [DBG] from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:45.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[84576]: cluster 2024-09-13T14:38:44.718437+0000 mon.a (mon.0) 892 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:38:45.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[84576]: cluster 2024-09-13T14:38:44.718437+0000 mon.a (mon.0) 892 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:38:45.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[84576]: cluster 2024-09-13T14:38:44.718468+0000 mon.a (mon.0) 893 : cluster [WRN] Health check failed: Degraded data redundancy: 110/1089 objects degraded (10.101%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:45.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[84576]: cluster 2024-09-13T14:38:44.718468+0000 mon.a (mon.0) 893 : cluster [WRN] Health check failed: Degraded data redundancy: 110/1089 objects degraded (10.101%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:45.808 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[84576]: cluster 2024-09-13T14:38:44.732079+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e659: 8 total, 7 up, 8 in 2024-09-13T14:38:45.808 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[84576]: cluster 2024-09-13T14:38:44.732079+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e659: 8 total, 7 up, 8 in 2024-09-13T14:38:45.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[87147]: cluster 2024-09-13T14:38:44.137300+0000 mgr.y (mgr.26252) 321 : cluster [DBG] pgmap v525: 457 pgs: 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:45.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[87147]: cluster 2024-09-13T14:38:44.137300+0000 mgr.y (mgr.26252) 321 : cluster [DBG] pgmap v525: 457 pgs: 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:45.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[87147]: audit 2024-09-13T14:38:44.618858+0000 mgr.y (mgr.26252) 322 : audit [DBG] from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:45.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[87147]: audit 2024-09-13T14:38:44.618858+0000 mgr.y (mgr.26252) 322 : audit [DBG] from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:45.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[87147]: cluster 2024-09-13T14:38:44.718437+0000 mon.a (mon.0) 892 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:38:45.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[87147]: cluster 2024-09-13T14:38:44.718437+0000 mon.a (mon.0) 892 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:38:45.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[87147]: cluster 2024-09-13T14:38:44.718468+0000 mon.a (mon.0) 893 : cluster [WRN] Health check failed: Degraded data redundancy: 110/1089 objects degraded (10.101%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:45.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[87147]: cluster 2024-09-13T14:38:44.718468+0000 mon.a (mon.0) 893 : cluster [WRN] Health check failed: Degraded data redundancy: 110/1089 objects degraded (10.101%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:45.809 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[87147]: cluster 2024-09-13T14:38:44.732079+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e659: 8 total, 7 up, 8 in 2024-09-13T14:38:45.809 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:45 smithi099 bash[87147]: cluster 2024-09-13T14:38:44.732079+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e659: 8 total, 7 up, 8 in 2024-09-13T14:38:46.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:45 smithi177 bash[68594]: cluster 2024-09-13T14:38:44.137300+0000 mgr.y (mgr.26252) 321 : cluster [DBG] pgmap v525: 457 pgs: 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:46.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:45 smithi177 bash[68594]: cluster 2024-09-13T14:38:44.137300+0000 mgr.y (mgr.26252) 321 : cluster [DBG] pgmap v525: 457 pgs: 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:46.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:45 smithi177 bash[68594]: audit 2024-09-13T14:38:44.618858+0000 mgr.y (mgr.26252) 322 : audit [DBG] from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:46.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:45 smithi177 bash[68594]: audit 2024-09-13T14:38:44.618858+0000 mgr.y (mgr.26252) 322 : audit [DBG] from='client.34635 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:46.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:45 smithi177 bash[68594]: cluster 2024-09-13T14:38:44.718437+0000 mon.a (mon.0) 892 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:38:46.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:45 smithi177 bash[68594]: cluster 2024-09-13T14:38:44.718437+0000 mon.a (mon.0) 892 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:38:46.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:45 smithi177 bash[68594]: cluster 2024-09-13T14:38:44.718468+0000 mon.a (mon.0) 893 : cluster [WRN] Health check failed: Degraded data redundancy: 110/1089 objects degraded (10.101%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:46.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:45 smithi177 bash[68594]: cluster 2024-09-13T14:38:44.718468+0000 mon.a (mon.0) 893 : cluster [WRN] Health check failed: Degraded data redundancy: 110/1089 objects degraded (10.101%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:46.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:45 smithi177 bash[68594]: cluster 2024-09-13T14:38:44.732079+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e659: 8 total, 7 up, 8 in 2024-09-13T14:38:46.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:45 smithi177 bash[68594]: cluster 2024-09-13T14:38:44.732079+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e659: 8 total, 7 up, 8 in 2024-09-13T14:38:46.669 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:46.669 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:46.669 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:46.670 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:46.670 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:46.670 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:46.671 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:46.671 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:46.671 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T14:38:46.671 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: Stopped Ceph osd.2 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:38:46.671 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:38:46.671 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:46 smithi099 systemd[1]: Started Ceph osd.2 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:38:47.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:44.949422+0000 mgr.y (mgr.26252) 323 : audit [DBG] from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:47.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:44.949422+0000 mgr.y (mgr.26252) 323 : audit [DBG] from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:47.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: cluster 2024-09-13T14:38:45.751740+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e660: 8 total, 7 up, 8 in 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: cluster 2024-09-13T14:38:45.751740+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e660: 8 total, 7 up, 8 in 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:45.755157+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.99:0/759021111' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:45.755157+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.99:0/759021111' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:45.760057+0000 mon.a (mon.0) 896 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:45.760057+0000 mon.a (mon.0) 896 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:46.092361+0000 mon.a (mon.0) 897 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:46.092361+0000 mon.a (mon.0) 897 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: cluster 2024-09-13T14:38:46.100814+0000 mon.a (mon.0) 898 : cluster [DBG] osdmap e661: 8 total, 7 up, 8 in 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: cluster 2024-09-13T14:38:46.100814+0000 mon.a (mon.0) 898 : cluster [DBG] osdmap e661: 8 total, 7 up, 8 in 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:46.716115+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:46.716115+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:46.723001+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:46.723001+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:46.724026+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:47.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[84576]: audit 2024-09-13T14:38:46.724026+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:47.048 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:44.949422+0000 mgr.y (mgr.26252) 323 : audit [DBG] from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:47.048 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:44.949422+0000 mgr.y (mgr.26252) 323 : audit [DBG] from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:47.048 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: cluster 2024-09-13T14:38:45.751740+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e660: 8 total, 7 up, 8 in 2024-09-13T14:38:47.048 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: cluster 2024-09-13T14:38:45.751740+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e660: 8 total, 7 up, 8 in 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:45.755157+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.99:0/759021111' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:45.755157+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.99:0/759021111' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:45.760057+0000 mon.a (mon.0) 896 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:45.760057+0000 mon.a (mon.0) 896 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:46.092361+0000 mon.a (mon.0) 897 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:46.092361+0000 mon.a (mon.0) 897 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: cluster 2024-09-13T14:38:46.100814+0000 mon.a (mon.0) 898 : cluster [DBG] osdmap e661: 8 total, 7 up, 8 in 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: cluster 2024-09-13T14:38:46.100814+0000 mon.a (mon.0) 898 : cluster [DBG] osdmap e661: 8 total, 7 up, 8 in 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:46.716115+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:46.716115+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:46.723001+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:46.723001+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:46.724026+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:47.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:46 smithi099 bash[87147]: audit 2024-09-13T14:38:46.724026+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:47.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:44.949422+0000 mgr.y (mgr.26252) 323 : audit [DBG] from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:47.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:44.949422+0000 mgr.y (mgr.26252) 323 : audit [DBG] from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:47.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: cluster 2024-09-13T14:38:45.751740+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e660: 8 total, 7 up, 8 in 2024-09-13T14:38:47.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: cluster 2024-09-13T14:38:45.751740+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e660: 8 total, 7 up, 8 in 2024-09-13T14:38:47.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:45.755157+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.99:0/759021111' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:45.755157+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.99:0/759021111' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:45.760057+0000 mon.a (mon.0) 896 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:45.760057+0000 mon.a (mon.0) 896 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:46.092361+0000 mon.a (mon.0) 897 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:46.092361+0000 mon.a (mon.0) 897 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: cluster 2024-09-13T14:38:46.100814+0000 mon.a (mon.0) 898 : cluster [DBG] osdmap e661: 8 total, 7 up, 8 in 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: cluster 2024-09-13T14:38:46.100814+0000 mon.a (mon.0) 898 : cluster [DBG] osdmap e661: 8 total, 7 up, 8 in 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:46.716115+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:46.716115+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:46.723001+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:46.723001+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:46.724026+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:46 smithi177 bash[68594]: audit 2024-09-13T14:38:46.724026+0000 mon.a (mon.0) 901 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:38:47] "GET /metrics HTTP/1.1" 200 63108 "" "Prometheus/2.51.0" 2024-09-13T14:38:47.586 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:48.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[84576]: cluster 2024-09-13T14:38:46.138240+0000 mgr.y (mgr.26252) 324 : cluster [DBG] pgmap v529: 489 pgs: 32 unknown, 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:48.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[84576]: cluster 2024-09-13T14:38:46.138240+0000 mgr.y (mgr.26252) 324 : cluster [DBG] pgmap v529: 489 pgs: 32 unknown, 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:48.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[84576]: cluster 2024-09-13T14:38:47.104082+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e662: 8 total, 7 up, 8 in 2024-09-13T14:38:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[84576]: cluster 2024-09-13T14:38:47.104082+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e662: 8 total, 7 up, 8 in 2024-09-13T14:38:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[84576]: audit 2024-09-13T14:38:47.105240+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.99:0/928753990' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[84576]: audit 2024-09-13T14:38:47.105240+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.99:0/928753990' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[84576]: audit 2024-09-13T14:38:47.106217+0000 mon.a (mon.0) 903 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[84576]: audit 2024-09-13T14:38:47.106217+0000 mon.a (mon.0) 903 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[84576]: audit 2024-09-13T14:38:47.269035+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.99:0/1367388071' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[84576]: audit 2024-09-13T14:38:47.269035+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.99:0/1367388071' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[87147]: cluster 2024-09-13T14:38:46.138240+0000 mgr.y (mgr.26252) 324 : cluster [DBG] pgmap v529: 489 pgs: 32 unknown, 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[87147]: cluster 2024-09-13T14:38:46.138240+0000 mgr.y (mgr.26252) 324 : cluster [DBG] pgmap v529: 489 pgs: 32 unknown, 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[87147]: cluster 2024-09-13T14:38:47.104082+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e662: 8 total, 7 up, 8 in 2024-09-13T14:38:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[87147]: cluster 2024-09-13T14:38:47.104082+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e662: 8 total, 7 up, 8 in 2024-09-13T14:38:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[87147]: audit 2024-09-13T14:38:47.105240+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.99:0/928753990' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[87147]: audit 2024-09-13T14:38:47.105240+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.99:0/928753990' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[87147]: audit 2024-09-13T14:38:47.106217+0000 mon.a (mon.0) 903 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[87147]: audit 2024-09-13T14:38:47.106217+0000 mon.a (mon.0) 903 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[87147]: audit 2024-09-13T14:38:47.269035+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.99:0/1367388071' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:48.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:47 smithi099 bash[87147]: audit 2024-09-13T14:38:47.269035+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.99:0/1367388071' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:48.117 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:47 smithi177 bash[68594]: cluster 2024-09-13T14:38:46.138240+0000 mgr.y (mgr.26252) 324 : cluster [DBG] pgmap v529: 489 pgs: 32 unknown, 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:48.117 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:47 smithi177 bash[68594]: cluster 2024-09-13T14:38:46.138240+0000 mgr.y (mgr.26252) 324 : cluster [DBG] pgmap v529: 489 pgs: 32 unknown, 111 active+undersized, 26 peering, 38 active+undersized+degraded, 282 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 30 KiB/s wr, 526 op/s; 110/1089 objects degraded (10.101%) 2024-09-13T14:38:48.118 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:47 smithi177 bash[68594]: cluster 2024-09-13T14:38:47.104082+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e662: 8 total, 7 up, 8 in 2024-09-13T14:38:48.118 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:47 smithi177 bash[68594]: cluster 2024-09-13T14:38:47.104082+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e662: 8 total, 7 up, 8 in 2024-09-13T14:38:48.118 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:47 smithi177 bash[68594]: audit 2024-09-13T14:38:47.105240+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.99:0/928753990' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.118 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:47 smithi177 bash[68594]: audit 2024-09-13T14:38:47.105240+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.99:0/928753990' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.118 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:47 smithi177 bash[68594]: audit 2024-09-13T14:38:47.106217+0000 mon.a (mon.0) 903 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.118 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:47 smithi177 bash[68594]: audit 2024-09-13T14:38:47.106217+0000 mon.a (mon.0) 903 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:48.119 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:47 smithi177 bash[68594]: audit 2024-09-13T14:38:47.269035+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.99:0/1367388071' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:48.119 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:47 smithi177 bash[68594]: audit 2024-09-13T14:38:47.269035+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.99:0/1367388071' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:48.127 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:38:48.461 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:48.461 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:48.462 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:38:48.462 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:48.462 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:38:48.749 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:38:48.749 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-13T14:38:49.072 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:38:49.072 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:38:49.072 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:38:49.072 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:38:49.072 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[105563]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-13T14:38:49.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[84576]: audit 2024-09-13T14:38:47.576080+0000 mgr.y (mgr.26252) 325 : audit [DBG] from='client.44555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:49.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[84576]: audit 2024-09-13T14:38:47.576080+0000 mgr.y (mgr.26252) 325 : audit [DBG] from='client.44555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[84576]: audit 2024-09-13T14:38:48.103228+0000 mon.a (mon.0) 905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[84576]: audit 2024-09-13T14:38:48.103228+0000 mon.a (mon.0) 905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[84576]: cluster 2024-09-13T14:38:48.111690+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e663: 8 total, 7 up, 8 in 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[84576]: cluster 2024-09-13T14:38:48.111690+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e663: 8 total, 7 up, 8 in 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[84576]: audit 2024-09-13T14:38:48.253223+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[84576]: audit 2024-09-13T14:38:48.253223+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[84576]: audit 2024-09-13T14:38:48.254419+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[84576]: audit 2024-09-13T14:38:48.254419+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[87147]: audit 2024-09-13T14:38:47.576080+0000 mgr.y (mgr.26252) 325 : audit [DBG] from='client.44555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[87147]: audit 2024-09-13T14:38:47.576080+0000 mgr.y (mgr.26252) 325 : audit [DBG] from='client.44555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:49.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[87147]: audit 2024-09-13T14:38:48.103228+0000 mon.a (mon.0) 905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:49.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[87147]: audit 2024-09-13T14:38:48.103228+0000 mon.a (mon.0) 905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:49.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[87147]: cluster 2024-09-13T14:38:48.111690+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e663: 8 total, 7 up, 8 in 2024-09-13T14:38:49.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[87147]: cluster 2024-09-13T14:38:48.111690+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e663: 8 total, 7 up, 8 in 2024-09-13T14:38:49.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[87147]: audit 2024-09-13T14:38:48.253223+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:49.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[87147]: audit 2024-09-13T14:38:48.253223+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:49.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[87147]: audit 2024-09-13T14:38:48.254419+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:49.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:48 smithi099 bash[87147]: audit 2024-09-13T14:38:48.254419+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:49.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:48 smithi177 bash[68594]: audit 2024-09-13T14:38:47.576080+0000 mgr.y (mgr.26252) 325 : audit [DBG] from='client.44555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:49.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:48 smithi177 bash[68594]: audit 2024-09-13T14:38:47.576080+0000 mgr.y (mgr.26252) 325 : audit [DBG] from='client.44555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:49.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:48 smithi177 bash[68594]: audit 2024-09-13T14:38:48.103228+0000 mon.a (mon.0) 905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:49.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:48 smithi177 bash[68594]: audit 2024-09-13T14:38:48.103228+0000 mon.a (mon.0) 905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:49.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:48 smithi177 bash[68594]: cluster 2024-09-13T14:38:48.111690+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e663: 8 total, 7 up, 8 in 2024-09-13T14:38:49.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:48 smithi177 bash[68594]: cluster 2024-09-13T14:38:48.111690+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e663: 8 total, 7 up, 8 in 2024-09-13T14:38:49.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:48 smithi177 bash[68594]: audit 2024-09-13T14:38:48.253223+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:49.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:48 smithi177 bash[68594]: audit 2024-09-13T14:38:48.253223+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:49.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:48 smithi177 bash[68594]: audit 2024-09-13T14:38:48.254419+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:49.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:48 smithi177 bash[68594]: audit 2024-09-13T14:38:48.254419+0000 mon.a (mon.0) 908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:50.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[84576]: audit 2024-09-13T14:38:47.884266+0000 mgr.y (mgr.26252) 326 : audit [DBG] from='client.34671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:50.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[84576]: audit 2024-09-13T14:38:47.884266+0000 mgr.y (mgr.26252) 326 : audit [DBG] from='client.34671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:50.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[84576]: cluster 2024-09-13T14:38:48.139262+0000 mgr.y (mgr.26252) 327 : cluster [DBG] pgmap v532: 521 pgs: 6 creating+peering, 53 unknown, 131 active+undersized, 46 active+undersized+degraded, 285 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 349 KiB/s rd, 25 KiB/s wr, 413 op/s; 128/1101 objects degraded (11.626%) 2024-09-13T14:38:50.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[84576]: cluster 2024-09-13T14:38:48.139262+0000 mgr.y (mgr.26252) 327 : cluster [DBG] pgmap v532: 521 pgs: 6 creating+peering, 53 unknown, 131 active+undersized, 46 active+undersized+degraded, 285 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 349 KiB/s rd, 25 KiB/s wr, 413 op/s; 128/1101 objects degraded (11.626%) 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[84576]: cluster 2024-09-13T14:38:49.103225+0000 mon.a (mon.0) 909 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[84576]: cluster 2024-09-13T14:38:49.103225+0000 mon.a (mon.0) 909 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[84576]: cluster 2024-09-13T14:38:49.119291+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e664: 8 total, 7 up, 8 in 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[84576]: cluster 2024-09-13T14:38:49.119291+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e664: 8 total, 7 up, 8 in 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[87147]: audit 2024-09-13T14:38:47.884266+0000 mgr.y (mgr.26252) 326 : audit [DBG] from='client.34671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[87147]: audit 2024-09-13T14:38:47.884266+0000 mgr.y (mgr.26252) 326 : audit [DBG] from='client.34671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[87147]: cluster 2024-09-13T14:38:48.139262+0000 mgr.y (mgr.26252) 327 : cluster [DBG] pgmap v532: 521 pgs: 6 creating+peering, 53 unknown, 131 active+undersized, 46 active+undersized+degraded, 285 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 349 KiB/s rd, 25 KiB/s wr, 413 op/s; 128/1101 objects degraded (11.626%) 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[87147]: cluster 2024-09-13T14:38:48.139262+0000 mgr.y (mgr.26252) 327 : cluster [DBG] pgmap v532: 521 pgs: 6 creating+peering, 53 unknown, 131 active+undersized, 46 active+undersized+degraded, 285 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 349 KiB/s rd, 25 KiB/s wr, 413 op/s; 128/1101 objects degraded (11.626%) 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[87147]: cluster 2024-09-13T14:38:49.103225+0000 mon.a (mon.0) 909 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[87147]: cluster 2024-09-13T14:38:49.103225+0000 mon.a (mon.0) 909 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-09-13T14:38:50.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[87147]: cluster 2024-09-13T14:38:49.119291+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e664: 8 total, 7 up, 8 in 2024-09-13T14:38:50.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:49 smithi099 bash[87147]: cluster 2024-09-13T14:38:49.119291+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e664: 8 total, 7 up, 8 in 2024-09-13T14:38:50.107 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:49 smithi177 bash[68594]: audit 2024-09-13T14:38:47.884266+0000 mgr.y (mgr.26252) 326 : audit [DBG] from='client.34671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:50.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:49 smithi177 bash[68594]: audit 2024-09-13T14:38:47.884266+0000 mgr.y (mgr.26252) 326 : audit [DBG] from='client.34671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:50.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:49 smithi177 bash[68594]: cluster 2024-09-13T14:38:48.139262+0000 mgr.y (mgr.26252) 327 : cluster [DBG] pgmap v532: 521 pgs: 6 creating+peering, 53 unknown, 131 active+undersized, 46 active+undersized+degraded, 285 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 349 KiB/s rd, 25 KiB/s wr, 413 op/s; 128/1101 objects degraded (11.626%) 2024-09-13T14:38:50.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:49 smithi177 bash[68594]: cluster 2024-09-13T14:38:48.139262+0000 mgr.y (mgr.26252) 327 : cluster [DBG] pgmap v532: 521 pgs: 6 creating+peering, 53 unknown, 131 active+undersized, 46 active+undersized+degraded, 285 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 349 KiB/s rd, 25 KiB/s wr, 413 op/s; 128/1101 objects degraded (11.626%) 2024-09-13T14:38:50.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:49 smithi177 bash[68594]: cluster 2024-09-13T14:38:49.103225+0000 mon.a (mon.0) 909 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-09-13T14:38:50.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:49 smithi177 bash[68594]: cluster 2024-09-13T14:38:49.103225+0000 mon.a (mon.0) 909 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-09-13T14:38:50.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:49 smithi177 bash[68594]: cluster 2024-09-13T14:38:49.119291+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e664: 8 total, 7 up, 8 in 2024-09-13T14:38:50.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:49 smithi177 bash[68594]: cluster 2024-09-13T14:38:49.119291+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e664: 8 total, 7 up, 8 in 2024-09-13T14:38:50.519 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:50.607 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:38:50 smithi177 bash[61439]: ts=2024-09-13T14:38:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:38:51.572 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[105859]: debug 2024-09-13T14:38:51.195+0000 7f394202d640 1 -- 172.21.15.99:0/3692750729 <== mon.2 v2:172.21.15.99:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55640fd7a960 con 0x55640fd9bc00 2024-09-13T14:38:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[84576]: cluster 2024-09-13T14:38:50.123656+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e665: 8 total, 7 up, 8 in 2024-09-13T14:38:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[84576]: cluster 2024-09-13T14:38:50.123656+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e665: 8 total, 7 up, 8 in 2024-09-13T14:38:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[84576]: cluster 2024-09-13T14:38:50.141262+0000 mgr.y (mgr.26252) 328 : cluster [DBG] pgmap v535: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 25 KiB/s wr, 399 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[84576]: cluster 2024-09-13T14:38:50.141262+0000 mgr.y (mgr.26252) 328 : cluster [DBG] pgmap v535: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 25 KiB/s wr, 399 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[84576]: audit 2024-09-13T14:38:50.200683+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.99:0/566462509' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[84576]: audit 2024-09-13T14:38:50.200683+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.99:0/566462509' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[84576]: audit 2024-09-13T14:38:50.509180+0000 mgr.y (mgr.26252) 329 : audit [DBG] from='client.44573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[84576]: audit 2024-09-13T14:38:50.509180+0000 mgr.y (mgr.26252) 329 : audit [DBG] from='client.44573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[84576]: cluster 2024-09-13T14:38:51.091359+0000 mon.a (mon.0) 913 : cluster [WRN] Health check update: Degraded data redundancy: 128/1101 objects degraded (11.626%), 46 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[84576]: cluster 2024-09-13T14:38:51.091359+0000 mon.a (mon.0) 913 : cluster [WRN] Health check update: Degraded data redundancy: 128/1101 objects degraded (11.626%), 46 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[87147]: cluster 2024-09-13T14:38:50.123656+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e665: 8 total, 7 up, 8 in 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[87147]: cluster 2024-09-13T14:38:50.123656+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e665: 8 total, 7 up, 8 in 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[87147]: cluster 2024-09-13T14:38:50.141262+0000 mgr.y (mgr.26252) 328 : cluster [DBG] pgmap v535: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 25 KiB/s wr, 399 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[87147]: cluster 2024-09-13T14:38:50.141262+0000 mgr.y (mgr.26252) 328 : cluster [DBG] pgmap v535: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 25 KiB/s wr, 399 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[87147]: audit 2024-09-13T14:38:50.200683+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.99:0/566462509' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[87147]: audit 2024-09-13T14:38:50.200683+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.99:0/566462509' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[87147]: audit 2024-09-13T14:38:50.509180+0000 mgr.y (mgr.26252) 329 : audit [DBG] from='client.44573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[87147]: audit 2024-09-13T14:38:50.509180+0000 mgr.y (mgr.26252) 329 : audit [DBG] from='client.44573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[87147]: cluster 2024-09-13T14:38:51.091359+0000 mon.a (mon.0) 913 : cluster [WRN] Health check update: Degraded data redundancy: 128/1101 objects degraded (11.626%), 46 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:51 smithi099 bash[87147]: cluster 2024-09-13T14:38:51.091359+0000 mon.a (mon.0) 913 : cluster [WRN] Health check update: Degraded data redundancy: 128/1101 objects degraded (11.626%), 46 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:51 smithi177 bash[68594]: cluster 2024-09-13T14:38:50.123656+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e665: 8 total, 7 up, 8 in 2024-09-13T14:38:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:51 smithi177 bash[68594]: cluster 2024-09-13T14:38:50.123656+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e665: 8 total, 7 up, 8 in 2024-09-13T14:38:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:51 smithi177 bash[68594]: cluster 2024-09-13T14:38:50.141262+0000 mgr.y (mgr.26252) 328 : cluster [DBG] pgmap v535: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 25 KiB/s wr, 399 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:51 smithi177 bash[68594]: cluster 2024-09-13T14:38:50.141262+0000 mgr.y (mgr.26252) 328 : cluster [DBG] pgmap v535: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 25 KiB/s wr, 399 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:51 smithi177 bash[68594]: audit 2024-09-13T14:38:50.200683+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.99:0/566462509' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:51 smithi177 bash[68594]: audit 2024-09-13T14:38:50.200683+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.99:0/566462509' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:51 smithi177 bash[68594]: audit 2024-09-13T14:38:50.509180+0000 mgr.y (mgr.26252) 329 : audit [DBG] from='client.44573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:51 smithi177 bash[68594]: audit 2024-09-13T14:38:50.509180+0000 mgr.y (mgr.26252) 329 : audit [DBG] from='client.44573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:51 smithi177 bash[68594]: cluster 2024-09-13T14:38:51.091359+0000 mon.a (mon.0) 913 : cluster [WRN] Health check update: Degraded data redundancy: 128/1101 objects degraded (11.626%), 46 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:51 smithi177 bash[68594]: cluster 2024-09-13T14:38:51.091359+0000 mon.a (mon.0) 913 : cluster [WRN] Health check update: Degraded data redundancy: 128/1101 objects degraded (11.626%), 46 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:52 smithi099 bash[84576]: audit 2024-09-13T14:38:50.807242+0000 mgr.y (mgr.26252) 330 : audit [DBG] from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:52 smithi099 bash[84576]: audit 2024-09-13T14:38:50.807242+0000 mgr.y (mgr.26252) 330 : audit [DBG] from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:52 smithi099 bash[84576]: cluster 2024-09-13T14:38:51.138278+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e666: 8 total, 7 up, 8 in 2024-09-13T14:38:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:52 smithi099 bash[84576]: cluster 2024-09-13T14:38:51.138278+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e666: 8 total, 7 up, 8 in 2024-09-13T14:38:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:52 smithi099 bash[87147]: audit 2024-09-13T14:38:50.807242+0000 mgr.y (mgr.26252) 330 : audit [DBG] from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:52 smithi099 bash[87147]: audit 2024-09-13T14:38:50.807242+0000 mgr.y (mgr.26252) 330 : audit [DBG] from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:52.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:52 smithi099 bash[87147]: cluster 2024-09-13T14:38:51.138278+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e666: 8 total, 7 up, 8 in 2024-09-13T14:38:52.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:52 smithi099 bash[87147]: cluster 2024-09-13T14:38:51.138278+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e666: 8 total, 7 up, 8 in 2024-09-13T14:38:52.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:52 smithi177 bash[68594]: audit 2024-09-13T14:38:50.807242+0000 mgr.y (mgr.26252) 330 : audit [DBG] from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:52.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:52 smithi177 bash[68594]: audit 2024-09-13T14:38:50.807242+0000 mgr.y (mgr.26252) 330 : audit [DBG] from='client.34689 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:52.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:52 smithi177 bash[68594]: cluster 2024-09-13T14:38:51.138278+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e666: 8 total, 7 up, 8 in 2024-09-13T14:38:52.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:52 smithi177 bash[68594]: cluster 2024-09-13T14:38:51.138278+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e666: 8 total, 7 up, 8 in 2024-09-13T14:38:53.071 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:52 smithi099 bash[105859]: debug 2024-09-13T14:38:52.739+0000 7f3944894640 -1 Falling back to public interface 2024-09-13T14:38:53.489 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[87147]: cluster 2024-09-13T14:38:52.142810+0000 mgr.y (mgr.26252) 331 : cluster [DBG] pgmap v537: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 3.5 KiB/s wr, 46 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[87147]: cluster 2024-09-13T14:38:52.142810+0000 mgr.y (mgr.26252) 331 : cluster [DBG] pgmap v537: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 3.5 KiB/s wr, 46 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[87147]: cluster 2024-09-13T14:38:52.149704+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e667: 8 total, 7 up, 8 in 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[87147]: cluster 2024-09-13T14:38:52.149704+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e667: 8 total, 7 up, 8 in 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[87147]: audit 2024-09-13T14:38:52.780094+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[87147]: audit 2024-09-13T14:38:52.780094+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[87147]: audit 2024-09-13T14:38:52.788420+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[87147]: audit 2024-09-13T14:38:52.788420+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[87147]: audit 2024-09-13T14:38:53.110270+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.99:0/3517954015' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[87147]: audit 2024-09-13T14:38:53.110270+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.99:0/3517954015' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[84576]: cluster 2024-09-13T14:38:52.142810+0000 mgr.y (mgr.26252) 331 : cluster [DBG] pgmap v537: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 3.5 KiB/s wr, 46 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[84576]: cluster 2024-09-13T14:38:52.142810+0000 mgr.y (mgr.26252) 331 : cluster [DBG] pgmap v537: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 3.5 KiB/s wr, 46 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:53.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[84576]: cluster 2024-09-13T14:38:52.149704+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e667: 8 total, 7 up, 8 in 2024-09-13T14:38:53.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[84576]: cluster 2024-09-13T14:38:52.149704+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e667: 8 total, 7 up, 8 in 2024-09-13T14:38:53.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[84576]: audit 2024-09-13T14:38:52.780094+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[84576]: audit 2024-09-13T14:38:52.780094+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[84576]: audit 2024-09-13T14:38:52.788420+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[84576]: audit 2024-09-13T14:38:52.788420+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[84576]: audit 2024-09-13T14:38:53.110270+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.99:0/3517954015' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:53.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:53 smithi099 bash[84576]: audit 2024-09-13T14:38:53.110270+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.99:0/3517954015' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:53 smithi177 bash[68594]: cluster 2024-09-13T14:38:52.142810+0000 mgr.y (mgr.26252) 331 : cluster [DBG] pgmap v537: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 3.5 KiB/s wr, 46 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:53 smithi177 bash[68594]: cluster 2024-09-13T14:38:52.142810+0000 mgr.y (mgr.26252) 331 : cluster [DBG] pgmap v537: 521 pgs: 6 creating+peering, 155 active+undersized, 46 active+undersized+degraded, 314 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 3.5 KiB/s wr, 46 op/s; 131/1113 objects degraded (11.770%) 2024-09-13T14:38:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:53 smithi177 bash[68594]: cluster 2024-09-13T14:38:52.149704+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e667: 8 total, 7 up, 8 in 2024-09-13T14:38:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:53 smithi177 bash[68594]: cluster 2024-09-13T14:38:52.149704+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e667: 8 total, 7 up, 8 in 2024-09-13T14:38:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:53 smithi177 bash[68594]: audit 2024-09-13T14:38:52.780094+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:53 smithi177 bash[68594]: audit 2024-09-13T14:38:52.780094+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:53 smithi177 bash[68594]: audit 2024-09-13T14:38:52.788420+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:53 smithi177 bash[68594]: audit 2024-09-13T14:38:52.788420+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:53 smithi177 bash[68594]: audit 2024-09-13T14:38:53.110270+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.99:0/3517954015' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:53 smithi177 bash[68594]: audit 2024-09-13T14:38:53.110270+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.99:0/3517954015' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:53.897 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 1 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout:child: image96 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96/85e318ba2f49 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.898 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 2 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout:child: image97 2024-09-13T14:38:53.899 INFO:tasks.workunit.client.0.smithi099.stdout:child: image96 2024-09-13T14:38:53.900 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96/85e318ba2f49 2024-09-13T14:38:53.900 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.900 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97/85e336efc871 2024-09-13T14:38:53.900 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.900 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.900 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.900 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.900 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.900 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:53.901 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 3 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 3 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout:child: image97 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout:child: image96 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout:child: image98 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96/85e318ba2f49 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97/85e336efc871 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image98/85e3e0000d31 2024-09-13T14:38:53.902 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 3 2024-09-13T14:38:53.903 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 3 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout:child: image97 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout:child: image96 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout:child: image98 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96/85e318ba2f49 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97/85e336efc871 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image98/85e3e0000d31 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.904 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.905 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.905 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.905 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.905 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.905 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.905 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.905 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.906 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.906 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.906 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:53.906 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.906 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.906 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:53.906 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:53.906 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.906 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 4 2024-09-13T14:38:53.906 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 4 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout:child: image97 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout:child: image96 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout:child: image98 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout:child: image99 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96/85e318ba2f49 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97/85e336efc871 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image98/85e3e0000d31 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image99/85e39b82e9a8 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.907 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.908 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.909 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:53.909 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:53.909 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.909 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:53.909 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:53.909 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:53.909 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image96 2024-09-13T14:38:53.909 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[84576]: cluster 2024-09-13T14:38:53.160257+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e668: 8 total, 7 up, 8 in 2024-09-13T14:38:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[84576]: cluster 2024-09-13T14:38:53.160257+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e668: 8 total, 7 up, 8 in 2024-09-13T14:38:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[84576]: audit 2024-09-13T14:38:53.478328+0000 mgr.y (mgr.26252) 332 : audit [DBG] from='client.44594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[84576]: audit 2024-09-13T14:38:53.478328+0000 mgr.y (mgr.26252) 332 : audit [DBG] from='client.44594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[84576]: audit 2024-09-13T14:38:54.050638+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[84576]: audit 2024-09-13T14:38:54.050638+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[84576]: audit 2024-09-13T14:38:54.055932+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[84576]: audit 2024-09-13T14:38:54.055932+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[87147]: cluster 2024-09-13T14:38:53.160257+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e668: 8 total, 7 up, 8 in 2024-09-13T14:38:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[87147]: cluster 2024-09-13T14:38:53.160257+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e668: 8 total, 7 up, 8 in 2024-09-13T14:38:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[87147]: audit 2024-09-13T14:38:53.478328+0000 mgr.y (mgr.26252) 332 : audit [DBG] from='client.44594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[87147]: audit 2024-09-13T14:38:53.478328+0000 mgr.y (mgr.26252) 332 : audit [DBG] from='client.44594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[87147]: audit 2024-09-13T14:38:54.050638+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[87147]: audit 2024-09-13T14:38:54.050638+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[87147]: audit 2024-09-13T14:38:54.055932+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:54 smithi099 bash[87147]: audit 2024-09-13T14:38:54.055932+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:54 smithi177 bash[68594]: cluster 2024-09-13T14:38:53.160257+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e668: 8 total, 7 up, 8 in 2024-09-13T14:38:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:54 smithi177 bash[68594]: cluster 2024-09-13T14:38:53.160257+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e668: 8 total, 7 up, 8 in 2024-09-13T14:38:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:54 smithi177 bash[68594]: audit 2024-09-13T14:38:53.478328+0000 mgr.y (mgr.26252) 332 : audit [DBG] from='client.44594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:54 smithi177 bash[68594]: audit 2024-09-13T14:38:53.478328+0000 mgr.y (mgr.26252) 332 : audit [DBG] from='client.44594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:54 smithi177 bash[68594]: audit 2024-09-13T14:38:54.050638+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:54 smithi177 bash[68594]: audit 2024-09-13T14:38:54.050638+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:54 smithi177 bash[68594]: audit 2024-09-13T14:38:54.055932+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:54 smithi177 bash[68594]: audit 2024-09-13T14:38:54.055932+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:55.314 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:55.314 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:55.314 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.314 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 4 2024-09-13T14:38:55.314 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 4 2024-09-13T14:38:55.314 INFO:tasks.workunit.client.0.smithi099.stdout:child: image97 2024-09-13T14:38:55.314 INFO:tasks.workunit.client.0.smithi099.stdout:child: image96 2024-09-13T14:38:55.314 INFO:tasks.workunit.client.0.smithi099.stdout:child: image98 2024-09-13T14:38:55.314 INFO:tasks.workunit.client.0.smithi099.stdout:child: image99 2024-09-13T14:38:55.314 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image96/85e318ba2f49 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97/85e336efc871 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image98/85e3e0000d31 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image99/85e39b82e9a8 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.315 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.316 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.316 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:55.316 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.316 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:55.316 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:55.316 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.316 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.316 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:55.316 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image98 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 3 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 3 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:child: image97 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:child: image98 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:child: image99 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97/85e336efc871 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image98/85e3e0000d31 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.317 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image99/85e39b82e9a8 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image99 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 2 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:38:55.318 INFO:tasks.workunit.client.0.smithi099.stdout:child: image97 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout:child: image99 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97/85e336efc871 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-27/image99/85e39b82e9a8 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 1 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:38:55.319 INFO:tasks.workunit.client.0.smithi099.stdout:child: image97 2024-09-13T14:38:55.320 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-26/image97/85e336efc871 2024-09-13T14:38:55.320 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:38:55.320 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 0 2024-09-13T14:38:55.320 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 0 2024-09-13T14:38:55.320 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ListChildren (10337 ms) 2024-09-13T14:38:55.320 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ListChildrenTiered 2024-09-13T14:38:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:55 smithi099 bash[84576]: audit 2024-09-13T14:38:53.834418+0000 mgr.y (mgr.26252) 333 : audit [DBG] from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:55.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:55 smithi099 bash[84576]: audit 2024-09-13T14:38:53.834418+0000 mgr.y (mgr.26252) 333 : audit [DBG] from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:55.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:55 smithi099 bash[84576]: cluster 2024-09-13T14:38:54.144842+0000 mgr.y (mgr.26252) 334 : cluster [DBG] pgmap v540: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 182 KiB/s rd, 7.9 KiB/s wr, 232 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:55.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:55 smithi099 bash[84576]: cluster 2024-09-13T14:38:54.144842+0000 mgr.y (mgr.26252) 334 : cluster [DBG] pgmap v540: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 182 KiB/s rd, 7.9 KiB/s wr, 232 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:55 smithi099 bash[87147]: audit 2024-09-13T14:38:53.834418+0000 mgr.y (mgr.26252) 333 : audit [DBG] from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:55 smithi099 bash[87147]: audit 2024-09-13T14:38:53.834418+0000 mgr.y (mgr.26252) 333 : audit [DBG] from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:55 smithi099 bash[87147]: cluster 2024-09-13T14:38:54.144842+0000 mgr.y (mgr.26252) 334 : cluster [DBG] pgmap v540: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 182 KiB/s rd, 7.9 KiB/s wr, 232 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:55 smithi099 bash[87147]: cluster 2024-09-13T14:38:54.144842+0000 mgr.y (mgr.26252) 334 : cluster [DBG] pgmap v540: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 182 KiB/s rd, 7.9 KiB/s wr, 232 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:55.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:55 smithi177 bash[68594]: audit 2024-09-13T14:38:53.834418+0000 mgr.y (mgr.26252) 333 : audit [DBG] from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:55.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:55 smithi177 bash[68594]: audit 2024-09-13T14:38:53.834418+0000 mgr.y (mgr.26252) 333 : audit [DBG] from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:55.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:55 smithi177 bash[68594]: cluster 2024-09-13T14:38:54.144842+0000 mgr.y (mgr.26252) 334 : cluster [DBG] pgmap v540: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 182 KiB/s rd, 7.9 KiB/s wr, 232 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:55.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:55 smithi177 bash[68594]: cluster 2024-09-13T14:38:54.144842+0000 mgr.y (mgr.26252) 334 : cluster [DBG] pgmap v540: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 182 KiB/s rd, 7.9 KiB/s wr, 232 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:56.432 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:56.551 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[84576]: cluster 2024-09-13T14:38:55.171672+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e669: 8 total, 7 up, 8 in 2024-09-13T14:38:56.551 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[84576]: cluster 2024-09-13T14:38:55.171672+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e669: 8 total, 7 up, 8 in 2024-09-13T14:38:56.551 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[84576]: audit 2024-09-13T14:38:55.182684+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[84576]: audit 2024-09-13T14:38:55.182684+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[84576]: audit 2024-09-13T14:38:55.192529+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[84576]: audit 2024-09-13T14:38:55.192529+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[84576]: cluster 2024-09-13T14:38:56.092715+0000 mon.a (mon.0) 924 : cluster [WRN] Health check update: Degraded data redundancy: 140/1170 objects degraded (11.966%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[84576]: cluster 2024-09-13T14:38:56.092715+0000 mon.a (mon.0) 924 : cluster [WRN] Health check update: Degraded data redundancy: 140/1170 objects degraded (11.966%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[84576]: audit 2024-09-13T14:38:56.118082+0000 mon.a (mon.0) 925 : audit [DBG] from='client.? 172.21.15.99:0/2125749568' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[84576]: audit 2024-09-13T14:38:56.118082+0000 mon.a (mon.0) 925 : audit [DBG] from='client.? 172.21.15.99:0/2125749568' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[87147]: cluster 2024-09-13T14:38:55.171672+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e669: 8 total, 7 up, 8 in 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[87147]: cluster 2024-09-13T14:38:55.171672+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e669: 8 total, 7 up, 8 in 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[87147]: audit 2024-09-13T14:38:55.182684+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.552 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[87147]: audit 2024-09-13T14:38:55.182684+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.553 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[87147]: audit 2024-09-13T14:38:55.192529+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.553 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[87147]: audit 2024-09-13T14:38:55.192529+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.553 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[87147]: cluster 2024-09-13T14:38:56.092715+0000 mon.a (mon.0) 924 : cluster [WRN] Health check update: Degraded data redundancy: 140/1170 objects degraded (11.966%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:56.553 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[87147]: cluster 2024-09-13T14:38:56.092715+0000 mon.a (mon.0) 924 : cluster [WRN] Health check update: Degraded data redundancy: 140/1170 objects degraded (11.966%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:56.553 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[87147]: audit 2024-09-13T14:38:56.118082+0000 mon.a (mon.0) 925 : audit [DBG] from='client.? 172.21.15.99:0/2125749568' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:56.553 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[87147]: audit 2024-09-13T14:38:56.118082+0000 mon.a (mon.0) 925 : audit [DBG] from='client.? 172.21.15.99:0/2125749568' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:56.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:56 smithi177 bash[68594]: cluster 2024-09-13T14:38:55.171672+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e669: 8 total, 7 up, 8 in 2024-09-13T14:38:56.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:56 smithi177 bash[68594]: cluster 2024-09-13T14:38:55.171672+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e669: 8 total, 7 up, 8 in 2024-09-13T14:38:56.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:56 smithi177 bash[68594]: audit 2024-09-13T14:38:55.182684+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:56 smithi177 bash[68594]: audit 2024-09-13T14:38:55.182684+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:56 smithi177 bash[68594]: audit 2024-09-13T14:38:55.192529+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:56 smithi177 bash[68594]: audit 2024-09-13T14:38:55.192529+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:38:56.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:56 smithi177 bash[68594]: cluster 2024-09-13T14:38:56.092715+0000 mon.a (mon.0) 924 : cluster [WRN] Health check update: Degraded data redundancy: 140/1170 objects degraded (11.966%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:56.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:56 smithi177 bash[68594]: cluster 2024-09-13T14:38:56.092715+0000 mon.a (mon.0) 924 : cluster [WRN] Health check update: Degraded data redundancy: 140/1170 objects degraded (11.966%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T14:38:56.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:56 smithi177 bash[68594]: audit 2024-09-13T14:38:56.118082+0000 mon.a (mon.0) 925 : audit [DBG] from='client.? 172.21.15.99:0/2125749568' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:56.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:56 smithi177 bash[68594]: audit 2024-09-13T14:38:56.118082+0000 mon.a (mon.0) 925 : audit [DBG] from='client.? 172.21.15.99:0/2125749568' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:56.821 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:56 smithi099 bash[105859]: debug 2024-09-13T14:38:56.551+0000 7f3944894640 -1 osd.2 656 log_to_monitors true 2024-09-13T14:38:57.296 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:38:57] "GET /metrics HTTP/1.1" 200 67920 "" "Prometheus/2.51.0" 2024-09-13T14:38:57.297 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[105859]: debug 2024-09-13T14:38:57.295+0000 7f393c63e640 -1 osd.2 656 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:38:57.297 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: cluster 2024-09-13T14:38:56.145526+0000 mgr.y (mgr.26252) 335 : cluster [DBG] pgmap v542: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 6.4 KiB/s wr, 186 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:57.297 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: cluster 2024-09-13T14:38:56.145526+0000 mgr.y (mgr.26252) 335 : cluster [DBG] pgmap v542: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 6.4 KiB/s wr, 186 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:57.297 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: cluster 2024-09-13T14:38:56.202254+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e670: 8 total, 7 up, 8 in 2024-09-13T14:38:57.297 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: cluster 2024-09-13T14:38:56.202254+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e670: 8 total, 7 up, 8 in 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.203681+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.99:0/1631605457' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.203681+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.99:0/1631605457' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.204888+0000 mon.a (mon.0) 927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.204888+0000 mon.a (mon.0) 927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.424019+0000 mgr.y (mgr.26252) 336 : audit [DBG] from='client.44618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.424019+0000 mgr.y (mgr.26252) 336 : audit [DBG] from='client.44618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.558320+0000 mon.c (mon.2) 27 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.558320+0000 mon.c (mon.2) 27 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.558653+0000 mon.a (mon.0) 928 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.558653+0000 mon.a (mon.0) 928 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.736689+0000 mgr.y (mgr.26252) 337 : audit [DBG] from='client.44624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.299 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[87147]: audit 2024-09-13T14:38:56.736689+0000 mgr.y (mgr.26252) 337 : audit [DBG] from='client.44624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: cluster 2024-09-13T14:38:56.145526+0000 mgr.y (mgr.26252) 335 : cluster [DBG] pgmap v542: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 6.4 KiB/s wr, 186 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:57.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: cluster 2024-09-13T14:38:56.145526+0000 mgr.y (mgr.26252) 335 : cluster [DBG] pgmap v542: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 6.4 KiB/s wr, 186 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:57.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: cluster 2024-09-13T14:38:56.202254+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e670: 8 total, 7 up, 8 in 2024-09-13T14:38:57.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: cluster 2024-09-13T14:38:56.202254+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e670: 8 total, 7 up, 8 in 2024-09-13T14:38:57.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.203681+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.99:0/1631605457' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.203681+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.99:0/1631605457' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.204888+0000 mon.a (mon.0) 927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.204888+0000 mon.a (mon.0) 927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.424019+0000 mgr.y (mgr.26252) 336 : audit [DBG] from='client.44618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.424019+0000 mgr.y (mgr.26252) 336 : audit [DBG] from='client.44618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.558320+0000 mon.c (mon.2) 27 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.558320+0000 mon.c (mon.2) 27 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.558653+0000 mon.a (mon.0) 928 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.558653+0000 mon.a (mon.0) 928 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.736689+0000 mgr.y (mgr.26252) 337 : audit [DBG] from='client.44624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:57 smithi099 bash[84576]: audit 2024-09-13T14:38:56.736689+0000 mgr.y (mgr.26252) 337 : audit [DBG] from='client.44624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: cluster 2024-09-13T14:38:56.145526+0000 mgr.y (mgr.26252) 335 : cluster [DBG] pgmap v542: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 6.4 KiB/s wr, 186 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:57.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: cluster 2024-09-13T14:38:56.145526+0000 mgr.y (mgr.26252) 335 : cluster [DBG] pgmap v542: 521 pgs: 152 active+undersized, 52 active+undersized+degraded, 317 active+clean; 131 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 6.4 KiB/s wr, 186 op/s; 140/1170 objects degraded (11.966%) 2024-09-13T14:38:57.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: cluster 2024-09-13T14:38:56.202254+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e670: 8 total, 7 up, 8 in 2024-09-13T14:38:57.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: cluster 2024-09-13T14:38:56.202254+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e670: 8 total, 7 up, 8 in 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.203681+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.99:0/1631605457' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.203681+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.99:0/1631605457' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.204888+0000 mon.a (mon.0) 927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.204888+0000 mon.a (mon.0) 927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.424019+0000 mgr.y (mgr.26252) 336 : audit [DBG] from='client.44618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.424019+0000 mgr.y (mgr.26252) 336 : audit [DBG] from='client.44618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.558320+0000 mon.c (mon.2) 27 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.558320+0000 mon.c (mon.2) 27 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.558653+0000 mon.a (mon.0) 928 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.558653+0000 mon.a (mon.0) 928 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.736689+0000 mgr.y (mgr.26252) 337 : audit [DBG] from='client.44624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:57 smithi177 bash[68594]: audit 2024-09-13T14:38:56.736689+0000 mgr.y (mgr.26252) 337 : audit [DBG] from='client.44624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[84576]: audit 2024-09-13T14:38:57.202402+0000 mon.a (mon.0) 929 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:58.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[84576]: audit 2024-09-13T14:38:57.202402+0000 mon.a (mon.0) 929 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[84576]: audit 2024-09-13T14:38:57.202608+0000 mon.a (mon.0) 930 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[84576]: audit 2024-09-13T14:38:57.202608+0000 mon.a (mon.0) 930 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[84576]: cluster 2024-09-13T14:38:57.210896+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e671: 8 total, 7 up, 8 in 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[84576]: cluster 2024-09-13T14:38:57.210896+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e671: 8 total, 7 up, 8 in 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[84576]: audit 2024-09-13T14:38:57.225046+0000 mon.c (mon.2) 28 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[84576]: audit 2024-09-13T14:38:57.225046+0000 mon.c (mon.2) 28 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[84576]: audit 2024-09-13T14:38:57.233805+0000 mon.a (mon.0) 932 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[84576]: audit 2024-09-13T14:38:57.233805+0000 mon.a (mon.0) 932 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[87147]: audit 2024-09-13T14:38:57.202402+0000 mon.a (mon.0) 929 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[87147]: audit 2024-09-13T14:38:57.202402+0000 mon.a (mon.0) 929 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:58.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[87147]: audit 2024-09-13T14:38:57.202608+0000 mon.a (mon.0) 930 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:38:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[87147]: audit 2024-09-13T14:38:57.202608+0000 mon.a (mon.0) 930 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:38:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[87147]: cluster 2024-09-13T14:38:57.210896+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e671: 8 total, 7 up, 8 in 2024-09-13T14:38:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[87147]: cluster 2024-09-13T14:38:57.210896+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e671: 8 total, 7 up, 8 in 2024-09-13T14:38:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[87147]: audit 2024-09-13T14:38:57.225046+0000 mon.c (mon.2) 28 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[87147]: audit 2024-09-13T14:38:57.225046+0000 mon.c (mon.2) 28 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[87147]: audit 2024-09-13T14:38:57.233805+0000 mon.a (mon.0) 932 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:58 smithi099 bash[87147]: audit 2024-09-13T14:38:57.233805+0000 mon.a (mon.0) 932 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:58 smithi177 bash[68594]: audit 2024-09-13T14:38:57.202402+0000 mon.a (mon.0) 929 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:58.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:58 smithi177 bash[68594]: audit 2024-09-13T14:38:57.202402+0000 mon.a (mon.0) 929 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:38:58.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:58 smithi177 bash[68594]: audit 2024-09-13T14:38:57.202608+0000 mon.a (mon.0) 930 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:38:58.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:58 smithi177 bash[68594]: audit 2024-09-13T14:38:57.202608+0000 mon.a (mon.0) 930 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:38:58.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:58 smithi177 bash[68594]: cluster 2024-09-13T14:38:57.210896+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e671: 8 total, 7 up, 8 in 2024-09-13T14:38:58.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:58 smithi177 bash[68594]: cluster 2024-09-13T14:38:57.210896+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e671: 8 total, 7 up, 8 in 2024-09-13T14:38:58.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:58 smithi177 bash[68594]: audit 2024-09-13T14:38:57.225046+0000 mon.c (mon.2) 28 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:58 smithi177 bash[68594]: audit 2024-09-13T14:38:57.225046+0000 mon.c (mon.2) 28 : audit [INF] from='osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:58 smithi177 bash[68594]: audit 2024-09-13T14:38:57.233805+0000 mon.a (mon.0) 932 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:58.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:58 smithi177 bash[68594]: audit 2024-09-13T14:38:57.233805+0000 mon.a (mon.0) 932 : audit [INF] from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:38:59.464 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:38:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: cluster 2024-09-13T14:38:58.147722+0000 mgr.y (mgr.26252) 338 : cluster [DBG] pgmap v545: 553 pgs: 28 unknown, 154 active+undersized, 51 active+undersized+degraded, 320 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 397 KiB/s rd, 8.8 KiB/s wr, 477 op/s; 139/1173 objects degraded (11.850%) 2024-09-13T14:38:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: cluster 2024-09-13T14:38:58.147722+0000 mgr.y (mgr.26252) 338 : cluster [DBG] pgmap v545: 553 pgs: 28 unknown, 154 active+undersized, 51 active+undersized+degraded, 320 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 397 KiB/s rd, 8.8 KiB/s wr, 477 op/s; 139/1173 objects degraded (11.850%) 2024-09-13T14:38:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: cluster 2024-09-13T14:38:58.221294+0000 mon.a (mon.0) 933 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: cluster 2024-09-13T14:38:58.221294+0000 mon.a (mon.0) 933 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: cluster 2024-09-13T14:38:58.240466+0000 mon.a (mon.0) 934 : cluster [INF] osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315] boot 2024-09-13T14:38:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: cluster 2024-09-13T14:38:58.240466+0000 mon.a (mon.0) 934 : cluster [INF] osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315] boot 2024-09-13T14:38:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: cluster 2024-09-13T14:38:58.240515+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-13T14:38:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: cluster 2024-09-13T14:38:58.240515+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-13T14:38:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: audit 2024-09-13T14:38:58.249498+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:38:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: audit 2024-09-13T14:38:58.249498+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: audit 2024-09-13T14:38:58.253588+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.99:0/535538870' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: audit 2024-09-13T14:38:58.253588+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.99:0/535538870' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: audit 2024-09-13T14:38:58.254156+0000 mon.a (mon.0) 937 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: audit 2024-09-13T14:38:58.254156+0000 mon.a (mon.0) 937 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: audit 2024-09-13T14:38:59.043495+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.99:0/3302280110' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[87147]: audit 2024-09-13T14:38:59.043495+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.99:0/3302280110' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: cluster 2024-09-13T14:38:58.147722+0000 mgr.y (mgr.26252) 338 : cluster [DBG] pgmap v545: 553 pgs: 28 unknown, 154 active+undersized, 51 active+undersized+degraded, 320 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 397 KiB/s rd, 8.8 KiB/s wr, 477 op/s; 139/1173 objects degraded (11.850%) 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: cluster 2024-09-13T14:38:58.147722+0000 mgr.y (mgr.26252) 338 : cluster [DBG] pgmap v545: 553 pgs: 28 unknown, 154 active+undersized, 51 active+undersized+degraded, 320 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 397 KiB/s rd, 8.8 KiB/s wr, 477 op/s; 139/1173 objects degraded (11.850%) 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: cluster 2024-09-13T14:38:58.221294+0000 mon.a (mon.0) 933 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: cluster 2024-09-13T14:38:58.221294+0000 mon.a (mon.0) 933 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:59.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: cluster 2024-09-13T14:38:58.240466+0000 mon.a (mon.0) 934 : cluster [INF] osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315] boot 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: cluster 2024-09-13T14:38:58.240466+0000 mon.a (mon.0) 934 : cluster [INF] osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315] boot 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: cluster 2024-09-13T14:38:58.240515+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: cluster 2024-09-13T14:38:58.240515+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: audit 2024-09-13T14:38:58.249498+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: audit 2024-09-13T14:38:58.249498+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: audit 2024-09-13T14:38:58.253588+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.99:0/535538870' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: audit 2024-09-13T14:38:58.253588+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.99:0/535538870' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: audit 2024-09-13T14:38:58.254156+0000 mon.a (mon.0) 937 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: audit 2024-09-13T14:38:58.254156+0000 mon.a (mon.0) 937 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: audit 2024-09-13T14:38:59.043495+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.99:0/3302280110' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:38:59 smithi099 bash[84576]: audit 2024-09-13T14:38:59.043495+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.99:0/3302280110' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:59.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: cluster 2024-09-13T14:38:58.147722+0000 mgr.y (mgr.26252) 338 : cluster [DBG] pgmap v545: 553 pgs: 28 unknown, 154 active+undersized, 51 active+undersized+degraded, 320 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 397 KiB/s rd, 8.8 KiB/s wr, 477 op/s; 139/1173 objects degraded (11.850%) 2024-09-13T14:38:59.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: cluster 2024-09-13T14:38:58.147722+0000 mgr.y (mgr.26252) 338 : cluster [DBG] pgmap v545: 553 pgs: 28 unknown, 154 active+undersized, 51 active+undersized+degraded, 320 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 397 KiB/s rd, 8.8 KiB/s wr, 477 op/s; 139/1173 objects degraded (11.850%) 2024-09-13T14:38:59.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: cluster 2024-09-13T14:38:58.221294+0000 mon.a (mon.0) 933 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:59.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: cluster 2024-09-13T14:38:58.221294+0000 mon.a (mon.0) 933 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:38:59.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: cluster 2024-09-13T14:38:58.240466+0000 mon.a (mon.0) 934 : cluster [INF] osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315] boot 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: cluster 2024-09-13T14:38:58.240466+0000 mon.a (mon.0) 934 : cluster [INF] osd.2 [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315] boot 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: cluster 2024-09-13T14:38:58.240515+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: cluster 2024-09-13T14:38:58.240515+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: audit 2024-09-13T14:38:58.249498+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: audit 2024-09-13T14:38:58.249498+0000 mon.a (mon.0) 936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: audit 2024-09-13T14:38:58.253588+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.99:0/535538870' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: audit 2024-09-13T14:38:58.253588+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.99:0/535538870' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: audit 2024-09-13T14:38:58.254156+0000 mon.a (mon.0) 937 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: audit 2024-09-13T14:38:58.254156+0000 mon.a (mon.0) 937 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: audit 2024-09-13T14:38:59.043495+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.99:0/3302280110' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:38:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:38:59 smithi177 bash[68594]: audit 2024-09-13T14:38:59.043495+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.99:0/3302280110' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:00.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[84576]: audit 2024-09-13T14:38:59.233025+0000 mon.a (mon.0) 938 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:00.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[84576]: audit 2024-09-13T14:38:59.233025+0000 mon.a (mon.0) 938 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:00.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[84576]: cluster 2024-09-13T14:38:59.251646+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-13T14:39:00.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[84576]: cluster 2024-09-13T14:38:59.251646+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-13T14:39:00.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[84576]: audit 2024-09-13T14:38:59.453008+0000 mgr.y (mgr.26252) 339 : audit [DBG] from='client.44645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[84576]: audit 2024-09-13T14:38:59.453008+0000 mgr.y (mgr.26252) 339 : audit [DBG] from='client.44645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[84576]: audit 2024-09-13T14:38:59.736039+0000 mgr.y (mgr.26252) 340 : audit [DBG] from='client.34764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[84576]: audit 2024-09-13T14:38:59.736039+0000 mgr.y (mgr.26252) 340 : audit [DBG] from='client.34764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[87147]: audit 2024-09-13T14:38:59.233025+0000 mon.a (mon.0) 938 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:00.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[87147]: audit 2024-09-13T14:38:59.233025+0000 mon.a (mon.0) 938 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:00.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[87147]: cluster 2024-09-13T14:38:59.251646+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-13T14:39:00.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[87147]: cluster 2024-09-13T14:38:59.251646+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-13T14:39:00.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[87147]: audit 2024-09-13T14:38:59.453008+0000 mgr.y (mgr.26252) 339 : audit [DBG] from='client.44645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[87147]: audit 2024-09-13T14:38:59.453008+0000 mgr.y (mgr.26252) 339 : audit [DBG] from='client.44645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[87147]: audit 2024-09-13T14:38:59.736039+0000 mgr.y (mgr.26252) 340 : audit [DBG] from='client.34764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:00 smithi099 bash[87147]: audit 2024-09-13T14:38:59.736039+0000 mgr.y (mgr.26252) 340 : audit [DBG] from='client.34764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:39:00 smithi177 bash[61439]: ts=2024-09-13T14:39:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:00.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:00 smithi177 bash[68594]: audit 2024-09-13T14:38:59.233025+0000 mon.a (mon.0) 938 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:00.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:00 smithi177 bash[68594]: audit 2024-09-13T14:38:59.233025+0000 mon.a (mon.0) 938 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:00.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:00 smithi177 bash[68594]: cluster 2024-09-13T14:38:59.251646+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-13T14:39:00.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:00 smithi177 bash[68594]: cluster 2024-09-13T14:38:59.251646+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-13T14:39:00.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:00 smithi177 bash[68594]: audit 2024-09-13T14:38:59.453008+0000 mgr.y (mgr.26252) 339 : audit [DBG] from='client.44645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:00 smithi177 bash[68594]: audit 2024-09-13T14:38:59.453008+0000 mgr.y (mgr.26252) 339 : audit [DBG] from='client.44645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:00 smithi177 bash[68594]: audit 2024-09-13T14:38:59.736039+0000 mgr.y (mgr.26252) 340 : audit [DBG] from='client.34764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:00.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:00 smithi177 bash[68594]: audit 2024-09-13T14:38:59.736039+0000 mgr.y (mgr.26252) 340 : audit [DBG] from='client.34764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[84576]: cluster 2024-09-13T14:39:00.149004+0000 mgr.y (mgr.26252) 341 : cluster [DBG] pgmap v548: 585 pgs: 22 creating+peering, 132 peering, 13 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 8.2 KiB/s wr, 494 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[84576]: cluster 2024-09-13T14:39:00.149004+0000 mgr.y (mgr.26252) 341 : cluster [DBG] pgmap v548: 585 pgs: 22 creating+peering, 132 peering, 13 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 8.2 KiB/s wr, 494 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[84576]: cluster 2024-09-13T14:39:00.286223+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-13T14:39:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[84576]: cluster 2024-09-13T14:39:00.286223+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-13T14:39:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[84576]: audit 2024-09-13T14:39:00.302803+0000 mon.c (mon.2) 31 : audit [INF] from='client.? 172.21.15.99:0/2421221295' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[84576]: audit 2024-09-13T14:39:00.302803+0000 mon.c (mon.2) 31 : audit [INF] from='client.? 172.21.15.99:0/2421221295' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[84576]: audit 2024-09-13T14:39:00.303205+0000 mon.a (mon.0) 941 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[84576]: audit 2024-09-13T14:39:00.303205+0000 mon.a (mon.0) 941 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[84576]: cluster 2024-09-13T14:39:01.094357+0000 mon.a (mon.0) 942 : cluster [WRN] Health check update: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:01.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[84576]: cluster 2024-09-13T14:39:01.094357+0000 mon.a (mon.0) 942 : cluster [WRN] Health check update: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:01.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[87147]: cluster 2024-09-13T14:39:00.149004+0000 mgr.y (mgr.26252) 341 : cluster [DBG] pgmap v548: 585 pgs: 22 creating+peering, 132 peering, 13 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 8.2 KiB/s wr, 494 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:01.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[87147]: cluster 2024-09-13T14:39:00.149004+0000 mgr.y (mgr.26252) 341 : cluster [DBG] pgmap v548: 585 pgs: 22 creating+peering, 132 peering, 13 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 8.2 KiB/s wr, 494 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:01.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[87147]: cluster 2024-09-13T14:39:00.286223+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-13T14:39:01.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[87147]: cluster 2024-09-13T14:39:00.286223+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-13T14:39:01.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[87147]: audit 2024-09-13T14:39:00.302803+0000 mon.c (mon.2) 31 : audit [INF] from='client.? 172.21.15.99:0/2421221295' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[87147]: audit 2024-09-13T14:39:00.302803+0000 mon.c (mon.2) 31 : audit [INF] from='client.? 172.21.15.99:0/2421221295' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[87147]: audit 2024-09-13T14:39:00.303205+0000 mon.a (mon.0) 941 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[87147]: audit 2024-09-13T14:39:00.303205+0000 mon.a (mon.0) 941 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[87147]: cluster 2024-09-13T14:39:01.094357+0000 mon.a (mon.0) 942 : cluster [WRN] Health check update: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:01.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:01 smithi099 bash[87147]: cluster 2024-09-13T14:39:01.094357+0000 mon.a (mon.0) 942 : cluster [WRN] Health check update: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:01.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:01 smithi177 bash[68594]: cluster 2024-09-13T14:39:00.149004+0000 mgr.y (mgr.26252) 341 : cluster [DBG] pgmap v548: 585 pgs: 22 creating+peering, 132 peering, 13 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 8.2 KiB/s wr, 494 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:01.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:01 smithi177 bash[68594]: cluster 2024-09-13T14:39:00.149004+0000 mgr.y (mgr.26252) 341 : cluster [DBG] pgmap v548: 585 pgs: 22 creating+peering, 132 peering, 13 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 411 KiB/s rd, 8.2 KiB/s wr, 494 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:01.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:01 smithi177 bash[68594]: cluster 2024-09-13T14:39:00.286223+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-13T14:39:01.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:01 smithi177 bash[68594]: cluster 2024-09-13T14:39:00.286223+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-13T14:39:01.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:01 smithi177 bash[68594]: audit 2024-09-13T14:39:00.302803+0000 mon.c (mon.2) 31 : audit [INF] from='client.? 172.21.15.99:0/2421221295' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:01 smithi177 bash[68594]: audit 2024-09-13T14:39:00.302803+0000 mon.c (mon.2) 31 : audit [INF] from='client.? 172.21.15.99:0/2421221295' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:01 smithi177 bash[68594]: audit 2024-09-13T14:39:00.303205+0000 mon.a (mon.0) 941 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:01 smithi177 bash[68594]: audit 2024-09-13T14:39:00.303205+0000 mon.a (mon.0) 941 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:39:01.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:01 smithi177 bash[68594]: cluster 2024-09-13T14:39:01.094357+0000 mon.a (mon.0) 942 : cluster [WRN] Health check update: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:01.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:01 smithi177 bash[68594]: cluster 2024-09-13T14:39:01.094357+0000 mon.a (mon.0) 942 : cluster [WRN] Health check update: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:02.384 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[84576]: audit 2024-09-13T14:39:01.283982+0000 mon.a (mon.0) 943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[84576]: audit 2024-09-13T14:39:01.283982+0000 mon.a (mon.0) 943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[84576]: cluster 2024-09-13T14:39:01.295012+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-13T14:39:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[84576]: cluster 2024-09-13T14:39:01.295012+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-13T14:39:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[84576]: audit 2024-09-13T14:39:01.301725+0000 mon.a (mon.0) 945 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""} : dispatch 2024-09-13T14:39:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[84576]: audit 2024-09-13T14:39:01.301725+0000 mon.a (mon.0) 945 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""} : dispatch 2024-09-13T14:39:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[84576]: audit 2024-09-13T14:39:02.043910+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.99:0/3779913842' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[84576]: audit 2024-09-13T14:39:02.043910+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.99:0/3779913842' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[84576]: audit 2024-09-13T14:39:02.286643+0000 mon.a (mon.0) 946 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""}]': finished 2024-09-13T14:39:02.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[84576]: audit 2024-09-13T14:39:02.286643+0000 mon.a (mon.0) 946 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""}]': finished 2024-09-13T14:39:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[87147]: audit 2024-09-13T14:39:01.283982+0000 mon.a (mon.0) 943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[87147]: audit 2024-09-13T14:39:01.283982+0000 mon.a (mon.0) 943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[87147]: cluster 2024-09-13T14:39:01.295012+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-13T14:39:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[87147]: cluster 2024-09-13T14:39:01.295012+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-13T14:39:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[87147]: audit 2024-09-13T14:39:01.301725+0000 mon.a (mon.0) 945 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""} : dispatch 2024-09-13T14:39:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[87147]: audit 2024-09-13T14:39:01.301725+0000 mon.a (mon.0) 945 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""} : dispatch 2024-09-13T14:39:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[87147]: audit 2024-09-13T14:39:02.043910+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.99:0/3779913842' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[87147]: audit 2024-09-13T14:39:02.043910+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.99:0/3779913842' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[87147]: audit 2024-09-13T14:39:02.286643+0000 mon.a (mon.0) 946 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""}]': finished 2024-09-13T14:39:02.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:02 smithi099 bash[87147]: audit 2024-09-13T14:39:02.286643+0000 mon.a (mon.0) 946 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""}]': finished 2024-09-13T14:39:02.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:02 smithi177 bash[68594]: audit 2024-09-13T14:39:01.283982+0000 mon.a (mon.0) 943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:02.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:02 smithi177 bash[68594]: audit 2024-09-13T14:39:01.283982+0000 mon.a (mon.0) 943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:39:02.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:02 smithi177 bash[68594]: cluster 2024-09-13T14:39:01.295012+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-13T14:39:02.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:02 smithi177 bash[68594]: cluster 2024-09-13T14:39:01.295012+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-13T14:39:02.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:02 smithi177 bash[68594]: audit 2024-09-13T14:39:01.301725+0000 mon.a (mon.0) 945 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""} : dispatch 2024-09-13T14:39:02.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:02 smithi177 bash[68594]: audit 2024-09-13T14:39:01.301725+0000 mon.a (mon.0) 945 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""} : dispatch 2024-09-13T14:39:02.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:02 smithi177 bash[68594]: audit 2024-09-13T14:39:02.043910+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.99:0/3779913842' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:02.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:02 smithi177 bash[68594]: audit 2024-09-13T14:39:02.043910+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.99:0/3779913842' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:02.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:02 smithi177 bash[68594]: audit 2024-09-13T14:39:02.286643+0000 mon.a (mon.0) 946 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""}]': finished 2024-09-13T14:39:02.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:02 smithi177 bash[68594]: audit 2024-09-13T14:39:02.286643+0000 mon.a (mon.0) 946 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30", "force_nonempty":""}]': finished 2024-09-13T14:39:03.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: cluster 2024-09-13T14:39:02.149925+0000 mgr.y (mgr.26252) 342 : cluster [DBG] pgmap v551: 617 pgs: 22 creating+peering, 132 peering, 45 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 4.0 KiB/s wr, 101 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:03.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: cluster 2024-09-13T14:39:02.149925+0000 mgr.y (mgr.26252) 342 : cluster [DBG] pgmap v551: 617 pgs: 22 creating+peering, 132 peering, 45 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 4.0 KiB/s wr, 101 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:03.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: cluster 2024-09-13T14:39:02.291513+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-13T14:39:03.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: cluster 2024-09-13T14:39:02.291513+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-13T14:39:03.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:02.307584+0000 mon.a (mon.0) 948 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"} : dispatch 2024-09-13T14:39:03.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:02.307584+0000 mon.a (mon.0) 948 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"} : dispatch 2024-09-13T14:39:03.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:02.374282+0000 mgr.y (mgr.26252) 343 : audit [DBG] from='client.44666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:02.374282+0000 mgr.y (mgr.26252) 343 : audit [DBG] from='client.44666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:02.681275+0000 mgr.y (mgr.26252) 344 : audit [DBG] from='client.44672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:02.681275+0000 mgr.y (mgr.26252) 344 : audit [DBG] from='client.44672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:03.257586+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:03.257586+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:03.259901+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:03.259901+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: cluster 2024-09-13T14:39:03.287184+0000 mon.a (mon.0) 951 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T14:39:03.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: cluster 2024-09-13T14:39:03.287184+0000 mon.a (mon.0) 951 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T14:39:03.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:03.289498+0000 mon.a (mon.0) 952 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"}]': finished 2024-09-13T14:39:03.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: cluster 2024-09-13T14:39:02.149925+0000 mgr.y (mgr.26252) 342 : cluster [DBG] pgmap v551: 617 pgs: 22 creating+peering, 132 peering, 45 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 4.0 KiB/s wr, 101 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:03.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: cluster 2024-09-13T14:39:02.149925+0000 mgr.y (mgr.26252) 342 : cluster [DBG] pgmap v551: 617 pgs: 22 creating+peering, 132 peering, 45 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 4.0 KiB/s wr, 101 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:03.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: cluster 2024-09-13T14:39:02.291513+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-13T14:39:03.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: cluster 2024-09-13T14:39:02.291513+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-13T14:39:03.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:02.307584+0000 mon.a (mon.0) 948 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"} : dispatch 2024-09-13T14:39:03.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:02.307584+0000 mon.a (mon.0) 948 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"} : dispatch 2024-09-13T14:39:03.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:02.374282+0000 mgr.y (mgr.26252) 343 : audit [DBG] from='client.44666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:02.374282+0000 mgr.y (mgr.26252) 343 : audit [DBG] from='client.44666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:02.681275+0000 mgr.y (mgr.26252) 344 : audit [DBG] from='client.44672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:02.681275+0000 mgr.y (mgr.26252) 344 : audit [DBG] from='client.44672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:03.257586+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:03.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:03.257586+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:03.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:03.259901+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:03.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:03.259901+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:03.579 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: cluster 2024-09-13T14:39:03.287184+0000 mon.a (mon.0) 951 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T14:39:03.579 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: cluster 2024-09-13T14:39:03.287184+0000 mon.a (mon.0) 951 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T14:39:03.579 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:03.289498+0000 mon.a (mon.0) 952 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"}]': finished 2024-09-13T14:39:03.579 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:03.289498+0000 mon.a (mon.0) 952 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"}]': finished 2024-09-13T14:39:03.579 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: cluster 2024-09-13T14:39:03.297691+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-13T14:39:03.580 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: cluster 2024-09-13T14:39:03.297691+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-13T14:39:03.580 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:03.301877+0000 mon.a (mon.0) 954 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:03.580 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[84576]: audit 2024-09-13T14:39:03.301877+0000 mon.a (mon.0) 954 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:03.580 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:03.289498+0000 mon.a (mon.0) 952 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"}]': finished 2024-09-13T14:39:03.580 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: cluster 2024-09-13T14:39:03.297691+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-13T14:39:03.581 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: cluster 2024-09-13T14:39:03.297691+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-13T14:39:03.581 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:03.301877+0000 mon.a (mon.0) 954 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:03.581 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:03 smithi099 bash[87147]: audit 2024-09-13T14:39:03.301877+0000 mon.a (mon.0) 954 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: cluster 2024-09-13T14:39:02.149925+0000 mgr.y (mgr.26252) 342 : cluster [DBG] pgmap v551: 617 pgs: 22 creating+peering, 132 peering, 45 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 4.0 KiB/s wr, 101 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: cluster 2024-09-13T14:39:02.149925+0000 mgr.y (mgr.26252) 342 : cluster [DBG] pgmap v551: 617 pgs: 22 creating+peering, 132 peering, 45 unknown, 59 active+undersized, 17 active+undersized+degraded, 342 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 4.0 KiB/s wr, 101 op/s; 45/1143 objects degraded (3.937%) 2024-09-13T14:39:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: cluster 2024-09-13T14:39:02.291513+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-13T14:39:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: cluster 2024-09-13T14:39:02.291513+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-13T14:39:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:02.307584+0000 mon.a (mon.0) 948 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"} : dispatch 2024-09-13T14:39:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:02.307584+0000 mon.a (mon.0) 948 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"} : dispatch 2024-09-13T14:39:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:02.374282+0000 mgr.y (mgr.26252) 343 : audit [DBG] from='client.44666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:02.374282+0000 mgr.y (mgr.26252) 343 : audit [DBG] from='client.44666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:02.681275+0000 mgr.y (mgr.26252) 344 : audit [DBG] from='client.44672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:02.681275+0000 mgr.y (mgr.26252) 344 : audit [DBG] from='client.44672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:03.257586+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:03.257586+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:03.259901+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:03.259901+0000 mon.a (mon.0) 950 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: cluster 2024-09-13T14:39:03.287184+0000 mon.a (mon.0) 951 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: cluster 2024-09-13T14:39:03.287184+0000 mon.a (mon.0) 951 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:03.289498+0000 mon.a (mon.0) 952 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"}]': finished 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:03.289498+0000 mon.a (mon.0) 952 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi099-69110-30", "mode":"writeback"}]': finished 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: cluster 2024-09-13T14:39:03.297691+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: cluster 2024-09-13T14:39:03.297691+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:03.301877+0000 mon.a (mon.0) 954 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:03 smithi177 bash[68594]: audit 2024-09-13T14:39:03.301877+0000 mon.a (mon.0) 954 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:04.299 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:05.325 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[84576]: cluster 2024-09-13T14:39:04.154499+0000 mgr.y (mgr.26252) 345 : cluster [DBG] pgmap v554: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 14 B/s, 39 keys/s, 3 objects/s recovering 2024-09-13T14:39:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[84576]: cluster 2024-09-13T14:39:04.154499+0000 mgr.y (mgr.26252) 345 : cluster [DBG] pgmap v554: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 14 B/s, 39 keys/s, 3 objects/s recovering 2024-09-13T14:39:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[84576]: audit 2024-09-13T14:39:04.292269+0000 mon.a (mon.0) 955 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[84576]: audit 2024-09-13T14:39:04.292269+0000 mon.a (mon.0) 955 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[84576]: cluster 2024-09-13T14:39:04.298931+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-13T14:39:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[84576]: cluster 2024-09-13T14:39:04.298931+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-13T14:39:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[84576]: cluster 2024-09-13T14:39:04.318011+0000 mon.a (mon.0) 957 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded) 2024-09-13T14:39:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[84576]: cluster 2024-09-13T14:39:04.318011+0000 mon.a (mon.0) 957 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded) 2024-09-13T14:39:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[84576]: audit 2024-09-13T14:39:04.989258+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.99:0/923112647' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[84576]: audit 2024-09-13T14:39:04.989258+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.99:0/923112647' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[87147]: cluster 2024-09-13T14:39:04.154499+0000 mgr.y (mgr.26252) 345 : cluster [DBG] pgmap v554: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 14 B/s, 39 keys/s, 3 objects/s recovering 2024-09-13T14:39:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[87147]: cluster 2024-09-13T14:39:04.154499+0000 mgr.y (mgr.26252) 345 : cluster [DBG] pgmap v554: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 14 B/s, 39 keys/s, 3 objects/s recovering 2024-09-13T14:39:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[87147]: audit 2024-09-13T14:39:04.292269+0000 mon.a (mon.0) 955 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[87147]: audit 2024-09-13T14:39:04.292269+0000 mon.a (mon.0) 955 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[87147]: cluster 2024-09-13T14:39:04.298931+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-13T14:39:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[87147]: cluster 2024-09-13T14:39:04.298931+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-13T14:39:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[87147]: cluster 2024-09-13T14:39:04.318011+0000 mon.a (mon.0) 957 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded) 2024-09-13T14:39:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[87147]: cluster 2024-09-13T14:39:04.318011+0000 mon.a (mon.0) 957 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded) 2024-09-13T14:39:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[87147]: audit 2024-09-13T14:39:04.989258+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.99:0/923112647' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:05 smithi099 bash[87147]: audit 2024-09-13T14:39:04.989258+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.99:0/923112647' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:05 smithi177 bash[68594]: cluster 2024-09-13T14:39:04.154499+0000 mgr.y (mgr.26252) 345 : cluster [DBG] pgmap v554: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 14 B/s, 39 keys/s, 3 objects/s recovering 2024-09-13T14:39:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:05 smithi177 bash[68594]: cluster 2024-09-13T14:39:04.154499+0000 mgr.y (mgr.26252) 345 : cluster [DBG] pgmap v554: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 14 B/s, 39 keys/s, 3 objects/s recovering 2024-09-13T14:39:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:05 smithi177 bash[68594]: audit 2024-09-13T14:39:04.292269+0000 mon.a (mon.0) 955 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:05 smithi177 bash[68594]: audit 2024-09-13T14:39:04.292269+0000 mon.a (mon.0) 955 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi099-69110-28", "overlaypool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:05.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:05 smithi177 bash[68594]: cluster 2024-09-13T14:39:04.298931+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-13T14:39:05.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:05 smithi177 bash[68594]: cluster 2024-09-13T14:39:04.298931+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-13T14:39:05.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:05 smithi177 bash[68594]: cluster 2024-09-13T14:39:04.318011+0000 mon.a (mon.0) 957 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded) 2024-09-13T14:39:05.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:05 smithi177 bash[68594]: cluster 2024-09-13T14:39:04.318011+0000 mon.a (mon.0) 957 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1143 objects degraded (3.937%), 17 pgs degraded) 2024-09-13T14:39:05.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:05 smithi177 bash[68594]: audit 2024-09-13T14:39:04.989258+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.99:0/923112647' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:05.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:05 smithi177 bash[68594]: audit 2024-09-13T14:39:04.989258+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.99:0/923112647' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[84576]: audit 2024-09-13T14:39:05.312502+0000 mgr.y (mgr.26252) 346 : audit [DBG] from='client.44684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[84576]: audit 2024-09-13T14:39:05.312502+0000 mgr.y (mgr.26252) 346 : audit [DBG] from='client.44684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[84576]: cluster 2024-09-13T14:39:05.327212+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-13T14:39:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[84576]: cluster 2024-09-13T14:39:05.327212+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-13T14:39:06.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[84576]: audit 2024-09-13T14:39:05.631999+0000 mgr.y (mgr.26252) 347 : audit [DBG] from='client.44690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[84576]: audit 2024-09-13T14:39:05.631999+0000 mgr.y (mgr.26252) 347 : audit [DBG] from='client.44690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[84576]: cluster 2024-09-13T14:39:06.103082+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-13T14:39:06.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[84576]: cluster 2024-09-13T14:39:06.103082+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-13T14:39:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[87147]: audit 2024-09-13T14:39:05.312502+0000 mgr.y (mgr.26252) 346 : audit [DBG] from='client.44684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[87147]: audit 2024-09-13T14:39:05.312502+0000 mgr.y (mgr.26252) 346 : audit [DBG] from='client.44684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[87147]: cluster 2024-09-13T14:39:05.327212+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-13T14:39:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[87147]: cluster 2024-09-13T14:39:05.327212+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-13T14:39:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[87147]: audit 2024-09-13T14:39:05.631999+0000 mgr.y (mgr.26252) 347 : audit [DBG] from='client.44690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[87147]: audit 2024-09-13T14:39:05.631999+0000 mgr.y (mgr.26252) 347 : audit [DBG] from='client.44690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[87147]: cluster 2024-09-13T14:39:06.103082+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-13T14:39:06.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:06 smithi099 bash[87147]: cluster 2024-09-13T14:39:06.103082+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-13T14:39:06.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:06 smithi177 bash[68594]: audit 2024-09-13T14:39:05.312502+0000 mgr.y (mgr.26252) 346 : audit [DBG] from='client.44684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:06 smithi177 bash[68594]: audit 2024-09-13T14:39:05.312502+0000 mgr.y (mgr.26252) 346 : audit [DBG] from='client.44684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:06 smithi177 bash[68594]: cluster 2024-09-13T14:39:05.327212+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-13T14:39:06.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:06 smithi177 bash[68594]: cluster 2024-09-13T14:39:05.327212+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-13T14:39:06.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:06 smithi177 bash[68594]: audit 2024-09-13T14:39:05.631999+0000 mgr.y (mgr.26252) 347 : audit [DBG] from='client.44690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:06 smithi177 bash[68594]: audit 2024-09-13T14:39:05.631999+0000 mgr.y (mgr.26252) 347 : audit [DBG] from='client.44690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:06.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:06 smithi177 bash[68594]: cluster 2024-09-13T14:39:06.103082+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-13T14:39:06.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:06 smithi177 bash[68594]: cluster 2024-09-13T14:39:06.103082+0000 mon.a (mon.0) 960 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-13T14:39:07.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:39:07] "GET /metrics HTTP/1.1" 200 75086 "" "Prometheus/2.51.0" 2024-09-13T14:39:07.326 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: cluster 2024-09-13T14:39:06.155406+0000 mgr.y (mgr.26252) 348 : cluster [DBG] pgmap v558: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 15 B/s, 41 keys/s, 4 objects/s recovering 2024-09-13T14:39:07.327 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: cluster 2024-09-13T14:39:06.155406+0000 mgr.y (mgr.26252) 348 : cluster [DBG] pgmap v558: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 15 B/s, 41 keys/s, 4 objects/s recovering 2024-09-13T14:39:07.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: cluster 2024-09-13T14:39:07.113617+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-13T14:39:07.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: cluster 2024-09-13T14:39:07.113617+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-13T14:39:07.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.127601+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.127601+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.142567+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.142567+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.145393+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.145393+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.147425+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.147425+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.166564+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.166564+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.267398+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.267398+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.269651+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.269651+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.271267+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.271267+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.272452+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.272452+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.273949+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:39:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:07 smithi177 bash[68594]: audit 2024-09-13T14:39:07.273949+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:39:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: cluster 2024-09-13T14:39:06.155406+0000 mgr.y (mgr.26252) 348 : cluster [DBG] pgmap v558: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 15 B/s, 41 keys/s, 4 objects/s recovering 2024-09-13T14:39:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: cluster 2024-09-13T14:39:06.155406+0000 mgr.y (mgr.26252) 348 : cluster [DBG] pgmap v558: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 15 B/s, 41 keys/s, 4 objects/s recovering 2024-09-13T14:39:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: cluster 2024-09-13T14:39:07.113617+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-13T14:39:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: cluster 2024-09-13T14:39:07.113617+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-13T14:39:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.127601+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.127601+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.142567+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.142567+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.145393+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.145393+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.147425+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.147425+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.166564+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.166564+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.267398+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.267398+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.269651+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.269651+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.271267+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.271267+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.272452+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.272452+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.273949+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:39:07.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[87147]: audit 2024-09-13T14:39:07.273949+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:39:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: cluster 2024-09-13T14:39:06.155406+0000 mgr.y (mgr.26252) 348 : cluster [DBG] pgmap v558: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 15 B/s, 41 keys/s, 4 objects/s recovering 2024-09-13T14:39:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: cluster 2024-09-13T14:39:06.155406+0000 mgr.y (mgr.26252) 348 : cluster [DBG] pgmap v558: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 15 B/s, 41 keys/s, 4 objects/s recovering 2024-09-13T14:39:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: cluster 2024-09-13T14:39:07.113617+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-13T14:39:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: cluster 2024-09-13T14:39:07.113617+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-13T14:39:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.127601+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.127601+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.142567+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.142567+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.145393+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.145393+0000 mon.a (mon.0) 964 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.147425+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.147425+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.166564+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.166564+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.267398+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.267398+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.269651+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.269651+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.271267+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.271267+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.272452+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.272452+0000 mon.a (mon.0) 970 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.273949+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:39:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:07 smithi099 bash[84576]: audit 2024-09-13T14:39:07.273949+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:39:08.289 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:08.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:08 smithi177 bash[68594]: audit 2024-09-13T14:39:07.274138+0000 mgr.y (mgr.26252) 349 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:39:08.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:08 smithi177 bash[68594]: audit 2024-09-13T14:39:07.274138+0000 mgr.y (mgr.26252) 349 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:39:08.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:08 smithi177 bash[68594]: cephadm 2024-09-13T14:39:07.276023+0000 mgr.y (mgr.26252) 350 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-13T14:39:08.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:08 smithi177 bash[68594]: cephadm 2024-09-13T14:39:07.276023+0000 mgr.y (mgr.26252) 350 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-13T14:39:08.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:08 smithi177 bash[68594]: audit 2024-09-13T14:39:07.947497+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.99:0/3006749126' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:08.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:08 smithi177 bash[68594]: audit 2024-09-13T14:39:07.947497+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.99:0/3006749126' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:08.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:08 smithi177 bash[68594]: cluster 2024-09-13T14:39:08.172276+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-13T14:39:08.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:08 smithi177 bash[68594]: cluster 2024-09-13T14:39:08.172276+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-13T14:39:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[84576]: audit 2024-09-13T14:39:07.274138+0000 mgr.y (mgr.26252) 349 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:39:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[84576]: audit 2024-09-13T14:39:07.274138+0000 mgr.y (mgr.26252) 349 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:39:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[84576]: cephadm 2024-09-13T14:39:07.276023+0000 mgr.y (mgr.26252) 350 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-13T14:39:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[84576]: cephadm 2024-09-13T14:39:07.276023+0000 mgr.y (mgr.26252) 350 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-13T14:39:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[84576]: audit 2024-09-13T14:39:07.947497+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.99:0/3006749126' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:08.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[84576]: audit 2024-09-13T14:39:07.947497+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.99:0/3006749126' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:08.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[84576]: cluster 2024-09-13T14:39:08.172276+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-13T14:39:08.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[84576]: cluster 2024-09-13T14:39:08.172276+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-13T14:39:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[87147]: audit 2024-09-13T14:39:07.274138+0000 mgr.y (mgr.26252) 349 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:39:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[87147]: audit 2024-09-13T14:39:07.274138+0000 mgr.y (mgr.26252) 349 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:39:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[87147]: cephadm 2024-09-13T14:39:07.276023+0000 mgr.y (mgr.26252) 350 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-13T14:39:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[87147]: cephadm 2024-09-13T14:39:07.276023+0000 mgr.y (mgr.26252) 350 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-13T14:39:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[87147]: audit 2024-09-13T14:39:07.947497+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.99:0/3006749126' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[87147]: audit 2024-09-13T14:39:07.947497+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.99:0/3006749126' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[87147]: cluster 2024-09-13T14:39:08.172276+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-13T14:39:08.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:08 smithi099 bash[87147]: cluster 2024-09-13T14:39:08.172276+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-13T14:39:09.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[84576]: cluster 2024-09-13T14:39:08.157503+0000 mgr.y (mgr.26252) 351 : cluster [DBG] pgmap v560: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 511 B/s wr, 5 op/s 2024-09-13T14:39:09.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[84576]: cluster 2024-09-13T14:39:08.157503+0000 mgr.y (mgr.26252) 351 : cluster [DBG] pgmap v560: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 511 B/s wr, 5 op/s 2024-09-13T14:39:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[84576]: audit 2024-09-13T14:39:08.278476+0000 mgr.y (mgr.26252) 352 : audit [DBG] from='client.44702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[84576]: audit 2024-09-13T14:39:08.278476+0000 mgr.y (mgr.26252) 352 : audit [DBG] from='client.44702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[84576]: audit 2024-09-13T14:39:08.601685+0000 mgr.y (mgr.26252) 353 : audit [DBG] from='client.44708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[84576]: audit 2024-09-13T14:39:08.601685+0000 mgr.y (mgr.26252) 353 : audit [DBG] from='client.44708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[84576]: cluster 2024-09-13T14:39:09.175572+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-13T14:39:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[84576]: cluster 2024-09-13T14:39:09.175572+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-13T14:39:09.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[87147]: cluster 2024-09-13T14:39:08.157503+0000 mgr.y (mgr.26252) 351 : cluster [DBG] pgmap v560: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 511 B/s wr, 5 op/s 2024-09-13T14:39:09.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[87147]: cluster 2024-09-13T14:39:08.157503+0000 mgr.y (mgr.26252) 351 : cluster [DBG] pgmap v560: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 511 B/s wr, 5 op/s 2024-09-13T14:39:09.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[87147]: audit 2024-09-13T14:39:08.278476+0000 mgr.y (mgr.26252) 352 : audit [DBG] from='client.44702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[87147]: audit 2024-09-13T14:39:08.278476+0000 mgr.y (mgr.26252) 352 : audit [DBG] from='client.44702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[87147]: audit 2024-09-13T14:39:08.601685+0000 mgr.y (mgr.26252) 353 : audit [DBG] from='client.44708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[87147]: audit 2024-09-13T14:39:08.601685+0000 mgr.y (mgr.26252) 353 : audit [DBG] from='client.44708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[87147]: cluster 2024-09-13T14:39:09.175572+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-13T14:39:09.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:09 smithi099 bash[87147]: cluster 2024-09-13T14:39:09.175572+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-13T14:39:09.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:09 smithi177 bash[68594]: cluster 2024-09-13T14:39:08.157503+0000 mgr.y (mgr.26252) 351 : cluster [DBG] pgmap v560: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 511 B/s wr, 5 op/s 2024-09-13T14:39:09.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:09 smithi177 bash[68594]: cluster 2024-09-13T14:39:08.157503+0000 mgr.y (mgr.26252) 351 : cluster [DBG] pgmap v560: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 511 B/s wr, 5 op/s 2024-09-13T14:39:09.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:09 smithi177 bash[68594]: audit 2024-09-13T14:39:08.278476+0000 mgr.y (mgr.26252) 352 : audit [DBG] from='client.44702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:09 smithi177 bash[68594]: audit 2024-09-13T14:39:08.278476+0000 mgr.y (mgr.26252) 352 : audit [DBG] from='client.44702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:09 smithi177 bash[68594]: audit 2024-09-13T14:39:08.601685+0000 mgr.y (mgr.26252) 353 : audit [DBG] from='client.44708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:09 smithi177 bash[68594]: audit 2024-09-13T14:39:08.601685+0000 mgr.y (mgr.26252) 353 : audit [DBG] from='client.44708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:09.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:09 smithi177 bash[68594]: cluster 2024-09-13T14:39:09.175572+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-13T14:39:09.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:09 smithi177 bash[68594]: cluster 2024-09-13T14:39:09.175572+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-13T14:39:10.231 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.231 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.231 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.231 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.231 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.231 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 1 2024-09-13T14:39:10.231 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:39:10.231 INFO:tasks.workunit.client.0.smithi099.stdout:child: image101 2024-09-13T14:39:10.231 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101/85e320d436e2 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.232 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 2 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout:child: image102 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout:child: image101 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101/85e320d436e2 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102/85e3aabd9e5f 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.233 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:10.234 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 3 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 3 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout:child: image102 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout:child: image101 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout:child: image103 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101/85e320d436e2 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102/85e3aabd9e5f 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image103/85e3eb692172 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.235 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 3 2024-09-13T14:39:10.236 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 3 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout:child: image102 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout:child: image101 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout:child: image103 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101/85e320d436e2 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102/85e3aabd9e5f 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image103/85e3eb692172 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.237 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:10.238 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 4 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 4 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout:child: image102 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout:child: image101 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout:child: image103 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout:child: image104 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101/85e320d436e2 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102/85e3aabd9e5f 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image103/85e3eb692172 2024-09-13T14:39:10.239 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image104/85e39d9593ce 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.240 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.241 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.241 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:10.241 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.241 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:10.241 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:10.241 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:10.241 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.241 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:10.241 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:10.241 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:10.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:39:10 smithi177 bash[61439]: ts=2024-09-13T14:39:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:11.336 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: cluster 2024-09-13T14:39:10.158892+0000 mgr.y (mgr.26252) 354 : cluster [DBG] pgmap v563: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 KiB/s wr, 42 op/s 2024-09-13T14:39:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: cluster 2024-09-13T14:39:10.158892+0000 mgr.y (mgr.26252) 354 : cluster [DBG] pgmap v563: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 KiB/s wr, 42 op/s 2024-09-13T14:39:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: cephadm 2024-09-13T14:39:10.478287+0000 mgr.y (mgr.26252) 355 : cephadm [INF] Upgrade: Updating osd.3 2024-09-13T14:39:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: cephadm 2024-09-13T14:39:10.478287+0000 mgr.y (mgr.26252) 355 : cephadm [INF] Upgrade: Updating osd.3 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: audit 2024-09-13T14:39:10.484232+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: audit 2024-09-13T14:39:10.484232+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: audit 2024-09-13T14:39:10.487973+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: audit 2024-09-13T14:39:10.487973+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: audit 2024-09-13T14:39:10.488995+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: audit 2024-09-13T14:39:10.488995+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: cephadm 2024-09-13T14:39:10.499307+0000 mgr.y (mgr.26252) 356 : cephadm [INF] Deploying daemon osd.3 on smithi099 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: cephadm 2024-09-13T14:39:10.499307+0000 mgr.y (mgr.26252) 356 : cephadm [INF] Deploying daemon osd.3 on smithi099 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: audit 2024-09-13T14:39:10.931010+0000 mon.a (mon.0) 977 : audit [DBG] from='client.? 172.21.15.99:0/2294975301' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: audit 2024-09-13T14:39:10.931010+0000 mon.a (mon.0) 977 : audit [DBG] from='client.? 172.21.15.99:0/2294975301' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: cluster 2024-09-13T14:39:11.107491+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-13T14:39:11.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: cluster 2024-09-13T14:39:11.107491+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-13T14:39:11.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: audit 2024-09-13T14:39:11.321454+0000 mon.a (mon.0) 979 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"} : dispatch 2024-09-13T14:39:11.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[87147]: audit 2024-09-13T14:39:11.321454+0000 mon.a (mon.0) 979 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"} : dispatch 2024-09-13T14:39:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: cluster 2024-09-13T14:39:10.158892+0000 mgr.y (mgr.26252) 354 : cluster [DBG] pgmap v563: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 KiB/s wr, 42 op/s 2024-09-13T14:39:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: cluster 2024-09-13T14:39:10.158892+0000 mgr.y (mgr.26252) 354 : cluster [DBG] pgmap v563: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 KiB/s wr, 42 op/s 2024-09-13T14:39:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: cephadm 2024-09-13T14:39:10.478287+0000 mgr.y (mgr.26252) 355 : cephadm [INF] Upgrade: Updating osd.3 2024-09-13T14:39:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: cephadm 2024-09-13T14:39:10.478287+0000 mgr.y (mgr.26252) 355 : cephadm [INF] Upgrade: Updating osd.3 2024-09-13T14:39:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: audit 2024-09-13T14:39:10.484232+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: audit 2024-09-13T14:39:10.484232+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: audit 2024-09-13T14:39:10.487973+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:39:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: audit 2024-09-13T14:39:10.487973+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:39:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: audit 2024-09-13T14:39:10.488995+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: audit 2024-09-13T14:39:10.488995+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: cephadm 2024-09-13T14:39:10.499307+0000 mgr.y (mgr.26252) 356 : cephadm [INF] Deploying daemon osd.3 on smithi099 2024-09-13T14:39:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: cephadm 2024-09-13T14:39:10.499307+0000 mgr.y (mgr.26252) 356 : cephadm [INF] Deploying daemon osd.3 on smithi099 2024-09-13T14:39:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: audit 2024-09-13T14:39:10.931010+0000 mon.a (mon.0) 977 : audit [DBG] from='client.? 172.21.15.99:0/2294975301' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: audit 2024-09-13T14:39:10.931010+0000 mon.a (mon.0) 977 : audit [DBG] from='client.? 172.21.15.99:0/2294975301' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: cluster 2024-09-13T14:39:11.107491+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-13T14:39:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: cluster 2024-09-13T14:39:11.107491+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-13T14:39:11.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: audit 2024-09-13T14:39:11.321454+0000 mon.a (mon.0) 979 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"} : dispatch 2024-09-13T14:39:11.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:11 smithi099 bash[84576]: audit 2024-09-13T14:39:11.321454+0000 mon.a (mon.0) 979 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"} : dispatch 2024-09-13T14:39:11.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: cluster 2024-09-13T14:39:10.158892+0000 mgr.y (mgr.26252) 354 : cluster [DBG] pgmap v563: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 KiB/s wr, 42 op/s 2024-09-13T14:39:11.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: cluster 2024-09-13T14:39:10.158892+0000 mgr.y (mgr.26252) 354 : cluster [DBG] pgmap v563: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 KiB/s wr, 42 op/s 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: cephadm 2024-09-13T14:39:10.478287+0000 mgr.y (mgr.26252) 355 : cephadm [INF] Upgrade: Updating osd.3 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: cephadm 2024-09-13T14:39:10.478287+0000 mgr.y (mgr.26252) 355 : cephadm [INF] Upgrade: Updating osd.3 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: audit 2024-09-13T14:39:10.484232+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: audit 2024-09-13T14:39:10.484232+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: audit 2024-09-13T14:39:10.487973+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: audit 2024-09-13T14:39:10.487973+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: audit 2024-09-13T14:39:10.488995+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: audit 2024-09-13T14:39:10.488995+0000 mon.a (mon.0) 976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: cephadm 2024-09-13T14:39:10.499307+0000 mgr.y (mgr.26252) 356 : cephadm [INF] Deploying daemon osd.3 on smithi099 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: cephadm 2024-09-13T14:39:10.499307+0000 mgr.y (mgr.26252) 356 : cephadm [INF] Deploying daemon osd.3 on smithi099 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: audit 2024-09-13T14:39:10.931010+0000 mon.a (mon.0) 977 : audit [DBG] from='client.? 172.21.15.99:0/2294975301' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: audit 2024-09-13T14:39:10.931010+0000 mon.a (mon.0) 977 : audit [DBG] from='client.? 172.21.15.99:0/2294975301' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: cluster 2024-09-13T14:39:11.107491+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-13T14:39:11.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: cluster 2024-09-13T14:39:11.107491+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-13T14:39:11.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: audit 2024-09-13T14:39:11.321454+0000 mon.a (mon.0) 979 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"} : dispatch 2024-09-13T14:39:11.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:11 smithi177 bash[68594]: audit 2024-09-13T14:39:11.321454+0000 mon.a (mon.0) 979 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"} : dispatch 2024-09-13T14:39:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[84576]: audit 2024-09-13T14:39:11.326472+0000 mgr.y (mgr.26252) 357 : audit [DBG] from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[84576]: audit 2024-09-13T14:39:11.326472+0000 mgr.y (mgr.26252) 357 : audit [DBG] from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[84576]: audit 2024-09-13T14:39:11.640185+0000 mgr.y (mgr.26252) 358 : audit [DBG] from='client.44726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[84576]: audit 2024-09-13T14:39:11.640185+0000 mgr.y (mgr.26252) 358 : audit [DBG] from='client.44726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[84576]: audit 2024-09-13T14:39:12.122917+0000 mon.a (mon.0) 980 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"}]': finished 2024-09-13T14:39:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[84576]: audit 2024-09-13T14:39:12.122917+0000 mon.a (mon.0) 980 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"}]': finished 2024-09-13T14:39:12.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[84576]: cluster 2024-09-13T14:39:12.135410+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-13T14:39:12.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[84576]: cluster 2024-09-13T14:39:12.135410+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-13T14:39:12.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[84576]: audit 2024-09-13T14:39:12.140976+0000 mon.a (mon.0) 982 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:12.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[84576]: audit 2024-09-13T14:39:12.140976+0000 mon.a (mon.0) 982 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:12.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[87147]: audit 2024-09-13T14:39:11.326472+0000 mgr.y (mgr.26252) 357 : audit [DBG] from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[87147]: audit 2024-09-13T14:39:11.326472+0000 mgr.y (mgr.26252) 357 : audit [DBG] from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[87147]: audit 2024-09-13T14:39:11.640185+0000 mgr.y (mgr.26252) 358 : audit [DBG] from='client.44726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[87147]: audit 2024-09-13T14:39:11.640185+0000 mgr.y (mgr.26252) 358 : audit [DBG] from='client.44726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[87147]: audit 2024-09-13T14:39:12.122917+0000 mon.a (mon.0) 980 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"}]': finished 2024-09-13T14:39:12.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[87147]: audit 2024-09-13T14:39:12.122917+0000 mon.a (mon.0) 980 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"}]': finished 2024-09-13T14:39:12.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[87147]: cluster 2024-09-13T14:39:12.135410+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-13T14:39:12.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[87147]: cluster 2024-09-13T14:39:12.135410+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-13T14:39:12.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[87147]: audit 2024-09-13T14:39:12.140976+0000 mon.a (mon.0) 982 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:12.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:12 smithi099 bash[87147]: audit 2024-09-13T14:39:12.140976+0000 mon.a (mon.0) 982 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:12.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:12 smithi177 bash[68594]: audit 2024-09-13T14:39:11.326472+0000 mgr.y (mgr.26252) 357 : audit [DBG] from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:12 smithi177 bash[68594]: audit 2024-09-13T14:39:11.326472+0000 mgr.y (mgr.26252) 357 : audit [DBG] from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:12 smithi177 bash[68594]: audit 2024-09-13T14:39:11.640185+0000 mgr.y (mgr.26252) 358 : audit [DBG] from='client.44726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:12 smithi177 bash[68594]: audit 2024-09-13T14:39:11.640185+0000 mgr.y (mgr.26252) 358 : audit [DBG] from='client.44726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:12.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:12 smithi177 bash[68594]: audit 2024-09-13T14:39:12.122917+0000 mon.a (mon.0) 980 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"}]': finished 2024-09-13T14:39:12.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:12 smithi177 bash[68594]: audit 2024-09-13T14:39:12.122917+0000 mon.a (mon.0) 980 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi099-69110-28"}]': finished 2024-09-13T14:39:12.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:12 smithi177 bash[68594]: cluster 2024-09-13T14:39:12.135410+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-13T14:39:12.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:12 smithi177 bash[68594]: cluster 2024-09-13T14:39:12.135410+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-13T14:39:12.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:12 smithi177 bash[68594]: audit 2024-09-13T14:39:12.140976+0000 mon.a (mon.0) 982 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:12.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:12 smithi177 bash[68594]: audit 2024-09-13T14:39:12.140976+0000 mon.a (mon.0) 982 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"} : dispatch 2024-09-13T14:39:13.131 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image101 2024-09-13T14:39:13.131 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 4 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 4 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout:child: image102 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout:child: image101 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout:child: image103 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout:child: image104 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image101/85e320d436e2 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.132 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102/85e3aabd9e5f 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image103/85e3eb692172 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image104/85e39d9593ce 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.133 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image103 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 3 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 3 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:child: image102 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:child: image103 2024-09-13T14:39:13.134 INFO:tasks.workunit.client.0.smithi099.stdout:child: image104 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102/85e3aabd9e5f 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image103/85e3eb692172 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image104/85e39d9593ce 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:13.135 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.136 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:13.136 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image104 2024-09-13T14:39:13.136 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.136 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 2 2024-09-13T14:39:13.136 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 2 2024-09-13T14:39:13.136 INFO:tasks.workunit.client.0.smithi099.stdout:child: image102 2024-09-13T14:39:13.136 INFO:tasks.workunit.client.0.smithi099.stdout:child: image104 2024-09-13T14:39:13.136 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102/85e3aabd9e5f 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-29/image104/85e39d9593ce 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout:trying to find test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout:checking test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 1 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 1 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout:child: image102 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-28/image102/85e3aabd9e5f 2024-09-13T14:39:13.137 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:39:13.139 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 0 2024-09-13T14:39:13.139 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 0 2024-09-13T14:39:13.139 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ListChildrenTiered (17818 ms) 2024-09-13T14:39:13.139 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.LockingPP 2024-09-13T14:39:13.139 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:13.254 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.LockingPP (123 ms) 2024-09-13T14:39:13.254 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.FlushAio 2024-09-13T14:39:13.254 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:13.629 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.FlushAio (375 ms) 2024-09-13T14:39:13.630 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.FlushAioPP 2024-09-13T14:39:13.630 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:13.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[87147]: cluster 2024-09-13T14:39:12.160017+0000 mgr.y (mgr.26252) 359 : cluster [DBG] pgmap v566: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.0 KiB/s wr, 37 op/s 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[87147]: cluster 2024-09-13T14:39:12.160017+0000 mgr.y (mgr.26252) 359 : cluster [DBG] pgmap v566: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.0 KiB/s wr, 37 op/s 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[87147]: cluster 2024-09-13T14:39:13.124215+0000 mon.a (mon.0) 983 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[87147]: cluster 2024-09-13T14:39:13.124215+0000 mon.a (mon.0) 983 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[87147]: cluster 2024-09-13T14:39:13.124279+0000 mon.a (mon.0) 984 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[87147]: cluster 2024-09-13T14:39:13.124279+0000 mon.a (mon.0) 984 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[87147]: audit 2024-09-13T14:39:13.134055+0000 mon.a (mon.0) 985 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[87147]: audit 2024-09-13T14:39:13.134055+0000 mon.a (mon.0) 985 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[87147]: cluster 2024-09-13T14:39:13.143771+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[87147]: cluster 2024-09-13T14:39:13.143771+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[84576]: cluster 2024-09-13T14:39:12.160017+0000 mgr.y (mgr.26252) 359 : cluster [DBG] pgmap v566: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.0 KiB/s wr, 37 op/s 2024-09-13T14:39:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[84576]: cluster 2024-09-13T14:39:12.160017+0000 mgr.y (mgr.26252) 359 : cluster [DBG] pgmap v566: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.0 KiB/s wr, 37 op/s 2024-09-13T14:39:13.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[84576]: cluster 2024-09-13T14:39:13.124215+0000 mon.a (mon.0) 983 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T14:39:13.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[84576]: cluster 2024-09-13T14:39:13.124215+0000 mon.a (mon.0) 983 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T14:39:13.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[84576]: cluster 2024-09-13T14:39:13.124279+0000 mon.a (mon.0) 984 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:13.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[84576]: cluster 2024-09-13T14:39:13.124279+0000 mon.a (mon.0) 984 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:13.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[84576]: audit 2024-09-13T14:39:13.134055+0000 mon.a (mon.0) 985 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:13.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[84576]: audit 2024-09-13T14:39:13.134055+0000 mon.a (mon.0) 985 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:13.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[84576]: cluster 2024-09-13T14:39:13.143771+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-13T14:39:13.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:13 smithi099 bash[84576]: cluster 2024-09-13T14:39:13.143771+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-13T14:39:13.840 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.FlushAioPP (211 ms) 2024-09-13T14:39:13.840 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SnapDiff 2024-09-13T14:39:13.840 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:13.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:13 smithi177 bash[68594]: cluster 2024-09-13T14:39:12.160017+0000 mgr.y (mgr.26252) 359 : cluster [DBG] pgmap v566: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.0 KiB/s wr, 37 op/s 2024-09-13T14:39:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:13 smithi177 bash[68594]: cluster 2024-09-13T14:39:12.160017+0000 mgr.y (mgr.26252) 359 : cluster [DBG] pgmap v566: 617 pgs: 617 active+clean; 131 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.0 KiB/s wr, 37 op/s 2024-09-13T14:39:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:13 smithi177 bash[68594]: cluster 2024-09-13T14:39:13.124215+0000 mon.a (mon.0) 983 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T14:39:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:13 smithi177 bash[68594]: cluster 2024-09-13T14:39:13.124215+0000 mon.a (mon.0) 983 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T14:39:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:13 smithi177 bash[68594]: cluster 2024-09-13T14:39:13.124279+0000 mon.a (mon.0) 984 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:13 smithi177 bash[68594]: cluster 2024-09-13T14:39:13.124279+0000 mon.a (mon.0) 984 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:13 smithi177 bash[68594]: audit 2024-09-13T14:39:13.134055+0000 mon.a (mon.0) 985 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:13 smithi177 bash[68594]: audit 2024-09-13T14:39:13.134055+0000 mon.a (mon.0) 985 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi099-69110-28", "tierpool":"test-librbd-smithi099-69110-30"}]': finished 2024-09-13T14:39:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:13 smithi177 bash[68594]: cluster 2024-09-13T14:39:13.143771+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-13T14:39:13.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:13 smithi177 bash[68594]: cluster 2024-09-13T14:39:13.143771+0000 mon.a (mon.0) 986 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-13T14:39:14.302 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:15.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:14 smithi099 bash[84576]: audit 2024-09-13T14:39:13.734931+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:15.021 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:14 smithi099 bash[84576]: audit 2024-09-13T14:39:13.734931+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:15.021 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:14 smithi099 bash[84576]: audit 2024-09-13T14:39:13.976372+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.99:0/2735575389' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:15.021 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:14 smithi099 bash[84576]: audit 2024-09-13T14:39:13.976372+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.99:0/2735575389' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:15.021 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:14 smithi099 bash[87147]: audit 2024-09-13T14:39:13.734931+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:15.021 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:14 smithi099 bash[87147]: audit 2024-09-13T14:39:13.734931+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:15.021 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:14 smithi099 bash[87147]: audit 2024-09-13T14:39:13.976372+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.99:0/2735575389' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:15.021 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:14 smithi099 bash[87147]: audit 2024-09-13T14:39:13.976372+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.99:0/2735575389' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:14 smithi177 bash[68594]: audit 2024-09-13T14:39:13.734931+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:14 smithi177 bash[68594]: audit 2024-09-13T14:39:13.734931+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:14 smithi177 bash[68594]: audit 2024-09-13T14:39:13.976372+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.99:0/2735575389' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:15.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:14 smithi177 bash[68594]: audit 2024-09-13T14:39:13.976372+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.99:0/2735575389' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:16.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[84576]: cluster 2024-09-13T14:39:14.164287+0000 mgr.y (mgr.26252) 360 : cluster [DBG] pgmap v568: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 523 KiB/s rd, 67 KiB/s wr, 697 op/s 2024-09-13T14:39:16.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[84576]: cluster 2024-09-13T14:39:14.164287+0000 mgr.y (mgr.26252) 360 : cluster [DBG] pgmap v568: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 523 KiB/s rd, 67 KiB/s wr, 697 op/s 2024-09-13T14:39:16.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[84576]: audit 2024-09-13T14:39:14.291833+0000 mgr.y (mgr.26252) 361 : audit [DBG] from='client.44738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[84576]: audit 2024-09-13T14:39:14.291833+0000 mgr.y (mgr.26252) 361 : audit [DBG] from='client.44738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[84576]: audit 2024-09-13T14:39:14.608647+0000 mgr.y (mgr.26252) 362 : audit [DBG] from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[84576]: audit 2024-09-13T14:39:14.608647+0000 mgr.y (mgr.26252) 362 : audit [DBG] from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[84576]: cluster 2024-09-13T14:39:14.752266+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-13T14:39:16.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[84576]: cluster 2024-09-13T14:39:14.752266+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-13T14:39:16.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[87147]: cluster 2024-09-13T14:39:14.164287+0000 mgr.y (mgr.26252) 360 : cluster [DBG] pgmap v568: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 523 KiB/s rd, 67 KiB/s wr, 697 op/s 2024-09-13T14:39:16.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[87147]: cluster 2024-09-13T14:39:14.164287+0000 mgr.y (mgr.26252) 360 : cluster [DBG] pgmap v568: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 523 KiB/s rd, 67 KiB/s wr, 697 op/s 2024-09-13T14:39:16.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[87147]: audit 2024-09-13T14:39:14.291833+0000 mgr.y (mgr.26252) 361 : audit [DBG] from='client.44738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[87147]: audit 2024-09-13T14:39:14.291833+0000 mgr.y (mgr.26252) 361 : audit [DBG] from='client.44738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[87147]: audit 2024-09-13T14:39:14.608647+0000 mgr.y (mgr.26252) 362 : audit [DBG] from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[87147]: audit 2024-09-13T14:39:14.608647+0000 mgr.y (mgr.26252) 362 : audit [DBG] from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[87147]: cluster 2024-09-13T14:39:14.752266+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-13T14:39:16.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:15 smithi099 bash[87147]: cluster 2024-09-13T14:39:14.752266+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-13T14:39:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:15 smithi177 bash[68594]: cluster 2024-09-13T14:39:14.164287+0000 mgr.y (mgr.26252) 360 : cluster [DBG] pgmap v568: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 523 KiB/s rd, 67 KiB/s wr, 697 op/s 2024-09-13T14:39:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:15 smithi177 bash[68594]: cluster 2024-09-13T14:39:14.164287+0000 mgr.y (mgr.26252) 360 : cluster [DBG] pgmap v568: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 523 KiB/s rd, 67 KiB/s wr, 697 op/s 2024-09-13T14:39:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:15 smithi177 bash[68594]: audit 2024-09-13T14:39:14.291833+0000 mgr.y (mgr.26252) 361 : audit [DBG] from='client.44738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:15 smithi177 bash[68594]: audit 2024-09-13T14:39:14.291833+0000 mgr.y (mgr.26252) 361 : audit [DBG] from='client.44738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:15 smithi177 bash[68594]: audit 2024-09-13T14:39:14.608647+0000 mgr.y (mgr.26252) 362 : audit [DBG] from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:15 smithi177 bash[68594]: audit 2024-09-13T14:39:14.608647+0000 mgr.y (mgr.26252) 362 : audit [DBG] from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:15 smithi177 bash[68594]: cluster 2024-09-13T14:39:14.752266+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-13T14:39:16.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:15 smithi177 bash[68594]: cluster 2024-09-13T14:39:14.752266+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-13T14:39:16.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:16 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:16.572 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:39:16 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:16.572 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:39:16 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:16.573 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:39:16 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:16.573 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:39:16 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:16.573 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:39:16 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:16.574 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:39:16 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:16.574 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:16 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:16.574 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:16 smithi099 systemd[1]: Stopping Ceph osd.3 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:39:16.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:16 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:17.049 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[51911]: debug 2024-09-13T14:39:16.598+0000 7f7b8f808640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T14:39:17.049 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[51911]: debug 2024-09-13T14:39:16.598+0000 7f7b8f808640 -1 osd.3 689 *** Got signal Terminated *** 2024-09-13T14:39:17.049 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[51911]: debug 2024-09-13T14:39:16.598+0000 7f7b8f808640 -1 osd.3 689 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:39:17.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[87147]: cluster 2024-09-13T14:39:15.753330+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-13T14:39:17.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[87147]: cluster 2024-09-13T14:39:15.753330+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-13T14:39:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[87147]: cluster 2024-09-13T14:39:16.108937+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-13T14:39:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[87147]: cluster 2024-09-13T14:39:16.108937+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-13T14:39:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[87147]: cluster 2024-09-13T14:39:16.604215+0000 mon.a (mon.0) 991 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:39:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[87147]: cluster 2024-09-13T14:39:16.604215+0000 mon.a (mon.0) 991 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:39:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[84576]: cluster 2024-09-13T14:39:15.753330+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-13T14:39:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[84576]: cluster 2024-09-13T14:39:15.753330+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-13T14:39:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[84576]: cluster 2024-09-13T14:39:16.108937+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-13T14:39:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[84576]: cluster 2024-09-13T14:39:16.108937+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-13T14:39:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[84576]: cluster 2024-09-13T14:39:16.604215+0000 mon.a (mon.0) 991 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:39:17.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:16 smithi099 bash[84576]: cluster 2024-09-13T14:39:16.604215+0000 mon.a (mon.0) 991 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:39:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:16 smithi177 bash[68594]: cluster 2024-09-13T14:39:15.753330+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-13T14:39:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:16 smithi177 bash[68594]: cluster 2024-09-13T14:39:15.753330+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-13T14:39:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:16 smithi177 bash[68594]: cluster 2024-09-13T14:39:16.108937+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-13T14:39:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:16 smithi177 bash[68594]: cluster 2024-09-13T14:39:16.108937+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-13T14:39:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:16 smithi177 bash[68594]: cluster 2024-09-13T14:39:16.604215+0000 mon.a (mon.0) 991 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:39:17.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:16 smithi177 bash[68594]: cluster 2024-09-13T14:39:16.604215+0000 mon.a (mon.0) 991 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:39:17.275 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:39:17] "GET /metrics HTTP/1.1" 200 75086 "" "Prometheus/2.51.0" 2024-09-13T14:39:18.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[84576]: cluster 2024-09-13T14:39:16.164916+0000 mgr.y (mgr.26252) 363 : cluster [DBG] pgmap v572: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 651 KiB/s rd, 84 KiB/s wr, 869 op/s 2024-09-13T14:39:18.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[84576]: cluster 2024-09-13T14:39:16.164916+0000 mgr.y (mgr.26252) 363 : cluster [DBG] pgmap v572: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 651 KiB/s rd, 84 KiB/s wr, 869 op/s 2024-09-13T14:39:18.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[84576]: audit 2024-09-13T14:39:16.910123+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.99:0/1500717094' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:18.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[84576]: audit 2024-09-13T14:39:16.910123+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.99:0/1500717094' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:18.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[84576]: cluster 2024-09-13T14:39:17.106653+0000 mon.a (mon.0) 993 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:18.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[84576]: cluster 2024-09-13T14:39:17.106653+0000 mon.a (mon.0) 993 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:18.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[84576]: cluster 2024-09-13T14:39:17.115757+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e690: 8 total, 7 up, 8 in 2024-09-13T14:39:18.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[84576]: cluster 2024-09-13T14:39:17.115757+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e690: 8 total, 7 up, 8 in 2024-09-13T14:39:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[87147]: cluster 2024-09-13T14:39:16.164916+0000 mgr.y (mgr.26252) 363 : cluster [DBG] pgmap v572: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 651 KiB/s rd, 84 KiB/s wr, 869 op/s 2024-09-13T14:39:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[87147]: cluster 2024-09-13T14:39:16.164916+0000 mgr.y (mgr.26252) 363 : cluster [DBG] pgmap v572: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 651 KiB/s rd, 84 KiB/s wr, 869 op/s 2024-09-13T14:39:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[87147]: audit 2024-09-13T14:39:16.910123+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.99:0/1500717094' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[87147]: audit 2024-09-13T14:39:16.910123+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.99:0/1500717094' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[87147]: cluster 2024-09-13T14:39:17.106653+0000 mon.a (mon.0) 993 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[87147]: cluster 2024-09-13T14:39:17.106653+0000 mon.a (mon.0) 993 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[87147]: cluster 2024-09-13T14:39:17.115757+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e690: 8 total, 7 up, 8 in 2024-09-13T14:39:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:17 smithi099 bash[87147]: cluster 2024-09-13T14:39:17.115757+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e690: 8 total, 7 up, 8 in 2024-09-13T14:39:18.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:17 smithi177 bash[68594]: cluster 2024-09-13T14:39:16.164916+0000 mgr.y (mgr.26252) 363 : cluster [DBG] pgmap v572: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 651 KiB/s rd, 84 KiB/s wr, 869 op/s 2024-09-13T14:39:18.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:17 smithi177 bash[68594]: cluster 2024-09-13T14:39:16.164916+0000 mgr.y (mgr.26252) 363 : cluster [DBG] pgmap v572: 617 pgs: 617 active+clean; 131 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 651 KiB/s rd, 84 KiB/s wr, 869 op/s 2024-09-13T14:39:18.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:17 smithi177 bash[68594]: audit 2024-09-13T14:39:16.910123+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.99:0/1500717094' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:18.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:17 smithi177 bash[68594]: audit 2024-09-13T14:39:16.910123+0000 mon.a (mon.0) 992 : audit [DBG] from='client.? 172.21.15.99:0/1500717094' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:18.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:17 smithi177 bash[68594]: cluster 2024-09-13T14:39:17.106653+0000 mon.a (mon.0) 993 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:18.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:17 smithi177 bash[68594]: cluster 2024-09-13T14:39:17.106653+0000 mon.a (mon.0) 993 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:18.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:17 smithi177 bash[68594]: cluster 2024-09-13T14:39:17.115757+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e690: 8 total, 7 up, 8 in 2024-09-13T14:39:18.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:17 smithi177 bash[68594]: cluster 2024-09-13T14:39:17.115757+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e690: 8 total, 7 up, 8 in 2024-09-13T14:39:18.297 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:39:18.297 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SnapDiff (4456 ms) 2024-09-13T14:39:18.297 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ZeroLengthWrite 2024-09-13T14:39:18.298 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:18.330 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ZeroLengthWrite (34 ms) 2024-09-13T14:39:18.331 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ZeroLengthDiscard 2024-09-13T14:39:18.331 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:18.388 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ZeroLengthDiscard (57 ms) 2024-09-13T14:39:18.388 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ZeroLengthRead 2024-09-13T14:39:18.388 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:18.430 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ZeroLengthRead (41 ms) 2024-09-13T14:39:18.430 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.LargeCacheRead 2024-09-13T14:39:18.430 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:18.519 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.LargeCacheRead (90 ms) 2024-09-13T14:39:18.519 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestPendingAio 2024-09-13T14:39:18.520 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:18.759 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[110356]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-3 2024-09-13T14:39:18.782 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestPendingAio (263 ms) 2024-09-13T14:39:18.782 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyup 2024-09-13T14:39:18.783 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:19.071 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[110496]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-3 2024-09-13T14:39:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[87147]: audit 2024-09-13T14:39:17.265700+0000 mgr.y (mgr.26252) 364 : audit [DBG] from='client.34863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[87147]: audit 2024-09-13T14:39:17.265700+0000 mgr.y (mgr.26252) 364 : audit [DBG] from='client.34863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[87147]: audit 2024-09-13T14:39:17.560616+0000 mgr.y (mgr.26252) 365 : audit [DBG] from='client.44762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[87147]: audit 2024-09-13T14:39:17.560616+0000 mgr.y (mgr.26252) 365 : audit [DBG] from='client.44762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[87147]: cluster 2024-09-13T14:39:18.120242+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e691: 8 total, 7 up, 8 in 2024-09-13T14:39:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[87147]: cluster 2024-09-13T14:39:18.120242+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e691: 8 total, 7 up, 8 in 2024-09-13T14:39:19.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[87147]: audit 2024-09-13T14:39:18.255055+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:19.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[87147]: audit 2024-09-13T14:39:18.255055+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:19.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[87147]: audit 2024-09-13T14:39:18.256752+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:19.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[87147]: audit 2024-09-13T14:39:18.256752+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:19.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[84576]: audit 2024-09-13T14:39:17.265700+0000 mgr.y (mgr.26252) 364 : audit [DBG] from='client.34863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[84576]: audit 2024-09-13T14:39:17.265700+0000 mgr.y (mgr.26252) 364 : audit [DBG] from='client.34863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[84576]: audit 2024-09-13T14:39:17.560616+0000 mgr.y (mgr.26252) 365 : audit [DBG] from='client.44762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[84576]: audit 2024-09-13T14:39:17.560616+0000 mgr.y (mgr.26252) 365 : audit [DBG] from='client.44762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[84576]: cluster 2024-09-13T14:39:18.120242+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e691: 8 total, 7 up, 8 in 2024-09-13T14:39:19.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[84576]: cluster 2024-09-13T14:39:18.120242+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e691: 8 total, 7 up, 8 in 2024-09-13T14:39:19.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[84576]: audit 2024-09-13T14:39:18.255055+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:19.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[84576]: audit 2024-09-13T14:39:18.255055+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:19.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[84576]: audit 2024-09-13T14:39:18.256752+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:19.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:18 smithi099 bash[84576]: audit 2024-09-13T14:39:18.256752+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:19.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:18 smithi177 bash[68594]: audit 2024-09-13T14:39:17.265700+0000 mgr.y (mgr.26252) 364 : audit [DBG] from='client.34863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:18 smithi177 bash[68594]: audit 2024-09-13T14:39:17.265700+0000 mgr.y (mgr.26252) 364 : audit [DBG] from='client.34863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:18 smithi177 bash[68594]: audit 2024-09-13T14:39:17.560616+0000 mgr.y (mgr.26252) 365 : audit [DBG] from='client.44762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:18 smithi177 bash[68594]: audit 2024-09-13T14:39:17.560616+0000 mgr.y (mgr.26252) 365 : audit [DBG] from='client.44762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:18 smithi177 bash[68594]: cluster 2024-09-13T14:39:18.120242+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e691: 8 total, 7 up, 8 in 2024-09-13T14:39:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:18 smithi177 bash[68594]: cluster 2024-09-13T14:39:18.120242+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e691: 8 total, 7 up, 8 in 2024-09-13T14:39:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:18 smithi177 bash[68594]: audit 2024-09-13T14:39:18.255055+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:18 smithi177 bash[68594]: audit 2024-09-13T14:39:18.255055+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:18 smithi177 bash[68594]: audit 2024-09-13T14:39:18.256752+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:18 smithi177 bash[68594]: audit 2024-09-13T14:39:18.256752+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:19.980 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:20.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:19 smithi099 bash[87147]: cluster 2024-09-13T14:39:18.166556+0000 mgr.y (mgr.26252) 366 : cluster [DBG] pgmap v575: 617 pgs: 65 peering, 78 stale+active+clean, 474 active+clean; 133 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s wr, 77 op/s 2024-09-13T14:39:20.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:19 smithi099 bash[87147]: cluster 2024-09-13T14:39:18.166556+0000 mgr.y (mgr.26252) 366 : cluster [DBG] pgmap v575: 617 pgs: 65 peering, 78 stale+active+clean, 474 active+clean; 133 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s wr, 77 op/s 2024-09-13T14:39:20.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:19 smithi099 bash[87147]: cluster 2024-09-13T14:39:19.112704+0000 mon.a (mon.0) 998 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:20.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:19 smithi099 bash[87147]: cluster 2024-09-13T14:39:19.112704+0000 mon.a (mon.0) 998 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:20.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:19 smithi099 bash[84576]: cluster 2024-09-13T14:39:18.166556+0000 mgr.y (mgr.26252) 366 : cluster [DBG] pgmap v575: 617 pgs: 65 peering, 78 stale+active+clean, 474 active+clean; 133 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s wr, 77 op/s 2024-09-13T14:39:20.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:19 smithi099 bash[84576]: cluster 2024-09-13T14:39:18.166556+0000 mgr.y (mgr.26252) 366 : cluster [DBG] pgmap v575: 617 pgs: 65 peering, 78 stale+active+clean, 474 active+clean; 133 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s wr, 77 op/s 2024-09-13T14:39:20.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:19 smithi099 bash[84576]: cluster 2024-09-13T14:39:19.112704+0000 mon.a (mon.0) 998 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:20.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:19 smithi099 bash[84576]: cluster 2024-09-13T14:39:19.112704+0000 mon.a (mon.0) 998 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:20.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:19 smithi177 bash[68594]: cluster 2024-09-13T14:39:18.166556+0000 mgr.y (mgr.26252) 366 : cluster [DBG] pgmap v575: 617 pgs: 65 peering, 78 stale+active+clean, 474 active+clean; 133 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s wr, 77 op/s 2024-09-13T14:39:20.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:19 smithi177 bash[68594]: cluster 2024-09-13T14:39:18.166556+0000 mgr.y (mgr.26252) 366 : cluster [DBG] pgmap v575: 617 pgs: 65 peering, 78 stale+active+clean, 474 active+clean; 133 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s wr, 77 op/s 2024-09-13T14:39:20.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:19 smithi177 bash[68594]: cluster 2024-09-13T14:39:19.112704+0000 mon.a (mon.0) 998 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:20.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:19 smithi177 bash[68594]: cluster 2024-09-13T14:39:19.112704+0000 mon.a (mon.0) 998 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:20.218 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:20.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[61439]: ts=2024-09-13T14:39:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:21.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[84576]: cluster 2024-09-13T14:39:19.777086+0000 mon.a (mon.0) 999 : cluster [DBG] osdmap e692: 8 total, 7 up, 8 in 2024-09-13T14:39:21.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[84576]: cluster 2024-09-13T14:39:19.777086+0000 mon.a (mon.0) 999 : cluster [DBG] osdmap e692: 8 total, 7 up, 8 in 2024-09-13T14:39:21.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[84576]: audit 2024-09-13T14:39:19.890352+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.99:0/1909127331' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:21.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[84576]: audit 2024-09-13T14:39:19.890352+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.99:0/1909127331' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:21.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[84576]: cluster 2024-09-13T14:39:20.167492+0000 mgr.y (mgr.26252) 367 : cluster [DBG] pgmap v577: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 509 KiB/s rd, 641 KiB/s wr, 167 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:21.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[84576]: cluster 2024-09-13T14:39:20.167492+0000 mgr.y (mgr.26252) 367 : cluster [DBG] pgmap v577: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 509 KiB/s rd, 641 KiB/s wr, 167 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:21.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[84576]: audit 2024-09-13T14:39:20.208541+0000 mgr.y (mgr.26252) 368 : audit [DBG] from='client.44774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[84576]: audit 2024-09-13T14:39:20.208541+0000 mgr.y (mgr.26252) 368 : audit [DBG] from='client.44774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[84576]: audit 2024-09-13T14:39:20.521836+0000 mgr.y (mgr.26252) 369 : audit [DBG] from='client.34887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[84576]: audit 2024-09-13T14:39:20.521836+0000 mgr.y (mgr.26252) 369 : audit [DBG] from='client.34887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[87147]: cluster 2024-09-13T14:39:19.777086+0000 mon.a (mon.0) 999 : cluster [DBG] osdmap e692: 8 total, 7 up, 8 in 2024-09-13T14:39:21.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[87147]: cluster 2024-09-13T14:39:19.777086+0000 mon.a (mon.0) 999 : cluster [DBG] osdmap e692: 8 total, 7 up, 8 in 2024-09-13T14:39:21.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[87147]: audit 2024-09-13T14:39:19.890352+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.99:0/1909127331' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:21.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[87147]: audit 2024-09-13T14:39:19.890352+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.99:0/1909127331' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:21.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[87147]: cluster 2024-09-13T14:39:20.167492+0000 mgr.y (mgr.26252) 367 : cluster [DBG] pgmap v577: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 509 KiB/s rd, 641 KiB/s wr, 167 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:21.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[87147]: cluster 2024-09-13T14:39:20.167492+0000 mgr.y (mgr.26252) 367 : cluster [DBG] pgmap v577: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 509 KiB/s rd, 641 KiB/s wr, 167 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:21.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[87147]: audit 2024-09-13T14:39:20.208541+0000 mgr.y (mgr.26252) 368 : audit [DBG] from='client.44774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[87147]: audit 2024-09-13T14:39:20.208541+0000 mgr.y (mgr.26252) 368 : audit [DBG] from='client.44774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[87147]: audit 2024-09-13T14:39:20.521836+0000 mgr.y (mgr.26252) 369 : audit [DBG] from='client.34887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:20 smithi099 bash[87147]: audit 2024-09-13T14:39:20.521836+0000 mgr.y (mgr.26252) 369 : audit [DBG] from='client.34887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[68594]: cluster 2024-09-13T14:39:19.777086+0000 mon.a (mon.0) 999 : cluster [DBG] osdmap e692: 8 total, 7 up, 8 in 2024-09-13T14:39:21.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[68594]: cluster 2024-09-13T14:39:19.777086+0000 mon.a (mon.0) 999 : cluster [DBG] osdmap e692: 8 total, 7 up, 8 in 2024-09-13T14:39:21.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[68594]: audit 2024-09-13T14:39:19.890352+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.99:0/1909127331' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:21.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[68594]: audit 2024-09-13T14:39:19.890352+0000 mon.a (mon.0) 1000 : audit [DBG] from='client.? 172.21.15.99:0/1909127331' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:21.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[68594]: cluster 2024-09-13T14:39:20.167492+0000 mgr.y (mgr.26252) 367 : cluster [DBG] pgmap v577: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 509 KiB/s rd, 641 KiB/s wr, 167 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:21.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[68594]: cluster 2024-09-13T14:39:20.167492+0000 mgr.y (mgr.26252) 367 : cluster [DBG] pgmap v577: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 509 KiB/s rd, 641 KiB/s wr, 167 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:21.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[68594]: audit 2024-09-13T14:39:20.208541+0000 mgr.y (mgr.26252) 368 : audit [DBG] from='client.44774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[68594]: audit 2024-09-13T14:39:20.208541+0000 mgr.y (mgr.26252) 368 : audit [DBG] from='client.44774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[68594]: audit 2024-09-13T14:39:20.521836+0000 mgr.y (mgr.26252) 369 : audit [DBG] from='client.34887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:20 smithi177 bash[68594]: audit 2024-09-13T14:39:20.521836+0000 mgr.y (mgr.26252) 369 : audit [DBG] from='client.34887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:21.205 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyup (2422 ms) 2024-09-13T14:39:21.205 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyupMismatch 2024-09-13T14:39:21.205 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:21.502 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:21.503 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:21.503 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:21.503 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:21.503 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:21.504 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T14:39:21.504 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: Stopped Ceph osd.3 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:39:21.504 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:21.504 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:21.505 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:21.505 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:21.781 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:21 smithi099 systemd[1]: Started Ceph osd.3 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:39:22.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: cluster 2024-09-13T14:39:20.771572+0000 mon.a (mon.0) 1001 : cluster [WRN] Health check failed: Degraded data redundancy: 134/1254 objects degraded (10.686%), 44 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:22.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: cluster 2024-09-13T14:39:20.771572+0000 mon.a (mon.0) 1001 : cluster [WRN] Health check failed: Degraded data redundancy: 134/1254 objects degraded (10.686%), 44 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:22.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: cluster 2024-09-13T14:39:20.790350+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e693: 8 total, 7 up, 8 in 2024-09-13T14:39:22.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: cluster 2024-09-13T14:39:20.790350+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e693: 8 total, 7 up, 8 in 2024-09-13T14:39:22.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: cluster 2024-09-13T14:39:21.114636+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e694: 8 total, 7 up, 8 in 2024-09-13T14:39:22.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: cluster 2024-09-13T14:39:21.114636+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e694: 8 total, 7 up, 8 in 2024-09-13T14:39:22.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: audit 2024-09-13T14:39:21.551101+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: audit 2024-09-13T14:39:21.551101+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: audit 2024-09-13T14:39:21.557683+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: audit 2024-09-13T14:39:21.557683+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: audit 2024-09-13T14:39:21.558643+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:22.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[84576]: audit 2024-09-13T14:39:21.558643+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:22.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: cluster 2024-09-13T14:39:20.771572+0000 mon.a (mon.0) 1001 : cluster [WRN] Health check failed: Degraded data redundancy: 134/1254 objects degraded (10.686%), 44 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:22.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: cluster 2024-09-13T14:39:20.771572+0000 mon.a (mon.0) 1001 : cluster [WRN] Health check failed: Degraded data redundancy: 134/1254 objects degraded (10.686%), 44 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:22.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: cluster 2024-09-13T14:39:20.790350+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e693: 8 total, 7 up, 8 in 2024-09-13T14:39:22.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: cluster 2024-09-13T14:39:20.790350+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e693: 8 total, 7 up, 8 in 2024-09-13T14:39:22.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: cluster 2024-09-13T14:39:21.114636+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e694: 8 total, 7 up, 8 in 2024-09-13T14:39:22.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: cluster 2024-09-13T14:39:21.114636+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e694: 8 total, 7 up, 8 in 2024-09-13T14:39:22.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: audit 2024-09-13T14:39:21.551101+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: audit 2024-09-13T14:39:21.551101+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: audit 2024-09-13T14:39:21.557683+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: audit 2024-09-13T14:39:21.557683+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: audit 2024-09-13T14:39:21.558643+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:22.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:21 smithi099 bash[87147]: audit 2024-09-13T14:39:21.558643+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:22.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: cluster 2024-09-13T14:39:20.771572+0000 mon.a (mon.0) 1001 : cluster [WRN] Health check failed: Degraded data redundancy: 134/1254 objects degraded (10.686%), 44 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:22.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: cluster 2024-09-13T14:39:20.771572+0000 mon.a (mon.0) 1001 : cluster [WRN] Health check failed: Degraded data redundancy: 134/1254 objects degraded (10.686%), 44 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:22.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: cluster 2024-09-13T14:39:20.790350+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e693: 8 total, 7 up, 8 in 2024-09-13T14:39:22.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: cluster 2024-09-13T14:39:20.790350+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e693: 8 total, 7 up, 8 in 2024-09-13T14:39:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: cluster 2024-09-13T14:39:21.114636+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e694: 8 total, 7 up, 8 in 2024-09-13T14:39:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: cluster 2024-09-13T14:39:21.114636+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e694: 8 total, 7 up, 8 in 2024-09-13T14:39:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: audit 2024-09-13T14:39:21.551101+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: audit 2024-09-13T14:39:21.551101+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: audit 2024-09-13T14:39:21.557683+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: audit 2024-09-13T14:39:21.557683+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: audit 2024-09-13T14:39:21.558643+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:21 smithi177 bash[68594]: audit 2024-09-13T14:39:21.558643+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:22.356 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:23.181 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:23.383 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[84576]: cluster 2024-09-13T14:39:22.111940+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e695: 8 total, 7 up, 8 in 2024-09-13T14:39:23.383 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[84576]: cluster 2024-09-13T14:39:22.111940+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e695: 8 total, 7 up, 8 in 2024-09-13T14:39:23.383 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[84576]: cluster 2024-09-13T14:39:22.168745+0000 mgr.y (mgr.26252) 370 : cluster [DBG] pgmap v581: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 532 KiB/s rd, 585 KiB/s wr, 115 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:23.384 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[84576]: cluster 2024-09-13T14:39:22.168745+0000 mgr.y (mgr.26252) 370 : cluster [DBG] pgmap v581: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 532 KiB/s rd, 585 KiB/s wr, 115 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:23.384 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[84576]: audit 2024-09-13T14:39:22.830066+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.99:0/390322550' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:23.384 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[84576]: audit 2024-09-13T14:39:22.830066+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.99:0/390322550' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:23.384 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[87147]: cluster 2024-09-13T14:39:22.111940+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e695: 8 total, 7 up, 8 in 2024-09-13T14:39:23.384 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[87147]: cluster 2024-09-13T14:39:22.111940+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e695: 8 total, 7 up, 8 in 2024-09-13T14:39:23.384 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[87147]: cluster 2024-09-13T14:39:22.168745+0000 mgr.y (mgr.26252) 370 : cluster [DBG] pgmap v581: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 532 KiB/s rd, 585 KiB/s wr, 115 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:23.384 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[87147]: cluster 2024-09-13T14:39:22.168745+0000 mgr.y (mgr.26252) 370 : cluster [DBG] pgmap v581: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 532 KiB/s rd, 585 KiB/s wr, 115 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:23.384 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[87147]: audit 2024-09-13T14:39:22.830066+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.99:0/390322550' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:23.384 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[87147]: audit 2024-09-13T14:39:22.830066+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.99:0/390322550' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:23.385 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:23.385 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:23.385 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:39:23.385 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:23.385 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:23.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:23 smithi177 bash[68594]: cluster 2024-09-13T14:39:22.111940+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e695: 8 total, 7 up, 8 in 2024-09-13T14:39:23.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:23 smithi177 bash[68594]: cluster 2024-09-13T14:39:22.111940+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e695: 8 total, 7 up, 8 in 2024-09-13T14:39:23.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:23 smithi177 bash[68594]: cluster 2024-09-13T14:39:22.168745+0000 mgr.y (mgr.26252) 370 : cluster [DBG] pgmap v581: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 532 KiB/s rd, 585 KiB/s wr, 115 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:23.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:23 smithi177 bash[68594]: cluster 2024-09-13T14:39:22.168745+0000 mgr.y (mgr.26252) 370 : cluster [DBG] pgmap v581: 617 pgs: 153 active+undersized, 65 peering, 44 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 532 KiB/s rd, 585 KiB/s wr, 115 op/s; 134/1254 objects degraded (10.686%) 2024-09-13T14:39:23.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:23 smithi177 bash[68594]: audit 2024-09-13T14:39:22.830066+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.99:0/390322550' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:23.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:23 smithi177 bash[68594]: audit 2024-09-13T14:39:22.830066+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.99:0/390322550' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:23.690 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:39:23.690 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T14:39:23.690 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:39:24.072 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:39:24.072 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:39:24.072 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:39:24.072 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:23 smithi099 bash[110713]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T14:39:24.249 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyupMismatch (3045 ms) 2024-09-13T14:39:24.249 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.Flatten 2024-09-13T14:39:24.249 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:24.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[84576]: cluster 2024-09-13T14:39:23.122008+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e696: 8 total, 7 up, 8 in 2024-09-13T14:39:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[84576]: cluster 2024-09-13T14:39:23.122008+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e696: 8 total, 7 up, 8 in 2024-09-13T14:39:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[84576]: audit 2024-09-13T14:39:23.167260+0000 mgr.y (mgr.26252) 371 : audit [DBG] from='client.34899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[84576]: audit 2024-09-13T14:39:23.167260+0000 mgr.y (mgr.26252) 371 : audit [DBG] from='client.34899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[84576]: audit 2024-09-13T14:39:23.490182+0000 mgr.y (mgr.26252) 372 : audit [DBG] from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[84576]: audit 2024-09-13T14:39:23.490182+0000 mgr.y (mgr.26252) 372 : audit [DBG] from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[87147]: cluster 2024-09-13T14:39:23.122008+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e696: 8 total, 7 up, 8 in 2024-09-13T14:39:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[87147]: cluster 2024-09-13T14:39:23.122008+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e696: 8 total, 7 up, 8 in 2024-09-13T14:39:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[87147]: audit 2024-09-13T14:39:23.167260+0000 mgr.y (mgr.26252) 371 : audit [DBG] from='client.34899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[87147]: audit 2024-09-13T14:39:23.167260+0000 mgr.y (mgr.26252) 371 : audit [DBG] from='client.34899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[87147]: audit 2024-09-13T14:39:23.490182+0000 mgr.y (mgr.26252) 372 : audit [DBG] from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:24 smithi099 bash[87147]: audit 2024-09-13T14:39:23.490182+0000 mgr.y (mgr.26252) 372 : audit [DBG] from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:24 smithi177 bash[68594]: cluster 2024-09-13T14:39:23.122008+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e696: 8 total, 7 up, 8 in 2024-09-13T14:39:24.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:24 smithi177 bash[68594]: cluster 2024-09-13T14:39:23.122008+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e696: 8 total, 7 up, 8 in 2024-09-13T14:39:24.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:24 smithi177 bash[68594]: audit 2024-09-13T14:39:23.167260+0000 mgr.y (mgr.26252) 371 : audit [DBG] from='client.34899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:24 smithi177 bash[68594]: audit 2024-09-13T14:39:23.167260+0000 mgr.y (mgr.26252) 371 : audit [DBG] from='client.34899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:24 smithi177 bash[68594]: audit 2024-09-13T14:39:23.490182+0000 mgr.y (mgr.26252) 372 : audit [DBG] from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:24.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:24 smithi177 bash[68594]: audit 2024-09-13T14:39:23.490182+0000 mgr.y (mgr.26252) 372 : audit [DBG] from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:25.402 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.Flatten (1153 ms) 2024-09-13T14:39:25.402 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.Sparsify 2024-09-13T14:39:25.402 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:25.492 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 32768 2024-09-13T14:39:25.492 INFO:tasks.workunit.client.0.smithi099.stdout:read: 32768 2024-09-13T14:39:25.492 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.Sparsify (90 ms) 2024-09-13T14:39:25.492 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SparsifyPP 2024-09-13T14:39:25.492 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:25.558 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SparsifyPP (66 ms) 2024-09-13T14:39:25.558 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SnapshotLimit 2024-09-13T14:39:25.558 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:25 smithi099 bash[84576]: cluster 2024-09-13T14:39:24.133562+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e697: 8 total, 7 up, 8 in 2024-09-13T14:39:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:25 smithi099 bash[84576]: cluster 2024-09-13T14:39:24.133562+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e697: 8 total, 7 up, 8 in 2024-09-13T14:39:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:25 smithi099 bash[84576]: cluster 2024-09-13T14:39:24.170383+0000 mgr.y (mgr.26252) 373 : cluster [DBG] pgmap v584: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 318 KiB/s rd, 34 KiB/s wr, 429 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:25 smithi099 bash[84576]: cluster 2024-09-13T14:39:24.170383+0000 mgr.y (mgr.26252) 373 : cluster [DBG] pgmap v584: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 318 KiB/s rd, 34 KiB/s wr, 429 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:25 smithi099 bash[87147]: cluster 2024-09-13T14:39:24.133562+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e697: 8 total, 7 up, 8 in 2024-09-13T14:39:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:25 smithi099 bash[87147]: cluster 2024-09-13T14:39:24.133562+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e697: 8 total, 7 up, 8 in 2024-09-13T14:39:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:25 smithi099 bash[87147]: cluster 2024-09-13T14:39:24.170383+0000 mgr.y (mgr.26252) 373 : cluster [DBG] pgmap v584: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 318 KiB/s rd, 34 KiB/s wr, 429 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:25 smithi099 bash[87147]: cluster 2024-09-13T14:39:24.170383+0000 mgr.y (mgr.26252) 373 : cluster [DBG] pgmap v584: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 318 KiB/s rd, 34 KiB/s wr, 429 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:25 smithi177 bash[68594]: cluster 2024-09-13T14:39:24.133562+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e697: 8 total, 7 up, 8 in 2024-09-13T14:39:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:25 smithi177 bash[68594]: cluster 2024-09-13T14:39:24.133562+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e697: 8 total, 7 up, 8 in 2024-09-13T14:39:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:25 smithi177 bash[68594]: cluster 2024-09-13T14:39:24.170383+0000 mgr.y (mgr.26252) 373 : cluster [DBG] pgmap v584: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 318 KiB/s rd, 34 KiB/s wr, 429 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:25.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:25 smithi177 bash[68594]: cluster 2024-09-13T14:39:24.170383+0000 mgr.y (mgr.26252) 373 : cluster [DBG] pgmap v584: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 318 KiB/s rd, 34 KiB/s wr, 429 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:26.138 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:26.321 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[111000]: debug 2024-09-13T14:39:26.038+0000 7f247b379640 1 -- 172.21.15.99:0/509674037 <== mon.2 v2:172.21.15.99:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561896820960 con 0x56189760e000 2024-09-13T14:39:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[84576]: cluster 2024-09-13T14:39:25.127787+0000 mon.a (mon.0) 1011 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2024-09-13T14:39:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[84576]: cluster 2024-09-13T14:39:25.127787+0000 mon.a (mon.0) 1011 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2024-09-13T14:39:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[84576]: cluster 2024-09-13T14:39:25.140708+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e698: 8 total, 7 up, 8 in 2024-09-13T14:39:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[84576]: cluster 2024-09-13T14:39:25.140708+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e698: 8 total, 7 up, 8 in 2024-09-13T14:39:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[84576]: audit 2024-09-13T14:39:25.801497+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.99:0/1348727672' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[84576]: audit 2024-09-13T14:39:25.801497+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.99:0/1348727672' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[84576]: cluster 2024-09-13T14:39:26.106254+0000 mon.a (mon.0) 1014 : cluster [WRN] Health check update: Degraded data redundancy: 193/1383 objects degraded (13.955%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:26.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[84576]: cluster 2024-09-13T14:39:26.106254+0000 mon.a (mon.0) 1014 : cluster [WRN] Health check update: Degraded data redundancy: 193/1383 objects degraded (13.955%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:26.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[87147]: cluster 2024-09-13T14:39:25.127787+0000 mon.a (mon.0) 1011 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2024-09-13T14:39:26.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[87147]: cluster 2024-09-13T14:39:25.127787+0000 mon.a (mon.0) 1011 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2024-09-13T14:39:26.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[87147]: cluster 2024-09-13T14:39:25.140708+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e698: 8 total, 7 up, 8 in 2024-09-13T14:39:26.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[87147]: cluster 2024-09-13T14:39:25.140708+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e698: 8 total, 7 up, 8 in 2024-09-13T14:39:26.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[87147]: audit 2024-09-13T14:39:25.801497+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.99:0/1348727672' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:26.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[87147]: audit 2024-09-13T14:39:25.801497+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.99:0/1348727672' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:26.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[87147]: cluster 2024-09-13T14:39:26.106254+0000 mon.a (mon.0) 1014 : cluster [WRN] Health check update: Degraded data redundancy: 193/1383 objects degraded (13.955%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:26.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:26 smithi099 bash[87147]: cluster 2024-09-13T14:39:26.106254+0000 mon.a (mon.0) 1014 : cluster [WRN] Health check update: Degraded data redundancy: 193/1383 objects degraded (13.955%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:26.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:26 smithi177 bash[68594]: cluster 2024-09-13T14:39:25.127787+0000 mon.a (mon.0) 1011 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2024-09-13T14:39:26.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:26 smithi177 bash[68594]: cluster 2024-09-13T14:39:25.127787+0000 mon.a (mon.0) 1011 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 6 pgs peering) 2024-09-13T14:39:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:26 smithi177 bash[68594]: cluster 2024-09-13T14:39:25.140708+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e698: 8 total, 7 up, 8 in 2024-09-13T14:39:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:26 smithi177 bash[68594]: cluster 2024-09-13T14:39:25.140708+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e698: 8 total, 7 up, 8 in 2024-09-13T14:39:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:26 smithi177 bash[68594]: audit 2024-09-13T14:39:25.801497+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.99:0/1348727672' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:26 smithi177 bash[68594]: audit 2024-09-13T14:39:25.801497+0000 mon.a (mon.0) 1013 : audit [DBG] from='client.? 172.21.15.99:0/1348727672' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:26 smithi177 bash[68594]: cluster 2024-09-13T14:39:26.106254+0000 mon.a (mon.0) 1014 : cluster [WRN] Health check update: Degraded data redundancy: 193/1383 objects degraded (13.955%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:26 smithi177 bash[68594]: cluster 2024-09-13T14:39:26.106254+0000 mon.a (mon.0) 1014 : cluster [WRN] Health check update: Degraded data redundancy: 193/1383 objects degraded (13.955%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:27.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:39:27] "GET /metrics HTTP/1.1" 200 75165 "" "Prometheus/2.51.0" 2024-09-13T14:39:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[84576]: audit 2024-09-13T14:39:26.126940+0000 mgr.y (mgr.26252) 374 : audit [DBG] from='client.44810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[84576]: audit 2024-09-13T14:39:26.126940+0000 mgr.y (mgr.26252) 374 : audit [DBG] from='client.44810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[84576]: cluster 2024-09-13T14:39:26.150812+0000 mon.a (mon.0) 1015 : cluster [DBG] osdmap e699: 8 total, 7 up, 8 in 2024-09-13T14:39:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[84576]: cluster 2024-09-13T14:39:26.150812+0000 mon.a (mon.0) 1015 : cluster [DBG] osdmap e699: 8 total, 7 up, 8 in 2024-09-13T14:39:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[84576]: cluster 2024-09-13T14:39:26.171384+0000 mgr.y (mgr.26252) 375 : cluster [DBG] pgmap v587: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 29 KiB/s wr, 363 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[84576]: cluster 2024-09-13T14:39:26.171384+0000 mgr.y (mgr.26252) 375 : cluster [DBG] pgmap v587: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 29 KiB/s wr, 363 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:27.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[84576]: audit 2024-09-13T14:39:26.452646+0000 mgr.y (mgr.26252) 376 : audit [DBG] from='client.44819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[84576]: audit 2024-09-13T14:39:26.452646+0000 mgr.y (mgr.26252) 376 : audit [DBG] from='client.44819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[87147]: audit 2024-09-13T14:39:26.126940+0000 mgr.y (mgr.26252) 374 : audit [DBG] from='client.44810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[87147]: audit 2024-09-13T14:39:26.126940+0000 mgr.y (mgr.26252) 374 : audit [DBG] from='client.44810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[87147]: cluster 2024-09-13T14:39:26.150812+0000 mon.a (mon.0) 1015 : cluster [DBG] osdmap e699: 8 total, 7 up, 8 in 2024-09-13T14:39:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[87147]: cluster 2024-09-13T14:39:26.150812+0000 mon.a (mon.0) 1015 : cluster [DBG] osdmap e699: 8 total, 7 up, 8 in 2024-09-13T14:39:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[87147]: cluster 2024-09-13T14:39:26.171384+0000 mgr.y (mgr.26252) 375 : cluster [DBG] pgmap v587: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 29 KiB/s wr, 363 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[87147]: cluster 2024-09-13T14:39:26.171384+0000 mgr.y (mgr.26252) 375 : cluster [DBG] pgmap v587: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 29 KiB/s wr, 363 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[87147]: audit 2024-09-13T14:39:26.452646+0000 mgr.y (mgr.26252) 376 : audit [DBG] from='client.44819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[87147]: audit 2024-09-13T14:39:26.452646+0000 mgr.y (mgr.26252) 376 : audit [DBG] from='client.44819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:27 smithi177 bash[68594]: audit 2024-09-13T14:39:26.126940+0000 mgr.y (mgr.26252) 374 : audit [DBG] from='client.44810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:27 smithi177 bash[68594]: audit 2024-09-13T14:39:26.126940+0000 mgr.y (mgr.26252) 374 : audit [DBG] from='client.44810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:27 smithi177 bash[68594]: cluster 2024-09-13T14:39:26.150812+0000 mon.a (mon.0) 1015 : cluster [DBG] osdmap e699: 8 total, 7 up, 8 in 2024-09-13T14:39:27.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:27 smithi177 bash[68594]: cluster 2024-09-13T14:39:26.150812+0000 mon.a (mon.0) 1015 : cluster [DBG] osdmap e699: 8 total, 7 up, 8 in 2024-09-13T14:39:27.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:27 smithi177 bash[68594]: cluster 2024-09-13T14:39:26.171384+0000 mgr.y (mgr.26252) 375 : cluster [DBG] pgmap v587: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 29 KiB/s wr, 363 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:27.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:27 smithi177 bash[68594]: cluster 2024-09-13T14:39:26.171384+0000 mgr.y (mgr.26252) 375 : cluster [DBG] pgmap v587: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 29 KiB/s wr, 363 op/s; 193/1383 objects degraded (13.955%) 2024-09-13T14:39:27.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:27 smithi177 bash[68594]: audit 2024-09-13T14:39:26.452646+0000 mgr.y (mgr.26252) 376 : audit [DBG] from='client.44819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:27 smithi177 bash[68594]: audit 2024-09-13T14:39:26.452646+0000 mgr.y (mgr.26252) 376 : audit [DBG] from='client.44819 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:27.822 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:27 smithi099 bash[111000]: debug 2024-09-13T14:39:27.542+0000 7f247dbe0640 -1 Falling back to public interface 2024-09-13T14:39:28.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:28 smithi099 bash[84576]: cluster 2024-09-13T14:39:27.175897+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e700: 8 total, 7 up, 8 in 2024-09-13T14:39:28.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:28 smithi099 bash[84576]: cluster 2024-09-13T14:39:27.175897+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e700: 8 total, 7 up, 8 in 2024-09-13T14:39:28.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:28 smithi099 bash[87147]: cluster 2024-09-13T14:39:27.175897+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e700: 8 total, 7 up, 8 in 2024-09-13T14:39:28.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:28 smithi099 bash[87147]: cluster 2024-09-13T14:39:27.175897+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e700: 8 total, 7 up, 8 in 2024-09-13T14:39:28.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:28 smithi177 bash[68594]: cluster 2024-09-13T14:39:27.175897+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e700: 8 total, 7 up, 8 in 2024-09-13T14:39:28.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:28 smithi177 bash[68594]: cluster 2024-09-13T14:39:27.175897+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e700: 8 total, 7 up, 8 in 2024-09-13T14:39:29.071 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:29.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[84576]: cluster 2024-09-13T14:39:28.172491+0000 mgr.y (mgr.26252) 377 : cluster [DBG] pgmap v590: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 5.0 KiB/s wr, 77 op/s; 196/1407 objects degraded (13.930%) 2024-09-13T14:39:29.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[84576]: cluster 2024-09-13T14:39:28.172491+0000 mgr.y (mgr.26252) 377 : cluster [DBG] pgmap v590: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 5.0 KiB/s wr, 77 op/s; 196/1407 objects degraded (13.930%) 2024-09-13T14:39:29.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[84576]: cluster 2024-09-13T14:39:28.174101+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e701: 8 total, 7 up, 8 in 2024-09-13T14:39:29.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[84576]: cluster 2024-09-13T14:39:28.174101+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e701: 8 total, 7 up, 8 in 2024-09-13T14:39:29.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[84576]: audit 2024-09-13T14:39:28.754699+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.99:0/391871219' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:29.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[84576]: audit 2024-09-13T14:39:28.754699+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.99:0/391871219' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:29.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[84576]: audit 2024-09-13T14:39:28.767288+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[84576]: audit 2024-09-13T14:39:28.767288+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[84576]: audit 2024-09-13T14:39:28.772771+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[84576]: audit 2024-09-13T14:39:28.772771+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[87147]: cluster 2024-09-13T14:39:28.172491+0000 mgr.y (mgr.26252) 377 : cluster [DBG] pgmap v590: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 5.0 KiB/s wr, 77 op/s; 196/1407 objects degraded (13.930%) 2024-09-13T14:39:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[87147]: cluster 2024-09-13T14:39:28.172491+0000 mgr.y (mgr.26252) 377 : cluster [DBG] pgmap v590: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 5.0 KiB/s wr, 77 op/s; 196/1407 objects degraded (13.930%) 2024-09-13T14:39:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[87147]: cluster 2024-09-13T14:39:28.174101+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e701: 8 total, 7 up, 8 in 2024-09-13T14:39:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[87147]: cluster 2024-09-13T14:39:28.174101+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e701: 8 total, 7 up, 8 in 2024-09-13T14:39:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[87147]: audit 2024-09-13T14:39:28.754699+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.99:0/391871219' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[87147]: audit 2024-09-13T14:39:28.754699+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.99:0/391871219' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[87147]: audit 2024-09-13T14:39:28.767288+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[87147]: audit 2024-09-13T14:39:28.767288+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[87147]: audit 2024-09-13T14:39:28.772771+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:29 smithi099 bash[87147]: audit 2024-09-13T14:39:28.772771+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:29 smithi177 bash[68594]: cluster 2024-09-13T14:39:28.172491+0000 mgr.y (mgr.26252) 377 : cluster [DBG] pgmap v590: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 5.0 KiB/s wr, 77 op/s; 196/1407 objects degraded (13.930%) 2024-09-13T14:39:29.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:29 smithi177 bash[68594]: cluster 2024-09-13T14:39:28.172491+0000 mgr.y (mgr.26252) 377 : cluster [DBG] pgmap v590: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 5.0 KiB/s wr, 77 op/s; 196/1407 objects degraded (13.930%) 2024-09-13T14:39:29.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:29 smithi177 bash[68594]: cluster 2024-09-13T14:39:28.174101+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e701: 8 total, 7 up, 8 in 2024-09-13T14:39:29.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:29 smithi177 bash[68594]: cluster 2024-09-13T14:39:28.174101+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e701: 8 total, 7 up, 8 in 2024-09-13T14:39:29.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:29 smithi177 bash[68594]: audit 2024-09-13T14:39:28.754699+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.99:0/391871219' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:29.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:29 smithi177 bash[68594]: audit 2024-09-13T14:39:28.754699+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.99:0/391871219' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:29.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:29 smithi177 bash[68594]: audit 2024-09-13T14:39:28.767288+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:29 smithi177 bash[68594]: audit 2024-09-13T14:39:28.767288+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:29 smithi177 bash[68594]: audit 2024-09-13T14:39:28.772771+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:29.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:29 smithi177 bash[68594]: audit 2024-09-13T14:39:28.772771+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.265 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SnapshotLimit (4707 ms) 2024-09-13T14:39:30.265 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SnapshotLimitPP 2024-09-13T14:39:30.265 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[84576]: audit 2024-09-13T14:39:29.060994+0000 mgr.y (mgr.26252) 378 : audit [DBG] from='client.44831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[84576]: audit 2024-09-13T14:39:29.060994+0000 mgr.y (mgr.26252) 378 : audit [DBG] from='client.44831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[84576]: cluster 2024-09-13T14:39:29.184203+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e702: 8 total, 7 up, 8 in 2024-09-13T14:39:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[84576]: cluster 2024-09-13T14:39:29.184203+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e702: 8 total, 7 up, 8 in 2024-09-13T14:39:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[84576]: audit 2024-09-13T14:39:29.370801+0000 mgr.y (mgr.26252) 379 : audit [DBG] from='client.34944 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[84576]: audit 2024-09-13T14:39:29.370801+0000 mgr.y (mgr.26252) 379 : audit [DBG] from='client.34944 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[84576]: audit 2024-09-13T14:39:29.891500+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[84576]: audit 2024-09-13T14:39:29.891500+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[84576]: audit 2024-09-13T14:39:29.897039+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[84576]: audit 2024-09-13T14:39:29.897039+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[87147]: audit 2024-09-13T14:39:29.060994+0000 mgr.y (mgr.26252) 378 : audit [DBG] from='client.44831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[87147]: audit 2024-09-13T14:39:29.060994+0000 mgr.y (mgr.26252) 378 : audit [DBG] from='client.44831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[87147]: cluster 2024-09-13T14:39:29.184203+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e702: 8 total, 7 up, 8 in 2024-09-13T14:39:30.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[87147]: cluster 2024-09-13T14:39:29.184203+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e702: 8 total, 7 up, 8 in 2024-09-13T14:39:30.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[87147]: audit 2024-09-13T14:39:29.370801+0000 mgr.y (mgr.26252) 379 : audit [DBG] from='client.34944 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[87147]: audit 2024-09-13T14:39:29.370801+0000 mgr.y (mgr.26252) 379 : audit [DBG] from='client.34944 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[87147]: audit 2024-09-13T14:39:29.891500+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[87147]: audit 2024-09-13T14:39:29.891500+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[87147]: audit 2024-09-13T14:39:29.897039+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:30 smithi099 bash[87147]: audit 2024-09-13T14:39:29.897039+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[61439]: ts=2024-09-13T14:39:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:30.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[68594]: audit 2024-09-13T14:39:29.060994+0000 mgr.y (mgr.26252) 378 : audit [DBG] from='client.44831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[68594]: audit 2024-09-13T14:39:29.060994+0000 mgr.y (mgr.26252) 378 : audit [DBG] from='client.44831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[68594]: cluster 2024-09-13T14:39:29.184203+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e702: 8 total, 7 up, 8 in 2024-09-13T14:39:30.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[68594]: cluster 2024-09-13T14:39:29.184203+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e702: 8 total, 7 up, 8 in 2024-09-13T14:39:30.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[68594]: audit 2024-09-13T14:39:29.370801+0000 mgr.y (mgr.26252) 379 : audit [DBG] from='client.34944 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[68594]: audit 2024-09-13T14:39:29.370801+0000 mgr.y (mgr.26252) 379 : audit [DBG] from='client.34944 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:30.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[68594]: audit 2024-09-13T14:39:29.891500+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[68594]: audit 2024-09-13T14:39:29.891500+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[68594]: audit 2024-09-13T14:39:29.897039+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:30.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:30 smithi177 bash[68594]: audit 2024-09-13T14:39:29.897039+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[84576]: cluster 2024-09-13T14:39:30.174044+0000 mgr.y (mgr.26252) 380 : cluster [DBG] pgmap v592: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 152 KiB/s rd, 34 KiB/s wr, 185 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[84576]: cluster 2024-09-13T14:39:30.174044+0000 mgr.y (mgr.26252) 380 : cluster [DBG] pgmap v592: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 152 KiB/s rd, 34 KiB/s wr, 185 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[84576]: cluster 2024-09-13T14:39:30.195652+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e703: 8 total, 7 up, 8 in 2024-09-13T14:39:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[84576]: cluster 2024-09-13T14:39:30.195652+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e703: 8 total, 7 up, 8 in 2024-09-13T14:39:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[84576]: cluster 2024-09-13T14:39:31.107156+0000 mon.a (mon.0) 1024 : cluster [WRN] Health check update: Degraded data redundancy: 201/1458 objects degraded (13.786%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[84576]: cluster 2024-09-13T14:39:31.107156+0000 mon.a (mon.0) 1024 : cluster [WRN] Health check update: Degraded data redundancy: 201/1458 objects degraded (13.786%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:31.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[87147]: cluster 2024-09-13T14:39:30.174044+0000 mgr.y (mgr.26252) 380 : cluster [DBG] pgmap v592: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 152 KiB/s rd, 34 KiB/s wr, 185 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:31.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[87147]: cluster 2024-09-13T14:39:30.174044+0000 mgr.y (mgr.26252) 380 : cluster [DBG] pgmap v592: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 152 KiB/s rd, 34 KiB/s wr, 185 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[87147]: cluster 2024-09-13T14:39:30.195652+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e703: 8 total, 7 up, 8 in 2024-09-13T14:39:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[87147]: cluster 2024-09-13T14:39:30.195652+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e703: 8 total, 7 up, 8 in 2024-09-13T14:39:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[87147]: cluster 2024-09-13T14:39:31.107156+0000 mon.a (mon.0) 1024 : cluster [WRN] Health check update: Degraded data redundancy: 201/1458 objects degraded (13.786%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[87147]: cluster 2024-09-13T14:39:31.107156+0000 mon.a (mon.0) 1024 : cluster [WRN] Health check update: Degraded data redundancy: 201/1458 objects degraded (13.786%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:31.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:31 smithi177 bash[68594]: cluster 2024-09-13T14:39:30.174044+0000 mgr.y (mgr.26252) 380 : cluster [DBG] pgmap v592: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 152 KiB/s rd, 34 KiB/s wr, 185 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:31.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:31 smithi177 bash[68594]: cluster 2024-09-13T14:39:30.174044+0000 mgr.y (mgr.26252) 380 : cluster [DBG] pgmap v592: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 152 KiB/s rd, 34 KiB/s wr, 185 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:31.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:31 smithi177 bash[68594]: cluster 2024-09-13T14:39:30.195652+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e703: 8 total, 7 up, 8 in 2024-09-13T14:39:31.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:31 smithi177 bash[68594]: cluster 2024-09-13T14:39:30.195652+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e703: 8 total, 7 up, 8 in 2024-09-13T14:39:31.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:31 smithi177 bash[68594]: cluster 2024-09-13T14:39:31.107156+0000 mon.a (mon.0) 1024 : cluster [WRN] Health check update: Degraded data redundancy: 201/1458 objects degraded (13.786%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:31.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:31 smithi177 bash[68594]: cluster 2024-09-13T14:39:31.107156+0000 mon.a (mon.0) 1024 : cluster [WRN] Health check update: Degraded data redundancy: 201/1458 objects degraded (13.786%), 58 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:31.986 INFO:teuthology.orchestra.run.smithi099.stdout:true 2024-09-13T14:39:32.071 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:31 smithi099 bash[111000]: debug 2024-09-13T14:39:31.754+0000 7f247dbe0640 -1 osd.3 689 log_to_monitors true 2024-09-13T14:39:32.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[87147]: cluster 2024-09-13T14:39:31.199160+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e704: 8 total, 7 up, 8 in 2024-09-13T14:39:32.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[87147]: cluster 2024-09-13T14:39:31.199160+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e704: 8 total, 7 up, 8 in 2024-09-13T14:39:32.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[87147]: audit 2024-09-13T14:39:31.664098+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.99:0/1562509319' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:32.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[87147]: audit 2024-09-13T14:39:31.664098+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.99:0/1562509319' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:32.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[87147]: audit 2024-09-13T14:39:31.762499+0000 mon.c (mon.2) 37 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[87147]: audit 2024-09-13T14:39:31.762499+0000 mon.c (mon.2) 37 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[87147]: audit 2024-09-13T14:39:31.762885+0000 mon.a (mon.0) 1026 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[87147]: audit 2024-09-13T14:39:31.762885+0000 mon.a (mon.0) 1026 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[84576]: cluster 2024-09-13T14:39:31.199160+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e704: 8 total, 7 up, 8 in 2024-09-13T14:39:32.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[84576]: cluster 2024-09-13T14:39:31.199160+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e704: 8 total, 7 up, 8 in 2024-09-13T14:39:32.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[84576]: audit 2024-09-13T14:39:31.664098+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.99:0/1562509319' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:32.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[84576]: audit 2024-09-13T14:39:31.664098+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.99:0/1562509319' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:32.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[84576]: audit 2024-09-13T14:39:31.762499+0000 mon.c (mon.2) 37 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[84576]: audit 2024-09-13T14:39:31.762499+0000 mon.c (mon.2) 37 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[84576]: audit 2024-09-13T14:39:31.762885+0000 mon.a (mon.0) 1026 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[84576]: audit 2024-09-13T14:39:31.762885+0000 mon.a (mon.0) 1026 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:32 smithi177 bash[68594]: cluster 2024-09-13T14:39:31.199160+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e704: 8 total, 7 up, 8 in 2024-09-13T14:39:32.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:32 smithi177 bash[68594]: cluster 2024-09-13T14:39:31.199160+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e704: 8 total, 7 up, 8 in 2024-09-13T14:39:32.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:32 smithi177 bash[68594]: audit 2024-09-13T14:39:31.664098+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.99:0/1562509319' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:32.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:32 smithi177 bash[68594]: audit 2024-09-13T14:39:31.664098+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.99:0/1562509319' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:32.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:32 smithi177 bash[68594]: audit 2024-09-13T14:39:31.762499+0000 mon.c (mon.2) 37 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:32 smithi177 bash[68594]: audit 2024-09-13T14:39:31.762499+0000 mon.c (mon.2) 37 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:32 smithi177 bash[68594]: audit 2024-09-13T14:39:31.762885+0000 mon.a (mon.0) 1026 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:32.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:32 smithi177 bash[68594]: audit 2024-09-13T14:39:31.762885+0000 mon.a (mon.0) 1026 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:39:33.072 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:39:32 smithi099 bash[111000]: debug 2024-09-13T14:39:32.758+0000 7f2471982640 -1 osd.3 689 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:39:33.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:31.977584+0000 mgr.y (mgr.26252) 381 : audit [DBG] from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:31.977584+0000 mgr.y (mgr.26252) 381 : audit [DBG] from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: cluster 2024-09-13T14:39:32.175120+0000 mgr.y (mgr.26252) 382 : cluster [DBG] pgmap v595: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 29 KiB/s wr, 109 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:33.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: cluster 2024-09-13T14:39:32.175120+0000 mgr.y (mgr.26252) 382 : cluster [DBG] pgmap v595: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 29 KiB/s wr, 109 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:32.218859+0000 mon.a (mon.0) 1027 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:32.218859+0000 mon.a (mon.0) 1027 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: cluster 2024-09-13T14:39:32.229576+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e705: 8 total, 7 up, 8 in 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: cluster 2024-09-13T14:39:32.229576+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e705: 8 total, 7 up, 8 in 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:32.232780+0000 mon.c (mon.2) 38 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:32.232780+0000 mon.c (mon.2) 38 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:32.233295+0000 mon.a (mon.0) 1029 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:32.233295+0000 mon.a (mon.0) 1029 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:32.308131+0000 mgr.y (mgr.26252) 383 : audit [DBG] from='client.34965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:32.308131+0000 mgr.y (mgr.26252) 383 : audit [DBG] from='client.34965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:32.741291+0000 mon.a (mon.0) 1030 : audit [INF] from='osd.3 ' entity='osd.3' 2024-09-13T14:39:33.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[84576]: audit 2024-09-13T14:39:32.741291+0000 mon.a (mon.0) 1030 : audit [INF] from='osd.3 ' entity='osd.3' 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:31.977584+0000 mgr.y (mgr.26252) 381 : audit [DBG] from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:31.977584+0000 mgr.y (mgr.26252) 381 : audit [DBG] from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: cluster 2024-09-13T14:39:32.175120+0000 mgr.y (mgr.26252) 382 : cluster [DBG] pgmap v595: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 29 KiB/s wr, 109 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: cluster 2024-09-13T14:39:32.175120+0000 mgr.y (mgr.26252) 382 : cluster [DBG] pgmap v595: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 29 KiB/s wr, 109 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:32.218859+0000 mon.a (mon.0) 1027 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:32.218859+0000 mon.a (mon.0) 1027 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: cluster 2024-09-13T14:39:32.229576+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e705: 8 total, 7 up, 8 in 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: cluster 2024-09-13T14:39:32.229576+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e705: 8 total, 7 up, 8 in 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:32.232780+0000 mon.c (mon.2) 38 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:32.232780+0000 mon.c (mon.2) 38 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:32.233295+0000 mon.a (mon.0) 1029 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:32.233295+0000 mon.a (mon.0) 1029 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:32.308131+0000 mgr.y (mgr.26252) 383 : audit [DBG] from='client.34965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:32.308131+0000 mgr.y (mgr.26252) 383 : audit [DBG] from='client.34965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:32.741291+0000 mon.a (mon.0) 1030 : audit [INF] from='osd.3 ' entity='osd.3' 2024-09-13T14:39:33.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:33 smithi099 bash[87147]: audit 2024-09-13T14:39:32.741291+0000 mon.a (mon.0) 1030 : audit [INF] from='osd.3 ' entity='osd.3' 2024-09-13T14:39:33.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:31.977584+0000 mgr.y (mgr.26252) 381 : audit [DBG] from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:31.977584+0000 mgr.y (mgr.26252) 381 : audit [DBG] from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: cluster 2024-09-13T14:39:32.175120+0000 mgr.y (mgr.26252) 382 : cluster [DBG] pgmap v595: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 29 KiB/s wr, 109 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: cluster 2024-09-13T14:39:32.175120+0000 mgr.y (mgr.26252) 382 : cluster [DBG] pgmap v595: 617 pgs: 204 active+undersized, 58 active+undersized+degraded, 355 active+clean; 139 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 29 KiB/s wr, 109 op/s; 201/1458 objects degraded (13.786%) 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:32.218859+0000 mon.a (mon.0) 1027 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:32.218859+0000 mon.a (mon.0) 1027 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: cluster 2024-09-13T14:39:32.229576+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e705: 8 total, 7 up, 8 in 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: cluster 2024-09-13T14:39:32.229576+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e705: 8 total, 7 up, 8 in 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:32.232780+0000 mon.c (mon.2) 38 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:32.232780+0000 mon.c (mon.2) 38 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:32.233295+0000 mon.a (mon.0) 1029 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:32.233295+0000 mon.a (mon.0) 1029 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:32.308131+0000 mgr.y (mgr.26252) 383 : audit [DBG] from='client.34965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:32.308131+0000 mgr.y (mgr.26252) 383 : audit [DBG] from='client.34965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:33.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:32.741291+0000 mon.a (mon.0) 1030 : audit [INF] from='osd.3 ' entity='osd.3' 2024-09-13T14:39:33.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:33 smithi177 bash[68594]: audit 2024-09-13T14:39:32.741291+0000 mon.a (mon.0) 1030 : audit [INF] from='osd.3 ' entity='osd.3' 2024-09-13T14:39:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: cluster 2024-09-13T14:39:33.222081+0000 mon.a (mon.0) 1031 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:39:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: cluster 2024-09-13T14:39:33.222081+0000 mon.a (mon.0) 1031 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:39:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: cluster 2024-09-13T14:39:33.247774+0000 mon.a (mon.0) 1032 : cluster [INF] osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003] boot 2024-09-13T14:39:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: cluster 2024-09-13T14:39:33.247774+0000 mon.a (mon.0) 1032 : cluster [INF] osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003] boot 2024-09-13T14:39:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: cluster 2024-09-13T14:39:33.247878+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-13T14:39:34.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: cluster 2024-09-13T14:39:33.247878+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-13T14:39:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: audit 2024-09-13T14:39:33.249237+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:39:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: audit 2024-09-13T14:39:33.249237+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:39:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: audit 2024-09-13T14:39:33.280571+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: audit 2024-09-13T14:39:33.280571+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: audit 2024-09-13T14:39:33.282542+0000 mon.a (mon.0) 1036 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[84576]: audit 2024-09-13T14:39:33.282542+0000 mon.a (mon.0) 1036 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:34.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: cluster 2024-09-13T14:39:33.222081+0000 mon.a (mon.0) 1031 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:39:34.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: cluster 2024-09-13T14:39:33.222081+0000 mon.a (mon.0) 1031 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:39:34.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: cluster 2024-09-13T14:39:33.247774+0000 mon.a (mon.0) 1032 : cluster [INF] osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003] boot 2024-09-13T14:39:34.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: cluster 2024-09-13T14:39:33.247774+0000 mon.a (mon.0) 1032 : cluster [INF] osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003] boot 2024-09-13T14:39:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: cluster 2024-09-13T14:39:33.247878+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-13T14:39:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: cluster 2024-09-13T14:39:33.247878+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-13T14:39:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: audit 2024-09-13T14:39:33.249237+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:39:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: audit 2024-09-13T14:39:33.249237+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:39:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: audit 2024-09-13T14:39:33.280571+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: audit 2024-09-13T14:39:33.280571+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: audit 2024-09-13T14:39:33.282542+0000 mon.a (mon.0) 1036 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:34 smithi099 bash[87147]: audit 2024-09-13T14:39:33.282542+0000 mon.a (mon.0) 1036 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: cluster 2024-09-13T14:39:33.222081+0000 mon.a (mon.0) 1031 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:39:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: cluster 2024-09-13T14:39:33.222081+0000 mon.a (mon.0) 1031 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:39:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: cluster 2024-09-13T14:39:33.247774+0000 mon.a (mon.0) 1032 : cluster [INF] osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003] boot 2024-09-13T14:39:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: cluster 2024-09-13T14:39:33.247774+0000 mon.a (mon.0) 1032 : cluster [INF] osd.3 [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003] boot 2024-09-13T14:39:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: cluster 2024-09-13T14:39:33.247878+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-13T14:39:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: cluster 2024-09-13T14:39:33.247878+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-13T14:39:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: audit 2024-09-13T14:39:33.249237+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:39:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: audit 2024-09-13T14:39:33.249237+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:39:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: audit 2024-09-13T14:39:33.280571+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: audit 2024-09-13T14:39:33.280571+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: audit 2024-09-13T14:39:33.282542+0000 mon.a (mon.0) 1036 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:34 smithi177 bash[68594]: audit 2024-09-13T14:39:33.282542+0000 mon.a (mon.0) 1036 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:34.658 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 4 2024-09-13T14:39:35.502 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SnapshotLimitPP (5237 ms) 2024-09-13T14:39:35.502 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.RebuildObjectMapViaLockOwner 2024-09-13T14:39:35.502 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[84576]: cluster 2024-09-13T14:39:34.178974+0000 mgr.y (mgr.26252) 384 : cluster [DBG] pgmap v598: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[84576]: cluster 2024-09-13T14:39:34.178974+0000 mgr.y (mgr.26252) 384 : cluster [DBG] pgmap v598: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[84576]: cluster 2024-09-13T14:39:34.244482+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-13T14:39:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[84576]: cluster 2024-09-13T14:39:34.244482+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-13T14:39:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[84576]: audit 2024-09-13T14:39:34.648272+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.99:0/3460590251' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[84576]: audit 2024-09-13T14:39:34.648272+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.99:0/3460590251' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[87147]: cluster 2024-09-13T14:39:34.178974+0000 mgr.y (mgr.26252) 384 : cluster [DBG] pgmap v598: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[87147]: cluster 2024-09-13T14:39:34.178974+0000 mgr.y (mgr.26252) 384 : cluster [DBG] pgmap v598: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[87147]: cluster 2024-09-13T14:39:34.244482+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-13T14:39:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[87147]: cluster 2024-09-13T14:39:34.244482+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-13T14:39:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[87147]: audit 2024-09-13T14:39:34.648272+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.99:0/3460590251' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:35 smithi099 bash[87147]: audit 2024-09-13T14:39:34.648272+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.99:0/3460590251' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:35.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:35 smithi177 bash[68594]: cluster 2024-09-13T14:39:34.178974+0000 mgr.y (mgr.26252) 384 : cluster [DBG] pgmap v598: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:35.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:35 smithi177 bash[68594]: cluster 2024-09-13T14:39:34.178974+0000 mgr.y (mgr.26252) 384 : cluster [DBG] pgmap v598: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:35.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:35 smithi177 bash[68594]: cluster 2024-09-13T14:39:34.244482+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-13T14:39:35.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:35 smithi177 bash[68594]: cluster 2024-09-13T14:39:34.244482+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-13T14:39:35.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:35 smithi177 bash[68594]: audit 2024-09-13T14:39:34.648272+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.99:0/3460590251' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:35.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:35 smithi177 bash[68594]: audit 2024-09-13T14:39:34.648272+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.99:0/3460590251' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-13T14:39:35.647 INFO:tasks.workunit.client.0.smithi099.stdout:0.00% done 2024-09-13T14:39:35.647 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.RebuildObjectMapViaLockOwner (145 ms) 2024-09-13T14:39:35.647 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.RenameViaLockOwner 2024-09-13T14:39:35.647 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:35.831 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.RenameViaLockOwner (184 ms) 2024-09-13T14:39:35.831 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SnapCreateViaLockOwner 2024-09-13T14:39:35.831 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[84576]: cluster 2024-09-13T14:39:35.246474+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-13T14:39:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[84576]: cluster 2024-09-13T14:39:35.246474+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-13T14:39:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[84576]: cluster 2024-09-13T14:39:36.108501+0000 mon.a (mon.0) 1039 : cluster [WRN] Health check update: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[84576]: cluster 2024-09-13T14:39:36.108501+0000 mon.a (mon.0) 1039 : cluster [WRN] Health check update: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[84576]: cluster 2024-09-13T14:39:36.123705+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-13T14:39:36.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[84576]: cluster 2024-09-13T14:39:36.123705+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-13T14:39:36.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[87147]: cluster 2024-09-13T14:39:35.246474+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-13T14:39:36.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[87147]: cluster 2024-09-13T14:39:35.246474+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-13T14:39:36.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[87147]: cluster 2024-09-13T14:39:36.108501+0000 mon.a (mon.0) 1039 : cluster [WRN] Health check update: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:36.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[87147]: cluster 2024-09-13T14:39:36.108501+0000 mon.a (mon.0) 1039 : cluster [WRN] Health check update: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:36.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[87147]: cluster 2024-09-13T14:39:36.123705+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-13T14:39:36.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:36 smithi099 bash[87147]: cluster 2024-09-13T14:39:36.123705+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-13T14:39:36.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:36 smithi177 bash[68594]: cluster 2024-09-13T14:39:35.246474+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-13T14:39:36.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:36 smithi177 bash[68594]: cluster 2024-09-13T14:39:35.246474+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-13T14:39:36.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:36 smithi177 bash[68594]: cluster 2024-09-13T14:39:36.108501+0000 mon.a (mon.0) 1039 : cluster [WRN] Health check update: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:36.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:36 smithi177 bash[68594]: cluster 2024-09-13T14:39:36.108501+0000 mon.a (mon.0) 1039 : cluster [WRN] Health check update: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:36.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:36 smithi177 bash[68594]: cluster 2024-09-13T14:39:36.123705+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-13T14:39:36.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:36 smithi177 bash[68594]: cluster 2024-09-13T14:39:36.123705+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-13T14:39:37.225 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SnapCreateViaLockOwner (1394 ms) 2024-09-13T14:39:37.225 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SnapRemoveViaLockOwner 2024-09-13T14:39:37.225 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:39:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:39:37] "GET /metrics HTTP/1.1" 200 75031 "" "Prometheus/2.51.0" 2024-09-13T14:39:37.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:37 smithi099 bash[84576]: cluster 2024-09-13T14:39:36.180251+0000 mgr.y (mgr.26252) 385 : cluster [DBG] pgmap v602: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.3 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:37.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:37 smithi099 bash[84576]: cluster 2024-09-13T14:39:36.180251+0000 mgr.y (mgr.26252) 385 : cluster [DBG] pgmap v602: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.3 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:37.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:37 smithi099 bash[84576]: cluster 2024-09-13T14:39:37.153383+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-13T14:39:37.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:37 smithi099 bash[84576]: cluster 2024-09-13T14:39:37.153383+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-13T14:39:37.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:37 smithi099 bash[87147]: cluster 2024-09-13T14:39:36.180251+0000 mgr.y (mgr.26252) 385 : cluster [DBG] pgmap v602: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.3 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:37.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:37 smithi099 bash[87147]: cluster 2024-09-13T14:39:36.180251+0000 mgr.y (mgr.26252) 385 : cluster [DBG] pgmap v602: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.3 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:37.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:37 smithi099 bash[87147]: cluster 2024-09-13T14:39:37.153383+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-13T14:39:37.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:37 smithi099 bash[87147]: cluster 2024-09-13T14:39:37.153383+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-13T14:39:37.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade pause' 2024-09-13T14:39:37.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:37 smithi177 bash[68594]: cluster 2024-09-13T14:39:36.180251+0000 mgr.y (mgr.26252) 385 : cluster [DBG] pgmap v602: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.3 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:37.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:37 smithi177 bash[68594]: cluster 2024-09-13T14:39:36.180251+0000 mgr.y (mgr.26252) 385 : cluster [DBG] pgmap v602: 617 pgs: 119 peering, 111 active+undersized, 32 active+undersized+degraded, 355 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.3 KiB/s wr, 36 op/s; 123/1473 objects degraded (8.350%) 2024-09-13T14:39:37.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:37 smithi177 bash[68594]: cluster 2024-09-13T14:39:37.153383+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-13T14:39:37.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:37 smithi177 bash[68594]: cluster 2024-09-13T14:39:37.153383+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-13T14:39:38.461 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[84576]: cluster 2024-09-13T14:39:38.254204+0000 mon.a (mon.0) 1042 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded) 2024-09-13T14:39:38.461 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[84576]: cluster 2024-09-13T14:39:38.254204+0000 mon.a (mon.0) 1042 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded) 2024-09-13T14:39:38.462 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[84576]: cluster 2024-09-13T14:39:38.254234+0000 mon.a (mon.0) 1043 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:38.462 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[84576]: cluster 2024-09-13T14:39:38.254234+0000 mon.a (mon.0) 1043 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:38.462 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[84576]: cluster 2024-09-13T14:39:38.267038+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-13T14:39:38.462 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[84576]: cluster 2024-09-13T14:39:38.267038+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-13T14:39:38.462 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[87147]: cluster 2024-09-13T14:39:38.254204+0000 mon.a (mon.0) 1042 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded) 2024-09-13T14:39:38.462 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[87147]: cluster 2024-09-13T14:39:38.254204+0000 mon.a (mon.0) 1042 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded) 2024-09-13T14:39:38.462 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[87147]: cluster 2024-09-13T14:39:38.254234+0000 mon.a (mon.0) 1043 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:38.462 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[87147]: cluster 2024-09-13T14:39:38.254234+0000 mon.a (mon.0) 1043 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:38.462 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[87147]: cluster 2024-09-13T14:39:38.267038+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-13T14:39:38.462 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:38 smithi099 bash[87147]: cluster 2024-09-13T14:39:38.267038+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-13T14:39:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:38 smithi177 bash[68594]: cluster 2024-09-13T14:39:38.254204+0000 mon.a (mon.0) 1042 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded) 2024-09-13T14:39:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:38 smithi177 bash[68594]: cluster 2024-09-13T14:39:38.254204+0000 mon.a (mon.0) 1042 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 123/1473 objects degraded (8.350%), 32 pgs degraded) 2024-09-13T14:39:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:38 smithi177 bash[68594]: cluster 2024-09-13T14:39:38.254234+0000 mon.a (mon.0) 1043 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:38.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:38 smithi177 bash[68594]: cluster 2024-09-13T14:39:38.254234+0000 mon.a (mon.0) 1043 : cluster [INF] Cluster is now healthy 2024-09-13T14:39:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:38 smithi177 bash[68594]: cluster 2024-09-13T14:39:38.267038+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-13T14:39:38.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:38 smithi177 bash[68594]: cluster 2024-09-13T14:39:38.267038+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-13T14:39:39.391 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SnapRemoveViaLockOwner (2166 ms) 2024-09-13T14:39:39.392 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.UpdateFeaturesViaLockOwner 2024-09-13T14:39:39.392 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:39.492 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.UpdateFeaturesViaLockOwner (100 ms) 2024-09-13T14:39:39.492 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.EnableJournalingViaLockOwner 2024-09-13T14:39:39.492 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:39:39.492 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.EnableJournalingViaLockOwner (0 ms) 2024-09-13T14:39:39.492 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SnapRemove2 2024-09-13T14:39:39.492 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:39.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:39 smithi099 bash[84576]: cluster 2024-09-13T14:39:38.181786+0000 mgr.y (mgr.26252) 386 : cluster [DBG] pgmap v604: 617 pgs: 119 peering, 498 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 45 op/s; 34 KiB/s, 9 keys/s, 6 objects/s recovering 2024-09-13T14:39:39.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:39 smithi099 bash[84576]: cluster 2024-09-13T14:39:38.181786+0000 mgr.y (mgr.26252) 386 : cluster [DBG] pgmap v604: 617 pgs: 119 peering, 498 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 45 op/s; 34 KiB/s, 9 keys/s, 6 objects/s recovering 2024-09-13T14:39:39.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:39 smithi099 bash[87147]: cluster 2024-09-13T14:39:38.181786+0000 mgr.y (mgr.26252) 386 : cluster [DBG] pgmap v604: 617 pgs: 119 peering, 498 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 45 op/s; 34 KiB/s, 9 keys/s, 6 objects/s recovering 2024-09-13T14:39:39.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:39 smithi099 bash[87147]: cluster 2024-09-13T14:39:38.181786+0000 mgr.y (mgr.26252) 386 : cluster [DBG] pgmap v604: 617 pgs: 119 peering, 498 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 45 op/s; 34 KiB/s, 9 keys/s, 6 objects/s recovering 2024-09-13T14:39:39.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:39 smithi177 bash[68594]: cluster 2024-09-13T14:39:38.181786+0000 mgr.y (mgr.26252) 386 : cluster [DBG] pgmap v604: 617 pgs: 119 peering, 498 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 45 op/s; 34 KiB/s, 9 keys/s, 6 objects/s recovering 2024-09-13T14:39:39.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:39 smithi177 bash[68594]: cluster 2024-09-13T14:39:38.181786+0000 mgr.y (mgr.26252) 386 : cluster [DBG] pgmap v604: 617 pgs: 119 peering, 498 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 45 op/s; 34 KiB/s, 9 keys/s, 6 objects/s recovering 2024-09-13T14:39:40.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:40 smithi177 bash[68594]: cluster 2024-09-13T14:39:39.323376+0000 mon.a (mon.0) 1045 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-13T14:39:40.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:40 smithi177 bash[68594]: cluster 2024-09-13T14:39:39.323376+0000 mon.a (mon.0) 1045 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-13T14:39:40.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:39:40 smithi177 bash[61439]: ts=2024-09-13T14:39:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:40.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:40 smithi099 bash[84576]: cluster 2024-09-13T14:39:39.323376+0000 mon.a (mon.0) 1045 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-13T14:39:40.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:40 smithi099 bash[84576]: cluster 2024-09-13T14:39:39.323376+0000 mon.a (mon.0) 1045 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-13T14:39:40.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:40 smithi099 bash[87147]: cluster 2024-09-13T14:39:39.323376+0000 mon.a (mon.0) 1045 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-13T14:39:40.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:40 smithi099 bash[87147]: cluster 2024-09-13T14:39:39.323376+0000 mon.a (mon.0) 1045 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-13T14:39:41.188 INFO:tasks.workunit.client.0.smithi099.stdout:100.00% done 2024-09-13T14:39:41.188 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SnapRemove2 (1697 ms) 2024-09-13T14:39:41.188 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SnapRenameViaLockOwner 2024-09-13T14:39:41.188 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:41.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:41 smithi177 bash[68594]: cluster 2024-09-13T14:39:40.182894+0000 mgr.y (mgr.26252) 387 : cluster [DBG] pgmap v607: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 11 KiB/s wr, 150 op/s; 36 KiB/s, 14 keys/s, 8 objects/s recovering 2024-09-13T14:39:41.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:41 smithi177 bash[68594]: cluster 2024-09-13T14:39:40.182894+0000 mgr.y (mgr.26252) 387 : cluster [DBG] pgmap v607: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 11 KiB/s wr, 150 op/s; 36 KiB/s, 14 keys/s, 8 objects/s recovering 2024-09-13T14:39:41.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:41 smithi177 bash[68594]: cluster 2024-09-13T14:39:40.329999+0000 mon.a (mon.0) 1046 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-13T14:39:41.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:41 smithi177 bash[68594]: cluster 2024-09-13T14:39:40.329999+0000 mon.a (mon.0) 1046 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-13T14:39:41.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:41 smithi177 bash[68594]: cluster 2024-09-13T14:39:41.126715+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-13T14:39:41.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:41 smithi177 bash[68594]: cluster 2024-09-13T14:39:41.126715+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-13T14:39:41.618 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[84576]: cluster 2024-09-13T14:39:40.182894+0000 mgr.y (mgr.26252) 387 : cluster [DBG] pgmap v607: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 11 KiB/s wr, 150 op/s; 36 KiB/s, 14 keys/s, 8 objects/s recovering 2024-09-13T14:39:41.619 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[84576]: cluster 2024-09-13T14:39:40.182894+0000 mgr.y (mgr.26252) 387 : cluster [DBG] pgmap v607: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 11 KiB/s wr, 150 op/s; 36 KiB/s, 14 keys/s, 8 objects/s recovering 2024-09-13T14:39:41.619 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[84576]: cluster 2024-09-13T14:39:40.329999+0000 mon.a (mon.0) 1046 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-13T14:39:41.619 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[84576]: cluster 2024-09-13T14:39:40.329999+0000 mon.a (mon.0) 1046 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-13T14:39:41.619 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[84576]: cluster 2024-09-13T14:39:41.126715+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-13T14:39:41.619 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[84576]: cluster 2024-09-13T14:39:41.126715+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-13T14:39:41.619 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[87147]: cluster 2024-09-13T14:39:40.182894+0000 mgr.y (mgr.26252) 387 : cluster [DBG] pgmap v607: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 11 KiB/s wr, 150 op/s; 36 KiB/s, 14 keys/s, 8 objects/s recovering 2024-09-13T14:39:41.620 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[87147]: cluster 2024-09-13T14:39:40.182894+0000 mgr.y (mgr.26252) 387 : cluster [DBG] pgmap v607: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 11 KiB/s wr, 150 op/s; 36 KiB/s, 14 keys/s, 8 objects/s recovering 2024-09-13T14:39:41.620 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[87147]: cluster 2024-09-13T14:39:40.329999+0000 mon.a (mon.0) 1046 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-13T14:39:41.620 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[87147]: cluster 2024-09-13T14:39:40.329999+0000 mon.a (mon.0) 1046 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-13T14:39:41.620 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[87147]: cluster 2024-09-13T14:39:41.126715+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-13T14:39:41.620 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:41 smithi099 bash[87147]: cluster 2024-09-13T14:39:41.126715+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-13T14:39:42.246 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SnapRenameViaLockOwner (1058 ms) 2024-09-13T14:39:42.246 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SnapProtectViaLockOwner 2024-09-13T14:39:42.247 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:42.813 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:43.245 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SnapProtectViaLockOwner (999 ms) 2024-09-13T14:39:43.245 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SnapUnprotectViaLockOwner 2024-09-13T14:39:43.245 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:43.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: cluster 2024-09-13T14:39:42.129686+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-13T14:39:43.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: cluster 2024-09-13T14:39:42.129686+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-13T14:39:43.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: cluster 2024-09-13T14:39:42.184120+0000 mgr.y (mgr.26252) 388 : cluster [DBG] pgmap v611: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.9 KiB/s wr, 110 op/s; 2.1 KiB/s, 4 keys/s, 2 objects/s recovering 2024-09-13T14:39:43.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: cluster 2024-09-13T14:39:42.184120+0000 mgr.y (mgr.26252) 388 : cluster [DBG] pgmap v611: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.9 KiB/s wr, 110 op/s; 2.1 KiB/s, 4 keys/s, 2 objects/s recovering 2024-09-13T14:39:43.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.356261+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.356261+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.365812+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.365812+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.367374+0000 mon.a (mon.0) 1051 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.367374+0000 mon.a (mon.0) 1051 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.368524+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.368524+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.378297+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.378297+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.464670+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.464670+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.466762+0000 mon.a (mon.0) 1055 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.466762+0000 mon.a (mon.0) 1055 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.468339+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.468339+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.469379+0000 mon.a (mon.0) 1057 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.469379+0000 mon.a (mon.0) 1057 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.470793+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:39:43.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.470793+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:39:43.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.470944+0000 mgr.y (mgr.26252) 389 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:39:43.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: audit 2024-09-13T14:39:42.470944+0000 mgr.y (mgr.26252) 389 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:39:43.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: cephadm 2024-09-13T14:39:42.472533+0000 mgr.y (mgr.26252) 390 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-13T14:39:43.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[87147]: cephadm 2024-09-13T14:39:42.472533+0000 mgr.y (mgr.26252) 390 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: cluster 2024-09-13T14:39:42.129686+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: cluster 2024-09-13T14:39:42.129686+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: cluster 2024-09-13T14:39:42.184120+0000 mgr.y (mgr.26252) 388 : cluster [DBG] pgmap v611: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.9 KiB/s wr, 110 op/s; 2.1 KiB/s, 4 keys/s, 2 objects/s recovering 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: cluster 2024-09-13T14:39:42.184120+0000 mgr.y (mgr.26252) 388 : cluster [DBG] pgmap v611: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.9 KiB/s wr, 110 op/s; 2.1 KiB/s, 4 keys/s, 2 objects/s recovering 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.356261+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.356261+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.365812+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.365812+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.367374+0000 mon.a (mon.0) 1051 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.367374+0000 mon.a (mon.0) 1051 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.368524+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.368524+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.378297+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.378297+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.464670+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.464670+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.466762+0000 mon.a (mon.0) 1055 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.466762+0000 mon.a (mon.0) 1055 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.468339+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.468339+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.469379+0000 mon.a (mon.0) 1057 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.469379+0000 mon.a (mon.0) 1057 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.470793+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.470793+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.470944+0000 mgr.y (mgr.26252) 389 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:39:43.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: audit 2024-09-13T14:39:42.470944+0000 mgr.y (mgr.26252) 389 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:39:43.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: cephadm 2024-09-13T14:39:42.472533+0000 mgr.y (mgr.26252) 390 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-13T14:39:43.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:43 smithi099 bash[84576]: cephadm 2024-09-13T14:39:42.472533+0000 mgr.y (mgr.26252) 390 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-13T14:39:43.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: cluster 2024-09-13T14:39:42.129686+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-13T14:39:43.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: cluster 2024-09-13T14:39:42.129686+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-13T14:39:43.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: cluster 2024-09-13T14:39:42.184120+0000 mgr.y (mgr.26252) 388 : cluster [DBG] pgmap v611: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.9 KiB/s wr, 110 op/s; 2.1 KiB/s, 4 keys/s, 2 objects/s recovering 2024-09-13T14:39:43.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: cluster 2024-09-13T14:39:42.184120+0000 mgr.y (mgr.26252) 388 : cluster [DBG] pgmap v611: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.9 KiB/s wr, 110 op/s; 2.1 KiB/s, 4 keys/s, 2 objects/s recovering 2024-09-13T14:39:43.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.356261+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.356261+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.365812+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.365812+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.367374+0000 mon.a (mon.0) 1051 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:43.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.367374+0000 mon.a (mon.0) 1051 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.368524+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.368524+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.378297+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.378297+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.464670+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.464670+0000 mon.a (mon.0) 1054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.466762+0000 mon.a (mon.0) 1055 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.466762+0000 mon.a (mon.0) 1055 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.468339+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.468339+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.469379+0000 mon.a (mon.0) 1057 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.469379+0000 mon.a (mon.0) 1057 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:43.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.470793+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:39:43.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.470793+0000 mon.a (mon.0) 1058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T14:39:43.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.470944+0000 mgr.y (mgr.26252) 389 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:39:43.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: audit 2024-09-13T14:39:42.470944+0000 mgr.y (mgr.26252) 389 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T14:39:43.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: cephadm 2024-09-13T14:39:42.472533+0000 mgr.y (mgr.26252) 390 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-13T14:39:43.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:43 smithi177 bash[68594]: cephadm 2024-09-13T14:39:42.472533+0000 mgr.y (mgr.26252) 390 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-13T14:39:44.283 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SnapUnprotectViaLockOwner (1037 ms) 2024-09-13T14:39:44.283 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.FlattenViaLockOwner 2024-09-13T14:39:44.283 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:44.423 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:44 smithi099 bash[84576]: cluster 2024-09-13T14:39:43.145642+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-13T14:39:44.423 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:44 smithi099 bash[84576]: cluster 2024-09-13T14:39:43.145642+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-13T14:39:44.424 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:44 smithi099 bash[87147]: cluster 2024-09-13T14:39:43.145642+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-13T14:39:44.424 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:44 smithi099 bash[87147]: cluster 2024-09-13T14:39:43.145642+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-13T14:39:44.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:44 smithi177 bash[68594]: cluster 2024-09-13T14:39:43.145642+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-13T14:39:44.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:44 smithi177 bash[68594]: cluster 2024-09-13T14:39:43.145642+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-13T14:39:45.338 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.FlattenViaLockOwner (1056 ms) 2024-09-13T14:39:45.339 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ResizeViaLockOwner 2024-09-13T14:39:45.339 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:45.387 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ResizeViaLockOwner (49 ms) 2024-09-13T14:39:45.387 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SparsifyViaLockOwner 2024-09-13T14:39:45.387 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:45.437 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SparsifyViaLockOwner (50 ms) 2024-09-13T14:39:45.437 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ObjectMapConsistentSnap 2024-09-13T14:39:45.437 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:45.451 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:45 smithi177 bash[68594]: cluster 2024-09-13T14:39:44.150544+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-13T14:39:45.451 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:45 smithi177 bash[68594]: cluster 2024-09-13T14:39:44.150544+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-13T14:39:45.451 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:45 smithi177 bash[68594]: cluster 2024-09-13T14:39:44.185651+0000 mgr.y (mgr.26252) 391 : cluster [DBG] pgmap v614: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 12 KiB/s wr, 208 op/s 2024-09-13T14:39:45.451 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:45 smithi177 bash[68594]: cluster 2024-09-13T14:39:44.185651+0000 mgr.y (mgr.26252) 391 : cluster [DBG] pgmap v614: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 12 KiB/s wr, 208 op/s 2024-09-13T14:39:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:45 smithi099 bash[84576]: cluster 2024-09-13T14:39:44.150544+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-13T14:39:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:45 smithi099 bash[84576]: cluster 2024-09-13T14:39:44.150544+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-13T14:39:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:45 smithi099 bash[84576]: cluster 2024-09-13T14:39:44.185651+0000 mgr.y (mgr.26252) 391 : cluster [DBG] pgmap v614: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 12 KiB/s wr, 208 op/s 2024-09-13T14:39:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:45 smithi099 bash[84576]: cluster 2024-09-13T14:39:44.185651+0000 mgr.y (mgr.26252) 391 : cluster [DBG] pgmap v614: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 12 KiB/s wr, 208 op/s 2024-09-13T14:39:45.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:45 smithi099 bash[87147]: cluster 2024-09-13T14:39:44.150544+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-13T14:39:45.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:45 smithi099 bash[87147]: cluster 2024-09-13T14:39:44.150544+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-13T14:39:45.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:45 smithi099 bash[87147]: cluster 2024-09-13T14:39:44.185651+0000 mgr.y (mgr.26252) 391 : cluster [DBG] pgmap v614: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 12 KiB/s wr, 208 op/s 2024-09-13T14:39:45.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:45 smithi099 bash[87147]: cluster 2024-09-13T14:39:44.185651+0000 mgr.y (mgr.26252) 391 : cluster [DBG] pgmap v614: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 12 KiB/s wr, 208 op/s 2024-09-13T14:39:46.540 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:46 smithi099 bash[84576]: cluster 2024-09-13T14:39:45.161676+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-13T14:39:46.540 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:46 smithi099 bash[84576]: cluster 2024-09-13T14:39:45.161676+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-13T14:39:46.540 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:46 smithi099 bash[84576]: cluster 2024-09-13T14:39:46.129326+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-13T14:39:46.540 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:46 smithi099 bash[84576]: cluster 2024-09-13T14:39:46.129326+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-13T14:39:46.540 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:46 smithi099 bash[87147]: cluster 2024-09-13T14:39:45.161676+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-13T14:39:46.540 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:46 smithi099 bash[87147]: cluster 2024-09-13T14:39:45.161676+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-13T14:39:46.540 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:46 smithi099 bash[87147]: cluster 2024-09-13T14:39:46.129326+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-13T14:39:46.541 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:46 smithi099 bash[87147]: cluster 2024-09-13T14:39:46.129326+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-13T14:39:46.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:46 smithi177 bash[68594]: cluster 2024-09-13T14:39:45.161676+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-13T14:39:46.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:46 smithi177 bash[68594]: cluster 2024-09-13T14:39:45.161676+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-13T14:39:46.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:46 smithi177 bash[68594]: cluster 2024-09-13T14:39:46.129326+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-13T14:39:46.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:46 smithi177 bash[68594]: cluster 2024-09-13T14:39:46.129326+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-13T14:39:46.919 INFO:teuthology.orchestra.run.smithi099.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:39:47.211 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:39:47] "GET /metrics HTTP/1.1" 200 75031 "" "Prometheus/2.51.0" 2024-09-13T14:39:47.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: cluster 2024-09-13T14:39:46.192883+0000 mgr.y (mgr.26252) 392 : cluster [DBG] pgmap v617: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s rd, 11 KiB/s wr, 201 op/s 2024-09-13T14:39:47.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: cluster 2024-09-13T14:39:46.192883+0000 mgr.y (mgr.26252) 392 : cluster [DBG] pgmap v617: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s rd, 11 KiB/s wr, 201 op/s 2024-09-13T14:39:47.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: cephadm 2024-09-13T14:39:46.250500+0000 mgr.y (mgr.26252) 393 : cephadm [INF] Upgrade: Updating osd.4 2024-09-13T14:39:47.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: cephadm 2024-09-13T14:39:46.250500+0000 mgr.y (mgr.26252) 393 : cephadm [INF] Upgrade: Updating osd.4 2024-09-13T14:39:47.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: audit 2024-09-13T14:39:46.255976+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: audit 2024-09-13T14:39:46.255976+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: audit 2024-09-13T14:39:46.259191+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:39:47.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: audit 2024-09-13T14:39:46.259191+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:39:47.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: audit 2024-09-13T14:39:46.260400+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:47.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: audit 2024-09-13T14:39:46.260400+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:47.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: cephadm 2024-09-13T14:39:46.268416+0000 mgr.y (mgr.26252) 394 : cephadm [INF] Deploying daemon osd.4 on smithi177 2024-09-13T14:39:47.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: cephadm 2024-09-13T14:39:46.268416+0000 mgr.y (mgr.26252) 394 : cephadm [INF] Deploying daemon osd.4 on smithi177 2024-09-13T14:39:47.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: audit 2024-09-13T14:39:46.921710+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: audit 2024-09-13T14:39:46.921710+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: cluster 2024-09-13T14:39:47.179460+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-13T14:39:47.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[87147]: cluster 2024-09-13T14:39:47.179460+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-13T14:39:47.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: cluster 2024-09-13T14:39:46.192883+0000 mgr.y (mgr.26252) 392 : cluster [DBG] pgmap v617: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s rd, 11 KiB/s wr, 201 op/s 2024-09-13T14:39:47.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: cluster 2024-09-13T14:39:46.192883+0000 mgr.y (mgr.26252) 392 : cluster [DBG] pgmap v617: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s rd, 11 KiB/s wr, 201 op/s 2024-09-13T14:39:47.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: cephadm 2024-09-13T14:39:46.250500+0000 mgr.y (mgr.26252) 393 : cephadm [INF] Upgrade: Updating osd.4 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: cephadm 2024-09-13T14:39:46.250500+0000 mgr.y (mgr.26252) 393 : cephadm [INF] Upgrade: Updating osd.4 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: audit 2024-09-13T14:39:46.255976+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: audit 2024-09-13T14:39:46.255976+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: audit 2024-09-13T14:39:46.259191+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: audit 2024-09-13T14:39:46.259191+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: audit 2024-09-13T14:39:46.260400+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: audit 2024-09-13T14:39:46.260400+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: cephadm 2024-09-13T14:39:46.268416+0000 mgr.y (mgr.26252) 394 : cephadm [INF] Deploying daemon osd.4 on smithi177 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: cephadm 2024-09-13T14:39:46.268416+0000 mgr.y (mgr.26252) 394 : cephadm [INF] Deploying daemon osd.4 on smithi177 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: audit 2024-09-13T14:39:46.921710+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: audit 2024-09-13T14:39:46.921710+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: cluster 2024-09-13T14:39:47.179460+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-13T14:39:47.605 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:47 smithi099 bash[84576]: cluster 2024-09-13T14:39:47.179460+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-13T14:39:47.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: cluster 2024-09-13T14:39:46.192883+0000 mgr.y (mgr.26252) 392 : cluster [DBG] pgmap v617: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s rd, 11 KiB/s wr, 201 op/s 2024-09-13T14:39:47.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: cluster 2024-09-13T14:39:46.192883+0000 mgr.y (mgr.26252) 392 : cluster [DBG] pgmap v617: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s rd, 11 KiB/s wr, 201 op/s 2024-09-13T14:39:47.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: cephadm 2024-09-13T14:39:46.250500+0000 mgr.y (mgr.26252) 393 : cephadm [INF] Upgrade: Updating osd.4 2024-09-13T14:39:47.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: cephadm 2024-09-13T14:39:46.250500+0000 mgr.y (mgr.26252) 393 : cephadm [INF] Upgrade: Updating osd.4 2024-09-13T14:39:47.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: audit 2024-09-13T14:39:46.255976+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: audit 2024-09-13T14:39:46.255976+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: audit 2024-09-13T14:39:46.259191+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:39:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: audit 2024-09-13T14:39:46.259191+0000 mon.a (mon.0) 1064 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:39:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: audit 2024-09-13T14:39:46.260400+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: audit 2024-09-13T14:39:46.260400+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: cephadm 2024-09-13T14:39:46.268416+0000 mgr.y (mgr.26252) 394 : cephadm [INF] Deploying daemon osd.4 on smithi177 2024-09-13T14:39:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: cephadm 2024-09-13T14:39:46.268416+0000 mgr.y (mgr.26252) 394 : cephadm [INF] Deploying daemon osd.4 on smithi177 2024-09-13T14:39:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: audit 2024-09-13T14:39:46.921710+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: audit 2024-09-13T14:39:46.921710+0000 mon.a (mon.0) 1066 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:47.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: cluster 2024-09-13T14:39:47.179460+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-13T14:39:47.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:47 smithi177 bash[68594]: cluster 2024-09-13T14:39:47.179460+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-13T14:39:47.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:39:48.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:48 smithi177 bash[68594]: audit 2024-09-13T14:39:46.914674+0000 mgr.y (mgr.26252) 395 : audit [DBG] from='client.34977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:48 smithi177 bash[68594]: audit 2024-09-13T14:39:46.914674+0000 mgr.y (mgr.26252) 395 : audit [DBG] from='client.34977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:48 smithi177 bash[68594]: cephadm 2024-09-13T14:39:46.915080+0000 mgr.y (mgr.26252) 396 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:39:48.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:48 smithi177 bash[68594]: cephadm 2024-09-13T14:39:46.915080+0000 mgr.y (mgr.26252) 396 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:39:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:48 smithi099 bash[84576]: audit 2024-09-13T14:39:46.914674+0000 mgr.y (mgr.26252) 395 : audit [DBG] from='client.34977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:48 smithi099 bash[84576]: audit 2024-09-13T14:39:46.914674+0000 mgr.y (mgr.26252) 395 : audit [DBG] from='client.34977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:48 smithi099 bash[84576]: cephadm 2024-09-13T14:39:46.915080+0000 mgr.y (mgr.26252) 396 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:39:48.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:48 smithi099 bash[84576]: cephadm 2024-09-13T14:39:46.915080+0000 mgr.y (mgr.26252) 396 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:39:48.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:48 smithi099 bash[87147]: audit 2024-09-13T14:39:46.914674+0000 mgr.y (mgr.26252) 395 : audit [DBG] from='client.34977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:48 smithi099 bash[87147]: audit 2024-09-13T14:39:46.914674+0000 mgr.y (mgr.26252) 395 : audit [DBG] from='client.34977 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:48 smithi099 bash[87147]: cephadm 2024-09-13T14:39:46.915080+0000 mgr.y (mgr.26252) 396 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:39:48.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:48 smithi099 bash[87147]: cephadm 2024-09-13T14:39:46.915080+0000 mgr.y (mgr.26252) 396 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:39:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: cluster 2024-09-13T14:39:48.194377+0000 mgr.y (mgr.26252) 397 : cluster [DBG] pgmap v619: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 6.9 KiB/s wr, 161 op/s 2024-09-13T14:39:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: cluster 2024-09-13T14:39:48.194377+0000 mgr.y (mgr.26252) 397 : cluster [DBG] pgmap v619: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 6.9 KiB/s wr, 161 op/s 2024-09-13T14:39:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.257639+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.257639+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.260681+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.260681+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: cluster 2024-09-13T14:39:48.277090+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-13T14:39:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: cluster 2024-09-13T14:39:48.277090+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-13T14:39:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.709441+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]} : dispatch 2024-09-13T14:39:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.709441+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]} : dispatch 2024-09-13T14:39:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.710159+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.710159+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.710543+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.710543+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.710903+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]} : dispatch 2024-09-13T14:39:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.710903+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]} : dispatch 2024-09-13T14:39:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.711265+0000 mon.a (mon.0) 1075 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: cluster 2024-09-13T14:39:48.194377+0000 mgr.y (mgr.26252) 397 : cluster [DBG] pgmap v619: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 6.9 KiB/s wr, 161 op/s 2024-09-13T14:39:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: cluster 2024-09-13T14:39:48.194377+0000 mgr.y (mgr.26252) 397 : cluster [DBG] pgmap v619: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 6.9 KiB/s wr, 161 op/s 2024-09-13T14:39:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.257639+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.257639+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.260681+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.260681+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: cluster 2024-09-13T14:39:48.277090+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-13T14:39:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: cluster 2024-09-13T14:39:48.277090+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-13T14:39:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.709441+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]} : dispatch 2024-09-13T14:39:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.709441+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]} : dispatch 2024-09-13T14:39:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.710159+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.710159+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.710543+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.710543+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.710903+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]} : dispatch 2024-09-13T14:39:49.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.710903+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]} : dispatch 2024-09-13T14:39:49.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.711265+0000 mon.a (mon.0) 1075 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.578 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.711265+0000 mon.a (mon.0) 1075 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.578 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.711622+0000 mon.a (mon.0) 1076 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]} : dispatch 2024-09-13T14:39:49.578 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.711622+0000 mon.a (mon.0) 1076 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]} : dispatch 2024-09-13T14:39:49.578 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.711976+0000 mon.a (mon.0) 1077 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.579 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.711976+0000 mon.a (mon.0) 1077 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.579 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.712332+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]} : dispatch 2024-09-13T14:39:49.579 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.712332+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]} : dispatch 2024-09-13T14:39:49.579 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.712678+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]} : dispatch 2024-09-13T14:39:49.579 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.712678+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]} : dispatch 2024-09-13T14:39:49.580 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.799282+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.580 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[84576]: audit 2024-09-13T14:39:48.799282+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.580 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.711265+0000 mon.a (mon.0) 1075 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.580 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.711622+0000 mon.a (mon.0) 1076 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]} : dispatch 2024-09-13T14:39:49.580 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.711622+0000 mon.a (mon.0) 1076 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]} : dispatch 2024-09-13T14:39:49.580 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.711976+0000 mon.a (mon.0) 1077 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.581 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.711976+0000 mon.a (mon.0) 1077 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.581 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.712332+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]} : dispatch 2024-09-13T14:39:49.581 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.712332+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]} : dispatch 2024-09-13T14:39:49.581 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.712678+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]} : dispatch 2024-09-13T14:39:49.581 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.712678+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]} : dispatch 2024-09-13T14:39:49.581 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.799282+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.581 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:49 smithi099 bash[87147]: audit 2024-09-13T14:39:48.799282+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: cluster 2024-09-13T14:39:48.194377+0000 mgr.y (mgr.26252) 397 : cluster [DBG] pgmap v619: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 6.9 KiB/s wr, 161 op/s 2024-09-13T14:39:49.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: cluster 2024-09-13T14:39:48.194377+0000 mgr.y (mgr.26252) 397 : cluster [DBG] pgmap v619: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 6.9 KiB/s wr, 161 op/s 2024-09-13T14:39:49.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.257639+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.257639+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.260681+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.260681+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: cluster 2024-09-13T14:39:48.277090+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: cluster 2024-09-13T14:39:48.277090+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.709441+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.709441+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.710159+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.710159+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.710543+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.710543+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.710903+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.710903+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.711265+0000 mon.a (mon.0) 1075 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.711265+0000 mon.a (mon.0) 1075 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]} : dispatch 2024-09-13T14:39:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.711622+0000 mon.a (mon.0) 1076 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]} : dispatch 2024-09-13T14:39:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.711622+0000 mon.a (mon.0) 1076 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]} : dispatch 2024-09-13T14:39:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.711976+0000 mon.a (mon.0) 1077 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.711976+0000 mon.a (mon.0) 1077 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-09-13T14:39:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.712332+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]} : dispatch 2024-09-13T14:39:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.712332+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]} : dispatch 2024-09-13T14:39:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.712678+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]} : dispatch 2024-09-13T14:39:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.712678+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]} : dispatch 2024-09-13T14:39:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.799282+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:49 smithi177 bash[68594]: audit 2024-09-13T14:39:48.799282+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:50.473 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[61439]: ts=2024-09-13T14:39:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:39:50.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.286293+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]}]': finished 2024-09-13T14:39:50.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.286293+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]}]': finished 2024-09-13T14:39:50.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.286483+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]': finished 2024-09-13T14:39:50.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.286483+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]': finished 2024-09-13T14:39:50.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.286652+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]}]': finished 2024-09-13T14:39:50.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.286652+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]}]': finished 2024-09-13T14:39:50.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.286810+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.286810+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.286965+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.286965+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.287119+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.287119+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.287277+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.287277+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.287432+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.287432+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.287586+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: audit 2024-09-13T14:39:49.287586+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]}]': finished 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: cluster 2024-09-13T14:39:49.295578+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-13T14:39:50.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:50 smithi177 bash[68594]: cluster 2024-09-13T14:39:49.295578+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-13T14:39:50.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.286293+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]}]': finished 2024-09-13T14:39:50.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.286293+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.286483+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.286483+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.286652+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.286652+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.286810+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.286810+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.286965+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.286965+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.287119+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.287119+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.287277+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.287277+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-09-13T14:39:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.287432+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]}]': finished 2024-09-13T14:39:50.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.287432+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]}]': finished 2024-09-13T14:39:50.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.287586+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]}]': finished 2024-09-13T14:39:50.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: audit 2024-09-13T14:39:49.287586+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]}]': finished 2024-09-13T14:39:50.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: cluster 2024-09-13T14:39:49.295578+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-13T14:39:50.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[84576]: cluster 2024-09-13T14:39:49.295578+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-13T14:39:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.286293+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]}]': finished 2024-09-13T14:39:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.286293+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.14", "id": [6, 0]}]': finished 2024-09-13T14:39:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.286483+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]': finished 2024-09-13T14:39:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.286483+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.286652+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.286652+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [3, 5]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.286810+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.286810+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.286965+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.286965+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.7", "id": [3, 5]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.287119+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.287119+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.d", "id": [2, 1]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.287277+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.287277+0000 mon.a (mon.0) 1087 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.287432+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.287432+0000 mon.a (mon.0) 1088 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.16", "id": [7, 1]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.287586+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]}]': finished 2024-09-13T14:39:50.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: audit 2024-09-13T14:39:49.287586+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.17", "id": [3, 4]}]': finished 2024-09-13T14:39:50.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: cluster 2024-09-13T14:39:49.295578+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-13T14:39:50.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:50 smithi099 bash[87147]: cluster 2024-09-13T14:39:49.295578+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-13T14:39:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:51 smithi099 bash[84576]: cluster 2024-09-13T14:39:50.195531+0000 mgr.y (mgr.26252) 398 : cluster [DBG] pgmap v622: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 190 KiB/s rd, 11 KiB/s wr, 249 op/s 2024-09-13T14:39:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:51 smithi099 bash[84576]: cluster 2024-09-13T14:39:50.195531+0000 mgr.y (mgr.26252) 398 : cluster [DBG] pgmap v622: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 190 KiB/s rd, 11 KiB/s wr, 249 op/s 2024-09-13T14:39:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:51 smithi099 bash[84576]: cluster 2024-09-13T14:39:50.306426+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-13T14:39:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:51 smithi099 bash[84576]: cluster 2024-09-13T14:39:50.306426+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-13T14:39:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:51 smithi099 bash[87147]: cluster 2024-09-13T14:39:50.195531+0000 mgr.y (mgr.26252) 398 : cluster [DBG] pgmap v622: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 190 KiB/s rd, 11 KiB/s wr, 249 op/s 2024-09-13T14:39:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:51 smithi099 bash[87147]: cluster 2024-09-13T14:39:50.195531+0000 mgr.y (mgr.26252) 398 : cluster [DBG] pgmap v622: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 190 KiB/s rd, 11 KiB/s wr, 249 op/s 2024-09-13T14:39:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:51 smithi099 bash[87147]: cluster 2024-09-13T14:39:50.306426+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-13T14:39:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:51 smithi099 bash[87147]: cluster 2024-09-13T14:39:50.306426+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-13T14:39:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:51 smithi177 bash[68594]: cluster 2024-09-13T14:39:50.195531+0000 mgr.y (mgr.26252) 398 : cluster [DBG] pgmap v622: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 190 KiB/s rd, 11 KiB/s wr, 249 op/s 2024-09-13T14:39:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:51 smithi177 bash[68594]: cluster 2024-09-13T14:39:50.195531+0000 mgr.y (mgr.26252) 398 : cluster [DBG] pgmap v622: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 190 KiB/s rd, 11 KiB/s wr, 249 op/s 2024-09-13T14:39:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:51 smithi177 bash[68594]: cluster 2024-09-13T14:39:50.306426+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-13T14:39:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:51 smithi177 bash[68594]: cluster 2024-09-13T14:39:50.306426+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-13T14:39:51.961 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:51 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:51.962 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:39:51 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:51.962 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:51 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:51.962 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:39:51 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:51.963 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:39:51 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:51.963 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:39:51 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:51.963 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:39:51 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:51.964 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:39:51 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:51.964 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:39:51 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:52.317 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:51 smithi177 systemd[1]: Stopping Ceph osd.4 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:39:52.317 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:52 smithi177 bash[27989]: debug 2024-09-13T14:39:52.066+0000 7fc61d60c640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T14:39:52.317 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:52 smithi177 bash[27989]: debug 2024-09-13T14:39:52.066+0000 7fc61d60c640 -1 osd.4 724 *** Got signal Terminated *** 2024-09-13T14:39:52.317 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:52 smithi177 bash[27989]: debug 2024-09-13T14:39:52.066+0000 7fc61d60c640 -1 osd.4 724 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:39:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:52 smithi099 bash[84576]: cluster 2024-09-13T14:39:51.308225+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-13T14:39:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:52 smithi099 bash[84576]: cluster 2024-09-13T14:39:51.308225+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-13T14:39:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:52 smithi099 bash[84576]: cluster 2024-09-13T14:39:52.070624+0000 mon.a (mon.0) 1093 : cluster [INF] osd.4 marked itself down and dead 2024-09-13T14:39:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:52 smithi099 bash[84576]: cluster 2024-09-13T14:39:52.070624+0000 mon.a (mon.0) 1093 : cluster [INF] osd.4 marked itself down and dead 2024-09-13T14:39:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:52 smithi099 bash[87147]: cluster 2024-09-13T14:39:51.308225+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-13T14:39:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:52 smithi099 bash[87147]: cluster 2024-09-13T14:39:51.308225+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-13T14:39:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:52 smithi099 bash[87147]: cluster 2024-09-13T14:39:52.070624+0000 mon.a (mon.0) 1093 : cluster [INF] osd.4 marked itself down and dead 2024-09-13T14:39:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:52 smithi099 bash[87147]: cluster 2024-09-13T14:39:52.070624+0000 mon.a (mon.0) 1093 : cluster [INF] osd.4 marked itself down and dead 2024-09-13T14:39:52.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:52 smithi177 bash[68594]: cluster 2024-09-13T14:39:51.308225+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-13T14:39:52.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:52 smithi177 bash[68594]: cluster 2024-09-13T14:39:51.308225+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-13T14:39:52.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:52 smithi177 bash[68594]: cluster 2024-09-13T14:39:52.070624+0000 mon.a (mon.0) 1093 : cluster [INF] osd.4 marked itself down and dead 2024-09-13T14:39:52.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:52 smithi177 bash[68594]: cluster 2024-09-13T14:39:52.070624+0000 mon.a (mon.0) 1093 : cluster [INF] osd.4 marked itself down and dead 2024-09-13T14:39:52.757 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[84576]: cluster 2024-09-13T14:39:52.196661+0000 mgr.y (mgr.26252) 399 : cluster [DBG] pgmap v625: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s rd, 4.2 KiB/s wr, 90 op/s 2024-09-13T14:39:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[84576]: cluster 2024-09-13T14:39:52.196661+0000 mgr.y (mgr.26252) 399 : cluster [DBG] pgmap v625: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s rd, 4.2 KiB/s wr, 90 op/s 2024-09-13T14:39:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[84576]: cluster 2024-09-13T14:39:52.317646+0000 mon.a (mon.0) 1094 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[84576]: cluster 2024-09-13T14:39:52.317646+0000 mon.a (mon.0) 1094 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[84576]: cluster 2024-09-13T14:39:52.338606+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e725: 8 total, 7 up, 8 in 2024-09-13T14:39:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[84576]: cluster 2024-09-13T14:39:52.338606+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e725: 8 total, 7 up, 8 in 2024-09-13T14:39:53.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[87147]: cluster 2024-09-13T14:39:52.196661+0000 mgr.y (mgr.26252) 399 : cluster [DBG] pgmap v625: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s rd, 4.2 KiB/s wr, 90 op/s 2024-09-13T14:39:53.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[87147]: cluster 2024-09-13T14:39:52.196661+0000 mgr.y (mgr.26252) 399 : cluster [DBG] pgmap v625: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s rd, 4.2 KiB/s wr, 90 op/s 2024-09-13T14:39:53.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[87147]: cluster 2024-09-13T14:39:52.317646+0000 mon.a (mon.0) 1094 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:53.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[87147]: cluster 2024-09-13T14:39:52.317646+0000 mon.a (mon.0) 1094 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:53.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[87147]: cluster 2024-09-13T14:39:52.338606+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e725: 8 total, 7 up, 8 in 2024-09-13T14:39:53.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:53 smithi099 bash[87147]: cluster 2024-09-13T14:39:52.338606+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e725: 8 total, 7 up, 8 in 2024-09-13T14:39:53.608 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:53 smithi177 bash[73916]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-4 2024-09-13T14:39:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:53 smithi177 bash[68594]: cluster 2024-09-13T14:39:52.196661+0000 mgr.y (mgr.26252) 399 : cluster [DBG] pgmap v625: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s rd, 4.2 KiB/s wr, 90 op/s 2024-09-13T14:39:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:53 smithi177 bash[68594]: cluster 2024-09-13T14:39:52.196661+0000 mgr.y (mgr.26252) 399 : cluster [DBG] pgmap v625: 617 pgs: 617 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s rd, 4.2 KiB/s wr, 90 op/s 2024-09-13T14:39:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:53 smithi177 bash[68594]: cluster 2024-09-13T14:39:52.317646+0000 mon.a (mon.0) 1094 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:53 smithi177 bash[68594]: cluster 2024-09-13T14:39:52.317646+0000 mon.a (mon.0) 1094 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:53 smithi177 bash[68594]: cluster 2024-09-13T14:39:52.338606+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e725: 8 total, 7 up, 8 in 2024-09-13T14:39:53.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:53 smithi177 bash[68594]: cluster 2024-09-13T14:39:52.338606+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e725: 8 total, 7 up, 8 in 2024-09-13T14:39:54.108 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:53 smithi177 bash[73979]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-4 2024-09-13T14:39:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:54 smithi177 bash[68594]: cluster 2024-09-13T14:39:53.346416+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e726: 8 total, 7 up, 8 in 2024-09-13T14:39:54.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:54 smithi177 bash[68594]: cluster 2024-09-13T14:39:53.346416+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e726: 8 total, 7 up, 8 in 2024-09-13T14:39:54.652 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:54 smithi099 bash[84576]: cluster 2024-09-13T14:39:53.346416+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e726: 8 total, 7 up, 8 in 2024-09-13T14:39:54.652 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:54 smithi099 bash[84576]: cluster 2024-09-13T14:39:53.346416+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e726: 8 total, 7 up, 8 in 2024-09-13T14:39:54.653 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:54 smithi099 bash[87147]: cluster 2024-09-13T14:39:53.346416+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e726: 8 total, 7 up, 8 in 2024-09-13T14:39:54.653 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:54 smithi099 bash[87147]: cluster 2024-09-13T14:39:53.346416+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e726: 8 total, 7 up, 8 in 2024-09-13T14:39:55.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:55 smithi177 bash[68594]: cluster 2024-09-13T14:39:54.200480+0000 mgr.y (mgr.26252) 400 : cluster [DBG] pgmap v628: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:55.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:55 smithi177 bash[68594]: cluster 2024-09-13T14:39:54.200480+0000 mgr.y (mgr.26252) 400 : cluster [DBG] pgmap v628: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:55.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:55 smithi177 bash[68594]: cluster 2024-09-13T14:39:54.333199+0000 mon.a (mon.0) 1097 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:55.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:55 smithi177 bash[68594]: cluster 2024-09-13T14:39:54.333199+0000 mon.a (mon.0) 1097 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:55.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:55 smithi177 bash[68594]: cluster 2024-09-13T14:39:54.333246+0000 mon.a (mon.0) 1098 : cluster [WRN] Health check failed: Degraded data redundancy: 99/1653 objects degraded (5.989%), 30 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:55.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:55 smithi177 bash[68594]: cluster 2024-09-13T14:39:54.333246+0000 mon.a (mon.0) 1098 : cluster [WRN] Health check failed: Degraded data redundancy: 99/1653 objects degraded (5.989%), 30 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:55.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:55 smithi177 bash[68594]: cluster 2024-09-13T14:39:54.350014+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e727: 8 total, 7 up, 8 in 2024-09-13T14:39:55.458 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:55 smithi177 bash[68594]: cluster 2024-09-13T14:39:54.350014+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e727: 8 total, 7 up, 8 in 2024-09-13T14:39:55.794 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[87147]: cluster 2024-09-13T14:39:54.200480+0000 mgr.y (mgr.26252) 400 : cluster [DBG] pgmap v628: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:55.794 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[87147]: cluster 2024-09-13T14:39:54.200480+0000 mgr.y (mgr.26252) 400 : cluster [DBG] pgmap v628: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:55.794 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[87147]: cluster 2024-09-13T14:39:54.333199+0000 mon.a (mon.0) 1097 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:55.794 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[87147]: cluster 2024-09-13T14:39:54.333199+0000 mon.a (mon.0) 1097 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:55.794 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[87147]: cluster 2024-09-13T14:39:54.333246+0000 mon.a (mon.0) 1098 : cluster [WRN] Health check failed: Degraded data redundancy: 99/1653 objects degraded (5.989%), 30 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:55.794 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[87147]: cluster 2024-09-13T14:39:54.333246+0000 mon.a (mon.0) 1098 : cluster [WRN] Health check failed: Degraded data redundancy: 99/1653 objects degraded (5.989%), 30 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:55.795 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[87147]: cluster 2024-09-13T14:39:54.350014+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e727: 8 total, 7 up, 8 in 2024-09-13T14:39:55.795 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[87147]: cluster 2024-09-13T14:39:54.350014+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e727: 8 total, 7 up, 8 in 2024-09-13T14:39:55.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[84576]: cluster 2024-09-13T14:39:54.200480+0000 mgr.y (mgr.26252) 400 : cluster [DBG] pgmap v628: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:55.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[84576]: cluster 2024-09-13T14:39:54.200480+0000 mgr.y (mgr.26252) 400 : cluster [DBG] pgmap v628: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:55.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[84576]: cluster 2024-09-13T14:39:54.333199+0000 mon.a (mon.0) 1097 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:55.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[84576]: cluster 2024-09-13T14:39:54.333199+0000 mon.a (mon.0) 1097 : cluster [WRN] Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:39:55.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[84576]: cluster 2024-09-13T14:39:54.333246+0000 mon.a (mon.0) 1098 : cluster [WRN] Health check failed: Degraded data redundancy: 99/1653 objects degraded (5.989%), 30 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:55.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[84576]: cluster 2024-09-13T14:39:54.333246+0000 mon.a (mon.0) 1098 : cluster [WRN] Health check failed: Degraded data redundancy: 99/1653 objects degraded (5.989%), 30 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:55.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[84576]: cluster 2024-09-13T14:39:54.350014+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e727: 8 total, 7 up, 8 in 2024-09-13T14:39:55.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:55 smithi099 bash[84576]: cluster 2024-09-13T14:39:54.350014+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e727: 8 total, 7 up, 8 in 2024-09-13T14:39:56.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:56.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:56 smithi177 bash[68594]: cluster 2024-09-13T14:39:55.363494+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e728: 8 total, 7 up, 8 in 2024-09-13T14:39:56.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:56 smithi177 bash[68594]: cluster 2024-09-13T14:39:55.363494+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e728: 8 total, 7 up, 8 in 2024-09-13T14:39:56.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:56.530 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:56.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:56.531 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:56.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-13T14:39:56.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: Stopped Ceph osd.4 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:39:56.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:56.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: Started Ceph osd.4 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:39:56.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:56.532 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:56.532 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 14:39:56 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T14:39:56.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:56 smithi099 bash[84576]: cluster 2024-09-13T14:39:55.363494+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e728: 8 total, 7 up, 8 in 2024-09-13T14:39:56.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:56 smithi099 bash[84576]: cluster 2024-09-13T14:39:55.363494+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e728: 8 total, 7 up, 8 in 2024-09-13T14:39:56.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:56 smithi099 bash[87147]: cluster 2024-09-13T14:39:55.363494+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e728: 8 total, 7 up, 8 in 2024-09-13T14:39:56.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:56 smithi099 bash[87147]: cluster 2024-09-13T14:39:55.363494+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e728: 8 total, 7 up, 8 in 2024-09-13T14:39:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:39:57] "GET /metrics HTTP/1.1" 200 75161 "" "Prometheus/2.51.0" 2024-09-13T14:39:57.524 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ObjectMapConsistentSnap (12087 ms) 2024-09-13T14:39:57.524 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.Metadata 2024-09-13T14:39:57.524 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:57.727 INFO:teuthology.orchestra.run.smithi099.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:39:57.727 INFO:teuthology.orchestra.run.smithi099.stdout:alertmanager.a smithi099 *:9093,9094 running (10m) 28s ago 18m 14.7M - 0.27.0 11f11916f8cd 45a2ce75edd3 2024-09-13T14:39:57.727 INFO:teuthology.orchestra.run.smithi099.stdout:grafana.a smithi177 *:3000 running (9m) 64s ago 17m 75.4M - 9.4.7 954c08fa6188 443ce4187a0d 2024-09-13T14:39:57.727 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi099.yyrtld smithi099 running (15m) 28s ago 15m 15.2M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 7bce2f3fd8d5 2024-09-13T14:39:57.727 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi177.outtkv smithi177 running (15m) 64s ago 15m 20.2M - 17.2.7-1627-gbbdadfa3 a994b00deb3a b976db2e3d1b 2024-09-13T14:39:57.727 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.x smithi177 *:8443,9283,8765 running (8m) 64s ago 27m 442M - 19.3.0-4697-gd58f7306 fec07016a2b0 24a222989f2c 2024-09-13T14:39:57.727 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.y smithi099 *:8443,9283,8765 running (9m) 28s ago 29m 528M - 19.3.0-4697-gd58f7306 fec07016a2b0 8ebc4af8542d 2024-09-13T14:39:57.727 INFO:teuthology.orchestra.run.smithi099.stdout:mon.a smithi099 running (8m) 28s ago 29m 74.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 7bae834be590 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:mon.b smithi177 running (5m) 64s ago 27m 47.5M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 6ddaaf41777b 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:mon.c smithi099 running (6m) 28s ago 27m 52.2M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 8e92ca9c3d4b 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.a smithi099 *:9100 running (10m) 28s ago 18m 8391k - 1.7.0 72c9c2088986 adc238b4fabf 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.b smithi177 *:9100 running (9m) 64s ago 18m 8219k - 1.7.0 72c9c2088986 0e333c475514 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0 smithi099 running (3m) 28s ago 26m 99.6M 2524M 19.3.0-4697-gd58f7306 fec07016a2b0 ebecc5a7ecad 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1 smithi099 running (117s) 28s ago 25m 111M 2524M 19.3.0-4697-gd58f7306 fec07016a2b0 5ed58cf87b9c 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2 smithi099 running (67s) 28s ago 24m 91.8M 2524M 19.3.0-4697-gd58f7306 fec07016a2b0 8bf83563f2b4 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3 smithi099 running (33s) 28s ago 23m - 2524M 19.3.0-4697-gd58f7306 fec07016a2b0 b0bde9c79880 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4 smithi177 starting - - - 2524M 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5 smithi177 running (21m) 64s ago 21m 140M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a f285f4fee6d3 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6 smithi177 running (20m) 64s ago 20m 148M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 2318776c78d4 2024-09-13T14:39:57.728 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7 smithi177 running (19m) 64s ago 20m 166M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a e2d33a0b16ca 2024-09-13T14:39:57.729 INFO:teuthology.orchestra.run.smithi099.stdout:prometheus.a smithi177 *:9095 running (8m) 64s ago 18m 51.5M - 2.51.0 1d3b7f56885b d360e28d5f8b 2024-09-13T14:39:57.739 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[84576]: cluster 2024-09-13T14:39:56.202076+0000 mgr.y (mgr.26252) 401 : cluster [DBG] pgmap v631: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:57.739 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[84576]: cluster 2024-09-13T14:39:56.202076+0000 mgr.y (mgr.26252) 401 : cluster [DBG] pgmap v631: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[84576]: cluster 2024-09-13T14:39:56.377845+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e729: 8 total, 7 up, 8 in 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[84576]: cluster 2024-09-13T14:39:56.377845+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e729: 8 total, 7 up, 8 in 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[84576]: audit 2024-09-13T14:39:56.586653+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[84576]: audit 2024-09-13T14:39:56.586653+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[84576]: audit 2024-09-13T14:39:56.597000+0000 mon.a (mon.0) 1103 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[84576]: audit 2024-09-13T14:39:56.597000+0000 mon.a (mon.0) 1103 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[84576]: audit 2024-09-13T14:39:56.599227+0000 mon.a (mon.0) 1104 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[84576]: audit 2024-09-13T14:39:56.599227+0000 mon.a (mon.0) 1104 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[87147]: cluster 2024-09-13T14:39:56.202076+0000 mgr.y (mgr.26252) 401 : cluster [DBG] pgmap v631: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[87147]: cluster 2024-09-13T14:39:56.202076+0000 mgr.y (mgr.26252) 401 : cluster [DBG] pgmap v631: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:57.740 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[87147]: cluster 2024-09-13T14:39:56.377845+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e729: 8 total, 7 up, 8 in 2024-09-13T14:39:57.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[87147]: cluster 2024-09-13T14:39:56.377845+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e729: 8 total, 7 up, 8 in 2024-09-13T14:39:57.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[87147]: audit 2024-09-13T14:39:56.586653+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[87147]: audit 2024-09-13T14:39:56.586653+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[87147]: audit 2024-09-13T14:39:56.597000+0000 mon.a (mon.0) 1103 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[87147]: audit 2024-09-13T14:39:56.597000+0000 mon.a (mon.0) 1103 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[87147]: audit 2024-09-13T14:39:56.599227+0000 mon.a (mon.0) 1104 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:57.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:57 smithi099 bash[87147]: audit 2024-09-13T14:39:56.599227+0000 mon.a (mon.0) 1104 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:57.760 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[68594]: cluster 2024-09-13T14:39:56.202076+0000 mgr.y (mgr.26252) 401 : cluster [DBG] pgmap v631: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:57.760 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[68594]: cluster 2024-09-13T14:39:56.202076+0000 mgr.y (mgr.26252) 401 : cluster [DBG] pgmap v631: 617 pgs: 92 active+undersized, 25 peering, 30 stale+active+clean, 30 active+undersized+degraded, 440 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 KiB/s wr, 38 op/s; 99/1653 objects degraded (5.989%) 2024-09-13T14:39:57.760 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[68594]: cluster 2024-09-13T14:39:56.377845+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e729: 8 total, 7 up, 8 in 2024-09-13T14:39:57.760 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[68594]: cluster 2024-09-13T14:39:56.377845+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e729: 8 total, 7 up, 8 in 2024-09-13T14:39:57.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[68594]: audit 2024-09-13T14:39:56.586653+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[68594]: audit 2024-09-13T14:39:56.586653+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[68594]: audit 2024-09-13T14:39:56.597000+0000 mon.a (mon.0) 1103 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[68594]: audit 2024-09-13T14:39:56.597000+0000 mon.a (mon.0) 1103 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:39:57.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[68594]: audit 2024-09-13T14:39:56.599227+0000 mon.a (mon.0) 1104 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:57.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[68594]: audit 2024-09-13T14:39:56.599227+0000 mon.a (mon.0) 1104 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:58.026 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[74126]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:58.026 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:57 smithi177 bash[74126]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:58.026 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[74126]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:39:58.026 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[74126]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:58.358 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[74126]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:58.358 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[74126]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T14:39:58.358 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[74126]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-13T14:39:58.628 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.Metadata (1104 ms) 2024-09-13T14:39:58.628 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.MetadataPP 2024-09-13T14:39:58.629 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:58.728 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[68594]: audit 2024-09-13T14:39:57.716976+0000 mgr.y (mgr.26252) 402 : audit [DBG] from='client.44876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:58.728 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[68594]: audit 2024-09-13T14:39:57.716976+0000 mgr.y (mgr.26252) 402 : audit [DBG] from='client.44876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:58.729 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[74126]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T14:39:58.729 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[74126]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-13T14:39:58.729 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[74126]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:39:58.729 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[74126]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T14:39:58.729 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:39:58 smithi177 bash[74126]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-13T14:39:58.763 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:58 smithi099 bash[84576]: audit 2024-09-13T14:39:57.716976+0000 mgr.y (mgr.26252) 402 : audit [DBG] from='client.44876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:58.763 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:58 smithi099 bash[84576]: audit 2024-09-13T14:39:57.716976+0000 mgr.y (mgr.26252) 402 : audit [DBG] from='client.44876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:58.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:58 smithi099 bash[87147]: audit 2024-09-13T14:39:57.716976+0000 mgr.y (mgr.26252) 402 : audit [DBG] from='client.44876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:58.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:58 smithi099 bash[87147]: audit 2024-09-13T14:39:57.716976+0000 mgr.y (mgr.26252) 402 : audit [DBG] from='client.44876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:58.764 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch ps' 2024-09-13T14:39:59.711 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.MetadataPP (1083 ms) 2024-09-13T14:39:59.711 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.UpdateFeatures 2024-09-13T14:39:59.712 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:39:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[84576]: cluster 2024-09-13T14:39:58.205737+0000 mgr.y (mgr.26252) 403 : cluster [DBG] pgmap v633: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 110 KiB/s wr, 413 op/s; 268/1929 objects degraded (13.893%) 2024-09-13T14:39:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[84576]: cluster 2024-09-13T14:39:58.205737+0000 mgr.y (mgr.26252) 403 : cluster [DBG] pgmap v633: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 110 KiB/s wr, 413 op/s; 268/1929 objects degraded (13.893%) 2024-09-13T14:39:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[84576]: cluster 2024-09-13T14:39:58.370509+0000 mon.a (mon.0) 1105 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-13T14:39:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[84576]: cluster 2024-09-13T14:39:58.370509+0000 mon.a (mon.0) 1105 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-13T14:39:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[84576]: cluster 2024-09-13T14:39:58.396891+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e730: 8 total, 7 up, 8 in 2024-09-13T14:39:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[84576]: cluster 2024-09-13T14:39:58.396891+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e730: 8 total, 7 up, 8 in 2024-09-13T14:39:59.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[87147]: cluster 2024-09-13T14:39:58.205737+0000 mgr.y (mgr.26252) 403 : cluster [DBG] pgmap v633: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 110 KiB/s wr, 413 op/s; 268/1929 objects degraded (13.893%) 2024-09-13T14:39:59.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[87147]: cluster 2024-09-13T14:39:58.205737+0000 mgr.y (mgr.26252) 403 : cluster [DBG] pgmap v633: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 110 KiB/s wr, 413 op/s; 268/1929 objects degraded (13.893%) 2024-09-13T14:39:59.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[87147]: cluster 2024-09-13T14:39:58.370509+0000 mon.a (mon.0) 1105 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-13T14:39:59.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[87147]: cluster 2024-09-13T14:39:58.370509+0000 mon.a (mon.0) 1105 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-13T14:39:59.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[87147]: cluster 2024-09-13T14:39:58.396891+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e730: 8 total, 7 up, 8 in 2024-09-13T14:39:59.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:39:59 smithi099 bash[87147]: cluster 2024-09-13T14:39:58.396891+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e730: 8 total, 7 up, 8 in 2024-09-13T14:39:59.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:59 smithi177 bash[68594]: cluster 2024-09-13T14:39:58.205737+0000 mgr.y (mgr.26252) 403 : cluster [DBG] pgmap v633: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 110 KiB/s wr, 413 op/s; 268/1929 objects degraded (13.893%) 2024-09-13T14:39:59.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:59 smithi177 bash[68594]: cluster 2024-09-13T14:39:58.205737+0000 mgr.y (mgr.26252) 403 : cluster [DBG] pgmap v633: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 110 KiB/s wr, 413 op/s; 268/1929 objects degraded (13.893%) 2024-09-13T14:39:59.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:59 smithi177 bash[68594]: cluster 2024-09-13T14:39:58.370509+0000 mon.a (mon.0) 1105 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-13T14:39:59.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:59 smithi177 bash[68594]: cluster 2024-09-13T14:39:58.370509+0000 mon.a (mon.0) 1105 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-13T14:39:59.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:59 smithi177 bash[68594]: cluster 2024-09-13T14:39:58.396891+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e730: 8 total, 7 up, 8 in 2024-09-13T14:39:59.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:39:59 smithi177 bash[68594]: cluster 2024-09-13T14:39:58.396891+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e730: 8 total, 7 up, 8 in 2024-09-13T14:39:59.983 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.UpdateFeatures (272 ms) 2024-09-13T14:39:59.983 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.FeaturesBitmaskString 2024-09-13T14:39:59.983 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.FeaturesBitmaskString (0 ms) 2024-09-13T14:39:59.983 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.RebuildObjectMap 2024-09-13T14:39:59.983 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:00.558 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[61439]: ts=2024-09-13T14:40:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:00.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:39:59.406255+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e731: 8 total, 7 up, 8 in 2024-09-13T14:40:00.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:39:59.406255+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e731: 8 total, 7 up, 8 in 2024-09-13T14:40:00.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000178+0000 mon.a (mon.0) 1108 : cluster [WRN] Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000178+0000 mon.a (mon.0) 1108 : cluster [WRN] Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000238+0000 mon.a (mon.0) 1109 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000238+0000 mon.a (mon.0) 1109 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000259+0000 mon.a (mon.0) 1110 : cluster [WRN] osd.4 (root=default,host=smithi177) is down 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000259+0000 mon.a (mon.0) 1110 : cluster [WRN] osd.4 (root=default,host=smithi177) is down 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000284+0000 mon.a (mon.0) 1111 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000284+0000 mon.a (mon.0) 1111 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000326+0000 mon.a (mon.0) 1112 : cluster [WRN] pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000326+0000 mon.a (mon.0) 1112 : cluster [WRN] pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000347+0000 mon.a (mon.0) 1113 : cluster [WRN] pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000347+0000 mon.a (mon.0) 1113 : cluster [WRN] pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000363+0000 mon.a (mon.0) 1114 : cluster [WRN] pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000363+0000 mon.a (mon.0) 1114 : cluster [WRN] pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000376+0000 mon.a (mon.0) 1115 : cluster [WRN] pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000376+0000 mon.a (mon.0) 1115 : cluster [WRN] pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000391+0000 mon.a (mon.0) 1116 : cluster [WRN] pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000391+0000 mon.a (mon.0) 1116 : cluster [WRN] pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000407+0000 mon.a (mon.0) 1117 : cluster [WRN] pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000407+0000 mon.a (mon.0) 1117 : cluster [WRN] pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000426+0000 mon.a (mon.0) 1118 : cluster [WRN] pg 18.4 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000426+0000 mon.a (mon.0) 1118 : cluster [WRN] pg 18.4 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000443+0000 mon.a (mon.0) 1119 : cluster [WRN] pg 18.6 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000443+0000 mon.a (mon.0) 1119 : cluster [WRN] pg 18.6 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000462+0000 mon.a (mon.0) 1120 : cluster [WRN] pg 18.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000462+0000 mon.a (mon.0) 1120 : cluster [WRN] pg 18.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000478+0000 mon.a (mon.0) 1121 : cluster [WRN] pg 18.a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000478+0000 mon.a (mon.0) 1121 : cluster [WRN] pg 18.a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000493+0000 mon.a (mon.0) 1122 : cluster [WRN] pg 18.b is active+undersized+degraded, acting [5,2] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000493+0000 mon.a (mon.0) 1122 : cluster [WRN] pg 18.b is active+undersized+degraded, acting [5,2] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000508+0000 mon.a (mon.0) 1123 : cluster [WRN] pg 18.d is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000508+0000 mon.a (mon.0) 1123 : cluster [WRN] pg 18.d is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000523+0000 mon.a (mon.0) 1124 : cluster [WRN] pg 18.12 is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000523+0000 mon.a (mon.0) 1124 : cluster [WRN] pg 18.12 is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000538+0000 mon.a (mon.0) 1125 : cluster [WRN] pg 18.19 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000538+0000 mon.a (mon.0) 1125 : cluster [WRN] pg 18.19 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000568+0000 mon.a (mon.0) 1126 : cluster [WRN] pg 18.1b is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000568+0000 mon.a (mon.0) 1126 : cluster [WRN] pg 18.1b is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000586+0000 mon.a (mon.0) 1127 : cluster [WRN] pg 19.5 is active+undersized+degraded, acting [2,1] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000586+0000 mon.a (mon.0) 1127 : cluster [WRN] pg 19.5 is active+undersized+degraded, acting [2,1] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000604+0000 mon.a (mon.0) 1128 : cluster [WRN] pg 19.1a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000604+0000 mon.a (mon.0) 1128 : cluster [WRN] pg 19.1a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000619+0000 mon.a (mon.0) 1129 : cluster [WRN] pg 19.1b is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000619+0000 mon.a (mon.0) 1129 : cluster [WRN] pg 19.1b is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000635+0000 mon.a (mon.0) 1130 : cluster [WRN] pg 19.1c is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000635+0000 mon.a (mon.0) 1130 : cluster [WRN] pg 19.1c is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000651+0000 mon.a (mon.0) 1131 : cluster [WRN] pg 20.3 is active+undersized+degraded, acting [0,7] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000651+0000 mon.a (mon.0) 1131 : cluster [WRN] pg 20.3 is active+undersized+degraded, acting [0,7] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000669+0000 mon.a (mon.0) 1132 : cluster [WRN] pg 20.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000669+0000 mon.a (mon.0) 1132 : cluster [WRN] pg 20.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000684+0000 mon.a (mon.0) 1133 : cluster [WRN] pg 20.1b is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000684+0000 mon.a (mon.0) 1133 : cluster [WRN] pg 20.1b is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000700+0000 mon.a (mon.0) 1134 : cluster [WRN] pg 20.1c is active+undersized+degraded, acting [1,5] 2024-09-13T14:40:00.562 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000700+0000 mon.a (mon.0) 1134 : cluster [WRN] pg 20.1c is active+undersized+degraded, acting [1,5] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000715+0000 mon.a (mon.0) 1135 : cluster [WRN] pg 21.2 is active+undersized+degraded, acting [2,5] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000715+0000 mon.a (mon.0) 1135 : cluster [WRN] pg 21.2 is active+undersized+degraded, acting [2,5] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000728+0000 mon.a (mon.0) 1136 : cluster [WRN] pg 21.a is active+undersized+degraded, acting [1,0] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000728+0000 mon.a (mon.0) 1136 : cluster [WRN] pg 21.a is active+undersized+degraded, acting [1,0] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000742+0000 mon.a (mon.0) 1137 : cluster [WRN] pg 21.c is active+undersized+degraded, acting [6,5] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000742+0000 mon.a (mon.0) 1137 : cluster [WRN] pg 21.c is active+undersized+degraded, acting [6,5] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000759+0000 mon.a (mon.0) 1138 : cluster [WRN] pg 21.12 is active+undersized+degraded, acting [6,1] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000759+0000 mon.a (mon.0) 1138 : cluster [WRN] pg 21.12 is active+undersized+degraded, acting [6,1] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000775+0000 mon.a (mon.0) 1139 : cluster [WRN] pg 21.15 is active+undersized+degraded, acting [2,0] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000775+0000 mon.a (mon.0) 1139 : cluster [WRN] pg 21.15 is active+undersized+degraded, acting [2,0] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000791+0000 mon.a (mon.0) 1140 : cluster [WRN] pg 21.1c is active+undersized+degraded, acting [1,2] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000791+0000 mon.a (mon.0) 1140 : cluster [WRN] pg 21.1c is active+undersized+degraded, acting [1,2] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000816+0000 mon.a (mon.0) 1141 : cluster [WRN] pg 22.f is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000816+0000 mon.a (mon.0) 1141 : cluster [WRN] pg 22.f is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.564 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000831+0000 mon.a (mon.0) 1142 : cluster [WRN] pg 22.10 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.564 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000831+0000 mon.a (mon.0) 1142 : cluster [WRN] pg 22.10 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.564 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000849+0000 mon.a (mon.0) 1143 : cluster [WRN] pg 22.14 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000849+0000 mon.a (mon.0) 1143 : cluster [WRN] pg 22.14 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000864+0000 mon.a (mon.0) 1144 : cluster [WRN] pg 22.15 is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000864+0000 mon.a (mon.0) 1144 : cluster [WRN] pg 22.15 is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000880+0000 mon.a (mon.0) 1145 : cluster [WRN] pg 22.1b is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000880+0000 mon.a (mon.0) 1145 : cluster [WRN] pg 22.1b is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000896+0000 mon.a (mon.0) 1146 : cluster [WRN] pg 23.3 is active+undersized+degraded, acting [0,5] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000896+0000 mon.a (mon.0) 1146 : cluster [WRN] pg 23.3 is active+undersized+degraded, acting [0,5] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000909+0000 mon.a (mon.0) 1147 : cluster [WRN] pg 23.8 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000909+0000 mon.a (mon.0) 1147 : cluster [WRN] pg 23.8 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000924+0000 mon.a (mon.0) 1148 : cluster [WRN] pg 23.18 is active+undersized+degraded, acting [1,7] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000924+0000 mon.a (mon.0) 1148 : cluster [WRN] pg 23.18 is active+undersized+degraded, acting [1,7] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000939+0000 mon.a (mon.0) 1149 : cluster [WRN] pg 23.1c is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000939+0000 mon.a (mon.0) 1149 : cluster [WRN] pg 23.1c is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000954+0000 mon.a (mon.0) 1150 : cluster [WRN] pg 23.1f is active+undersized+degraded, acting [3,2] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000954+0000 mon.a (mon.0) 1150 : cluster [WRN] pg 23.1f is active+undersized+degraded, acting [3,2] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000969+0000 mon.a (mon.0) 1151 : cluster [WRN] pg 24.4 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000969+0000 mon.a (mon.0) 1151 : cluster [WRN] pg 24.4 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000986+0000 mon.a (mon.0) 1152 : cluster [WRN] pg 24.e is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.000986+0000 mon.a (mon.0) 1152 : cluster [WRN] pg 24.e is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001003+0000 mon.a (mon.0) 1153 : cluster [WRN] pg 24.16 is active+undersized+degraded, acting [2,6] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001003+0000 mon.a (mon.0) 1153 : cluster [WRN] pg 24.16 is active+undersized+degraded, acting [2,6] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001018+0000 mon.a (mon.0) 1154 : cluster [WRN] pg 24.17 is active+undersized+degraded, acting [5,3] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001018+0000 mon.a (mon.0) 1154 : cluster [WRN] pg 24.17 is active+undersized+degraded, acting [5,3] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001038+0000 mon.a (mon.0) 1155 : cluster [WRN] pg 25.3 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001038+0000 mon.a (mon.0) 1155 : cluster [WRN] pg 25.3 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001051+0000 mon.a (mon.0) 1156 : cluster [WRN] pg 25.1c is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.566 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001051+0000 mon.a (mon.0) 1156 : cluster [WRN] pg 25.1c is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001070+0000 mon.a (mon.0) 1157 : cluster [WRN] pg 26.1a is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001070+0000 mon.a (mon.0) 1157 : cluster [WRN] pg 26.1a is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001087+0000 mon.a (mon.0) 1158 : cluster [WRN] pg 27.1c is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001087+0000 mon.a (mon.0) 1158 : cluster [WRN] pg 27.1c is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001103+0000 mon.a (mon.0) 1159 : cluster [WRN] pg 30.3 is active+undersized+degraded, acting [6,2] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001103+0000 mon.a (mon.0) 1159 : cluster [WRN] pg 30.3 is active+undersized+degraded, acting [6,2] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001119+0000 mon.a (mon.0) 1160 : cluster [WRN] pg 30.a is active+undersized+degraded, acting [0,6] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001119+0000 mon.a (mon.0) 1160 : cluster [WRN] pg 30.a is active+undersized+degraded, acting [0,6] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001134+0000 mon.a (mon.0) 1161 : cluster [WRN] pg 31.3 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001134+0000 mon.a (mon.0) 1161 : cluster [WRN] pg 31.3 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001150+0000 mon.a (mon.0) 1162 : cluster [WRN] pg 31.a is active+undersized+degraded, acting [3,1] 2024-09-13T14:40:00.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.001150+0000 mon.a (mon.0) 1162 : cluster [WRN] pg 31.a is active+undersized+degraded, acting [3,1] 2024-09-13T14:40:00.596 INFO:tasks.workunit.client.0.smithi099.stdout:0.00% done 2024-09-13T14:40:00.596 INFO:tasks.workunit.client.0.smithi099.stdout:25.00% done 2024-09-13T14:40:00.596 INFO:tasks.workunit.client.0.smithi099.stdout:50.00% done 2024-09-13T14:40:00.596 INFO:tasks.workunit.client.0.smithi099.stdout:75.00% done 2024-09-13T14:40:00.596 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.RebuildObjectMap (613 ms) 2024-09-13T14:40:00.596 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.RebuildNewObjectMap 2024-09-13T14:40:00.676 INFO:tasks.workunit.client.0.smithi099.stdout:0.00% done 2024-09-13T14:40:00.676 INFO:tasks.workunit.client.0.smithi099.stdout:25.00% done 2024-09-13T14:40:00.677 INFO:tasks.workunit.client.0.smithi099.stdout:50.00% done 2024-09-13T14:40:00.677 INFO:tasks.workunit.client.0.smithi099.stdout:75.00% done 2024-09-13T14:40:00.677 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.RebuildNewObjectMap (80 ms) 2024-09-13T14:40:00.677 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.CheckObjectMap 2024-09-13T14:40:00.677 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:00.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:39:59.406255+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e731: 8 total, 7 up, 8 in 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:39:59.406255+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e731: 8 total, 7 up, 8 in 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000178+0000 mon.a (mon.0) 1108 : cluster [WRN] Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000178+0000 mon.a (mon.0) 1108 : cluster [WRN] Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000238+0000 mon.a (mon.0) 1109 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000238+0000 mon.a (mon.0) 1109 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000259+0000 mon.a (mon.0) 1110 : cluster [WRN] osd.4 (root=default,host=smithi177) is down 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000259+0000 mon.a (mon.0) 1110 : cluster [WRN] osd.4 (root=default,host=smithi177) is down 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000284+0000 mon.a (mon.0) 1111 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000284+0000 mon.a (mon.0) 1111 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000326+0000 mon.a (mon.0) 1112 : cluster [WRN] pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000326+0000 mon.a (mon.0) 1112 : cluster [WRN] pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000347+0000 mon.a (mon.0) 1113 : cluster [WRN] pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-13T14:40:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000347+0000 mon.a (mon.0) 1113 : cluster [WRN] pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-13T14:40:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000363+0000 mon.a (mon.0) 1114 : cluster [WRN] pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000363+0000 mon.a (mon.0) 1114 : cluster [WRN] pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000376+0000 mon.a (mon.0) 1115 : cluster [WRN] pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000376+0000 mon.a (mon.0) 1115 : cluster [WRN] pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000391+0000 mon.a (mon.0) 1116 : cluster [WRN] pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T14:40:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000391+0000 mon.a (mon.0) 1116 : cluster [WRN] pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T14:40:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000407+0000 mon.a (mon.0) 1117 : cluster [WRN] pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000407+0000 mon.a (mon.0) 1117 : cluster [WRN] pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000426+0000 mon.a (mon.0) 1118 : cluster [WRN] pg 18.4 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:39:59.406255+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e731: 8 total, 7 up, 8 in 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:39:59.406255+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e731: 8 total, 7 up, 8 in 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000178+0000 mon.a (mon.0) 1108 : cluster [WRN] Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000178+0000 mon.a (mon.0) 1108 : cluster [WRN] Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000238+0000 mon.a (mon.0) 1109 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000238+0000 mon.a (mon.0) 1109 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000259+0000 mon.a (mon.0) 1110 : cluster [WRN] osd.4 (root=default,host=smithi177) is down 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000259+0000 mon.a (mon.0) 1110 : cluster [WRN] osd.4 (root=default,host=smithi177) is down 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000284+0000 mon.a (mon.0) 1111 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000284+0000 mon.a (mon.0) 1111 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 268/1929 objects degraded (13.893%), 60 pgs degraded 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000326+0000 mon.a (mon.0) 1112 : cluster [WRN] pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000326+0000 mon.a (mon.0) 1112 : cluster [WRN] pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000347+0000 mon.a (mon.0) 1113 : cluster [WRN] pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000347+0000 mon.a (mon.0) 1113 : cluster [WRN] pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000363+0000 mon.a (mon.0) 1114 : cluster [WRN] pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000363+0000 mon.a (mon.0) 1114 : cluster [WRN] pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000376+0000 mon.a (mon.0) 1115 : cluster [WRN] pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000376+0000 mon.a (mon.0) 1115 : cluster [WRN] pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000391+0000 mon.a (mon.0) 1116 : cluster [WRN] pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000391+0000 mon.a (mon.0) 1116 : cluster [WRN] pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000407+0000 mon.a (mon.0) 1117 : cluster [WRN] pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000407+0000 mon.a (mon.0) 1117 : cluster [WRN] pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000426+0000 mon.a (mon.0) 1118 : cluster [WRN] pg 18.4 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000426+0000 mon.a (mon.0) 1118 : cluster [WRN] pg 18.4 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000443+0000 mon.a (mon.0) 1119 : cluster [WRN] pg 18.6 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000443+0000 mon.a (mon.0) 1119 : cluster [WRN] pg 18.6 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000462+0000 mon.a (mon.0) 1120 : cluster [WRN] pg 18.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000462+0000 mon.a (mon.0) 1120 : cluster [WRN] pg 18.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000478+0000 mon.a (mon.0) 1121 : cluster [WRN] pg 18.a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000478+0000 mon.a (mon.0) 1121 : cluster [WRN] pg 18.a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000493+0000 mon.a (mon.0) 1122 : cluster [WRN] pg 18.b is active+undersized+degraded, acting [5,2] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000493+0000 mon.a (mon.0) 1122 : cluster [WRN] pg 18.b is active+undersized+degraded, acting [5,2] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000508+0000 mon.a (mon.0) 1123 : cluster [WRN] pg 18.d is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000508+0000 mon.a (mon.0) 1123 : cluster [WRN] pg 18.d is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000523+0000 mon.a (mon.0) 1124 : cluster [WRN] pg 18.12 is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000523+0000 mon.a (mon.0) 1124 : cluster [WRN] pg 18.12 is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000538+0000 mon.a (mon.0) 1125 : cluster [WRN] pg 18.19 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000538+0000 mon.a (mon.0) 1125 : cluster [WRN] pg 18.19 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000568+0000 mon.a (mon.0) 1126 : cluster [WRN] pg 18.1b is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000568+0000 mon.a (mon.0) 1126 : cluster [WRN] pg 18.1b is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000586+0000 mon.a (mon.0) 1127 : cluster [WRN] pg 19.5 is active+undersized+degraded, acting [2,1] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000586+0000 mon.a (mon.0) 1127 : cluster [WRN] pg 19.5 is active+undersized+degraded, acting [2,1] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000604+0000 mon.a (mon.0) 1128 : cluster [WRN] pg 19.1a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000604+0000 mon.a (mon.0) 1128 : cluster [WRN] pg 19.1a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000619+0000 mon.a (mon.0) 1129 : cluster [WRN] pg 19.1b is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000619+0000 mon.a (mon.0) 1129 : cluster [WRN] pg 19.1b is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000635+0000 mon.a (mon.0) 1130 : cluster [WRN] pg 19.1c is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000426+0000 mon.a (mon.0) 1118 : cluster [WRN] pg 18.4 is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000443+0000 mon.a (mon.0) 1119 : cluster [WRN] pg 18.6 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000443+0000 mon.a (mon.0) 1119 : cluster [WRN] pg 18.6 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000462+0000 mon.a (mon.0) 1120 : cluster [WRN] pg 18.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000462+0000 mon.a (mon.0) 1120 : cluster [WRN] pg 18.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000478+0000 mon.a (mon.0) 1121 : cluster [WRN] pg 18.a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000478+0000 mon.a (mon.0) 1121 : cluster [WRN] pg 18.a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000493+0000 mon.a (mon.0) 1122 : cluster [WRN] pg 18.b is active+undersized+degraded, acting [5,2] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000493+0000 mon.a (mon.0) 1122 : cluster [WRN] pg 18.b is active+undersized+degraded, acting [5,2] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000508+0000 mon.a (mon.0) 1123 : cluster [WRN] pg 18.d is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000508+0000 mon.a (mon.0) 1123 : cluster [WRN] pg 18.d is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000523+0000 mon.a (mon.0) 1124 : cluster [WRN] pg 18.12 is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000523+0000 mon.a (mon.0) 1124 : cluster [WRN] pg 18.12 is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000538+0000 mon.a (mon.0) 1125 : cluster [WRN] pg 18.19 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000538+0000 mon.a (mon.0) 1125 : cluster [WRN] pg 18.19 is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000568+0000 mon.a (mon.0) 1126 : cluster [WRN] pg 18.1b is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000568+0000 mon.a (mon.0) 1126 : cluster [WRN] pg 18.1b is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000586+0000 mon.a (mon.0) 1127 : cluster [WRN] pg 19.5 is active+undersized+degraded, acting [2,1] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000586+0000 mon.a (mon.0) 1127 : cluster [WRN] pg 19.5 is active+undersized+degraded, acting [2,1] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000604+0000 mon.a (mon.0) 1128 : cluster [WRN] pg 19.1a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000604+0000 mon.a (mon.0) 1128 : cluster [WRN] pg 19.1a is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000619+0000 mon.a (mon.0) 1129 : cluster [WRN] pg 19.1b is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000619+0000 mon.a (mon.0) 1129 : cluster [WRN] pg 19.1b is active+undersized+degraded, acting [0,2] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000635+0000 mon.a (mon.0) 1130 : cluster [WRN] pg 19.1c is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.830 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000635+0000 mon.a (mon.0) 1130 : cluster [WRN] pg 19.1c is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000651+0000 mon.a (mon.0) 1131 : cluster [WRN] pg 20.3 is active+undersized+degraded, acting [0,7] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000651+0000 mon.a (mon.0) 1131 : cluster [WRN] pg 20.3 is active+undersized+degraded, acting [0,7] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000669+0000 mon.a (mon.0) 1132 : cluster [WRN] pg 20.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000669+0000 mon.a (mon.0) 1132 : cluster [WRN] pg 20.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000684+0000 mon.a (mon.0) 1133 : cluster [WRN] pg 20.1b is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000684+0000 mon.a (mon.0) 1133 : cluster [WRN] pg 20.1b is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000700+0000 mon.a (mon.0) 1134 : cluster [WRN] pg 20.1c is active+undersized+degraded, acting [1,5] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000700+0000 mon.a (mon.0) 1134 : cluster [WRN] pg 20.1c is active+undersized+degraded, acting [1,5] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000715+0000 mon.a (mon.0) 1135 : cluster [WRN] pg 21.2 is active+undersized+degraded, acting [2,5] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000715+0000 mon.a (mon.0) 1135 : cluster [WRN] pg 21.2 is active+undersized+degraded, acting [2,5] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000728+0000 mon.a (mon.0) 1136 : cluster [WRN] pg 21.a is active+undersized+degraded, acting [1,0] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000728+0000 mon.a (mon.0) 1136 : cluster [WRN] pg 21.a is active+undersized+degraded, acting [1,0] 2024-09-13T14:40:00.831 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000742+0000 mon.a (mon.0) 1137 : cluster [WRN] pg 21.c is active+undersized+degraded, acting [6,5] 2024-09-13T14:40:00.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000742+0000 mon.a (mon.0) 1137 : cluster [WRN] pg 21.c is active+undersized+degraded, acting [6,5] 2024-09-13T14:40:00.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000759+0000 mon.a (mon.0) 1138 : cluster [WRN] pg 21.12 is active+undersized+degraded, acting [6,1] 2024-09-13T14:40:00.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000759+0000 mon.a (mon.0) 1138 : cluster [WRN] pg 21.12 is active+undersized+degraded, acting [6,1] 2024-09-13T14:40:00.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000775+0000 mon.a (mon.0) 1139 : cluster [WRN] pg 21.15 is active+undersized+degraded, acting [2,0] 2024-09-13T14:40:00.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000775+0000 mon.a (mon.0) 1139 : cluster [WRN] pg 21.15 is active+undersized+degraded, acting [2,0] 2024-09-13T14:40:00.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000791+0000 mon.a (mon.0) 1140 : cluster [WRN] pg 21.1c is active+undersized+degraded, acting [1,2] 2024-09-13T14:40:00.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000791+0000 mon.a (mon.0) 1140 : cluster [WRN] pg 21.1c is active+undersized+degraded, acting [1,2] 2024-09-13T14:40:00.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000816+0000 mon.a (mon.0) 1141 : cluster [WRN] pg 22.f is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000816+0000 mon.a (mon.0) 1141 : cluster [WRN] pg 22.f is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.832 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000831+0000 mon.a (mon.0) 1142 : cluster [WRN] pg 22.10 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000635+0000 mon.a (mon.0) 1130 : cluster [WRN] pg 19.1c is active+undersized+degraded, acting [7,3] 2024-09-13T14:40:00.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000651+0000 mon.a (mon.0) 1131 : cluster [WRN] pg 20.3 is active+undersized+degraded, acting [0,7] 2024-09-13T14:40:00.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000651+0000 mon.a (mon.0) 1131 : cluster [WRN] pg 20.3 is active+undersized+degraded, acting [0,7] 2024-09-13T14:40:00.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000669+0000 mon.a (mon.0) 1132 : cluster [WRN] pg 20.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000669+0000 mon.a (mon.0) 1132 : cluster [WRN] pg 20.9 is active+undersized+degraded, acting [3,5] 2024-09-13T14:40:00.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000684+0000 mon.a (mon.0) 1133 : cluster [WRN] pg 20.1b is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000684+0000 mon.a (mon.0) 1133 : cluster [WRN] pg 20.1b is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000700+0000 mon.a (mon.0) 1134 : cluster [WRN] pg 20.1c is active+undersized+degraded, acting [1,5] 2024-09-13T14:40:00.833 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000700+0000 mon.a (mon.0) 1134 : cluster [WRN] pg 20.1c is active+undersized+degraded, acting [1,5] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000715+0000 mon.a (mon.0) 1135 : cluster [WRN] pg 21.2 is active+undersized+degraded, acting [2,5] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000715+0000 mon.a (mon.0) 1135 : cluster [WRN] pg 21.2 is active+undersized+degraded, acting [2,5] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000728+0000 mon.a (mon.0) 1136 : cluster [WRN] pg 21.a is active+undersized+degraded, acting [1,0] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000728+0000 mon.a (mon.0) 1136 : cluster [WRN] pg 21.a is active+undersized+degraded, acting [1,0] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000742+0000 mon.a (mon.0) 1137 : cluster [WRN] pg 21.c is active+undersized+degraded, acting [6,5] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000742+0000 mon.a (mon.0) 1137 : cluster [WRN] pg 21.c is active+undersized+degraded, acting [6,5] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000759+0000 mon.a (mon.0) 1138 : cluster [WRN] pg 21.12 is active+undersized+degraded, acting [6,1] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000759+0000 mon.a (mon.0) 1138 : cluster [WRN] pg 21.12 is active+undersized+degraded, acting [6,1] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000775+0000 mon.a (mon.0) 1139 : cluster [WRN] pg 21.15 is active+undersized+degraded, acting [2,0] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000775+0000 mon.a (mon.0) 1139 : cluster [WRN] pg 21.15 is active+undersized+degraded, acting [2,0] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000791+0000 mon.a (mon.0) 1140 : cluster [WRN] pg 21.1c is active+undersized+degraded, acting [1,2] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000791+0000 mon.a (mon.0) 1140 : cluster [WRN] pg 21.1c is active+undersized+degraded, acting [1,2] 2024-09-13T14:40:00.834 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000816+0000 mon.a (mon.0) 1141 : cluster [WRN] pg 22.f is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000816+0000 mon.a (mon.0) 1141 : cluster [WRN] pg 22.f is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.835 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000831+0000 mon.a (mon.0) 1142 : cluster [WRN] pg 22.10 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.835 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000831+0000 mon.a (mon.0) 1142 : cluster [WRN] pg 22.10 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.835 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000849+0000 mon.a (mon.0) 1143 : cluster [WRN] pg 22.14 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.835 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000849+0000 mon.a (mon.0) 1143 : cluster [WRN] pg 22.14 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.835 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000864+0000 mon.a (mon.0) 1144 : cluster [WRN] pg 22.15 is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.835 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000864+0000 mon.a (mon.0) 1144 : cluster [WRN] pg 22.15 is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.835 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000880+0000 mon.a (mon.0) 1145 : cluster [WRN] pg 22.1b is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000880+0000 mon.a (mon.0) 1145 : cluster [WRN] pg 22.1b is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000896+0000 mon.a (mon.0) 1146 : cluster [WRN] pg 23.3 is active+undersized+degraded, acting [0,5] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000896+0000 mon.a (mon.0) 1146 : cluster [WRN] pg 23.3 is active+undersized+degraded, acting [0,5] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000909+0000 mon.a (mon.0) 1147 : cluster [WRN] pg 23.8 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000909+0000 mon.a (mon.0) 1147 : cluster [WRN] pg 23.8 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000924+0000 mon.a (mon.0) 1148 : cluster [WRN] pg 23.18 is active+undersized+degraded, acting [1,7] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000924+0000 mon.a (mon.0) 1148 : cluster [WRN] pg 23.18 is active+undersized+degraded, acting [1,7] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000939+0000 mon.a (mon.0) 1149 : cluster [WRN] pg 23.1c is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000939+0000 mon.a (mon.0) 1149 : cluster [WRN] pg 23.1c is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000954+0000 mon.a (mon.0) 1150 : cluster [WRN] pg 23.1f is active+undersized+degraded, acting [3,2] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000954+0000 mon.a (mon.0) 1150 : cluster [WRN] pg 23.1f is active+undersized+degraded, acting [3,2] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000969+0000 mon.a (mon.0) 1151 : cluster [WRN] pg 24.4 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.836 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000969+0000 mon.a (mon.0) 1151 : cluster [WRN] pg 24.4 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000986+0000 mon.a (mon.0) 1152 : cluster [WRN] pg 24.e is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.000986+0000 mon.a (mon.0) 1152 : cluster [WRN] pg 24.e is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001003+0000 mon.a (mon.0) 1153 : cluster [WRN] pg 24.16 is active+undersized+degraded, acting [2,6] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001003+0000 mon.a (mon.0) 1153 : cluster [WRN] pg 24.16 is active+undersized+degraded, acting [2,6] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001018+0000 mon.a (mon.0) 1154 : cluster [WRN] pg 24.17 is active+undersized+degraded, acting [5,3] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001018+0000 mon.a (mon.0) 1154 : cluster [WRN] pg 24.17 is active+undersized+degraded, acting [5,3] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001038+0000 mon.a (mon.0) 1155 : cluster [WRN] pg 25.3 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001038+0000 mon.a (mon.0) 1155 : cluster [WRN] pg 25.3 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001051+0000 mon.a (mon.0) 1156 : cluster [WRN] pg 25.1c is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001051+0000 mon.a (mon.0) 1156 : cluster [WRN] pg 25.1c is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001070+0000 mon.a (mon.0) 1157 : cluster [WRN] pg 26.1a is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001070+0000 mon.a (mon.0) 1157 : cluster [WRN] pg 26.1a is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001087+0000 mon.a (mon.0) 1158 : cluster [WRN] pg 27.1c is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001087+0000 mon.a (mon.0) 1158 : cluster [WRN] pg 27.1c is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001103+0000 mon.a (mon.0) 1159 : cluster [WRN] pg 30.3 is active+undersized+degraded, acting [6,2] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001103+0000 mon.a (mon.0) 1159 : cluster [WRN] pg 30.3 is active+undersized+degraded, acting [6,2] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001119+0000 mon.a (mon.0) 1160 : cluster [WRN] pg 30.a is active+undersized+degraded, acting [0,6] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001119+0000 mon.a (mon.0) 1160 : cluster [WRN] pg 30.a is active+undersized+degraded, acting [0,6] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001134+0000 mon.a (mon.0) 1161 : cluster [WRN] pg 31.3 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001134+0000 mon.a (mon.0) 1161 : cluster [WRN] pg 31.3 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001150+0000 mon.a (mon.0) 1162 : cluster [WRN] pg 31.a is active+undersized+degraded, acting [3,1] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.001150+0000 mon.a (mon.0) 1162 : cluster [WRN] pg 31.a is active+undersized+degraded, acting [3,1] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000831+0000 mon.a (mon.0) 1142 : cluster [WRN] pg 22.10 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000849+0000 mon.a (mon.0) 1143 : cluster [WRN] pg 22.14 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000849+0000 mon.a (mon.0) 1143 : cluster [WRN] pg 22.14 is active+undersized+degraded, acting [7,2] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000864+0000 mon.a (mon.0) 1144 : cluster [WRN] pg 22.15 is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000864+0000 mon.a (mon.0) 1144 : cluster [WRN] pg 22.15 is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000880+0000 mon.a (mon.0) 1145 : cluster [WRN] pg 22.1b is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000880+0000 mon.a (mon.0) 1145 : cluster [WRN] pg 22.1b is active+undersized+degraded, acting [5,6] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000896+0000 mon.a (mon.0) 1146 : cluster [WRN] pg 23.3 is active+undersized+degraded, acting [0,5] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000896+0000 mon.a (mon.0) 1146 : cluster [WRN] pg 23.3 is active+undersized+degraded, acting [0,5] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000909+0000 mon.a (mon.0) 1147 : cluster [WRN] pg 23.8 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000909+0000 mon.a (mon.0) 1147 : cluster [WRN] pg 23.8 is active+undersized+degraded, acting [7,6] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000924+0000 mon.a (mon.0) 1148 : cluster [WRN] pg 23.18 is active+undersized+degraded, acting [1,7] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000924+0000 mon.a (mon.0) 1148 : cluster [WRN] pg 23.18 is active+undersized+degraded, acting [1,7] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000939+0000 mon.a (mon.0) 1149 : cluster [WRN] pg 23.1c is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000939+0000 mon.a (mon.0) 1149 : cluster [WRN] pg 23.1c is active+undersized+degraded, acting [2,7] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000954+0000 mon.a (mon.0) 1150 : cluster [WRN] pg 23.1f is active+undersized+degraded, acting [3,2] 2024-09-13T14:40:00.839 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000954+0000 mon.a (mon.0) 1150 : cluster [WRN] pg 23.1f is active+undersized+degraded, acting [3,2] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000969+0000 mon.a (mon.0) 1151 : cluster [WRN] pg 24.4 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000969+0000 mon.a (mon.0) 1151 : cluster [WRN] pg 24.4 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000986+0000 mon.a (mon.0) 1152 : cluster [WRN] pg 24.e is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.000986+0000 mon.a (mon.0) 1152 : cluster [WRN] pg 24.e is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001003+0000 mon.a (mon.0) 1153 : cluster [WRN] pg 24.16 is active+undersized+degraded, acting [2,6] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001003+0000 mon.a (mon.0) 1153 : cluster [WRN] pg 24.16 is active+undersized+degraded, acting [2,6] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001018+0000 mon.a (mon.0) 1154 : cluster [WRN] pg 24.17 is active+undersized+degraded, acting [5,3] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001018+0000 mon.a (mon.0) 1154 : cluster [WRN] pg 24.17 is active+undersized+degraded, acting [5,3] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001038+0000 mon.a (mon.0) 1155 : cluster [WRN] pg 25.3 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001038+0000 mon.a (mon.0) 1155 : cluster [WRN] pg 25.3 is active+undersized+degraded, acting [7,1] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001051+0000 mon.a (mon.0) 1156 : cluster [WRN] pg 25.1c is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001051+0000 mon.a (mon.0) 1156 : cluster [WRN] pg 25.1c is active+undersized+degraded, acting [2,3] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001070+0000 mon.a (mon.0) 1157 : cluster [WRN] pg 26.1a is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001070+0000 mon.a (mon.0) 1157 : cluster [WRN] pg 26.1a is active+undersized+degraded, acting [1,3] 2024-09-13T14:40:00.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001087+0000 mon.a (mon.0) 1158 : cluster [WRN] pg 27.1c is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001087+0000 mon.a (mon.0) 1158 : cluster [WRN] pg 27.1c is active+undersized+degraded, acting [0,1] 2024-09-13T14:40:00.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001103+0000 mon.a (mon.0) 1159 : cluster [WRN] pg 30.3 is active+undersized+degraded, acting [6,2] 2024-09-13T14:40:00.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001103+0000 mon.a (mon.0) 1159 : cluster [WRN] pg 30.3 is active+undersized+degraded, acting [6,2] 2024-09-13T14:40:00.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001119+0000 mon.a (mon.0) 1160 : cluster [WRN] pg 30.a is active+undersized+degraded, acting [0,6] 2024-09-13T14:40:00.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001119+0000 mon.a (mon.0) 1160 : cluster [WRN] pg 30.a is active+undersized+degraded, acting [0,6] 2024-09-13T14:40:00.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001134+0000 mon.a (mon.0) 1161 : cluster [WRN] pg 31.3 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001134+0000 mon.a (mon.0) 1161 : cluster [WRN] pg 31.3 is active+undersized+degraded, acting [3,6] 2024-09-13T14:40:00.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001150+0000 mon.a (mon.0) 1162 : cluster [WRN] pg 31.a is active+undersized+degraded, acting [3,1] 2024-09-13T14:40:00.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.001150+0000 mon.a (mon.0) 1162 : cluster [WRN] pg 31.a is active+undersized+degraded, acting [3,1] 2024-09-13T14:40:00.858 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:40:00 smithi177 bash[74346]: debug 2024-09-13T14:40:00.650+0000 7f6b6d1f1640 1 -- 172.21.15.177:0/3317420771 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55997b24e960 con 0x55997c03a000 2024-09-13T14:40:01.563 INFO:tasks.workunit.client.0.smithi099.stdout:0.00% done 2024-09-13T14:40:01.564 INFO:tasks.workunit.client.0.smithi099.stdout:25.00% done 2024-09-13T14:40:01.564 INFO:tasks.workunit.client.0.smithi099.stdout:50.00% done 2024-09-13T14:40:01.564 INFO:tasks.workunit.client.0.smithi099.stdout:75.00% done 2024-09-13T14:40:01.564 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.CheckObjectMap (887 ms) 2024-09-13T14:40:01.564 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.BlockingAIO 2024-09-13T14:40:01.564 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:01.604 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.BlockingAIO (41 ms) 2024-09-13T14:40:01.604 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ExclusiveLockTransition 2024-09-13T14:40:01.604 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:01.802 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ExclusiveLockTransition (198 ms) 2024-09-13T14:40:01.802 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ExclusiveLockReadTransition 2024-09-13T14:40:01.802 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:40:01.802 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ExclusiveLockReadTransition (0 ms) 2024-09-13T14:40:01.802 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.CacheMayCopyOnWrite 2024-09-13T14:40:01.802 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.207485+0000 mgr.y (mgr.26252) 404 : cluster [DBG] pgmap v636: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 352 KiB/s rd, 145 KiB/s wr, 576 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:01.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.207485+0000 mgr.y (mgr.26252) 404 : cluster [DBG] pgmap v636: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 352 KiB/s rd, 145 KiB/s wr, 576 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:01.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.398989+0000 mon.a (mon.0) 1163 : cluster [WRN] Health check update: Degraded data redundancy: 313/2403 objects degraded (13.025%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:01.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.398989+0000 mon.a (mon.0) 1163 : cluster [WRN] Health check update: Degraded data redundancy: 313/2403 objects degraded (13.025%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:01.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.436216+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e732: 8 total, 7 up, 8 in 2024-09-13T14:40:01.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[84576]: cluster 2024-09-13T14:40:00.436216+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e732: 8 total, 7 up, 8 in 2024-09-13T14:40:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.207485+0000 mgr.y (mgr.26252) 404 : cluster [DBG] pgmap v636: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 352 KiB/s rd, 145 KiB/s wr, 576 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.207485+0000 mgr.y (mgr.26252) 404 : cluster [DBG] pgmap v636: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 352 KiB/s rd, 145 KiB/s wr, 576 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:01.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.398989+0000 mon.a (mon.0) 1163 : cluster [WRN] Health check update: Degraded data redundancy: 313/2403 objects degraded (13.025%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:01.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.398989+0000 mon.a (mon.0) 1163 : cluster [WRN] Health check update: Degraded data redundancy: 313/2403 objects degraded (13.025%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:01.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.436216+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e732: 8 total, 7 up, 8 in 2024-09-13T14:40:01.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:01 smithi099 bash[87147]: cluster 2024-09-13T14:40:00.436216+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e732: 8 total, 7 up, 8 in 2024-09-13T14:40:01.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:01 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.207485+0000 mgr.y (mgr.26252) 404 : cluster [DBG] pgmap v636: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 352 KiB/s rd, 145 KiB/s wr, 576 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:02.074 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:01 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.207485+0000 mgr.y (mgr.26252) 404 : cluster [DBG] pgmap v636: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 352 KiB/s rd, 145 KiB/s wr, 576 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:02.074 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:01 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.398989+0000 mon.a (mon.0) 1163 : cluster [WRN] Health check update: Degraded data redundancy: 313/2403 objects degraded (13.025%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:02.074 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:01 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.398989+0000 mon.a (mon.0) 1163 : cluster [WRN] Health check update: Degraded data redundancy: 313/2403 objects degraded (13.025%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:02.074 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:01 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.436216+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e732: 8 total, 7 up, 8 in 2024-09-13T14:40:02.074 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:01 smithi177 bash[68594]: cluster 2024-09-13T14:40:00.436216+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e732: 8 total, 7 up, 8 in 2024-09-13T14:40:02.639 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.CacheMayCopyOnWrite (837 ms) 2024-09-13T14:40:02.639 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot 2024-09-13T14:40:02.639 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:02.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:02 smithi099 bash[84576]: cluster 2024-09-13T14:40:01.441621+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e733: 8 total, 7 up, 8 in 2024-09-13T14:40:02.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:02 smithi099 bash[84576]: cluster 2024-09-13T14:40:01.441621+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e733: 8 total, 7 up, 8 in 2024-09-13T14:40:02.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:02 smithi099 bash[87147]: cluster 2024-09-13T14:40:01.441621+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e733: 8 total, 7 up, 8 in 2024-09-13T14:40:02.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:02 smithi099 bash[87147]: cluster 2024-09-13T14:40:01.441621+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e733: 8 total, 7 up, 8 in 2024-09-13T14:40:02.858 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:40:02 smithi177 bash[74346]: debug 2024-09-13T14:40:02.426+0000 7f6b6fa58640 -1 Falling back to public interface 2024-09-13T14:40:02.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:02 smithi177 bash[68594]: cluster 2024-09-13T14:40:01.441621+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e733: 8 total, 7 up, 8 in 2024-09-13T14:40:02.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:02 smithi177 bash[68594]: cluster 2024-09-13T14:40:01.441621+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e733: 8 total, 7 up, 8 in 2024-09-13T14:40:03.556 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:03.558 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot (919 ms) 2024-09-13T14:40:03.558 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestImageOptions 2024-09-13T14:40:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[84576]: cluster 2024-09-13T14:40:02.209117+0000 mgr.y (mgr.26252) 405 : cluster [DBG] pgmap v639: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 105 KiB/s rd, 43 KiB/s wr, 174 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[84576]: cluster 2024-09-13T14:40:02.209117+0000 mgr.y (mgr.26252) 405 : cluster [DBG] pgmap v639: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 105 KiB/s rd, 43 KiB/s wr, 174 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[84576]: cluster 2024-09-13T14:40:02.449530+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e734: 8 total, 7 up, 8 in 2024-09-13T14:40:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[84576]: cluster 2024-09-13T14:40:02.449530+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e734: 8 total, 7 up, 8 in 2024-09-13T14:40:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[84576]: audit 2024-09-13T14:40:03.256206+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[84576]: audit 2024-09-13T14:40:03.256206+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[84576]: audit 2024-09-13T14:40:03.257863+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[84576]: audit 2024-09-13T14:40:03.257863+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[87147]: cluster 2024-09-13T14:40:02.209117+0000 mgr.y (mgr.26252) 405 : cluster [DBG] pgmap v639: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 105 KiB/s rd, 43 KiB/s wr, 174 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[87147]: cluster 2024-09-13T14:40:02.209117+0000 mgr.y (mgr.26252) 405 : cluster [DBG] pgmap v639: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 105 KiB/s rd, 43 KiB/s wr, 174 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[87147]: cluster 2024-09-13T14:40:02.449530+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e734: 8 total, 7 up, 8 in 2024-09-13T14:40:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[87147]: cluster 2024-09-13T14:40:02.449530+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e734: 8 total, 7 up, 8 in 2024-09-13T14:40:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[87147]: audit 2024-09-13T14:40:03.256206+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[87147]: audit 2024-09-13T14:40:03.256206+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[87147]: audit 2024-09-13T14:40:03.257863+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:03 smithi099 bash[87147]: audit 2024-09-13T14:40:03.257863+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:03.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:03 smithi177 bash[68594]: cluster 2024-09-13T14:40:02.209117+0000 mgr.y (mgr.26252) 405 : cluster [DBG] pgmap v639: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 105 KiB/s rd, 43 KiB/s wr, 174 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:03.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:03 smithi177 bash[68594]: cluster 2024-09-13T14:40:02.209117+0000 mgr.y (mgr.26252) 405 : cluster [DBG] pgmap v639: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 105 KiB/s rd, 43 KiB/s wr, 174 op/s; 313/2403 objects degraded (13.025%) 2024-09-13T14:40:03.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:03 smithi177 bash[68594]: cluster 2024-09-13T14:40:02.449530+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e734: 8 total, 7 up, 8 in 2024-09-13T14:40:03.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:03 smithi177 bash[68594]: cluster 2024-09-13T14:40:02.449530+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e734: 8 total, 7 up, 8 in 2024-09-13T14:40:03.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:03 smithi177 bash[68594]: audit 2024-09-13T14:40:03.256206+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:03.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:03 smithi177 bash[68594]: audit 2024-09-13T14:40:03.256206+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:03.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:03 smithi177 bash[68594]: audit 2024-09-13T14:40:03.257863+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:03.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:03 smithi177 bash[68594]: audit 2024-09-13T14:40:03.257863+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:04.748 INFO:tasks.workunit.client.0.smithi099.stdout:0.00% done 2024-09-13T14:40:04.748 INFO:tasks.workunit.client.0.smithi099.stdout:100.00% done 2024-09-13T14:40:04.749 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestImageOptions (1190 ms) 2024-09-13T14:40:04.749 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestImageOptionsPP 2024-09-13T14:40:04.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[84576]: cluster 2024-09-13T14:40:03.450781+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e735: 8 total, 7 up, 8 in 2024-09-13T14:40:04.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[84576]: cluster 2024-09-13T14:40:03.450781+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e735: 8 total, 7 up, 8 in 2024-09-13T14:40:04.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[84576]: audit 2024-09-13T14:40:03.831641+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[84576]: audit 2024-09-13T14:40:03.831641+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[84576]: audit 2024-09-13T14:40:03.842591+0000 mon.a (mon.0) 1171 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[84576]: audit 2024-09-13T14:40:03.842591+0000 mon.a (mon.0) 1171 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[87147]: cluster 2024-09-13T14:40:03.450781+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e735: 8 total, 7 up, 8 in 2024-09-13T14:40:04.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[87147]: cluster 2024-09-13T14:40:03.450781+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e735: 8 total, 7 up, 8 in 2024-09-13T14:40:04.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[87147]: audit 2024-09-13T14:40:03.831641+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[87147]: audit 2024-09-13T14:40:03.831641+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[87147]: audit 2024-09-13T14:40:03.842591+0000 mon.a (mon.0) 1171 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:04 smithi099 bash[87147]: audit 2024-09-13T14:40:03.842591+0000 mon.a (mon.0) 1171 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:04 smithi177 bash[68594]: cluster 2024-09-13T14:40:03.450781+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e735: 8 total, 7 up, 8 in 2024-09-13T14:40:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:04 smithi177 bash[68594]: cluster 2024-09-13T14:40:03.450781+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e735: 8 total, 7 up, 8 in 2024-09-13T14:40:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:04 smithi177 bash[68594]: audit 2024-09-13T14:40:03.831641+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:04 smithi177 bash[68594]: audit 2024-09-13T14:40:03.831641+0000 mon.a (mon.0) 1170 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:04 smithi177 bash[68594]: audit 2024-09-13T14:40:03.842591+0000 mon.a (mon.0) 1171 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:04.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:04 smithi177 bash[68594]: audit 2024-09-13T14:40:03.842591+0000 mon.a (mon.0) 1171 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.716 INFO:tasks.workunit.client.0.smithi099.stdout:0.00% done 2024-09-13T14:40:05.717 INFO:tasks.workunit.client.0.smithi099.stdout:100.00% done 2024-09-13T14:40:05.717 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestImageOptionsPP (968 ms) 2024-09-13T14:40:05.717 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.EventSocketPipe 2024-09-13T14:40:05.717 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.EventSocketPipe (0 ms) 2024-09-13T14:40:05.717 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.EventSocketEventfd 2024-09-13T14:40:05.717 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.EventSocketEventfd (0 ms) 2024-09-13T14:40:05.717 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ImagePollIO 2024-09-13T14:40:05.717 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:05.741 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[84576]: cluster 2024-09-13T14:40:04.212875+0000 mgr.y (mgr.26252) 406 : cluster [DBG] pgmap v642: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:05.741 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[84576]: cluster 2024-09-13T14:40:04.212875+0000 mgr.y (mgr.26252) 406 : cluster [DBG] pgmap v642: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:05.741 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[84576]: cluster 2024-09-13T14:40:04.463701+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e736: 8 total, 7 up, 8 in 2024-09-13T14:40:05.741 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[84576]: cluster 2024-09-13T14:40:04.463701+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e736: 8 total, 7 up, 8 in 2024-09-13T14:40:05.741 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[84576]: audit 2024-09-13T14:40:05.055341+0000 mon.a (mon.0) 1173 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[84576]: audit 2024-09-13T14:40:05.055341+0000 mon.a (mon.0) 1173 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[84576]: audit 2024-09-13T14:40:05.065529+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[84576]: audit 2024-09-13T14:40:05.065529+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[87147]: cluster 2024-09-13T14:40:04.212875+0000 mgr.y (mgr.26252) 406 : cluster [DBG] pgmap v642: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[87147]: cluster 2024-09-13T14:40:04.212875+0000 mgr.y (mgr.26252) 406 : cluster [DBG] pgmap v642: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[87147]: cluster 2024-09-13T14:40:04.463701+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e736: 8 total, 7 up, 8 in 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[87147]: cluster 2024-09-13T14:40:04.463701+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e736: 8 total, 7 up, 8 in 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[87147]: audit 2024-09-13T14:40:05.055341+0000 mon.a (mon.0) 1173 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[87147]: audit 2024-09-13T14:40:05.055341+0000 mon.a (mon.0) 1173 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[87147]: audit 2024-09-13T14:40:05.065529+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.742 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:05 smithi099 bash[87147]: audit 2024-09-13T14:40:05.065529+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.782 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:40:05.782 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:40:05.782 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:40:05.782 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:40:05.782 INFO:tasks.workunit.client.0.smithi099.stdout:wrote: 512 2024-09-13T14:40:05.782 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:40:05.782 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:40:05.783 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:started write 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:write completion cb called! 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 0 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:finished write 2024-09-13T14:40:05.784 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:40:05.785 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:40:05.786 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:40:05.786 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:40:05.786 INFO:tasks.workunit.client.0.smithi099.stdout:created completion 2024-09-13T14:40:05.786 INFO:tasks.workunit.client.0.smithi099.stdout:started read 2024-09-13T14:40:05.786 INFO:tasks.workunit.client.0.smithi099.stdout:read completion cb called! 2024-09-13T14:40:05.786 INFO:tasks.workunit.client.0.smithi099.stdout:return value is: 512 2024-09-13T14:40:05.786 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ImagePollIO (66 ms) 2024-09-13T14:40:05.786 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.Mirror 2024-09-13T14:40:05.790 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:05.803 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:05.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:05 smithi177 bash[68594]: cluster 2024-09-13T14:40:04.212875+0000 mgr.y (mgr.26252) 406 : cluster [DBG] pgmap v642: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:05.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:05 smithi177 bash[68594]: cluster 2024-09-13T14:40:04.212875+0000 mgr.y (mgr.26252) 406 : cluster [DBG] pgmap v642: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:05.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:05 smithi177 bash[68594]: cluster 2024-09-13T14:40:04.463701+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e736: 8 total, 7 up, 8 in 2024-09-13T14:40:05.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:05 smithi177 bash[68594]: cluster 2024-09-13T14:40:04.463701+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e736: 8 total, 7 up, 8 in 2024-09-13T14:40:05.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:05 smithi177 bash[68594]: audit 2024-09-13T14:40:05.055341+0000 mon.a (mon.0) 1173 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:05 smithi177 bash[68594]: audit 2024-09-13T14:40:05.055341+0000 mon.a (mon.0) 1173 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:05 smithi177 bash[68594]: audit 2024-09-13T14:40:05.065529+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:05.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:05 smithi177 bash[68594]: audit 2024-09-13T14:40:05.065529+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:06.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[84576]: cluster 2024-09-13T14:40:05.478568+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e737: 8 total, 7 up, 8 in 2024-09-13T14:40:06.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[84576]: cluster 2024-09-13T14:40:05.478568+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e737: 8 total, 7 up, 8 in 2024-09-13T14:40:06.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[84576]: cluster 2024-09-13T14:40:06.125895+0000 mon.a (mon.0) 1176 : cluster [WRN] Health check update: Degraded data redundancy: 366/2772 objects degraded (13.203%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:06.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[84576]: cluster 2024-09-13T14:40:06.125895+0000 mon.a (mon.0) 1176 : cluster [WRN] Health check update: Degraded data redundancy: 366/2772 objects degraded (13.203%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:06.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[84576]: audit 2024-09-13T14:40:06.374905+0000 mon.a (mon.0) 1177 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[84576]: audit 2024-09-13T14:40:06.374905+0000 mon.a (mon.0) 1177 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[84576]: audit 2024-09-13T14:40:06.375269+0000 mon.b (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[84576]: audit 2024-09-13T14:40:06.375269+0000 mon.b (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[87147]: cluster 2024-09-13T14:40:05.478568+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e737: 8 total, 7 up, 8 in 2024-09-13T14:40:06.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[87147]: cluster 2024-09-13T14:40:05.478568+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e737: 8 total, 7 up, 8 in 2024-09-13T14:40:06.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[87147]: cluster 2024-09-13T14:40:06.125895+0000 mon.a (mon.0) 1176 : cluster [WRN] Health check update: Degraded data redundancy: 366/2772 objects degraded (13.203%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:06.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[87147]: cluster 2024-09-13T14:40:06.125895+0000 mon.a (mon.0) 1176 : cluster [WRN] Health check update: Degraded data redundancy: 366/2772 objects degraded (13.203%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:06.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[87147]: audit 2024-09-13T14:40:06.374905+0000 mon.a (mon.0) 1177 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[87147]: audit 2024-09-13T14:40:06.374905+0000 mon.a (mon.0) 1177 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[87147]: audit 2024-09-13T14:40:06.375269+0000 mon.b (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:06 smithi099 bash[87147]: audit 2024-09-13T14:40:06.375269+0000 mon.b (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.858 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:40:06 smithi177 bash[74346]: debug 2024-09-13T14:40:06.366+0000 7f6b6fa58640 -1 osd.4 724 log_to_monitors true 2024-09-13T14:40:06.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:06 smithi177 bash[68594]: cluster 2024-09-13T14:40:05.478568+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e737: 8 total, 7 up, 8 in 2024-09-13T14:40:06.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:06 smithi177 bash[68594]: cluster 2024-09-13T14:40:05.478568+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e737: 8 total, 7 up, 8 in 2024-09-13T14:40:06.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:06 smithi177 bash[68594]: cluster 2024-09-13T14:40:06.125895+0000 mon.a (mon.0) 1176 : cluster [WRN] Health check update: Degraded data redundancy: 366/2772 objects degraded (13.203%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:06.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:06 smithi177 bash[68594]: cluster 2024-09-13T14:40:06.125895+0000 mon.a (mon.0) 1176 : cluster [WRN] Health check update: Degraded data redundancy: 366/2772 objects degraded (13.203%), 60 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:06.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:06 smithi177 bash[68594]: audit 2024-09-13T14:40:06.374905+0000 mon.a (mon.0) 1177 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:06 smithi177 bash[68594]: audit 2024-09-13T14:40:06.374905+0000 mon.a (mon.0) 1177 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:06 smithi177 bash[68594]: audit 2024-09-13T14:40:06.375269+0000 mon.b (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:06.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:06 smithi177 bash[68594]: audit 2024-09-13T14:40:06.375269+0000 mon.b (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T14:40:07.082 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.Mirror (1300 ms) 2024-09-13T14:40:07.082 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.MirrorPeerAttributes 2024-09-13T14:40:07.119 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.MirrorPeerAttributes (37 ms) 2024-09-13T14:40:07.119 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.CreateWithMirrorEnabled 2024-09-13T14:40:07.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:40:07] "GET /metrics HTTP/1.1" 200 75188 "" "Prometheus/2.51.0" 2024-09-13T14:40:07.359 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[74346]: debug 2024-09-13T14:40:07.062+0000 7f6b637fa640 -1 osd.4 724 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: cluster 2024-09-13T14:40:06.214651+0000 mgr.y (mgr.26252) 407 : cluster [DBG] pgmap v645: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: cluster 2024-09-13T14:40:06.214651+0000 mgr.y (mgr.26252) 407 : cluster [DBG] pgmap v645: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:06.498534+0000 mon.a (mon.0) 1178 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:06.498534+0000 mon.a (mon.0) 1178 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:06.507650+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:06.507650+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: cluster 2024-09-13T14:40:06.511382+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e738: 8 total, 7 up, 8 in 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: cluster 2024-09-13T14:40:06.511382+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e738: 8 total, 7 up, 8 in 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:06.513205+0000 mon.a (mon.0) 1180 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:06.513205+0000 mon.a (mon.0) 1180 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.018775+0000 mon.a (mon.0) 1181 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"} : dispatch 2024-09-13T14:40:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.018775+0000 mon.a (mon.0) 1181 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"} : dispatch 2024-09-13T14:40:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.025166+0000 mon.a (mon.0) 1182 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-09-13T14:40:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.025166+0000 mon.a (mon.0) 1182 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-09-13T14:40:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.029277+0000 mon.a (mon.0) 1183 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"} : dispatch 2024-09-13T14:40:07.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: cluster 2024-09-13T14:40:06.214651+0000 mgr.y (mgr.26252) 407 : cluster [DBG] pgmap v645: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:07.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: cluster 2024-09-13T14:40:06.214651+0000 mgr.y (mgr.26252) 407 : cluster [DBG] pgmap v645: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:07.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:06.498534+0000 mon.a (mon.0) 1178 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:40:07.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:06.498534+0000 mon.a (mon.0) 1178 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:40:07.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:06.507650+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:06.507650+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: cluster 2024-09-13T14:40:06.511382+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e738: 8 total, 7 up, 8 in 2024-09-13T14:40:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: cluster 2024-09-13T14:40:06.511382+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e738: 8 total, 7 up, 8 in 2024-09-13T14:40:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:06.513205+0000 mon.a (mon.0) 1180 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:06.513205+0000 mon.a (mon.0) 1180 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.018775+0000 mon.a (mon.0) 1181 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"} : dispatch 2024-09-13T14:40:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.018775+0000 mon.a (mon.0) 1181 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"} : dispatch 2024-09-13T14:40:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.025166+0000 mon.a (mon.0) 1182 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-09-13T14:40:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.025166+0000 mon.a (mon.0) 1182 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-09-13T14:40:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.029277+0000 mon.a (mon.0) 1183 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"} : dispatch 2024-09-13T14:40:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.029277+0000 mon.a (mon.0) 1183 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"} : dispatch 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.035359+0000 mon.a (mon.0) 1184 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"}]': finished 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.035359+0000 mon.a (mon.0) 1184 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"}]': finished 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.050628+0000 mon.a (mon.0) 1185 : audit [INF] from='osd.4 ' entity='osd.4' 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.050628+0000 mon.a (mon.0) 1185 : audit [INF] from='osd.4 ' entity='osd.4' 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.054727+0000 mon.a (mon.0) 1186 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"} : dispatch 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.054727+0000 mon.a (mon.0) 1186 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"} : dispatch 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.063149+0000 mon.a (mon.0) 1187 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"}]': finished 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.063149+0000 mon.a (mon.0) 1187 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"}]': finished 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.067446+0000 mon.a (mon.0) 1188 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"} : dispatch 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.067446+0000 mon.a (mon.0) 1188 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"} : dispatch 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.075536+0000 mon.a (mon.0) 1189 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"}]': finished 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.075536+0000 mon.a (mon.0) 1189 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"}]': finished 2024-09-13T14:40:07.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.091576+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.091576+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.101594+0000 mon.a (mon.0) 1191 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.029277+0000 mon.a (mon.0) 1183 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"} : dispatch 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.035359+0000 mon.a (mon.0) 1184 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"}]': finished 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.035359+0000 mon.a (mon.0) 1184 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"}]': finished 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.050628+0000 mon.a (mon.0) 1185 : audit [INF] from='osd.4 ' entity='osd.4' 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.050628+0000 mon.a (mon.0) 1185 : audit [INF] from='osd.4 ' entity='osd.4' 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.054727+0000 mon.a (mon.0) 1186 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"} : dispatch 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.054727+0000 mon.a (mon.0) 1186 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"} : dispatch 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.063149+0000 mon.a (mon.0) 1187 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"}]': finished 2024-09-13T14:40:07.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.063149+0000 mon.a (mon.0) 1187 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"}]': finished 2024-09-13T14:40:07.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.067446+0000 mon.a (mon.0) 1188 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"} : dispatch 2024-09-13T14:40:07.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.067446+0000 mon.a (mon.0) 1188 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"} : dispatch 2024-09-13T14:40:07.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.075536+0000 mon.a (mon.0) 1189 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"}]': finished 2024-09-13T14:40:07.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.075536+0000 mon.a (mon.0) 1189 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"}]': finished 2024-09-13T14:40:07.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.091576+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.091576+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.101594+0000 mon.a (mon.0) 1191 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' 2024-09-13T14:40:07.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.101594+0000 mon.a (mon.0) 1191 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' 2024-09-13T14:40:07.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.102511+0000 mon.a (mon.0) 1192 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.102511+0000 mon.a (mon.0) 1192 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.103248+0000 mon.a (mon.0) 1193 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.103248+0000 mon.a (mon.0) 1193 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.111287+0000 mon.a (mon.0) 1194 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"}]': finished 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[84576]: audit 2024-09-13T14:40:07.111287+0000 mon.a (mon.0) 1194 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"}]': finished 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.101594+0000 mon.a (mon.0) 1191 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.102511+0000 mon.a (mon.0) 1192 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.102511+0000 mon.a (mon.0) 1192 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.103248+0000 mon.a (mon.0) 1193 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.103248+0000 mon.a (mon.0) 1193 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.111287+0000 mon.a (mon.0) 1194 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"}]': finished 2024-09-13T14:40:07.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:07 smithi099 bash[87147]: audit 2024-09-13T14:40:07.111287+0000 mon.a (mon.0) 1194 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"}]': finished 2024-09-13T14:40:07.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: cluster 2024-09-13T14:40:06.214651+0000 mgr.y (mgr.26252) 407 : cluster [DBG] pgmap v645: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:07.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: cluster 2024-09-13T14:40:06.214651+0000 mgr.y (mgr.26252) 407 : cluster [DBG] pgmap v645: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 139 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 533 KiB/s rd, 155 KiB/s wr, 715 op/s; 366/2772 objects degraded (13.203%) 2024-09-13T14:40:07.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:06.498534+0000 mon.a (mon.0) 1178 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:40:07.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:06.498534+0000 mon.a (mon.0) 1178 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:40:07.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:06.507650+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:06.507650+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: cluster 2024-09-13T14:40:06.511382+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e738: 8 total, 7 up, 8 in 2024-09-13T14:40:07.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: cluster 2024-09-13T14:40:06.511382+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e738: 8 total, 7 up, 8 in 2024-09-13T14:40:07.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:06.513205+0000 mon.a (mon.0) 1180 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:06.513205+0000 mon.a (mon.0) 1180 : audit [INF] from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]} : dispatch 2024-09-13T14:40:07.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.018775+0000 mon.a (mon.0) 1181 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"} : dispatch 2024-09-13T14:40:07.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.018775+0000 mon.a (mon.0) 1181 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"} : dispatch 2024-09-13T14:40:07.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.025166+0000 mon.a (mon.0) 1182 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-09-13T14:40:07.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.025166+0000 mon.a (mon.0) 1182 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-09-13T14:40:07.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.029277+0000 mon.a (mon.0) 1183 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"} : dispatch 2024-09-13T14:40:07.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.029277+0000 mon.a (mon.0) 1183 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"} : dispatch 2024-09-13T14:40:07.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.035359+0000 mon.a (mon.0) 1184 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"}]': finished 2024-09-13T14:40:07.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.035359+0000 mon.a (mon.0) 1184 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c09ecf4f-c059-4c22-b725-45493b2cf4e3"}]': finished 2024-09-13T14:40:07.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.050628+0000 mon.a (mon.0) 1185 : audit [INF] from='osd.4 ' entity='osd.4' 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.050628+0000 mon.a (mon.0) 1185 : audit [INF] from='osd.4 ' entity='osd.4' 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.054727+0000 mon.a (mon.0) 1186 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"} : dispatch 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.054727+0000 mon.a (mon.0) 1186 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"} : dispatch 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.063149+0000 mon.a (mon.0) 1187 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"}]': finished 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.063149+0000 mon.a (mon.0) 1187 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c89b7d15-803c-4807-ae84-3e3d71064c38"}]': finished 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.067446+0000 mon.a (mon.0) 1188 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"} : dispatch 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.067446+0000 mon.a (mon.0) 1188 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"} : dispatch 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.075536+0000 mon.a (mon.0) 1189 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"}]': finished 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.075536+0000 mon.a (mon.0) 1189 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b02b3026-f82d-446a-aea9-eb62c353bfa6"}]': finished 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.091576+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.091576+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.101594+0000 mon.a (mon.0) 1191 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' 2024-09-13T14:40:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.101594+0000 mon.a (mon.0) 1191 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' 2024-09-13T14:40:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.102511+0000 mon.a (mon.0) 1192 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.102511+0000 mon.a (mon.0) 1192 : audit [DBG] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.103248+0000 mon.a (mon.0) 1193 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.103248+0000 mon.a (mon.0) 1193 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"} : dispatch 2024-09-13T14:40:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.111287+0000 mon.a (mon.0) 1194 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"}]': finished 2024-09-13T14:40:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:07 smithi177 bash[68594]: audit 2024-09-13T14:40:07.111287+0000 mon.a (mon.0) 1194 : audit [INF] from='client.? 172.21.15.99:0/4091065626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/c7f3d6c2-5400-48b8-9c60-fea3582788ac"}]': finished 2024-09-13T14:40:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[84576]: cluster 2024-09-13T14:40:08.073915+0000 mon.a (mon.0) 1195 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[84576]: cluster 2024-09-13T14:40:08.073915+0000 mon.a (mon.0) 1195 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[84576]: cluster 2024-09-13T14:40:08.084891+0000 mon.a (mon.0) 1196 : cluster [INF] osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964] boot 2024-09-13T14:40:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[84576]: cluster 2024-09-13T14:40:08.084891+0000 mon.a (mon.0) 1196 : cluster [INF] osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964] boot 2024-09-13T14:40:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[84576]: cluster 2024-09-13T14:40:08.085101+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-13T14:40:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[84576]: cluster 2024-09-13T14:40:08.085101+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-13T14:40:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[84576]: audit 2024-09-13T14:40:08.102366+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:40:08.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[84576]: audit 2024-09-13T14:40:08.102366+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:40:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[87147]: cluster 2024-09-13T14:40:08.073915+0000 mon.a (mon.0) 1195 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:08.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[87147]: cluster 2024-09-13T14:40:08.073915+0000 mon.a (mon.0) 1195 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[87147]: cluster 2024-09-13T14:40:08.084891+0000 mon.a (mon.0) 1196 : cluster [INF] osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964] boot 2024-09-13T14:40:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[87147]: cluster 2024-09-13T14:40:08.084891+0000 mon.a (mon.0) 1196 : cluster [INF] osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964] boot 2024-09-13T14:40:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[87147]: cluster 2024-09-13T14:40:08.085101+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-13T14:40:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[87147]: cluster 2024-09-13T14:40:08.085101+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-13T14:40:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[87147]: audit 2024-09-13T14:40:08.102366+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:40:08.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:08 smithi099 bash[87147]: audit 2024-09-13T14:40:08.102366+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:40:08.849 INFO:teuthology.orchestra.run.smithi099.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:40:08.849 INFO:teuthology.orchestra.run.smithi099.stdout:alertmanager.a smithi099 *:9093,9094 running (10m) 40s ago 18m 14.7M - 0.27.0 11f11916f8cd 45a2ce75edd3 2024-09-13T14:40:08.849 INFO:teuthology.orchestra.run.smithi099.stdout:grafana.a smithi177 *:3000 running (9m) 5s ago 18m 74.5M - 9.4.7 954c08fa6188 443ce4187a0d 2024-09-13T14:40:08.849 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi099.yyrtld smithi099 running (15m) 40s ago 15m 15.2M - 17.2.7-1627-gbbdadfa3 a994b00deb3a 7bce2f3fd8d5 2024-09-13T14:40:08.849 INFO:teuthology.orchestra.run.smithi099.stdout:mds.foo.smithi177.outtkv smithi177 running (15m) 5s ago 15m 20.8M - 17.2.7-1627-gbbdadfa3 a994b00deb3a b976db2e3d1b 2024-09-13T14:40:08.849 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.x smithi177 *:8443,9283,8765 running (8m) 5s ago 27m 442M - 19.3.0-4697-gd58f7306 fec07016a2b0 24a222989f2c 2024-09-13T14:40:08.849 INFO:teuthology.orchestra.run.smithi099.stdout:mgr.y smithi099 *:8443,9283,8765 running (9m) 40s ago 29m 528M - 19.3.0-4697-gd58f7306 fec07016a2b0 8ebc4af8542d 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:mon.a smithi099 running (8m) 40s ago 29m 74.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 7bae834be590 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:mon.b smithi177 running (5m) 5s ago 27m 52.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 6ddaaf41777b 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:mon.c smithi099 running (6m) 40s ago 27m 52.2M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 8e92ca9c3d4b 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.a smithi099 *:9100 running (10m) 40s ago 18m 8391k - 1.7.0 72c9c2088986 adc238b4fabf 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:node-exporter.b smithi177 *:9100 running (10m) 5s ago 18m 8416k - 1.7.0 72c9c2088986 0e333c475514 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0 smithi099 running (4m) 40s ago 26m 99.6M 2524M 19.3.0-4697-gd58f7306 fec07016a2b0 ebecc5a7ecad 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1 smithi099 running (2m) 40s ago 25m 111M 2524M 19.3.0-4697-gd58f7306 fec07016a2b0 5ed58cf87b9c 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2 smithi099 running (78s) 40s ago 24m 91.8M 2524M 19.3.0-4697-gd58f7306 fec07016a2b0 8bf83563f2b4 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3 smithi099 running (44s) 40s ago 23m - 2524M 19.3.0-4697-gd58f7306 fec07016a2b0 b0bde9c79880 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4 smithi177 running (9s) 5s ago 22m - 2524M 19.3.0-4697-gd58f7306 fec07016a2b0 a14869e52246 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5 smithi177 running (21m) 5s ago 22m 151M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a f285f4fee6d3 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6 smithi177 running (21m) 5s ago 21m 159M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a 2318776c78d4 2024-09-13T14:40:08.850 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7 smithi177 running (20m) 5s ago 20m 178M 2524M 17.2.7-1627-gbbdadfa3 a994b00deb3a e2d33a0b16ca 2024-09-13T14:40:08.851 INFO:teuthology.orchestra.run.smithi099.stdout:prometheus.a smithi177 *:9095 running (9m) 5s ago 19m 53.8M - 2.51.0 1d3b7f56885b d360e28d5f8b 2024-09-13T14:40:08.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:08 smithi177 bash[68594]: cluster 2024-09-13T14:40:08.073915+0000 mon.a (mon.0) 1195 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:08.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:08 smithi177 bash[68594]: cluster 2024-09-13T14:40:08.073915+0000 mon.a (mon.0) 1195 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:08.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:08 smithi177 bash[68594]: cluster 2024-09-13T14:40:08.084891+0000 mon.a (mon.0) 1196 : cluster [INF] osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964] boot 2024-09-13T14:40:08.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:08 smithi177 bash[68594]: cluster 2024-09-13T14:40:08.084891+0000 mon.a (mon.0) 1196 : cluster [INF] osd.4 [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964] boot 2024-09-13T14:40:08.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:08 smithi177 bash[68594]: cluster 2024-09-13T14:40:08.085101+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-13T14:40:08.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:08 smithi177 bash[68594]: cluster 2024-09-13T14:40:08.085101+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-13T14:40:08.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:08 smithi177 bash[68594]: audit 2024-09-13T14:40:08.102366+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:40:08.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:08 smithi177 bash[68594]: audit 2024-09-13T14:40:08.102366+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:40:09.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:09 smithi099 bash[84576]: cluster 2024-09-13T14:40:08.215656+0000 mgr.y (mgr.26252) 408 : cluster [DBG] pgmap v648: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 141 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 518 KiB/s wr, 100 op/s; 378/2844 objects degraded (13.291%) 2024-09-13T14:40:09.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:09 smithi099 bash[84576]: cluster 2024-09-13T14:40:08.215656+0000 mgr.y (mgr.26252) 408 : cluster [DBG] pgmap v648: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 141 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 518 KiB/s wr, 100 op/s; 378/2844 objects degraded (13.291%) 2024-09-13T14:40:09.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:09 smithi099 bash[84576]: cluster 2024-09-13T14:40:09.092369+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-13T14:40:09.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:09 smithi099 bash[84576]: cluster 2024-09-13T14:40:09.092369+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-13T14:40:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:09 smithi099 bash[87147]: cluster 2024-09-13T14:40:08.215656+0000 mgr.y (mgr.26252) 408 : cluster [DBG] pgmap v648: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 141 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 518 KiB/s wr, 100 op/s; 378/2844 objects degraded (13.291%) 2024-09-13T14:40:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:09 smithi099 bash[87147]: cluster 2024-09-13T14:40:08.215656+0000 mgr.y (mgr.26252) 408 : cluster [DBG] pgmap v648: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 141 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 518 KiB/s wr, 100 op/s; 378/2844 objects degraded (13.291%) 2024-09-13T14:40:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:09 smithi099 bash[87147]: cluster 2024-09-13T14:40:09.092369+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-13T14:40:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:09 smithi099 bash[87147]: cluster 2024-09-13T14:40:09.092369+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-13T14:40:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:09 smithi177 bash[68594]: cluster 2024-09-13T14:40:08.215656+0000 mgr.y (mgr.26252) 408 : cluster [DBG] pgmap v648: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 141 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 518 KiB/s wr, 100 op/s; 378/2844 objects degraded (13.291%) 2024-09-13T14:40:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:09 smithi177 bash[68594]: cluster 2024-09-13T14:40:08.215656+0000 mgr.y (mgr.26252) 408 : cluster [DBG] pgmap v648: 617 pgs: 175 active+undersized, 60 active+undersized+degraded, 382 active+clean; 141 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 518 KiB/s wr, 100 op/s; 378/2844 objects degraded (13.291%) 2024-09-13T14:40:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:09 smithi177 bash[68594]: cluster 2024-09-13T14:40:09.092369+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-13T14:40:09.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:09 smithi177 bash[68594]: cluster 2024-09-13T14:40:09.092369+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-13T14:40:10.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph versions' 2024-09-13T14:40:10.505 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:40:10 smithi177 bash[61439]: ts=2024-09-13T14:40:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:10.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:10 smithi099 bash[84576]: audit 2024-09-13T14:40:08.815464+0000 mgr.y (mgr.26252) 409 : audit [DBG] from='client.34989 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:10.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:10 smithi099 bash[84576]: audit 2024-09-13T14:40:08.815464+0000 mgr.y (mgr.26252) 409 : audit [DBG] from='client.34989 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:10.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:10 smithi099 bash[84576]: cluster 2024-09-13T14:40:10.095842+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-13T14:40:10.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:10 smithi099 bash[84576]: cluster 2024-09-13T14:40:10.095842+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-13T14:40:10.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:10 smithi099 bash[87147]: audit 2024-09-13T14:40:08.815464+0000 mgr.y (mgr.26252) 409 : audit [DBG] from='client.34989 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:10.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:10 smithi099 bash[87147]: audit 2024-09-13T14:40:08.815464+0000 mgr.y (mgr.26252) 409 : audit [DBG] from='client.34989 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:10.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:10 smithi099 bash[87147]: cluster 2024-09-13T14:40:10.095842+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-13T14:40:10.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:10 smithi099 bash[87147]: cluster 2024-09-13T14:40:10.095842+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-13T14:40:10.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:10 smithi177 bash[68594]: audit 2024-09-13T14:40:08.815464+0000 mgr.y (mgr.26252) 409 : audit [DBG] from='client.34989 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:10 smithi177 bash[68594]: audit 2024-09-13T14:40:08.815464+0000 mgr.y (mgr.26252) 409 : audit [DBG] from='client.34989 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:10 smithi177 bash[68594]: cluster 2024-09-13T14:40:10.095842+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-13T14:40:10.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:10 smithi177 bash[68594]: cluster 2024-09-13T14:40:10.095842+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-13T14:40:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[84576]: cluster 2024-09-13T14:40:10.216931+0000 mgr.y (mgr.26252) 410 : cluster [DBG] pgmap v651: 617 pgs: 85 peering, 112 active+undersized, 38 active+undersized+degraded, 382 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 196 KiB/s rd, 1.8 MiB/s wr, 270 op/s; 257/2955 objects degraded (8.697%) 2024-09-13T14:40:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[84576]: cluster 2024-09-13T14:40:10.216931+0000 mgr.y (mgr.26252) 410 : cluster [DBG] pgmap v651: 617 pgs: 85 peering, 112 active+undersized, 38 active+undersized+degraded, 382 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 196 KiB/s rd, 1.8 MiB/s wr, 270 op/s; 257/2955 objects degraded (8.697%) 2024-09-13T14:40:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[84576]: cluster 2024-09-13T14:40:11.106169+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-13T14:40:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[84576]: cluster 2024-09-13T14:40:11.106169+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-13T14:40:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[84576]: cluster 2024-09-13T14:40:11.127130+0000 mon.a (mon.0) 1202 : cluster [WRN] Health check update: Degraded data redundancy: 257/2955 objects degraded (8.697%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[84576]: cluster 2024-09-13T14:40:11.127130+0000 mon.a (mon.0) 1202 : cluster [WRN] Health check update: Degraded data redundancy: 257/2955 objects degraded (8.697%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[87147]: cluster 2024-09-13T14:40:10.216931+0000 mgr.y (mgr.26252) 410 : cluster [DBG] pgmap v651: 617 pgs: 85 peering, 112 active+undersized, 38 active+undersized+degraded, 382 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 196 KiB/s rd, 1.8 MiB/s wr, 270 op/s; 257/2955 objects degraded (8.697%) 2024-09-13T14:40:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[87147]: cluster 2024-09-13T14:40:10.216931+0000 mgr.y (mgr.26252) 410 : cluster [DBG] pgmap v651: 617 pgs: 85 peering, 112 active+undersized, 38 active+undersized+degraded, 382 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 196 KiB/s rd, 1.8 MiB/s wr, 270 op/s; 257/2955 objects degraded (8.697%) 2024-09-13T14:40:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[87147]: cluster 2024-09-13T14:40:11.106169+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-13T14:40:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[87147]: cluster 2024-09-13T14:40:11.106169+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-13T14:40:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[87147]: cluster 2024-09-13T14:40:11.127130+0000 mon.a (mon.0) 1202 : cluster [WRN] Health check update: Degraded data redundancy: 257/2955 objects degraded (8.697%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:11.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:11 smithi099 bash[87147]: cluster 2024-09-13T14:40:11.127130+0000 mon.a (mon.0) 1202 : cluster [WRN] Health check update: Degraded data redundancy: 257/2955 objects degraded (8.697%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:11.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:11 smithi177 bash[68594]: cluster 2024-09-13T14:40:10.216931+0000 mgr.y (mgr.26252) 410 : cluster [DBG] pgmap v651: 617 pgs: 85 peering, 112 active+undersized, 38 active+undersized+degraded, 382 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 196 KiB/s rd, 1.8 MiB/s wr, 270 op/s; 257/2955 objects degraded (8.697%) 2024-09-13T14:40:11.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:11 smithi177 bash[68594]: cluster 2024-09-13T14:40:10.216931+0000 mgr.y (mgr.26252) 410 : cluster [DBG] pgmap v651: 617 pgs: 85 peering, 112 active+undersized, 38 active+undersized+degraded, 382 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 196 KiB/s rd, 1.8 MiB/s wr, 270 op/s; 257/2955 objects degraded (8.697%) 2024-09-13T14:40:11.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:11 smithi177 bash[68594]: cluster 2024-09-13T14:40:11.106169+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-13T14:40:11.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:11 smithi177 bash[68594]: cluster 2024-09-13T14:40:11.106169+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-13T14:40:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:11 smithi177 bash[68594]: cluster 2024-09-13T14:40:11.127130+0000 mon.a (mon.0) 1202 : cluster [WRN] Health check update: Degraded data redundancy: 257/2955 objects degraded (8.697%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:11.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:11 smithi177 bash[68594]: cluster 2024-09-13T14:40:11.127130+0000 mon.a (mon.0) 1202 : cluster [WRN] Health check update: Degraded data redundancy: 257/2955 objects degraded (8.697%), 38 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:13.219 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.CreateWithMirrorEnabled (6100 ms) 2024-09-13T14:40:13.220 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot 2024-09-13T14:40:13.220 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:13.504 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:13 smithi177 bash[68594]: cluster 2024-09-13T14:40:12.109327+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-13T14:40:13.504 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:13 smithi177 bash[68594]: cluster 2024-09-13T14:40:12.109327+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-13T14:40:13.504 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:13 smithi177 bash[68594]: cluster 2024-09-13T14:40:12.218344+0000 mgr.y (mgr.26252) 411 : cluster [DBG] pgmap v654: 617 pgs: 85 peering, 54 active+undersized, 20 active+undersized+degraded, 458 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 1.5 MiB/s wr, 215 op/s; 178/2973 objects degraded (5.987%); 197 KiB/s, 5 keys/s, 8 objects/s recovering 2024-09-13T14:40:13.504 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:13 smithi177 bash[68594]: cluster 2024-09-13T14:40:12.218344+0000 mgr.y (mgr.26252) 411 : cluster [DBG] pgmap v654: 617 pgs: 85 peering, 54 active+undersized, 20 active+undersized+degraded, 458 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 1.5 MiB/s wr, 215 op/s; 178/2973 objects degraded (5.987%); 197 KiB/s, 5 keys/s, 8 objects/s recovering 2024-09-13T14:40:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:13 smithi099 bash[84576]: cluster 2024-09-13T14:40:12.109327+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-13T14:40:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:13 smithi099 bash[84576]: cluster 2024-09-13T14:40:12.109327+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-13T14:40:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:13 smithi099 bash[84576]: cluster 2024-09-13T14:40:12.218344+0000 mgr.y (mgr.26252) 411 : cluster [DBG] pgmap v654: 617 pgs: 85 peering, 54 active+undersized, 20 active+undersized+degraded, 458 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 1.5 MiB/s wr, 215 op/s; 178/2973 objects degraded (5.987%); 197 KiB/s, 5 keys/s, 8 objects/s recovering 2024-09-13T14:40:13.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:13 smithi099 bash[84576]: cluster 2024-09-13T14:40:12.218344+0000 mgr.y (mgr.26252) 411 : cluster [DBG] pgmap v654: 617 pgs: 85 peering, 54 active+undersized, 20 active+undersized+degraded, 458 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 1.5 MiB/s wr, 215 op/s; 178/2973 objects degraded (5.987%); 197 KiB/s, 5 keys/s, 8 objects/s recovering 2024-09-13T14:40:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:13 smithi099 bash[87147]: cluster 2024-09-13T14:40:12.109327+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-13T14:40:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:13 smithi099 bash[87147]: cluster 2024-09-13T14:40:12.109327+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-13T14:40:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:13 smithi099 bash[87147]: cluster 2024-09-13T14:40:12.218344+0000 mgr.y (mgr.26252) 411 : cluster [DBG] pgmap v654: 617 pgs: 85 peering, 54 active+undersized, 20 active+undersized+degraded, 458 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 1.5 MiB/s wr, 215 op/s; 178/2973 objects degraded (5.987%); 197 KiB/s, 5 keys/s, 8 objects/s recovering 2024-09-13T14:40:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:13 smithi099 bash[87147]: cluster 2024-09-13T14:40:12.218344+0000 mgr.y (mgr.26252) 411 : cluster [DBG] pgmap v654: 617 pgs: 85 peering, 54 active+undersized, 20 active+undersized+degraded, 458 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 1.5 MiB/s wr, 215 op/s; 178/2973 objects degraded (5.987%); 197 KiB/s, 5 keys/s, 8 objects/s recovering 2024-09-13T14:40:14.450 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:14 smithi177 bash[68594]: cluster 2024-09-13T14:40:13.122156+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-13T14:40:14.450 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:14 smithi177 bash[68594]: cluster 2024-09-13T14:40:13.122156+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-13T14:40:14.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:14 smithi099 bash[84576]: cluster 2024-09-13T14:40:13.122156+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-13T14:40:14.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:14 smithi099 bash[84576]: cluster 2024-09-13T14:40:13.122156+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-13T14:40:14.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:14 smithi099 bash[87147]: cluster 2024-09-13T14:40:13.122156+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-13T14:40:14.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:14 smithi099 bash[87147]: cluster 2024-09-13T14:40:13.122156+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-13T14:40:14.722 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:40:14 smithi177 bash[60145]: logger=sqlstore.transactions t=2024-09-13T14:40:14.454062597Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:40:14.722 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:40:14 smithi177 bash[60145]: logger=cleanup t=2024-09-13T14:40:14.561024275Z level=info msg="Completed cleanup jobs" duration=118.403598ms 2024-09-13T14:40:14.894 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:15.190 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot (1971 ms) 2024-09-13T14:40:15.190 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ExclusiveLock 2024-09-13T14:40:15.190 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:15 smithi099 bash[84576]: cluster 2024-09-13T14:40:14.131442+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-13T14:40:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:15 smithi099 bash[84576]: cluster 2024-09-13T14:40:14.131442+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-13T14:40:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:15 smithi099 bash[84576]: cluster 2024-09-13T14:40:14.219460+0000 mgr.y (mgr.26252) 412 : cluster [DBG] pgmap v657: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 151 KiB/s rd, 204 KiB/s wr, 201 op/s; 788 KiB/s, 34 keys/s, 45 objects/s recovering 2024-09-13T14:40:15.239 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:15 smithi099 bash[84576]: cluster 2024-09-13T14:40:14.219460+0000 mgr.y (mgr.26252) 412 : cluster [DBG] pgmap v657: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 151 KiB/s rd, 204 KiB/s wr, 201 op/s; 788 KiB/s, 34 keys/s, 45 objects/s recovering 2024-09-13T14:40:15.311 INFO:tasks.workunit.client.0.smithi099.stdout:1: acquiring exclusive lock 2024-09-13T14:40:15.311 INFO:tasks.workunit.client.0.smithi099.stdout:2: acquiring exclusive lock 2024-09-13T14:40:15.317 INFO:tasks.workunit.client.0.smithi099.stdout:2: exclusive lock acquired 2024-09-13T14:40:15.329 INFO:tasks.workunit.client.0.smithi099.stdout:2: releasing exclusive lock 2024-09-13T14:40:15.334 INFO:tasks.workunit.client.0.smithi099.stdout:2: exclusive lock released 2024-09-13T14:40:15.334 INFO:tasks.workunit.client.0.smithi099.stdout:2: acquiring exclusive lock 2024-09-13T14:40:15.340 INFO:tasks.workunit.client.0.smithi099.stdout:1: exclusive lock acquired 2024-09-13T14:40:15.374 INFO:tasks.workunit.client.0.smithi099.stdout:1: releasing exclusive lock 2024-09-13T14:40:15.378 INFO:tasks.workunit.client.0.smithi099.stdout:1: exclusive lock released 2024-09-13T14:40:15.378 INFO:tasks.workunit.client.0.smithi099.stdout:1: acquiring exclusive lock 2024-09-13T14:40:15.384 INFO:tasks.workunit.client.0.smithi099.stdout:1: exclusive lock acquired 2024-09-13T14:40:15.425 INFO:tasks.workunit.client.0.smithi099.stdout:1: releasing exclusive lock 2024-09-13T14:40:15.429 INFO:tasks.workunit.client.0.smithi099.stdout:1: exclusive lock released 2024-09-13T14:40:15.429 INFO:tasks.workunit.client.0.smithi099.stdout:1: acquiring exclusive lock 2024-09-13T14:40:15.434 INFO:tasks.workunit.client.0.smithi099.stdout:2: exclusive lock acquired 2024-09-13T14:40:15.475 INFO:tasks.workunit.client.0.smithi099.stdout:2: releasing exclusive lock 2024-09-13T14:40:15.480 INFO:tasks.workunit.client.0.smithi099.stdout:2: exclusive lock released 2024-09-13T14:40:15.480 INFO:tasks.workunit.client.0.smithi099.stdout:2: acquiring exclusive lock 2024-09-13T14:40:15.487 INFO:tasks.workunit.client.0.smithi099.stdout:1: exclusive lock acquired 2024-09-13T14:40:15.516 INFO:tasks.workunit.client.0.smithi099.stdout:1: releasing exclusive lock 2024-09-13T14:40:15.521 INFO:tasks.workunit.client.0.smithi099.stdout:1: exclusive lock released 2024-09-13T14:40:15.521 INFO:tasks.workunit.client.0.smithi099.stdout:1: acquiring exclusive lock 2024-09-13T14:40:15.527 INFO:tasks.workunit.client.0.smithi099.stdout:2: exclusive lock acquired 2024-09-13T14:40:15.538 INFO:tasks.workunit.client.0.smithi099.stdout:2: releasing exclusive lock 2024-09-13T14:40:15.544 INFO:tasks.workunit.client.0.smithi099.stdout:2: exclusive lock released 2024-09-13T14:40:15.544 INFO:tasks.workunit.client.0.smithi099.stdout:2: acquiring exclusive lock 2024-09-13T14:40:15.551 INFO:tasks.workunit.client.0.smithi099.stdout:1: exclusive lock acquired 2024-09-13T14:40:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:15 smithi099 bash[87147]: cluster 2024-09-13T14:40:14.131442+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-13T14:40:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:15 smithi099 bash[87147]: cluster 2024-09-13T14:40:14.131442+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-13T14:40:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:15 smithi099 bash[87147]: cluster 2024-09-13T14:40:14.219460+0000 mgr.y (mgr.26252) 412 : cluster [DBG] pgmap v657: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 151 KiB/s rd, 204 KiB/s wr, 201 op/s; 788 KiB/s, 34 keys/s, 45 objects/s recovering 2024-09-13T14:40:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:15 smithi099 bash[87147]: cluster 2024-09-13T14:40:14.219460+0000 mgr.y (mgr.26252) 412 : cluster [DBG] pgmap v657: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 151 KiB/s rd, 204 KiB/s wr, 201 op/s; 788 KiB/s, 34 keys/s, 45 objects/s recovering 2024-09-13T14:40:15.599 INFO:tasks.workunit.client.0.smithi099.stdout:1: releasing exclusive lock 2024-09-13T14:40:15.604 INFO:tasks.workunit.client.0.smithi099.stdout:1: exclusive lock released 2024-09-13T14:40:15.604 INFO:tasks.workunit.client.0.smithi099.stdout:1: acquiring exclusive lock 2024-09-13T14:40:15.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:15 smithi177 bash[68594]: cluster 2024-09-13T14:40:14.131442+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-13T14:40:15.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:15 smithi177 bash[68594]: cluster 2024-09-13T14:40:14.131442+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-13T14:40:15.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:15 smithi177 bash[68594]: cluster 2024-09-13T14:40:14.219460+0000 mgr.y (mgr.26252) 412 : cluster [DBG] pgmap v657: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 151 KiB/s rd, 204 KiB/s wr, 201 op/s; 788 KiB/s, 34 keys/s, 45 objects/s recovering 2024-09-13T14:40:15.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:15 smithi177 bash[68594]: cluster 2024-09-13T14:40:14.219460+0000 mgr.y (mgr.26252) 412 : cluster [DBG] pgmap v657: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 151 KiB/s rd, 204 KiB/s wr, 201 op/s; 788 KiB/s, 34 keys/s, 45 objects/s recovering 2024-09-13T14:40:15.610 INFO:tasks.workunit.client.0.smithi099.stdout:2: exclusive lock acquired 2024-09-13T14:40:15.624 INFO:tasks.workunit.client.0.smithi099.stdout:2: releasing exclusive lock 2024-09-13T14:40:15.629 INFO:tasks.workunit.client.0.smithi099.stdout:2: exclusive lock released 2024-09-13T14:40:15.630 INFO:tasks.workunit.client.0.smithi099.stdout:2: acquiring exclusive lock 2024-09-13T14:40:15.637 INFO:tasks.workunit.client.0.smithi099.stdout:1: exclusive lock acquired 2024-09-13T14:40:15.649 INFO:tasks.workunit.client.0.smithi099.stdout:1: releasing exclusive lock 2024-09-13T14:40:15.654 INFO:tasks.workunit.client.0.smithi099.stdout:1: exclusive lock released 2024-09-13T14:40:15.662 INFO:tasks.workunit.client.0.smithi099.stdout:2: exclusive lock acquired 2024-09-13T14:40:15.711 INFO:tasks.workunit.client.0.smithi099.stdout:2: releasing exclusive lock 2024-09-13T14:40:15.717 INFO:tasks.workunit.client.0.smithi099.stdout:2: exclusive lock released 2024-09-13T14:40:15.744 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ExclusiveLock (554 ms) 2024-09-13T14:40:15.744 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.BreakLock 2024-09-13T14:40:15.781 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:16.227 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.BreakLock (483 ms) 2024-09-13T14:40:16.227 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.DiscardAfterWrite 2024-09-13T14:40:16.227 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:16.295 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.DiscardAfterWrite (68 ms) 2024-09-13T14:40:16.295 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.DefaultFeatures 2024-09-13T14:40:16.295 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.DefaultFeatures (0 ms) 2024-09-13T14:40:16.295 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurge 2024-09-13T14:40:16.296 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:16.436 INFO:tasks.workunit.client.0.smithi099.stdout:0.00% done 2024-09-13T14:40:16.436 INFO:tasks.workunit.client.0.smithi099.stdout:1.56% done 2024-09-13T14:40:16.436 INFO:tasks.workunit.client.0.smithi099.stdout:3.12% done 2024-09-13T14:40:16.437 INFO:tasks.workunit.client.0.smithi099.stdout:4.69% done 2024-09-13T14:40:16.437 INFO:tasks.workunit.client.0.smithi099.stdout:6.25% done 2024-09-13T14:40:16.437 INFO:tasks.workunit.client.0.smithi099.stdout:7.81% done 2024-09-13T14:40:16.437 INFO:tasks.workunit.client.0.smithi099.stdout:9.38% done 2024-09-13T14:40:16.437 INFO:tasks.workunit.client.0.smithi099.stdout:10.94% done 2024-09-13T14:40:16.438 INFO:tasks.workunit.client.0.smithi099.stdout:12.50% done 2024-09-13T14:40:16.438 INFO:tasks.workunit.client.0.smithi099.stdout:14.06% done 2024-09-13T14:40:16.438 INFO:tasks.workunit.client.0.smithi099.stdout:15.62% done 2024-09-13T14:40:16.438 INFO:tasks.workunit.client.0.smithi099.stdout:17.19% done 2024-09-13T14:40:16.439 INFO:tasks.workunit.client.0.smithi099.stdout:18.75% done 2024-09-13T14:40:16.439 INFO:tasks.workunit.client.0.smithi099.stdout:20.31% done 2024-09-13T14:40:16.439 INFO:tasks.workunit.client.0.smithi099.stdout:21.88% done 2024-09-13T14:40:16.439 INFO:tasks.workunit.client.0.smithi099.stdout:23.44% done 2024-09-13T14:40:16.440 INFO:tasks.workunit.client.0.smithi099.stdout:25.00% done 2024-09-13T14:40:16.440 INFO:tasks.workunit.client.0.smithi099.stdout:26.56% done 2024-09-13T14:40:16.440 INFO:tasks.workunit.client.0.smithi099.stdout:28.12% done 2024-09-13T14:40:16.440 INFO:tasks.workunit.client.0.smithi099.stdout:29.69% done 2024-09-13T14:40:16.440 INFO:tasks.workunit.client.0.smithi099.stdout:31.25% done 2024-09-13T14:40:16.441 INFO:tasks.workunit.client.0.smithi099.stdout:32.81% done 2024-09-13T14:40:16.441 INFO:tasks.workunit.client.0.smithi099.stdout:34.38% done 2024-09-13T14:40:16.441 INFO:tasks.workunit.client.0.smithi099.stdout:35.94% done 2024-09-13T14:40:16.441 INFO:tasks.workunit.client.0.smithi099.stdout:37.50% done 2024-09-13T14:40:16.442 INFO:tasks.workunit.client.0.smithi099.stdout:39.06% done 2024-09-13T14:40:16.442 INFO:tasks.workunit.client.0.smithi099.stdout:40.62% done 2024-09-13T14:40:16.442 INFO:tasks.workunit.client.0.smithi099.stdout:42.19% done 2024-09-13T14:40:16.442 INFO:tasks.workunit.client.0.smithi099.stdout:43.75% done 2024-09-13T14:40:16.442 INFO:tasks.workunit.client.0.smithi099.stdout:45.31% done 2024-09-13T14:40:16.443 INFO:tasks.workunit.client.0.smithi099.stdout:46.88% done 2024-09-13T14:40:16.443 INFO:tasks.workunit.client.0.smithi099.stdout:48.44% done 2024-09-13T14:40:16.443 INFO:tasks.workunit.client.0.smithi099.stdout:50.00% done 2024-09-13T14:40:16.443 INFO:tasks.workunit.client.0.smithi099.stdout:51.56% done 2024-09-13T14:40:16.443 INFO:tasks.workunit.client.0.smithi099.stdout:53.12% done 2024-09-13T14:40:16.444 INFO:tasks.workunit.client.0.smithi099.stdout:54.69% done 2024-09-13T14:40:16.444 INFO:tasks.workunit.client.0.smithi099.stdout:56.25% done 2024-09-13T14:40:16.444 INFO:tasks.workunit.client.0.smithi099.stdout:57.81% done 2024-09-13T14:40:16.444 INFO:tasks.workunit.client.0.smithi099.stdout:59.38% done 2024-09-13T14:40:16.445 INFO:tasks.workunit.client.0.smithi099.stdout:60.94% done 2024-09-13T14:40:16.445 INFO:tasks.workunit.client.0.smithi099.stdout:62.50% done 2024-09-13T14:40:16.445 INFO:tasks.workunit.client.0.smithi099.stdout:64.06% done 2024-09-13T14:40:16.445 INFO:tasks.workunit.client.0.smithi099.stdout:65.62% done 2024-09-13T14:40:16.445 INFO:tasks.workunit.client.0.smithi099.stdout:67.19% done 2024-09-13T14:40:16.446 INFO:tasks.workunit.client.0.smithi099.stdout:68.75% done 2024-09-13T14:40:16.446 INFO:tasks.workunit.client.0.smithi099.stdout:70.31% done 2024-09-13T14:40:16.446 INFO:tasks.workunit.client.0.smithi099.stdout:71.88% done 2024-09-13T14:40:16.446 INFO:tasks.workunit.client.0.smithi099.stdout:73.44% done 2024-09-13T14:40:16.446 INFO:tasks.workunit.client.0.smithi099.stdout:75.00% done 2024-09-13T14:40:16.447 INFO:tasks.workunit.client.0.smithi099.stdout:76.56% done 2024-09-13T14:40:16.447 INFO:tasks.workunit.client.0.smithi099.stdout:78.12% done 2024-09-13T14:40:16.447 INFO:tasks.workunit.client.0.smithi099.stdout:79.69% done 2024-09-13T14:40:16.447 INFO:tasks.workunit.client.0.smithi099.stdout:81.25% done 2024-09-13T14:40:16.447 INFO:tasks.workunit.client.0.smithi099.stdout:82.81% done 2024-09-13T14:40:16.448 INFO:tasks.workunit.client.0.smithi099.stdout:84.38% done 2024-09-13T14:40:16.448 INFO:tasks.workunit.client.0.smithi099.stdout:85.94% done 2024-09-13T14:40:16.448 INFO:tasks.workunit.client.0.smithi099.stdout:87.50% done 2024-09-13T14:40:16.448 INFO:tasks.workunit.client.0.smithi099.stdout:89.06% done 2024-09-13T14:40:16.448 INFO:tasks.workunit.client.0.smithi099.stdout:90.62% done 2024-09-13T14:40:16.448 INFO:tasks.workunit.client.0.smithi099.stdout:92.19% done 2024-09-13T14:40:16.449 INFO:tasks.workunit.client.0.smithi099.stdout:93.75% done 2024-09-13T14:40:16.449 INFO:tasks.workunit.client.0.smithi099.stdout:95.31% done 2024-09-13T14:40:16.449 INFO:tasks.workunit.client.0.smithi099.stdout:96.88% done 2024-09-13T14:40:16.449 INFO:tasks.workunit.client.0.smithi099.stdout:98.44% done 2024-09-13T14:40:16.450 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurge (140 ms) 2024-09-13T14:40:16.450 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay 2024-09-13T14:40:16.450 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:16.557 INFO:tasks.workunit.client.0.smithi099.stdout:0.00% done 2024-09-13T14:40:16.557 INFO:tasks.workunit.client.0.smithi099.stdout:1.56% done 2024-09-13T14:40:16.557 INFO:tasks.workunit.client.0.smithi099.stdout:3.12% done 2024-09-13T14:40:16.558 INFO:tasks.workunit.client.0.smithi099.stdout:4.69% done 2024-09-13T14:40:16.558 INFO:tasks.workunit.client.0.smithi099.stdout:6.25% done 2024-09-13T14:40:16.558 INFO:tasks.workunit.client.0.smithi099.stdout:7.81% done 2024-09-13T14:40:16.558 INFO:tasks.workunit.client.0.smithi099.stdout:9.38% done 2024-09-13T14:40:16.558 INFO:tasks.workunit.client.0.smithi099.stdout:10.94% done 2024-09-13T14:40:16.559 INFO:tasks.workunit.client.0.smithi099.stdout:12.50% done 2024-09-13T14:40:16.559 INFO:tasks.workunit.client.0.smithi099.stdout:14.06% done 2024-09-13T14:40:16.559 INFO:tasks.workunit.client.0.smithi099.stdout:15.62% done 2024-09-13T14:40:16.559 INFO:tasks.workunit.client.0.smithi099.stdout:17.19% done 2024-09-13T14:40:16.560 INFO:tasks.workunit.client.0.smithi099.stdout:18.75% done 2024-09-13T14:40:16.560 INFO:tasks.workunit.client.0.smithi099.stdout:20.31% done 2024-09-13T14:40:16.560 INFO:tasks.workunit.client.0.smithi099.stdout:21.88% done 2024-09-13T14:40:16.560 INFO:tasks.workunit.client.0.smithi099.stdout:23.44% done 2024-09-13T14:40:16.560 INFO:tasks.workunit.client.0.smithi099.stdout:25.00% done 2024-09-13T14:40:16.561 INFO:tasks.workunit.client.0.smithi099.stdout:26.56% done 2024-09-13T14:40:16.561 INFO:tasks.workunit.client.0.smithi099.stdout:28.12% done 2024-09-13T14:40:16.561 INFO:tasks.workunit.client.0.smithi099.stdout:29.69% done 2024-09-13T14:40:16.561 INFO:tasks.workunit.client.0.smithi099.stdout:31.25% done 2024-09-13T14:40:16.562 INFO:tasks.workunit.client.0.smithi099.stdout:32.81% done 2024-09-13T14:40:16.562 INFO:tasks.workunit.client.0.smithi099.stdout:34.38% done 2024-09-13T14:40:16.562 INFO:tasks.workunit.client.0.smithi099.stdout:35.94% done 2024-09-13T14:40:16.562 INFO:tasks.workunit.client.0.smithi099.stdout:37.50% done 2024-09-13T14:40:16.562 INFO:tasks.workunit.client.0.smithi099.stdout:39.06% done 2024-09-13T14:40:16.563 INFO:tasks.workunit.client.0.smithi099.stdout:40.62% done 2024-09-13T14:40:16.563 INFO:tasks.workunit.client.0.smithi099.stdout:42.19% done 2024-09-13T14:40:16.563 INFO:tasks.workunit.client.0.smithi099.stdout:43.75% done 2024-09-13T14:40:16.563 INFO:tasks.workunit.client.0.smithi099.stdout:45.31% done 2024-09-13T14:40:16.563 INFO:tasks.workunit.client.0.smithi099.stdout:46.88% done 2024-09-13T14:40:16.564 INFO:tasks.workunit.client.0.smithi099.stdout:48.44% done 2024-09-13T14:40:16.564 INFO:tasks.workunit.client.0.smithi099.stdout:50.00% done 2024-09-13T14:40:16.564 INFO:tasks.workunit.client.0.smithi099.stdout:51.56% done 2024-09-13T14:40:16.564 INFO:tasks.workunit.client.0.smithi099.stdout:53.12% done 2024-09-13T14:40:16.565 INFO:tasks.workunit.client.0.smithi099.stdout:54.69% done 2024-09-13T14:40:16.565 INFO:tasks.workunit.client.0.smithi099.stdout:56.25% done 2024-09-13T14:40:16.565 INFO:tasks.workunit.client.0.smithi099.stdout:57.81% done 2024-09-13T14:40:16.565 INFO:tasks.workunit.client.0.smithi099.stdout:59.38% done 2024-09-13T14:40:16.565 INFO:tasks.workunit.client.0.smithi099.stdout:60.94% done 2024-09-13T14:40:16.566 INFO:tasks.workunit.client.0.smithi099.stdout:62.50% done 2024-09-13T14:40:16.566 INFO:tasks.workunit.client.0.smithi099.stdout:64.06% done 2024-09-13T14:40:16.566 INFO:tasks.workunit.client.0.smithi099.stdout:65.62% done 2024-09-13T14:40:16.566 INFO:tasks.workunit.client.0.smithi099.stdout:67.19% done 2024-09-13T14:40:16.567 INFO:tasks.workunit.client.0.smithi099.stdout:68.75% done 2024-09-13T14:40:16.567 INFO:tasks.workunit.client.0.smithi099.stdout:70.31% done 2024-09-13T14:40:16.567 INFO:tasks.workunit.client.0.smithi099.stdout:71.88% done 2024-09-13T14:40:16.567 INFO:tasks.workunit.client.0.smithi099.stdout:73.44% done 2024-09-13T14:40:16.567 INFO:tasks.workunit.client.0.smithi099.stdout:75.00% done 2024-09-13T14:40:16.567 INFO:tasks.workunit.client.0.smithi099.stdout:76.56% done 2024-09-13T14:40:16.568 INFO:tasks.workunit.client.0.smithi099.stdout:78.12% done 2024-09-13T14:40:16.568 INFO:tasks.workunit.client.0.smithi099.stdout:79.69% done 2024-09-13T14:40:16.568 INFO:tasks.workunit.client.0.smithi099.stdout:81.25% done 2024-09-13T14:40:16.568 INFO:tasks.workunit.client.0.smithi099.stdout:82.81% done 2024-09-13T14:40:16.568 INFO:tasks.workunit.client.0.smithi099.stdout:84.38% done 2024-09-13T14:40:16.568 INFO:tasks.workunit.client.0.smithi099.stdout:85.94% done 2024-09-13T14:40:16.569 INFO:tasks.workunit.client.0.smithi099.stdout:87.50% done 2024-09-13T14:40:16.569 INFO:tasks.workunit.client.0.smithi099.stdout:89.06% done 2024-09-13T14:40:16.569 INFO:tasks.workunit.client.0.smithi099.stdout:90.62% done 2024-09-13T14:40:16.569 INFO:tasks.workunit.client.0.smithi099.stdout:92.19% done 2024-09-13T14:40:16.569 INFO:tasks.workunit.client.0.smithi099.stdout:93.75% done 2024-09-13T14:40:16.569 INFO:tasks.workunit.client.0.smithi099.stdout:95.31% done 2024-09-13T14:40:16.569 INFO:tasks.workunit.client.0.smithi099.stdout:96.88% done 2024-09-13T14:40:16.570 INFO:tasks.workunit.client.0.smithi099.stdout:98.44% done 2024-09-13T14:40:16.570 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay (121 ms) 2024-09-13T14:40:16.570 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestTrashPurge 2024-09-13T14:40:16.570 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:16.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[84576]: cluster 2024-09-13T14:40:15.123716+0000 mon.a (mon.0) 1206 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 178/2973 objects degraded (5.987%), 20 pgs degraded) 2024-09-13T14:40:16.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[84576]: cluster 2024-09-13T14:40:15.123716+0000 mon.a (mon.0) 1206 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 178/2973 objects degraded (5.987%), 20 pgs degraded) 2024-09-13T14:40:16.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[84576]: cluster 2024-09-13T14:40:15.123788+0000 mon.a (mon.0) 1207 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:16.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[84576]: cluster 2024-09-13T14:40:15.123788+0000 mon.a (mon.0) 1207 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:16.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[84576]: cluster 2024-09-13T14:40:15.146018+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-13T14:40:16.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[84576]: cluster 2024-09-13T14:40:15.146018+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-13T14:40:16.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[84576]: audit 2024-09-13T14:40:15.846327+0000 mon.a (mon.0) 1209 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"} : dispatch 2024-09-13T14:40:16.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[84576]: audit 2024-09-13T14:40:15.846327+0000 mon.a (mon.0) 1209 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"} : dispatch 2024-09-13T14:40:16.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[87147]: cluster 2024-09-13T14:40:15.123716+0000 mon.a (mon.0) 1206 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 178/2973 objects degraded (5.987%), 20 pgs degraded) 2024-09-13T14:40:16.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[87147]: cluster 2024-09-13T14:40:15.123716+0000 mon.a (mon.0) 1206 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 178/2973 objects degraded (5.987%), 20 pgs degraded) 2024-09-13T14:40:16.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[87147]: cluster 2024-09-13T14:40:15.123788+0000 mon.a (mon.0) 1207 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:16.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[87147]: cluster 2024-09-13T14:40:15.123788+0000 mon.a (mon.0) 1207 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:16.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[87147]: cluster 2024-09-13T14:40:15.146018+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-13T14:40:16.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[87147]: cluster 2024-09-13T14:40:15.146018+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-13T14:40:16.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[87147]: audit 2024-09-13T14:40:15.846327+0000 mon.a (mon.0) 1209 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"} : dispatch 2024-09-13T14:40:16.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:16 smithi099 bash[87147]: audit 2024-09-13T14:40:15.846327+0000 mon.a (mon.0) 1209 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"} : dispatch 2024-09-13T14:40:16.576 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:16.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:16 smithi177 bash[68594]: cluster 2024-09-13T14:40:15.123716+0000 mon.a (mon.0) 1206 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 178/2973 objects degraded (5.987%), 20 pgs degraded) 2024-09-13T14:40:16.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:16 smithi177 bash[68594]: cluster 2024-09-13T14:40:15.123716+0000 mon.a (mon.0) 1206 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 178/2973 objects degraded (5.987%), 20 pgs degraded) 2024-09-13T14:40:16.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:16 smithi177 bash[68594]: cluster 2024-09-13T14:40:15.123788+0000 mon.a (mon.0) 1207 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:16.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:16 smithi177 bash[68594]: cluster 2024-09-13T14:40:15.123788+0000 mon.a (mon.0) 1207 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:16.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:16 smithi177 bash[68594]: cluster 2024-09-13T14:40:15.146018+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-13T14:40:16.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:16 smithi177 bash[68594]: cluster 2024-09-13T14:40:15.146018+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-13T14:40:16.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:16 smithi177 bash[68594]: audit 2024-09-13T14:40:15.846327+0000 mon.a (mon.0) 1209 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"} : dispatch 2024-09-13T14:40:16.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:16 smithi177 bash[68594]: audit 2024-09-13T14:40:15.846327+0000 mon.a (mon.0) 1209 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"} : dispatch 2024-09-13T14:40:16.829 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestTrashPurge (272 ms) 2024-09-13T14:40:16.829 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndRestore 2024-09-13T14:40:16.829 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:16.887 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestTrashMoveAndRestore (59 ms) 2024-09-13T14:40:16.887 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestListWatchers 2024-09-13T14:40:16.887 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:16.920 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestListWatchers (33 ms) 2024-09-13T14:40:16.920 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestSetSnapById 2024-09-13T14:40:16.920 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:40:17] "GET /metrics HTTP/1.1" 200 75188 "" "Prometheus/2.51.0" 2024-09-13T14:40:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.138671+0000 mon.a (mon.0) 1210 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"}]': finished 2024-09-13T14:40:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.138671+0000 mon.a (mon.0) 1210 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"}]': finished 2024-09-13T14:40:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: cluster 2024-09-13T14:40:16.145712+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-13T14:40:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: cluster 2024-09-13T14:40:16.145712+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-13T14:40:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: cluster 2024-09-13T14:40:16.220553+0000 mgr.y (mgr.26252) 413 : cluster [DBG] pgmap v660: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 9.5 KiB/s wr, 156 op/s; 591 KiB/s, 28 keys/s, 36 objects/s recovering 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: cluster 2024-09-13T14:40:16.220553+0000 mgr.y (mgr.26252) 413 : cluster [DBG] pgmap v660: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 9.5 KiB/s wr, 156 op/s; 591 KiB/s, 28 keys/s, 36 objects/s recovering 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.649968+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.649968+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.662090+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.662090+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.664688+0000 mon.a (mon.0) 1214 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.664688+0000 mon.a (mon.0) 1214 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.666941+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.666941+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.679328+0000 mon.a (mon.0) 1216 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.679328+0000 mon.a (mon.0) 1216 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.799248+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:17.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.799248+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:17.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.801442+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.801442+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.802353+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.802353+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.808661+0000 mon.a (mon.0) 1220 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[87147]: audit 2024-09-13T14:40:16.808661+0000 mon.a (mon.0) 1220 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.138671+0000 mon.a (mon.0) 1210 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"}]': finished 2024-09-13T14:40:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.138671+0000 mon.a (mon.0) 1210 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"}]': finished 2024-09-13T14:40:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: cluster 2024-09-13T14:40:16.145712+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: cluster 2024-09-13T14:40:16.145712+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: cluster 2024-09-13T14:40:16.220553+0000 mgr.y (mgr.26252) 413 : cluster [DBG] pgmap v660: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 9.5 KiB/s wr, 156 op/s; 591 KiB/s, 28 keys/s, 36 objects/s recovering 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: cluster 2024-09-13T14:40:16.220553+0000 mgr.y (mgr.26252) 413 : cluster [DBG] pgmap v660: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 9.5 KiB/s wr, 156 op/s; 591 KiB/s, 28 keys/s, 36 objects/s recovering 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.649968+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.649968+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.662090+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.662090+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.664688+0000 mon.a (mon.0) 1214 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.664688+0000 mon.a (mon.0) 1214 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.666941+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.666941+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.679328+0000 mon.a (mon.0) 1216 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.679328+0000 mon.a (mon.0) 1216 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.799248+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:17.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.799248+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:17.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.801442+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.801442+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.802353+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.802353+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.808661+0000 mon.a (mon.0) 1220 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:17 smithi099 bash[84576]: audit 2024-09-13T14:40:16.808661+0000 mon.a (mon.0) 1220 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.138671+0000 mon.a (mon.0) 1210 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"}]': finished 2024-09-13T14:40:17.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.138671+0000 mon.a (mon.0) 1210 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.99:0/3935330909"}]': finished 2024-09-13T14:40:17.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: cluster 2024-09-13T14:40:16.145712+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-13T14:40:17.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: cluster 2024-09-13T14:40:16.145712+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-13T14:40:17.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: cluster 2024-09-13T14:40:16.220553+0000 mgr.y (mgr.26252) 413 : cluster [DBG] pgmap v660: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 9.5 KiB/s wr, 156 op/s; 591 KiB/s, 28 keys/s, 36 objects/s recovering 2024-09-13T14:40:17.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: cluster 2024-09-13T14:40:16.220553+0000 mgr.y (mgr.26252) 413 : cluster [DBG] pgmap v660: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 9.5 KiB/s wr, 156 op/s; 591 KiB/s, 28 keys/s, 36 objects/s recovering 2024-09-13T14:40:17.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.649968+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.649968+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.662090+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.662090+0000 mon.a (mon.0) 1213 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.664688+0000 mon.a (mon.0) 1214 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.664688+0000 mon.a (mon.0) 1214 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.666941+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.666941+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.679328+0000 mon.a (mon.0) 1216 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.679328+0000 mon.a (mon.0) 1216 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.799248+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.799248+0000 mon.a (mon.0) 1217 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.801442+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.801442+0000 mon.a (mon.0) 1218 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.802353+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.802353+0000 mon.a (mon.0) 1219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:17.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.808661+0000 mon.a (mon.0) 1220 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:17 smithi177 bash[68594]: audit 2024-09-13T14:40:16.808661+0000 mon.a (mon.0) 1220 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:17.895 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestSetSnapById (975 ms) 2024-09-13T14:40:17.895 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.Namespaces 2024-09-13T14:40:17.941 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.Namespaces (46 ms) 2024-09-13T14:40:17.942 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.NamespacesPP 2024-09-13T14:40:18.008 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:18.017 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:18.153 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.NamespacesPP (212 ms) 2024-09-13T14:40:18.153 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.Migration 2024-09-13T14:40:18.153 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:18.153 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:19.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[87147]: cluster 2024-09-13T14:40:17.816307+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-13T14:40:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[87147]: cluster 2024-09-13T14:40:17.816307+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-13T14:40:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[87147]: cluster 2024-09-13T14:40:18.221825+0000 mgr.y (mgr.26252) 414 : cluster [DBG] pgmap v662: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 131 KiB/s wr, 168 op/s 2024-09-13T14:40:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[87147]: cluster 2024-09-13T14:40:18.221825+0000 mgr.y (mgr.26252) 414 : cluster [DBG] pgmap v662: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 131 KiB/s wr, 168 op/s 2024-09-13T14:40:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[87147]: audit 2024-09-13T14:40:18.256834+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[87147]: audit 2024-09-13T14:40:18.256834+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[87147]: audit 2024-09-13T14:40:18.258531+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:19.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[87147]: audit 2024-09-13T14:40:18.258531+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:19.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[84576]: cluster 2024-09-13T14:40:17.816307+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-13T14:40:19.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[84576]: cluster 2024-09-13T14:40:17.816307+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-13T14:40:19.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[84576]: cluster 2024-09-13T14:40:18.221825+0000 mgr.y (mgr.26252) 414 : cluster [DBG] pgmap v662: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 131 KiB/s wr, 168 op/s 2024-09-13T14:40:19.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[84576]: cluster 2024-09-13T14:40:18.221825+0000 mgr.y (mgr.26252) 414 : cluster [DBG] pgmap v662: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 131 KiB/s wr, 168 op/s 2024-09-13T14:40:19.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[84576]: audit 2024-09-13T14:40:18.256834+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:19.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[84576]: audit 2024-09-13T14:40:18.256834+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:19.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[84576]: audit 2024-09-13T14:40:18.258531+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:19.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:18 smithi099 bash[84576]: audit 2024-09-13T14:40:18.258531+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:19.089 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.Migration (936 ms) 2024-09-13T14:40:19.089 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.MigrationPP 2024-09-13T14:40:19.090 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:19.090 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:19.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:18 smithi177 bash[68594]: cluster 2024-09-13T14:40:17.816307+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-13T14:40:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:18 smithi177 bash[68594]: cluster 2024-09-13T14:40:17.816307+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-13T14:40:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:18 smithi177 bash[68594]: cluster 2024-09-13T14:40:18.221825+0000 mgr.y (mgr.26252) 414 : cluster [DBG] pgmap v662: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 131 KiB/s wr, 168 op/s 2024-09-13T14:40:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:18 smithi177 bash[68594]: cluster 2024-09-13T14:40:18.221825+0000 mgr.y (mgr.26252) 414 : cluster [DBG] pgmap v662: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 131 KiB/s wr, 168 op/s 2024-09-13T14:40:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:18 smithi177 bash[68594]: audit 2024-09-13T14:40:18.256834+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:18 smithi177 bash[68594]: audit 2024-09-13T14:40:18.256834+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:18 smithi177 bash[68594]: audit 2024-09-13T14:40:18.258531+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:19.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:18 smithi177 bash[68594]: audit 2024-09-13T14:40:18.258531+0000 mon.a (mon.0) 1223 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: "mon": { 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr": { 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: "osd": { 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3, 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 5 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:40:19.767 INFO:teuthology.orchestra.run.smithi099.stdout: "mds": { 2024-09-13T14:40:19.768 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T14:40:19.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:40:19.768 INFO:teuthology.orchestra.run.smithi099.stdout: "overall": { 2024-09-13T14:40:19.768 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 5, 2024-09-13T14:40:19.768 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 10 2024-09-13T14:40:19.768 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:40:19.768 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:40:20.043 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:19 smithi099 bash[84576]: audit 2024-09-13T14:40:19.769217+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.99:0/2027687467' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:20.043 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:19 smithi099 bash[84576]: audit 2024-09-13T14:40:19.769217+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.99:0/2027687467' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:20.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:19 smithi099 bash[87147]: audit 2024-09-13T14:40:19.769217+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.99:0/2027687467' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:20.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:19 smithi099 bash[87147]: audit 2024-09-13T14:40:19.769217+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.99:0/2027687467' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:20.048 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.MigrationPP (959 ms) 2024-09-13T14:40:20.048 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestGetAccessTimestamp 2024-09-13T14:40:20.048 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:20.091 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestGetAccessTimestamp (43 ms) 2024-09-13T14:40:20.091 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.TestGetModifyTimestamp 2024-09-13T14:40:20.091 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:20.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:19 smithi177 bash[68594]: audit 2024-09-13T14:40:19.769217+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.99:0/2027687467' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:20.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:19 smithi177 bash[68594]: audit 2024-09-13T14:40:19.769217+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.99:0/2027687467' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:20.130 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.TestGetModifyTimestamp (39 ms) 2024-09-13T14:40:20.130 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ZeroOverlapFlatten 2024-09-13T14:40:20.130 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:20.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:40:20 smithi177 bash[61439]: ts=2024-09-13T14:40:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:20.798 DEBUG:teuthology.parallel:result is None 2024-09-13T14:40:21.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:20 smithi099 bash[84576]: cluster 2024-09-13T14:40:20.225333+0000 mgr.y (mgr.26252) 415 : cluster [DBG] pgmap v663: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 260 KiB/s rd, 103 KiB/s wr, 323 op/s 2024-09-13T14:40:21.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:20 smithi099 bash[84576]: cluster 2024-09-13T14:40:20.225333+0000 mgr.y (mgr.26252) 415 : cluster [DBG] pgmap v663: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 260 KiB/s rd, 103 KiB/s wr, 323 op/s 2024-09-13T14:40:21.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:20 smithi099 bash[87147]: cluster 2024-09-13T14:40:20.225333+0000 mgr.y (mgr.26252) 415 : cluster [DBG] pgmap v663: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 260 KiB/s rd, 103 KiB/s wr, 323 op/s 2024-09-13T14:40:21.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:20 smithi099 bash[87147]: cluster 2024-09-13T14:40:20.225333+0000 mgr.y (mgr.26252) 415 : cluster [DBG] pgmap v663: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 260 KiB/s rd, 103 KiB/s wr, 323 op/s 2024-09-13T14:40:21.082 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ZeroOverlapFlatten (952 ms) 2024-09-13T14:40:21.082 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.PoolMetadata 2024-09-13T14:40:21.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:20 smithi177 bash[68594]: cluster 2024-09-13T14:40:20.225333+0000 mgr.y (mgr.26252) 415 : cluster [DBG] pgmap v663: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 260 KiB/s rd, 103 KiB/s wr, 323 op/s 2024-09-13T14:40:21.108 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:20 smithi177 bash[68594]: cluster 2024-09-13T14:40:20.225333+0000 mgr.y (mgr.26252) 415 : cluster [DBG] pgmap v663: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 260 KiB/s rd, 103 KiB/s wr, 323 op/s 2024-09-13T14:40:21.179 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.PoolMetadata (97 ms) 2024-09-13T14:40:21.179 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.PoolMetadataPP 2024-09-13T14:40:21.207 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.PoolMetadataPP (28 ms) 2024-09-13T14:40:21.207 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.Config 2024-09-13T14:40:21.213 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:21.293 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.Config (86 ms) 2024-09-13T14:40:21.293 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ConfigPP 2024-09-13T14:40:21.299 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:21.354 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ConfigPP (62 ms) 2024-09-13T14:40:21.355 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.PoolStatsPP 2024-09-13T14:40:21.555 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[76324]: [13/Sep/2024:14:40:21] ENGINE Bus STOPPING 2024-09-13T14:40:21.822 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[76324]: [13/Sep/2024:14:40:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:40:21.822 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[76324]: [13/Sep/2024:14:40:21] ENGINE Bus STOPPED 2024-09-13T14:40:21.822 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[76324]: [13/Sep/2024:14:40:21] ENGINE Bus STARTING 2024-09-13T14:40:21.822 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[76324]: [13/Sep/2024:14:40:21] ENGINE Serving on http://:::9283 2024-09-13T14:40:21.822 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[76324]: [13/Sep/2024:14:40:21] ENGINE Bus STARTED 2024-09-13T14:40:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: cluster 2024-09-13T14:40:20.841208+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-13T14:40:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: cluster 2024-09-13T14:40:20.841208+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-13T14:40:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.119241+0000 mon.a (mon.0) 1226 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:40:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.119241+0000 mon.a (mon.0) 1226 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:40:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: cluster 2024-09-13T14:40:21.140434+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-13T14:40:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: cluster 2024-09-13T14:40:21.140434+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-13T14:40:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.155654+0000 mon.a (mon.0) 1228 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:22.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.155654+0000 mon.a (mon.0) 1228 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.162713+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.162713+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.165034+0000 mon.a (mon.0) 1230 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.165034+0000 mon.a (mon.0) 1230 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.176157+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.176157+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.559381+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:40:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.559381+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:40:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.561140+0000 mgr.y (mgr.26252) 416 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:40:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:21 smithi177 bash[68594]: audit 2024-09-13T14:40:21.561140+0000 mgr.y (mgr.26252) 416 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:40:22.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: cluster 2024-09-13T14:40:20.841208+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-13T14:40:22.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: cluster 2024-09-13T14:40:20.841208+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-13T14:40:22.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.119241+0000 mon.a (mon.0) 1226 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:40:22.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.119241+0000 mon.a (mon.0) 1226 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:40:22.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: cluster 2024-09-13T14:40:21.140434+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-13T14:40:22.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: cluster 2024-09-13T14:40:21.140434+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-13T14:40:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.155654+0000 mon.a (mon.0) 1228 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.155654+0000 mon.a (mon.0) 1228 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.162713+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.162713+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.165034+0000 mon.a (mon.0) 1230 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.165034+0000 mon.a (mon.0) 1230 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.176157+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.176157+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:22.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.559381+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:40:22.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.559381+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:40:22.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.561140+0000 mgr.y (mgr.26252) 416 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:40:22.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[84576]: audit 2024-09-13T14:40:21.561140+0000 mgr.y (mgr.26252) 416 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:40:22.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: cluster 2024-09-13T14:40:20.841208+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-13T14:40:22.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: cluster 2024-09-13T14:40:20.841208+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-13T14:40:22.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.119241+0000 mon.a (mon.0) 1226 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:40:22.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.119241+0000 mon.a (mon.0) 1226 : audit [INF] from='client.? 172.21.15.99:0/1935692935' entity='client.admin' 2024-09-13T14:40:22.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: cluster 2024-09-13T14:40:21.140434+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-13T14:40:22.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: cluster 2024-09-13T14:40:21.140434+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-13T14:40:22.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.155654+0000 mon.a (mon.0) 1228 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:22.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.155654+0000 mon.a (mon.0) 1228 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:22.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.162713+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:22.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.162713+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:22.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.165034+0000 mon.a (mon.0) 1230 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:22.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.165034+0000 mon.a (mon.0) 1230 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:22.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.176157+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:22.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.176157+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:22.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.559381+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:40:22.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.559381+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:40:22.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.561140+0000 mgr.y (mgr.26252) 416 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:40:22.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:21 smithi099 bash[87147]: audit 2024-09-13T14:40:21.561140+0000 mgr.y (mgr.26252) 416 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:40:23.210 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[84576]: cluster 2024-09-13T14:40:22.185063+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-13T14:40:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[84576]: cluster 2024-09-13T14:40:22.185063+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-13T14:40:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[84576]: audit 2024-09-13T14:40:22.194748+0000 mon.c (mon.2) 40 : audit [INF] from='client.? 172.21.15.99:0/1912358925' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[84576]: audit 2024-09-13T14:40:22.194748+0000 mon.c (mon.2) 40 : audit [INF] from='client.? 172.21.15.99:0/1912358925' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[84576]: audit 2024-09-13T14:40:22.195984+0000 mon.a (mon.0) 1234 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[84576]: audit 2024-09-13T14:40:22.195984+0000 mon.a (mon.0) 1234 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[84576]: cluster 2024-09-13T14:40:22.227483+0000 mgr.y (mgr.26252) 417 : cluster [DBG] pgmap v667: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 318 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[84576]: cluster 2024-09-13T14:40:22.227483+0000 mgr.y (mgr.26252) 417 : cluster [DBG] pgmap v667: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 318 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[87147]: cluster 2024-09-13T14:40:22.185063+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[87147]: cluster 2024-09-13T14:40:22.185063+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[87147]: audit 2024-09-13T14:40:22.194748+0000 mon.c (mon.2) 40 : audit [INF] from='client.? 172.21.15.99:0/1912358925' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[87147]: audit 2024-09-13T14:40:22.194748+0000 mon.c (mon.2) 40 : audit [INF] from='client.? 172.21.15.99:0/1912358925' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[87147]: audit 2024-09-13T14:40:22.195984+0000 mon.a (mon.0) 1234 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[87147]: audit 2024-09-13T14:40:22.195984+0000 mon.a (mon.0) 1234 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[87147]: cluster 2024-09-13T14:40:22.227483+0000 mgr.y (mgr.26252) 417 : cluster [DBG] pgmap v667: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 318 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-13T14:40:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:23 smithi099 bash[87147]: cluster 2024-09-13T14:40:22.227483+0000 mgr.y (mgr.26252) 417 : cluster [DBG] pgmap v667: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 318 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-13T14:40:23.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:23 smithi177 bash[68594]: cluster 2024-09-13T14:40:22.185063+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-13T14:40:23.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:23 smithi177 bash[68594]: cluster 2024-09-13T14:40:22.185063+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-13T14:40:23.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:23 smithi177 bash[68594]: audit 2024-09-13T14:40:22.194748+0000 mon.c (mon.2) 40 : audit [INF] from='client.? 172.21.15.99:0/1912358925' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:23 smithi177 bash[68594]: audit 2024-09-13T14:40:22.194748+0000 mon.c (mon.2) 40 : audit [INF] from='client.? 172.21.15.99:0/1912358925' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:23 smithi177 bash[68594]: audit 2024-09-13T14:40:22.195984+0000 mon.a (mon.0) 1234 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:23 smithi177 bash[68594]: audit 2024-09-13T14:40:22.195984+0000 mon.a (mon.0) 1234 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:23.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:23 smithi177 bash[68594]: cluster 2024-09-13T14:40:22.227483+0000 mgr.y (mgr.26252) 417 : cluster [DBG] pgmap v667: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 318 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-13T14:40:23.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:23 smithi177 bash[68594]: cluster 2024-09-13T14:40:22.227483+0000 mgr.y (mgr.26252) 417 : cluster [DBG] pgmap v667: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 318 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-13T14:40:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[84576]: cluster 2024-09-13T14:40:23.176704+0000 mon.a (mon.0) 1235 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[84576]: cluster 2024-09-13T14:40:23.176704+0000 mon.a (mon.0) 1235 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[84576]: audit 2024-09-13T14:40:23.190501+0000 mon.a (mon.0) 1236 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[84576]: audit 2024-09-13T14:40:23.190501+0000 mon.a (mon.0) 1236 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[84576]: cluster 2024-09-13T14:40:23.205932+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-13T14:40:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[84576]: cluster 2024-09-13T14:40:23.205932+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-13T14:40:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[84576]: audit 2024-09-13T14:40:23.876737+0000 mon.a (mon.0) 1238 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[84576]: audit 2024-09-13T14:40:23.876737+0000 mon.a (mon.0) 1238 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[87147]: cluster 2024-09-13T14:40:23.176704+0000 mon.a (mon.0) 1235 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[87147]: cluster 2024-09-13T14:40:23.176704+0000 mon.a (mon.0) 1235 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[87147]: audit 2024-09-13T14:40:23.190501+0000 mon.a (mon.0) 1236 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[87147]: audit 2024-09-13T14:40:23.190501+0000 mon.a (mon.0) 1236 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[87147]: cluster 2024-09-13T14:40:23.205932+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-13T14:40:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[87147]: cluster 2024-09-13T14:40:23.205932+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-13T14:40:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[87147]: audit 2024-09-13T14:40:23.876737+0000 mon.a (mon.0) 1238 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:24 smithi099 bash[87147]: audit 2024-09-13T14:40:23.876737+0000 mon.a (mon.0) 1238 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:24.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:24 smithi177 bash[68594]: cluster 2024-09-13T14:40:23.176704+0000 mon.a (mon.0) 1235 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:24.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:24 smithi177 bash[68594]: cluster 2024-09-13T14:40:23.176704+0000 mon.a (mon.0) 1235 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:24.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:24 smithi177 bash[68594]: audit 2024-09-13T14:40:23.190501+0000 mon.a (mon.0) 1236 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:24.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:24 smithi177 bash[68594]: audit 2024-09-13T14:40:23.190501+0000 mon.a (mon.0) 1236 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:24.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:24 smithi177 bash[68594]: cluster 2024-09-13T14:40:23.205932+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-13T14:40:24.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:24 smithi177 bash[68594]: cluster 2024-09-13T14:40:23.205932+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-13T14:40:24.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:24 smithi177 bash[68594]: audit 2024-09-13T14:40:23.876737+0000 mon.a (mon.0) 1238 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:24.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:24 smithi177 bash[68594]: audit 2024-09-13T14:40:23.876737+0000 mon.a (mon.0) 1238 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:25.291 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:25.316 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:25.331 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:25 smithi099 bash[84576]: cluster 2024-09-13T14:40:24.202240+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-13T14:40:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:25 smithi099 bash[84576]: cluster 2024-09-13T14:40:24.202240+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-13T14:40:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:25 smithi099 bash[84576]: cluster 2024-09-13T14:40:24.228854+0000 mgr.y (mgr.26252) 418 : cluster [DBG] pgmap v670: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 742 KiB/s rd, 42 KiB/s wr, 939 op/s 2024-09-13T14:40:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:25 smithi099 bash[84576]: cluster 2024-09-13T14:40:24.228854+0000 mgr.y (mgr.26252) 418 : cluster [DBG] pgmap v670: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 742 KiB/s rd, 42 KiB/s wr, 939 op/s 2024-09-13T14:40:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:25 smithi099 bash[87147]: cluster 2024-09-13T14:40:24.202240+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-13T14:40:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:25 smithi099 bash[87147]: cluster 2024-09-13T14:40:24.202240+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-13T14:40:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:25 smithi099 bash[87147]: cluster 2024-09-13T14:40:24.228854+0000 mgr.y (mgr.26252) 418 : cluster [DBG] pgmap v670: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 742 KiB/s rd, 42 KiB/s wr, 939 op/s 2024-09-13T14:40:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:25 smithi099 bash[87147]: cluster 2024-09-13T14:40:24.228854+0000 mgr.y (mgr.26252) 418 : cluster [DBG] pgmap v670: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 742 KiB/s rd, 42 KiB/s wr, 939 op/s 2024-09-13T14:40:25.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:25 smithi177 bash[68594]: cluster 2024-09-13T14:40:24.202240+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-13T14:40:25.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:25 smithi177 bash[68594]: cluster 2024-09-13T14:40:24.202240+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-13T14:40:25.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:25 smithi177 bash[68594]: cluster 2024-09-13T14:40:24.228854+0000 mgr.y (mgr.26252) 418 : cluster [DBG] pgmap v670: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 742 KiB/s rd, 42 KiB/s wr, 939 op/s 2024-09-13T14:40:25.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:25 smithi177 bash[68594]: cluster 2024-09-13T14:40:24.228854+0000 mgr.y (mgr.26252) 418 : cluster [DBG] pgmap v670: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 742 KiB/s rd, 42 KiB/s wr, 939 op/s 2024-09-13T14:40:26.421 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.PoolStatsPP (5066 ms) 2024-09-13T14:40:26.421 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ImageSpec 2024-09-13T14:40:26.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[84576]: cluster 2024-09-13T14:40:25.208808+0000 mon.a (mon.0) 1240 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:26.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[84576]: cluster 2024-09-13T14:40:25.208808+0000 mon.a (mon.0) 1240 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:26.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[84576]: cluster 2024-09-13T14:40:25.208891+0000 mon.a (mon.0) 1241 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:26.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[84576]: cluster 2024-09-13T14:40:25.208891+0000 mon.a (mon.0) 1241 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:26.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[84576]: cluster 2024-09-13T14:40:25.233550+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-13T14:40:26.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[84576]: cluster 2024-09-13T14:40:25.233550+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-13T14:40:26.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[87147]: cluster 2024-09-13T14:40:25.208808+0000 mon.a (mon.0) 1240 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:26.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[87147]: cluster 2024-09-13T14:40:25.208808+0000 mon.a (mon.0) 1240 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:26.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[87147]: cluster 2024-09-13T14:40:25.208891+0000 mon.a (mon.0) 1241 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:26.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[87147]: cluster 2024-09-13T14:40:25.208891+0000 mon.a (mon.0) 1241 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:26.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[87147]: cluster 2024-09-13T14:40:25.233550+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-13T14:40:26.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:26 smithi099 bash[87147]: cluster 2024-09-13T14:40:25.233550+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-13T14:40:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:26 smithi177 bash[68594]: cluster 2024-09-13T14:40:25.208808+0000 mon.a (mon.0) 1240 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:26 smithi177 bash[68594]: cluster 2024-09-13T14:40:25.208808+0000 mon.a (mon.0) 1240 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:26 smithi177 bash[68594]: cluster 2024-09-13T14:40:25.208891+0000 mon.a (mon.0) 1241 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:26 smithi177 bash[68594]: cluster 2024-09-13T14:40:25.208891+0000 mon.a (mon.0) 1241 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:26 smithi177 bash[68594]: cluster 2024-09-13T14:40:25.233550+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-13T14:40:26.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:26 smithi177 bash[68594]: cluster 2024-09-13T14:40:25.233550+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-13T14:40:27.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:40:27] "GET /metrics HTTP/1.1" 200 75165 "" "Prometheus/2.51.0" 2024-09-13T14:40:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:27 smithi099 bash[87147]: cluster 2024-09-13T14:40:26.230359+0000 mgr.y (mgr.26252) 419 : cluster [DBG] pgmap v672: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 474 KiB/s rd, 31 KiB/s wr, 615 op/s 2024-09-13T14:40:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:27 smithi099 bash[87147]: cluster 2024-09-13T14:40:26.230359+0000 mgr.y (mgr.26252) 419 : cluster [DBG] pgmap v672: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 474 KiB/s rd, 31 KiB/s wr, 615 op/s 2024-09-13T14:40:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:27 smithi099 bash[87147]: cluster 2024-09-13T14:40:26.242280+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-13T14:40:27.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:27 smithi099 bash[87147]: cluster 2024-09-13T14:40:26.242280+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-13T14:40:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:27 smithi099 bash[84576]: cluster 2024-09-13T14:40:26.230359+0000 mgr.y (mgr.26252) 419 : cluster [DBG] pgmap v672: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 474 KiB/s rd, 31 KiB/s wr, 615 op/s 2024-09-13T14:40:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:27 smithi099 bash[84576]: cluster 2024-09-13T14:40:26.230359+0000 mgr.y (mgr.26252) 419 : cluster [DBG] pgmap v672: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 474 KiB/s rd, 31 KiB/s wr, 615 op/s 2024-09-13T14:40:27.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:27 smithi099 bash[84576]: cluster 2024-09-13T14:40:26.242280+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-13T14:40:27.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:27 smithi099 bash[84576]: cluster 2024-09-13T14:40:26.242280+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-13T14:40:27.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:27 smithi177 bash[68594]: cluster 2024-09-13T14:40:26.230359+0000 mgr.y (mgr.26252) 419 : cluster [DBG] pgmap v672: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 474 KiB/s rd, 31 KiB/s wr, 615 op/s 2024-09-13T14:40:27.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:27 smithi177 bash[68594]: cluster 2024-09-13T14:40:26.230359+0000 mgr.y (mgr.26252) 419 : cluster [DBG] pgmap v672: 649 pgs: 10 creating+peering, 3 unknown, 636 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 474 KiB/s rd, 31 KiB/s wr, 615 op/s 2024-09-13T14:40:27.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:27 smithi177 bash[68594]: cluster 2024-09-13T14:40:26.242280+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-13T14:40:27.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:27 smithi177 bash[68594]: cluster 2024-09-13T14:40:26.242280+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-13T14:40:28.261 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[84576]: cluster 2024-09-13T14:40:27.247308+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-13T14:40:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[84576]: cluster 2024-09-13T14:40:27.247308+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-13T14:40:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[84576]: audit 2024-09-13T14:40:27.258760+0000 mon.a (mon.0) 1245 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[84576]: audit 2024-09-13T14:40:27.258760+0000 mon.a (mon.0) 1245 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[84576]: audit 2024-09-13T14:40:28.243611+0000 mon.a (mon.0) 1246 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[84576]: audit 2024-09-13T14:40:28.243611+0000 mon.a (mon.0) 1246 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[84576]: cluster 2024-09-13T14:40:28.252298+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-13T14:40:28.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[84576]: cluster 2024-09-13T14:40:28.252298+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-13T14:40:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[87147]: cluster 2024-09-13T14:40:27.247308+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-13T14:40:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[87147]: cluster 2024-09-13T14:40:27.247308+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-13T14:40:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[87147]: audit 2024-09-13T14:40:27.258760+0000 mon.a (mon.0) 1245 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[87147]: audit 2024-09-13T14:40:27.258760+0000 mon.a (mon.0) 1245 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[87147]: audit 2024-09-13T14:40:28.243611+0000 mon.a (mon.0) 1246 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[87147]: audit 2024-09-13T14:40:28.243611+0000 mon.a (mon.0) 1246 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[87147]: cluster 2024-09-13T14:40:28.252298+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-13T14:40:28.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:28 smithi099 bash[87147]: cluster 2024-09-13T14:40:28.252298+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-13T14:40:28.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:28 smithi177 bash[68594]: cluster 2024-09-13T14:40:27.247308+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-13T14:40:28.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:28 smithi177 bash[68594]: cluster 2024-09-13T14:40:27.247308+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-13T14:40:28.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:28 smithi177 bash[68594]: audit 2024-09-13T14:40:27.258760+0000 mon.a (mon.0) 1245 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:28.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:28 smithi177 bash[68594]: audit 2024-09-13T14:40:27.258760+0000 mon.a (mon.0) 1245 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:28.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:28 smithi177 bash[68594]: audit 2024-09-13T14:40:28.243611+0000 mon.a (mon.0) 1246 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:28.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:28 smithi177 bash[68594]: audit 2024-09-13T14:40:28.243611+0000 mon.a (mon.0) 1246 : audit [INF] from='client.? 172.21.15.99:0/4019029482' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:28.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:28 smithi177 bash[68594]: cluster 2024-09-13T14:40:28.252298+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-13T14:40:28.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:28 smithi177 bash[68594]: cluster 2024-09-13T14:40:28.252298+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-13T14:40:29.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:29 smithi099 bash[84576]: cluster 2024-09-13T14:40:28.233506+0000 mgr.y (mgr.26252) 420 : cluster [DBG] pgmap v675: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.2 KiB/s wr, 42 op/s 2024-09-13T14:40:29.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:29 smithi099 bash[84576]: cluster 2024-09-13T14:40:28.233506+0000 mgr.y (mgr.26252) 420 : cluster [DBG] pgmap v675: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.2 KiB/s wr, 42 op/s 2024-09-13T14:40:29.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:29 smithi099 bash[84576]: cluster 2024-09-13T14:40:28.262081+0000 mon.a (mon.0) 1248 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:29.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:29 smithi099 bash[84576]: cluster 2024-09-13T14:40:28.262081+0000 mon.a (mon.0) 1248 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:29.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:29 smithi099 bash[87147]: cluster 2024-09-13T14:40:28.233506+0000 mgr.y (mgr.26252) 420 : cluster [DBG] pgmap v675: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.2 KiB/s wr, 42 op/s 2024-09-13T14:40:29.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:29 smithi099 bash[87147]: cluster 2024-09-13T14:40:28.233506+0000 mgr.y (mgr.26252) 420 : cluster [DBG] pgmap v675: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.2 KiB/s wr, 42 op/s 2024-09-13T14:40:29.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:29 smithi099 bash[87147]: cluster 2024-09-13T14:40:28.262081+0000 mon.a (mon.0) 1248 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:29.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:29 smithi099 bash[87147]: cluster 2024-09-13T14:40:28.262081+0000 mon.a (mon.0) 1248 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:29.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:29 smithi177 bash[68594]: cluster 2024-09-13T14:40:28.233506+0000 mgr.y (mgr.26252) 420 : cluster [DBG] pgmap v675: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.2 KiB/s wr, 42 op/s 2024-09-13T14:40:29.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:29 smithi177 bash[68594]: cluster 2024-09-13T14:40:28.233506+0000 mgr.y (mgr.26252) 420 : cluster [DBG] pgmap v675: 681 pgs: 15 creating+peering, 17 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.2 KiB/s wr, 42 op/s 2024-09-13T14:40:29.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:29 smithi177 bash[68594]: cluster 2024-09-13T14:40:28.262081+0000 mon.a (mon.0) 1248 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:29.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:29 smithi177 bash[68594]: cluster 2024-09-13T14:40:28.262081+0000 mon.a (mon.0) 1248 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:30.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:30 smithi099 bash[84576]: cluster 2024-09-13T14:40:29.291642+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-13T14:40:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:30 smithi099 bash[84576]: cluster 2024-09-13T14:40:29.291642+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-13T14:40:30.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:30 smithi099 bash[87147]: cluster 2024-09-13T14:40:29.291642+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-13T14:40:30.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:30 smithi099 bash[87147]: cluster 2024-09-13T14:40:29.291642+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-13T14:40:30.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:30 smithi177 bash[68594]: cluster 2024-09-13T14:40:29.291642+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-13T14:40:30.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:30 smithi177 bash[68594]: cluster 2024-09-13T14:40:29.291642+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-13T14:40:30.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:40:30 smithi177 bash[61439]: ts=2024-09-13T14:40:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[84576]: cluster 2024-09-13T14:40:30.234663+0000 mgr.y (mgr.26252) 421 : cluster [DBG] pgmap v678: 681 pgs: 15 creating+peering, 5 unknown, 661 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-13T14:40:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[84576]: cluster 2024-09-13T14:40:30.234663+0000 mgr.y (mgr.26252) 421 : cluster [DBG] pgmap v678: 681 pgs: 15 creating+peering, 5 unknown, 661 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-13T14:40:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[84576]: cluster 2024-09-13T14:40:30.295961+0000 mon.a (mon.0) 1250 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[84576]: cluster 2024-09-13T14:40:30.295961+0000 mon.a (mon.0) 1250 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[84576]: cluster 2024-09-13T14:40:30.296006+0000 mon.a (mon.0) 1251 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[84576]: cluster 2024-09-13T14:40:30.296006+0000 mon.a (mon.0) 1251 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:31.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[84576]: cluster 2024-09-13T14:40:30.318599+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[84576]: cluster 2024-09-13T14:40:30.318599+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[84576]: cluster 2024-09-13T14:40:31.150001+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[84576]: cluster 2024-09-13T14:40:31.150001+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[87147]: cluster 2024-09-13T14:40:30.234663+0000 mgr.y (mgr.26252) 421 : cluster [DBG] pgmap v678: 681 pgs: 15 creating+peering, 5 unknown, 661 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[87147]: cluster 2024-09-13T14:40:30.234663+0000 mgr.y (mgr.26252) 421 : cluster [DBG] pgmap v678: 681 pgs: 15 creating+peering, 5 unknown, 661 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[87147]: cluster 2024-09-13T14:40:30.295961+0000 mon.a (mon.0) 1250 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[87147]: cluster 2024-09-13T14:40:30.295961+0000 mon.a (mon.0) 1250 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[87147]: cluster 2024-09-13T14:40:30.296006+0000 mon.a (mon.0) 1251 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[87147]: cluster 2024-09-13T14:40:30.296006+0000 mon.a (mon.0) 1251 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[87147]: cluster 2024-09-13T14:40:30.318599+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-13T14:40:31.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[87147]: cluster 2024-09-13T14:40:30.318599+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-13T14:40:31.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[87147]: cluster 2024-09-13T14:40:31.150001+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-13T14:40:31.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:31 smithi099 bash[87147]: cluster 2024-09-13T14:40:31.150001+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-13T14:40:31.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:31 smithi177 bash[68594]: cluster 2024-09-13T14:40:30.234663+0000 mgr.y (mgr.26252) 421 : cluster [DBG] pgmap v678: 681 pgs: 15 creating+peering, 5 unknown, 661 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-13T14:40:31.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:31 smithi177 bash[68594]: cluster 2024-09-13T14:40:30.234663+0000 mgr.y (mgr.26252) 421 : cluster [DBG] pgmap v678: 681 pgs: 15 creating+peering, 5 unknown, 661 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-13T14:40:31.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:31 smithi177 bash[68594]: cluster 2024-09-13T14:40:30.295961+0000 mon.a (mon.0) 1250 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:31.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:31 smithi177 bash[68594]: cluster 2024-09-13T14:40:30.295961+0000 mon.a (mon.0) 1250 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:31.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:31 smithi177 bash[68594]: cluster 2024-09-13T14:40:30.296006+0000 mon.a (mon.0) 1251 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:31.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:31 smithi177 bash[68594]: cluster 2024-09-13T14:40:30.296006+0000 mon.a (mon.0) 1251 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:31.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:31 smithi177 bash[68594]: cluster 2024-09-13T14:40:30.318599+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-13T14:40:31.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:31 smithi177 bash[68594]: cluster 2024-09-13T14:40:30.318599+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-13T14:40:31.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:31 smithi177 bash[68594]: cluster 2024-09-13T14:40:31.150001+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-13T14:40:31.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:31 smithi177 bash[68594]: cluster 2024-09-13T14:40:31.150001+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-13T14:40:32.681 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ImageSpec (6260 ms) 2024-09-13T14:40:32.681 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.SnapRemoveWithChildMissing 2024-09-13T14:40:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:33 smithi177 bash[68594]: cluster 2024-09-13T14:40:32.236530+0000 mgr.y (mgr.26252) 422 : cluster [DBG] pgmap v681: 681 pgs: 15 creating+peering, 666 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 3.2 KiB/s wr, 20 op/s 2024-09-13T14:40:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:33 smithi177 bash[68594]: cluster 2024-09-13T14:40:32.236530+0000 mgr.y (mgr.26252) 422 : cluster [DBG] pgmap v681: 681 pgs: 15 creating+peering, 666 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 3.2 KiB/s wr, 20 op/s 2024-09-13T14:40:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:33 smithi177 bash[68594]: cluster 2024-09-13T14:40:32.323912+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-13T14:40:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:33 smithi177 bash[68594]: cluster 2024-09-13T14:40:32.323912+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-13T14:40:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:33 smithi177 bash[68594]: audit 2024-09-13T14:40:33.254225+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:33.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:33 smithi177 bash[68594]: audit 2024-09-13T14:40:33.254225+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[84576]: cluster 2024-09-13T14:40:32.236530+0000 mgr.y (mgr.26252) 422 : cluster [DBG] pgmap v681: 681 pgs: 15 creating+peering, 666 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 3.2 KiB/s wr, 20 op/s 2024-09-13T14:40:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[84576]: cluster 2024-09-13T14:40:32.236530+0000 mgr.y (mgr.26252) 422 : cluster [DBG] pgmap v681: 681 pgs: 15 creating+peering, 666 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 3.2 KiB/s wr, 20 op/s 2024-09-13T14:40:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[84576]: cluster 2024-09-13T14:40:32.323912+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-13T14:40:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[84576]: cluster 2024-09-13T14:40:32.323912+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-13T14:40:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[84576]: audit 2024-09-13T14:40:33.254225+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[84576]: audit 2024-09-13T14:40:33.254225+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:33.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[87147]: cluster 2024-09-13T14:40:32.236530+0000 mgr.y (mgr.26252) 422 : cluster [DBG] pgmap v681: 681 pgs: 15 creating+peering, 666 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 3.2 KiB/s wr, 20 op/s 2024-09-13T14:40:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[87147]: cluster 2024-09-13T14:40:32.236530+0000 mgr.y (mgr.26252) 422 : cluster [DBG] pgmap v681: 681 pgs: 15 creating+peering, 666 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 3.2 KiB/s wr, 20 op/s 2024-09-13T14:40:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[87147]: cluster 2024-09-13T14:40:32.323912+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-13T14:40:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[87147]: cluster 2024-09-13T14:40:32.323912+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-13T14:40:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[87147]: audit 2024-09-13T14:40:33.254225+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:33 smithi099 bash[87147]: audit 2024-09-13T14:40:33.254225+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:34.342 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: cluster 2024-09-13T14:40:33.327577+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-13T14:40:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: cluster 2024-09-13T14:40:33.327577+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-13T14:40:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: audit 2024-09-13T14:40:33.336121+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.99:0/1715357093' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: audit 2024-09-13T14:40:33.336121+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.99:0/1715357093' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: audit 2024-09-13T14:40:33.358309+0000 mon.a (mon.0) 1257 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: audit 2024-09-13T14:40:33.358309+0000 mon.a (mon.0) 1257 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: audit 2024-09-13T14:40:33.948070+0000 mon.a (mon.0) 1258 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: audit 2024-09-13T14:40:33.948070+0000 mon.a (mon.0) 1258 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: audit 2024-09-13T14:40:34.325021+0000 mon.a (mon.0) 1259 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: audit 2024-09-13T14:40:34.325021+0000 mon.a (mon.0) 1259 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: cluster 2024-09-13T14:40:34.333323+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-13T14:40:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:34 smithi177 bash[68594]: cluster 2024-09-13T14:40:34.333323+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-13T14:40:34.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: cluster 2024-09-13T14:40:33.327577+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-13T14:40:34.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: cluster 2024-09-13T14:40:33.327577+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-13T14:40:34.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: audit 2024-09-13T14:40:33.336121+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.99:0/1715357093' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: audit 2024-09-13T14:40:33.336121+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.99:0/1715357093' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: audit 2024-09-13T14:40:33.358309+0000 mon.a (mon.0) 1257 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: audit 2024-09-13T14:40:33.358309+0000 mon.a (mon.0) 1257 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: audit 2024-09-13T14:40:33.948070+0000 mon.a (mon.0) 1258 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: audit 2024-09-13T14:40:33.948070+0000 mon.a (mon.0) 1258 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: audit 2024-09-13T14:40:34.325021+0000 mon.a (mon.0) 1259 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: audit 2024-09-13T14:40:34.325021+0000 mon.a (mon.0) 1259 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: cluster 2024-09-13T14:40:34.333323+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[87147]: cluster 2024-09-13T14:40:34.333323+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: cluster 2024-09-13T14:40:33.327577+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: cluster 2024-09-13T14:40:33.327577+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: audit 2024-09-13T14:40:33.336121+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.99:0/1715357093' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: audit 2024-09-13T14:40:33.336121+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.99:0/1715357093' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: audit 2024-09-13T14:40:33.358309+0000 mon.a (mon.0) 1257 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: audit 2024-09-13T14:40:33.358309+0000 mon.a (mon.0) 1257 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:34.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: audit 2024-09-13T14:40:33.948070+0000 mon.a (mon.0) 1258 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:34.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: audit 2024-09-13T14:40:33.948070+0000 mon.a (mon.0) 1258 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:34.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: audit 2024-09-13T14:40:34.325021+0000 mon.a (mon.0) 1259 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:34.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: audit 2024-09-13T14:40:34.325021+0000 mon.a (mon.0) 1259 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:34.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: cluster 2024-09-13T14:40:34.333323+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-13T14:40:34.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:34 smithi099 bash[84576]: cluster 2024-09-13T14:40:34.333323+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-13T14:40:35.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:35 smithi099 bash[84576]: cluster 2024-09-13T14:40:34.240320+0000 mgr.y (mgr.26252) 423 : cluster [DBG] pgmap v684: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 5.7 KiB/s wr, 118 op/s 2024-09-13T14:40:35.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:35 smithi099 bash[84576]: cluster 2024-09-13T14:40:34.240320+0000 mgr.y (mgr.26252) 423 : cluster [DBG] pgmap v684: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 5.7 KiB/s wr, 118 op/s 2024-09-13T14:40:35.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:35 smithi099 bash[84576]: cluster 2024-09-13T14:40:34.943448+0000 mon.a (mon.0) 1261 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:35.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:35 smithi099 bash[84576]: cluster 2024-09-13T14:40:34.943448+0000 mon.a (mon.0) 1261 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:35 smithi099 bash[87147]: cluster 2024-09-13T14:40:34.240320+0000 mgr.y (mgr.26252) 423 : cluster [DBG] pgmap v684: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 5.7 KiB/s wr, 118 op/s 2024-09-13T14:40:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:35 smithi099 bash[87147]: cluster 2024-09-13T14:40:34.240320+0000 mgr.y (mgr.26252) 423 : cluster [DBG] pgmap v684: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 5.7 KiB/s wr, 118 op/s 2024-09-13T14:40:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:35 smithi099 bash[87147]: cluster 2024-09-13T14:40:34.943448+0000 mon.a (mon.0) 1261 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:35 smithi099 bash[87147]: cluster 2024-09-13T14:40:34.943448+0000 mon.a (mon.0) 1261 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:35.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:35 smithi177 bash[68594]: cluster 2024-09-13T14:40:34.240320+0000 mgr.y (mgr.26252) 423 : cluster [DBG] pgmap v684: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 5.7 KiB/s wr, 118 op/s 2024-09-13T14:40:35.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:35 smithi177 bash[68594]: cluster 2024-09-13T14:40:34.240320+0000 mgr.y (mgr.26252) 423 : cluster [DBG] pgmap v684: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 5.7 KiB/s wr, 118 op/s 2024-09-13T14:40:35.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:35 smithi177 bash[68594]: cluster 2024-09-13T14:40:34.943448+0000 mon.a (mon.0) 1261 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:35.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:35 smithi177 bash[68594]: cluster 2024-09-13T14:40:34.943448+0000 mon.a (mon.0) 1261 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:36 smithi099 bash[84576]: cluster 2024-09-13T14:40:35.358518+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-13T14:40:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:36 smithi099 bash[84576]: cluster 2024-09-13T14:40:35.358518+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-13T14:40:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:36 smithi099 bash[84576]: cluster 2024-09-13T14:40:36.146919+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-13T14:40:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:36 smithi099 bash[84576]: cluster 2024-09-13T14:40:36.146919+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-13T14:40:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:36 smithi099 bash[87147]: cluster 2024-09-13T14:40:35.358518+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-13T14:40:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:36 smithi099 bash[87147]: cluster 2024-09-13T14:40:35.358518+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-13T14:40:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:36 smithi099 bash[87147]: cluster 2024-09-13T14:40:36.146919+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-13T14:40:36.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:36 smithi099 bash[87147]: cluster 2024-09-13T14:40:36.146919+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-13T14:40:36.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:36 smithi177 bash[68594]: cluster 2024-09-13T14:40:35.358518+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-13T14:40:36.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:36 smithi177 bash[68594]: cluster 2024-09-13T14:40:35.358518+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-13T14:40:36.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:36 smithi177 bash[68594]: cluster 2024-09-13T14:40:36.146919+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-13T14:40:36.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:36 smithi177 bash[68594]: cluster 2024-09-13T14:40:36.146919+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-13T14:40:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:40:37] "GET /metrics HTTP/1.1" 200 79974 "" "Prometheus/2.51.0" 2024-09-13T14:40:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[84576]: cluster 2024-09-13T14:40:36.241374+0000 mgr.y (mgr.26252) 424 : cluster [DBG] pgmap v688: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 92 KiB/s rd, 5.6 KiB/s wr, 118 op/s 2024-09-13T14:40:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[84576]: cluster 2024-09-13T14:40:36.241374+0000 mgr.y (mgr.26252) 424 : cluster [DBG] pgmap v688: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 92 KiB/s rd, 5.6 KiB/s wr, 118 op/s 2024-09-13T14:40:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[84576]: cluster 2024-09-13T14:40:37.140299+0000 mon.a (mon.0) 1264 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[84576]: cluster 2024-09-13T14:40:37.140299+0000 mon.a (mon.0) 1264 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[84576]: cluster 2024-09-13T14:40:37.140347+0000 mon.a (mon.0) 1265 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[84576]: cluster 2024-09-13T14:40:37.140347+0000 mon.a (mon.0) 1265 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:37.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[84576]: cluster 2024-09-13T14:40:37.161608+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-13T14:40:37.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[84576]: cluster 2024-09-13T14:40:37.161608+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-13T14:40:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[87147]: cluster 2024-09-13T14:40:36.241374+0000 mgr.y (mgr.26252) 424 : cluster [DBG] pgmap v688: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 92 KiB/s rd, 5.6 KiB/s wr, 118 op/s 2024-09-13T14:40:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[87147]: cluster 2024-09-13T14:40:36.241374+0000 mgr.y (mgr.26252) 424 : cluster [DBG] pgmap v688: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 92 KiB/s rd, 5.6 KiB/s wr, 118 op/s 2024-09-13T14:40:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[87147]: cluster 2024-09-13T14:40:37.140299+0000 mon.a (mon.0) 1264 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[87147]: cluster 2024-09-13T14:40:37.140299+0000 mon.a (mon.0) 1264 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[87147]: cluster 2024-09-13T14:40:37.140347+0000 mon.a (mon.0) 1265 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[87147]: cluster 2024-09-13T14:40:37.140347+0000 mon.a (mon.0) 1265 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[87147]: cluster 2024-09-13T14:40:37.161608+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-13T14:40:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:37 smithi099 bash[87147]: cluster 2024-09-13T14:40:37.161608+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-13T14:40:37.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:37 smithi177 bash[68594]: cluster 2024-09-13T14:40:36.241374+0000 mgr.y (mgr.26252) 424 : cluster [DBG] pgmap v688: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 92 KiB/s rd, 5.6 KiB/s wr, 118 op/s 2024-09-13T14:40:37.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:37 smithi177 bash[68594]: cluster 2024-09-13T14:40:36.241374+0000 mgr.y (mgr.26252) 424 : cluster [DBG] pgmap v688: 713 pgs: 15 creating+peering, 17 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 92 KiB/s rd, 5.6 KiB/s wr, 118 op/s 2024-09-13T14:40:37.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:37 smithi177 bash[68594]: cluster 2024-09-13T14:40:37.140299+0000 mon.a (mon.0) 1264 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:37.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:37 smithi177 bash[68594]: cluster 2024-09-13T14:40:37.140299+0000 mon.a (mon.0) 1264 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:37.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:37 smithi177 bash[68594]: cluster 2024-09-13T14:40:37.140347+0000 mon.a (mon.0) 1265 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:37.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:37 smithi177 bash[68594]: cluster 2024-09-13T14:40:37.140347+0000 mon.a (mon.0) 1265 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:37.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:37 smithi177 bash[68594]: cluster 2024-09-13T14:40:37.161608+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-13T14:40:37.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:37 smithi177 bash[68594]: cluster 2024-09-13T14:40:37.161608+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-13T14:40:39.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:39 smithi099 bash[84576]: cluster 2024-09-13T14:40:38.163148+0000 mon.a (mon.0) 1267 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-13T14:40:39.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:39 smithi099 bash[84576]: cluster 2024-09-13T14:40:38.163148+0000 mon.a (mon.0) 1267 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-13T14:40:39.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:39 smithi099 bash[84576]: cluster 2024-09-13T14:40:38.242603+0000 mgr.y (mgr.26252) 425 : cluster [DBG] pgmap v691: 713 pgs: 15 creating+peering, 698 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 1.3 KiB/s wr, 61 op/s 2024-09-13T14:40:39.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:39 smithi099 bash[84576]: cluster 2024-09-13T14:40:38.242603+0000 mgr.y (mgr.26252) 425 : cluster [DBG] pgmap v691: 713 pgs: 15 creating+peering, 698 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 1.3 KiB/s wr, 61 op/s 2024-09-13T14:40:39.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:39 smithi099 bash[87147]: cluster 2024-09-13T14:40:38.163148+0000 mon.a (mon.0) 1267 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-13T14:40:39.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:39 smithi099 bash[87147]: cluster 2024-09-13T14:40:38.163148+0000 mon.a (mon.0) 1267 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-13T14:40:39.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:39 smithi099 bash[87147]: cluster 2024-09-13T14:40:38.242603+0000 mgr.y (mgr.26252) 425 : cluster [DBG] pgmap v691: 713 pgs: 15 creating+peering, 698 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 1.3 KiB/s wr, 61 op/s 2024-09-13T14:40:39.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:39 smithi099 bash[87147]: cluster 2024-09-13T14:40:38.242603+0000 mgr.y (mgr.26252) 425 : cluster [DBG] pgmap v691: 713 pgs: 15 creating+peering, 698 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 1.3 KiB/s wr, 61 op/s 2024-09-13T14:40:39.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:39 smithi177 bash[68594]: cluster 2024-09-13T14:40:38.163148+0000 mon.a (mon.0) 1267 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-13T14:40:39.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:39 smithi177 bash[68594]: cluster 2024-09-13T14:40:38.163148+0000 mon.a (mon.0) 1267 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-13T14:40:39.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:39 smithi177 bash[68594]: cluster 2024-09-13T14:40:38.242603+0000 mgr.y (mgr.26252) 425 : cluster [DBG] pgmap v691: 713 pgs: 15 creating+peering, 698 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 1.3 KiB/s wr, 61 op/s 2024-09-13T14:40:39.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:39 smithi177 bash[68594]: cluster 2024-09-13T14:40:38.242603+0000 mgr.y (mgr.26252) 425 : cluster [DBG] pgmap v691: 713 pgs: 15 creating+peering, 698 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 1.3 KiB/s wr, 61 op/s 2024-09-13T14:40:40.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:40 smithi099 bash[84576]: cluster 2024-09-13T14:40:39.181201+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-13T14:40:40.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:40 smithi099 bash[84576]: cluster 2024-09-13T14:40:39.181201+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-13T14:40:40.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:40 smithi099 bash[87147]: cluster 2024-09-13T14:40:39.181201+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-13T14:40:40.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:40 smithi099 bash[87147]: cluster 2024-09-13T14:40:39.181201+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-13T14:40:40.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:40 smithi177 bash[68594]: cluster 2024-09-13T14:40:39.181201+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-13T14:40:40.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:40 smithi177 bash[68594]: cluster 2024-09-13T14:40:39.181201+0000 mon.a (mon.0) 1268 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-13T14:40:40.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:40:40 smithi177 bash[61439]: ts=2024-09-13T14:40:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:41.271 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 3 2024-09-13T14:40:41.271 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 3 2024-09-13T14:40:41.271 INFO:tasks.workunit.client.0.smithi099.stdout:child: image202 2024-09-13T14:40:41.271 INFO:tasks.workunit.client.0.smithi099.stdout:child: image204 2024-09-13T14:40:41.271 INFO:tasks.workunit.client.0.smithi099.stdout:child: image203 2024-09-13T14:40:41.271 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-14/image202/85e3ccee7929 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-33/image203/85e34d04da21 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout:found child test-librbd-smithi099-69110-14/image204/85e32eb21538 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout:num children is: 0 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 0 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout:num snaps is: 0 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout:expected: 0 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.SnapRemoveWithChildMissing (8590 ms) 2024-09-13T14:40:41.272 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.QuiesceWatch 2024-09-13T14:40:41.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:41 smithi099 bash[84576]: cluster 2024-09-13T14:40:40.191523+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-13T14:40:41.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:41 smithi099 bash[84576]: cluster 2024-09-13T14:40:40.191523+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-13T14:40:41.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:41 smithi099 bash[84576]: cluster 2024-09-13T14:40:40.243563+0000 mgr.y (mgr.26252) 426 : cluster [DBG] pgmap v694: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:41.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:41 smithi099 bash[84576]: cluster 2024-09-13T14:40:40.243563+0000 mgr.y (mgr.26252) 426 : cluster [DBG] pgmap v694: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:41.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:41 smithi099 bash[87147]: cluster 2024-09-13T14:40:40.191523+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-13T14:40:41.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:41 smithi099 bash[87147]: cluster 2024-09-13T14:40:40.191523+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-13T14:40:41.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:41 smithi099 bash[87147]: cluster 2024-09-13T14:40:40.243563+0000 mgr.y (mgr.26252) 426 : cluster [DBG] pgmap v694: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:41.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:41 smithi099 bash[87147]: cluster 2024-09-13T14:40:40.243563+0000 mgr.y (mgr.26252) 426 : cluster [DBG] pgmap v694: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:41.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:41 smithi177 bash[68594]: cluster 2024-09-13T14:40:40.191523+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-13T14:40:41.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:41 smithi177 bash[68594]: cluster 2024-09-13T14:40:40.191523+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-13T14:40:41.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:41 smithi177 bash[68594]: cluster 2024-09-13T14:40:40.243563+0000 mgr.y (mgr.26252) 426 : cluster [DBG] pgmap v694: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:41.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:41 smithi177 bash[68594]: cluster 2024-09-13T14:40:40.243563+0000 mgr.y (mgr.26252) 426 : cluster [DBG] pgmap v694: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:42.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:42 smithi099 bash[84576]: cluster 2024-09-13T14:40:41.203065+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-13T14:40:42.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:42 smithi099 bash[84576]: cluster 2024-09-13T14:40:41.203065+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-13T14:40:42.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:42 smithi099 bash[87147]: cluster 2024-09-13T14:40:41.203065+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-13T14:40:42.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:42 smithi099 bash[87147]: cluster 2024-09-13T14:40:41.203065+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-13T14:40:42.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:42 smithi177 bash[68594]: cluster 2024-09-13T14:40:41.203065+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-13T14:40:42.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:42 smithi177 bash[68594]: cluster 2024-09-13T14:40:41.203065+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-13T14:40:43.212 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[84576]: cluster 2024-09-13T14:40:42.208660+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-13T14:40:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[84576]: cluster 2024-09-13T14:40:42.208660+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-13T14:40:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[84576]: audit 2024-09-13T14:40:42.213970+0000 mon.a (mon.0) 1272 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[84576]: audit 2024-09-13T14:40:42.213970+0000 mon.a (mon.0) 1272 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[84576]: cluster 2024-09-13T14:40:42.244745+0000 mgr.y (mgr.26252) 427 : cluster [DBG] pgmap v697: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[84576]: cluster 2024-09-13T14:40:42.244745+0000 mgr.y (mgr.26252) 427 : cluster [DBG] pgmap v697: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[84576]: audit 2024-09-13T14:40:43.200192+0000 mon.a (mon.0) 1273 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[84576]: audit 2024-09-13T14:40:43.200192+0000 mon.a (mon.0) 1273 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:43.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[84576]: cluster 2024-09-13T14:40:43.209126+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-13T14:40:43.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[84576]: cluster 2024-09-13T14:40:43.209126+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-13T14:40:43.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[87147]: cluster 2024-09-13T14:40:42.208660+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-13T14:40:43.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[87147]: cluster 2024-09-13T14:40:42.208660+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-13T14:40:43.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[87147]: audit 2024-09-13T14:40:42.213970+0000 mon.a (mon.0) 1272 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:43.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[87147]: audit 2024-09-13T14:40:42.213970+0000 mon.a (mon.0) 1272 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:43.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[87147]: cluster 2024-09-13T14:40:42.244745+0000 mgr.y (mgr.26252) 427 : cluster [DBG] pgmap v697: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:43.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[87147]: cluster 2024-09-13T14:40:42.244745+0000 mgr.y (mgr.26252) 427 : cluster [DBG] pgmap v697: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:43.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[87147]: audit 2024-09-13T14:40:43.200192+0000 mon.a (mon.0) 1273 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:43.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[87147]: audit 2024-09-13T14:40:43.200192+0000 mon.a (mon.0) 1273 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:43.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[87147]: cluster 2024-09-13T14:40:43.209126+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-13T14:40:43.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:43 smithi099 bash[87147]: cluster 2024-09-13T14:40:43.209126+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-13T14:40:43.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:43 smithi177 bash[68594]: cluster 2024-09-13T14:40:42.208660+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-13T14:40:43.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:43 smithi177 bash[68594]: cluster 2024-09-13T14:40:42.208660+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-13T14:40:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:43 smithi177 bash[68594]: audit 2024-09-13T14:40:42.213970+0000 mon.a (mon.0) 1272 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:43 smithi177 bash[68594]: audit 2024-09-13T14:40:42.213970+0000 mon.a (mon.0) 1272 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:40:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:43 smithi177 bash[68594]: cluster 2024-09-13T14:40:42.244745+0000 mgr.y (mgr.26252) 427 : cluster [DBG] pgmap v697: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:43 smithi177 bash[68594]: cluster 2024-09-13T14:40:42.244745+0000 mgr.y (mgr.26252) 427 : cluster [DBG] pgmap v697: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:40:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:43 smithi177 bash[68594]: audit 2024-09-13T14:40:43.200192+0000 mon.a (mon.0) 1273 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:43 smithi177 bash[68594]: audit 2024-09-13T14:40:43.200192+0000 mon.a (mon.0) 1273 : audit [INF] from='client.? 172.21.15.99:0/131019296' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:40:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:43 smithi177 bash[68594]: cluster 2024-09-13T14:40:43.209126+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-13T14:40:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:43 smithi177 bash[68594]: cluster 2024-09-13T14:40:43.209126+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[84576]: cluster 2024-09-13T14:40:43.217610+0000 mon.a (mon.0) 1275 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[84576]: cluster 2024-09-13T14:40:43.217610+0000 mon.a (mon.0) 1275 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[84576]: audit 2024-09-13T14:40:44.024874+0000 mon.a (mon.0) 1276 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[84576]: audit 2024-09-13T14:40:44.024874+0000 mon.a (mon.0) 1276 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[84576]: cluster 2024-09-13T14:40:44.243838+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[84576]: cluster 2024-09-13T14:40:44.243838+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-13T14:40:44.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[87147]: cluster 2024-09-13T14:40:43.217610+0000 mon.a (mon.0) 1275 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:44.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[87147]: cluster 2024-09-13T14:40:43.217610+0000 mon.a (mon.0) 1275 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:44.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[87147]: audit 2024-09-13T14:40:44.024874+0000 mon.a (mon.0) 1276 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:44.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[87147]: audit 2024-09-13T14:40:44.024874+0000 mon.a (mon.0) 1276 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:44.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[87147]: cluster 2024-09-13T14:40:44.243838+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-13T14:40:44.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:44 smithi099 bash[87147]: cluster 2024-09-13T14:40:44.243838+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-13T14:40:44.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:44 smithi177 bash[68594]: cluster 2024-09-13T14:40:43.217610+0000 mon.a (mon.0) 1275 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:44.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:44 smithi177 bash[68594]: cluster 2024-09-13T14:40:43.217610+0000 mon.a (mon.0) 1275 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:40:44.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:44 smithi177 bash[68594]: audit 2024-09-13T14:40:44.024874+0000 mon.a (mon.0) 1276 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:44.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:44 smithi177 bash[68594]: audit 2024-09-13T14:40:44.024874+0000 mon.a (mon.0) 1276 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:44.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:44 smithi177 bash[68594]: cluster 2024-09-13T14:40:44.243838+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-13T14:40:44.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:44 smithi177 bash[68594]: cluster 2024-09-13T14:40:44.243838+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-13T14:40:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[84576]: cluster 2024-09-13T14:40:44.249257+0000 mgr.y (mgr.26252) 428 : cluster [DBG] pgmap v699: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.9 KiB/s wr, 74 op/s 2024-09-13T14:40:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[84576]: cluster 2024-09-13T14:40:44.249257+0000 mgr.y (mgr.26252) 428 : cluster [DBG] pgmap v699: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.9 KiB/s wr, 74 op/s 2024-09-13T14:40:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[84576]: cluster 2024-09-13T14:40:45.237869+0000 mon.a (mon.0) 1278 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[84576]: cluster 2024-09-13T14:40:45.237869+0000 mon.a (mon.0) 1278 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[84576]: cluster 2024-09-13T14:40:45.237950+0000 mon.a (mon.0) 1279 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[84576]: cluster 2024-09-13T14:40:45.237950+0000 mon.a (mon.0) 1279 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[87147]: cluster 2024-09-13T14:40:44.249257+0000 mgr.y (mgr.26252) 428 : cluster [DBG] pgmap v699: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.9 KiB/s wr, 74 op/s 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[87147]: cluster 2024-09-13T14:40:44.249257+0000 mgr.y (mgr.26252) 428 : cluster [DBG] pgmap v699: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.9 KiB/s wr, 74 op/s 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[87147]: cluster 2024-09-13T14:40:45.237869+0000 mon.a (mon.0) 1278 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[87147]: cluster 2024-09-13T14:40:45.237869+0000 mon.a (mon.0) 1278 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[87147]: cluster 2024-09-13T14:40:45.237950+0000 mon.a (mon.0) 1279 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:45.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:45 smithi099 bash[87147]: cluster 2024-09-13T14:40:45.237950+0000 mon.a (mon.0) 1279 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:45.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:45 smithi177 bash[68594]: cluster 2024-09-13T14:40:44.249257+0000 mgr.y (mgr.26252) 428 : cluster [DBG] pgmap v699: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.9 KiB/s wr, 74 op/s 2024-09-13T14:40:45.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:45 smithi177 bash[68594]: cluster 2024-09-13T14:40:44.249257+0000 mgr.y (mgr.26252) 428 : cluster [DBG] pgmap v699: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.9 KiB/s wr, 74 op/s 2024-09-13T14:40:45.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:45 smithi177 bash[68594]: cluster 2024-09-13T14:40:45.237869+0000 mon.a (mon.0) 1278 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:45.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:45 smithi177 bash[68594]: cluster 2024-09-13T14:40:45.237869+0000 mon.a (mon.0) 1278 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:40:45.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:45 smithi177 bash[68594]: cluster 2024-09-13T14:40:45.237950+0000 mon.a (mon.0) 1279 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:45.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:45 smithi177 bash[68594]: cluster 2024-09-13T14:40:45.237950+0000 mon.a (mon.0) 1279 : cluster [INF] Cluster is now healthy 2024-09-13T14:40:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:46 smithi099 bash[84576]: cluster 2024-09-13T14:40:45.276347+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-13T14:40:46.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:46 smithi099 bash[84576]: cluster 2024-09-13T14:40:45.276347+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-13T14:40:46.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:46 smithi099 bash[84576]: cluster 2024-09-13T14:40:46.150532+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-13T14:40:46.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:46 smithi099 bash[84576]: cluster 2024-09-13T14:40:46.150532+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-13T14:40:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:46 smithi099 bash[87147]: cluster 2024-09-13T14:40:45.276347+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-13T14:40:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:46 smithi099 bash[87147]: cluster 2024-09-13T14:40:45.276347+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-13T14:40:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:46 smithi099 bash[87147]: cluster 2024-09-13T14:40:46.150532+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-13T14:40:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:46 smithi099 bash[87147]: cluster 2024-09-13T14:40:46.150532+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-13T14:40:46.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:46 smithi177 bash[68594]: cluster 2024-09-13T14:40:45.276347+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-13T14:40:46.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:46 smithi177 bash[68594]: cluster 2024-09-13T14:40:45.276347+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-13T14:40:46.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:46 smithi177 bash[68594]: cluster 2024-09-13T14:40:46.150532+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-13T14:40:46.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:46 smithi177 bash[68594]: cluster 2024-09-13T14:40:46.150532+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-13T14:40:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:40:47] "GET /metrics HTTP/1.1" 200 79974 "" "Prometheus/2.51.0" 2024-09-13T14:40:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:47 smithi099 bash[84576]: cluster 2024-09-13T14:40:46.250634+0000 mgr.y (mgr.26252) 429 : cluster [DBG] pgmap v703: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.7 KiB/s wr, 73 op/s 2024-09-13T14:40:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:47 smithi099 bash[84576]: cluster 2024-09-13T14:40:46.250634+0000 mgr.y (mgr.26252) 429 : cluster [DBG] pgmap v703: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.7 KiB/s wr, 73 op/s 2024-09-13T14:40:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:47 smithi099 bash[84576]: cluster 2024-09-13T14:40:47.160702+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-13T14:40:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:47 smithi099 bash[84576]: cluster 2024-09-13T14:40:47.160702+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-13T14:40:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:47 smithi099 bash[87147]: cluster 2024-09-13T14:40:46.250634+0000 mgr.y (mgr.26252) 429 : cluster [DBG] pgmap v703: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.7 KiB/s wr, 73 op/s 2024-09-13T14:40:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:47 smithi099 bash[87147]: cluster 2024-09-13T14:40:46.250634+0000 mgr.y (mgr.26252) 429 : cluster [DBG] pgmap v703: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.7 KiB/s wr, 73 op/s 2024-09-13T14:40:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:47 smithi099 bash[87147]: cluster 2024-09-13T14:40:47.160702+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-13T14:40:47.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:47 smithi099 bash[87147]: cluster 2024-09-13T14:40:47.160702+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-13T14:40:47.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:47 smithi177 bash[68594]: cluster 2024-09-13T14:40:46.250634+0000 mgr.y (mgr.26252) 429 : cluster [DBG] pgmap v703: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.7 KiB/s wr, 73 op/s 2024-09-13T14:40:47.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:47 smithi177 bash[68594]: cluster 2024-09-13T14:40:46.250634+0000 mgr.y (mgr.26252) 429 : cluster [DBG] pgmap v703: 713 pgs: 14 creating+peering, 4 unknown, 695 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 3.7 KiB/s wr, 73 op/s 2024-09-13T14:40:47.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:47 smithi177 bash[68594]: cluster 2024-09-13T14:40:47.160702+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-13T14:40:47.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:47 smithi177 bash[68594]: cluster 2024-09-13T14:40:47.160702+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-13T14:40:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[84576]: cluster 2024-09-13T14:40:48.163703+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-13T14:40:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[84576]: cluster 2024-09-13T14:40:48.163703+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-13T14:40:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[84576]: cluster 2024-09-13T14:40:48.251988+0000 mgr.y (mgr.26252) 430 : cluster [DBG] pgmap v706: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 768 B/s wr, 27 op/s 2024-09-13T14:40:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[84576]: cluster 2024-09-13T14:40:48.251988+0000 mgr.y (mgr.26252) 430 : cluster [DBG] pgmap v706: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 768 B/s wr, 27 op/s 2024-09-13T14:40:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[84576]: audit 2024-09-13T14:40:48.253742+0000 mon.a (mon.0) 1284 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[84576]: audit 2024-09-13T14:40:48.253742+0000 mon.a (mon.0) 1284 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:49.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[87147]: cluster 2024-09-13T14:40:48.163703+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-13T14:40:49.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[87147]: cluster 2024-09-13T14:40:48.163703+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-13T14:40:49.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[87147]: cluster 2024-09-13T14:40:48.251988+0000 mgr.y (mgr.26252) 430 : cluster [DBG] pgmap v706: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 768 B/s wr, 27 op/s 2024-09-13T14:40:49.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[87147]: cluster 2024-09-13T14:40:48.251988+0000 mgr.y (mgr.26252) 430 : cluster [DBG] pgmap v706: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 768 B/s wr, 27 op/s 2024-09-13T14:40:49.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[87147]: audit 2024-09-13T14:40:48.253742+0000 mon.a (mon.0) 1284 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:49.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:49 smithi099 bash[87147]: audit 2024-09-13T14:40:48.253742+0000 mon.a (mon.0) 1284 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:49.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:49 smithi177 bash[68594]: cluster 2024-09-13T14:40:48.163703+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-13T14:40:49.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:49 smithi177 bash[68594]: cluster 2024-09-13T14:40:48.163703+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-13T14:40:49.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:49 smithi177 bash[68594]: cluster 2024-09-13T14:40:48.251988+0000 mgr.y (mgr.26252) 430 : cluster [DBG] pgmap v706: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 768 B/s wr, 27 op/s 2024-09-13T14:40:49.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:49 smithi177 bash[68594]: cluster 2024-09-13T14:40:48.251988+0000 mgr.y (mgr.26252) 430 : cluster [DBG] pgmap v706: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 768 B/s wr, 27 op/s 2024-09-13T14:40:49.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:49 smithi177 bash[68594]: audit 2024-09-13T14:40:48.253742+0000 mon.a (mon.0) 1284 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:49 smithi177 bash[68594]: audit 2024-09-13T14:40:48.253742+0000 mon.a (mon.0) 1284 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:50.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: cluster 2024-09-13T14:40:49.180799+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-13T14:40:50.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: cluster 2024-09-13T14:40:49.180799+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-13T14:40:50.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.408383+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]} : dispatch 2024-09-13T14:40:50.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.408383+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]} : dispatch 2024-09-13T14:40:50.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.408954+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]} : dispatch 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.408954+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]} : dispatch 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.409430+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.409430+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.409787+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.409787+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.410194+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.410194+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.410548+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[84576]: audit 2024-09-13T14:40:49.410548+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: cluster 2024-09-13T14:40:49.180799+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-13T14:40:50.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: cluster 2024-09-13T14:40:49.180799+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.408383+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.408383+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.408954+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.408954+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.409430+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.409430+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.409787+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.409787+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.410194+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.410194+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.410548+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:50 smithi099 bash[87147]: audit 2024-09-13T14:40:49.410548+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[61439]: ts=2024-09-13T14:40:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: cluster 2024-09-13T14:40:49.180799+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: cluster 2024-09-13T14:40:49.180799+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.408383+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]} : dispatch 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.408383+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]} : dispatch 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.408954+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]} : dispatch 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.408954+0000 mon.a (mon.0) 1287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]} : dispatch 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.409430+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.409430+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.409787+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.409787+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]} : dispatch 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.410194+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.410194+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.410548+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]} : dispatch 2024-09-13T14:40:50.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:50 smithi177 bash[68594]: audit 2024-09-13T14:40:49.410548+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]} : dispatch 2024-09-13T14:40:51.370 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.QuiesceWatch (10098 ms) 2024-09-13T14:40:51.370 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.QuiesceWatchPP 2024-09-13T14:40:51.370 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.172346+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]': finished 2024-09-13T14:40:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.172346+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]': finished 2024-09-13T14:40:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.172550+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]}]': finished 2024-09-13T14:40:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.172550+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]}]': finished 2024-09-13T14:40:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.172696+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]}]': finished 2024-09-13T14:40:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.172696+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]}]': finished 2024-09-13T14:40:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.172789+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]}]': finished 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.172789+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]}]': finished 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.172901+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]}]': finished 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.172901+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]}]': finished 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.173059+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]}]': finished 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: audit 2024-09-13T14:40:50.173059+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]}]': finished 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: cluster 2024-09-13T14:40:50.179857+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: cluster 2024-09-13T14:40:50.179857+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: cluster 2024-09-13T14:40:50.253054+0000 mgr.y (mgr.26252) 431 : cluster [DBG] pgmap v709: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[84576]: cluster 2024-09-13T14:40:50.253054+0000 mgr.y (mgr.26252) 431 : cluster [DBG] pgmap v709: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.172346+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]': finished 2024-09-13T14:40:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.172346+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.172550+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.172550+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.172696+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.172696+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.172789+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.172789+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.172901+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.172901+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.173059+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: audit 2024-09-13T14:40:50.173059+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]}]': finished 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: cluster 2024-09-13T14:40:50.179857+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: cluster 2024-09-13T14:40:50.179857+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: cluster 2024-09-13T14:40:50.253054+0000 mgr.y (mgr.26252) 431 : cluster [DBG] pgmap v709: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-09-13T14:40:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:51 smithi099 bash[87147]: cluster 2024-09-13T14:40:50.253054+0000 mgr.y (mgr.26252) 431 : cluster [DBG] pgmap v709: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-09-13T14:40:51.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.172346+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.172346+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.172550+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.172550+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.0", "id": [6, 3]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.172696+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.172696+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.5", "id": [0, 3]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.172789+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.172789+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.11", "id": [0, 3]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.172901+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.172901+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [0, 1]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.173059+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: audit 2024-09-13T14:40:50.173059+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.d", "id": [0, 1]}]': finished 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: cluster 2024-09-13T14:40:50.179857+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-13T14:40:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: cluster 2024-09-13T14:40:50.179857+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-13T14:40:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: cluster 2024-09-13T14:40:50.253054+0000 mgr.y (mgr.26252) 431 : cluster [DBG] pgmap v709: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-09-13T14:40:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:51 smithi177 bash[68594]: cluster 2024-09-13T14:40:50.253054+0000 mgr.y (mgr.26252) 431 : cluster [DBG] pgmap v709: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-09-13T14:40:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:52 smithi099 bash[84576]: cluster 2024-09-13T14:40:51.210969+0000 mon.a (mon.0) 1299 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-13T14:40:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:52 smithi099 bash[84576]: cluster 2024-09-13T14:40:51.210969+0000 mon.a (mon.0) 1299 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-13T14:40:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:52 smithi099 bash[87147]: cluster 2024-09-13T14:40:51.210969+0000 mon.a (mon.0) 1299 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-13T14:40:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:52 smithi099 bash[87147]: cluster 2024-09-13T14:40:51.210969+0000 mon.a (mon.0) 1299 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-13T14:40:52.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:52 smithi177 bash[68594]: cluster 2024-09-13T14:40:51.210969+0000 mon.a (mon.0) 1299 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-13T14:40:52.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:52 smithi177 bash[68594]: cluster 2024-09-13T14:40:51.210969+0000 mon.a (mon.0) 1299 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-13T14:40:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:53 smithi099 bash[84576]: cluster 2024-09-13T14:40:52.215394+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-13T14:40:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:53 smithi099 bash[84576]: cluster 2024-09-13T14:40:52.215394+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-13T14:40:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:53 smithi099 bash[84576]: cluster 2024-09-13T14:40:52.254591+0000 mgr.y (mgr.26252) 432 : cluster [DBG] pgmap v712: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 6.0 KiB/s wr, 80 op/s 2024-09-13T14:40:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:53 smithi099 bash[84576]: cluster 2024-09-13T14:40:52.254591+0000 mgr.y (mgr.26252) 432 : cluster [DBG] pgmap v712: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 6.0 KiB/s wr, 80 op/s 2024-09-13T14:40:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:53 smithi099 bash[87147]: cluster 2024-09-13T14:40:52.215394+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-13T14:40:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:53 smithi099 bash[87147]: cluster 2024-09-13T14:40:52.215394+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-13T14:40:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:53 smithi099 bash[87147]: cluster 2024-09-13T14:40:52.254591+0000 mgr.y (mgr.26252) 432 : cluster [DBG] pgmap v712: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 6.0 KiB/s wr, 80 op/s 2024-09-13T14:40:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:53 smithi099 bash[87147]: cluster 2024-09-13T14:40:52.254591+0000 mgr.y (mgr.26252) 432 : cluster [DBG] pgmap v712: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 6.0 KiB/s wr, 80 op/s 2024-09-13T14:40:53.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:53 smithi177 bash[68594]: cluster 2024-09-13T14:40:52.215394+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-13T14:40:53.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:53 smithi177 bash[68594]: cluster 2024-09-13T14:40:52.215394+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-13T14:40:53.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:53 smithi177 bash[68594]: cluster 2024-09-13T14:40:52.254591+0000 mgr.y (mgr.26252) 432 : cluster [DBG] pgmap v712: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 6.0 KiB/s wr, 80 op/s 2024-09-13T14:40:53.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:53 smithi177 bash[68594]: cluster 2024-09-13T14:40:52.254591+0000 mgr.y (mgr.26252) 432 : cluster [DBG] pgmap v712: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 6.0 KiB/s wr, 80 op/s 2024-09-13T14:40:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:54 smithi099 bash[84576]: cluster 2024-09-13T14:40:53.227054+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-13T14:40:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:54 smithi099 bash[84576]: cluster 2024-09-13T14:40:53.227054+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-13T14:40:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:54 smithi099 bash[84576]: audit 2024-09-13T14:40:54.101694+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:54 smithi099 bash[84576]: audit 2024-09-13T14:40:54.101694+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:54 smithi099 bash[87147]: cluster 2024-09-13T14:40:53.227054+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-13T14:40:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:54 smithi099 bash[87147]: cluster 2024-09-13T14:40:53.227054+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-13T14:40:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:54 smithi099 bash[87147]: audit 2024-09-13T14:40:54.101694+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:54 smithi099 bash[87147]: audit 2024-09-13T14:40:54.101694+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:54.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:54 smithi177 bash[68594]: cluster 2024-09-13T14:40:53.227054+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-13T14:40:54.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:54 smithi177 bash[68594]: cluster 2024-09-13T14:40:53.227054+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-13T14:40:54.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:54 smithi177 bash[68594]: audit 2024-09-13T14:40:54.101694+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:54.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:54 smithi177 bash[68594]: audit 2024-09-13T14:40:54.101694+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:40:55.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:55 smithi099 bash[84576]: cluster 2024-09-13T14:40:54.233811+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-13T14:40:55.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:55 smithi099 bash[84576]: cluster 2024-09-13T14:40:54.233811+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-13T14:40:55.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:55 smithi099 bash[84576]: cluster 2024-09-13T14:40:54.256238+0000 mgr.y (mgr.26252) 433 : cluster [DBG] pgmap v715: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 6.7 KiB/s wr, 117 op/s 2024-09-13T14:40:55.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:55 smithi099 bash[84576]: cluster 2024-09-13T14:40:54.256238+0000 mgr.y (mgr.26252) 433 : cluster [DBG] pgmap v715: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 6.7 KiB/s wr, 117 op/s 2024-09-13T14:40:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:55 smithi099 bash[87147]: cluster 2024-09-13T14:40:54.233811+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-13T14:40:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:55 smithi099 bash[87147]: cluster 2024-09-13T14:40:54.233811+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-13T14:40:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:55 smithi099 bash[87147]: cluster 2024-09-13T14:40:54.256238+0000 mgr.y (mgr.26252) 433 : cluster [DBG] pgmap v715: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 6.7 KiB/s wr, 117 op/s 2024-09-13T14:40:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:55 smithi099 bash[87147]: cluster 2024-09-13T14:40:54.256238+0000 mgr.y (mgr.26252) 433 : cluster [DBG] pgmap v715: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 6.7 KiB/s wr, 117 op/s 2024-09-13T14:40:55.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:55 smithi177 bash[68594]: cluster 2024-09-13T14:40:54.233811+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-13T14:40:55.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:55 smithi177 bash[68594]: cluster 2024-09-13T14:40:54.233811+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-13T14:40:55.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:55 smithi177 bash[68594]: cluster 2024-09-13T14:40:54.256238+0000 mgr.y (mgr.26252) 433 : cluster [DBG] pgmap v715: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 6.7 KiB/s wr, 117 op/s 2024-09-13T14:40:55.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:55 smithi177 bash[68594]: cluster 2024-09-13T14:40:54.256238+0000 mgr.y (mgr.26252) 433 : cluster [DBG] pgmap v715: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 6.7 KiB/s wr, 117 op/s 2024-09-13T14:40:56.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:56 smithi099 bash[84576]: cluster 2024-09-13T14:40:55.250471+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-13T14:40:56.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:56 smithi099 bash[84576]: cluster 2024-09-13T14:40:55.250471+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-13T14:40:56.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:56 smithi099 bash[84576]: cluster 2024-09-13T14:40:56.158594+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-13T14:40:56.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:56 smithi099 bash[84576]: cluster 2024-09-13T14:40:56.158594+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-13T14:40:56.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:56 smithi099 bash[87147]: cluster 2024-09-13T14:40:55.250471+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-13T14:40:56.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:56 smithi099 bash[87147]: cluster 2024-09-13T14:40:55.250471+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-13T14:40:56.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:56 smithi099 bash[87147]: cluster 2024-09-13T14:40:56.158594+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-13T14:40:56.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:56 smithi099 bash[87147]: cluster 2024-09-13T14:40:56.158594+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-13T14:40:56.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:56 smithi177 bash[68594]: cluster 2024-09-13T14:40:55.250471+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-13T14:40:56.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:56 smithi177 bash[68594]: cluster 2024-09-13T14:40:55.250471+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-13T14:40:56.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:56 smithi177 bash[68594]: cluster 2024-09-13T14:40:56.158594+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-13T14:40:56.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:56 smithi177 bash[68594]: cluster 2024-09-13T14:40:56.158594+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-13T14:40:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:40:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:40:57] "GET /metrics HTTP/1.1" 200 82352 "" "Prometheus/2.51.0" 2024-09-13T14:40:57.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:57 smithi099 bash[84576]: cluster 2024-09-13T14:40:56.257467+0000 mgr.y (mgr.26252) 434 : cluster [DBG] pgmap v718: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 KiB/s wr, 48 op/s 2024-09-13T14:40:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:57 smithi099 bash[84576]: cluster 2024-09-13T14:40:56.257467+0000 mgr.y (mgr.26252) 434 : cluster [DBG] pgmap v718: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 KiB/s wr, 48 op/s 2024-09-13T14:40:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:57 smithi099 bash[84576]: cluster 2024-09-13T14:40:57.158980+0000 mon.a (mon.0) 1306 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-13T14:40:57.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:57 smithi099 bash[84576]: cluster 2024-09-13T14:40:57.158980+0000 mon.a (mon.0) 1306 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-13T14:40:57.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:57 smithi099 bash[87147]: cluster 2024-09-13T14:40:56.257467+0000 mgr.y (mgr.26252) 434 : cluster [DBG] pgmap v718: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 KiB/s wr, 48 op/s 2024-09-13T14:40:57.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:57 smithi099 bash[87147]: cluster 2024-09-13T14:40:56.257467+0000 mgr.y (mgr.26252) 434 : cluster [DBG] pgmap v718: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 KiB/s wr, 48 op/s 2024-09-13T14:40:57.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:57 smithi099 bash[87147]: cluster 2024-09-13T14:40:57.158980+0000 mon.a (mon.0) 1306 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-13T14:40:57.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:57 smithi099 bash[87147]: cluster 2024-09-13T14:40:57.158980+0000 mon.a (mon.0) 1306 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-13T14:40:57.329 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.QuiesceWatchPP (5959 ms) 2024-09-13T14:40:57.329 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.QuiesceWatchError 2024-09-13T14:40:57.329 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:40:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:57 smithi177 bash[68594]: cluster 2024-09-13T14:40:56.257467+0000 mgr.y (mgr.26252) 434 : cluster [DBG] pgmap v718: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 KiB/s wr, 48 op/s 2024-09-13T14:40:57.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:57 smithi177 bash[68594]: cluster 2024-09-13T14:40:56.257467+0000 mgr.y (mgr.26252) 434 : cluster [DBG] pgmap v718: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 KiB/s wr, 48 op/s 2024-09-13T14:40:57.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:57 smithi177 bash[68594]: cluster 2024-09-13T14:40:57.158980+0000 mon.a (mon.0) 1306 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-13T14:40:57.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:57 smithi177 bash[68594]: cluster 2024-09-13T14:40:57.158980+0000 mon.a (mon.0) 1306 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-13T14:40:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:59 smithi099 bash[84576]: cluster 2024-09-13T14:40:58.258925+0000 mgr.y (mgr.26252) 435 : cluster [DBG] pgmap v721: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.5 KiB/s wr, 80 op/s 2024-09-13T14:40:59.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:59 smithi099 bash[84576]: cluster 2024-09-13T14:40:58.258925+0000 mgr.y (mgr.26252) 435 : cluster [DBG] pgmap v721: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.5 KiB/s wr, 80 op/s 2024-09-13T14:40:59.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:59 smithi099 bash[84576]: cluster 2024-09-13T14:40:58.259815+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-13T14:40:59.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:40:59 smithi099 bash[84576]: cluster 2024-09-13T14:40:58.259815+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-13T14:40:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:59 smithi099 bash[87147]: cluster 2024-09-13T14:40:58.258925+0000 mgr.y (mgr.26252) 435 : cluster [DBG] pgmap v721: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.5 KiB/s wr, 80 op/s 2024-09-13T14:40:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:59 smithi099 bash[87147]: cluster 2024-09-13T14:40:58.258925+0000 mgr.y (mgr.26252) 435 : cluster [DBG] pgmap v721: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.5 KiB/s wr, 80 op/s 2024-09-13T14:40:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:59 smithi099 bash[87147]: cluster 2024-09-13T14:40:58.259815+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-13T14:40:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:40:59 smithi099 bash[87147]: cluster 2024-09-13T14:40:58.259815+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-13T14:40:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:59 smithi177 bash[68594]: cluster 2024-09-13T14:40:58.258925+0000 mgr.y (mgr.26252) 435 : cluster [DBG] pgmap v721: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.5 KiB/s wr, 80 op/s 2024-09-13T14:40:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:59 smithi177 bash[68594]: cluster 2024-09-13T14:40:58.258925+0000 mgr.y (mgr.26252) 435 : cluster [DBG] pgmap v721: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.5 KiB/s wr, 80 op/s 2024-09-13T14:40:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:59 smithi177 bash[68594]: cluster 2024-09-13T14:40:58.259815+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-13T14:40:59.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:40:59 smithi177 bash[68594]: cluster 2024-09-13T14:40:58.259815+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-13T14:41:00.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:59.271155+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-13T14:41:00.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:00 smithi099 bash[84576]: cluster 2024-09-13T14:40:59.271155+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-13T14:41:00.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:59.271155+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-13T14:41:00.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:00 smithi099 bash[87147]: cluster 2024-09-13T14:40:59.271155+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-13T14:41:00.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:59.271155+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-13T14:41:00.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:00 smithi177 bash[68594]: cluster 2024-09-13T14:40:59.271155+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-13T14:41:00.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:41:00 smithi177 bash[61439]: ts=2024-09-13T14:41:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[84576]: cluster 2024-09-13T14:41:00.260112+0000 mgr.y (mgr.26252) 436 : cluster [DBG] pgmap v723: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 6.3 KiB/s wr, 110 op/s 2024-09-13T14:41:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[84576]: cluster 2024-09-13T14:41:00.260112+0000 mgr.y (mgr.26252) 436 : cluster [DBG] pgmap v723: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 6.3 KiB/s wr, 110 op/s 2024-09-13T14:41:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[84576]: cluster 2024-09-13T14:41:00.271866+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-13T14:41:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[84576]: cluster 2024-09-13T14:41:00.271866+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-13T14:41:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[84576]: cluster 2024-09-13T14:41:01.158114+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-13T14:41:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[84576]: cluster 2024-09-13T14:41:01.158114+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-13T14:41:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[87147]: cluster 2024-09-13T14:41:00.260112+0000 mgr.y (mgr.26252) 436 : cluster [DBG] pgmap v723: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 6.3 KiB/s wr, 110 op/s 2024-09-13T14:41:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[87147]: cluster 2024-09-13T14:41:00.260112+0000 mgr.y (mgr.26252) 436 : cluster [DBG] pgmap v723: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 6.3 KiB/s wr, 110 op/s 2024-09-13T14:41:01.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[87147]: cluster 2024-09-13T14:41:00.271866+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-13T14:41:01.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[87147]: cluster 2024-09-13T14:41:00.271866+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-13T14:41:01.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[87147]: cluster 2024-09-13T14:41:01.158114+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-13T14:41:01.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:01 smithi099 bash[87147]: cluster 2024-09-13T14:41:01.158114+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-13T14:41:01.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:01 smithi177 bash[68594]: cluster 2024-09-13T14:41:00.260112+0000 mgr.y (mgr.26252) 436 : cluster [DBG] pgmap v723: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 6.3 KiB/s wr, 110 op/s 2024-09-13T14:41:01.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:01 smithi177 bash[68594]: cluster 2024-09-13T14:41:00.260112+0000 mgr.y (mgr.26252) 436 : cluster [DBG] pgmap v723: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 6.3 KiB/s wr, 110 op/s 2024-09-13T14:41:01.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:01 smithi177 bash[68594]: cluster 2024-09-13T14:41:00.271866+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-13T14:41:01.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:01 smithi177 bash[68594]: cluster 2024-09-13T14:41:00.271866+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-13T14:41:01.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:01 smithi177 bash[68594]: cluster 2024-09-13T14:41:01.158114+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-13T14:41:01.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:01 smithi177 bash[68594]: cluster 2024-09-13T14:41:01.158114+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-13T14:41:03.353 INFO:tasks.workunit.client.0.smithi099.stdout:10.00% done 2024-09-13T14:41:03.353 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.QuiesceWatchError (6023 ms) 2024-09-13T14:41:03.353 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.QuiesceWatchTimeout 2024-09-13T14:41:03.353 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:41:03.389 INFO:tasks.workunit.client.0.smithi099.stderr:test quiesce is not long enough to time out 2024-09-13T14:41:03.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:03 smithi099 bash[84576]: cluster 2024-09-13T14:41:02.161850+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-13T14:41:03.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:03 smithi099 bash[84576]: cluster 2024-09-13T14:41:02.161850+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-13T14:41:03.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:03 smithi099 bash[84576]: cluster 2024-09-13T14:41:02.261282+0000 mgr.y (mgr.26252) 437 : cluster [DBG] pgmap v727: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.0 KiB/s wr, 32 op/s 2024-09-13T14:41:03.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:03 smithi099 bash[84576]: cluster 2024-09-13T14:41:02.261282+0000 mgr.y (mgr.26252) 437 : cluster [DBG] pgmap v727: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.0 KiB/s wr, 32 op/s 2024-09-13T14:41:03.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:03 smithi099 bash[87147]: cluster 2024-09-13T14:41:02.161850+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-13T14:41:03.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:03 smithi099 bash[87147]: cluster 2024-09-13T14:41:02.161850+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-13T14:41:03.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:03 smithi099 bash[87147]: cluster 2024-09-13T14:41:02.261282+0000 mgr.y (mgr.26252) 437 : cluster [DBG] pgmap v727: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.0 KiB/s wr, 32 op/s 2024-09-13T14:41:03.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:03 smithi099 bash[87147]: cluster 2024-09-13T14:41:02.261282+0000 mgr.y (mgr.26252) 437 : cluster [DBG] pgmap v727: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.0 KiB/s wr, 32 op/s 2024-09-13T14:41:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:03 smithi177 bash[68594]: cluster 2024-09-13T14:41:02.161850+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-13T14:41:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:03 smithi177 bash[68594]: cluster 2024-09-13T14:41:02.161850+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-13T14:41:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:03 smithi177 bash[68594]: cluster 2024-09-13T14:41:02.261282+0000 mgr.y (mgr.26252) 437 : cluster [DBG] pgmap v727: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.0 KiB/s wr, 32 op/s 2024-09-13T14:41:03.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:03 smithi177 bash[68594]: cluster 2024-09-13T14:41:02.261282+0000 mgr.y (mgr.26252) 437 : cluster [DBG] pgmap v727: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.0 KiB/s wr, 32 op/s 2024-09-13T14:41:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:04 smithi099 bash[84576]: cluster 2024-09-13T14:41:03.174454+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-13T14:41:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:04 smithi099 bash[84576]: cluster 2024-09-13T14:41:03.174454+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-13T14:41:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:04 smithi099 bash[84576]: audit 2024-09-13T14:41:03.253652+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:04 smithi099 bash[84576]: audit 2024-09-13T14:41:03.253652+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:04 smithi099 bash[87147]: cluster 2024-09-13T14:41:03.174454+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-13T14:41:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:04 smithi099 bash[87147]: cluster 2024-09-13T14:41:03.174454+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-13T14:41:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:04 smithi099 bash[87147]: audit 2024-09-13T14:41:03.253652+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:04 smithi099 bash[87147]: audit 2024-09-13T14:41:03.253652+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:04.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:04 smithi177 bash[68594]: cluster 2024-09-13T14:41:03.174454+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-13T14:41:04.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:04 smithi177 bash[68594]: cluster 2024-09-13T14:41:03.174454+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-13T14:41:04.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:04 smithi177 bash[68594]: audit 2024-09-13T14:41:03.253652+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:04.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:04 smithi177 bash[68594]: audit 2024-09-13T14:41:03.253652+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:05 smithi099 bash[84576]: cluster 2024-09-13T14:41:04.264901+0000 mgr.y (mgr.26252) 438 : cluster [DBG] pgmap v729: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 6.0 KiB/s wr, 99 op/s 2024-09-13T14:41:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:05 smithi099 bash[84576]: cluster 2024-09-13T14:41:04.264901+0000 mgr.y (mgr.26252) 438 : cluster [DBG] pgmap v729: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 6.0 KiB/s wr, 99 op/s 2024-09-13T14:41:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:05 smithi099 bash[87147]: cluster 2024-09-13T14:41:04.264901+0000 mgr.y (mgr.26252) 438 : cluster [DBG] pgmap v729: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 6.0 KiB/s wr, 99 op/s 2024-09-13T14:41:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:05 smithi099 bash[87147]: cluster 2024-09-13T14:41:04.264901+0000 mgr.y (mgr.26252) 438 : cluster [DBG] pgmap v729: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 6.0 KiB/s wr, 99 op/s 2024-09-13T14:41:05.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:05 smithi177 bash[68594]: cluster 2024-09-13T14:41:04.264901+0000 mgr.y (mgr.26252) 438 : cluster [DBG] pgmap v729: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 6.0 KiB/s wr, 99 op/s 2024-09-13T14:41:05.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:05 smithi177 bash[68594]: cluster 2024-09-13T14:41:04.264901+0000 mgr.y (mgr.26252) 438 : cluster [DBG] pgmap v729: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 6.0 KiB/s wr, 99 op/s 2024-09-13T14:41:07.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:41:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:41:07] "GET /metrics HTTP/1.1" 200 82354 "" "Prometheus/2.51.0" 2024-09-13T14:41:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:07 smithi099 bash[84576]: cluster 2024-09-13T14:41:06.162504+0000 mon.a (mon.0) 1314 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-13T14:41:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:07 smithi099 bash[84576]: cluster 2024-09-13T14:41:06.162504+0000 mon.a (mon.0) 1314 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-13T14:41:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:07 smithi099 bash[84576]: cluster 2024-09-13T14:41:06.266106+0000 mgr.y (mgr.26252) 439 : cluster [DBG] pgmap v731: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.7 KiB/s wr, 78 op/s 2024-09-13T14:41:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:07 smithi099 bash[84576]: cluster 2024-09-13T14:41:06.266106+0000 mgr.y (mgr.26252) 439 : cluster [DBG] pgmap v731: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.7 KiB/s wr, 78 op/s 2024-09-13T14:41:07.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:07 smithi099 bash[87147]: cluster 2024-09-13T14:41:06.162504+0000 mon.a (mon.0) 1314 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-13T14:41:07.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:07 smithi099 bash[87147]: cluster 2024-09-13T14:41:06.162504+0000 mon.a (mon.0) 1314 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-13T14:41:07.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:07 smithi099 bash[87147]: cluster 2024-09-13T14:41:06.266106+0000 mgr.y (mgr.26252) 439 : cluster [DBG] pgmap v731: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.7 KiB/s wr, 78 op/s 2024-09-13T14:41:07.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:07 smithi099 bash[87147]: cluster 2024-09-13T14:41:06.266106+0000 mgr.y (mgr.26252) 439 : cluster [DBG] pgmap v731: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.7 KiB/s wr, 78 op/s 2024-09-13T14:41:07.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:07 smithi177 bash[68594]: cluster 2024-09-13T14:41:06.162504+0000 mon.a (mon.0) 1314 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-13T14:41:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:07 smithi177 bash[68594]: cluster 2024-09-13T14:41:06.162504+0000 mon.a (mon.0) 1314 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-13T14:41:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:07 smithi177 bash[68594]: cluster 2024-09-13T14:41:06.266106+0000 mgr.y (mgr.26252) 439 : cluster [DBG] pgmap v731: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.7 KiB/s wr, 78 op/s 2024-09-13T14:41:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:07 smithi177 bash[68594]: cluster 2024-09-13T14:41:06.266106+0000 mgr.y (mgr.26252) 439 : cluster [DBG] pgmap v731: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.7 KiB/s wr, 78 op/s 2024-09-13T14:41:09.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:09 smithi177 bash[68594]: cluster 2024-09-13T14:41:08.269765+0000 mgr.y (mgr.26252) 440 : cluster [DBG] pgmap v732: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.3 KiB/s wr, 70 op/s 2024-09-13T14:41:09.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:09 smithi177 bash[68594]: cluster 2024-09-13T14:41:08.269765+0000 mgr.y (mgr.26252) 440 : cluster [DBG] pgmap v732: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.3 KiB/s wr, 70 op/s 2024-09-13T14:41:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:09 smithi099 bash[84576]: cluster 2024-09-13T14:41:08.269765+0000 mgr.y (mgr.26252) 440 : cluster [DBG] pgmap v732: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.3 KiB/s wr, 70 op/s 2024-09-13T14:41:09.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:09 smithi099 bash[84576]: cluster 2024-09-13T14:41:08.269765+0000 mgr.y (mgr.26252) 440 : cluster [DBG] pgmap v732: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.3 KiB/s wr, 70 op/s 2024-09-13T14:41:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:09 smithi099 bash[87147]: cluster 2024-09-13T14:41:08.269765+0000 mgr.y (mgr.26252) 440 : cluster [DBG] pgmap v732: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.3 KiB/s wr, 70 op/s 2024-09-13T14:41:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:09 smithi099 bash[87147]: cluster 2024-09-13T14:41:08.269765+0000 mgr.y (mgr.26252) 440 : cluster [DBG] pgmap v732: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.3 KiB/s wr, 70 op/s 2024-09-13T14:41:10.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:41:10 smithi177 bash[61439]: ts=2024-09-13T14:41:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:11.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:11 smithi177 bash[68594]: cluster 2024-09-13T14:41:10.273399+0000 mgr.y (mgr.26252) 441 : cluster [DBG] pgmap v733: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 3.2 KiB/s wr, 54 op/s 2024-09-13T14:41:11.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:11 smithi177 bash[68594]: cluster 2024-09-13T14:41:10.273399+0000 mgr.y (mgr.26252) 441 : cluster [DBG] pgmap v733: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 3.2 KiB/s wr, 54 op/s 2024-09-13T14:41:11.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:11 smithi177 bash[68594]: cluster 2024-09-13T14:41:11.167085+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-13T14:41:11.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:11 smithi177 bash[68594]: cluster 2024-09-13T14:41:11.167085+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-13T14:41:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:11 smithi099 bash[84576]: cluster 2024-09-13T14:41:10.273399+0000 mgr.y (mgr.26252) 441 : cluster [DBG] pgmap v733: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 3.2 KiB/s wr, 54 op/s 2024-09-13T14:41:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:11 smithi099 bash[84576]: cluster 2024-09-13T14:41:10.273399+0000 mgr.y (mgr.26252) 441 : cluster [DBG] pgmap v733: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 3.2 KiB/s wr, 54 op/s 2024-09-13T14:41:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:11 smithi099 bash[84576]: cluster 2024-09-13T14:41:11.167085+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-13T14:41:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:11 smithi099 bash[84576]: cluster 2024-09-13T14:41:11.167085+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-13T14:41:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:11 smithi099 bash[87147]: cluster 2024-09-13T14:41:10.273399+0000 mgr.y (mgr.26252) 441 : cluster [DBG] pgmap v733: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 3.2 KiB/s wr, 54 op/s 2024-09-13T14:41:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:11 smithi099 bash[87147]: cluster 2024-09-13T14:41:10.273399+0000 mgr.y (mgr.26252) 441 : cluster [DBG] pgmap v733: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 3.2 KiB/s wr, 54 op/s 2024-09-13T14:41:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:11 smithi099 bash[87147]: cluster 2024-09-13T14:41:11.167085+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-13T14:41:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:11 smithi099 bash[87147]: cluster 2024-09-13T14:41:11.167085+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-13T14:41:12.422 INFO:tasks.workunit.client.0.smithi099.stderr:test quiesce is timed out 2024-09-13T14:41:12.428 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 0 2024-09-13T14:41:13.428 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 1 2024-09-13T14:41:13.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:13 smithi177 bash[68594]: cluster 2024-09-13T14:41:12.275352+0000 mgr.y (mgr.26252) 442 : cluster [DBG] pgmap v735: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 255 B/s wr, 4 op/s 2024-09-13T14:41:13.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:13 smithi177 bash[68594]: cluster 2024-09-13T14:41:12.275352+0000 mgr.y (mgr.26252) 442 : cluster [DBG] pgmap v735: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 255 B/s wr, 4 op/s 2024-09-13T14:41:13.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:13 smithi177 bash[68594]: cluster 2024-09-13T14:41:12.351025+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-13T14:41:13.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:13 smithi177 bash[68594]: cluster 2024-09-13T14:41:12.351025+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-13T14:41:13.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:13 smithi099 bash[84576]: cluster 2024-09-13T14:41:12.275352+0000 mgr.y (mgr.26252) 442 : cluster [DBG] pgmap v735: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 255 B/s wr, 4 op/s 2024-09-13T14:41:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:13 smithi099 bash[84576]: cluster 2024-09-13T14:41:12.275352+0000 mgr.y (mgr.26252) 442 : cluster [DBG] pgmap v735: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 255 B/s wr, 4 op/s 2024-09-13T14:41:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:13 smithi099 bash[84576]: cluster 2024-09-13T14:41:12.351025+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-13T14:41:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:13 smithi099 bash[84576]: cluster 2024-09-13T14:41:12.351025+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-13T14:41:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:13 smithi099 bash[87147]: cluster 2024-09-13T14:41:12.275352+0000 mgr.y (mgr.26252) 442 : cluster [DBG] pgmap v735: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 255 B/s wr, 4 op/s 2024-09-13T14:41:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:13 smithi099 bash[87147]: cluster 2024-09-13T14:41:12.275352+0000 mgr.y (mgr.26252) 442 : cluster [DBG] pgmap v735: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 255 B/s wr, 4 op/s 2024-09-13T14:41:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:13 smithi099 bash[87147]: cluster 2024-09-13T14:41:12.351025+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-13T14:41:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:13 smithi099 bash[87147]: cluster 2024-09-13T14:41:12.351025+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-13T14:41:14.428 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 2 2024-09-13T14:41:15.428 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 3 2024-09-13T14:41:15.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:15 smithi177 bash[68594]: cluster 2024-09-13T14:41:14.280672+0000 mgr.y (mgr.26252) 443 : cluster [DBG] pgmap v737: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 639 B/s wr, 8 op/s 2024-09-13T14:41:15.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:15 smithi177 bash[68594]: cluster 2024-09-13T14:41:14.280672+0000 mgr.y (mgr.26252) 443 : cluster [DBG] pgmap v737: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 639 B/s wr, 8 op/s 2024-09-13T14:41:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:15 smithi099 bash[84576]: cluster 2024-09-13T14:41:14.280672+0000 mgr.y (mgr.26252) 443 : cluster [DBG] pgmap v737: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 639 B/s wr, 8 op/s 2024-09-13T14:41:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:15 smithi099 bash[84576]: cluster 2024-09-13T14:41:14.280672+0000 mgr.y (mgr.26252) 443 : cluster [DBG] pgmap v737: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 639 B/s wr, 8 op/s 2024-09-13T14:41:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:15 smithi099 bash[87147]: cluster 2024-09-13T14:41:14.280672+0000 mgr.y (mgr.26252) 443 : cluster [DBG] pgmap v737: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 639 B/s wr, 8 op/s 2024-09-13T14:41:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:15 smithi099 bash[87147]: cluster 2024-09-13T14:41:14.280672+0000 mgr.y (mgr.26252) 443 : cluster [DBG] pgmap v737: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 639 B/s wr, 8 op/s 2024-09-13T14:41:16.429 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 4 2024-09-13T14:41:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:41:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:41:17] "GET /metrics HTTP/1.1" 200 82354 "" "Prometheus/2.51.0" 2024-09-13T14:41:17.428 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 5 2024-09-13T14:41:17.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:17 smithi177 bash[68594]: cluster 2024-09-13T14:41:16.281850+0000 mgr.y (mgr.26252) 444 : cluster [DBG] pgmap v738: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T14:41:17.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:17 smithi177 bash[68594]: cluster 2024-09-13T14:41:16.281850+0000 mgr.y (mgr.26252) 444 : cluster [DBG] pgmap v738: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T14:41:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:17 smithi099 bash[84576]: cluster 2024-09-13T14:41:16.281850+0000 mgr.y (mgr.26252) 444 : cluster [DBG] pgmap v738: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T14:41:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:17 smithi099 bash[84576]: cluster 2024-09-13T14:41:16.281850+0000 mgr.y (mgr.26252) 444 : cluster [DBG] pgmap v738: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T14:41:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:17 smithi099 bash[87147]: cluster 2024-09-13T14:41:16.281850+0000 mgr.y (mgr.26252) 444 : cluster [DBG] pgmap v738: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T14:41:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:17 smithi099 bash[87147]: cluster 2024-09-13T14:41:16.281850+0000 mgr.y (mgr.26252) 444 : cluster [DBG] pgmap v738: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T14:41:18.429 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 6 2024-09-13T14:41:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:18 smithi099 bash[84576]: audit 2024-09-13T14:41:18.254304+0000 mon.a (mon.0) 1317 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:18 smithi099 bash[84576]: audit 2024-09-13T14:41:18.254304+0000 mon.a (mon.0) 1317 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:18 smithi099 bash[87147]: audit 2024-09-13T14:41:18.254304+0000 mon.a (mon.0) 1317 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:18 smithi099 bash[87147]: audit 2024-09-13T14:41:18.254304+0000 mon.a (mon.0) 1317 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:18.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:18 smithi177 bash[68594]: audit 2024-09-13T14:41:18.254304+0000 mon.a (mon.0) 1317 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:18.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:18 smithi177 bash[68594]: audit 2024-09-13T14:41:18.254304+0000 mon.a (mon.0) 1317 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:19.429 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 7 2024-09-13T14:41:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:19 smithi099 bash[84576]: cluster 2024-09-13T14:41:18.310812+0000 mgr.y (mgr.26252) 445 : cluster [DBG] pgmap v739: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 382 B/s wr, 4 op/s 2024-09-13T14:41:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:19 smithi099 bash[84576]: cluster 2024-09-13T14:41:18.310812+0000 mgr.y (mgr.26252) 445 : cluster [DBG] pgmap v739: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 382 B/s wr, 4 op/s 2024-09-13T14:41:19.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:19 smithi099 bash[87147]: cluster 2024-09-13T14:41:18.310812+0000 mgr.y (mgr.26252) 445 : cluster [DBG] pgmap v739: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 382 B/s wr, 4 op/s 2024-09-13T14:41:19.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:19 smithi099 bash[87147]: cluster 2024-09-13T14:41:18.310812+0000 mgr.y (mgr.26252) 445 : cluster [DBG] pgmap v739: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 382 B/s wr, 4 op/s 2024-09-13T14:41:19.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:19 smithi177 bash[68594]: cluster 2024-09-13T14:41:18.310812+0000 mgr.y (mgr.26252) 445 : cluster [DBG] pgmap v739: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 382 B/s wr, 4 op/s 2024-09-13T14:41:19.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:19 smithi177 bash[68594]: cluster 2024-09-13T14:41:18.310812+0000 mgr.y (mgr.26252) 445 : cluster [DBG] pgmap v739: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 382 B/s wr, 4 op/s 2024-09-13T14:41:20.429 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 8 2024-09-13T14:41:20.609 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:41:20 smithi177 bash[61439]: ts=2024-09-13T14:41:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:21.429 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 9 2024-09-13T14:41:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:21 smithi099 bash[84576]: cluster 2024-09-13T14:41:20.314428+0000 mgr.y (mgr.26252) 446 : cluster [DBG] pgmap v740: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 335 B/s wr, 4 op/s 2024-09-13T14:41:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:21 smithi099 bash[84576]: cluster 2024-09-13T14:41:20.314428+0000 mgr.y (mgr.26252) 446 : cluster [DBG] pgmap v740: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 335 B/s wr, 4 op/s 2024-09-13T14:41:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:21 smithi099 bash[84576]: audit 2024-09-13T14:41:21.274434+0000 mon.a (mon.0) 1318 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:21 smithi099 bash[84576]: audit 2024-09-13T14:41:21.274434+0000 mon.a (mon.0) 1318 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:21 smithi099 bash[87147]: cluster 2024-09-13T14:41:20.314428+0000 mgr.y (mgr.26252) 446 : cluster [DBG] pgmap v740: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 335 B/s wr, 4 op/s 2024-09-13T14:41:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:21 smithi099 bash[87147]: cluster 2024-09-13T14:41:20.314428+0000 mgr.y (mgr.26252) 446 : cluster [DBG] pgmap v740: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 335 B/s wr, 4 op/s 2024-09-13T14:41:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:21 smithi099 bash[87147]: audit 2024-09-13T14:41:21.274434+0000 mon.a (mon.0) 1318 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:21 smithi099 bash[87147]: audit 2024-09-13T14:41:21.274434+0000 mon.a (mon.0) 1318 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:21.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:21 smithi177 bash[68594]: cluster 2024-09-13T14:41:20.314428+0000 mgr.y (mgr.26252) 446 : cluster [DBG] pgmap v740: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 335 B/s wr, 4 op/s 2024-09-13T14:41:21.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:21 smithi177 bash[68594]: cluster 2024-09-13T14:41:20.314428+0000 mgr.y (mgr.26252) 446 : cluster [DBG] pgmap v740: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 335 B/s wr, 4 op/s 2024-09-13T14:41:21.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:21 smithi177 bash[68594]: audit 2024-09-13T14:41:21.274434+0000 mon.a (mon.0) 1318 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:21.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:21 smithi177 bash[68594]: audit 2024-09-13T14:41:21.274434+0000 mon.a (mon.0) 1318 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:22.429 INFO:tasks.workunit.client.0.smithi099.stderr:waiting for timed out ... 10 2024-09-13T14:41:23.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[84576]: audit 2024-09-13T14:41:21.875131+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[84576]: audit 2024-09-13T14:41:21.875131+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[84576]: audit 2024-09-13T14:41:21.880678+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[84576]: audit 2024-09-13T14:41:21.880678+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[84576]: cluster 2024-09-13T14:41:22.316216+0000 mgr.y (mgr.26252) 447 : cluster [DBG] pgmap v741: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 305 B/s wr, 3 op/s 2024-09-13T14:41:23.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[84576]: cluster 2024-09-13T14:41:22.316216+0000 mgr.y (mgr.26252) 447 : cluster [DBG] pgmap v741: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 305 B/s wr, 3 op/s 2024-09-13T14:41:23.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[87147]: audit 2024-09-13T14:41:21.875131+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[87147]: audit 2024-09-13T14:41:21.875131+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[87147]: audit 2024-09-13T14:41:21.880678+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[87147]: audit 2024-09-13T14:41:21.880678+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[87147]: cluster 2024-09-13T14:41:22.316216+0000 mgr.y (mgr.26252) 447 : cluster [DBG] pgmap v741: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 305 B/s wr, 3 op/s 2024-09-13T14:41:23.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:22 smithi099 bash[87147]: cluster 2024-09-13T14:41:22.316216+0000 mgr.y (mgr.26252) 447 : cluster [DBG] pgmap v741: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 305 B/s wr, 3 op/s 2024-09-13T14:41:23.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:22 smithi177 bash[68594]: audit 2024-09-13T14:41:21.875131+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:22 smithi177 bash[68594]: audit 2024-09-13T14:41:21.875131+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:22 smithi177 bash[68594]: audit 2024-09-13T14:41:21.880678+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:22 smithi177 bash[68594]: audit 2024-09-13T14:41:21.880678+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:23.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:22 smithi177 bash[68594]: cluster 2024-09-13T14:41:22.316216+0000 mgr.y (mgr.26252) 447 : cluster [DBG] pgmap v741: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 305 B/s wr, 3 op/s 2024-09-13T14:41:23.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:22 smithi177 bash[68594]: cluster 2024-09-13T14:41:22.316216+0000 mgr.y (mgr.26252) 447 : cluster [DBG] pgmap v741: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 305 B/s wr, 3 op/s 2024-09-13T14:41:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:25 smithi099 bash[84576]: cluster 2024-09-13T14:41:24.321749+0000 mgr.y (mgr.26252) 448 : cluster [DBG] pgmap v742: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 256 B/s wr, 3 op/s 2024-09-13T14:41:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:25 smithi099 bash[84576]: cluster 2024-09-13T14:41:24.321749+0000 mgr.y (mgr.26252) 448 : cluster [DBG] pgmap v742: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 256 B/s wr, 3 op/s 2024-09-13T14:41:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:25 smithi099 bash[87147]: cluster 2024-09-13T14:41:24.321749+0000 mgr.y (mgr.26252) 448 : cluster [DBG] pgmap v742: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 256 B/s wr, 3 op/s 2024-09-13T14:41:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:25 smithi099 bash[87147]: cluster 2024-09-13T14:41:24.321749+0000 mgr.y (mgr.26252) 448 : cluster [DBG] pgmap v742: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 256 B/s wr, 3 op/s 2024-09-13T14:41:25.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:25 smithi177 bash[68594]: cluster 2024-09-13T14:41:24.321749+0000 mgr.y (mgr.26252) 448 : cluster [DBG] pgmap v742: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 256 B/s wr, 3 op/s 2024-09-13T14:41:25.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:25 smithi177 bash[68594]: cluster 2024-09-13T14:41:24.321749+0000 mgr.y (mgr.26252) 448 : cluster [DBG] pgmap v742: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 256 B/s wr, 3 op/s 2024-09-13T14:41:27.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:41:27] "GET /metrics HTTP/1.1" 200 82355 "" "Prometheus/2.51.0" 2024-09-13T14:41:27.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[87147]: cluster 2024-09-13T14:41:26.322974+0000 mgr.y (mgr.26252) 449 : cluster [DBG] pgmap v743: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[87147]: cluster 2024-09-13T14:41:26.322974+0000 mgr.y (mgr.26252) 449 : cluster [DBG] pgmap v743: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[87147]: audit 2024-09-13T14:41:27.345853+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[87147]: audit 2024-09-13T14:41:27.345853+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[87147]: audit 2024-09-13T14:41:27.353082+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[87147]: audit 2024-09-13T14:41:27.353082+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[84576]: cluster 2024-09-13T14:41:26.322974+0000 mgr.y (mgr.26252) 449 : cluster [DBG] pgmap v743: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[84576]: cluster 2024-09-13T14:41:26.322974+0000 mgr.y (mgr.26252) 449 : cluster [DBG] pgmap v743: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[84576]: audit 2024-09-13T14:41:27.345853+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[84576]: audit 2024-09-13T14:41:27.345853+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[84576]: audit 2024-09-13T14:41:27.353082+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:27 smithi099 bash[84576]: audit 2024-09-13T14:41:27.353082+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:27 smithi177 bash[68594]: cluster 2024-09-13T14:41:26.322974+0000 mgr.y (mgr.26252) 449 : cluster [DBG] pgmap v743: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:27.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:27 smithi177 bash[68594]: cluster 2024-09-13T14:41:26.322974+0000 mgr.y (mgr.26252) 449 : cluster [DBG] pgmap v743: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:27.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:27 smithi177 bash[68594]: audit 2024-09-13T14:41:27.345853+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:27 smithi177 bash[68594]: audit 2024-09-13T14:41:27.345853+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:27 smithi177 bash[68594]: audit 2024-09-13T14:41:27.353082+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:27.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:27 smithi177 bash[68594]: audit 2024-09-13T14:41:27.353082+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[84576]: audit 2024-09-13T14:41:27.670070+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[84576]: audit 2024-09-13T14:41:27.670070+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[84576]: audit 2024-09-13T14:41:27.676354+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[84576]: audit 2024-09-13T14:41:27.676354+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[84576]: audit 2024-09-13T14:41:28.322414+0000 mon.a (mon.0) 1325 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:29.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[84576]: audit 2024-09-13T14:41:28.322414+0000 mon.a (mon.0) 1325 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:29.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[84576]: audit 2024-09-13T14:41:28.323570+0000 mon.a (mon.0) 1326 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:29.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[84576]: audit 2024-09-13T14:41:28.323570+0000 mon.a (mon.0) 1326 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:29.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[84576]: audit 2024-09-13T14:41:28.332863+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[84576]: audit 2024-09-13T14:41:28.332863+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[87147]: audit 2024-09-13T14:41:27.670070+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[87147]: audit 2024-09-13T14:41:27.670070+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[87147]: audit 2024-09-13T14:41:27.676354+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[87147]: audit 2024-09-13T14:41:27.676354+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[87147]: audit 2024-09-13T14:41:28.322414+0000 mon.a (mon.0) 1325 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:29.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[87147]: audit 2024-09-13T14:41:28.322414+0000 mon.a (mon.0) 1325 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:29.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[87147]: audit 2024-09-13T14:41:28.323570+0000 mon.a (mon.0) 1326 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:29.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[87147]: audit 2024-09-13T14:41:28.323570+0000 mon.a (mon.0) 1326 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:29.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[87147]: audit 2024-09-13T14:41:28.332863+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:28 smithi099 bash[87147]: audit 2024-09-13T14:41:28.332863+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:28 smithi177 bash[68594]: audit 2024-09-13T14:41:27.670070+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:28 smithi177 bash[68594]: audit 2024-09-13T14:41:27.670070+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:28 smithi177 bash[68594]: audit 2024-09-13T14:41:27.676354+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:28 smithi177 bash[68594]: audit 2024-09-13T14:41:27.676354+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:28 smithi177 bash[68594]: audit 2024-09-13T14:41:28.322414+0000 mon.a (mon.0) 1325 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:28 smithi177 bash[68594]: audit 2024-09-13T14:41:28.322414+0000 mon.a (mon.0) 1325 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:29.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:28 smithi177 bash[68594]: audit 2024-09-13T14:41:28.323570+0000 mon.a (mon.0) 1326 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:29.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:28 smithi177 bash[68594]: audit 2024-09-13T14:41:28.323570+0000 mon.a (mon.0) 1326 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:29.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:28 smithi177 bash[68594]: audit 2024-09-13T14:41:28.332863+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:29.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:28 smithi177 bash[68594]: audit 2024-09-13T14:41:28.332863+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:41:30.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:29 smithi099 bash[84576]: cluster 2024-09-13T14:41:28.324723+0000 mgr.y (mgr.26252) 450 : cluster [DBG] pgmap v744: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:30.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:29 smithi099 bash[84576]: cluster 2024-09-13T14:41:28.324723+0000 mgr.y (mgr.26252) 450 : cluster [DBG] pgmap v744: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:30.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:29 smithi099 bash[87147]: cluster 2024-09-13T14:41:28.324723+0000 mgr.y (mgr.26252) 450 : cluster [DBG] pgmap v744: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:30.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:29 smithi099 bash[87147]: cluster 2024-09-13T14:41:28.324723+0000 mgr.y (mgr.26252) 450 : cluster [DBG] pgmap v744: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:30.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:29 smithi177 bash[68594]: cluster 2024-09-13T14:41:28.324723+0000 mgr.y (mgr.26252) 450 : cluster [DBG] pgmap v744: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:30.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:29 smithi177 bash[68594]: cluster 2024-09-13T14:41:28.324723+0000 mgr.y (mgr.26252) 450 : cluster [DBG] pgmap v744: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T14:41:30.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:41:30 smithi177 bash[61439]: ts=2024-09-13T14:41:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:32.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:31 smithi099 bash[84576]: cluster 2024-09-13T14:41:30.328342+0000 mgr.y (mgr.26252) 451 : cluster [DBG] pgmap v745: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:32.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:31 smithi099 bash[84576]: cluster 2024-09-13T14:41:30.328342+0000 mgr.y (mgr.26252) 451 : cluster [DBG] pgmap v745: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:32.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:31 smithi099 bash[87147]: cluster 2024-09-13T14:41:30.328342+0000 mgr.y (mgr.26252) 451 : cluster [DBG] pgmap v745: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:32.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:31 smithi099 bash[87147]: cluster 2024-09-13T14:41:30.328342+0000 mgr.y (mgr.26252) 451 : cluster [DBG] pgmap v745: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:32.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:31 smithi177 bash[68594]: cluster 2024-09-13T14:41:30.328342+0000 mgr.y (mgr.26252) 451 : cluster [DBG] pgmap v745: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:32.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:31 smithi177 bash[68594]: cluster 2024-09-13T14:41:30.328342+0000 mgr.y (mgr.26252) 451 : cluster [DBG] pgmap v745: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:33.430 INFO:tasks.workunit.client.0.smithi099.stderr:test retry succeeds 2024-09-13T14:41:34.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:33 smithi099 bash[84576]: cluster 2024-09-13T14:41:32.330340+0000 mgr.y (mgr.26252) 452 : cluster [DBG] pgmap v746: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:33 smithi099 bash[84576]: cluster 2024-09-13T14:41:32.330340+0000 mgr.y (mgr.26252) 452 : cluster [DBG] pgmap v746: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:33 smithi099 bash[84576]: audit 2024-09-13T14:41:33.254622+0000 mon.a (mon.0) 1328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:33 smithi099 bash[84576]: audit 2024-09-13T14:41:33.254622+0000 mon.a (mon.0) 1328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:33 smithi099 bash[87147]: cluster 2024-09-13T14:41:32.330340+0000 mgr.y (mgr.26252) 452 : cluster [DBG] pgmap v746: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:33 smithi099 bash[87147]: cluster 2024-09-13T14:41:32.330340+0000 mgr.y (mgr.26252) 452 : cluster [DBG] pgmap v746: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:33 smithi099 bash[87147]: audit 2024-09-13T14:41:33.254622+0000 mon.a (mon.0) 1328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:33 smithi099 bash[87147]: audit 2024-09-13T14:41:33.254622+0000 mon.a (mon.0) 1328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:33 smithi177 bash[68594]: cluster 2024-09-13T14:41:32.330340+0000 mgr.y (mgr.26252) 452 : cluster [DBG] pgmap v746: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:33 smithi177 bash[68594]: cluster 2024-09-13T14:41:32.330340+0000 mgr.y (mgr.26252) 452 : cluster [DBG] pgmap v746: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:41:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:33 smithi177 bash[68594]: audit 2024-09-13T14:41:33.254622+0000 mon.a (mon.0) 1328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:33 smithi177 bash[68594]: audit 2024-09-13T14:41:33.254622+0000 mon.a (mon.0) 1328 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:35.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:34 smithi099 bash[84576]: cluster 2024-09-13T14:41:33.700733+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-13T14:41:35.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:34 smithi099 bash[84576]: cluster 2024-09-13T14:41:33.700733+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-13T14:41:35.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:34 smithi099 bash[87147]: cluster 2024-09-13T14:41:33.700733+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-13T14:41:35.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:34 smithi099 bash[87147]: cluster 2024-09-13T14:41:33.700733+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-13T14:41:35.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:34 smithi177 bash[68594]: cluster 2024-09-13T14:41:33.700733+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-13T14:41:35.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:34 smithi177 bash[68594]: cluster 2024-09-13T14:41:33.700733+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-13T14:41:35.996 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.QuiesceWatchTimeout (32643 ms) 2024-09-13T14:41:35.996 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.WriteZeroes 2024-09-13T14:41:35.996 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:41:36.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:35 smithi099 bash[84576]: cluster 2024-09-13T14:41:34.334794+0000 mgr.y (mgr.26252) 453 : cluster [DBG] pgmap v748: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 511 B/s wr, 4 op/s 2024-09-13T14:41:36.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:35 smithi099 bash[84576]: cluster 2024-09-13T14:41:34.334794+0000 mgr.y (mgr.26252) 453 : cluster [DBG] pgmap v748: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 511 B/s wr, 4 op/s 2024-09-13T14:41:36.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:35 smithi099 bash[84576]: cluster 2024-09-13T14:41:34.732699+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-13T14:41:36.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:35 smithi099 bash[84576]: cluster 2024-09-13T14:41:34.732699+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-13T14:41:36.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:35 smithi099 bash[87147]: cluster 2024-09-13T14:41:34.334794+0000 mgr.y (mgr.26252) 453 : cluster [DBG] pgmap v748: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 511 B/s wr, 4 op/s 2024-09-13T14:41:36.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:35 smithi099 bash[87147]: cluster 2024-09-13T14:41:34.334794+0000 mgr.y (mgr.26252) 453 : cluster [DBG] pgmap v748: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 511 B/s wr, 4 op/s 2024-09-13T14:41:36.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:35 smithi099 bash[87147]: cluster 2024-09-13T14:41:34.732699+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-13T14:41:36.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:35 smithi099 bash[87147]: cluster 2024-09-13T14:41:34.732699+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-13T14:41:36.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:35 smithi177 bash[68594]: cluster 2024-09-13T14:41:34.334794+0000 mgr.y (mgr.26252) 453 : cluster [DBG] pgmap v748: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 511 B/s wr, 4 op/s 2024-09-13T14:41:36.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:35 smithi177 bash[68594]: cluster 2024-09-13T14:41:34.334794+0000 mgr.y (mgr.26252) 453 : cluster [DBG] pgmap v748: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 511 B/s wr, 4 op/s 2024-09-13T14:41:36.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:35 smithi177 bash[68594]: cluster 2024-09-13T14:41:34.732699+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-13T14:41:36.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:35 smithi177 bash[68594]: cluster 2024-09-13T14:41:34.732699+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-13T14:41:36.140 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.WriteZeroes (145 ms) 2024-09-13T14:41:36.141 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.WriteZeroesThickProvision 2024-09-13T14:41:36.141 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:41:36.300 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.WriteZeroesThickProvision (158 ms) 2024-09-13T14:41:36.300 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestLibRBD.ConcurentOperations 2024-09-13T14:41:36.300 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:41:37.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:36 smithi099 bash[84576]: cluster 2024-09-13T14:41:35.741308+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-13T14:41:37.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:36 smithi099 bash[84576]: cluster 2024-09-13T14:41:35.741308+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-13T14:41:37.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:36 smithi099 bash[87147]: cluster 2024-09-13T14:41:35.741308+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-13T14:41:37.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:36 smithi099 bash[87147]: cluster 2024-09-13T14:41:35.741308+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-13T14:41:37.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:36 smithi177 bash[68594]: cluster 2024-09-13T14:41:35.741308+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-13T14:41:37.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:36 smithi177 bash[68594]: cluster 2024-09-13T14:41:35.741308+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-13T14:41:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:41:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:41:37] "GET /metrics HTTP/1.1" 200 82354 "" "Prometheus/2.51.0" 2024-09-13T14:41:38.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:37 smithi099 bash[84576]: cluster 2024-09-13T14:41:36.336216+0000 mgr.y (mgr.26252) 454 : cluster [DBG] pgmap v751: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 851 B/s wr, 8 op/s 2024-09-13T14:41:38.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:37 smithi099 bash[84576]: cluster 2024-09-13T14:41:36.336216+0000 mgr.y (mgr.26252) 454 : cluster [DBG] pgmap v751: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 851 B/s wr, 8 op/s 2024-09-13T14:41:38.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:37 smithi099 bash[84576]: cluster 2024-09-13T14:41:36.757505+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-13T14:41:38.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:37 smithi099 bash[84576]: cluster 2024-09-13T14:41:36.757505+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-13T14:41:38.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:37 smithi099 bash[87147]: cluster 2024-09-13T14:41:36.336216+0000 mgr.y (mgr.26252) 454 : cluster [DBG] pgmap v751: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 851 B/s wr, 8 op/s 2024-09-13T14:41:38.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:37 smithi099 bash[87147]: cluster 2024-09-13T14:41:36.336216+0000 mgr.y (mgr.26252) 454 : cluster [DBG] pgmap v751: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 851 B/s wr, 8 op/s 2024-09-13T14:41:38.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:37 smithi099 bash[87147]: cluster 2024-09-13T14:41:36.757505+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-13T14:41:38.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:37 smithi099 bash[87147]: cluster 2024-09-13T14:41:36.757505+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-13T14:41:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:37 smithi177 bash[68594]: cluster 2024-09-13T14:41:36.336216+0000 mgr.y (mgr.26252) 454 : cluster [DBG] pgmap v751: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 851 B/s wr, 8 op/s 2024-09-13T14:41:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:37 smithi177 bash[68594]: cluster 2024-09-13T14:41:36.336216+0000 mgr.y (mgr.26252) 454 : cluster [DBG] pgmap v751: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 851 B/s wr, 8 op/s 2024-09-13T14:41:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:37 smithi177 bash[68594]: cluster 2024-09-13T14:41:36.757505+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-13T14:41:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:37 smithi177 bash[68594]: cluster 2024-09-13T14:41:36.757505+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-13T14:41:39.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:38 smithi099 bash[84576]: cluster 2024-09-13T14:41:37.756849+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-13T14:41:39.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:38 smithi099 bash[84576]: cluster 2024-09-13T14:41:37.756849+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-13T14:41:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:38 smithi099 bash[87147]: cluster 2024-09-13T14:41:37.756849+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-13T14:41:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:38 smithi099 bash[87147]: cluster 2024-09-13T14:41:37.756849+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-13T14:41:39.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:38 smithi177 bash[68594]: cluster 2024-09-13T14:41:37.756849+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-13T14:41:39.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:38 smithi177 bash[68594]: cluster 2024-09-13T14:41:37.756849+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-13T14:41:40.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:39 smithi099 bash[84576]: cluster 2024-09-13T14:41:38.337428+0000 mgr.y (mgr.26252) 455 : cluster [DBG] pgmap v754: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 3.7 KiB/s wr, 113 op/s 2024-09-13T14:41:40.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:39 smithi099 bash[84576]: cluster 2024-09-13T14:41:38.337428+0000 mgr.y (mgr.26252) 455 : cluster [DBG] pgmap v754: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 3.7 KiB/s wr, 113 op/s 2024-09-13T14:41:40.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:39 smithi099 bash[84576]: cluster 2024-09-13T14:41:38.782386+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-13T14:41:40.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:39 smithi099 bash[84576]: cluster 2024-09-13T14:41:38.782386+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-13T14:41:40.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:39 smithi099 bash[87147]: cluster 2024-09-13T14:41:38.337428+0000 mgr.y (mgr.26252) 455 : cluster [DBG] pgmap v754: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 3.7 KiB/s wr, 113 op/s 2024-09-13T14:41:40.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:39 smithi099 bash[87147]: cluster 2024-09-13T14:41:38.337428+0000 mgr.y (mgr.26252) 455 : cluster [DBG] pgmap v754: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 3.7 KiB/s wr, 113 op/s 2024-09-13T14:41:40.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:39 smithi099 bash[87147]: cluster 2024-09-13T14:41:38.782386+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-13T14:41:40.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:39 smithi099 bash[87147]: cluster 2024-09-13T14:41:38.782386+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-13T14:41:40.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:39 smithi177 bash[68594]: cluster 2024-09-13T14:41:38.337428+0000 mgr.y (mgr.26252) 455 : cluster [DBG] pgmap v754: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 3.7 KiB/s wr, 113 op/s 2024-09-13T14:41:40.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:39 smithi177 bash[68594]: cluster 2024-09-13T14:41:38.337428+0000 mgr.y (mgr.26252) 455 : cluster [DBG] pgmap v754: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 3.7 KiB/s wr, 113 op/s 2024-09-13T14:41:40.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:39 smithi177 bash[68594]: cluster 2024-09-13T14:41:38.782386+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-13T14:41:40.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:39 smithi177 bash[68594]: cluster 2024-09-13T14:41:38.782386+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-13T14:41:40.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:41:40 smithi177 bash[61439]: ts=2024-09-13T14:41:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:41.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:40 smithi099 bash[84576]: cluster 2024-09-13T14:41:39.791017+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-13T14:41:41.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:40 smithi099 bash[84576]: cluster 2024-09-13T14:41:39.791017+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-13T14:41:41.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:40 smithi099 bash[87147]: cluster 2024-09-13T14:41:39.791017+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-13T14:41:41.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:40 smithi099 bash[87147]: cluster 2024-09-13T14:41:39.791017+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-13T14:41:41.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:40 smithi177 bash[68594]: cluster 2024-09-13T14:41:39.791017+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-13T14:41:41.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:40 smithi177 bash[68594]: cluster 2024-09-13T14:41:39.791017+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-13T14:41:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[84576]: cluster 2024-09-13T14:41:40.338605+0000 mgr.y (mgr.26252) 456 : cluster [DBG] pgmap v757: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 9.0 KiB/s wr, 169 op/s 2024-09-13T14:41:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[84576]: cluster 2024-09-13T14:41:40.338605+0000 mgr.y (mgr.26252) 456 : cluster [DBG] pgmap v757: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 9.0 KiB/s wr, 169 op/s 2024-09-13T14:41:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[84576]: cluster 2024-09-13T14:41:40.809373+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-13T14:41:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[84576]: cluster 2024-09-13T14:41:40.809373+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-13T14:41:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[84576]: cluster 2024-09-13T14:41:41.173848+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-13T14:41:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[84576]: cluster 2024-09-13T14:41:41.173848+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-13T14:41:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[87147]: cluster 2024-09-13T14:41:40.338605+0000 mgr.y (mgr.26252) 456 : cluster [DBG] pgmap v757: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 9.0 KiB/s wr, 169 op/s 2024-09-13T14:41:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[87147]: cluster 2024-09-13T14:41:40.338605+0000 mgr.y (mgr.26252) 456 : cluster [DBG] pgmap v757: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 9.0 KiB/s wr, 169 op/s 2024-09-13T14:41:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[87147]: cluster 2024-09-13T14:41:40.809373+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-13T14:41:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[87147]: cluster 2024-09-13T14:41:40.809373+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-13T14:41:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[87147]: cluster 2024-09-13T14:41:41.173848+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-13T14:41:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:41 smithi099 bash[87147]: cluster 2024-09-13T14:41:41.173848+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-13T14:41:42.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:41 smithi177 bash[68594]: cluster 2024-09-13T14:41:40.338605+0000 mgr.y (mgr.26252) 456 : cluster [DBG] pgmap v757: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 9.0 KiB/s wr, 169 op/s 2024-09-13T14:41:42.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:41 smithi177 bash[68594]: cluster 2024-09-13T14:41:40.338605+0000 mgr.y (mgr.26252) 456 : cluster [DBG] pgmap v757: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 9.0 KiB/s wr, 169 op/s 2024-09-13T14:41:42.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:41 smithi177 bash[68594]: cluster 2024-09-13T14:41:40.809373+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-13T14:41:42.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:41 smithi177 bash[68594]: cluster 2024-09-13T14:41:40.809373+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-13T14:41:42.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:41 smithi177 bash[68594]: cluster 2024-09-13T14:41:41.173848+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-13T14:41:42.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:41 smithi177 bash[68594]: cluster 2024-09-13T14:41:41.173848+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-13T14:41:43.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:43 smithi099 bash[84576]: cluster 2024-09-13T14:41:42.174880+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-13T14:41:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:43 smithi099 bash[84576]: cluster 2024-09-13T14:41:42.174880+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-13T14:41:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:43 smithi099 bash[84576]: cluster 2024-09-13T14:41:42.340072+0000 mgr.y (mgr.26252) 457 : cluster [DBG] pgmap v761: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 5.9 KiB/s wr, 62 op/s 2024-09-13T14:41:43.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:43 smithi099 bash[84576]: cluster 2024-09-13T14:41:42.340072+0000 mgr.y (mgr.26252) 457 : cluster [DBG] pgmap v761: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 5.9 KiB/s wr, 62 op/s 2024-09-13T14:41:43.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:43 smithi099 bash[87147]: cluster 2024-09-13T14:41:42.174880+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-13T14:41:43.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:43 smithi099 bash[87147]: cluster 2024-09-13T14:41:42.174880+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-13T14:41:43.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:43 smithi099 bash[87147]: cluster 2024-09-13T14:41:42.340072+0000 mgr.y (mgr.26252) 457 : cluster [DBG] pgmap v761: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 5.9 KiB/s wr, 62 op/s 2024-09-13T14:41:43.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:43 smithi099 bash[87147]: cluster 2024-09-13T14:41:42.340072+0000 mgr.y (mgr.26252) 457 : cluster [DBG] pgmap v761: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 5.9 KiB/s wr, 62 op/s 2024-09-13T14:41:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:43 smithi177 bash[68594]: cluster 2024-09-13T14:41:42.174880+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-13T14:41:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:43 smithi177 bash[68594]: cluster 2024-09-13T14:41:42.174880+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-13T14:41:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:43 smithi177 bash[68594]: cluster 2024-09-13T14:41:42.340072+0000 mgr.y (mgr.26252) 457 : cluster [DBG] pgmap v761: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 5.9 KiB/s wr, 62 op/s 2024-09-13T14:41:43.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:43 smithi177 bash[68594]: cluster 2024-09-13T14:41:42.340072+0000 mgr.y (mgr.26252) 457 : cluster [DBG] pgmap v761: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 5.9 KiB/s wr, 62 op/s 2024-09-13T14:41:44.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:44 smithi099 bash[84576]: cluster 2024-09-13T14:41:43.194320+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-13T14:41:44.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:44 smithi099 bash[84576]: cluster 2024-09-13T14:41:43.194320+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-13T14:41:44.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:44 smithi099 bash[87147]: cluster 2024-09-13T14:41:43.194320+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-13T14:41:44.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:44 smithi099 bash[87147]: cluster 2024-09-13T14:41:43.194320+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-13T14:41:44.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:44 smithi177 bash[68594]: cluster 2024-09-13T14:41:43.194320+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-13T14:41:44.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:44 smithi177 bash[68594]: cluster 2024-09-13T14:41:43.194320+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-13T14:41:45.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:45 smithi099 bash[84576]: cluster 2024-09-13T14:41:44.191676+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-13T14:41:45.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:45 smithi099 bash[84576]: cluster 2024-09-13T14:41:44.191676+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-13T14:41:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:45 smithi099 bash[84576]: cluster 2024-09-13T14:41:44.341296+0000 mgr.y (mgr.26252) 458 : cluster [DBG] pgmap v764: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.1 KiB/s wr, 22 op/s 2024-09-13T14:41:45.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:45 smithi099 bash[84576]: cluster 2024-09-13T14:41:44.341296+0000 mgr.y (mgr.26252) 458 : cluster [DBG] pgmap v764: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.1 KiB/s wr, 22 op/s 2024-09-13T14:41:45.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:45 smithi099 bash[87147]: cluster 2024-09-13T14:41:44.191676+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-13T14:41:45.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:45 smithi099 bash[87147]: cluster 2024-09-13T14:41:44.191676+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-13T14:41:45.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:45 smithi099 bash[87147]: cluster 2024-09-13T14:41:44.341296+0000 mgr.y (mgr.26252) 458 : cluster [DBG] pgmap v764: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.1 KiB/s wr, 22 op/s 2024-09-13T14:41:45.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:45 smithi099 bash[87147]: cluster 2024-09-13T14:41:44.341296+0000 mgr.y (mgr.26252) 458 : cluster [DBG] pgmap v764: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.1 KiB/s wr, 22 op/s 2024-09-13T14:41:45.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:45 smithi177 bash[68594]: cluster 2024-09-13T14:41:44.191676+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-13T14:41:45.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:45 smithi177 bash[68594]: cluster 2024-09-13T14:41:44.191676+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-13T14:41:45.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:45 smithi177 bash[68594]: cluster 2024-09-13T14:41:44.341296+0000 mgr.y (mgr.26252) 458 : cluster [DBG] pgmap v764: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.1 KiB/s wr, 22 op/s 2024-09-13T14:41:45.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:45 smithi177 bash[68594]: cluster 2024-09-13T14:41:44.341296+0000 mgr.y (mgr.26252) 458 : cluster [DBG] pgmap v764: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.1 KiB/s wr, 22 op/s 2024-09-13T14:41:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:46 smithi099 bash[84576]: cluster 2024-09-13T14:41:45.225302+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-13T14:41:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:46 smithi099 bash[84576]: cluster 2024-09-13T14:41:45.225302+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-13T14:41:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:46 smithi099 bash[87147]: cluster 2024-09-13T14:41:45.225302+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-13T14:41:46.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:46 smithi099 bash[87147]: cluster 2024-09-13T14:41:45.225302+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-13T14:41:46.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:46 smithi177 bash[68594]: cluster 2024-09-13T14:41:45.225302+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-13T14:41:46.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:46 smithi177 bash[68594]: cluster 2024-09-13T14:41:45.225302+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-13T14:41:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:41:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:41:47] "GET /metrics HTTP/1.1" 200 82354 "" "Prometheus/2.51.0" 2024-09-13T14:41:47.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:47 smithi099 bash[84576]: cluster 2024-09-13T14:41:46.231300+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-13T14:41:47.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:47 smithi099 bash[84576]: cluster 2024-09-13T14:41:46.231300+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-13T14:41:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:47 smithi099 bash[84576]: cluster 2024-09-13T14:41:46.343006+0000 mgr.y (mgr.26252) 459 : cluster [DBG] pgmap v767: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.5 KiB/s wr, 19 op/s 2024-09-13T14:41:47.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:47 smithi099 bash[84576]: cluster 2024-09-13T14:41:46.343006+0000 mgr.y (mgr.26252) 459 : cluster [DBG] pgmap v767: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.5 KiB/s wr, 19 op/s 2024-09-13T14:41:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:47 smithi099 bash[87147]: cluster 2024-09-13T14:41:46.231300+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-13T14:41:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:47 smithi099 bash[87147]: cluster 2024-09-13T14:41:46.231300+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-13T14:41:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:47 smithi099 bash[87147]: cluster 2024-09-13T14:41:46.343006+0000 mgr.y (mgr.26252) 459 : cluster [DBG] pgmap v767: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.5 KiB/s wr, 19 op/s 2024-09-13T14:41:47.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:47 smithi099 bash[87147]: cluster 2024-09-13T14:41:46.343006+0000 mgr.y (mgr.26252) 459 : cluster [DBG] pgmap v767: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.5 KiB/s wr, 19 op/s 2024-09-13T14:41:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:47 smithi177 bash[68594]: cluster 2024-09-13T14:41:46.231300+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-13T14:41:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:47 smithi177 bash[68594]: cluster 2024-09-13T14:41:46.231300+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-13T14:41:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:47 smithi177 bash[68594]: cluster 2024-09-13T14:41:46.343006+0000 mgr.y (mgr.26252) 459 : cluster [DBG] pgmap v767: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.5 KiB/s wr, 19 op/s 2024-09-13T14:41:47.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:47 smithi177 bash[68594]: cluster 2024-09-13T14:41:46.343006+0000 mgr.y (mgr.26252) 459 : cluster [DBG] pgmap v767: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.5 KiB/s wr, 19 op/s 2024-09-13T14:41:48.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:48 smithi099 bash[84576]: cluster 2024-09-13T14:41:47.260394+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-13T14:41:48.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:48 smithi099 bash[84576]: cluster 2024-09-13T14:41:47.260394+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-13T14:41:48.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:48 smithi099 bash[87147]: cluster 2024-09-13T14:41:47.260394+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-13T14:41:48.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:48 smithi099 bash[87147]: cluster 2024-09-13T14:41:47.260394+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-13T14:41:48.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:48 smithi177 bash[68594]: cluster 2024-09-13T14:41:47.260394+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-13T14:41:48.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:48 smithi177 bash[68594]: cluster 2024-09-13T14:41:47.260394+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-13T14:41:49.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[84576]: cluster 2024-09-13T14:41:48.250785+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[84576]: cluster 2024-09-13T14:41:48.250785+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[84576]: audit 2024-09-13T14:41:48.255431+0000 mon.a (mon.0) 1345 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[84576]: audit 2024-09-13T14:41:48.255431+0000 mon.a (mon.0) 1345 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[84576]: cluster 2024-09-13T14:41:48.343834+0000 mgr.y (mgr.26252) 460 : cluster [DBG] pgmap v770: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 4.7 KiB/s wr, 235 op/s 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[84576]: cluster 2024-09-13T14:41:48.343834+0000 mgr.y (mgr.26252) 460 : cluster [DBG] pgmap v770: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 4.7 KiB/s wr, 235 op/s 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[87147]: cluster 2024-09-13T14:41:48.250785+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[87147]: cluster 2024-09-13T14:41:48.250785+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[87147]: audit 2024-09-13T14:41:48.255431+0000 mon.a (mon.0) 1345 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[87147]: audit 2024-09-13T14:41:48.255431+0000 mon.a (mon.0) 1345 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:49.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[87147]: cluster 2024-09-13T14:41:48.343834+0000 mgr.y (mgr.26252) 460 : cluster [DBG] pgmap v770: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 4.7 KiB/s wr, 235 op/s 2024-09-13T14:41:49.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:49 smithi099 bash[87147]: cluster 2024-09-13T14:41:48.343834+0000 mgr.y (mgr.26252) 460 : cluster [DBG] pgmap v770: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 4.7 KiB/s wr, 235 op/s 2024-09-13T14:41:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:49 smithi177 bash[68594]: cluster 2024-09-13T14:41:48.250785+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-13T14:41:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:49 smithi177 bash[68594]: cluster 2024-09-13T14:41:48.250785+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-13T14:41:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:49 smithi177 bash[68594]: audit 2024-09-13T14:41:48.255431+0000 mon.a (mon.0) 1345 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:49 smithi177 bash[68594]: audit 2024-09-13T14:41:48.255431+0000 mon.a (mon.0) 1345 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:49 smithi177 bash[68594]: cluster 2024-09-13T14:41:48.343834+0000 mgr.y (mgr.26252) 460 : cluster [DBG] pgmap v770: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 4.7 KiB/s wr, 235 op/s 2024-09-13T14:41:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:49 smithi177 bash[68594]: cluster 2024-09-13T14:41:48.343834+0000 mgr.y (mgr.26252) 460 : cluster [DBG] pgmap v770: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 4.7 KiB/s wr, 235 op/s 2024-09-13T14:41:50.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:50 smithi099 bash[84576]: cluster 2024-09-13T14:41:49.280639+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-13T14:41:50.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:50 smithi099 bash[84576]: cluster 2024-09-13T14:41:49.280639+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-13T14:41:50.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:50 smithi099 bash[87147]: cluster 2024-09-13T14:41:49.280639+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-13T14:41:50.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:50 smithi099 bash[87147]: cluster 2024-09-13T14:41:49.280639+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-13T14:41:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:50 smithi177 bash[68594]: cluster 2024-09-13T14:41:49.280639+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-13T14:41:50.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:50 smithi177 bash[68594]: cluster 2024-09-13T14:41:49.280639+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-13T14:41:50.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:41:50 smithi177 bash[61439]: ts=2024-09-13T14:41:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:41:51.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:51 smithi099 bash[84576]: cluster 2024-09-13T14:41:50.292599+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-13T14:41:51.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:51 smithi099 bash[84576]: cluster 2024-09-13T14:41:50.292599+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-13T14:41:51.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:51 smithi099 bash[84576]: cluster 2024-09-13T14:41:50.345112+0000 mgr.y (mgr.26252) 461 : cluster [DBG] pgmap v773: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 4.7 KiB/s wr, 244 op/s 2024-09-13T14:41:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:51 smithi099 bash[84576]: cluster 2024-09-13T14:41:50.345112+0000 mgr.y (mgr.26252) 461 : cluster [DBG] pgmap v773: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 4.7 KiB/s wr, 244 op/s 2024-09-13T14:41:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:51 smithi099 bash[87147]: cluster 2024-09-13T14:41:50.292599+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-13T14:41:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:51 smithi099 bash[87147]: cluster 2024-09-13T14:41:50.292599+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-13T14:41:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:51 smithi099 bash[87147]: cluster 2024-09-13T14:41:50.345112+0000 mgr.y (mgr.26252) 461 : cluster [DBG] pgmap v773: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 4.7 KiB/s wr, 244 op/s 2024-09-13T14:41:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:51 smithi099 bash[87147]: cluster 2024-09-13T14:41:50.345112+0000 mgr.y (mgr.26252) 461 : cluster [DBG] pgmap v773: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 4.7 KiB/s wr, 244 op/s 2024-09-13T14:41:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:51 smithi177 bash[68594]: cluster 2024-09-13T14:41:50.292599+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-13T14:41:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:51 smithi177 bash[68594]: cluster 2024-09-13T14:41:50.292599+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-13T14:41:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:51 smithi177 bash[68594]: cluster 2024-09-13T14:41:50.345112+0000 mgr.y (mgr.26252) 461 : cluster [DBG] pgmap v773: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 4.7 KiB/s wr, 244 op/s 2024-09-13T14:41:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:51 smithi177 bash[68594]: cluster 2024-09-13T14:41:50.345112+0000 mgr.y (mgr.26252) 461 : cluster [DBG] pgmap v773: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 4.7 KiB/s wr, 244 op/s 2024-09-13T14:41:52.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:52 smithi099 bash[84576]: cluster 2024-09-13T14:41:51.300036+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-13T14:41:52.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:52 smithi099 bash[84576]: cluster 2024-09-13T14:41:51.300036+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-13T14:41:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:52 smithi099 bash[87147]: cluster 2024-09-13T14:41:51.300036+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-13T14:41:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:52 smithi099 bash[87147]: cluster 2024-09-13T14:41:51.300036+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-13T14:41:52.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:52 smithi177 bash[68594]: cluster 2024-09-13T14:41:51.300036+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-13T14:41:52.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:52 smithi177 bash[68594]: cluster 2024-09-13T14:41:51.300036+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-13T14:41:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:53 smithi099 bash[84576]: cluster 2024-09-13T14:41:52.301078+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-13T14:41:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:53 smithi099 bash[84576]: cluster 2024-09-13T14:41:52.301078+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-13T14:41:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:53 smithi099 bash[84576]: cluster 2024-09-13T14:41:52.346436+0000 mgr.y (mgr.26252) 462 : cluster [DBG] pgmap v776: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:41:53.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:53 smithi099 bash[84576]: cluster 2024-09-13T14:41:52.346436+0000 mgr.y (mgr.26252) 462 : cluster [DBG] pgmap v776: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:41:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:53 smithi099 bash[87147]: cluster 2024-09-13T14:41:52.301078+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-13T14:41:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:53 smithi099 bash[87147]: cluster 2024-09-13T14:41:52.301078+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-13T14:41:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:53 smithi099 bash[87147]: cluster 2024-09-13T14:41:52.346436+0000 mgr.y (mgr.26252) 462 : cluster [DBG] pgmap v776: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:41:53.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:53 smithi099 bash[87147]: cluster 2024-09-13T14:41:52.346436+0000 mgr.y (mgr.26252) 462 : cluster [DBG] pgmap v776: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:41:53.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:53 smithi177 bash[68594]: cluster 2024-09-13T14:41:52.301078+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-13T14:41:53.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:53 smithi177 bash[68594]: cluster 2024-09-13T14:41:52.301078+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-13T14:41:53.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:53 smithi177 bash[68594]: cluster 2024-09-13T14:41:52.346436+0000 mgr.y (mgr.26252) 462 : cluster [DBG] pgmap v776: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:41:53.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:53 smithi177 bash[68594]: cluster 2024-09-13T14:41:52.346436+0000 mgr.y (mgr.26252) 462 : cluster [DBG] pgmap v776: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T14:41:54.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:54 smithi177 bash[68594]: cluster 2024-09-13T14:41:53.333750+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-13T14:41:54.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:54 smithi177 bash[68594]: cluster 2024-09-13T14:41:53.333750+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-13T14:41:54.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:54 smithi099 bash[84576]: cluster 2024-09-13T14:41:53.333750+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-13T14:41:54.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:54 smithi099 bash[84576]: cluster 2024-09-13T14:41:53.333750+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-13T14:41:54.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:54 smithi099 bash[87147]: cluster 2024-09-13T14:41:53.333750+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-13T14:41:54.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:54 smithi099 bash[87147]: cluster 2024-09-13T14:41:53.333750+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-13T14:41:55.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:55 smithi177 bash[68594]: cluster 2024-09-13T14:41:54.345800+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-13T14:41:55.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:55 smithi177 bash[68594]: cluster 2024-09-13T14:41:54.345800+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-13T14:41:55.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:55 smithi177 bash[68594]: cluster 2024-09-13T14:41:54.349324+0000 mgr.y (mgr.26252) 463 : cluster [DBG] pgmap v779: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:55.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:55 smithi177 bash[68594]: cluster 2024-09-13T14:41:54.349324+0000 mgr.y (mgr.26252) 463 : cluster [DBG] pgmap v779: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:55.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:55 smithi099 bash[84576]: cluster 2024-09-13T14:41:54.345800+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-13T14:41:55.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:55 smithi099 bash[84576]: cluster 2024-09-13T14:41:54.345800+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-13T14:41:55.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:55 smithi099 bash[84576]: cluster 2024-09-13T14:41:54.349324+0000 mgr.y (mgr.26252) 463 : cluster [DBG] pgmap v779: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:55.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:55 smithi099 bash[84576]: cluster 2024-09-13T14:41:54.349324+0000 mgr.y (mgr.26252) 463 : cluster [DBG] pgmap v779: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:55.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:55 smithi099 bash[87147]: cluster 2024-09-13T14:41:54.345800+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-13T14:41:55.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:55 smithi099 bash[87147]: cluster 2024-09-13T14:41:54.345800+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-13T14:41:55.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:55 smithi099 bash[87147]: cluster 2024-09-13T14:41:54.349324+0000 mgr.y (mgr.26252) 463 : cluster [DBG] pgmap v779: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:55.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:55 smithi099 bash[87147]: cluster 2024-09-13T14:41:54.349324+0000 mgr.y (mgr.26252) 463 : cluster [DBG] pgmap v779: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:56.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:56 smithi177 bash[68594]: cluster 2024-09-13T14:41:55.360229+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-13T14:41:56.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:56 smithi177 bash[68594]: cluster 2024-09-13T14:41:55.360229+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-13T14:41:56.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:56 smithi177 bash[68594]: cluster 2024-09-13T14:41:56.177992+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-13T14:41:56.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:56 smithi177 bash[68594]: cluster 2024-09-13T14:41:56.177992+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-13T14:41:56.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:56 smithi099 bash[84576]: cluster 2024-09-13T14:41:55.360229+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-13T14:41:56.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:56 smithi099 bash[84576]: cluster 2024-09-13T14:41:55.360229+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-13T14:41:56.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:56 smithi099 bash[84576]: cluster 2024-09-13T14:41:56.177992+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-13T14:41:56.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:56 smithi099 bash[84576]: cluster 2024-09-13T14:41:56.177992+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-13T14:41:56.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:56 smithi099 bash[87147]: cluster 2024-09-13T14:41:55.360229+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-13T14:41:56.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:56 smithi099 bash[87147]: cluster 2024-09-13T14:41:55.360229+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-13T14:41:56.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:56 smithi099 bash[87147]: cluster 2024-09-13T14:41:56.177992+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-13T14:41:56.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:56 smithi099 bash[87147]: cluster 2024-09-13T14:41:56.177992+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-13T14:41:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:41:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:41:57] "GET /metrics HTTP/1.1" 200 82364 "" "Prometheus/2.51.0" 2024-09-13T14:41:57.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:57 smithi177 bash[68594]: cluster 2024-09-13T14:41:56.350587+0000 mgr.y (mgr.26252) 464 : cluster [DBG] pgmap v782: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:57.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:57 smithi177 bash[68594]: cluster 2024-09-13T14:41:56.350587+0000 mgr.y (mgr.26252) 464 : cluster [DBG] pgmap v782: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:57.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:57 smithi177 bash[68594]: cluster 2024-09-13T14:41:57.188255+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-13T14:41:57.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:57 smithi177 bash[68594]: cluster 2024-09-13T14:41:57.188255+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-13T14:41:57.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:57 smithi099 bash[84576]: cluster 2024-09-13T14:41:56.350587+0000 mgr.y (mgr.26252) 464 : cluster [DBG] pgmap v782: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:57.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:57 smithi099 bash[84576]: cluster 2024-09-13T14:41:56.350587+0000 mgr.y (mgr.26252) 464 : cluster [DBG] pgmap v782: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:57.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:57 smithi099 bash[84576]: cluster 2024-09-13T14:41:57.188255+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-13T14:41:57.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:57 smithi099 bash[84576]: cluster 2024-09-13T14:41:57.188255+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-13T14:41:57.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:57 smithi099 bash[87147]: cluster 2024-09-13T14:41:56.350587+0000 mgr.y (mgr.26252) 464 : cluster [DBG] pgmap v782: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:57.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:57 smithi099 bash[87147]: cluster 2024-09-13T14:41:56.350587+0000 mgr.y (mgr.26252) 464 : cluster [DBG] pgmap v782: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 24 op/s 2024-09-13T14:41:57.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:57 smithi099 bash[87147]: cluster 2024-09-13T14:41:57.188255+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-13T14:41:57.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:57 smithi099 bash[87147]: cluster 2024-09-13T14:41:57.188255+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-13T14:41:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:59 smithi099 bash[84576]: cluster 2024-09-13T14:41:58.192541+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-13T14:41:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:59 smithi099 bash[84576]: cluster 2024-09-13T14:41:58.192541+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-13T14:41:59.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:59 smithi099 bash[84576]: cluster 2024-09-13T14:41:58.351731+0000 mgr.y (mgr.26252) 465 : cluster [DBG] pgmap v785: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.5 KiB/s wr, 46 op/s 2024-09-13T14:41:59.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:41:59 smithi099 bash[84576]: cluster 2024-09-13T14:41:58.351731+0000 mgr.y (mgr.26252) 465 : cluster [DBG] pgmap v785: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.5 KiB/s wr, 46 op/s 2024-09-13T14:41:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:59 smithi099 bash[87147]: cluster 2024-09-13T14:41:58.192541+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-13T14:41:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:59 smithi099 bash[87147]: cluster 2024-09-13T14:41:58.192541+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-13T14:41:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:59 smithi099 bash[87147]: cluster 2024-09-13T14:41:58.351731+0000 mgr.y (mgr.26252) 465 : cluster [DBG] pgmap v785: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.5 KiB/s wr, 46 op/s 2024-09-13T14:41:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:41:59 smithi099 bash[87147]: cluster 2024-09-13T14:41:58.351731+0000 mgr.y (mgr.26252) 465 : cluster [DBG] pgmap v785: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.5 KiB/s wr, 46 op/s 2024-09-13T14:41:59.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:59 smithi177 bash[68594]: cluster 2024-09-13T14:41:58.192541+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-13T14:41:59.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:59 smithi177 bash[68594]: cluster 2024-09-13T14:41:58.192541+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-13T14:41:59.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:59 smithi177 bash[68594]: cluster 2024-09-13T14:41:58.351731+0000 mgr.y (mgr.26252) 465 : cluster [DBG] pgmap v785: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.5 KiB/s wr, 46 op/s 2024-09-13T14:41:59.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:41:59 smithi177 bash[68594]: cluster 2024-09-13T14:41:58.351731+0000 mgr.y (mgr.26252) 465 : cluster [DBG] pgmap v785: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 3.5 KiB/s wr, 46 op/s 2024-09-13T14:42:00.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:00 smithi099 bash[84576]: cluster 2024-09-13T14:41:59.203285+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-13T14:42:00.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:00 smithi099 bash[84576]: cluster 2024-09-13T14:41:59.203285+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-13T14:42:00.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:00 smithi099 bash[87147]: cluster 2024-09-13T14:41:59.203285+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-13T14:42:00.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:00 smithi099 bash[87147]: cluster 2024-09-13T14:41:59.203285+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-13T14:42:00.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:42:00 smithi177 bash[61439]: ts=2024-09-13T14:42:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:00.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:00 smithi177 bash[68594]: cluster 2024-09-13T14:41:59.203285+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-13T14:42:00.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:00 smithi177 bash[68594]: cluster 2024-09-13T14:41:59.203285+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-13T14:42:01.417 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestLibRBD.ConcurentOperations (25117 ms) 2024-09-13T14:42:01.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[84576]: cluster 2024-09-13T14:42:00.202130+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-13T14:42:01.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[84576]: cluster 2024-09-13T14:42:00.202130+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[84576]: cluster 2024-09-13T14:42:00.352920+0000 mgr.y (mgr.26252) 466 : cluster [DBG] pgmap v788: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 54 op/s 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[84576]: cluster 2024-09-13T14:42:00.352920+0000 mgr.y (mgr.26252) 466 : cluster [DBG] pgmap v788: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 54 op/s 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[84576]: cluster 2024-09-13T14:42:01.180731+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[84576]: cluster 2024-09-13T14:42:01.180731+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[87147]: cluster 2024-09-13T14:42:00.202130+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[87147]: cluster 2024-09-13T14:42:00.202130+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[87147]: cluster 2024-09-13T14:42:00.352920+0000 mgr.y (mgr.26252) 466 : cluster [DBG] pgmap v788: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 54 op/s 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[87147]: cluster 2024-09-13T14:42:00.352920+0000 mgr.y (mgr.26252) 466 : cluster [DBG] pgmap v788: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 54 op/s 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[87147]: cluster 2024-09-13T14:42:01.180731+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-13T14:42:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:01 smithi099 bash[87147]: cluster 2024-09-13T14:42:01.180731+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-13T14:42:01.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:01 smithi177 bash[68594]: cluster 2024-09-13T14:42:00.202130+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-13T14:42:01.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:01 smithi177 bash[68594]: cluster 2024-09-13T14:42:00.202130+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-13T14:42:01.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:01 smithi177 bash[68594]: cluster 2024-09-13T14:42:00.352920+0000 mgr.y (mgr.26252) 466 : cluster [DBG] pgmap v788: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 54 op/s 2024-09-13T14:42:01.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:01 smithi177 bash[68594]: cluster 2024-09-13T14:42:00.352920+0000 mgr.y (mgr.26252) 466 : cluster [DBG] pgmap v788: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 54 op/s 2024-09-13T14:42:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:01 smithi177 bash[68594]: cluster 2024-09-13T14:42:01.180731+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-13T14:42:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:01 smithi177 bash[68594]: cluster 2024-09-13T14:42:01.180731+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-13T14:42:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:03 smithi099 bash[84576]: cluster 2024-09-13T14:42:02.232775+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-13T14:42:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:03 smithi099 bash[84576]: cluster 2024-09-13T14:42:02.232775+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-13T14:42:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:03 smithi099 bash[84576]: cluster 2024-09-13T14:42:02.354137+0000 mgr.y (mgr.26252) 467 : cluster [DBG] pgmap v791: 681 pgs: 681 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:03 smithi099 bash[84576]: cluster 2024-09-13T14:42:02.354137+0000 mgr.y (mgr.26252) 467 : cluster [DBG] pgmap v791: 681 pgs: 681 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:03 smithi099 bash[87147]: cluster 2024-09-13T14:42:02.232775+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-13T14:42:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:03 smithi099 bash[87147]: cluster 2024-09-13T14:42:02.232775+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-13T14:42:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:03 smithi099 bash[87147]: cluster 2024-09-13T14:42:02.354137+0000 mgr.y (mgr.26252) 467 : cluster [DBG] pgmap v791: 681 pgs: 681 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:03 smithi099 bash[87147]: cluster 2024-09-13T14:42:02.354137+0000 mgr.y (mgr.26252) 467 : cluster [DBG] pgmap v791: 681 pgs: 681 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:03 smithi177 bash[68594]: cluster 2024-09-13T14:42:02.232775+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-13T14:42:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:03 smithi177 bash[68594]: cluster 2024-09-13T14:42:02.232775+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-13T14:42:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:03 smithi177 bash[68594]: cluster 2024-09-13T14:42:02.354137+0000 mgr.y (mgr.26252) 467 : cluster [DBG] pgmap v791: 681 pgs: 681 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:03.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:03 smithi177 bash[68594]: cluster 2024-09-13T14:42:02.354137+0000 mgr.y (mgr.26252) 467 : cluster [DBG] pgmap v791: 681 pgs: 681 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:04 smithi099 bash[84576]: cluster 2024-09-13T14:42:03.248074+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-13T14:42:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:04 smithi099 bash[84576]: cluster 2024-09-13T14:42:03.248074+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-13T14:42:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:04 smithi099 bash[84576]: audit 2024-09-13T14:42:03.256146+0000 mon.a (mon.0) 1361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:04.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:04 smithi099 bash[84576]: audit 2024-09-13T14:42:03.256146+0000 mon.a (mon.0) 1361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:04 smithi099 bash[87147]: cluster 2024-09-13T14:42:03.248074+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-13T14:42:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:04 smithi099 bash[87147]: cluster 2024-09-13T14:42:03.248074+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-13T14:42:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:04 smithi099 bash[87147]: audit 2024-09-13T14:42:03.256146+0000 mon.a (mon.0) 1361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:04 smithi099 bash[87147]: audit 2024-09-13T14:42:03.256146+0000 mon.a (mon.0) 1361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:04.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:04 smithi177 bash[68594]: cluster 2024-09-13T14:42:03.248074+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-13T14:42:04.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:04 smithi177 bash[68594]: cluster 2024-09-13T14:42:03.248074+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-13T14:42:04.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:04 smithi177 bash[68594]: audit 2024-09-13T14:42:03.256146+0000 mon.a (mon.0) 1361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:04.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:04 smithi177 bash[68594]: audit 2024-09-13T14:42:03.256146+0000 mon.a (mon.0) 1361 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[84576]: cluster 2024-09-13T14:42:04.246047+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-13T14:42:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[84576]: cluster 2024-09-13T14:42:04.246047+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-13T14:42:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[84576]: cluster 2024-09-13T14:42:04.355217+0000 mgr.y (mgr.26252) 468 : cluster [DBG] pgmap v794: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[84576]: cluster 2024-09-13T14:42:04.355217+0000 mgr.y (mgr.26252) 468 : cluster [DBG] pgmap v794: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[84576]: cluster 2024-09-13T14:42:05.247316+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-13T14:42:05.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[84576]: cluster 2024-09-13T14:42:05.247316+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-13T14:42:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[87147]: cluster 2024-09-13T14:42:04.246047+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-13T14:42:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[87147]: cluster 2024-09-13T14:42:04.246047+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-13T14:42:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[87147]: cluster 2024-09-13T14:42:04.355217+0000 mgr.y (mgr.26252) 468 : cluster [DBG] pgmap v794: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[87147]: cluster 2024-09-13T14:42:04.355217+0000 mgr.y (mgr.26252) 468 : cluster [DBG] pgmap v794: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[87147]: cluster 2024-09-13T14:42:05.247316+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-13T14:42:05.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:05 smithi099 bash[87147]: cluster 2024-09-13T14:42:05.247316+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-13T14:42:05.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:05 smithi177 bash[68594]: cluster 2024-09-13T14:42:04.246047+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-13T14:42:05.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:05 smithi177 bash[68594]: cluster 2024-09-13T14:42:04.246047+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-13T14:42:05.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:05 smithi177 bash[68594]: cluster 2024-09-13T14:42:04.355217+0000 mgr.y (mgr.26252) 468 : cluster [DBG] pgmap v794: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:05 smithi177 bash[68594]: cluster 2024-09-13T14:42:04.355217+0000 mgr.y (mgr.26252) 468 : cluster [DBG] pgmap v794: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:05 smithi177 bash[68594]: cluster 2024-09-13T14:42:05.247316+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-13T14:42:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:05 smithi177 bash[68594]: cluster 2024-09-13T14:42:05.247316+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-13T14:42:07.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:42:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:42:07] "GET /metrics HTTP/1.1" 200 77556 "" "Prometheus/2.51.0" 2024-09-13T14:42:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:07 smithi099 bash[84576]: cluster 2024-09-13T14:42:06.183110+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-13T14:42:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:07 smithi099 bash[84576]: cluster 2024-09-13T14:42:06.183110+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-13T14:42:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:07 smithi099 bash[84576]: cluster 2024-09-13T14:42:06.356265+0000 mgr.y (mgr.26252) 469 : cluster [DBG] pgmap v797: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:07 smithi099 bash[84576]: cluster 2024-09-13T14:42:06.356265+0000 mgr.y (mgr.26252) 469 : cluster [DBG] pgmap v797: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:07 smithi099 bash[87147]: cluster 2024-09-13T14:42:06.183110+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-13T14:42:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:07 smithi099 bash[87147]: cluster 2024-09-13T14:42:06.183110+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-13T14:42:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:07 smithi099 bash[87147]: cluster 2024-09-13T14:42:06.356265+0000 mgr.y (mgr.26252) 469 : cluster [DBG] pgmap v797: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:07 smithi099 bash[87147]: cluster 2024-09-13T14:42:06.356265+0000 mgr.y (mgr.26252) 469 : cluster [DBG] pgmap v797: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:07 smithi177 bash[68594]: cluster 2024-09-13T14:42:06.183110+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-13T14:42:07.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:07 smithi177 bash[68594]: cluster 2024-09-13T14:42:06.183110+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-13T14:42:07.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:07 smithi177 bash[68594]: cluster 2024-09-13T14:42:06.356265+0000 mgr.y (mgr.26252) 469 : cluster [DBG] pgmap v797: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:07.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:07 smithi177 bash[68594]: cluster 2024-09-13T14:42:06.356265+0000 mgr.y (mgr.26252) 469 : cluster [DBG] pgmap v797: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:08 smithi099 bash[84576]: cluster 2024-09-13T14:42:07.195833+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-13T14:42:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:08 smithi099 bash[84576]: cluster 2024-09-13T14:42:07.195833+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-13T14:42:08.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:08 smithi099 bash[87147]: cluster 2024-09-13T14:42:07.195833+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-13T14:42:08.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:08 smithi099 bash[87147]: cluster 2024-09-13T14:42:07.195833+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-13T14:42:08.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:08 smithi177 bash[68594]: cluster 2024-09-13T14:42:07.195833+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-13T14:42:08.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:08 smithi177 bash[68594]: cluster 2024-09-13T14:42:07.195833+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-13T14:42:09.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:09 smithi099 bash[84576]: cluster 2024-09-13T14:42:08.204219+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-13T14:42:09.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:09 smithi099 bash[84576]: cluster 2024-09-13T14:42:08.204219+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:09 smithi099 bash[84576]: cluster 2024-09-13T14:42:08.357233+0000 mgr.y (mgr.26252) 470 : cluster [DBG] pgmap v800: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:09 smithi099 bash[84576]: cluster 2024-09-13T14:42:08.357233+0000 mgr.y (mgr.26252) 470 : cluster [DBG] pgmap v800: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:09 smithi099 bash[87147]: cluster 2024-09-13T14:42:08.204219+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:09 smithi099 bash[87147]: cluster 2024-09-13T14:42:08.204219+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:09 smithi099 bash[87147]: cluster 2024-09-13T14:42:08.357233+0000 mgr.y (mgr.26252) 470 : cluster [DBG] pgmap v800: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:09.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:09 smithi099 bash[87147]: cluster 2024-09-13T14:42:08.357233+0000 mgr.y (mgr.26252) 470 : cluster [DBG] pgmap v800: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:09.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:09 smithi177 bash[68594]: cluster 2024-09-13T14:42:08.204219+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-13T14:42:09.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:09 smithi177 bash[68594]: cluster 2024-09-13T14:42:08.204219+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-13T14:42:09.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:09 smithi177 bash[68594]: cluster 2024-09-13T14:42:08.357233+0000 mgr.y (mgr.26252) 470 : cluster [DBG] pgmap v800: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:09.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:09 smithi177 bash[68594]: cluster 2024-09-13T14:42:08.357233+0000 mgr.y (mgr.26252) 470 : cluster [DBG] pgmap v800: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:10.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:10 smithi099 bash[84576]: cluster 2024-09-13T14:42:09.223743+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-13T14:42:10.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:10 smithi099 bash[84576]: cluster 2024-09-13T14:42:09.223743+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-13T14:42:10.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:10 smithi099 bash[87147]: cluster 2024-09-13T14:42:09.223743+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-13T14:42:10.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:10 smithi099 bash[87147]: cluster 2024-09-13T14:42:09.223743+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-13T14:42:10.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:10 smithi177 bash[68594]: cluster 2024-09-13T14:42:09.223743+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-13T14:42:10.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:10 smithi177 bash[68594]: cluster 2024-09-13T14:42:09.223743+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-13T14:42:10.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:42:10 smithi177 bash[61439]: ts=2024-09-13T14:42:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:11.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:11 smithi099 bash[84576]: cluster 2024-09-13T14:42:10.226919+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-13T14:42:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:11 smithi099 bash[84576]: cluster 2024-09-13T14:42:10.226919+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-13T14:42:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:11 smithi099 bash[84576]: cluster 2024-09-13T14:42:10.358387+0000 mgr.y (mgr.26252) 471 : cluster [DBG] pgmap v803: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:11 smithi099 bash[84576]: cluster 2024-09-13T14:42:10.358387+0000 mgr.y (mgr.26252) 471 : cluster [DBG] pgmap v803: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:11.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:11 smithi099 bash[87147]: cluster 2024-09-13T14:42:10.226919+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-13T14:42:11.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:11 smithi099 bash[87147]: cluster 2024-09-13T14:42:10.226919+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-13T14:42:11.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:11 smithi099 bash[87147]: cluster 2024-09-13T14:42:10.358387+0000 mgr.y (mgr.26252) 471 : cluster [DBG] pgmap v803: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:11.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:11 smithi099 bash[87147]: cluster 2024-09-13T14:42:10.358387+0000 mgr.y (mgr.26252) 471 : cluster [DBG] pgmap v803: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:11.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:11 smithi177 bash[68594]: cluster 2024-09-13T14:42:10.226919+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-13T14:42:11.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:11 smithi177 bash[68594]: cluster 2024-09-13T14:42:10.226919+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-13T14:42:11.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:11 smithi177 bash[68594]: cluster 2024-09-13T14:42:10.358387+0000 mgr.y (mgr.26252) 471 : cluster [DBG] pgmap v803: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:11.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:11 smithi177 bash[68594]: cluster 2024-09-13T14:42:10.358387+0000 mgr.y (mgr.26252) 471 : cluster [DBG] pgmap v803: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:12 smithi099 bash[84576]: cluster 2024-09-13T14:42:11.230335+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-13T14:42:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:12 smithi099 bash[84576]: cluster 2024-09-13T14:42:11.230335+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-13T14:42:12.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:12 smithi099 bash[84576]: cluster 2024-09-13T14:42:12.233722+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-13T14:42:12.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:12 smithi099 bash[84576]: cluster 2024-09-13T14:42:12.233722+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-13T14:42:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:12 smithi099 bash[87147]: cluster 2024-09-13T14:42:11.230335+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-13T14:42:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:12 smithi099 bash[87147]: cluster 2024-09-13T14:42:11.230335+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-13T14:42:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:12 smithi099 bash[87147]: cluster 2024-09-13T14:42:12.233722+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-13T14:42:12.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:12 smithi099 bash[87147]: cluster 2024-09-13T14:42:12.233722+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-13T14:42:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:12 smithi177 bash[68594]: cluster 2024-09-13T14:42:11.230335+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-13T14:42:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:12 smithi177 bash[68594]: cluster 2024-09-13T14:42:11.230335+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-13T14:42:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:12 smithi177 bash[68594]: cluster 2024-09-13T14:42:12.233722+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-13T14:42:12.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:12 smithi177 bash[68594]: cluster 2024-09-13T14:42:12.233722+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-13T14:42:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:13 smithi099 bash[84576]: cluster 2024-09-13T14:42:12.359283+0000 mgr.y (mgr.26252) 472 : cluster [DBG] pgmap v806: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:13.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:13 smithi099 bash[84576]: cluster 2024-09-13T14:42:12.359283+0000 mgr.y (mgr.26252) 472 : cluster [DBG] pgmap v806: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:13.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:13 smithi099 bash[87147]: cluster 2024-09-13T14:42:12.359283+0000 mgr.y (mgr.26252) 472 : cluster [DBG] pgmap v806: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:13.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:13 smithi099 bash[87147]: cluster 2024-09-13T14:42:12.359283+0000 mgr.y (mgr.26252) 472 : cluster [DBG] pgmap v806: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:13.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:13 smithi177 bash[68594]: cluster 2024-09-13T14:42:12.359283+0000 mgr.y (mgr.26252) 472 : cluster [DBG] pgmap v806: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:13.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:13 smithi177 bash[68594]: cluster 2024-09-13T14:42:12.359283+0000 mgr.y (mgr.26252) 472 : cluster [DBG] pgmap v806: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:14.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:14 smithi099 bash[84576]: cluster 2024-09-13T14:42:13.244766+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-13T14:42:14.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:14 smithi099 bash[84576]: cluster 2024-09-13T14:42:13.244766+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-13T14:42:14.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:14 smithi099 bash[84576]: cluster 2024-09-13T14:42:14.240795+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-13T14:42:14.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:14 smithi099 bash[84576]: cluster 2024-09-13T14:42:14.240795+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-13T14:42:14.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:14 smithi099 bash[87147]: cluster 2024-09-13T14:42:13.244766+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-13T14:42:14.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:14 smithi099 bash[87147]: cluster 2024-09-13T14:42:13.244766+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-13T14:42:14.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:14 smithi099 bash[87147]: cluster 2024-09-13T14:42:14.240795+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-13T14:42:14.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:14 smithi099 bash[87147]: cluster 2024-09-13T14:42:14.240795+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-13T14:42:14.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:14 smithi177 bash[68594]: cluster 2024-09-13T14:42:13.244766+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-13T14:42:14.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:14 smithi177 bash[68594]: cluster 2024-09-13T14:42:13.244766+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-13T14:42:14.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:14 smithi177 bash[68594]: cluster 2024-09-13T14:42:14.240795+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-13T14:42:14.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:14 smithi177 bash[68594]: cluster 2024-09-13T14:42:14.240795+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-13T14:42:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:15 smithi099 bash[84576]: cluster 2024-09-13T14:42:14.360038+0000 mgr.y (mgr.26252) 473 : cluster [DBG] pgmap v809: 297 pgs: 297 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:15.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:15 smithi099 bash[84576]: cluster 2024-09-13T14:42:14.360038+0000 mgr.y (mgr.26252) 473 : cluster [DBG] pgmap v809: 297 pgs: 297 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:15.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:15 smithi099 bash[84576]: cluster 2024-09-13T14:42:15.245207+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-13T14:42:15.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:15 smithi099 bash[84576]: cluster 2024-09-13T14:42:15.245207+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-13T14:42:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:15 smithi099 bash[87147]: cluster 2024-09-13T14:42:14.360038+0000 mgr.y (mgr.26252) 473 : cluster [DBG] pgmap v809: 297 pgs: 297 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:15 smithi099 bash[87147]: cluster 2024-09-13T14:42:14.360038+0000 mgr.y (mgr.26252) 473 : cluster [DBG] pgmap v809: 297 pgs: 297 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:15 smithi099 bash[87147]: cluster 2024-09-13T14:42:15.245207+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-13T14:42:15.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:15 smithi099 bash[87147]: cluster 2024-09-13T14:42:15.245207+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-13T14:42:15.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:15 smithi177 bash[68594]: cluster 2024-09-13T14:42:14.360038+0000 mgr.y (mgr.26252) 473 : cluster [DBG] pgmap v809: 297 pgs: 297 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:15.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:15 smithi177 bash[68594]: cluster 2024-09-13T14:42:14.360038+0000 mgr.y (mgr.26252) 473 : cluster [DBG] pgmap v809: 297 pgs: 297 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:15.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:15 smithi177 bash[68594]: cluster 2024-09-13T14:42:15.245207+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-13T14:42:15.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:15 smithi177 bash[68594]: cluster 2024-09-13T14:42:15.245207+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-13T14:42:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:42:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:42:17] "GET /metrics HTTP/1.1" 200 77556 "" "Prometheus/2.51.0" 2024-09-13T14:42:17.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:17 smithi099 bash[87147]: cluster 2024-09-13T14:42:16.245326+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-13T14:42:17.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:17 smithi099 bash[87147]: cluster 2024-09-13T14:42:16.245326+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-13T14:42:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:17 smithi099 bash[87147]: cluster 2024-09-13T14:42:16.360845+0000 mgr.y (mgr.26252) 474 : cluster [DBG] pgmap v812: 233 pgs: 233 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:17.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:17 smithi099 bash[87147]: cluster 2024-09-13T14:42:16.360845+0000 mgr.y (mgr.26252) 474 : cluster [DBG] pgmap v812: 233 pgs: 233 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:17.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:17 smithi099 bash[84576]: cluster 2024-09-13T14:42:16.245326+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-13T14:42:17.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:17 smithi099 bash[84576]: cluster 2024-09-13T14:42:16.245326+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-13T14:42:17.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:17 smithi099 bash[84576]: cluster 2024-09-13T14:42:16.360845+0000 mgr.y (mgr.26252) 474 : cluster [DBG] pgmap v812: 233 pgs: 233 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:17.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:17 smithi099 bash[84576]: cluster 2024-09-13T14:42:16.360845+0000 mgr.y (mgr.26252) 474 : cluster [DBG] pgmap v812: 233 pgs: 233 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:17.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:17 smithi177 bash[68594]: cluster 2024-09-13T14:42:16.245326+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-13T14:42:17.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:17 smithi177 bash[68594]: cluster 2024-09-13T14:42:16.245326+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-13T14:42:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:17 smithi177 bash[68594]: cluster 2024-09-13T14:42:16.360845+0000 mgr.y (mgr.26252) 474 : cluster [DBG] pgmap v812: 233 pgs: 233 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:17.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:17 smithi177 bash[68594]: cluster 2024-09-13T14:42:16.360845+0000 mgr.y (mgr.26252) 474 : cluster [DBG] pgmap v812: 233 pgs: 233 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:18.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:18 smithi099 bash[84576]: cluster 2024-09-13T14:42:17.257341+0000 mon.a (mon.0) 1375 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-13T14:42:18.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:18 smithi099 bash[84576]: cluster 2024-09-13T14:42:17.257341+0000 mon.a (mon.0) 1375 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-13T14:42:18.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:18 smithi099 bash[87147]: cluster 2024-09-13T14:42:17.257341+0000 mon.a (mon.0) 1375 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-13T14:42:18.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:18 smithi099 bash[87147]: cluster 2024-09-13T14:42:17.257341+0000 mon.a (mon.0) 1375 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-13T14:42:18.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:18 smithi177 bash[68594]: cluster 2024-09-13T14:42:17.257341+0000 mon.a (mon.0) 1375 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-13T14:42:18.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:18 smithi177 bash[68594]: cluster 2024-09-13T14:42:17.257341+0000 mon.a (mon.0) 1375 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-13T14:42:19.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[84576]: audit 2024-09-13T14:42:18.256112+0000 mon.a (mon.0) 1376 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:19.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[84576]: audit 2024-09-13T14:42:18.256112+0000 mon.a (mon.0) 1376 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[84576]: cluster 2024-09-13T14:42:18.263601+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-13T14:42:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[84576]: cluster 2024-09-13T14:42:18.263601+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-13T14:42:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[84576]: cluster 2024-09-13T14:42:18.361526+0000 mgr.y (mgr.26252) 475 : cluster [DBG] pgmap v815: 169 pgs: 169 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[84576]: cluster 2024-09-13T14:42:18.361526+0000 mgr.y (mgr.26252) 475 : cluster [DBG] pgmap v815: 169 pgs: 169 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:19.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[87147]: audit 2024-09-13T14:42:18.256112+0000 mon.a (mon.0) 1376 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:19.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[87147]: audit 2024-09-13T14:42:18.256112+0000 mon.a (mon.0) 1376 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:19.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[87147]: cluster 2024-09-13T14:42:18.263601+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-13T14:42:19.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[87147]: cluster 2024-09-13T14:42:18.263601+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-13T14:42:19.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[87147]: cluster 2024-09-13T14:42:18.361526+0000 mgr.y (mgr.26252) 475 : cluster [DBG] pgmap v815: 169 pgs: 169 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:19.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:19 smithi099 bash[87147]: cluster 2024-09-13T14:42:18.361526+0000 mgr.y (mgr.26252) 475 : cluster [DBG] pgmap v815: 169 pgs: 169 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:19.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:19 smithi177 bash[68594]: audit 2024-09-13T14:42:18.256112+0000 mon.a (mon.0) 1376 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:19 smithi177 bash[68594]: audit 2024-09-13T14:42:18.256112+0000 mon.a (mon.0) 1376 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:19 smithi177 bash[68594]: cluster 2024-09-13T14:42:18.263601+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-13T14:42:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:19 smithi177 bash[68594]: cluster 2024-09-13T14:42:18.263601+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-13T14:42:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:19 smithi177 bash[68594]: cluster 2024-09-13T14:42:18.361526+0000 mgr.y (mgr.26252) 475 : cluster [DBG] pgmap v815: 169 pgs: 169 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:19 smithi177 bash[68594]: cluster 2024-09-13T14:42:18.361526+0000 mgr.y (mgr.26252) 475 : cluster [DBG] pgmap v815: 169 pgs: 169 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:20.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:20 smithi099 bash[84576]: cluster 2024-09-13T14:42:19.275964+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-13T14:42:20.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:20 smithi099 bash[84576]: cluster 2024-09-13T14:42:19.275964+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-13T14:42:20.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:20 smithi099 bash[87147]: cluster 2024-09-13T14:42:19.275964+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-13T14:42:20.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:20 smithi099 bash[87147]: cluster 2024-09-13T14:42:19.275964+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-13T14:42:20.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:20 smithi177 bash[68594]: cluster 2024-09-13T14:42:19.275964+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-13T14:42:20.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:20 smithi177 bash[68594]: cluster 2024-09-13T14:42:19.275964+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-13T14:42:20.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:42:20 smithi177 bash[61439]: ts=2024-09-13T14:42:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:21.304 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 160 tests from TestLibRBD (366261 ms total) 2024-09-13T14:42:21.304 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:42:21.304 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 11 tests from DiffIterateTest/0, where TypeParam = DiffIterateParams 2024-09-13T14:42:21.364 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterate 2024-09-13T14:42:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:21 smithi099 bash[84576]: cluster 2024-09-13T14:42:20.288905+0000 mon.a (mon.0) 1379 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-13T14:42:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:21 smithi099 bash[84576]: cluster 2024-09-13T14:42:20.288905+0000 mon.a (mon.0) 1379 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-13T14:42:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:21 smithi099 bash[84576]: cluster 2024-09-13T14:42:20.362237+0000 mgr.y (mgr.26252) 476 : cluster [DBG] pgmap v818: 105 pgs: 105 active+clean; 586 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:21 smithi099 bash[84576]: cluster 2024-09-13T14:42:20.362237+0000 mgr.y (mgr.26252) 476 : cluster [DBG] pgmap v818: 105 pgs: 105 active+clean; 586 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:21 smithi099 bash[87147]: cluster 2024-09-13T14:42:20.288905+0000 mon.a (mon.0) 1379 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-13T14:42:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:21 smithi099 bash[87147]: cluster 2024-09-13T14:42:20.288905+0000 mon.a (mon.0) 1379 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-13T14:42:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:21 smithi099 bash[87147]: cluster 2024-09-13T14:42:20.362237+0000 mgr.y (mgr.26252) 476 : cluster [DBG] pgmap v818: 105 pgs: 105 active+clean; 586 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:21 smithi099 bash[87147]: cluster 2024-09-13T14:42:20.362237+0000 mgr.y (mgr.26252) 476 : cluster [DBG] pgmap v818: 105 pgs: 105 active+clean; 586 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:21.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:21 smithi177 bash[68594]: cluster 2024-09-13T14:42:20.288905+0000 mon.a (mon.0) 1379 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-13T14:42:21.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:21 smithi177 bash[68594]: cluster 2024-09-13T14:42:20.288905+0000 mon.a (mon.0) 1379 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-13T14:42:21.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:21 smithi177 bash[68594]: cluster 2024-09-13T14:42:20.362237+0000 mgr.y (mgr.26252) 476 : cluster [DBG] pgmap v818: 105 pgs: 105 active+clean; 586 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:21.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:21 smithi177 bash[68594]: cluster 2024-09-13T14:42:20.362237+0000 mgr.y (mgr.26252) 476 : cluster [DBG] pgmap v818: 105 pgs: 105 active+clean; 586 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:22.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:22 smithi099 bash[84576]: cluster 2024-09-13T14:42:21.300081+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-13T14:42:22.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:22 smithi099 bash[84576]: cluster 2024-09-13T14:42:21.300081+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-13T14:42:22.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:22 smithi099 bash[87147]: cluster 2024-09-13T14:42:21.300081+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-13T14:42:22.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:22 smithi099 bash[87147]: cluster 2024-09-13T14:42:21.300081+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-13T14:42:22.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:22 smithi177 bash[68594]: cluster 2024-09-13T14:42:21.300081+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-13T14:42:22.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:22 smithi177 bash[68594]: cluster 2024-09-13T14:42:21.300081+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-13T14:42:23.314 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:42:23.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: cluster 2024-09-13T14:42:22.307208+0000 mon.a (mon.0) 1381 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-13T14:42:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: cluster 2024-09-13T14:42:22.307208+0000 mon.a (mon.0) 1381 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-13T14:42:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: audit 2024-09-13T14:42:22.310972+0000 mon.c (mon.2) 42 : audit [INF] from='client.? 172.21.15.99:0/897123246' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: audit 2024-09-13T14:42:22.310972+0000 mon.c (mon.2) 42 : audit [INF] from='client.? 172.21.15.99:0/897123246' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: audit 2024-09-13T14:42:22.321058+0000 mon.a (mon.0) 1382 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: audit 2024-09-13T14:42:22.321058+0000 mon.a (mon.0) 1382 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: cluster 2024-09-13T14:42:22.362949+0000 mgr.y (mgr.26252) 477 : cluster [DBG] pgmap v821: 105 pgs: 3 creating+peering, 29 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: cluster 2024-09-13T14:42:22.362949+0000 mgr.y (mgr.26252) 477 : cluster [DBG] pgmap v821: 105 pgs: 3 creating+peering, 29 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: audit 2024-09-13T14:42:23.297653+0000 mon.a (mon.0) 1383 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:42:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: audit 2024-09-13T14:42:23.297653+0000 mon.a (mon.0) 1383 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:42:23.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: cluster 2024-09-13T14:42:23.305791+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-13T14:42:23.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[84576]: cluster 2024-09-13T14:42:23.305791+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-13T14:42:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: cluster 2024-09-13T14:42:22.307208+0000 mon.a (mon.0) 1381 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-13T14:42:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: cluster 2024-09-13T14:42:22.307208+0000 mon.a (mon.0) 1381 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-13T14:42:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: audit 2024-09-13T14:42:22.310972+0000 mon.c (mon.2) 42 : audit [INF] from='client.? 172.21.15.99:0/897123246' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: audit 2024-09-13T14:42:22.310972+0000 mon.c (mon.2) 42 : audit [INF] from='client.? 172.21.15.99:0/897123246' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: audit 2024-09-13T14:42:22.321058+0000 mon.a (mon.0) 1382 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: audit 2024-09-13T14:42:22.321058+0000 mon.a (mon.0) 1382 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: cluster 2024-09-13T14:42:22.362949+0000 mgr.y (mgr.26252) 477 : cluster [DBG] pgmap v821: 105 pgs: 3 creating+peering, 29 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:23.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: cluster 2024-09-13T14:42:22.362949+0000 mgr.y (mgr.26252) 477 : cluster [DBG] pgmap v821: 105 pgs: 3 creating+peering, 29 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:23.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: audit 2024-09-13T14:42:23.297653+0000 mon.a (mon.0) 1383 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:42:23.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: audit 2024-09-13T14:42:23.297653+0000 mon.a (mon.0) 1383 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:42:23.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: cluster 2024-09-13T14:42:23.305791+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-13T14:42:23.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:23 smithi099 bash[87147]: cluster 2024-09-13T14:42:23.305791+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-13T14:42:23.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: cluster 2024-09-13T14:42:22.307208+0000 mon.a (mon.0) 1381 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: cluster 2024-09-13T14:42:22.307208+0000 mon.a (mon.0) 1381 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: audit 2024-09-13T14:42:22.310972+0000 mon.c (mon.2) 42 : audit [INF] from='client.? 172.21.15.99:0/897123246' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: audit 2024-09-13T14:42:22.310972+0000 mon.c (mon.2) 42 : audit [INF] from='client.? 172.21.15.99:0/897123246' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: audit 2024-09-13T14:42:22.321058+0000 mon.a (mon.0) 1382 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: audit 2024-09-13T14:42:22.321058+0000 mon.a (mon.0) 1382 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: cluster 2024-09-13T14:42:22.362949+0000 mgr.y (mgr.26252) 477 : cluster [DBG] pgmap v821: 105 pgs: 3 creating+peering, 29 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: cluster 2024-09-13T14:42:22.362949+0000 mgr.y (mgr.26252) 477 : cluster [DBG] pgmap v821: 105 pgs: 3 creating+peering, 29 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: audit 2024-09-13T14:42:23.297653+0000 mon.a (mon.0) 1383 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: audit 2024-09-13T14:42:23.297653+0000 mon.a (mon.0) 1383 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: cluster 2024-09-13T14:42:23.305791+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-13T14:42:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:23 smithi177 bash[68594]: cluster 2024-09-13T14:42:23.305791+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-13T14:42:24.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:24 smithi099 bash[84576]: cluster 2024-09-13T14:42:23.314837+0000 mon.a (mon.0) 1385 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:42:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:24 smithi099 bash[84576]: cluster 2024-09-13T14:42:23.314837+0000 mon.a (mon.0) 1385 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:42:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:24 smithi099 bash[84576]: cluster 2024-09-13T14:42:24.306612+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-13T14:42:24.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:24 smithi099 bash[84576]: cluster 2024-09-13T14:42:24.306612+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-13T14:42:24.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:24 smithi099 bash[87147]: cluster 2024-09-13T14:42:23.314837+0000 mon.a (mon.0) 1385 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:42:24.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:24 smithi099 bash[87147]: cluster 2024-09-13T14:42:23.314837+0000 mon.a (mon.0) 1385 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:42:24.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:24 smithi099 bash[87147]: cluster 2024-09-13T14:42:24.306612+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-13T14:42:24.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:24 smithi099 bash[87147]: cluster 2024-09-13T14:42:24.306612+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-13T14:42:24.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:24 smithi177 bash[68594]: cluster 2024-09-13T14:42:23.314837+0000 mon.a (mon.0) 1385 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:42:24.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:24 smithi177 bash[68594]: cluster 2024-09-13T14:42:23.314837+0000 mon.a (mon.0) 1385 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:42:24.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:24 smithi177 bash[68594]: cluster 2024-09-13T14:42:24.306612+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-13T14:42:24.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:24 smithi177 bash[68594]: cluster 2024-09-13T14:42:24.306612+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-13T14:42:25.380 INFO:tasks.workunit.client.0.smithi099.stdout: wrote [2220996~69280,7992788~5927,8362887~9316,10160593~42483,11000523~58229,11408448~67305,12719047~1215,13734720~8758,15490004~3315,20527505~66246] 2024-09-13T14:42:25.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:25 smithi177 bash[68594]: cluster 2024-09-13T14:42:24.363602+0000 mgr.y (mgr.26252) 478 : cluster [DBG] pgmap v824: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:25.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:25 smithi177 bash[68594]: cluster 2024-09-13T14:42:24.363602+0000 mgr.y (mgr.26252) 478 : cluster [DBG] pgmap v824: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:25.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:25 smithi177 bash[68594]: cluster 2024-09-13T14:42:25.310168+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-13T14:42:25.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:25 smithi177 bash[68594]: cluster 2024-09-13T14:42:25.310168+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-13T14:42:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:25 smithi099 bash[84576]: cluster 2024-09-13T14:42:24.363602+0000 mgr.y (mgr.26252) 478 : cluster [DBG] pgmap v824: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:25 smithi099 bash[84576]: cluster 2024-09-13T14:42:24.363602+0000 mgr.y (mgr.26252) 478 : cluster [DBG] pgmap v824: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:25.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:25 smithi099 bash[84576]: cluster 2024-09-13T14:42:25.310168+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-13T14:42:25.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:25 smithi099 bash[84576]: cluster 2024-09-13T14:42:25.310168+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-13T14:42:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:25 smithi099 bash[87147]: cluster 2024-09-13T14:42:24.363602+0000 mgr.y (mgr.26252) 478 : cluster [DBG] pgmap v824: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:25 smithi099 bash[87147]: cluster 2024-09-13T14:42:24.363602+0000 mgr.y (mgr.26252) 478 : cluster [DBG] pgmap v824: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:25 smithi099 bash[87147]: cluster 2024-09-13T14:42:25.310168+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-13T14:42:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:25 smithi099 bash[87147]: cluster 2024-09-13T14:42:25.310168+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-13T14:42:26.380 INFO:tasks.workunit.client.0.smithi099.stdout: wrote [1068234~10160,3798202~95355,4205881~2546,5884827~99195,7619478~86668,8034881~16010,10972110~59347,18145547~92202,19986181~16006,20475068~35322] 2024-09-13T14:42:26.398 INFO:tasks.workunit.client.0.smithi099.stdout: diff was [1068234~10160,2290276~1603281,4205881~2546,5884827~99195,7619478~86668,8034881~16010,10972110~59347,18145547~92202,19986181~16006,20475068~35322] 2024-09-13T14:42:26.406 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterate (5043 ms) 2024-09-13T14:42:26.406 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDiscard 2024-09-13T14:42:26.406 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:42:26.463 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~256 2024-09-13T14:42:26.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:26 smithi177 bash[68594]: cluster 2024-09-13T14:42:25.323760+0000 mon.a (mon.0) 1388 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:42:26.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:26 smithi177 bash[68594]: cluster 2024-09-13T14:42:25.323760+0000 mon.a (mon.0) 1388 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:42:26.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:26 smithi177 bash[68594]: cluster 2024-09-13T14:42:25.323834+0000 mon.a (mon.0) 1389 : cluster [INF] Cluster is now healthy 2024-09-13T14:42:26.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:26 smithi177 bash[68594]: cluster 2024-09-13T14:42:25.323834+0000 mon.a (mon.0) 1389 : cluster [INF] Cluster is now healthy 2024-09-13T14:42:26.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:26 smithi099 bash[84576]: cluster 2024-09-13T14:42:25.323760+0000 mon.a (mon.0) 1388 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:42:26.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:26 smithi099 bash[84576]: cluster 2024-09-13T14:42:25.323760+0000 mon.a (mon.0) 1388 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:42:26.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:26 smithi099 bash[84576]: cluster 2024-09-13T14:42:25.323834+0000 mon.a (mon.0) 1389 : cluster [INF] Cluster is now healthy 2024-09-13T14:42:26.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:26 smithi099 bash[84576]: cluster 2024-09-13T14:42:25.323834+0000 mon.a (mon.0) 1389 : cluster [INF] Cluster is now healthy 2024-09-13T14:42:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:26 smithi099 bash[87147]: cluster 2024-09-13T14:42:25.323760+0000 mon.a (mon.0) 1388 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:42:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:26 smithi099 bash[87147]: cluster 2024-09-13T14:42:25.323760+0000 mon.a (mon.0) 1388 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:42:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:26 smithi099 bash[87147]: cluster 2024-09-13T14:42:25.323834+0000 mon.a (mon.0) 1389 : cluster [INF] Cluster is now healthy 2024-09-13T14:42:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:26 smithi099 bash[87147]: cluster 2024-09-13T14:42:25.323834+0000 mon.a (mon.0) 1389 : cluster [INF] Cluster is now healthy 2024-09-13T14:42:27.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:42:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:42:27] "GET /metrics HTTP/1.1" 200 42427 "" "Prometheus/2.51.0" 2024-09-13T14:42:27.401 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~256 2024-09-13T14:42:27.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:27 smithi177 bash[68594]: cluster 2024-09-13T14:42:26.354588+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-13T14:42:27.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:27 smithi177 bash[68594]: cluster 2024-09-13T14:42:26.354588+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-13T14:42:27.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:27 smithi177 bash[68594]: cluster 2024-09-13T14:42:26.364222+0000 mgr.y (mgr.26252) 479 : cluster [DBG] pgmap v827: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:27.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:27 smithi177 bash[68594]: cluster 2024-09-13T14:42:26.364222+0000 mgr.y (mgr.26252) 479 : cluster [DBG] pgmap v827: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:27 smithi099 bash[84576]: cluster 2024-09-13T14:42:26.354588+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-13T14:42:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:27 smithi099 bash[84576]: cluster 2024-09-13T14:42:26.354588+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-13T14:42:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:27 smithi099 bash[84576]: cluster 2024-09-13T14:42:26.364222+0000 mgr.y (mgr.26252) 479 : cluster [DBG] pgmap v827: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:27.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:27 smithi099 bash[84576]: cluster 2024-09-13T14:42:26.364222+0000 mgr.y (mgr.26252) 479 : cluster [DBG] pgmap v827: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:27 smithi099 bash[87147]: cluster 2024-09-13T14:42:26.354588+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-13T14:42:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:27 smithi099 bash[87147]: cluster 2024-09-13T14:42:26.354588+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-13T14:42:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:27 smithi099 bash[87147]: cluster 2024-09-13T14:42:26.364222+0000 mgr.y (mgr.26252) 479 : cluster [DBG] pgmap v827: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:27 smithi099 bash[87147]: cluster 2024-09-13T14:42:26.364222+0000 mgr.y (mgr.26252) 479 : cluster [DBG] pgmap v827: 105 pgs: 15 creating+peering, 4 unknown, 86 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:42:28.414 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~256 2024-09-13T14:42:28.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:28 smithi099 bash[87147]: cluster 2024-09-13T14:42:27.372104+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-13T14:42:28.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:28 smithi099 bash[87147]: cluster 2024-09-13T14:42:27.372104+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-13T14:42:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:28 smithi099 bash[84576]: cluster 2024-09-13T14:42:27.372104+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-13T14:42:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:28 smithi099 bash[84576]: cluster 2024-09-13T14:42:27.372104+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-13T14:42:28.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:28 smithi177 bash[68594]: cluster 2024-09-13T14:42:27.372104+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-13T14:42:28.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:28 smithi177 bash[68594]: cluster 2024-09-13T14:42:27.372104+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-13T14:42:29.428 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~256 2024-09-13T14:42:29.432 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterateDiscard (3025 ms) 2024-09-13T14:42:29.432 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStress 2024-09-13T14:42:29.432 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:42:29.570 INFO:tasks.workunit.client.0.smithi099.stdout: i=0 exists [24095272~3469942,34763164~3883094,55338128~6476200,79873207~5711554,98935619~4770731,135394590~7539557,234272105~1146148,244032850~6299094,312344726~6385006,366681712~3018802] wrote [24095272~3469942,34763164~3883094,55338128~6476200,79873207~5711554,98935619~4770731,135394590~7539557,234272105~1146148,244032850~6299094,312344726~6385006,366681712~3018802] 2024-09-13T14:42:29.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[84576]: cluster 2024-09-13T14:42:28.365203+0000 mgr.y (mgr.26252) 480 : cluster [DBG] pgmap v829: 105 pgs: 105 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 252 B/s rd, 148 KiB/s wr, 4 op/s 2024-09-13T14:42:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[84576]: cluster 2024-09-13T14:42:28.365203+0000 mgr.y (mgr.26252) 480 : cluster [DBG] pgmap v829: 105 pgs: 105 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 252 B/s rd, 148 KiB/s wr, 4 op/s 2024-09-13T14:42:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[84576]: cluster 2024-09-13T14:42:28.373964+0000 mon.a (mon.0) 1392 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-13T14:42:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[84576]: cluster 2024-09-13T14:42:28.373964+0000 mon.a (mon.0) 1392 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-13T14:42:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[84576]: audit 2024-09-13T14:42:28.421667+0000 mon.a (mon.0) 1393 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[84576]: audit 2024-09-13T14:42:28.421667+0000 mon.a (mon.0) 1393 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[84576]: audit 2024-09-13T14:42:29.019860+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[84576]: audit 2024-09-13T14:42:29.019860+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[84576]: audit 2024-09-13T14:42:29.027047+0000 mon.a (mon.0) 1395 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[84576]: audit 2024-09-13T14:42:29.027047+0000 mon.a (mon.0) 1395 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[87147]: cluster 2024-09-13T14:42:28.365203+0000 mgr.y (mgr.26252) 480 : cluster [DBG] pgmap v829: 105 pgs: 105 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 252 B/s rd, 148 KiB/s wr, 4 op/s 2024-09-13T14:42:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[87147]: cluster 2024-09-13T14:42:28.365203+0000 mgr.y (mgr.26252) 480 : cluster [DBG] pgmap v829: 105 pgs: 105 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 252 B/s rd, 148 KiB/s wr, 4 op/s 2024-09-13T14:42:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[87147]: cluster 2024-09-13T14:42:28.373964+0000 mon.a (mon.0) 1392 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-13T14:42:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[87147]: cluster 2024-09-13T14:42:28.373964+0000 mon.a (mon.0) 1392 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-13T14:42:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[87147]: audit 2024-09-13T14:42:28.421667+0000 mon.a (mon.0) 1393 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[87147]: audit 2024-09-13T14:42:28.421667+0000 mon.a (mon.0) 1393 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[87147]: audit 2024-09-13T14:42:29.019860+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[87147]: audit 2024-09-13T14:42:29.019860+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[87147]: audit 2024-09-13T14:42:29.027047+0000 mon.a (mon.0) 1395 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:29 smithi099 bash[87147]: audit 2024-09-13T14:42:29.027047+0000 mon.a (mon.0) 1395 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:29 smithi177 bash[68594]: cluster 2024-09-13T14:42:28.365203+0000 mgr.y (mgr.26252) 480 : cluster [DBG] pgmap v829: 105 pgs: 105 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 252 B/s rd, 148 KiB/s wr, 4 op/s 2024-09-13T14:42:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:29 smithi177 bash[68594]: cluster 2024-09-13T14:42:28.365203+0000 mgr.y (mgr.26252) 480 : cluster [DBG] pgmap v829: 105 pgs: 105 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 252 B/s rd, 148 KiB/s wr, 4 op/s 2024-09-13T14:42:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:29 smithi177 bash[68594]: cluster 2024-09-13T14:42:28.373964+0000 mon.a (mon.0) 1392 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-13T14:42:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:29 smithi177 bash[68594]: cluster 2024-09-13T14:42:28.373964+0000 mon.a (mon.0) 1392 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-13T14:42:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:29 smithi177 bash[68594]: audit 2024-09-13T14:42:28.421667+0000 mon.a (mon.0) 1393 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:29 smithi177 bash[68594]: audit 2024-09-13T14:42:28.421667+0000 mon.a (mon.0) 1393 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:29 smithi177 bash[68594]: audit 2024-09-13T14:42:29.019860+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:29 smithi177 bash[68594]: audit 2024-09-13T14:42:29.019860+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:29 smithi177 bash[68594]: audit 2024-09-13T14:42:29.027047+0000 mon.a (mon.0) 1395 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:29 smithi177 bash[68594]: audit 2024-09-13T14:42:29.027047+0000 mon.a (mon.0) 1395 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:30.503 INFO:tasks.workunit.client.0.smithi099.stdout: i=1 exists [14140835~6473260,24095272~3469942,34763164~7628796,55338128~6476200,63605087~1204012,79873207~5711554,98935619~4770731,124729756~3807852,135394590~7539557,149941644~3016691,182438565~2356224,219834611~3809766,234272105~1146148,240866559~221559,244032850~6299094,312344726~6385006,326980227~7932989,366681712~3018802,406669382~2394616] wrote [14140835~6473260,35900567~6491393,63605087~1204012,124729756~3807852,149941644~3016691,182438565~2356224,219834611~3809766,240866559~221559,326980227~7932989,406669382~2394616] 2024-09-13T14:42:30.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:30 smithi177 bash[68594]: cluster 2024-09-13T14:42:29.388838+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-13T14:42:30.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:30 smithi177 bash[68594]: cluster 2024-09-13T14:42:29.388838+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-13T14:42:30.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:42:30 smithi177 bash[61439]: ts=2024-09-13T14:42:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:30 smithi099 bash[84576]: cluster 2024-09-13T14:42:29.388838+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-13T14:42:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:30 smithi099 bash[84576]: cluster 2024-09-13T14:42:29.388838+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-13T14:42:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:30 smithi099 bash[87147]: cluster 2024-09-13T14:42:29.388838+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-13T14:42:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:30 smithi099 bash[87147]: cluster 2024-09-13T14:42:29.388838+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-13T14:42:31.344 INFO:tasks.workunit.client.0.smithi099.stdout: i=2 exists [3148125~2497343,11616606~8997489,24095272~3469942,34763164~7628796,55338128~6476200,63605087~1204012,79873207~5711554,90104149~7458565,97704586~6220681,124729756~3807852,135394590~7539557,149941644~7147220,182438565~2356224,219834611~3809766,234272105~1146148,240866559~221559,244032850~6299094,312344726~6385006,322678965~1585454,326980227~9040796,366681712~3018802,379850485~2642533,405003943~6147882] wrote [3148125~2497343,11616606~4641737,90104149~7458565,97704586~6220681,151511398~5577466,322678965~1585454,329696827~6324196,379850485~2642533,405003943~6147882] 2024-09-13T14:42:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[84576]: cluster 2024-09-13T14:42:30.366047+0000 mgr.y (mgr.26252) 481 : cluster [DBG] pgmap v832: 105 pgs: 105 active+clean; 18 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 221 KiB/s wr, 95 op/s 2024-09-13T14:42:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[84576]: cluster 2024-09-13T14:42:30.366047+0000 mgr.y (mgr.26252) 481 : cluster [DBG] pgmap v832: 105 pgs: 105 active+clean; 18 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 221 KiB/s wr, 95 op/s 2024-09-13T14:42:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[84576]: cluster 2024-09-13T14:42:30.411003+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-13T14:42:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[84576]: cluster 2024-09-13T14:42:30.411003+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-13T14:42:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[84576]: cluster 2024-09-13T14:42:31.191666+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-13T14:42:31.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[84576]: cluster 2024-09-13T14:42:31.191666+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-13T14:42:31.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[87147]: cluster 2024-09-13T14:42:30.366047+0000 mgr.y (mgr.26252) 481 : cluster [DBG] pgmap v832: 105 pgs: 105 active+clean; 18 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 221 KiB/s wr, 95 op/s 2024-09-13T14:42:31.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[87147]: cluster 2024-09-13T14:42:30.366047+0000 mgr.y (mgr.26252) 481 : cluster [DBG] pgmap v832: 105 pgs: 105 active+clean; 18 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 221 KiB/s wr, 95 op/s 2024-09-13T14:42:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[87147]: cluster 2024-09-13T14:42:30.411003+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-13T14:42:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[87147]: cluster 2024-09-13T14:42:30.411003+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-13T14:42:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[87147]: cluster 2024-09-13T14:42:31.191666+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-13T14:42:31.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:31 smithi099 bash[87147]: cluster 2024-09-13T14:42:31.191666+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-13T14:42:31.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:31 smithi177 bash[68594]: cluster 2024-09-13T14:42:30.366047+0000 mgr.y (mgr.26252) 481 : cluster [DBG] pgmap v832: 105 pgs: 105 active+clean; 18 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 221 KiB/s wr, 95 op/s 2024-09-13T14:42:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:31 smithi177 bash[68594]: cluster 2024-09-13T14:42:30.366047+0000 mgr.y (mgr.26252) 481 : cluster [DBG] pgmap v832: 105 pgs: 105 active+clean; 18 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 221 KiB/s wr, 95 op/s 2024-09-13T14:42:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:31 smithi177 bash[68594]: cluster 2024-09-13T14:42:30.411003+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-13T14:42:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:31 smithi177 bash[68594]: cluster 2024-09-13T14:42:30.411003+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-13T14:42:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:31 smithi177 bash[68594]: cluster 2024-09-13T14:42:31.191666+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-13T14:42:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:31 smithi177 bash[68594]: cluster 2024-09-13T14:42:31.191666+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-13T14:42:32.293 INFO:tasks.workunit.client.0.smithi099.stdout: i=3 exists [3148125~2497343,10538482~10075613,24095272~3469942,34435575~7956385,55338128~6476200,63605087~1204012,79873207~5711554,90104149~13821118,124729756~3807852,132706774~10227373,149941644~7147220,182438565~2356224,198449267~1940830,219834611~3809766,234272105~1146148,240866559~221559,242466485~7865459,253893291~1225143,263022958~2091010,274100517~3132520,312344726~6385006,322678965~1585454,326980227~9040796,366681712~3018802,379850485~2642533,405003943~6147882] wrote [10538482~7692670,34435575~1894216,97410719~3652484,132706774~6904275,198449267~1940830,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,274100517~3132520] 2024-09-13T14:42:33.343 INFO:tasks.workunit.client.0.smithi099.stdout: i=4 exists [3148125~2497343,9390236~11223859,24095272~3469942,34435575~7956385,55338128~6476200,63605087~1204012,79873207~5711554,90104149~13821118,106191246~11701461,124729756~3807852,132706774~10227373,149941644~7147220,169300642~11861796,182438565~2356224,198449267~1940830,219834611~3809766,231958926~3459327,240866559~221559,242466485~7865459,253893291~1225143,263022958~2091010,273326128~4621377,312344726~6385006,322678965~1585454,326980227~9040796,366681712~3018802,379850485~2642533,389565511~8081080,405003943~6147882] wrote [9390236~6883116,91011347~3717169,106191246~11701461,152231961~2578327,169300642~11861796,231958926~3416707,273326128~4621377,389565511~8081080] 2024-09-13T14:42:33.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:33 smithi099 bash[84576]: cluster 2024-09-13T14:42:32.194718+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-13T14:42:33.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:33 smithi099 bash[84576]: cluster 2024-09-13T14:42:32.194718+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-13T14:42:33.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:33 smithi099 bash[84576]: cluster 2024-09-13T14:42:32.366824+0000 mgr.y (mgr.26252) 482 : cluster [DBG] pgmap v836: 105 pgs: 105 active+clean; 63 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 7.3 MiB/s wr, 95 op/s 2024-09-13T14:42:33.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:33 smithi099 bash[84576]: cluster 2024-09-13T14:42:32.366824+0000 mgr.y (mgr.26252) 482 : cluster [DBG] pgmap v836: 105 pgs: 105 active+clean; 63 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 7.3 MiB/s wr, 95 op/s 2024-09-13T14:42:33.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:33 smithi099 bash[87147]: cluster 2024-09-13T14:42:32.194718+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-13T14:42:33.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:33 smithi099 bash[87147]: cluster 2024-09-13T14:42:32.194718+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-13T14:42:33.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:33 smithi099 bash[87147]: cluster 2024-09-13T14:42:32.366824+0000 mgr.y (mgr.26252) 482 : cluster [DBG] pgmap v836: 105 pgs: 105 active+clean; 63 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 7.3 MiB/s wr, 95 op/s 2024-09-13T14:42:33.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:33 smithi099 bash[87147]: cluster 2024-09-13T14:42:32.366824+0000 mgr.y (mgr.26252) 482 : cluster [DBG] pgmap v836: 105 pgs: 105 active+clean; 63 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 7.3 MiB/s wr, 95 op/s 2024-09-13T14:42:33.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:33 smithi177 bash[68594]: cluster 2024-09-13T14:42:32.194718+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-13T14:42:33.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:33 smithi177 bash[68594]: cluster 2024-09-13T14:42:32.194718+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-13T14:42:33.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:33 smithi177 bash[68594]: cluster 2024-09-13T14:42:32.366824+0000 mgr.y (mgr.26252) 482 : cluster [DBG] pgmap v836: 105 pgs: 105 active+clean; 63 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 7.3 MiB/s wr, 95 op/s 2024-09-13T14:42:33.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:33 smithi177 bash[68594]: cluster 2024-09-13T14:42:32.366824+0000 mgr.y (mgr.26252) 482 : cluster [DBG] pgmap v836: 105 pgs: 105 active+clean; 63 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 7.3 MiB/s wr, 95 op/s 2024-09-13T14:42:34.285 INFO:tasks.workunit.client.0.smithi099.stdout: i=5 exists [3148125~2497343,9390236~11223859,24095272~3469942,34435575~7956385,50256085~3817320,55338128~6476200,63605087~1204012,79873207~5711554,87809486~16115781,106191246~17539866,124729756~3807852,132706774~10227373,149941644~7147220,162346201~1803261,169300642~11861796,181824389~2970400,198449267~1940830,219834611~3809766,231958926~3459327,240866559~221559,242466485~7865459,253893291~1225143,263022958~2091010,273326128~4621377,312344726~6385006,322678965~1585454,326980227~9040796,342881049~4268842,366681712~3018802,379850485~2642533,389565511~8081080,399873735~4931489,405003943~6147882] wrote [37838725~3913129,50256085~3817320,79983070~11130,87809486~7971187,115764491~7966621,162346201~1803261,177418312~599734,181824389~1807088,342881049~4268842,399873735~4931489] 2024-09-13T14:42:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[84576]: cluster 2024-09-13T14:42:33.206509+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-13T14:42:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[84576]: cluster 2024-09-13T14:42:33.206509+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-13T14:42:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[84576]: audit 2024-09-13T14:42:33.256328+0000 mon.a (mon.0) 1401 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[84576]: audit 2024-09-13T14:42:33.256328+0000 mon.a (mon.0) 1401 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[84576]: audit 2024-09-13T14:42:34.190550+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[84576]: audit 2024-09-13T14:42:34.190550+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:34.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[87147]: cluster 2024-09-13T14:42:33.206509+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-13T14:42:34.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[87147]: cluster 2024-09-13T14:42:33.206509+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-13T14:42:34.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[87147]: audit 2024-09-13T14:42:33.256328+0000 mon.a (mon.0) 1401 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:34.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[87147]: audit 2024-09-13T14:42:33.256328+0000 mon.a (mon.0) 1401 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:34.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[87147]: audit 2024-09-13T14:42:34.190550+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:34.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:34 smithi099 bash[87147]: audit 2024-09-13T14:42:34.190550+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:34 smithi177 bash[68594]: cluster 2024-09-13T14:42:33.206509+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-13T14:42:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:34 smithi177 bash[68594]: cluster 2024-09-13T14:42:33.206509+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-13T14:42:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:34 smithi177 bash[68594]: audit 2024-09-13T14:42:33.256328+0000 mon.a (mon.0) 1401 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:34 smithi177 bash[68594]: audit 2024-09-13T14:42:33.256328+0000 mon.a (mon.0) 1401 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:34 smithi177 bash[68594]: audit 2024-09-13T14:42:34.190550+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:34 smithi177 bash[68594]: audit 2024-09-13T14:42:34.190550+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.428 INFO:tasks.workunit.client.0.smithi099.stdout: i=6 exists [3148125~17465970,24095272~3469942,34435575~8477011,45672656~8400749,55338128~6476200,63605087~1204012,79873207~5711554,87809486~16115781,106191246~17539866,124729756~3807852,132706774~10227373,149941644~7147220,162346201~1803261,169300642~11861796,181824389~2970400,198449267~1940830,207136903~808770,219834611~3809766,231958926~3459327,240563824~9768120,253893291~1225143,261549486~6211154,273326128~4621377,312344726~6385006,322678965~1585454,326980227~9040796,342881049~4268842,353815301~1081416,366681712~3018802,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] wrote [5525499~14331290,40640954~2271632,45672656~6965434,207136903~808770,240563824~2533590,261549486~6211154,353815301~1081416,376986335~2574090,385538942~4898061] 2024-09-13T14:42:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: cluster 2024-09-13T14:42:34.208990+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: cluster 2024-09-13T14:42:34.208990+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: audit 2024-09-13T14:42:34.286408+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: audit 2024-09-13T14:42:34.286408+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: audit 2024-09-13T14:42:34.300352+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: audit 2024-09-13T14:42:34.300352+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: cluster 2024-09-13T14:42:34.367355+0000 mgr.y (mgr.26252) 483 : cluster [DBG] pgmap v839: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 KiB/s rd, 44 MiB/s wr, 283 op/s 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: cluster 2024-09-13T14:42:34.367355+0000 mgr.y (mgr.26252) 483 : cluster [DBG] pgmap v839: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 KiB/s rd, 44 MiB/s wr, 283 op/s 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: audit 2024-09-13T14:42:35.137558+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: audit 2024-09-13T14:42:35.137558+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: audit 2024-09-13T14:42:35.147399+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[84576]: audit 2024-09-13T14:42:35.147399+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: cluster 2024-09-13T14:42:34.208990+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: cluster 2024-09-13T14:42:34.208990+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: audit 2024-09-13T14:42:34.286408+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: audit 2024-09-13T14:42:34.286408+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: audit 2024-09-13T14:42:34.300352+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: audit 2024-09-13T14:42:34.300352+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: cluster 2024-09-13T14:42:34.367355+0000 mgr.y (mgr.26252) 483 : cluster [DBG] pgmap v839: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 KiB/s rd, 44 MiB/s wr, 283 op/s 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: cluster 2024-09-13T14:42:34.367355+0000 mgr.y (mgr.26252) 483 : cluster [DBG] pgmap v839: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 KiB/s rd, 44 MiB/s wr, 283 op/s 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: audit 2024-09-13T14:42:35.137558+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: audit 2024-09-13T14:42:35.137558+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: audit 2024-09-13T14:42:35.147399+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:35 smithi099 bash[87147]: audit 2024-09-13T14:42:35.147399+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: cluster 2024-09-13T14:42:34.208990+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-13T14:42:35.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: cluster 2024-09-13T14:42:34.208990+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-13T14:42:35.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: audit 2024-09-13T14:42:34.286408+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: audit 2024-09-13T14:42:34.286408+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: audit 2024-09-13T14:42:34.300352+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: audit 2024-09-13T14:42:34.300352+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: cluster 2024-09-13T14:42:34.367355+0000 mgr.y (mgr.26252) 483 : cluster [DBG] pgmap v839: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 KiB/s rd, 44 MiB/s wr, 283 op/s 2024-09-13T14:42:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: cluster 2024-09-13T14:42:34.367355+0000 mgr.y (mgr.26252) 483 : cluster [DBG] pgmap v839: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 KiB/s rd, 44 MiB/s wr, 283 op/s 2024-09-13T14:42:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: audit 2024-09-13T14:42:35.137558+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: audit 2024-09-13T14:42:35.137558+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: audit 2024-09-13T14:42:35.147399+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:35 smithi177 bash[68594]: audit 2024-09-13T14:42:35.147399+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:36.494 INFO:tasks.workunit.client.0.smithi099.stdout: i=7 exists [3148125~18106797,24095272~3469942,34435575~8477011,45672656~9209629,55338128~6476200,63605087~1204012,79873207~5711554,87809486~16115781,106191246~17539866,124729756~3807852,132706774~10227373,149941644~7147220,162346201~18816237,181824389~2970400,198449267~1940830,207136903~808770,219834611~3809766,231958926~3459327,240563824~9768120,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~8275965,322678965~1585454,326980227~9040796,342881049~4268842,353815301~1081416,366681712~3018802,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] wrote [13749627~7505295,35833134~7018729,49220521~5661764,97211542~4109022,163962679~7798051,272681980~2423441,282281434~976289,288360988~5294094,310453767~7798987] 2024-09-13T14:42:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[84576]: cluster 2024-09-13T14:42:35.316470+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-13T14:42:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[84576]: cluster 2024-09-13T14:42:35.316470+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-13T14:42:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[84576]: audit 2024-09-13T14:42:35.853055+0000 mon.a (mon.0) 1409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[84576]: audit 2024-09-13T14:42:35.853055+0000 mon.a (mon.0) 1409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[84576]: audit 2024-09-13T14:42:35.854589+0000 mon.a (mon.0) 1410 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[84576]: audit 2024-09-13T14:42:35.854589+0000 mon.a (mon.0) 1410 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[84576]: audit 2024-09-13T14:42:35.864136+0000 mon.a (mon.0) 1411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:36.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[84576]: audit 2024-09-13T14:42:35.864136+0000 mon.a (mon.0) 1411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:36.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[87147]: cluster 2024-09-13T14:42:35.316470+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-13T14:42:36.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[87147]: cluster 2024-09-13T14:42:35.316470+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-13T14:42:36.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[87147]: audit 2024-09-13T14:42:35.853055+0000 mon.a (mon.0) 1409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:36.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[87147]: audit 2024-09-13T14:42:35.853055+0000 mon.a (mon.0) 1409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:36.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[87147]: audit 2024-09-13T14:42:35.854589+0000 mon.a (mon.0) 1410 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:36.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[87147]: audit 2024-09-13T14:42:35.854589+0000 mon.a (mon.0) 1410 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:36.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[87147]: audit 2024-09-13T14:42:35.864136+0000 mon.a (mon.0) 1411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:36.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:36 smithi099 bash[87147]: audit 2024-09-13T14:42:35.864136+0000 mon.a (mon.0) 1411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:36.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:36 smithi177 bash[68594]: cluster 2024-09-13T14:42:35.316470+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-13T14:42:36.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:36 smithi177 bash[68594]: cluster 2024-09-13T14:42:35.316470+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-13T14:42:36.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:36 smithi177 bash[68594]: audit 2024-09-13T14:42:35.853055+0000 mon.a (mon.0) 1409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:36.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:36 smithi177 bash[68594]: audit 2024-09-13T14:42:35.853055+0000 mon.a (mon.0) 1409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:42:36.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:36 smithi177 bash[68594]: audit 2024-09-13T14:42:35.854589+0000 mon.a (mon.0) 1410 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:36.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:36 smithi177 bash[68594]: audit 2024-09-13T14:42:35.854589+0000 mon.a (mon.0) 1410 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:42:36.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:36 smithi177 bash[68594]: audit 2024-09-13T14:42:35.864136+0000 mon.a (mon.0) 1411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:36.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:36 smithi177 bash[68594]: audit 2024-09-13T14:42:35.864136+0000 mon.a (mon.0) 1411 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:42:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:42:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:42:37] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.51.0" 2024-09-13T14:42:37.514 INFO:tasks.workunit.client.0.smithi099.stdout: i=8 exists [160166~21094756,24095272~3469942,34435575~8477011,45672656~9209629,55338128~6476200,63605087~1204012,79873207~5711554,87809486~16115781,106191246~22346362,132706774~10227373,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3459327,236203174~14128770,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~8275965,322678965~1585454,326980227~9040796,342881049~4268842,353815301~1081416,366681712~3018802,371152217~386800,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] wrote [160166~7567734,12488826~4353156,117874817~7108375,149400161~2594737,156094323~7350508,193540217~5568432,236203174~7525653,273974549~2728127,371152217~386800] 2024-09-13T14:42:37.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:37 smithi177 bash[68594]: cluster 2024-09-13T14:42:36.320133+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-13T14:42:37.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:37 smithi177 bash[68594]: cluster 2024-09-13T14:42:36.320133+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-13T14:42:37.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:37 smithi177 bash[68594]: cluster 2024-09-13T14:42:36.367989+0000 mgr.y (mgr.26252) 484 : cluster [DBG] pgmap v842: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 168 KiB/s rd, 37 MiB/s wr, 276 op/s 2024-09-13T14:42:37.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:37 smithi177 bash[68594]: cluster 2024-09-13T14:42:36.367989+0000 mgr.y (mgr.26252) 484 : cluster [DBG] pgmap v842: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 168 KiB/s rd, 37 MiB/s wr, 276 op/s 2024-09-13T14:42:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:37 smithi099 bash[84576]: cluster 2024-09-13T14:42:36.320133+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-13T14:42:38.132 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:37 smithi099 bash[84576]: cluster 2024-09-13T14:42:36.320133+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-13T14:42:38.133 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:37 smithi099 bash[84576]: cluster 2024-09-13T14:42:36.367989+0000 mgr.y (mgr.26252) 484 : cluster [DBG] pgmap v842: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 168 KiB/s rd, 37 MiB/s wr, 276 op/s 2024-09-13T14:42:38.133 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:37 smithi099 bash[84576]: cluster 2024-09-13T14:42:36.367989+0000 mgr.y (mgr.26252) 484 : cluster [DBG] pgmap v842: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 168 KiB/s rd, 37 MiB/s wr, 276 op/s 2024-09-13T14:42:38.133 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:37 smithi099 bash[87147]: cluster 2024-09-13T14:42:36.320133+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-13T14:42:38.133 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:37 smithi099 bash[87147]: cluster 2024-09-13T14:42:36.320133+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-13T14:42:38.133 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:37 smithi099 bash[87147]: cluster 2024-09-13T14:42:36.367989+0000 mgr.y (mgr.26252) 484 : cluster [DBG] pgmap v842: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 168 KiB/s rd, 37 MiB/s wr, 276 op/s 2024-09-13T14:42:38.133 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:37 smithi099 bash[87147]: cluster 2024-09-13T14:42:36.367989+0000 mgr.y (mgr.26252) 484 : cluster [DBG] pgmap v842: 105 pgs: 105 active+clean; 251 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 168 KiB/s rd, 37 MiB/s wr, 276 op/s 2024-09-13T14:42:38.492 INFO:tasks.workunit.client.0.smithi099.stdout: i=9 exists [160166~21094756,24095272~3469942,34435575~8477011,45672656~9209629,55338128~6476200,63605087~1204012,78353736~7231025,87809486~16115781,106191246~22346362,132706774~10227373,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3459327,236203174~14128770,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~8275965,321000567~15020456,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,379850485~2642533,384511452~13135139,399873735~4931489,405003943~6147882] wrote [5830009~3359988,35554568~6496363,78353736~1561412,165534750~3793518,175109128~4337853,321000567~7975347,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:38.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:38 smithi177 bash[68594]: cluster 2024-09-13T14:42:37.344794+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-13T14:42:38.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:38 smithi177 bash[68594]: cluster 2024-09-13T14:42:37.344794+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-13T14:42:38.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:38 smithi099 bash[84576]: cluster 2024-09-13T14:42:37.344794+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-13T14:42:38.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:38 smithi099 bash[84576]: cluster 2024-09-13T14:42:37.344794+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-13T14:42:38.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:38 smithi099 bash[87147]: cluster 2024-09-13T14:42:37.344794+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-13T14:42:38.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:38 smithi099 bash[87147]: cluster 2024-09-13T14:42:37.344794+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-13T14:42:39.483 INFO:tasks.workunit.client.0.smithi099.stdout: i=10 exists [160166~21094756,24095272~3469942,34435575~8477011,45672656~9209629,55338128~6476200,63605087~1204012,78353736~9448422,87809486~16115781,106191246~22346362,132706774~10227373,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3459327,236203174~14128770,253893291~1225143,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~8275965,321000567~16003404,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,379850485~2642533,384511452~13135139,399873735~4931489,405003943~6147882] wrote [3197767~7603533,83876008~3926150,96810451~3825869,194953710~5225570,273856102~5889748,287287224~1839976,294322084~7881482,329573765~7430206,351783666~1823863] 2024-09-13T14:42:39.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:39 smithi177 bash[68594]: cluster 2024-09-13T14:42:38.352981+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-13T14:42:39.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:39 smithi177 bash[68594]: cluster 2024-09-13T14:42:38.352981+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-13T14:42:39.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:39 smithi177 bash[68594]: cluster 2024-09-13T14:42:38.368683+0000 mgr.y (mgr.26252) 485 : cluster [DBG] pgmap v845: 105 pgs: 105 active+clean; 391 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 30 MiB/s wr, 98 op/s 2024-09-13T14:42:39.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:39 smithi177 bash[68594]: cluster 2024-09-13T14:42:38.368683+0000 mgr.y (mgr.26252) 485 : cluster [DBG] pgmap v845: 105 pgs: 105 active+clean; 391 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 30 MiB/s wr, 98 op/s 2024-09-13T14:42:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:39 smithi099 bash[84576]: cluster 2024-09-13T14:42:38.352981+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-13T14:42:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:39 smithi099 bash[84576]: cluster 2024-09-13T14:42:38.352981+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-13T14:42:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:39 smithi099 bash[84576]: cluster 2024-09-13T14:42:38.368683+0000 mgr.y (mgr.26252) 485 : cluster [DBG] pgmap v845: 105 pgs: 105 active+clean; 391 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 30 MiB/s wr, 98 op/s 2024-09-13T14:42:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:39 smithi099 bash[84576]: cluster 2024-09-13T14:42:38.368683+0000 mgr.y (mgr.26252) 485 : cluster [DBG] pgmap v845: 105 pgs: 105 active+clean; 391 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 30 MiB/s wr, 98 op/s 2024-09-13T14:42:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:39 smithi099 bash[87147]: cluster 2024-09-13T14:42:38.352981+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-13T14:42:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:39 smithi099 bash[87147]: cluster 2024-09-13T14:42:38.352981+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-13T14:42:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:39 smithi099 bash[87147]: cluster 2024-09-13T14:42:38.368683+0000 mgr.y (mgr.26252) 485 : cluster [DBG] pgmap v845: 105 pgs: 105 active+clean; 391 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 30 MiB/s wr, 98 op/s 2024-09-13T14:42:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:39 smithi099 bash[87147]: cluster 2024-09-13T14:42:38.368683+0000 mgr.y (mgr.26252) 485 : cluster [DBG] pgmap v845: 105 pgs: 105 active+clean; 391 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 30 MiB/s wr, 98 op/s 2024-09-13T14:42:40.520 INFO:tasks.workunit.client.0.smithi099.stdout: i=11 exists [160166~21094756,24095272~3469942,33702107~9210479,45672656~9209629,55338128~6476200,63605087~1204012,77439947~10362211,87809486~16115781,105957581~22580027,132706774~10227373,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3459327,236203174~14128770,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~8275965,321000567~16003404,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~13135139,399873735~4931489,405003943~6147882] wrote [33702107~2581660,77439947~6884463,99104350~2644211,105957581~4841492,250358186~6972862,273089521~3348875,297590923~1695684,375818016~7111856] 2024-09-13T14:42:40.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:40 smithi177 bash[68594]: cluster 2024-09-13T14:42:39.367443+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-13T14:42:40.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:40 smithi177 bash[68594]: cluster 2024-09-13T14:42:39.367443+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-13T14:42:40.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:42:40 smithi177 bash[61439]: ts=2024-09-13T14:42:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:40.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:40 smithi099 bash[84576]: cluster 2024-09-13T14:42:39.367443+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-13T14:42:40.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:40 smithi099 bash[84576]: cluster 2024-09-13T14:42:39.367443+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-13T14:42:40.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:40 smithi099 bash[87147]: cluster 2024-09-13T14:42:39.367443+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-13T14:42:40.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:40 smithi099 bash[87147]: cluster 2024-09-13T14:42:39.367443+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-13T14:42:41.502 INFO:tasks.workunit.client.0.smithi099.stdout: i=12 exists [160166~21094756,24095272~3469942,33702107~9210479,45672656~9209629,55338128~6476200,63605087~1204012,77439947~26485320,105957581~23292082,132706774~10227373,149400161~41040903,193540217~6849880,203924372~7818857,219834611~9950791,231958926~3459327,236203174~14128770,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~8275965,321000567~16003404,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489,405003943~6147882] wrote [81203912~656330,87193675~4350095,117570724~853772,127749407~1500256,176994513~6534112,184405588~6035476,203924372~7818857,223004483~6780919,375414778~4901517,381872637~2044679] 2024-09-13T14:42:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:41 smithi099 bash[84576]: cluster 2024-09-13T14:42:40.367757+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-13T14:42:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:41 smithi099 bash[84576]: cluster 2024-09-13T14:42:40.367757+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-13T14:42:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:41 smithi099 bash[84576]: cluster 2024-09-13T14:42:40.369626+0000 mgr.y (mgr.26252) 486 : cluster [DBG] pgmap v848: 105 pgs: 105 active+clean; 474 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 187 KiB/s rd, 49 MiB/s wr, 306 op/s 2024-09-13T14:42:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:41 smithi099 bash[84576]: cluster 2024-09-13T14:42:40.369626+0000 mgr.y (mgr.26252) 486 : cluster [DBG] pgmap v848: 105 pgs: 105 active+clean; 474 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 187 KiB/s rd, 49 MiB/s wr, 306 op/s 2024-09-13T14:42:41.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:41 smithi099 bash[87147]: cluster 2024-09-13T14:42:40.367757+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-13T14:42:41.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:41 smithi099 bash[87147]: cluster 2024-09-13T14:42:40.367757+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-13T14:42:41.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:41 smithi099 bash[87147]: cluster 2024-09-13T14:42:40.369626+0000 mgr.y (mgr.26252) 486 : cluster [DBG] pgmap v848: 105 pgs: 105 active+clean; 474 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 187 KiB/s rd, 49 MiB/s wr, 306 op/s 2024-09-13T14:42:41.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:41 smithi099 bash[87147]: cluster 2024-09-13T14:42:40.369626+0000 mgr.y (mgr.26252) 486 : cluster [DBG] pgmap v848: 105 pgs: 105 active+clean; 474 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 187 KiB/s rd, 49 MiB/s wr, 306 op/s 2024-09-13T14:42:41.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:41 smithi177 bash[68594]: cluster 2024-09-13T14:42:40.367757+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-13T14:42:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:41 smithi177 bash[68594]: cluster 2024-09-13T14:42:40.367757+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-13T14:42:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:41 smithi177 bash[68594]: cluster 2024-09-13T14:42:40.369626+0000 mgr.y (mgr.26252) 486 : cluster [DBG] pgmap v848: 105 pgs: 105 active+clean; 474 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 187 KiB/s rd, 49 MiB/s wr, 306 op/s 2024-09-13T14:42:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:41 smithi177 bash[68594]: cluster 2024-09-13T14:42:40.369626+0000 mgr.y (mgr.26252) 486 : cluster [DBG] pgmap v848: 105 pgs: 105 active+clean; 474 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 187 KiB/s rd, 49 MiB/s wr, 306 op/s 2024-09-13T14:42:42.549 INFO:tasks.workunit.client.0.smithi099.stdout: i=13 exists [160166~22596730,24095272~3469942,33702107~9210479,45672656~9209629,55338128~6476200,63605087~7307385,77439947~26485320,105957581~23292082,132351677~10582470,149400161~41040903,193540217~6849880,203924372~7818857,219834611~9950791,231958926~3459327,236203174~14128770,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~8275965,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489,405003943~6147882] wrote [6870776~7158901,15268134~7488762,64579445~6333027,132351677~3360072,179806240~7716975,222718339~5152715,260623526~1785751,321943280~5551255,353246342~5833435] 2024-09-13T14:42:42.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:42 smithi099 bash[84576]: cluster 2024-09-13T14:42:41.389557+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-13T14:42:42.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:42 smithi099 bash[84576]: cluster 2024-09-13T14:42:41.389557+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-13T14:42:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:42 smithi099 bash[87147]: cluster 2024-09-13T14:42:41.389557+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-13T14:42:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:42 smithi099 bash[87147]: cluster 2024-09-13T14:42:41.389557+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-13T14:42:42.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:42 smithi177 bash[68594]: cluster 2024-09-13T14:42:41.389557+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-13T14:42:42.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:42 smithi177 bash[68594]: cluster 2024-09-13T14:42:41.389557+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-13T14:42:43.507 INFO:tasks.workunit.client.0.smithi099.stdout: i=14 exists [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,55338128~6476200,63605087~7307385,77439947~26485320,105957581~23292082,132351677~10582470,149400161~41040903,193540217~6849880,203924372~7818857,219834611~9950791,231958926~3459327,236203174~14128770,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~8275965,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~13135139,399873735~4931489,405003943~6147882] wrote [13376238~7133337,23836099~4227684,44219630~803772,196783755~39519,197409751~530452,243273429~1227157,287173785~3975876,299549539~4652901,327957029~5068897,373574339~7928023] 2024-09-13T14:42:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:43 smithi099 bash[84576]: cluster 2024-09-13T14:42:42.370634+0000 mgr.y (mgr.26252) 487 : cluster [DBG] pgmap v850: 105 pgs: 105 active+clean; 528 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 124 KiB/s rd, 32 MiB/s wr, 213 op/s 2024-09-13T14:42:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:43 smithi099 bash[84576]: cluster 2024-09-13T14:42:42.370634+0000 mgr.y (mgr.26252) 487 : cluster [DBG] pgmap v850: 105 pgs: 105 active+clean; 528 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 124 KiB/s rd, 32 MiB/s wr, 213 op/s 2024-09-13T14:42:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:43 smithi099 bash[84576]: cluster 2024-09-13T14:42:42.400605+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-13T14:42:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:43 smithi099 bash[84576]: cluster 2024-09-13T14:42:42.400605+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-13T14:42:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:43 smithi099 bash[87147]: cluster 2024-09-13T14:42:42.370634+0000 mgr.y (mgr.26252) 487 : cluster [DBG] pgmap v850: 105 pgs: 105 active+clean; 528 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 124 KiB/s rd, 32 MiB/s wr, 213 op/s 2024-09-13T14:42:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:43 smithi099 bash[87147]: cluster 2024-09-13T14:42:42.370634+0000 mgr.y (mgr.26252) 487 : cluster [DBG] pgmap v850: 105 pgs: 105 active+clean; 528 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 124 KiB/s rd, 32 MiB/s wr, 213 op/s 2024-09-13T14:42:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:43 smithi099 bash[87147]: cluster 2024-09-13T14:42:42.400605+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-13T14:42:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:43 smithi099 bash[87147]: cluster 2024-09-13T14:42:42.400605+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-13T14:42:43.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:43 smithi177 bash[68594]: cluster 2024-09-13T14:42:42.370634+0000 mgr.y (mgr.26252) 487 : cluster [DBG] pgmap v850: 105 pgs: 105 active+clean; 528 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 124 KiB/s rd, 32 MiB/s wr, 213 op/s 2024-09-13T14:42:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:43 smithi177 bash[68594]: cluster 2024-09-13T14:42:42.370634+0000 mgr.y (mgr.26252) 487 : cluster [DBG] pgmap v850: 105 pgs: 105 active+clean; 528 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 124 KiB/s rd, 32 MiB/s wr, 213 op/s 2024-09-13T14:42:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:43 smithi177 bash[68594]: cluster 2024-09-13T14:42:42.400605+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-13T14:42:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:43 smithi177 bash[68594]: cluster 2024-09-13T14:42:42.400605+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-13T14:42:44.505 INFO:tasks.workunit.client.0.smithi099.stdout: i=15 exists [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,55338128~6476200,63605087~7307385,77439947~26485320,105957581~23292082,132351677~10582470,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3459327,236203174~14128770,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~8275965,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307,405003943~6147882] wrote [9317267~7996811,90842417~1915499,208345786~7311803,270284536~3735996,284903576~1106294,349598203~3802019,364542714~535737,375206719~1944506,399621917~2387960] 2024-09-13T14:42:44.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:44 smithi099 bash[84576]: cluster 2024-09-13T14:42:43.402401+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-13T14:42:44.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:44 smithi099 bash[84576]: cluster 2024-09-13T14:42:43.402401+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-13T14:42:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:44 smithi099 bash[87147]: cluster 2024-09-13T14:42:43.402401+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-13T14:42:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:44 smithi099 bash[87147]: cluster 2024-09-13T14:42:43.402401+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-13T14:42:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:44 smithi177 bash[68594]: cluster 2024-09-13T14:42:43.402401+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-13T14:42:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:44 smithi177 bash[68594]: cluster 2024-09-13T14:42:43.402401+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-13T14:42:45.547 INFO:tasks.workunit.client.0.smithi099.stdout: i=16 exists [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,55338128~6476200,63605087~11660952,77439947~26485320,105957581~23292082,132351677~10582470,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3459327,236203174~14128770,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~8275965,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307,405003943~6147882] wrote [2195112~2825918,67408128~7857911,141404955~444745,162734548~2602407,209566024~3865350,283377402~1311205,322101591~969515,353805247~8189214,386940757~6159434] 2024-09-13T14:42:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:45 smithi099 bash[84576]: cluster 2024-09-13T14:42:44.371657+0000 mgr.y (mgr.26252) 488 : cluster [DBG] pgmap v853: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 55 MiB/s wr, 309 op/s 2024-09-13T14:42:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:45 smithi099 bash[84576]: cluster 2024-09-13T14:42:44.371657+0000 mgr.y (mgr.26252) 488 : cluster [DBG] pgmap v853: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 55 MiB/s wr, 309 op/s 2024-09-13T14:42:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:45 smithi099 bash[84576]: cluster 2024-09-13T14:42:44.425037+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-13T14:42:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:45 smithi099 bash[84576]: cluster 2024-09-13T14:42:44.425037+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-13T14:42:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:45 smithi099 bash[87147]: cluster 2024-09-13T14:42:44.371657+0000 mgr.y (mgr.26252) 488 : cluster [DBG] pgmap v853: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 55 MiB/s wr, 309 op/s 2024-09-13T14:42:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:45 smithi099 bash[87147]: cluster 2024-09-13T14:42:44.371657+0000 mgr.y (mgr.26252) 488 : cluster [DBG] pgmap v853: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 55 MiB/s wr, 309 op/s 2024-09-13T14:42:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:45 smithi099 bash[87147]: cluster 2024-09-13T14:42:44.425037+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-13T14:42:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:45 smithi099 bash[87147]: cluster 2024-09-13T14:42:44.425037+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-13T14:42:45.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:45 smithi177 bash[68594]: cluster 2024-09-13T14:42:44.371657+0000 mgr.y (mgr.26252) 488 : cluster [DBG] pgmap v853: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 55 MiB/s wr, 309 op/s 2024-09-13T14:42:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:45 smithi177 bash[68594]: cluster 2024-09-13T14:42:44.371657+0000 mgr.y (mgr.26252) 488 : cluster [DBG] pgmap v853: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 55 MiB/s wr, 309 op/s 2024-09-13T14:42:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:45 smithi177 bash[68594]: cluster 2024-09-13T14:42:44.425037+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-13T14:42:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:45 smithi177 bash[68594]: cluster 2024-09-13T14:42:44.425037+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-13T14:42:46.573 INFO:tasks.workunit.client.0.smithi099.stdout: i=17 exists [160166~22596730,23836099~4227684,33702107~21180178,55338128~6476200,63605087~11660952,77439947~26485320,105957581~23292082,132351677~13674491,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3459327,236203174~14128770,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~8275965,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~31522735,399621917~5183307,405003943~6147882] wrote [37198352~5213903,42755322~6801038,82473537~389924,139193095~6833073,178949854~4791653,221004607~1059472,273733736~3446844,321062763~2189426,380109914~10082291] 2024-09-13T14:42:46.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:46 smithi099 bash[84576]: cluster 2024-09-13T14:42:45.422557+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-13T14:42:46.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:46 smithi099 bash[84576]: cluster 2024-09-13T14:42:45.422557+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-13T14:42:46.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:46 smithi099 bash[87147]: cluster 2024-09-13T14:42:45.422557+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-13T14:42:46.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:46 smithi099 bash[87147]: cluster 2024-09-13T14:42:45.422557+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-13T14:42:46.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:46 smithi177 bash[68594]: cluster 2024-09-13T14:42:45.422557+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-13T14:42:46.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:46 smithi177 bash[68594]: cluster 2024-09-13T14:42:45.422557+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-13T14:42:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:42:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:42:47] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.51.0" 2024-09-13T14:42:47.608 INFO:tasks.workunit.client.0.smithi099.stdout: i=18 exists [160166~22596730,23836099~4227684,33702107~21180178,55338128~6476200,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3459327,236203174~14128770,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~8275965,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~6147882] wrote [7608321~5148929,57087283~3131861,129289827~5230086,206286216~7363235,264059082~6625470,330795407~13472687,345304694~6260654,366861183~3878902] 2024-09-13T14:42:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:47 smithi099 bash[84576]: cluster 2024-09-13T14:42:46.372418+0000 mgr.y (mgr.26252) 489 : cluster [DBG] pgmap v856: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 43 MiB/s wr, 302 op/s 2024-09-13T14:42:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:47 smithi099 bash[84576]: cluster 2024-09-13T14:42:46.372418+0000 mgr.y (mgr.26252) 489 : cluster [DBG] pgmap v856: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 43 MiB/s wr, 302 op/s 2024-09-13T14:42:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:47 smithi099 bash[84576]: cluster 2024-09-13T14:42:46.446311+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-13T14:42:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:47 smithi099 bash[84576]: cluster 2024-09-13T14:42:46.446311+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-13T14:42:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:47 smithi099 bash[87147]: cluster 2024-09-13T14:42:46.372418+0000 mgr.y (mgr.26252) 489 : cluster [DBG] pgmap v856: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 43 MiB/s wr, 302 op/s 2024-09-13T14:42:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:47 smithi099 bash[87147]: cluster 2024-09-13T14:42:46.372418+0000 mgr.y (mgr.26252) 489 : cluster [DBG] pgmap v856: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 43 MiB/s wr, 302 op/s 2024-09-13T14:42:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:47 smithi099 bash[87147]: cluster 2024-09-13T14:42:46.446311+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-13T14:42:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:47 smithi099 bash[87147]: cluster 2024-09-13T14:42:46.446311+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-13T14:42:47.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:47 smithi177 bash[68594]: cluster 2024-09-13T14:42:46.372418+0000 mgr.y (mgr.26252) 489 : cluster [DBG] pgmap v856: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 43 MiB/s wr, 302 op/s 2024-09-13T14:42:47.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:47 smithi177 bash[68594]: cluster 2024-09-13T14:42:46.372418+0000 mgr.y (mgr.26252) 489 : cluster [DBG] pgmap v856: 105 pgs: 105 active+clean; 701 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 189 KiB/s rd, 43 MiB/s wr, 302 op/s 2024-09-13T14:42:47.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:47 smithi177 bash[68594]: cluster 2024-09-13T14:42:46.446311+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-13T14:42:47.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:47 smithi177 bash[68594]: cluster 2024-09-13T14:42:46.446311+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-13T14:42:48.565 INFO:tasks.workunit.client.0.smithi099.stdout: i=19 exists [160166~22596730,23836099~4227684,33702107~21180178,55338128~6476200,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3459327,236203174~14128770,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] wrote [17566233~2418993,47167836~5353769,300197640~5022742,313405246~6003536,338181584~6121488,371897357~977757,386597183~6467330,399959864~2122617,410436271~3113586] 2024-09-13T14:42:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:48 smithi099 bash[84576]: cluster 2024-09-13T14:42:47.457860+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-13T14:42:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:48 smithi099 bash[84576]: cluster 2024-09-13T14:42:47.457860+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-13T14:42:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:48 smithi099 bash[84576]: audit 2024-09-13T14:42:48.256416+0000 mon.a (mon.0) 1424 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:48 smithi099 bash[84576]: audit 2024-09-13T14:42:48.256416+0000 mon.a (mon.0) 1424 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:48 smithi099 bash[87147]: cluster 2024-09-13T14:42:47.457860+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-13T14:42:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:48 smithi099 bash[87147]: cluster 2024-09-13T14:42:47.457860+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-13T14:42:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:48 smithi099 bash[87147]: audit 2024-09-13T14:42:48.256416+0000 mon.a (mon.0) 1424 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:48 smithi099 bash[87147]: audit 2024-09-13T14:42:48.256416+0000 mon.a (mon.0) 1424 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:48.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:48 smithi177 bash[68594]: cluster 2024-09-13T14:42:47.457860+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-13T14:42:48.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:48 smithi177 bash[68594]: cluster 2024-09-13T14:42:47.457860+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-13T14:42:48.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:48 smithi177 bash[68594]: audit 2024-09-13T14:42:48.256416+0000 mon.a (mon.0) 1424 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:48.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:48 smithi177 bash[68594]: audit 2024-09-13T14:42:48.256416+0000 mon.a (mon.0) 1424 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:49.493 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 1 diff [14140835~6473260,35900567~6491393,63605087~1204012,124729756~3807852,149941644~3016691,182438565~2356224,219834611~3809766,240866559~221559,326980227~7932989,406669382~2394616] 2024-09-13T14:42:49.493 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [14140835~6473260,35900567~6491393,63605087~1204012,124729756~3807852,149941644~3016691,182438565~2356224,219834611~3809766,240866559~221559,326980227~7932989,406669382~2394616] 2024-09-13T14:42:49.535 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [12582912~8031183,35900567~6491393,62914560~1894539,121634816~6902792,146800640~6157695,180355072~4439717,218103808~5540569,239075328~2012790,322961408~11951808,402653184~6410814] 2024-09-13T14:42:49.535 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 2 diff [3148125~2497343,11616606~8997489,35900567~6491393,63605087~1204012,90104149~7458565,97704586~6220681,124729756~3807852,149941644~7147220,182438565~2356224,219834611~3809766,240866559~221559,322678965~1585454,326980227~9040796,379850485~2642533,405003943~6147882] 2024-09-13T14:42:49.535 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~2497343,11616606~8997489,35900567~6491393,63605087~1204012,90104149~7458565,97704586~6220681,124729756~3807852,149941644~7147220,182438565~2356224,219834611~3809766,240866559~221559,322678965~1585454,326980227~9040796,379850485~2642533,405003943~6147882] 2024-09-13T14:42:49.553 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~5645468,8388608~12225487,35900567~6491393,62914560~1894539,88080384~9482330,97704586~6220681,121634816~6902792,146800640~10288224,180355072~4439717,218103808~5540569,239075328~2012790,318767104~17253919,377487360~5005658,402653184~8498641] 2024-09-13T14:42:49.553 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 3 diff [3148125~2497343,10538482~10075613,34435575~7956385,63605087~1204012,90104149~13821118,124729756~3807852,132706774~6904275,149941644~7147220,182438565~2356224,198449267~1940830,219834611~3809766,240866559~221559,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,274100517~3132520,322678965~1585454,326980227~9040796,379850485~2642533,405003943~6147882] 2024-09-13T14:42:49.553 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~2497343,10538482~10075613,34435575~7956385,63605087~1204012,90104149~13821118,124729756~3807852,132706774~6904275,149941644~7147220,182438565~2356224,198449267~1940830,219834611~3809766,240866559~221559,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,274100517~3132520,322678965~1585454,326980227~9040796,379850485~2642533,405003943~6147882] 2024-09-13T14:42:49.567 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~5645468,8388608~12225487,34435575~7956385,62914560~1894539,88080384~15844883,121634816~6902792,130023424~9587625,146800640~10288224,180355072~4439717,197132288~3257809,218103808~5540569,239075328~6611303,245777751~2963301,251658240~3460194,260046848~5067120,272629760~4603277,318767104~17253919,377487360~5005658,402653184~8498641] 2024-09-13T14:42:49.567 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 4 diff [3148125~2497343,9390236~11223859,34435575~7956385,63605087~1204012,90104149~13821118,106191246~11701461,124729756~3807852,132706774~6904275,149941644~7147220,169300642~11861796,182438565~2356224,198449267~1940830,219834611~3809766,231958926~3416707,240866559~221559,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,322678965~1585454,326980227~9040796,379850485~2642533,389565511~8081080,405003943~6147882] 2024-09-13T14:42:49.567 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~2497343,9390236~11223859,34435575~7956385,63605087~1204012,90104149~13821118,106191246~11701461,124729756~3807852,132706774~6904275,149941644~7147220,169300642~11861796,182438565~2356224,198449267~1940830,219834611~3809766,231958926~3416707,240866559~221559,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,322678965~1585454,326980227~9040796,379850485~2642533,389565511~8081080,405003943~6147882] 2024-09-13T14:42:49.579 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~5645468,8388608~12225487,34435575~7956385,62914560~1894539,88080384~15844883,104857600~13035107,121634816~6902792,130023424~9587625,146800640~10288224,167772160~17022629,197132288~3257809,218103808~5540569,231958926~3416707,239075328~6611303,245777751~2963301,251658240~3460194,260046848~5067120,272629760~5317745,318767104~17253919,377487360~5005658,385875968~11770623,402653184~8498641] 2024-09-13T14:42:49.579 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 5 diff [3148125~2497343,9390236~11223859,34435575~7956385,50256085~3817320,63605087~1204012,79983070~11130,87809486~16115781,106191246~17539866,124729756~3807852,132706774~6904275,149941644~7147220,162346201~1803261,169300642~11861796,181824389~2970400,198449267~1940830,219834611~3809766,231958926~3416707,240866559~221559,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,322678965~1585454,326980227~9040796,342881049~4268842,379850485~2642533,389565511~8081080,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.580 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~2497343,9390236~11223859,34435575~7956385,50256085~3817320,63605087~1204012,79983070~11130,87809486~16115781,106191246~17539866,124729756~3807852,132706774~6904275,149941644~7147220,162346201~1803261,169300642~11861796,181824389~2970400,198449267~1940830,219834611~3809766,231958926~3416707,240866559~221559,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,322678965~1585454,326980227~9040796,342881049~4268842,379850485~2642533,389565511~8081080,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.591 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~5645468,8388608~12225487,34435575~7956385,46137344~7936061,62914560~1894539,79983070~11130,85584761~18340506,104857600~23680008,130023424~9587625,146800640~10288224,159383552~4765910,167772160~17022629,197132288~3257809,218103808~5540569,231958926~3416707,239075328~6611303,245777751~2963301,251658240~3460194,260046848~5067120,272629760~5317745,318767104~17253919,339738624~7411267,377487360~5005658,385875968~11770623,398458880~12692945] 2024-09-13T14:42:49.591 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 6 diff [3148125~17465970,34435575~8477011,45672656~8400749,63605087~1204012,79983070~11130,87809486~16115781,106191246~17539866,124729756~3807852,132706774~6904275,149941644~7147220,162346201~1803261,169300642~11861796,181824389~2970400,198449267~1940830,207136903~808770,219834611~3809766,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,273326128~4621377,322678965~1585454,326980227~9040796,342881049~4268842,353815301~1081416,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.591 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~17465970,34435575~8477011,45672656~8400749,63605087~1204012,79983070~11130,87809486~16115781,106191246~17539866,124729756~3807852,132706774~6904275,149941644~7147220,162346201~1803261,169300642~11861796,181824389~2970400,198449267~1940830,207136903~808770,219834611~3809766,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,273326128~4621377,322678965~1585454,326980227~9040796,342881049~4268842,353815301~1081416,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.602 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~20614095,34435575~19637830,62914560~1894539,79983070~11130,85584761~18340506,104857600~23680008,130023424~9587625,146800640~10288224,159383552~4765910,167772160~17022629,197132288~3257809,205520896~2424777,218103808~5540569,231958926~3416707,239075328~6611303,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,318767104~17253919,339738624~7411267,352321536~2575181,373293056~24353535,398458880~12692945] 2024-09-13T14:42:49.602 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 7 diff [3148125~18106797,34435575~8477011,45672656~9209629,63605087~1204012,79983070~11130,87809486~16115781,106191246~17539866,124729756~3807852,132706774~6904275,149941644~7147220,162346201~18816237,181824389~2970400,198449267~1940830,207136903~808770,219834611~3809766,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,322678965~1585454,326980227~9040796,342881049~4268842,353815301~1081416,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.603 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~18106797,34435575~8477011,45672656~9209629,63605087~1204012,79983070~11130,87809486~16115781,106191246~17539866,124729756~3807852,132706774~6904275,149941644~7147220,162346201~18816237,181824389~2970400,198449267~1940830,207136903~808770,219834611~3809766,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,322678965~1585454,326980227~9040796,342881049~4268842,353815301~1081416,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.613 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,34435575~20446710,62914560~1894539,79983070~11130,85584761~18340506,104857600~23680008,130023424~9587625,146800640~10288224,159383552~25411237,197132288~3257809,205520896~2424777,218103808~5540569,231958926~3416707,239075328~6611303,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,281018368~2239355,285212672~8442410,310453767~7798987,318767104~17253919,339738624~7411267,352321536~2575181,373293056~24353535,398458880~12692945] 2024-09-13T14:42:49.613 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 8 diff [160166~21094756,34435575~8477011,45672656~9209629,63605087~1204012,79983070~11130,87809486~16115781,106191246~22346362,132706774~6904275,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,322678965~1585454,326980227~9040796,342881049~4268842,353815301~1081416,371152217~386800,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.613 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,34435575~8477011,45672656~9209629,63605087~1204012,79983070~11130,87809486~16115781,106191246~22346362,132706774~6904275,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,322678965~1585454,326980227~9040796,342881049~4268842,353815301~1081416,371152217~386800,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.623 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,34435575~20446710,62914560~1894539,79983070~11130,85584761~18340506,104857600~23680008,130023424~9587625,146800640~37994149,192937984~7452113,205520896~2424777,218103808~5540569,231958926~3416707,235418253~10268378,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,281018368~2239355,285212672~8442410,310453767~7798987,318767104~17253919,339738624~7411267,352321536~2575181,369700514~1838503,373293056~24353535,398458880~12692945] 2024-09-13T14:42:49.623 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 9 diff [160166~21094756,34435575~8477011,45672656~9209629,63605087~1204012,78353736~1561412,79983070~11130,87809486~16115781,106191246~22346362,132706774~6904275,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,321000567~15020456,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,379850485~2642533,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.623 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,34435575~8477011,45672656~9209629,63605087~1204012,78353736~1561412,79983070~11130,87809486~16115781,106191246~22346362,132706774~6904275,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,321000567~15020456,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,379850485~2642533,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.632 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,34435575~20446710,62914560~1894539,75497472~4417676,79983070~11130,85584761~18340506,104857600~23680008,130023424~9587625,146800640~37994149,192937984~7452113,205520896~2424777,218103808~5540569,231958926~3416707,235418253~10268378,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,281018368~2239355,285212672~8442410,310453767~7798987,318767104~17253919,339738624~7411267,352321536~10810425,366123856~31522735,398458880~12692945] 2024-09-13T14:42:49.632 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 10 diff [160166~21094756,34435575~8477011,45672656~9209629,63605087~1204012,78353736~1561412,79983070~11130,83876008~3926150,87809486~16115781,106191246~22346362,132706774~6904275,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~16003404,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,379850485~2642533,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.632 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,34435575~8477011,45672656~9209629,63605087~1204012,78353736~1561412,79983070~11130,83876008~3926150,87809486~16115781,106191246~22346362,132706774~6904275,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~16003404,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,379850485~2642533,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.641 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,34435575~20446710,62914560~1894539,75497472~4417676,79983070~11130,83876008~20049259,104857600~23680008,130023424~9587625,146800640~37994149,192937984~7452113,205520896~2424777,218103808~5540569,231958926~3416707,235418253~10268378,245777751~2963301,251658240~3460194,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,318767104~18236867,339738624~7411267,348127232~15004729,366123856~31522735,398458880~12692945] 2024-09-13T14:42:49.641 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 11 diff [160166~21094756,33702107~9210479,45672656~9209629,63605087~1204012,77439947~10362211,87809486~16115781,105957581~22580027,132706774~6904275,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~16003404,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.642 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,63605087~1204012,77439947~10362211,87809486~16115781,105957581~22580027,132706774~6904275,149400161~31762277,181824389~2970400,193540217~6849880,207136903~808770,219834611~3809766,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~16003404,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.650 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,33702107~21180178,62914560~1894539,75497472~28427795,104857600~23680008,130023424~9587625,146800640~37994149,192937984~7452113,205520896~2424777,218103808~5540569,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,318767104~18236867,339738624~7411267,348127232~15004729,366123856~31522735,398458880~12692945] 2024-09-13T14:42:49.650 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 12 diff [160166~21094756,33702107~9210479,45672656~9209629,63605087~1204012,77439947~26485320,105957581~23292082,132706774~6904275,149400161~41040903,193540217~6849880,203924372~7818857,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~16003404,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.650 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,63605087~1204012,77439947~26485320,105957581~23292082,132706774~6904275,149400161~41040903,193540217~6849880,203924372~7818857,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~16003404,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.657 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,33702107~21180178,62914560~1894539,75497472~28427795,104857600~24392063,130023424~9587625,146800640~43640424,192937984~7452113,201326592~10416637,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,318767104~18236867,339738624~7411267,348127232~15004729,366123856~31522735,398458880~12692945] 2024-09-13T14:42:49.658 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 13 diff [160166~22596730,33702107~9210479,45672656~9209629,63605087~7307385,77439947~26485320,105957581~23292082,132351677~7259372,149400161~41040903,193540217~6849880,203924372~7818857,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.658 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,33702107~9210479,45672656~9209629,63605087~7307385,77439947~26485320,105957581~23292082,132351677~7259372,149400161~41040903,193540217~6849880,203924372~7818857,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.665 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,33702107~21180178,62914560~7997912,75497472~28427795,104857600~24392063,130023424~9587625,146800640~43640424,192937984~7452113,201326592~10416637,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,318767104~18236867,339738624~7411267,348127232~15004729,366123856~31522735,398458880~12692945] 2024-09-13T14:42:49.665 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 14 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,63605087~7307385,77439947~26485320,105957581~23292082,132351677~7259372,149400161~41040903,193540217~6849880,203924372~7818857,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.665 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,63605087~7307385,77439947~26485320,105957581~23292082,132351677~7259372,149400161~41040903,193540217~6849880,203924372~7818857,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.673 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,62914560~7997912,75497472~28427795,104857600~24392063,130023424~9587625,146800640~43640424,192937984~7452113,201326592~10416637,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~18989768,310453767~7798987,318767104~18236867,339738624~7411267,348127232~15004729,366123856~31522735,398458880~12692945] 2024-09-13T14:42:49.673 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 15 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,63605087~7307385,77439947~26485320,105957581~23292082,132351677~7259372,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.673 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,63605087~7307385,77439947~26485320,105957581~23292082,132351677~7259372,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.681 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,62914560~7997912,75497472~28427795,104857600~24392063,130023424~9587625,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,268435456~11310394,281018368~23184072,310453767~7798987,318767104~18236867,339738624~7411267,348127232~16951219,366123856~31522735,398458880~12692945] 2024-09-13T14:42:49.681 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 16 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,63605087~11660952,77439947~26485320,105957581~23292082,132351677~7259372,141404955~444745,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.681 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,63605087~11660952,77439947~26485320,105957581~23292082,132351677~7259372,141404955~444745,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.690 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,62914560~12351479,75497472~28427795,104857600~24392063,130023424~9587625,141404955~444745,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,268435456~11310394,281018368~23184072,310453767~7798987,318767104~18236867,339738624~7411267,348127232~16951219,366123856~31522735,398458880~12692945] 2024-09-13T14:42:49.690 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 17 diff [160166~22596730,23836099~4227684,33702107~21180178,63605087~11660952,77439947~26485320,105957581~23292082,132351677~13674491,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~31522735,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.690 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,63605087~11660952,77439947~26485320,105957581~23292082,132351677~13674491,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~31522735,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.698 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,62914560~12351479,75497472~28427795,104857600~24392063,130023424~16002744,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,268435456~11310394,281018368~23184072,310453767~7798987,318767104~18236867,339738624~7411267,348127232~16951219,366123856~31522735,398458880~12692945] 2024-09-13T14:42:49.699 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 18 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.699 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.707 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~23184072,310453767~7798987,318767104~46311347,366123856~31522735,398458880~12692945] 2024-09-13T14:42:49.707 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 19 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:49.707 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:49.715 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:42:49.715 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 2 diff [3148125~2497343,11616606~4641737,90104149~7458565,97704586~6220681,151511398~5577466,322678965~1585454,329696827~6324196,379850485~2642533,405003943~6147882] 2024-09-13T14:42:49.715 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~2497343,11616606~4641737,90104149~7458565,97704586~6220681,151511398~5577466,322678965~1585454,329696827~6324196,379850485~2642533,405003943~6147882] 2024-09-13T14:42:49.726 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~5645468,8388608~7869735,88080384~9482330,97704586~6220681,151511398~5577466,318767104~5497315,329696827~6324196,377487360~5005658,405003943~6147882] 2024-09-13T14:42:49.726 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 3 diff [3148125~2497343,10538482~7692670,34435575~1894216,90104149~13821118,132706774~6904275,151511398~5577466,198449267~1940830,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,274100517~3132520,322678965~1585454,329696827~6324196,379850485~2642533,405003943~6147882] 2024-09-13T14:42:49.726 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~2497343,10538482~7692670,34435575~1894216,90104149~13821118,132706774~6904275,151511398~5577466,198449267~1940830,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,274100517~3132520,322678965~1585454,329696827~6324196,379850485~2642533,405003943~6147882] 2024-09-13T14:42:49.733 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~5645468,8388608~9842544,34435575~1894216,88080384~15844883,130023424~9587625,151511398~5577466,197132288~3257809,241088118~4598513,245777751~2963301,251658240~3460194,260046848~5067120,272629760~4603277,318767104~5497315,329696827~6324196,377487360~5005658,405003943~6147882] 2024-09-13T14:42:49.733 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 4 diff [3148125~2497343,9390236~8840916,34435575~1894216,90104149~13821118,106191246~11701461,132706774~6904275,151511398~5577466,169300642~11861796,198449267~1940830,231958926~3416707,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,322678965~1585454,329696827~6324196,379850485~2642533,389565511~8081080,405003943~6147882] 2024-09-13T14:42:49.733 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~2497343,9390236~8840916,34435575~1894216,90104149~13821118,106191246~11701461,132706774~6904275,151511398~5577466,169300642~11861796,198449267~1940830,231958926~3416707,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,322678965~1585454,329696827~6324196,379850485~2642533,389565511~8081080,405003943~6147882] 2024-09-13T14:42:49.740 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~5645468,8388608~9842544,34435575~1894216,88080384~15844883,104857600~13035107,130023424~9587625,151511398~5577466,167772160~13390278,197132288~3257809,231958926~3416707,241088118~4598513,245777751~2963301,251658240~3460194,260046848~5067120,272629760~5317745,318767104~5497315,329696827~6324196,377487360~5005658,385875968~11770623,405003943~6147882] 2024-09-13T14:42:49.740 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 5 diff [3148125~2497343,9390236~8840916,34435575~1894216,37838725~3913129,50256085~3817320,79983070~11130,87809486~16115781,106191246~17539866,132706774~6904275,151511398~5577466,162346201~1803261,169300642~11861796,181824389~1807088,198449267~1940830,231958926~3416707,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,322678965~1585454,329696827~6324196,342881049~4268842,379850485~2642533,389565511~8081080,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.740 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~2497343,9390236~8840916,34435575~1894216,37838725~3913129,50256085~3817320,79983070~11130,87809486~16115781,106191246~17539866,132706774~6904275,151511398~5577466,162346201~1803261,169300642~11861796,181824389~1807088,198449267~1940830,231958926~3416707,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,322678965~1585454,329696827~6324196,342881049~4268842,379850485~2642533,389565511~8081080,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.748 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~5645468,8388608~9842544,34435575~1894216,37838725~3913129,46137344~7936061,79983070~11130,85584761~18340506,104857600~18873512,130023424~9587625,151511398~5577466,159383552~4765910,167772160~13390278,181824389~1807088,197132288~3257809,231958926~3416707,241088118~4598513,245777751~2963301,251658240~3460194,260046848~5067120,272629760~5317745,318767104~5497315,329696827~6324196,339738624~7411267,377487360~5005658,385875968~11770623,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.748 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 6 diff [3148125~16708664,34435575~1894216,37838725~5073861,45672656~8400749,79983070~11130,87809486~16115781,106191246~17539866,132706774~6904275,151511398~5577466,162346201~1803261,169300642~11861796,181824389~1807088,198449267~1940830,207136903~808770,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,273326128~4621377,322678965~1585454,329696827~6324196,342881049~4268842,353815301~1081416,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.748 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~16708664,34435575~1894216,37838725~5073861,45672656~8400749,79983070~11130,87809486~16115781,106191246~17539866,132706774~6904275,151511398~5577466,162346201~1803261,169300642~11861796,181824389~1807088,198449267~1940830,207136903~808770,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,273326128~4621377,322678965~1585454,329696827~6324196,342881049~4268842,353815301~1081416,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.756 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~19856789,34435575~1894216,37838725~16234680,79983070~11130,85584761~18340506,104857600~18873512,130023424~9587625,151511398~5577466,159383552~4765910,167772160~13390278,181824389~1807088,197132288~3257809,205520896~2424777,231958926~3416707,240563824~5122807,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,318767104~5497315,329696827~6324196,339738624~7411267,352321536~2575181,373293056~24353535,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.756 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 7 diff [3148125~18106797,34435575~8477011,45672656~9209629,79983070~11130,87809486~16115781,106191246~17539866,132706774~6904275,151511398~5577466,162346201~18816237,181824389~1807088,198449267~1940830,207136903~808770,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,322678965~1585454,329696827~6324196,342881049~4268842,353815301~1081416,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.756 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3148125~18106797,34435575~8477011,45672656~9209629,79983070~11130,87809486~16115781,106191246~17539866,132706774~6904275,151511398~5577466,162346201~18816237,181824389~1807088,198449267~1940830,207136903~808770,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,322678965~1585454,329696827~6324196,342881049~4268842,353815301~1081416,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.764 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,34435575~20446710,79983070~11130,85584761~18340506,104857600~18873512,130023424~9587625,151511398~5577466,159383552~21778886,181824389~1807088,197132288~3257809,205520896~2424777,231958926~3416707,240563824~5122807,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,281018368~2239355,285212672~8442410,310453767~7798987,318767104~5497315,329696827~6324196,339738624~7411267,352321536~2575181,373293056~24353535,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.764 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 8 diff [160166~21094756,34435575~8477011,45672656~9209629,79983070~11130,87809486~16115781,106191246~18791946,132706774~6904275,149400161~31762277,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,322678965~1585454,329696827~6324196,342881049~4268842,353815301~1081416,371152217~386800,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.764 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,34435575~8477011,45672656~9209629,79983070~11130,87809486~16115781,106191246~18791946,132706774~6904275,149400161~31762277,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,322678965~1585454,329696827~6324196,342881049~4268842,353815301~1081416,371152217~386800,376986335~2574090,379850485~2642533,385538942~12107649,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.772 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,34435575~20446710,79983070~11130,85584761~18340506,104857600~20125592,130023424~9587625,149400161~31762277,181824389~1807088,192937984~7452113,205520896~2424777,231958926~3416707,235418253~10268378,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,281018368~2239355,285212672~8442410,310453767~7798987,318767104~5497315,329696827~6324196,339738624~7411267,352321536~2575181,369700514~1838503,373293056~24353535,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.772 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 9 diff [160166~21094756,34435575~8477011,45672656~9209629,78353736~1561412,79983070~11130,87809486~16115781,106191246~18791946,132706774~6904275,149400161~31762277,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,329696827~6324196,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,379850485~2642533,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.772 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,34435575~8477011,45672656~9209629,78353736~1561412,79983070~11130,87809486~16115781,106191246~18791946,132706774~6904275,149400161~31762277,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,329696827~6324196,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,379850485~2642533,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.780 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,34435575~20446710,75497472~4417676,79983070~11130,85584761~18340506,104857600~20125592,130023424~9587625,149400161~31762277,181824389~1807088,192937984~7452113,205520896~2424777,231958926~3416707,235418253~10268378,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,281018368~2239355,285212672~8442410,310453767~7798987,318767104~10208810,329696827~6324196,339738624~7411267,352321536~10810425,366123856~31522735,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.780 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 10 diff [160166~21094756,34435575~8477011,45672656~9209629,78353736~1561412,79983070~11130,83876008~3926150,87809486~16115781,106191246~18791946,132706774~6904275,149400161~31762277,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,379850485~2642533,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.780 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,34435575~8477011,45672656~9209629,78353736~1561412,79983070~11130,83876008~3926150,87809486~16115781,106191246~18791946,132706774~6904275,149400161~31762277,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,379850485~2642533,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.788 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,34435575~20446710,75497472~4417676,79983070~11130,83876008~20049259,104857600~20125592,130023424~9587625,149400161~31762277,181824389~1807088,192937984~7452113,205520896~2424777,231958926~3416707,235418253~10268378,245777751~2963301,251658240~3460194,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,318767104~10208810,329573765~7430206,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.788 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 11 diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~10362211,87809486~16115781,105957581~19025611,132706774~6904275,149400161~31762277,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.789 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~10362211,87809486~16115781,105957581~19025611,132706774~6904275,149400161~31762277,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.800 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,33702107~21180178,75497472~28427795,104857600~20125592,130023424~9587625,149400161~31762277,181824389~1807088,192937984~7452113,205520896~2424777,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,318767104~10208810,329573765~7430206,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.800 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 12 diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~26485320,105957581~19025611,127749407~1500256,132706774~6904275,149400161~34231316,184405588~6035476,193540217~6849880,203924372~7818857,223004483~6780919,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.800 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~26485320,105957581~19025611,127749407~1500256,132706774~6904275,149400161~34231316,184405588~6035476,193540217~6849880,203924372~7818857,223004483~6780919,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.807 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~21254922,33702107~21180178,75497472~28427795,104857600~20125592,127749407~1500256,130023424~9587625,149400161~34231316,184405588~6035476,192937984~7452113,201326592~10416637,223004483~6780919,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,318767104~10208810,329573765~7430206,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.807 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 13 diff [160166~22596730,33702107~9210479,45672656~9209629,64579445~6333027,77439947~26485320,105957581~19025611,127749407~1500256,132351677~7259372,149400161~41040903,193540217~6849880,203924372~7818857,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.807 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,33702107~9210479,45672656~9209629,64579445~6333027,77439947~26485320,105957581~19025611,127749407~1500256,132351677~7259372,149400161~41040903,193540217~6849880,203924372~7818857,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.815 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,33702107~21180178,64579445~6333027,75497472~28427795,104857600~20125592,127749407~1500256,130023424~9587625,149400161~41040903,192937984~7452113,201326592~10416637,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,318767104~10208810,329573765~7430206,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.816 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 14 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~26485320,105957581~19025611,127749407~1500256,132351677~7259372,149400161~41040903,193540217~6849880,203924372~7818857,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.816 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~26485320,105957581~19025611,127749407~1500256,132351677~7259372,149400161~41040903,193540217~6849880,203924372~7818857,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~13135139,399873735~4931489,405003943~6147882] 2024-09-13T14:42:49.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:49 smithi099 bash[87147]: cluster 2024-09-13T14:42:48.373323+0000 mgr.y (mgr.26252) 490 : cluster [DBG] pgmap v859: 105 pgs: 105 active+clean; 806 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 26 MiB/s wr, 177 op/s 2024-09-13T14:42:49.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:49 smithi099 bash[87147]: cluster 2024-09-13T14:42:48.373323+0000 mgr.y (mgr.26252) 490 : cluster [DBG] pgmap v859: 105 pgs: 105 active+clean; 806 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 26 MiB/s wr, 177 op/s 2024-09-13T14:42:49.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:49 smithi099 bash[87147]: cluster 2024-09-13T14:42:48.469731+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-13T14:42:49.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:49 smithi099 bash[87147]: cluster 2024-09-13T14:42:48.469731+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-13T14:42:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:49 smithi099 bash[84576]: cluster 2024-09-13T14:42:48.373323+0000 mgr.y (mgr.26252) 490 : cluster [DBG] pgmap v859: 105 pgs: 105 active+clean; 806 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 26 MiB/s wr, 177 op/s 2024-09-13T14:42:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:49 smithi099 bash[84576]: cluster 2024-09-13T14:42:48.373323+0000 mgr.y (mgr.26252) 490 : cluster [DBG] pgmap v859: 105 pgs: 105 active+clean; 806 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 26 MiB/s wr, 177 op/s 2024-09-13T14:42:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:49 smithi099 bash[84576]: cluster 2024-09-13T14:42:48.469731+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-13T14:42:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:49 smithi099 bash[84576]: cluster 2024-09-13T14:42:48.469731+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-13T14:42:49.823 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,64579445~6333027,75497472~28427795,104857600~20125592,127749407~1500256,130023424~9587625,149400161~41040903,192937984~7452113,201326592~10416637,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~18989768,310453767~7798987,318767104~18236867,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.823 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 15 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~26485320,105957581~19025611,127749407~1500256,132351677~7259372,149400161~41040903,193540217~6849880,203924372~11733217,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.823 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~26485320,105957581~19025611,127749407~1500256,132351677~7259372,149400161~41040903,193540217~6849880,203924372~11733217,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.831 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,64579445~6333027,75497472~28427795,104857600~20125592,127749407~1500256,130023424~9587625,149400161~41040903,192937984~7452113,201326592~14330997,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,268435456~11310394,281018368~23184072,310453767~7798987,318767104~18236867,339738624~7411267,348127232~16951219,366123856~31522735,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.831 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 16 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,132351677~7259372,141404955~444745,149400161~41040903,193540217~6849880,203924372~11733217,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.831 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,132351677~7259372,141404955~444745,149400161~41040903,193540217~6849880,203924372~11733217,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.838 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,64579445~10686594,75497472~28427795,104857600~20125592,127749407~1500256,130023424~9587625,141404955~444745,149400161~41040903,192937984~7452113,201326592~14330997,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,268435456~11310394,281018368~23184072,310453767~7798987,318767104~18236867,339738624~7411267,348127232~16951219,366123856~31522735,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.838 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 17 diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,132351677~13674491,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~31522735,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.838 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,132351677~13674491,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~31522735,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.847 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,64579445~10686594,75497472~28427795,104857600~20125592,127749407~1500256,130023424~16002744,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,268435456~11310394,281018368~23184072,310453767~7798987,318767104~18236867,339738624~7411267,348127232~16951219,366123856~31522735,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.847 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 18 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.847 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~6147882] 2024-09-13T14:42:49.855 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~23184072,310453767~7798987,318767104~46311347,366123856~31522735,398458880~6346344,405003943~6147882] 2024-09-13T14:42:49.855 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 19 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:49.855 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:49.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:49 smithi177 bash[68594]: cluster 2024-09-13T14:42:48.373323+0000 mgr.y (mgr.26252) 490 : cluster [DBG] pgmap v859: 105 pgs: 105 active+clean; 806 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 26 MiB/s wr, 177 op/s 2024-09-13T14:42:49.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:49 smithi177 bash[68594]: cluster 2024-09-13T14:42:48.373323+0000 mgr.y (mgr.26252) 490 : cluster [DBG] pgmap v859: 105 pgs: 105 active+clean; 806 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 26 MiB/s wr, 177 op/s 2024-09-13T14:42:49.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:49 smithi177 bash[68594]: cluster 2024-09-13T14:42:48.469731+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-13T14:42:49.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:49 smithi177 bash[68594]: cluster 2024-09-13T14:42:48.469731+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-13T14:42:49.863 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:42:49.863 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 3 diff [10538482~7692670,34435575~1894216,97410719~3652484,132706774~6904275,198449267~1940830,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,274100517~3132520] 2024-09-13T14:42:49.863 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [10538482~7692670,34435575~1894216,97410719~3652484,132706774~6904275,198449267~1940830,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,274100517~3132520] 2024-09-13T14:42:49.870 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [10538482~7692670,34435575~1894216,97410719~3652484,130023424~9587625,197132288~3257809,241088118~4598513,245777751~2963301,251658240~3460194,260046848~5067120,272629760~4603277] 2024-09-13T14:42:49.870 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 4 diff [9390236~8840916,34435575~1894216,91011347~3717169,97410719~3652484,106191246~11701461,132706774~6904275,152231961~2578327,169300642~11861796,198449267~1940830,231958926~3416707,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,389565511~8081080] 2024-09-13T14:42:49.870 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [9390236~8840916,34435575~1894216,91011347~3717169,97410719~3652484,106191246~11701461,132706774~6904275,152231961~2578327,169300642~11861796,198449267~1940830,231958926~3416707,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,389565511~8081080] 2024-09-13T14:42:49.879 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [9390236~8840916,34435575~1894216,91011347~3717169,97410719~3652484,104857600~13035107,130023424~9587625,152231961~2578327,167772160~13390278,197132288~3257809,231958926~3416707,241088118~4598513,245777751~2963301,251658240~3460194,260046848~5067120,272629760~5317745,385875968~11770623] 2024-09-13T14:42:49.879 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 5 diff [9390236~8840916,34435575~1894216,37838725~3913129,50256085~3817320,79983070~11130,87809486~7971187,97410719~3652484,106191246~17539866,132706774~6904275,152231961~2578327,162346201~1803261,169300642~11861796,181824389~1807088,198449267~1940830,231958926~3416707,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,342881049~4268842,389565511~8081080,399873735~4931489] 2024-09-13T14:42:49.879 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [9390236~8840916,34435575~1894216,37838725~3913129,50256085~3817320,79983070~11130,87809486~7971187,97410719~3652484,106191246~17539866,132706774~6904275,152231961~2578327,162346201~1803261,169300642~11861796,181824389~1807088,198449267~1940830,231958926~3416707,242466485~3220146,245777751~2963301,253893291~1225143,263022958~2091010,273326128~4621377,342881049~4268842,389565511~8081080,399873735~4931489] 2024-09-13T14:42:49.886 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [9390236~8840916,34435575~1894216,37838725~3913129,46137344~7936061,79983070~11130,85584761~10195912,97410719~3652484,104857600~18873512,130023424~9587625,152231961~2578327,159383552~4765910,167772160~13390278,181824389~1807088,197132288~3257809,231958926~3416707,241088118~4598513,245777751~2963301,251658240~3460194,260046848~5067120,272629760~5317745,339738624~7411267,385875968~11770623,398458880~6346344] 2024-09-13T14:42:49.886 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 6 diff [5525499~14331290,34435575~1894216,37838725~5073861,45672656~8400749,79983070~11130,87809486~7971187,97410719~3652484,106191246~17539866,132706774~6904275,152231961~2578327,162346201~1803261,169300642~11861796,181824389~1807088,198449267~1940830,207136903~808770,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,273326128~4621377,342881049~4268842,353815301~1081416,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:49.886 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [5525499~14331290,34435575~1894216,37838725~5073861,45672656~8400749,79983070~11130,87809486~7971187,97410719~3652484,106191246~17539866,132706774~6904275,152231961~2578327,162346201~1803261,169300642~11861796,181824389~1807088,198449267~1940830,207136903~808770,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,273326128~4621377,342881049~4268842,353815301~1081416,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:49.893 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [5525499~14331290,34435575~1894216,37838725~16234680,79983070~11130,85584761~10195912,97410719~3652484,104857600~18873512,130023424~9587625,152231961~2578327,159383552~4765910,167772160~13390278,181824389~1807088,197132288~3257809,205520896~2424777,231958926~3416707,240563824~5122807,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,339738624~7411267,352321536~2575181,373293056~6267369,382493018~15153573,398458880~6346344] 2024-09-13T14:42:49.894 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 7 diff [5525499~15729423,34435575~8477011,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,106191246~17539866,132706774~6904275,152231961~2578327,162346201~18816237,181824389~1807088,198449267~1940830,207136903~808770,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:49.894 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [5525499~15729423,34435575~8477011,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,106191246~17539866,132706774~6904275,152231961~2578327,162346201~18816237,181824389~1807088,198449267~1940830,207136903~808770,231958926~3416707,240563824~5122807,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:49.903 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [5525499~15729423,34435575~20446710,79983070~11130,85584761~10195912,97211542~4109022,104857600~18873512,130023424~9587625,152231961~2578327,159383552~21778886,181824389~1807088,197132288~3257809,205520896~2424777,231958926~3416707,240563824~5122807,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,281018368~2239355,285212672~8442410,310453767~7798987,339738624~7411267,352321536~2575181,373293056~6267369,382493018~15153573,398458880~6346344] 2024-09-13T14:42:49.903 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 8 diff [160166~21094756,34435575~8477011,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,106191246~18791946,132706774~6904275,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,371152217~386800,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:49.903 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,34435575~8477011,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,106191246~18791946,132706774~6904275,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,371152217~386800,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:49.911 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,34435575~20446710,79983070~11130,85584761~10195912,97211542~4109022,104857600~20125592,130023424~9587625,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,192937984~7452113,205520896~2424777,231958926~3416707,235418253~10268378,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,281018368~2239355,285212672~8442410,310453767~7798987,339738624~7411267,352321536~2575181,369700514~1838503,373293056~6267369,382493018~15153573,398458880~6346344] 2024-09-13T14:42:49.912 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 9 diff [160166~21094756,34435575~8477011,45672656~9209629,78353736~1561412,79983070~11130,87809486~7971187,97211542~4109022,106191246~18791946,132706774~6904275,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.912 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,34435575~8477011,45672656~9209629,78353736~1561412,79983070~11130,87809486~7971187,97211542~4109022,106191246~18791946,132706774~6904275,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.919 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,34435575~20446710,75497472~4417676,79983070~11130,85584761~10195912,97211542~4109022,104857600~20125592,130023424~9587625,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,192937984~7452113,205520896~2424777,231958926~3416707,235418253~10268378,245777751~2963301,251658240~3460194,260046848~7713792,272629760~5317745,281018368~2239355,285212672~8442410,310453767~7798987,321000567~7975347,339738624~7411267,352321536~10810425,366123856~13436569,382493018~15153573,398458880~6346344] 2024-09-13T14:42:49.920 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 10 diff [160166~21094756,34435575~8477011,45672656~9209629,78353736~1561412,79983070~11130,83876008~3926150,87809486~7971187,96810451~4510113,106191246~18791946,132706774~6904275,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.920 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,34435575~8477011,45672656~9209629,78353736~1561412,79983070~11130,83876008~3926150,87809486~7971187,96810451~4510113,106191246~18791946,132706774~6904275,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,253893291~1225143,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.927 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,34435575~20446710,75497472~4417676,79983070~11130,83876008~11904665,96810451~4510113,104857600~20125592,130023424~9587625,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,192937984~7452113,205520896~2424777,231958926~3416707,235418253~10268378,245777751~2963301,251658240~3460194,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~13436569,382493018~15153573,398458880~6346344] 2024-09-13T14:42:49.927 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 11 diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~10362211,87809486~7971187,96810451~4938110,105957581~19025611,132706774~6904275,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.927 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~10362211,87809486~7971187,96810451~4938110,105957581~19025611,132706774~6904275,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6849880,207136903~808770,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.934 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,33702107~21180178,75497472~20283201,96810451~4938110,104857600~20125592,130023424~9587625,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,192937984~7452113,205520896~2424777,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344] 2024-09-13T14:42:49.934 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 12 diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132706774~6904275,149400161~2594737,152231961~2578327,156094323~27537154,184405588~6035476,193540217~6849880,203924372~7818857,223004483~6780919,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.934 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132706774~6904275,149400161~2594737,152231961~2578327,156094323~27537154,184405588~6035476,193540217~6849880,203924372~7818857,223004483~6780919,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.941 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,33702107~21180178,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,130023424~9587625,149400161~2594737,152231961~2578327,156094323~27537154,184405588~6035476,192937984~7452113,201326592~10416637,223004483~6780919,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344] 2024-09-13T14:42:49.942 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 13 diff [160166~22596730,33702107~9210479,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~7259372,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~7818857,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.942 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,33702107~9210479,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~7259372,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~7818857,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.949 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,33702107~21180178,64579445~6333027,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,130023424~9587625,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~10416637,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344] 2024-09-13T14:42:49.949 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 14 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~7259372,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~7818857,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.949 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~7259372,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~7818857,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~13135139,399873735~4931489] 2024-09-13T14:42:49.957 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~6333027,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,130023424~9587625,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~10416637,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,272629760~7116090,281018368~2239355,285212672~18989768,310453767~7798987,321000567~16003404,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344] 2024-09-13T14:42:49.958 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 15 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~7259372,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307] 2024-09-13T14:42:49.958 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~7259372,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307] 2024-09-13T14:42:49.965 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~6333027,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,130023424~9587625,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,339738624~7411267,348127232~16951219,366123856~31522735,398458880~6346344] 2024-09-13T14:42:49.965 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 16 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~7259372,141404955~444745,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307] 2024-09-13T14:42:49.965 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~7259372,141404955~444745,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307] 2024-09-13T14:42:49.972 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,130023424~9587625,141404955~444745,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,339738624~7411267,348127232~16951219,366123856~31522735,398458880~6346344] 2024-09-13T14:42:49.972 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 17 diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~13674491,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~31522735,399621917~5183307] 2024-09-13T14:42:49.972 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~13674491,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~31522735,399621917~5183307] 2024-09-13T14:42:49.981 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,130023424~16002744,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~7713792,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,339738624~7411267,348127232~16951219,366123856~31522735,398458880~6346344] 2024-09-13T14:42:49.981 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 18 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307] 2024-09-13T14:42:49.981 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307] 2024-09-13T14:42:49.990 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~23184072,310453767~7798987,321000567~44077884,366123856~31522735,398458880~6346344] 2024-09-13T14:42:49.990 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 19 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:49.991 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:49.998 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:49.999 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 4 diff [9390236~6883116,91011347~3717169,106191246~11701461,152231961~2578327,169300642~11861796,231958926~3416707,273326128~4621377,389565511~8081080] 2024-09-13T14:42:49.999 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [9390236~6883116,91011347~3717169,106191246~11701461,152231961~2578327,169300642~11861796,231958926~3416707,273326128~4621377,389565511~8081080] 2024-09-13T14:42:50.006 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [9390236~6883116,91011347~3717169,104857600~13035107,152231961~2578327,167772160~13390278,231958926~3416707,273326128~4621377,385875968~11770623] 2024-09-13T14:42:50.006 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 5 diff [9390236~6883116,37838725~3913129,50256085~3817320,79983070~11130,87809486~7971187,106191246~17539866,152231961~2578327,162346201~1803261,169300642~11861796,181824389~1807088,231958926~3416707,273326128~4621377,342881049~4268842,389565511~8081080,399873735~4931489] 2024-09-13T14:42:50.006 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [9390236~6883116,37838725~3913129,50256085~3817320,79983070~11130,87809486~7971187,106191246~17539866,152231961~2578327,162346201~1803261,169300642~11861796,181824389~1807088,231958926~3416707,273326128~4621377,342881049~4268842,389565511~8081080,399873735~4931489] 2024-09-13T14:42:50.015 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [9390236~6883116,37838725~3913129,46137344~7936061,79983070~11130,85584761~10195912,104857600~18873512,152231961~2578327,159383552~4765910,167772160~13390278,181824389~1807088,231958926~3416707,273326128~4621377,339738624~7411267,385875968~11770623,398458880~6346344] 2024-09-13T14:42:50.015 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 6 diff [5525499~14331290,37838725~5073861,45672656~8400749,79983070~11130,87809486~7971187,106191246~17539866,152231961~2578327,162346201~1803261,169300642~11861796,181824389~1807088,207136903~808770,231958926~3416707,240563824~2533590,261549486~6211154,273326128~4621377,342881049~4268842,353815301~1081416,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:50.015 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [5525499~14331290,37838725~5073861,45672656~8400749,79983070~11130,87809486~7971187,106191246~17539866,152231961~2578327,162346201~1803261,169300642~11861796,181824389~1807088,207136903~808770,231958926~3416707,240563824~2533590,261549486~6211154,273326128~4621377,342881049~4268842,353815301~1081416,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:50.021 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [5525499~14331290,37838725~16234680,79983070~11130,85584761~10195912,104857600~18873512,152231961~2578327,159383552~4765910,167772160~13390278,181824389~1807088,205520896~2424777,231958926~3416707,240563824~2533590,261549486~6211154,273326128~4621377,339738624~7411267,352321536~2575181,373293056~6267369,382493018~15153573,398458880~6346344] 2024-09-13T14:42:50.021 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 7 diff [5525499~15729423,35833134~7079452,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,106191246~17539866,152231961~2578327,162346201~18816237,181824389~1807088,207136903~808770,231958926~3416707,240563824~2533590,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:50.021 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [5525499~15729423,35833134~7079452,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,106191246~17539866,152231961~2578327,162346201~18816237,181824389~1807088,207136903~808770,231958926~3416707,240563824~2533590,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:50.029 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [5525499~15729423,35833134~19049151,79983070~11130,85584761~10195912,97211542~4109022,104857600~18873512,152231961~2578327,159383552~21778886,181824389~1807088,205520896~2424777,231958926~3416707,240563824~2533590,261549486~6211154,272681980~5265525,281018368~2239355,285212672~8442410,310453767~7798987,339738624~7411267,352321536~2575181,373293056~6267369,382493018~15153573,398458880~6346344] 2024-09-13T14:42:50.029 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 8 diff [160166~21094756,35833134~7079452,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,106191246~18791946,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~5568432,207136903~808770,231958926~3416707,236203174~7525653,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,371152217~386800,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:50.029 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,35833134~7079452,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,106191246~18791946,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~5568432,207136903~808770,231958926~3416707,236203174~7525653,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,371152217~386800,376986335~2574090,385538942~12107649,399873735~4931489] 2024-09-13T14:42:50.036 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,35833134~19049151,79983070~11130,85584761~10195912,97211542~4109022,104857600~20125592,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,192937984~6170665,205520896~2424777,231958926~3416707,235418253~8310574,261549486~6211154,272681980~5265525,281018368~2239355,285212672~8442410,310453767~7798987,339738624~7411267,352321536~2575181,369700514~1838503,373293056~6267369,382493018~15153573,398458880~6346344] 2024-09-13T14:42:50.036 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 9 diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,79983070~11130,87809486~7971187,97211542~4109022,106191246~18791946,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~5568432,207136903~808770,231958926~3416707,236203174~7525653,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.037 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,79983070~11130,87809486~7971187,97211542~4109022,106191246~18791946,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~5568432,207136903~808770,231958926~3416707,236203174~7525653,261549486~6211154,272681980~5265525,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.044 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,35554568~19327717,75497472~4417676,79983070~11130,85584761~10195912,97211542~4109022,104857600~20125592,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,192937984~6170665,205520896~2424777,231958926~3416707,235418253~8310574,261549486~6211154,272681980~5265525,281018368~2239355,285212672~8442410,310453767~7798987,321000567~7975347,339738624~7411267,352321536~10810425,366123856~13436569,382493018~15153573,398458880~6346344] 2024-09-13T14:42:50.044 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 10 diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,79983070~11130,83876008~3926150,87809486~7971187,96810451~4510113,106191246~18791946,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6639063,207136903~808770,231958926~3416707,236203174~7525653,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.044 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,79983070~11130,83876008~3926150,87809486~7971187,96810451~4510113,106191246~18791946,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6639063,207136903~808770,231958926~3416707,236203174~7525653,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.050 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,35554568~19327717,75497472~4417676,79983070~11130,83876008~11904665,96810451~4510113,104857600~20125592,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,192937984~7241296,205520896~2424777,231958926~3416707,235418253~8310574,261549486~6211154,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~13436569,382493018~15153573,398458880~6346344] 2024-09-13T14:42:50.051 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 11 diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~10362211,87809486~7971187,96810451~4938110,105957581~19025611,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6639063,207136903~808770,231958926~3416707,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.051 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~10362211,87809486~7971187,96810451~4938110,105957581~19025611,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,193540217~6639063,207136903~808770,231958926~3416707,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.059 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,33702107~21180178,75497472~20283201,96810451~4938110,104857600~20125592,149400161~2594737,152231961~2578327,156094323~25068115,181824389~1807088,192937984~7241296,205520896~2424777,231958926~3416707,235418253~8310574,250331944~6999104,261549486~6211154,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344] 2024-09-13T14:42:50.059 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 12 diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,149400161~2594737,152231961~2578327,156094323~27537154,184405588~6035476,193540217~6639063,203924372~7818857,223004483~6780919,231958926~3416707,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.059 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,149400161~2594737,152231961~2578327,156094323~27537154,184405588~6035476,193540217~6639063,203924372~7818857,223004483~6780919,231958926~3416707,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.067 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,33702107~21180178,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,149400161~2594737,152231961~2578327,156094323~27537154,184405588~6035476,192937984~7241296,201326592~10416637,223004483~6780919,231958926~3416707,235418253~8310574,250331944~6999104,261549486~6211154,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344] 2024-09-13T14:42:50.068 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 13 diff [160166~22596730,33702107~9210479,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~3360072,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~7818857,222718339~7067063,231958926~3416707,236203174~7525653,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.068 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,33702107~9210479,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~3360072,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~7818857,222718339~7067063,231958926~3416707,236203174~7525653,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.075 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,33702107~21180178,64579445~6333027,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,132351677~3360072,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~10416637,222718339~7067063,231958926~3416707,235418253~8310574,250331944~6999104,260623526~7137114,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344] 2024-09-13T14:42:50.075 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 14 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~3360072,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~7818857,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.075 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~3360072,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~7818857,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~13135139,399873735~4931489] 2024-09-13T14:42:50.082 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~6333027,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,132351677~3360072,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~10416637,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~7137114,272681980~7063870,281018368~2239355,285212672~18989768,310453767~7798987,321000567~16003404,339738624~7411267,348127232~15004729,366123856~31522735,398458880~6346344] 2024-09-13T14:42:50.082 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 15 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~3360072,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307] 2024-09-13T14:42:50.082 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~3360072,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307] 2024-09-13T14:42:50.090 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~6333027,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,132351677~3360072,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~7137114,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,339738624~7411267,348127232~16951219,366123856~31522735,398458880~6346344] 2024-09-13T14:42:50.090 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 16 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307] 2024-09-13T14:42:50.090 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~13135139,399621917~5183307] 2024-09-13T14:42:50.097 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~7137114,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,339738624~7411267,348127232~16951219,366123856~31522735,398458880~6346344] 2024-09-13T14:42:50.097 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 17 diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~31522735,399621917~5183307] 2024-09-13T14:42:50.098 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~31522735,399621917~5183307] 2024-09-13T14:42:50.105 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~7137114,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,339738624~7411267,348127232~16951219,366123856~31522735,398458880~6346344] 2024-09-13T14:42:50.105 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 18 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307] 2024-09-13T14:42:50.105 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307] 2024-09-13T14:42:50.113 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~23184072,310453767~7798987,321000567~44077884,366123856~31522735,398458880~6346344] 2024-09-13T14:42:50.113 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 19 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:50.113 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:50.121 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:50.121 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 5 diff [37838725~3913129,50256085~3817320,79983070~11130,87809486~7971187,115764491~7966621,162346201~1803261,177418312~599734,181824389~1807088,342881049~4268842,399873735~4931489] 2024-09-13T14:42:50.121 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [37838725~3913129,50256085~3817320,79983070~11130,87809486~7971187,115764491~7966621,162346201~1803261,177418312~599734,181824389~1807088,342881049~4268842,399873735~4931489] 2024-09-13T14:42:50.128 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [37838725~3913129,46137344~7936061,79983070~11130,85584761~10195912,115764491~7966621,159383552~4765910,177418312~599734,181824389~1807088,339738624~7411267,398458880~6346344] 2024-09-13T14:42:50.128 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 6 diff [5525499~14331290,37838725~5073861,45672656~8400749,79983070~11130,87809486~7971187,115764491~7966621,162346201~1803261,177418312~599734,181824389~1807088,207136903~808770,240563824~2533590,261549486~6211154,342881049~4268842,353815301~1081416,376986335~2574090,385538942~4898061,399873735~4931489] 2024-09-13T14:42:50.128 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [5525499~14331290,37838725~5073861,45672656~8400749,79983070~11130,87809486~7971187,115764491~7966621,162346201~1803261,177418312~599734,181824389~1807088,207136903~808770,240563824~2533590,261549486~6211154,342881049~4268842,353815301~1081416,376986335~2574090,385538942~4898061,399873735~4931489] 2024-09-13T14:42:50.135 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [5525499~14331290,37838725~16234680,79983070~11130,85584761~10195912,115764491~7966621,159383552~4765910,177418312~599734,181824389~1807088,205520896~2424777,240563824~2533590,261549486~6211154,339738624~7411267,352321536~2575181,373293056~6267369,382493018~7943985,398458880~6346344] 2024-09-13T14:42:50.135 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 7 diff [5525499~15729423,35833134~7079452,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,115764491~7966621,162346201~9414529,177418312~599734,181824389~1807088,207136903~808770,240563824~2533590,261549486~6211154,272681980~2423441,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,376986335~2574090,385538942~4898061,399873735~4931489] 2024-09-13T14:42:50.136 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [5525499~15729423,35833134~7079452,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,115764491~7966621,162346201~9414529,177418312~599734,181824389~1807088,207136903~808770,240563824~2533590,261549486~6211154,272681980~2423441,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,376986335~2574090,385538942~4898061,399873735~4931489] 2024-09-13T14:42:50.143 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [5525499~15729423,35833134~19049151,79983070~11130,85584761~10195912,97211542~4109022,115764491~7966621,159383552~12377178,177418312~599734,181824389~1807088,205520896~2424777,240563824~2533590,261549486~6211154,272681980~2423441,281018368~2239355,285212672~8442410,310453767~7798987,339738624~7411267,352321536~2575181,373293056~6267369,382493018~7943985,398458880~6346344] 2024-09-13T14:42:50.143 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 8 diff [160166~21094756,35833134~7079452,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,115764491~9218701,149400161~2594737,156094323~15666407,177418312~599734,181824389~1807088,193540217~5568432,207136903~808770,236203174~7525653,261549486~6211154,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,371152217~386800,376986335~2574090,385538942~4898061,399873735~4931489] 2024-09-13T14:42:50.144 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,35833134~7079452,45672656~9209629,79983070~11130,87809486~7971187,97211542~4109022,115764491~9218701,149400161~2594737,156094323~15666407,177418312~599734,181824389~1807088,193540217~5568432,207136903~808770,236203174~7525653,261549486~6211154,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,342881049~4268842,353815301~1081416,371152217~386800,376986335~2574090,385538942~4898061,399873735~4931489] 2024-09-13T14:42:50.151 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,35833134~19049151,79983070~11130,85584761~10195912,97211542~4109022,115764491~9218701,149400161~2594737,156094323~15666407,177418312~599734,181824389~1807088,192937984~6170665,205520896~2424777,235418253~8310574,261549486~6211154,272681980~4020696,281018368~2239355,285212672~8442410,310453767~7798987,339738624~7411267,352321536~2575181,369700514~1838503,373293056~6267369,382493018~7943985,398458880~6346344] 2024-09-13T14:42:50.151 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 9 diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,79983070~11130,87809486~7971187,97211542~4109022,115764491~9218701,149400161~2594737,156094323~15666407,175109128~4337853,181824389~1807088,193540217~5568432,207136903~808770,236203174~7525653,261549486~6211154,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.151 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,79983070~11130,87809486~7971187,97211542~4109022,115764491~9218701,149400161~2594737,156094323~15666407,175109128~4337853,181824389~1807088,193540217~5568432,207136903~808770,236203174~7525653,261549486~6211154,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,342881049~4268842,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.158 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,35554568~19327717,75497472~4417676,79983070~11130,85584761~10195912,97211542~4109022,115764491~9218701,149400161~2594737,156094323~15666407,175109128~4337853,181824389~1807088,192937984~6170665,205520896~2424777,235418253~8310574,261549486~6211154,272681980~4020696,281018368~2239355,285212672~8442410,310453767~7798987,321000567~7975347,339738624~7411267,352321536~10810425,366123856~13436569,382493018~7943985,398458880~6346344] 2024-09-13T14:42:50.158 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 10 diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,79983070~11130,83876008~3926150,87809486~7971187,96810451~4510113,115764491~9218701,149400161~2594737,156094323~15666407,175109128~4337853,181824389~1807088,193540217~6639063,207136903~808770,236203174~7525653,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.158 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,79983070~11130,83876008~3926150,87809486~7971187,96810451~4510113,115764491~9218701,149400161~2594737,156094323~15666407,175109128~4337853,181824389~1807088,193540217~6639063,207136903~808770,236203174~7525653,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.165 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,35554568~19327717,75497472~4417676,79983070~11130,83876008~11904665,96810451~4510113,115764491~9218701,149400161~2594737,156094323~15666407,175109128~4337853,181824389~1807088,192937984~7241296,205520896~2424777,235418253~8310574,261549486~6211154,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~13436569,382493018~7943985,398458880~6346344] 2024-09-13T14:42:50.165 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 11 diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~10362211,87809486~7971187,96810451~4938110,105957581~4841492,115764491~9218701,149400161~2594737,156094323~15666407,175109128~4337853,181824389~1807088,193540217~6639063,207136903~808770,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.165 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~10362211,87809486~7971187,96810451~4938110,105957581~4841492,115764491~9218701,149400161~2594737,156094323~15666407,175109128~4337853,181824389~1807088,193540217~6639063,207136903~808770,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.173 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,33702107~21180178,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,149400161~2594737,156094323~15666407,175109128~4337853,181824389~1807088,192937984~7241296,205520896~2424777,235418253~8310574,250331944~6999104,261549486~6211154,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~24313147,398458880~6346344] 2024-09-13T14:42:50.173 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 12 diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,149400161~2594737,156094323~15666407,175109128~8522349,184405588~6035476,193540217~6639063,203924372~7818857,223004483~6780919,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.173 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,149400161~2594737,156094323~15666407,175109128~8522349,184405588~6035476,193540217~6639063,203924372~7818857,223004483~6780919,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.182 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,33702107~21180178,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,149400161~2594737,156094323~15666407,175109128~8522349,184405588~6035476,192937984~7241296,201326592~10416637,223004483~6780919,235418253~8310574,250331944~6999104,261549486~6211154,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~24313147,398458880~6346344] 2024-09-13T14:42:50.182 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 13 diff [160166~22596730,33702107~9210479,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~7525653,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.182 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,33702107~9210479,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~7525653,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,342881049~4268842,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.191 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,33702107~21180178,64579445~6333027,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~10416637,222718339~7067063,235418253~8310574,250331944~6999104,260623526~7137114,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,339738624~7411267,348127232~15004729,366123856~24313147,398458880~6346344] 2024-09-13T14:42:50.191 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 14 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.191 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,351783666~7296111,359246174~3885787,366123856~17793460,384511452~5925551,399873735~4931489] 2024-09-13T14:42:50.198 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~6333027,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~10416637,222718339~7067063,235418253~9082333,250331944~6999104,260623526~7137114,272681980~7063870,281018368~2239355,285212672~18989768,310453767~7798987,321000567~16003404,339738624~7411267,348127232~15004729,366123856~24313147,398458880~6346344] 2024-09-13T14:42:50.198 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 15 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~5925551,399621917~5183307] 2024-09-13T14:42:50.198 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~5925551,399621917~5183307] 2024-09-13T14:42:50.205 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~6333027,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,222718339~7067063,235418253~9082333,250331944~6999104,260623526~7137114,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,339738624~7411267,348127232~16951219,366123856~24313147,398458880~6346344] 2024-09-13T14:42:50.205 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 16 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~8588739,399621917~5183307] 2024-09-13T14:42:50.205 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~17793460,384511452~8588739,399621917~5183307] 2024-09-13T14:42:50.214 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,222718339~7067063,235418253~9082333,250331944~6999104,260623526~7137114,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,339738624~7411267,348127232~16951219,366123856~26976335,398458880~6346344] 2024-09-13T14:42:50.214 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 17 diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~26976335,399621917~5183307] 2024-09-13T14:42:50.214 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,342881049~4268842,349598203~13533758,364542714~535737,366123856~26976335,399621917~5183307] 2024-09-13T14:42:50.221 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~7137114,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,339738624~7411267,348127232~16951219,366123856~26976335,398458880~6346344] 2024-09-13T14:42:50.221 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 18 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307] 2024-09-13T14:42:50.221 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307] 2024-09-13T14:42:50.228 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~23184072,310453767~7798987,321000567~44077884,366123856~26976335,398458880~6346344] 2024-09-13T14:42:50.228 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 19 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:50.229 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:50.236 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:42:50.236 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 6 diff [5525499~14331290,40640954~2271632,45672656~6965434,207136903~808770,240563824~2533590,261549486~6211154,353815301~1081416,376986335~2574090,385538942~4898061] 2024-09-13T14:42:50.236 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [5525499~14331290,40640954~2271632,45672656~6965434,207136903~808770,240563824~2533590,261549486~6211154,353815301~1081416,376986335~2574090,385538942~4898061] 2024-09-13T14:42:50.243 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [5525499~14331290,40640954~11997136,205520896~2424777,240563824~2533590,261549486~6211154,352321536~2575181,373293056~6267369,382493018~7943985] 2024-09-13T14:42:50.244 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 7 diff [5525499~15729423,35833134~7079452,45672656~9209629,97211542~4109022,163962679~7798051,207136903~808770,240563824~2533590,261549486~6211154,272681980~2423441,282281434~976289,288360988~5294094,310453767~7798987,353815301~1081416,376986335~2574090,385538942~4898061] 2024-09-13T14:42:50.244 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [5525499~15729423,35833134~7079452,45672656~9209629,97211542~4109022,163962679~7798051,207136903~808770,240563824~2533590,261549486~6211154,272681980~2423441,282281434~976289,288360988~5294094,310453767~7798987,353815301~1081416,376986335~2574090,385538942~4898061] 2024-09-13T14:42:50.251 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [5525499~15729423,35833134~19049151,97211542~4109022,163962679~7798051,205520896~2424777,240563824~2533590,261549486~6211154,272681980~2423441,281018368~2239355,285212672~8442410,310453767~7798987,352321536~2575181,373293056~6267369,382493018~7943985] 2024-09-13T14:42:50.252 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 8 diff [160166~21094756,35833134~7079452,45672656~9209629,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,193540217~5568432,207136903~808770,236203174~7525653,261549486~6211154,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,353815301~1081416,371152217~386800,376986335~2574090,385538942~4898061] 2024-09-13T14:42:50.252 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,35833134~7079452,45672656~9209629,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,193540217~5568432,207136903~808770,236203174~7525653,261549486~6211154,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,353815301~1081416,371152217~386800,376986335~2574090,385538942~4898061] 2024-09-13T14:42:50.258 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,35833134~19049151,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,192937984~6170665,205520896~2424777,235418253~8310574,261549486~6211154,272681980~4020696,281018368~2239355,285212672~8442410,310453767~7798987,352321536~2575181,369700514~1838503,373293056~6267369,382493018~7943985] 2024-09-13T14:42:50.259 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 9 diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~5568432,207136903~808770,236203174~7525653,261549486~6211154,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~5925551] 2024-09-13T14:42:50.259 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~5568432,207136903~808770,236203174~7525653,261549486~6211154,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~5925551] 2024-09-13T14:42:50.267 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,35554568~19327717,75497472~4417676,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,192937984~6170665,205520896~2424777,235418253~8310574,261549486~6211154,272681980~4020696,281018368~2239355,285212672~8442410,310453767~7798987,321000567~7975347,352321536~10810425,366123856~13436569,382493018~7943985] 2024-09-13T14:42:50.267 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 10 diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,83876008~3926150,96810451~4510113,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~6639063,207136903~808770,236203174~7525653,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~5925551] 2024-09-13T14:42:50.267 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,35554568~7358018,45672656~9209629,78353736~1561412,83876008~3926150,96810451~4510113,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~6639063,207136903~808770,236203174~7525653,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,376986335~2574090,384511452~5925551] 2024-09-13T14:42:50.274 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,35554568~19327717,75497472~4417676,83876008~3926150,96810451~4510113,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,192937984~7241296,205520896~2424777,235418253~8310574,261549486~6211154,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,348127232~15004729,366123856~13436569,382493018~7943985] 2024-09-13T14:42:50.274 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 11 diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~10362211,96810451~4938110,105957581~4841492,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~6639063,207136903~808770,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~5925551] 2024-09-13T14:42:50.274 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~10362211,96810451~4938110,105957581~4841492,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~6639063,207136903~808770,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375818016~7111856,384511452~5925551] 2024-09-13T14:42:50.282 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,33702107~21180178,75497472~12304686,96810451~4938110,105957581~4841492,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,192937984~7241296,205520896~2424777,235418253~8310574,250331944~6999104,261549486~6211154,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,348127232~15004729,366123856~24313147] 2024-09-13T14:42:50.282 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 12 diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,149400161~2594737,156094323~7350508,163962679~7798051,175109128~8419497,184405588~6035476,193540217~6639063,203924372~7818857,223004483~6780919,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~5925551] 2024-09-13T14:42:50.282 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~21094756,33702107~9210479,45672656~9209629,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,149400161~2594737,156094323~7350508,163962679~7798051,175109128~8419497,184405588~6035476,193540217~6639063,203924372~7818857,223004483~6780919,236203174~7525653,250358186~6972862,261549486~6211154,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,353815301~5089988,359246174~3885787,366123856~8156090,375414778~8502538,384511452~5925551] 2024-09-13T14:42:50.290 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~21094756,33702107~21180178,75497472~16046298,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,149400161~2594737,156094323~7350508,163962679~7798051,175109128~8419497,184405588~6035476,192937984~7241296,201326592~10416637,223004483~6780919,235418253~8310574,250331944~6999104,261549486~6211154,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,348127232~15004729,366123856~24313147] 2024-09-13T14:42:50.290 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 13 diff [160166~22596730,33702107~9210479,45672656~9209629,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~7525653,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~5925551] 2024-09-13T14:42:50.290 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,33702107~9210479,45672656~9209629,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~7525653,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~5925551] 2024-09-13T14:42:50.298 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,33702107~21180178,64579445~6333027,75497472~16046298,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,192937984~7241296,201326592~10416637,222718339~7067063,235418253~8310574,250331944~6999104,260623526~7137114,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,348127232~15004729,366123856~24313147] 2024-09-13T14:42:50.298 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 14 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,351783666~7296111,359246174~3885787,366123856~17793460,384511452~5925551] 2024-09-13T14:42:50.298 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,351783666~7296111,359246174~3885787,366123856~17793460,384511452~5925551] 2024-09-13T14:42:50.305 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~6333027,75497472~16046298,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,192937984~7241296,201326592~10416637,222718339~7067063,235418253~9082333,250331944~6999104,260623526~7137114,272681980~7063870,281018368~2239355,285212672~18989768,310453767~7798987,321000567~16003404,348127232~15004729,366123856~24313147] 2024-09-13T14:42:50.305 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 15 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~5925551,399621917~2387960] 2024-09-13T14:42:50.305 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~5925551,399621917~2387960] 2024-09-13T14:42:50.312 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~6333027,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,192937984~7241296,201326592~14330997,222718339~7067063,235418253~9082333,250331944~6999104,260623526~7137114,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,348127232~16951219,366123856~24313147,399621917~2387960] 2024-09-13T14:42:50.312 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 16 diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~13533758,364542714~535737,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.312 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9210479,44219630~803772,45672656~9209629,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~13533758,364542714~535737,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.320 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,222718339~7067063,235418253~9082333,250331944~6999104,260623526~7137114,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,348127232~16951219,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.320 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 17 diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~13533758,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.320 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~7137114,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~13533758,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.328 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~7137114,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,348127232~16951219,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.328 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 18 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~23267527,345304694~17827267,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.328 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~23267527,345304694~17827267,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.336 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~23184072,310453767~7798987,321000567~23267527,345304694~19773757,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.336 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 19 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.337 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.346 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.346 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 7 diff [13749627~7505295,35833134~7018729,49220521~5661764,97211542~4109022,163962679~7798051,272681980~2423441,282281434~976289,288360988~5294094,310453767~7798987] 2024-09-13T14:42:50.346 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [13749627~7505295,35833134~7018729,49220521~5661764,97211542~4109022,163962679~7798051,272681980~2423441,282281434~976289,288360988~5294094,310453767~7798987] 2024-09-13T14:42:50.353 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [13749627~7505295,35833134~7018729,49220521~5661764,97211542~4109022,163962679~7798051,272681980~2423441,281018368~2239355,285212672~8442410,310453767~7798987] 2024-09-13T14:42:50.353 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 8 diff [160166~7567734,12488826~8766096,35833134~7018729,49220521~5661764,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,193540217~5568432,236203174~7525653,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,371152217~386800] 2024-09-13T14:42:50.353 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~7567734,12488826~8766096,35833134~7018729,49220521~5661764,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,193540217~5568432,236203174~7525653,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,371152217~386800] 2024-09-13T14:42:50.361 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~7567734,12488826~8766096,35833134~7018729,49220521~5661764,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,192937984~6170665,235418253~8310574,272681980~4020696,281018368~2239355,285212672~8442410,310453767~7798987,369700514~1838503] 2024-09-13T14:42:50.361 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 9 diff [160166~9029831,12488826~8766096,35554568~7297295,49220521~5661764,78353736~1561412,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~5568432,236203174~7525653,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.361 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~9029831,12488826~8766096,35554568~7297295,49220521~5661764,78353736~1561412,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~5568432,236203174~7525653,272681980~4020696,282281434~976289,288360988~5294094,310453767~7798987,321000567~7975347,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.368 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~9029831,12488826~8766096,35554568~7297295,49220521~5661764,75497472~4417676,97211542~4109022,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,192937984~6170665,235418253~8310574,272681980~4020696,281018368~2239355,285212672~8442410,310453767~7798987,321000567~7975347,354823138~8308823,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.368 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 10 diff [160166~10641134,12488826~8766096,35554568~7297295,49220521~5661764,78353736~1561412,83876008~3926150,96810451~4510113,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~6639063,236203174~7525653,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.368 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~10641134,12488826~8766096,35554568~7297295,49220521~5661764,78353736~1561412,83876008~3926150,96810451~4510113,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~6639063,236203174~7525653,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.376 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~10641134,12488826~8766096,35554568~7297295,49220521~5661764,75497472~4417676,83876008~3926150,96810451~4510113,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,192937984~7241296,235418253~8310574,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,348127232~5480297,354823138~8308823,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.376 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 11 diff [160166~10641134,12488826~8766096,33702107~9149756,49220521~5661764,77439947~10362211,96810451~4938110,105957581~4841492,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~6639063,236203174~7525653,250358186~6972862,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375818016~7111856,384511452~4628330] 2024-09-13T14:42:50.376 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~10641134,12488826~8766096,33702107~9149756,49220521~5661764,77439947~10362211,96810451~4938110,105957581~4841492,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,193540217~6639063,236203174~7525653,250358186~6972862,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375818016~7111856,384511452~4628330] 2024-09-13T14:42:50.384 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~10641134,12488826~8766096,33702107~9149756,49220521~5661764,75497472~12304686,96810451~4938110,105957581~4841492,117874817~7108375,149400161~2594737,156094323~7350508,163962679~7798051,175109128~4337853,192937984~7241296,235418253~8310574,250331944~6999104,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,348127232~5480297,354823138~8308823,366123856~8156090,375818016~7111856,384511452~4628330] 2024-09-13T14:42:50.384 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 12 diff [160166~10641134,12488826~8766096,33702107~9149756,49220521~5661764,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,149400161~2594737,156094323~7350508,163962679~7798051,175109128~8419497,184405588~6035476,193540217~6639063,203924372~7818857,223004483~6780919,236203174~7525653,250358186~6972862,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.384 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~10641134,12488826~8766096,33702107~9149756,49220521~5661764,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,149400161~2594737,156094323~7350508,163962679~7798051,175109128~8419497,184405588~6035476,193540217~6639063,203924372~7818857,223004483~6780919,236203174~7525653,250358186~6972862,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.391 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~10641134,12488826~8766096,33702107~9149756,49220521~5661764,75497472~16046298,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,149400161~2594737,156094323~7350508,163962679~7798051,175109128~8419497,184405588~6035476,192937984~7241296,201326592~10416637,223004483~6780919,235418253~8310574,250331944~6999104,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,348127232~5480297,354823138~8308823,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.391 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 13 diff [160166~22596730,33702107~9149756,49220521~5661764,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~7525653,250358186~6972862,260623526~1785751,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.391 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,33702107~9149756,49220521~5661764,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~7525653,250358186~6972862,260623526~1785751,272681980~7063870,282281434~976289,287287224~6367858,294322084~7881482,310453767~7798987,321000567~7975347,329573765~7430206,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.398 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,33702107~9149756,49220521~5661764,64579445~6333027,75497472~16046298,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,192937984~7241296,201326592~10416637,222718339~7067063,235418253~8310574,250331944~6999104,260623526~1785751,272681980~7063870,281018368~2239355,285212672~16990894,310453767~7798987,321000567~7975347,329573765~7430206,348127232~15004729,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.398 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 14 diff [160166~22596730,23836099~4227684,33702107~9149756,44219630~803772,49220521~5661764,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,351783666~7296111,359246174~3885787,366123856~17793460,384511452~4628330] 2024-09-13T14:42:50.398 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9149756,44219630~803772,49220521~5661764,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,272681980~7063870,282281434~976289,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,351783666~7296111,359246174~3885787,366123856~17793460,384511452~4628330] 2024-09-13T14:42:50.406 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~9149756,44219630~803772,49220521~5661764,64579445~6333027,75497472~16046298,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,192937984~7241296,201326592~10416637,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,272681980~7063870,281018368~2239355,285212672~18989768,310453767~7798987,321000567~16003404,348127232~15004729,366123856~17793460,384511452~4628330] 2024-09-13T14:42:50.406 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 15 diff [160166~22596730,23836099~4227684,33702107~9149756,44219630~803772,49220521~5661764,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~4628330,399621917~2387960] 2024-09-13T14:42:50.406 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9149756,44219630~803772,49220521~5661764,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,282281434~976289,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~4628330,399621917~2387960] 2024-09-13T14:42:50.416 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~9149756,44219630~803772,49220521~5661764,64579445~6333027,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,163962679~7798051,175109128~15331936,192937984~7241296,201326592~14330997,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,348127232~16951219,366123856~17793460,384511452~4628330,399621917~2387960] 2024-09-13T14:42:50.417 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 16 diff [160166~22596730,23836099~4227684,33702107~9149756,44219630~803772,49220521~5661764,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~13533758,364542714~535737,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.417 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~9149756,44219630~803772,49220521~5661764,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~13533758,364542714~535737,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.420 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~9149756,44219630~803772,49220521~5661764,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,348127232~16951219,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.420 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 17 diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~13533758,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.420 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~16003404,349598203~13533758,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.427 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,268435456~11310394,281018368~23184072,310453767~7798987,321000567~16003404,348127232~16951219,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.427 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 18 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~23267527,345304694~17827267,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.427 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~9880356,310453767~7798987,321000567~23267527,345304694~17827267,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.434 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~23184072,310453767~7798987,321000567~23267527,345304694~19773757,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.434 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 19 diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.434 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.442 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.442 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 8 diff [160166~7567734,12488826~4353156,117874817~7108375,149400161~2594737,156094323~7350508,193540217~5568432,236203174~7525653,273974549~2728127,371152217~386800] 2024-09-13T14:42:50.442 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~7567734,12488826~4353156,117874817~7108375,149400161~2594737,156094323~7350508,193540217~5568432,236203174~7525653,273974549~2728127,371152217~386800] 2024-09-13T14:42:50.449 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~7567734,12488826~4353156,117874817~7108375,149400161~2594737,156094323~7350508,192937984~6170665,235418253~8310574,273974549~2728127,369700514~1838503] 2024-09-13T14:42:50.449 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 9 diff [160166~9029831,12488826~4353156,35554568~6496363,78353736~1561412,117874817~7108375,149400161~2594737,156094323~7350508,165534750~3793518,175109128~4337853,193540217~5568432,236203174~7525653,273974549~2728127,321000567~7975347,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.449 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~9029831,12488826~4353156,35554568~6496363,78353736~1561412,117874817~7108375,149400161~2594737,156094323~7350508,165534750~3793518,175109128~4337853,193540217~5568432,236203174~7525653,273974549~2728127,321000567~7975347,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.456 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~9029831,12488826~4353156,35554568~6496363,75497472~4417676,117874817~7108375,149400161~2594737,156094323~7350508,165534750~3793518,175109128~4337853,192937984~6170665,235418253~8310574,273974549~2728127,321000567~7975347,354823138~8308823,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.457 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 10 diff [160166~10641134,12488826~4353156,35554568~6496363,78353736~1561412,83876008~3926150,96810451~3825869,117874817~7108375,149400161~2594737,156094323~7350508,165534750~3793518,175109128~4337853,193540217~6639063,236203174~7525653,273856102~5889748,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.457 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~10641134,12488826~4353156,35554568~6496363,78353736~1561412,83876008~3926150,96810451~3825869,117874817~7108375,149400161~2594737,156094323~7350508,165534750~3793518,175109128~4337853,193540217~6639063,236203174~7525653,273856102~5889748,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.464 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~10641134,12488826~4353156,35554568~6496363,75497472~4417676,83876008~3926150,96810451~3825869,117874817~7108375,149400161~2594737,156094323~7350508,165534750~3793518,175109128~4337853,192937984~7241296,235418253~8310574,273856102~5889748,287287224~1839976,293655082~8548484,321000567~7975347,329573765~7430206,348127232~5480297,354823138~8308823,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.464 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 11 diff [160166~10641134,12488826~4353156,33702107~8348824,77439947~10362211,96810451~4938110,105957581~4841492,117874817~7108375,149400161~2594737,156094323~7350508,165534750~3793518,175109128~4337853,193540217~6639063,236203174~7525653,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375818016~7111856,384511452~4628330] 2024-09-13T14:42:50.464 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~10641134,12488826~4353156,33702107~8348824,77439947~10362211,96810451~4938110,105957581~4841492,117874817~7108375,149400161~2594737,156094323~7350508,165534750~3793518,175109128~4337853,193540217~6639063,236203174~7525653,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375818016~7111856,384511452~4628330] 2024-09-13T14:42:50.471 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~10641134,12488826~4353156,33702107~8348824,75497472~12304686,96810451~4938110,105957581~4841492,117874817~7108375,149400161~2594737,156094323~7350508,165534750~3793518,175109128~4337853,192937984~7241296,235418253~8310574,250331944~6999104,273089521~6656329,287287224~1839976,293655082~8548484,321000567~7975347,329573765~7430206,348127232~5480297,354823138~8308823,366123856~8156090,375818016~7111856,384511452~4628330] 2024-09-13T14:42:50.471 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 12 diff [160166~10641134,12488826~4353156,33702107~8348824,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,149400161~2594737,156094323~7350508,165534750~3793518,175109128~8419497,184405588~6035476,193540217~6639063,203924372~7818857,223004483~6780919,236203174~7525653,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.472 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~10641134,12488826~4353156,33702107~8348824,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,149400161~2594737,156094323~7350508,165534750~3793518,175109128~8419497,184405588~6035476,193540217~6639063,203924372~7818857,223004483~6780919,236203174~7525653,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.475 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:42:50 smithi177 bash[61439]: ts=2024-09-13T14:42:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:42:50.481 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~10641134,12488826~4353156,33702107~8348824,75497472~16046298,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,149400161~2594737,156094323~7350508,165534750~3793518,175109128~8419497,184405588~6035476,192937984~7241296,201326592~10416637,223004483~6780919,235418253~8310574,250331944~6999104,273089521~6656329,287287224~1839976,293655082~8548484,321000567~7975347,329573765~7430206,348127232~5480297,354823138~8308823,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.482 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 13 diff [160166~22596730,33702107~8348824,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,165534750~3793518,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~7525653,250358186~6972862,260623526~1785751,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.482 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,33702107~8348824,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,165534750~3793518,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~7525653,250358186~6972862,260623526~1785751,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.488 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,33702107~8348824,64579445~6333027,75497472~16046298,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,165534750~3793518,175109128~15331936,192937984~7241296,201326592~10416637,222718339~7067063,235418253~8310574,250331944~6999104,260623526~1785751,273089521~6656329,287287224~1839976,293655082~8548484,321000567~7975347,329573765~7430206,348127232~15004729,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.489 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 14 diff [160166~22596730,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,165534750~3793518,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,273089521~6656329,287173785~3975876,294322084~9880356,321000567~16003404,351783666~7296111,359246174~3885787,366123856~17793460,384511452~4628330] 2024-09-13T14:42:50.489 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,165534750~3793518,175109128~15331936,193540217~6639063,203924372~7818857,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,273089521~6656329,287173785~3975876,294322084~9880356,321000567~16003404,351783666~7296111,359246174~3885787,366123856~17793460,384511452~4628330] 2024-09-13T14:42:50.495 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,75497472~16046298,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,165534750~3793518,175109128~15331936,192937984~7241296,201326592~10416637,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,273089521~6656329,287173785~3975876,293655082~10547358,321000567~16003404,348127232~15004729,366123856~17793460,384511452~4628330] 2024-09-13T14:42:50.495 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 15 diff [160166~22596730,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~4628330,399621917~2387960] 2024-09-13T14:42:50.495 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~4628330,399621917~2387960] 2024-09-13T14:42:50.502 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,149400161~2594737,156094323~7350508,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,268435456~11310394,283257723~2752147,287173785~3975876,293655082~10547358,321000567~16003404,348127232~16951219,366123856~17793460,384511452~4628330,399621917~2387960] 2024-09-13T14:42:50.502 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 16 diff [160166~22596730,23836099~4227684,33702107~8348824,44219630~803772,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~13533758,364542714~535737,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.502 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8348824,44219630~803772,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~13533758,364542714~535737,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.512 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8348824,44219630~803772,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,141404955~444745,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,268435456~11310394,283257723~2752147,287173785~3975876,293655082~10547358,321000567~16003404,348127232~16951219,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.512 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 17 diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~6801038,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~13533758,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.512 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~6801038,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~13533758,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.519 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~6801038,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,132351677~3360072,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,268435456~11310394,283257723~2752147,287173785~3975876,293655082~10547358,321000567~16003404,348127232~16951219,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.519 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 18 diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~6801038,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~23267527,345304694~17827267,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.519 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~6801038,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~23267527,345304694~17827267,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.528 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~6801038,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~10547358,321000567~23267527,345304694~19773757,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.528 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 19 diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.528 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.535 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.535 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 9 diff [5830009~3359988,35554568~6496363,78353736~1561412,165534750~3793518,175109128~4337853,321000567~7975347,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.535 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [5830009~3359988,35554568~6496363,78353736~1561412,165534750~3793518,175109128~4337853,321000567~7975347,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.541 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [5830009~3359988,35554568~6496363,75497472~4417676,165534750~3793518,175109128~4337853,321000567~7975347,354823138~8308823,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.541 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 10 diff [3197767~7603533,35554568~6496363,78353736~1561412,83876008~3926150,96810451~3825869,165534750~3793518,175109128~4337853,194953710~5225570,273856102~5889748,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.541 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~7603533,35554568~6496363,78353736~1561412,83876008~3926150,96810451~3825869,165534750~3793518,175109128~4337853,194953710~5225570,273856102~5889748,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.548 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~7603533,35554568~6496363,75497472~4417676,83876008~3926150,96810451~3825869,165534750~3793518,175109128~4337853,194953710~5225570,273856102~5889748,287287224~1839976,293655082~8548484,321000567~7975347,329573765~7430206,348127232~5480297,354823138~8308823,366123856~8156090,384511452~4628330] 2024-09-13T14:42:50.548 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 11 diff [3197767~7603533,33702107~8348824,77439947~10362211,96810451~4938110,105957581~4841492,165534750~3793518,175109128~4337853,194953710~5225570,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375818016~7111856,384511452~4628330] 2024-09-13T14:42:50.548 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~7603533,33702107~8348824,77439947~10362211,96810451~4938110,105957581~4841492,165534750~3793518,175109128~4337853,194953710~5225570,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375818016~7111856,384511452~4628330] 2024-09-13T14:42:50.555 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~7603533,33702107~8348824,75497472~12304686,96810451~4938110,105957581~4841492,165534750~3793518,175109128~4337853,194953710~5225570,250331944~6999104,273089521~6656329,287287224~1839976,293655082~8548484,321000567~7975347,329573765~7430206,348127232~5480297,354823138~8308823,366123856~8156090,375818016~7111856,384511452~4628330] 2024-09-13T14:42:50.555 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 12 diff [3197767~7603533,33702107~8348824,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,165534750~3793518,175109128~8419497,184405588~6035476,194953710~5225570,203924372~7818857,223004483~6780919,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.555 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~7603533,33702107~8348824,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,165534750~3793518,175109128~8419497,184405588~6035476,194953710~5225570,203924372~7818857,223004483~6780919,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~1823863,354823138~4082151,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.563 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~7603533,33702107~8348824,75497472~16046298,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,165534750~3793518,175109128~8419497,184405588~6035476,194953710~5225570,201326592~10416637,223004483~6780919,250331944~6999104,273089521~6656329,287287224~1839976,293655082~8548484,321000567~7975347,329573765~7430206,348127232~5480297,354823138~8308823,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.563 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 13 diff [3197767~10831910,15268134~7488762,33702107~8348824,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,165534750~3793518,175109128~15331936,194953710~5225570,203924372~7818857,222718339~7067063,250358186~6972862,260623526~1785751,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.563 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~10831910,15268134~7488762,33702107~8348824,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,165534750~3793518,175109128~15331936,194953710~5225570,203924372~7818857,222718339~7067063,250358186~6972862,260623526~1785751,273089521~6656329,287287224~1839976,294322084~7881482,321000567~7975347,329573765~7430206,351783666~7296111,359246174~3885787,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.571 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~10831910,15268134~7488762,33702107~8348824,64579445~6333027,75497472~16046298,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,165534750~3793518,175109128~15331936,194953710~5225570,201326592~10416637,222718339~7067063,250331944~6999104,260623526~1785751,273089521~6656329,287287224~1839976,293655082~8548484,321000567~7975347,329573765~7430206,348127232~15004729,366123856~8156090,375414778~8502538,384511452~4628330] 2024-09-13T14:42:50.571 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 14 diff [3197767~19559129,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,165534750~3793518,175109128~15331936,194953710~5225570,203924372~7818857,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,273089521~6656329,287173785~3975876,294322084~9880356,321000567~16003404,351783666~7296111,359246174~3885787,366123856~17793460,384511452~4628330] 2024-09-13T14:42:50.571 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~19559129,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,165534750~3793518,175109128~15331936,194953710~5225570,203924372~7818857,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,273089521~6656329,287173785~3975876,294322084~9880356,321000567~16003404,351783666~7296111,359246174~3885787,366123856~17793460,384511452~4628330] 2024-09-13T14:42:50.578 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~19559129,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,75497472~16046298,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,165534750~3793518,175109128~15331936,194953710~5225570,201326592~10416637,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,273089521~6656329,287173785~3975876,293655082~10547358,321000567~16003404,348127232~15004729,366123856~17793460,384511452~4628330] 2024-09-13T14:42:50.579 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 15 diff [3197767~19559129,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~4628330,399621917~2387960] 2024-09-13T14:42:50.579 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~19559129,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~9481574,359246174~3885787,364542714~535737,366123856~17793460,384511452~4628330,399621917~2387960] 2024-09-13T14:42:50.586 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~19559129,23836099~4227684,33702107~8348824,44219630~803772,64579445~6333027,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,268435456~11310394,283257723~2752147,287173785~3975876,293655082~10547358,321000567~16003404,348127232~16951219,366123856~17793460,384511452~4628330,399621917~2387960] 2024-09-13T14:42:50.586 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 16 diff [2195112~20561784,23836099~4227684,33702107~8348824,44219630~803772,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~13533758,364542714~535737,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.586 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8348824,44219630~803772,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~13533758,364542714~535737,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.592 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8348824,44219630~803772,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,268435456~11310394,283257723~2752147,287173785~3975876,293655082~10547358,321000567~16003404,348127232~16951219,366123856~17793460,384511452~8588739,399621917~2387960] 2024-09-13T14:42:50.592 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 17 diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~6801038,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~13533758,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.592 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~6801038,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~16003404,349598203~13533758,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.599 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~6801038,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,268435456~11310394,283257723~2752147,287173785~3975876,293655082~10547358,321000567~16003404,348127232~16951219,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.599 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 18 diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~6801038,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~23267527,345304694~17827267,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.599 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~6801038,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321000567~23267527,345304694~17827267,364542714~535737,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.607 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~6801038,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~10547358,321000567~23267527,345304694~19773757,366123856~26976335,399621917~2387960] 2024-09-13T14:42:50.607 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 19 diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.607 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.615 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.615 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 10 diff [3197767~7603533,83876008~3926150,96810451~3825869,194953710~5225570,273856102~5889748,287287224~1839976,294322084~7881482,329573765~7430206,351783666~1823863] 2024-09-13T14:42:50.615 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~7603533,83876008~3926150,96810451~3825869,194953710~5225570,273856102~5889748,287287224~1839976,294322084~7881482,329573765~7430206,351783666~1823863] 2024-09-13T14:42:50.622 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~7603533,83876008~3926150,96810451~3825869,194953710~5225570,273856102~5889748,287287224~1839976,293655082~8548484,329573765~7430206,348127232~5480297] 2024-09-13T14:42:50.622 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 11 diff [3197767~7603533,33702107~2581660,77439947~10362211,96810451~4938110,105957581~4841492,194953710~5225570,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,329573765~7430206,351783666~1823863,375818016~7111856] 2024-09-13T14:42:50.622 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~7603533,33702107~2581660,77439947~10362211,96810451~4938110,105957581~4841492,194953710~5225570,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,329573765~7430206,351783666~1823863,375818016~7111856] 2024-09-13T14:42:50.629 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~7603533,33702107~2581660,77439947~10362211,96810451~4938110,105957581~4841492,194953710~5225570,250331944~6999104,273089521~6656329,287287224~1839976,293655082~8548484,329573765~7430206,348127232~5480297,375818016~7111856] 2024-09-13T14:42:50.629 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 12 diff [3197767~7603533,33702107~2581660,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,176994513~6534112,184405588~6035476,194953710~5225570,203924372~7818857,223004483~6780919,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,329573765~7430206,351783666~1823863,375414778~8502538] 2024-09-13T14:42:50.629 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~7603533,33702107~2581660,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,176994513~6534112,184405588~6035476,194953710~5225570,203924372~7818857,223004483~6780919,250358186~6972862,273089521~6656329,287287224~1839976,294322084~7881482,329573765~7430206,351783666~1823863,375414778~8502538] 2024-09-13T14:42:50.637 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~7603533,33702107~2581660,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,176994513~6534112,184405588~6035476,194953710~5225570,201326592~10416637,223004483~6780919,250331944~6999104,273089521~6656329,287287224~1839976,293655082~8548484,329573765~7430206,348127232~5480297,375414778~8502538] 2024-09-13T14:42:50.637 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 13 diff [3197767~10831910,15268134~7488762,33702107~2581660,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,194953710~5225570,203924372~7818857,222718339~7067063,250358186~6972862,260623526~1785751,273089521~6656329,287287224~1839976,294322084~7881482,321943280~5551255,329573765~7430206,351783666~7296111,375414778~8502538] 2024-09-13T14:42:50.637 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~10831910,15268134~7488762,33702107~2581660,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,194953710~5225570,203924372~7818857,222718339~7067063,250358186~6972862,260623526~1785751,273089521~6656329,287287224~1839976,294322084~7881482,321943280~5551255,329573765~7430206,351783666~7296111,375414778~8502538] 2024-09-13T14:42:50.643 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~10831910,15268134~7488762,33702107~2581660,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,194953710~5225570,201326592~10416637,222718339~7067063,250331944~6999104,260623526~1785751,273089521~6656329,287287224~1839976,293655082~8548484,321943280~5551255,329573765~7430206,348127232~10952545,375414778~8502538] 2024-09-13T14:42:50.643 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 14 diff [3197767~19559129,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,194953710~5225570,203924372~7818857,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,273089521~6656329,287173785~3975876,294322084~9880356,321943280~5551255,327957029~9046942,351783666~7296111,373574339~10342977] 2024-09-13T14:42:50.643 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~19559129,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,194953710~5225570,203924372~7818857,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,273089521~6656329,287173785~3975876,294322084~9880356,321943280~5551255,327957029~9046942,351783666~7296111,373574339~10342977] 2024-09-13T14:42:50.650 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~19559129,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~14103823,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,194953710~5225570,201326592~10416637,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,273089521~6656329,287173785~3975876,293655082~10547358,321943280~5551255,327957029~9046942,348127232~10952545,373574339~10342977] 2024-09-13T14:42:50.650 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 15 diff [3197767~19559129,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,194953710~5225570,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,284903576~1106294,287173785~3975876,294322084~9880356,321943280~5551255,327957029~9046942,349598203~9481574,364542714~535737,373574339~10342977,399621917~2387960] 2024-09-13T14:42:50.651 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [3197767~19559129,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,194953710~5225570,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,284903576~1106294,287173785~3975876,294322084~9880356,321943280~5551255,327957029~9046942,349598203~9481574,364542714~535737,373574339~10342977,399621917~2387960] 2024-09-13T14:42:50.658 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [3197767~19559129,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,194953710~5225570,201326592~14330997,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,268435456~11310394,283257723~2752147,287173785~3975876,293655082~10547358,321943280~5551255,327957029~9046942,348127232~10952545,363131961~1946490,373574339~10342977,399621917~2387960] 2024-09-13T14:42:50.658 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 16 diff [2195112~20561784,23836099~4227684,33702107~2581660,44219630~803772,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321943280~5551255,327957029~9046942,349598203~12396258,364542714~535737,373574339~10342977,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.659 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,44219630~803772,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321943280~5551255,327957029~9046942,349598203~12396258,364542714~535737,373574339~10342977,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.665 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,44219630~803772,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,268435456~11310394,283257723~2752147,287173785~3975876,293655082~10547358,321943280~5551255,327957029~9046942,348127232~13867229,363131961~1946490,373574339~10342977,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.665 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 17 diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321062763~6431772,327957029~9046942,349598203~12396258,364542714~535737,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.665 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~9461314,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321062763~6431772,327957029~9046942,349598203~12396258,364542714~535737,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.672 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,268435456~11310394,283257723~2752147,287173785~3975876,293655082~10547358,321062763~6431772,327957029~9046942,348127232~13867229,363131961~1946490,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.672 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 18 diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321062763~6431772,327957029~16311065,345304694~16689767,364542714~535737,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.672 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~9880356,321062763~6431772,327957029~16311065,345304694~16689767,364542714~535737,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.679 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~10547358,321062763~6431772,327957029~16311065,345304694~16689767,363131961~1946490,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.679 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 19 diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.679 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.686 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.686 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 11 diff [33702107~2581660,77439947~6884463,99104350~2644211,105957581~4841492,250358186~6972862,273089521~3348875,297590923~1695684,375818016~7111856] 2024-09-13T14:42:50.686 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [33702107~2581660,77439947~6884463,99104350~2644211,105957581~4841492,250358186~6972862,273089521~3348875,297590923~1695684,375818016~7111856] 2024-09-13T14:42:50.692 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [33702107~2581660,77439947~6884463,99104350~2644211,105957581~4841492,250331944~6999104,273089521~3348875,297590923~1695684,375818016~7111856] 2024-09-13T14:42:50.692 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 12 diff [33702107~2581660,77439947~6884463,87193675~4350095,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,176994513~6534112,184405588~6035476,203924372~7818857,223004483~6780919,250358186~6972862,273089521~3348875,297590923~1695684,375414778~8502538] 2024-09-13T14:42:50.692 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [33702107~2581660,77439947~6884463,87193675~4350095,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,176994513~6534112,184405588~6035476,203924372~7818857,223004483~6780919,250358186~6972862,273089521~3348875,297590923~1695684,375414778~8502538] 2024-09-13T14:42:50.699 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [33702107~2581660,77439947~6884463,87193675~4350095,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,176994513~6534112,184405588~6035476,201326592~10416637,223004483~6780919,250331944~6999104,273089521~3348875,297590923~1695684,375414778~8502538] 2024-09-13T14:42:50.699 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 13 diff [6870776~7158901,15268134~7488762,33702107~2581660,64579445~6333027,77439947~6884463,87193675~4350095,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,203924372~7818857,222718339~7067063,250358186~6972862,260623526~1785751,273089521~3348875,297590923~1695684,321943280~5551255,353246342~5833435,375414778~8502538] 2024-09-13T14:42:50.700 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [6870776~7158901,15268134~7488762,33702107~2581660,64579445~6333027,77439947~6884463,87193675~4350095,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,203924372~7818857,222718339~7067063,250358186~6972862,260623526~1785751,273089521~3348875,297590923~1695684,321943280~5551255,353246342~5833435,375414778~8502538] 2024-09-13T14:42:50.707 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [6870776~7158901,15268134~7488762,33702107~2581660,64579445~6333027,77439947~6884463,87193675~4350095,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,201326592~10416637,222718339~7067063,250331944~6999104,260623526~1785751,273089521~3348875,297590923~1695684,321943280~5551255,353246342~5833435,375414778~8502538] 2024-09-13T14:42:50.707 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 14 diff [6870776~15886120,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~6884463,87193675~4350095,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,203924372~7818857,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,273089521~3348875,287173785~3975876,297590923~1695684,299549539~4652901,321943280~5551255,327957029~5068897,353246342~5833435,373574339~10342977] 2024-09-13T14:42:50.707 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [6870776~15886120,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~6884463,87193675~4350095,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,203924372~7818857,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,273089521~3348875,287173785~3975876,297590923~1695684,299549539~4652901,321943280~5551255,327957029~5068897,353246342~5833435,373574339~10342977] 2024-09-13T14:42:50.714 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [6870776~15886120,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~6884463,87193675~4350095,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,201326592~10416637,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,273089521~3348875,287173785~3975876,297590923~1695684,299549539~4652901,321943280~5551255,327957029~5068897,353246342~5833435,373574339~10342977] 2024-09-13T14:42:50.714 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 15 diff [6870776~15886120,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~6153860,284903576~1106294,287173785~3975876,297590923~1695684,299549539~4652901,321943280~5551255,327957029~5068897,349598203~9481574,364542714~535737,373574339~10342977,399621917~2387960] 2024-09-13T14:42:50.714 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [6870776~15886120,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~6153860,284903576~1106294,287173785~3975876,297590923~1695684,299549539~4652901,321943280~5551255,327957029~5068897,349598203~9481574,364542714~535737,373574339~10342977,399621917~2387960] 2024-09-13T14:42:50.721 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [6870776~15886120,23836099~4227684,33702107~2581660,44219630~803772,64579445~6333027,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,268435456~8002940,283257723~2752147,287173785~3975876,297590923~1695684,299549539~4652901,321943280~5551255,327957029~5068897,349598203~9481574,363131961~1946490,373574339~10342977,399621917~2387960] 2024-09-13T14:42:50.721 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 16 diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,44219630~803772,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~6153860,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~4652901,321943280~5551255,327957029~5068897,349598203~12396258,364542714~535737,373574339~10342977,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.721 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,44219630~803772,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~6153860,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~4652901,321943280~5551255,327957029~5068897,349598203~12396258,364542714~535737,373574339~10342977,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.728 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,44219630~803772,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,268435456~8002940,283257723~2752147,287173785~3975876,297590923~1695684,299549539~4652901,321943280~5551255,327957029~5068897,349598203~12396258,363131961~1946490,373574339~10342977,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.728 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 17 diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~6896044,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~4652901,321062763~6431772,327957029~5068897,349598203~12396258,364542714~535737,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.728 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,270284536~6896044,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~4652901,321062763~6431772,327957029~5068897,349598203~12396258,364542714~535737,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.734 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,268435456~8745124,283257723~2752147,287173785~3975876,297590923~1695684,299549539~4652901,321062763~6431772,327957029~5068897,349598203~12396258,363131961~1946490,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.734 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 18 diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~4652901,321062763~6431772,327957029~16311065,345304694~16689767,364542714~535737,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.734 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~4652901,321062763~6431772,327957029~16311065,345304694~16689767,364542714~535737,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.741 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,297590923~1695684,299549539~4652901,321062763~6431772,327957029~16311065,345304694~16689767,363131961~1946490,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.742 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 19 diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.742 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.749 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.749 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 12 diff [81203912~656330,87193675~4350095,117570724~853772,127749407~1500256,176994513~6534112,184405588~6035476,203924372~7818857,223004483~6780919,375414778~4901517,381872637~2044679] 2024-09-13T14:42:50.749 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [81203912~656330,87193675~4350095,117570724~853772,127749407~1500256,176994513~6534112,184405588~6035476,203924372~7818857,223004483~6780919,375414778~4901517,381872637~2044679] 2024-09-13T14:42:50.756 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [81203912~656330,87193675~4350095,117570724~853772,127749407~1500256,176994513~6534112,184405588~6035476,201326592~10416637,223004483~6780919,375414778~4901517,381872637~2044679] 2024-09-13T14:42:50.756 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 13 diff [6870776~7158901,15268134~7488762,64579445~6333027,81203912~656330,87193675~4350095,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,203924372~7818857,222718339~7067063,260623526~1785751,321943280~5551255,353246342~5833435,375414778~4901517,381872637~2044679] 2024-09-13T14:42:50.756 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [6870776~7158901,15268134~7488762,64579445~6333027,81203912~656330,87193675~4350095,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,203924372~7818857,222718339~7067063,260623526~1785751,321943280~5551255,353246342~5833435,375414778~4901517,381872637~2044679] 2024-09-13T14:42:50.763 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [6870776~7158901,15268134~7488762,64579445~6333027,81203912~656330,87193675~4350095,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,201326592~10416637,222718339~7067063,260623526~1785751,321943280~5551255,353246342~5833435,375414778~4901517,381872637~2044679] 2024-09-13T14:42:50.763 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 14 diff [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,81203912~656330,87193675~4350095,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,203924372~7818857,222718339~7067063,243273429~1227157,260623526~1785751,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,353246342~5833435,373574339~7928023,381872637~2044679] 2024-09-13T14:42:50.763 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,81203912~656330,87193675~4350095,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,203924372~7818857,222718339~7067063,243273429~1227157,260623526~1785751,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,353246342~5833435,373574339~7928023,381872637~2044679] 2024-09-13T14:42:50.770 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,81203912~656330,87193675~4350095,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,201326592~10416637,222718339~7067063,243273429~1227157,260623526~1785751,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,353246342~5833435,373574339~7928023,381872637~2044679] 2024-09-13T14:42:50.771 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 15 diff [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,81203912~656330,87193675~5564241,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,222718339~7067063,243273429~1227157,260623526~1785751,270284536~3735996,284903576~1106294,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~9481574,364542714~535737,373574339~7928023,381872637~2044679,399621917~2387960] 2024-09-13T14:42:50.771 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,81203912~656330,87193675~5564241,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,222718339~7067063,243273429~1227157,260623526~1785751,270284536~3735996,284903576~1106294,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~9481574,364542714~535737,373574339~7928023,381872637~2044679,399621917~2387960] 2024-09-13T14:42:50.777 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,81203912~656330,87193675~5564241,117570724~853772,127749407~1500256,132351677~3360072,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,222718339~7067063,243273429~1227157,260623526~1785751,268435456~5585076,283257723~2752147,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~9481574,363131961~1946490,373574339~7928023,381872637~2044679,399621917~2387960] 2024-09-13T14:42:50.777 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 16 diff [2195112~2825918,6870776~15886120,23836099~4227684,44219630~803772,64579445~10686594,81203912~656330,87193675~5564241,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,222718339~7067063,243273429~1227157,260623526~1785751,270284536~3735996,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~12396258,364542714~535737,373574339~7928023,381872637~2044679,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.777 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,44219630~803772,64579445~10686594,81203912~656330,87193675~5564241,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,222718339~7067063,243273429~1227157,260623526~1785751,270284536~3735996,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~12396258,364542714~535737,373574339~7928023,381872637~2044679,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.785 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,44219630~803772,64579445~10686594,81203912~656330,87193675~5564241,117570724~853772,127749407~1500256,132351677~3360072,141404955~444745,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,222718339~7067063,243273429~1227157,260623526~1785751,268435456~5585076,283257723~2752147,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~12396258,363131961~1946490,373574339~7928023,381872637~2044679,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.785 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 17 diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,270284536~6896044,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~6431772,327957029~5068897,349598203~12396258,364542714~535737,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.785 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,270284536~6896044,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~6431772,327957029~5068897,349598203~12396258,364542714~535737,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.792 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,132351677~3360072,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,268435456~8745124,283257723~2752147,287173785~3975876,299549539~4652901,321062763~6431772,327957029~5068897,349598203~12396258,363131961~1946490,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.792 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 18 diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~6431772,327957029~16311065,345304694~16689767,364542714~535737,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.792 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~6431772,327957029~16311065,345304694~16689767,364542714~535737,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.799 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~4652901,321062763~6431772,327957029~16311065,345304694~16689767,363131961~1946490,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.799 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 19 diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.799 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.806 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.806 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 13 diff [6870776~7158901,15268134~7488762,64579445~6333027,132351677~3360072,179806240~7716975,222718339~5152715,260623526~1785751,321943280~5551255,353246342~5833435] 2024-09-13T14:42:50.806 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [6870776~7158901,15268134~7488762,64579445~6333027,132351677~3360072,179806240~7716975,222718339~5152715,260623526~1785751,321943280~5551255,353246342~5833435] 2024-09-13T14:42:50.813 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [6870776~7158901,15268134~7488762,64579445~6333027,132351677~3360072,179806240~7716975,222718339~5152715,260623526~1785751,321943280~5551255,353246342~5833435] 2024-09-13T14:42:50.813 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 14 diff [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,132351677~3360072,179806240~7716975,196783755~39519,197409751~530452,222718339~5152715,243273429~1227157,260623526~1785751,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,353246342~5833435,373574339~7928023] 2024-09-13T14:42:50.813 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,132351677~3360072,179806240~7716975,196783755~39519,197409751~530452,222718339~5152715,243273429~1227157,260623526~1785751,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,353246342~5833435,373574339~7928023] 2024-09-13T14:42:50.820 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,132351677~3360072,179806240~7716975,196783755~39519,197409751~530452,222718339~5152715,243273429~1227157,260623526~1785751,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,353246342~5833435,373574339~7928023] 2024-09-13T14:42:50.820 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 15 diff [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,90842417~1915499,132351677~3360072,179806240~7716975,196783755~39519,197409751~530452,208345786~7311803,222718339~5152715,243273429~1227157,260623526~1785751,270284536~3735996,284903576~1106294,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~9481574,364542714~535737,373574339~7928023,399621917~2387960] 2024-09-13T14:42:50.820 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,90842417~1915499,132351677~3360072,179806240~7716975,196783755~39519,197409751~530452,208345786~7311803,222718339~5152715,243273429~1227157,260623526~1785751,270284536~3735996,284903576~1106294,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~9481574,364542714~535737,373574339~7928023,399621917~2387960] 2024-09-13T14:42:50.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:50 smithi099 bash[84576]: cluster 2024-09-13T14:42:49.469616+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-13T14:42:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:50 smithi099 bash[84576]: cluster 2024-09-13T14:42:49.469616+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-13T14:42:50.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:50 smithi099 bash[87147]: cluster 2024-09-13T14:42:49.469616+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-13T14:42:50.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:50 smithi099 bash[87147]: cluster 2024-09-13T14:42:49.469616+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-13T14:42:50.827 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [6870776~15886120,23836099~4227684,44219630~803772,64579445~6333027,90842417~1915499,132351677~3360072,179806240~7716975,196783755~39519,197409751~530452,208345786~7311803,222718339~5152715,243273429~1227157,260623526~1785751,268435456~5585076,283257723~2752147,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~9481574,363131961~1946490,373574339~7928023,399621917~2387960] 2024-09-13T14:42:50.827 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 16 diff [2195112~2825918,6870776~15886120,23836099~4227684,44219630~803772,64579445~10686594,90842417~1915499,132351677~3360072,141404955~444745,162734548~2602407,179806240~7716975,196783755~39519,197409751~530452,208345786~7311803,222718339~5152715,243273429~1227157,260623526~1785751,270284536~3735996,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~12396258,364542714~535737,373574339~7928023,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.827 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,44219630~803772,64579445~10686594,90842417~1915499,132351677~3360072,141404955~444745,162734548~2602407,179806240~7716975,196783755~39519,197409751~530452,208345786~7311803,222718339~5152715,243273429~1227157,260623526~1785751,270284536~3735996,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~12396258,364542714~535737,373574339~7928023,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.835 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,44219630~803772,64579445~10686594,90842417~1915499,132351677~3360072,141404955~444745,162734548~2602407,179806240~7716975,196783755~39519,197409751~530452,208345786~7311803,222718339~5152715,243273429~1227157,260623526~1785751,268435456~5585076,283257723~2752147,287173785~3975876,299549539~4652901,321943280~5551255,327957029~5068897,349598203~12396258,363131961~1946490,373574339~7928023,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.835 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 17 diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,64579445~10686594,82473537~389924,90842417~1915499,132351677~3360072,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,208345786~7311803,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,270284536~6896044,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~6431772,327957029~5068897,349598203~12396258,364542714~535737,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.835 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,64579445~10686594,82473537~389924,90842417~1915499,132351677~3360072,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,208345786~7311803,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,270284536~6896044,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~6431772,327957029~5068897,349598203~12396258,364542714~535737,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.841 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,64579445~10686594,82473537~389924,90842417~1915499,132351677~3360072,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,208345786~7311803,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,268435456~8745124,283257723~2752147,287173785~3975876,299549539~4652901,321062763~6431772,327957029~5068897,349598203~12396258,363131961~1946490,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.842 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 18 diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~6431772,327957029~16311065,345304694~16689767,364542714~535737,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.842 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~6431772,327957029~16311065,345304694~16689767,364542714~535737,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.849 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~6801038,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129249663~6462086,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~4652901,321062763~6431772,327957029~16311065,345304694~16689767,363131961~1946490,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.849 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 19 diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.849 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.856 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129249663~6462086,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.856 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 14 diff [13376238~7133337,23836099~4227684,44219630~803772,196783755~39519,197409751~530452,243273429~1227157,287173785~3975876,299549539~4652901,327957029~5068897,373574339~7928023] 2024-09-13T14:42:50.856 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [13376238~7133337,23836099~4227684,44219630~803772,196783755~39519,197409751~530452,243273429~1227157,287173785~3975876,299549539~4652901,327957029~5068897,373574339~7928023] 2024-09-13T14:42:50.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:50 smithi177 bash[68594]: cluster 2024-09-13T14:42:49.469616+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-13T14:42:50.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:50 smithi177 bash[68594]: cluster 2024-09-13T14:42:49.469616+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-13T14:42:50.862 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [13376238~7133337,23836099~4227684,44219630~803772,196783755~39519,197409751~530452,243273429~1227157,287173785~3975876,299549539~4652901,327957029~5068897,373574339~7928023] 2024-09-13T14:42:50.862 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 15 diff [9317267~11192308,23836099~4227684,44219630~803772,90842417~1915499,196783755~39519,197409751~530452,208345786~7311803,243273429~1227157,270284536~3735996,284903576~1106294,287173785~3975876,299549539~4652901,327957029~5068897,349598203~3802019,364542714~535737,373574339~7928023,399621917~2387960] 2024-09-13T14:42:50.862 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [9317267~11192308,23836099~4227684,44219630~803772,90842417~1915499,196783755~39519,197409751~530452,208345786~7311803,243273429~1227157,270284536~3735996,284903576~1106294,287173785~3975876,299549539~4652901,327957029~5068897,349598203~3802019,364542714~535737,373574339~7928023,399621917~2387960] 2024-09-13T14:42:50.868 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [9317267~11192308,23836099~4227684,44219630~803772,90842417~1915499,196783755~39519,197409751~530452,208345786~7311803,243273429~1227157,268435456~5585076,283257723~2752147,287173785~3975876,299549539~4652901,327957029~5068897,349598203~3802019,363131961~1946490,373574339~7928023,399621917~2387960] 2024-09-13T14:42:50.868 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 16 diff [2195112~2825918,9317267~11192308,23836099~4227684,44219630~803772,67408128~7857911,90842417~1915499,141404955~444745,162734548~2602407,196783755~39519,197409751~530452,208345786~7311803,243273429~1227157,270284536~3735996,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,322101591~969515,327957029~5068897,349598203~3802019,353805247~8189214,364542714~535737,373574339~7928023,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.869 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,9317267~11192308,23836099~4227684,44219630~803772,67408128~7857911,90842417~1915499,141404955~444745,162734548~2602407,196783755~39519,197409751~530452,208345786~7311803,243273429~1227157,270284536~3735996,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,322101591~969515,327957029~5068897,349598203~3802019,353805247~8189214,364542714~535737,373574339~7928023,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.876 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,9317267~11192308,23836099~4227684,44219630~803772,67408128~7857911,90842417~1915499,141404955~444745,162734548~2602407,196783755~39519,197409751~530452,208345786~7311803,243273429~1227157,268435456~5585076,283257723~2752147,287173785~3975876,299549539~4652901,322101591~969515,327957029~5068897,349598203~3802019,353805247~8189214,363131961~1946490,373574339~7928023,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.876 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 17 diff [2195112~2825918,9317267~11192308,23836099~4227684,37198352~5213903,42755322~6801038,67408128~7857911,82473537~389924,90842417~1915499,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,208345786~7311803,221004607~1059472,243273429~1227157,270284536~6896044,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~2189426,327957029~5068897,349598203~3802019,353805247~8189214,364542714~535737,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.876 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,9317267~11192308,23836099~4227684,37198352~5213903,42755322~6801038,67408128~7857911,82473537~389924,90842417~1915499,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,208345786~7311803,221004607~1059472,243273429~1227157,270284536~6896044,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~2189426,327957029~5068897,349598203~3802019,353805247~8189214,364542714~535737,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.883 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,9317267~11192308,23836099~4227684,37198352~5213903,42755322~6801038,67408128~7857911,82473537~389924,90842417~1915499,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,208345786~7311803,221004607~1059472,243273429~1227157,268435456~8745124,283257723~2752147,287173785~3975876,299549539~4652901,321062763~2189426,327957029~5068897,349598203~3802019,353805247~8189214,363131961~1946490,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.883 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 18 diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~6801038,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~2189426,327957029~16311065,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.883 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~6801038,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~4652901,321062763~2189426,327957029~16311065,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.889 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~6801038,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283257723~2752147,287173785~3975876,299549539~4652901,321062763~2189426,327957029~16311065,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,373574339~19525852,399621917~2387960] 2024-09-13T14:42:50.889 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 19 diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.890 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.896 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.897 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to 15 diff [9317267~7996811,90842417~1915499,208345786~7311803,270284536~3735996,284903576~1106294,349598203~3802019,364542714~535737,375206719~1944506,399621917~2387960] 2024-09-13T14:42:50.897 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [9317267~7996811,90842417~1915499,208345786~7311803,270284536~3735996,284903576~1106294,349598203~3802019,364542714~535737,375206719~1944506,399621917~2387960] 2024-09-13T14:42:50.904 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [9317267~7996811,90842417~1915499,208345786~7311803,268435456~5585076,283257723~2752147,349598203~3802019,363131961~1946490,375206719~1944506,399621917~2387960] 2024-09-13T14:42:50.904 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to 16 diff [2195112~2825918,9317267~7996811,67408128~7857911,90842417~1915499,141404955~444745,162734548~2602407,208345786~7311803,270284536~3735996,283377402~1311205,284903576~1106294,322101591~969515,349598203~3802019,353805247~8189214,364542714~535737,375206719~1944506,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.904 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,9317267~7996811,67408128~7857911,90842417~1915499,141404955~444745,162734548~2602407,208345786~7311803,270284536~3735996,283377402~1311205,284903576~1106294,322101591~969515,349598203~3802019,353805247~8189214,364542714~535737,375206719~1944506,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.910 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,9317267~7996811,67408128~7857911,90842417~1915499,141404955~444745,162734548~2602407,208345786~7311803,268435456~5585076,283257723~2752147,322101591~969515,349598203~3802019,353805247~8189214,363131961~1946490,375206719~1944506,386940757~6159434,399621917~2387960] 2024-09-13T14:42:50.910 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to 17 diff [2195112~2825918,9317267~7996811,37198352~5213903,42755322~6801038,67408128~7857911,82473537~389924,90842417~1915499,139193095~6833073,162734548~2602407,178949854~4791653,208345786~7311803,221004607~1059472,270284536~6896044,283377402~1311205,284903576~1106294,321062763~2189426,349598203~3802019,353805247~8189214,364542714~535737,375206719~1944506,380109914~12990277,399621917~2387960] 2024-09-13T14:42:50.910 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,9317267~7996811,37198352~5213903,42755322~6801038,67408128~7857911,82473537~389924,90842417~1915499,139193095~6833073,162734548~2602407,178949854~4791653,208345786~7311803,221004607~1059472,270284536~6896044,283377402~1311205,284903576~1106294,321062763~2189426,349598203~3802019,353805247~8189214,364542714~535737,375206719~1944506,380109914~12990277,399621917~2387960] 2024-09-13T14:42:50.916 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,9317267~7996811,37198352~5213903,42755322~6801038,67408128~7857911,82473537~389924,90842417~1915499,139193095~6833073,162734548~2602407,178949854~4791653,208345786~7311803,221004607~1059472,268435456~8745124,283257723~2752147,321062763~2189426,349598203~3802019,353805247~8189214,363131961~1946490,375206719~1944506,380109914~12990277,399621917~2387960] 2024-09-13T14:42:50.916 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to 18 diff [2195112~2825918,7608321~9705757,37198352~5213903,42755322~6801038,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,321062763~2189426,330795407~13472687,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,375206719~1944506,380109914~12990277,399621917~2387960] 2024-09-13T14:42:50.916 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~9705757,37198352~5213903,42755322~6801038,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,321062763~2189426,330795407~13472687,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,375206719~1944506,380109914~12990277,399621917~2387960] 2024-09-13T14:42:50.923 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~9705757,37198352~5213903,42755322~6801038,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283257723~2752147,321062763~2189426,330795407~13472687,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,375206719~1944506,380109914~12990277,399621917~2387960] 2024-09-13T14:42:50.923 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to 19 diff [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.923 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.930 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283257723~2752147,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:50.930 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to 16 diff [2195112~2825918,67408128~7857911,141404955~444745,162734548~2602407,209566024~3865350,283377402~1311205,322101591~969515,353805247~8189214,386940757~6159434] 2024-09-13T14:42:50.930 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,67408128~7857911,141404955~444745,162734548~2602407,209566024~3865350,283377402~1311205,322101591~969515,353805247~8189214,386940757~6159434] 2024-09-13T14:42:50.937 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,67408128~7857911,141404955~444745,162734548~2602407,209566024~3865350,283377402~1311205,322101591~969515,353805247~8189214,386940757~6159434] 2024-09-13T14:42:50.937 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to 17 diff [2195112~2825918,37198352~5213903,42755322~6801038,67408128~7857911,82473537~389924,139193095~6833073,162734548~2602407,178949854~4791653,209566024~3865350,221004607~1059472,273733736~3446844,283377402~1311205,321062763~2189426,353805247~8189214,380109914~12990277] 2024-09-13T14:42:50.937 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,37198352~5213903,42755322~6801038,67408128~7857911,82473537~389924,139193095~6833073,162734548~2602407,178949854~4791653,209566024~3865350,221004607~1059472,273733736~3446844,283377402~1311205,321062763~2189426,353805247~8189214,380109914~12990277] 2024-09-13T14:42:50.944 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,37198352~5213903,42755322~6801038,67408128~7857911,82473537~389924,139193095~6833073,162734548~2602407,178949854~4791653,209566024~3865350,221004607~1059472,273733736~3446844,283377402~1311205,321062763~2189426,353805247~8189214,380109914~12990277] 2024-09-13T14:42:50.945 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to 18 diff [2195112~2825918,7608321~5148929,37198352~5213903,42755322~6801038,57087283~3131861,67408128~7857911,82473537~389924,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,321062763~2189426,330795407~13472687,345304694~6260654,353805247~8189214,366861183~3878902,380109914~12990277] 2024-09-13T14:42:50.945 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~5148929,37198352~5213903,42755322~6801038,57087283~3131861,67408128~7857911,82473537~389924,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,321062763~2189426,330795407~13472687,345304694~6260654,353805247~8189214,366861183~3878902,380109914~12990277] 2024-09-13T14:42:50.952 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~5148929,37198352~5213903,42755322~6801038,57087283~3131861,67408128~7857911,82473537~389924,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,321062763~2189426,330795407~13472687,345304694~6260654,353805247~8189214,366861183~3878902,380109914~12990277] 2024-09-13T14:42:50.952 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to 19 diff [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.952 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.959 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.959 INFO:tasks.workunit.client.0.smithi099.stdout:from 16 to 17 diff [37198352~5213903,42755322~6801038,82473537~389924,139193095~6833073,178949854~4791653,221004607~1059472,273733736~3446844,321062763~2189426,380109914~10082291] 2024-09-13T14:42:50.959 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [37198352~5213903,42755322~6801038,82473537~389924,139193095~6833073,178949854~4791653,221004607~1059472,273733736~3446844,321062763~2189426,380109914~10082291] 2024-09-13T14:42:50.965 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [37198352~5213903,42755322~6801038,82473537~389924,139193095~6833073,178949854~4791653,221004607~1059472,273733736~3446844,321062763~2189426,380109914~10082291] 2024-09-13T14:42:50.965 INFO:tasks.workunit.client.0.smithi099.stdout:from 16 to 18 diff [7608321~5148929,37198352~5213903,42755322~6801038,57087283~3131861,82473537~389924,129289827~5230086,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,321062763~2189426,330795407~13472687,345304694~6260654,366861183~3878902,380109914~10082291] 2024-09-13T14:42:50.965 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [7608321~5148929,37198352~5213903,42755322~6801038,57087283~3131861,82473537~389924,129289827~5230086,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,321062763~2189426,330795407~13472687,345304694~6260654,366861183~3878902,380109914~10082291] 2024-09-13T14:42:50.973 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [7608321~5148929,37198352~5213903,42755322~6801038,57087283~3131861,82473537~389924,129249663~5270250,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,321062763~2189426,330795407~13472687,345304694~6260654,366861183~3878902,380109914~10082291] 2024-09-13T14:42:50.973 INFO:tasks.workunit.client.0.smithi099.stdout:from 16 to 19 diff [7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,82473537~389924,129289827~5230086,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,380109914~12954599,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.973 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,82473537~389924,129289827~5230086,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,380109914~12954599,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.980 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,82473537~389924,129249663~5270250,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,380109914~12954599,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.980 INFO:tasks.workunit.client.0.smithi099.stdout:from 17 to 18 diff [7608321~5148929,57087283~3131861,129289827~5230086,206286216~7363235,264059082~6625470,330795407~13472687,345304694~6260654,366861183~3878902] 2024-09-13T14:42:50.980 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [7608321~5148929,57087283~3131861,129289827~5230086,206286216~7363235,264059082~6625470,330795407~13472687,345304694~6260654,366861183~3878902] 2024-09-13T14:42:50.986 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [7608321~5148929,57087283~3131861,129249663~5270250,206286216~7363235,264059082~6625470,330795407~13472687,345304694~6260654,366861183~3878902] 2024-09-13T14:42:50.986 INFO:tasks.workunit.client.0.smithi099.stdout:from 17 to 19 diff [7608321~5148929,17566233~2418993,47167836~5353769,57087283~3131861,129289827~5230086,206286216~7363235,264059082~6625470,300197640~5022742,313405246~6003536,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,386597183~6467330,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.986 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [7608321~5148929,17566233~2418993,47167836~5353769,57087283~3131861,129289827~5230086,206286216~7363235,264059082~6625470,300197640~5022742,313405246~6003536,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,386597183~6467330,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.992 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [7608321~5148929,17566233~2418993,47167836~5353769,57087283~3131861,129249663~5270250,206286216~7363235,264059082~6625470,300197640~5022742,313405246~6003536,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,386597183~6467330,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.992 INFO:tasks.workunit.client.0.smithi099.stdout:from 18 to 19 diff [17566233~2418993,47167836~5353769,300197640~5022742,313405246~6003536,338181584~6121488,371897357~977757,386597183~6467330,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.992 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [17566233~2418993,47167836~5353769,300197640~5022742,313405246~6003536,338181584~6121488,371897357~977757,386597183~6467330,399959864~2122617,410436271~3113586] 2024-09-13T14:42:50.999 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [17566233~2418993,47167836~5353769,300197640~5022742,313405246~6003536,338181584~6121488,371897357~977757,386597183~6467330,399959864~2122617,410436271~3113586] 2024-09-13T14:42:51.515 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:51.515 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:51.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[84576]: cluster 2024-09-13T14:42:50.373784+0000 mgr.y (mgr.26252) 491 : cluster [DBG] pgmap v862: 105 pgs: 105 active+clean; 885 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 240 KiB/s rd, 47 MiB/s wr, 376 op/s 2024-09-13T14:42:51.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[84576]: cluster 2024-09-13T14:42:50.373784+0000 mgr.y (mgr.26252) 491 : cluster [DBG] pgmap v862: 105 pgs: 105 active+clean; 885 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 240 KiB/s rd, 47 MiB/s wr, 376 op/s 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[84576]: audit 2024-09-13T14:42:50.515354+0000 mon.a (mon.0) 1427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[84576]: audit 2024-09-13T14:42:50.515354+0000 mon.a (mon.0) 1427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[84576]: audit 2024-09-13T14:42:50.515513+0000 mon.a (mon.0) 1428 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[84576]: audit 2024-09-13T14:42:50.515513+0000 mon.a (mon.0) 1428 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[84576]: audit 2024-09-13T14:42:50.515639+0000 mon.a (mon.0) 1429 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[84576]: audit 2024-09-13T14:42:50.515639+0000 mon.a (mon.0) 1429 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[84576]: audit 2024-09-13T14:42:50.515761+0000 mon.a (mon.0) 1430 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]} : dispatch 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[84576]: audit 2024-09-13T14:42:50.515761+0000 mon.a (mon.0) 1430 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]} : dispatch 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[87147]: cluster 2024-09-13T14:42:50.373784+0000 mgr.y (mgr.26252) 491 : cluster [DBG] pgmap v862: 105 pgs: 105 active+clean; 885 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 240 KiB/s rd, 47 MiB/s wr, 376 op/s 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[87147]: cluster 2024-09-13T14:42:50.373784+0000 mgr.y (mgr.26252) 491 : cluster [DBG] pgmap v862: 105 pgs: 105 active+clean; 885 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 240 KiB/s rd, 47 MiB/s wr, 376 op/s 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[87147]: audit 2024-09-13T14:42:50.515354+0000 mon.a (mon.0) 1427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[87147]: audit 2024-09-13T14:42:50.515354+0000 mon.a (mon.0) 1427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[87147]: audit 2024-09-13T14:42:50.515513+0000 mon.a (mon.0) 1428 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[87147]: audit 2024-09-13T14:42:50.515513+0000 mon.a (mon.0) 1428 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[87147]: audit 2024-09-13T14:42:50.515639+0000 mon.a (mon.0) 1429 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[87147]: audit 2024-09-13T14:42:50.515639+0000 mon.a (mon.0) 1429 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[87147]: audit 2024-09-13T14:42:50.515761+0000 mon.a (mon.0) 1430 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]} : dispatch 2024-09-13T14:42:51.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:51 smithi099 bash[87147]: audit 2024-09-13T14:42:50.515761+0000 mon.a (mon.0) 1430 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]} : dispatch 2024-09-13T14:42:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:51 smithi177 bash[68594]: cluster 2024-09-13T14:42:50.373784+0000 mgr.y (mgr.26252) 491 : cluster [DBG] pgmap v862: 105 pgs: 105 active+clean; 885 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 240 KiB/s rd, 47 MiB/s wr, 376 op/s 2024-09-13T14:42:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:51 smithi177 bash[68594]: cluster 2024-09-13T14:42:50.373784+0000 mgr.y (mgr.26252) 491 : cluster [DBG] pgmap v862: 105 pgs: 105 active+clean; 885 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 240 KiB/s rd, 47 MiB/s wr, 376 op/s 2024-09-13T14:42:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:51 smithi177 bash[68594]: audit 2024-09-13T14:42:50.515354+0000 mon.a (mon.0) 1427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:51 smithi177 bash[68594]: audit 2024-09-13T14:42:50.515354+0000 mon.a (mon.0) 1427 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:51 smithi177 bash[68594]: audit 2024-09-13T14:42:50.515513+0000 mon.a (mon.0) 1428 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:51 smithi177 bash[68594]: audit 2024-09-13T14:42:50.515513+0000 mon.a (mon.0) 1428 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:51 smithi177 bash[68594]: audit 2024-09-13T14:42:50.515639+0000 mon.a (mon.0) 1429 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:51 smithi177 bash[68594]: audit 2024-09-13T14:42:50.515639+0000 mon.a (mon.0) 1429 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-09-13T14:42:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:51 smithi177 bash[68594]: audit 2024-09-13T14:42:50.515761+0000 mon.a (mon.0) 1430 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]} : dispatch 2024-09-13T14:42:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:51 smithi177 bash[68594]: audit 2024-09-13T14:42:50.515761+0000 mon.a (mon.0) 1430 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]} : dispatch 2024-09-13T14:42:52.635 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:42:52.635 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:52.635 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:52.655 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:42:52.655 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:52.655 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:52.671 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:52.671 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:52.671 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:52.684 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:52.684 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:52.684 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:52.696 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:42:52.696 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.696 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.708 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.708 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.708 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.718 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.718 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.718 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.728 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.728 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.728 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.736 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.737 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.737 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.744 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.745 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.745 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.752 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.753 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.753 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.762 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.762 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.762 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.769 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129249663~6462086,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.769 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to HEAD diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.769 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.777 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.777 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to HEAD diff [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.777 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.786 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283257723~2752147,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:52.786 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to HEAD diff [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:52.786 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:52.795 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:52.795 INFO:tasks.workunit.client.0.smithi099.stdout:from 16 to HEAD diff [7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,82473537~389924,129289827~5230086,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,380109914~12954599,399959864~2122617,410436271~3113586] 2024-09-13T14:42:52.795 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,82473537~389924,129289827~5230086,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,380109914~12954599,399959864~2122617,410436271~3113586] 2024-09-13T14:42:52.803 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,82473537~389924,129249663~5270250,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,380109914~12954599,399959864~2122617,410436271~3113586] 2024-09-13T14:42:52.803 INFO:tasks.workunit.client.0.smithi099.stdout:from 17 to HEAD diff [7608321~5148929,17566233~2418993,47167836~5353769,57087283~3131861,129289827~5230086,206286216~7363235,264059082~6625470,300197640~5022742,313405246~6003536,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,386597183~6467330,399959864~2122617,410436271~3113586] 2024-09-13T14:42:52.803 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [7608321~5148929,17566233~2418993,47167836~5353769,57087283~3131861,129289827~5230086,206286216~7363235,264059082~6625470,300197640~5022742,313405246~6003536,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,386597183~6467330,399959864~2122617,410436271~3113586] 2024-09-13T14:42:52.812 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [7608321~5148929,17566233~2418993,47167836~5353769,57087283~3131861,129249663~5270250,206286216~7363235,264059082~6625470,300197640~5022742,313405246~6003536,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,386597183~6467330,399959864~2122617,410436271~3113586] 2024-09-13T14:42:52.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[87147]: audit 2024-09-13T14:42:51.491173+0000 mon.a (mon.0) 1431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-09-13T14:42:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[87147]: audit 2024-09-13T14:42:51.491173+0000 mon.a (mon.0) 1431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-09-13T14:42:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[87147]: audit 2024-09-13T14:42:51.491376+0000 mon.a (mon.0) 1432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-09-13T14:42:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[87147]: audit 2024-09-13T14:42:51.491376+0000 mon.a (mon.0) 1432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-09-13T14:42:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[87147]: audit 2024-09-13T14:42:51.491534+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-09-13T14:42:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[87147]: audit 2024-09-13T14:42:51.491534+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-09-13T14:42:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[87147]: audit 2024-09-13T14:42:51.491734+0000 mon.a (mon.0) 1434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-09-13T14:42:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[87147]: audit 2024-09-13T14:42:51.491734+0000 mon.a (mon.0) 1434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-09-13T14:42:52.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[87147]: cluster 2024-09-13T14:42:51.500242+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[87147]: cluster 2024-09-13T14:42:51.500242+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[84576]: audit 2024-09-13T14:42:51.491173+0000 mon.a (mon.0) 1431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[84576]: audit 2024-09-13T14:42:51.491173+0000 mon.a (mon.0) 1431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[84576]: audit 2024-09-13T14:42:51.491376+0000 mon.a (mon.0) 1432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[84576]: audit 2024-09-13T14:42:51.491376+0000 mon.a (mon.0) 1432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[84576]: audit 2024-09-13T14:42:51.491534+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[84576]: audit 2024-09-13T14:42:51.491534+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[84576]: audit 2024-09-13T14:42:51.491734+0000 mon.a (mon.0) 1434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[84576]: audit 2024-09-13T14:42:51.491734+0000 mon.a (mon.0) 1434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[84576]: cluster 2024-09-13T14:42:51.500242+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-13T14:42:52.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:52 smithi099 bash[84576]: cluster 2024-09-13T14:42:51.500242+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-13T14:42:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:52 smithi177 bash[68594]: audit 2024-09-13T14:42:51.491173+0000 mon.a (mon.0) 1431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-09-13T14:42:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:52 smithi177 bash[68594]: audit 2024-09-13T14:42:51.491173+0000 mon.a (mon.0) 1431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-09-13T14:42:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:52 smithi177 bash[68594]: audit 2024-09-13T14:42:51.491376+0000 mon.a (mon.0) 1432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-09-13T14:42:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:52 smithi177 bash[68594]: audit 2024-09-13T14:42:51.491376+0000 mon.a (mon.0) 1432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-09-13T14:42:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:52 smithi177 bash[68594]: audit 2024-09-13T14:42:51.491534+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-09-13T14:42:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:52 smithi177 bash[68594]: audit 2024-09-13T14:42:51.491534+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-09-13T14:42:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:52 smithi177 bash[68594]: audit 2024-09-13T14:42:51.491734+0000 mon.a (mon.0) 1434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-09-13T14:42:52.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:52 smithi177 bash[68594]: audit 2024-09-13T14:42:51.491734+0000 mon.a (mon.0) 1434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-09-13T14:42:52.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:52 smithi177 bash[68594]: cluster 2024-09-13T14:42:51.500242+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-13T14:42:52.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:52 smithi177 bash[68594]: cluster 2024-09-13T14:42:51.500242+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-13T14:42:53.543 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:53.543 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:53.569 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:42:53.569 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:53.569 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:53.581 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:42:53.581 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:53.581 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:53.592 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:53.592 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:53.592 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:53.604 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:53.604 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:53.604 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:53.615 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:42:53.615 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.615 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.626 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.626 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.626 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.634 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.634 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.635 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.643 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.643 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.643 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.650 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.650 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.650 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.657 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.658 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.658 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.665 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.665 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.665 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.672 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.672 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.672 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.680 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129249663~6462086,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.680 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to HEAD diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.680 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.688 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.688 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to HEAD diff [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.688 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.695 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283257723~2752147,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:53.696 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to HEAD diff [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:53.696 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:53.703 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:53.703 INFO:tasks.workunit.client.0.smithi099.stdout:from 16 to HEAD diff [7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,82473537~389924,129289827~5230086,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,380109914~12954599,399959864~2122617,410436271~3113586] 2024-09-13T14:42:53.704 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,82473537~389924,129289827~5230086,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,380109914~12954599,399959864~2122617,410436271~3113586] 2024-09-13T14:42:53.711 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,82473537~389924,129249663~5270250,139193095~6833073,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,366861183~3878902,371897357~977757,380109914~12954599,399959864~2122617,410436271~3113586] 2024-09-13T14:42:53.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:53 smithi099 bash[84576]: cluster 2024-09-13T14:42:52.374602+0000 mgr.y (mgr.26252) 492 : cluster [DBG] pgmap v864: 105 pgs: 105 active+clean; 913 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 107 KiB/s rd, 27 MiB/s wr, 1.16k op/s 2024-09-13T14:42:53.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:53 smithi099 bash[84576]: cluster 2024-09-13T14:42:52.374602+0000 mgr.y (mgr.26252) 492 : cluster [DBG] pgmap v864: 105 pgs: 105 active+clean; 913 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 107 KiB/s rd, 27 MiB/s wr, 1.16k op/s 2024-09-13T14:42:53.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:53 smithi099 bash[84576]: cluster 2024-09-13T14:42:52.512576+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-13T14:42:53.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:53 smithi099 bash[84576]: cluster 2024-09-13T14:42:52.512576+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-13T14:42:53.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:53 smithi099 bash[87147]: cluster 2024-09-13T14:42:52.374602+0000 mgr.y (mgr.26252) 492 : cluster [DBG] pgmap v864: 105 pgs: 105 active+clean; 913 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 107 KiB/s rd, 27 MiB/s wr, 1.16k op/s 2024-09-13T14:42:53.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:53 smithi099 bash[87147]: cluster 2024-09-13T14:42:52.374602+0000 mgr.y (mgr.26252) 492 : cluster [DBG] pgmap v864: 105 pgs: 105 active+clean; 913 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 107 KiB/s rd, 27 MiB/s wr, 1.16k op/s 2024-09-13T14:42:53.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:53 smithi099 bash[87147]: cluster 2024-09-13T14:42:52.512576+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-13T14:42:53.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:53 smithi099 bash[87147]: cluster 2024-09-13T14:42:52.512576+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-13T14:42:53.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:53 smithi177 bash[68594]: cluster 2024-09-13T14:42:52.374602+0000 mgr.y (mgr.26252) 492 : cluster [DBG] pgmap v864: 105 pgs: 105 active+clean; 913 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 107 KiB/s rd, 27 MiB/s wr, 1.16k op/s 2024-09-13T14:42:53.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:53 smithi177 bash[68594]: cluster 2024-09-13T14:42:52.374602+0000 mgr.y (mgr.26252) 492 : cluster [DBG] pgmap v864: 105 pgs: 105 active+clean; 913 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 107 KiB/s rd, 27 MiB/s wr, 1.16k op/s 2024-09-13T14:42:53.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:53 smithi177 bash[68594]: cluster 2024-09-13T14:42:52.512576+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-13T14:42:53.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:53 smithi177 bash[68594]: cluster 2024-09-13T14:42:52.512576+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-13T14:42:54.553 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:54.553 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:54.595 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:42:54.595 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:54.595 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:54.607 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:42:54.607 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:54.607 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:54.617 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:54.618 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:54.618 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:54.627 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:54.627 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:54.627 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:54.637 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:42:54.637 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.637 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.646 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.646 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.646 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.656 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.656 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.656 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.664 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.664 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.664 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.672 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.672 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.672 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.680 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.680 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.680 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.687 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.687 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.687 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.694 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.694 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.694 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.701 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129249663~6462086,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.701 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to HEAD diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.701 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.709 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.709 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to HEAD diff [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.709 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.716 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283257723~2752147,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:54.716 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to HEAD diff [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:54.717 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:54.723 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~5148929,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~7363235,221004607~1059472,264059082~6625470,273733736~3446844,283377402~1311205,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~6260654,353805247~8189214,366861183~3878902,371897357~977757,380109914~12990277,399959864~2122617,410436271~3113586] 2024-09-13T14:42:54.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:54 smithi099 bash[84576]: cluster 2024-09-13T14:42:53.511366+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-13T14:42:54.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:54 smithi099 bash[84576]: cluster 2024-09-13T14:42:53.511366+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-13T14:42:54.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:54 smithi099 bash[87147]: cluster 2024-09-13T14:42:53.511366+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-13T14:42:54.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:54 smithi099 bash[87147]: cluster 2024-09-13T14:42:53.511366+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-13T14:42:54.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:54 smithi177 bash[68594]: cluster 2024-09-13T14:42:53.511366+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-13T14:42:54.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:54 smithi177 bash[68594]: cluster 2024-09-13T14:42:53.511366+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-13T14:42:55.563 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:55.563 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:55.605 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:42:55.605 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:55.605 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:55.616 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:42:55.617 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:55.617 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:55.627 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:55.627 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:55.627 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:55.636 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:55.636 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:55.636 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:55.644 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:42:55.644 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.645 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.653 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.653 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.653 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.662 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.662 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.662 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.669 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.669 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.669 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.677 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.677 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.677 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.685 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.685 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.685 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.692 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.692 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.692 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.699 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.699 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.699 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.706 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129249663~6462086,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.706 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to HEAD diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.706 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.713 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.714 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to HEAD diff [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.714 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283377402~1311205,284903576~1106294,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.721 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~9705757,17566233~2418993,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,206286216~9371373,221004607~1059472,264059082~13121498,283257723~2752147,300197640~5022742,313405246~6003536,321062763~2189426,330795407~13507665,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,375206719~1944506,380109914~12990277,399621917~2460564,410436271~3113586] 2024-09-13T14:42:55.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:55 smithi099 bash[84576]: cluster 2024-09-13T14:42:54.375589+0000 mgr.y (mgr.26252) 493 : cluster [DBG] pgmap v867: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 11 MiB/s wr, 3.91k op/s; 19 MiB/s, 8 objects/s recovering 2024-09-13T14:42:55.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:55 smithi099 bash[84576]: cluster 2024-09-13T14:42:54.375589+0000 mgr.y (mgr.26252) 493 : cluster [DBG] pgmap v867: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 11 MiB/s wr, 3.91k op/s; 19 MiB/s, 8 objects/s recovering 2024-09-13T14:42:55.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:55 smithi099 bash[84576]: cluster 2024-09-13T14:42:54.531562+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-13T14:42:55.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:55 smithi099 bash[84576]: cluster 2024-09-13T14:42:54.531562+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-13T14:42:55.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:55 smithi099 bash[87147]: cluster 2024-09-13T14:42:54.375589+0000 mgr.y (mgr.26252) 493 : cluster [DBG] pgmap v867: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 11 MiB/s wr, 3.91k op/s; 19 MiB/s, 8 objects/s recovering 2024-09-13T14:42:55.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:55 smithi099 bash[87147]: cluster 2024-09-13T14:42:54.375589+0000 mgr.y (mgr.26252) 493 : cluster [DBG] pgmap v867: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 11 MiB/s wr, 3.91k op/s; 19 MiB/s, 8 objects/s recovering 2024-09-13T14:42:55.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:55 smithi099 bash[87147]: cluster 2024-09-13T14:42:54.531562+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-13T14:42:55.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:55 smithi099 bash[87147]: cluster 2024-09-13T14:42:54.531562+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-13T14:42:55.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:55 smithi177 bash[68594]: cluster 2024-09-13T14:42:54.375589+0000 mgr.y (mgr.26252) 493 : cluster [DBG] pgmap v867: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 11 MiB/s wr, 3.91k op/s; 19 MiB/s, 8 objects/s recovering 2024-09-13T14:42:55.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:55 smithi177 bash[68594]: cluster 2024-09-13T14:42:54.375589+0000 mgr.y (mgr.26252) 493 : cluster [DBG] pgmap v867: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 11 MiB/s wr, 3.91k op/s; 19 MiB/s, 8 objects/s recovering 2024-09-13T14:42:55.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:55 smithi177 bash[68594]: cluster 2024-09-13T14:42:54.531562+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-13T14:42:55.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:55 smithi177 bash[68594]: cluster 2024-09-13T14:42:54.531562+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-13T14:42:56.220 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:56.220 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:56.269 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:42:56.269 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:56.269 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:56.279 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:42:56.279 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:56.279 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:56.288 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:56.288 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:56.288 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:56.296 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:56.296 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:56.297 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:56.305 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:42:56.306 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.306 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.313 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.313 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.313 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.322 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.322 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.322 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.330 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.330 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.330 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.337 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.338 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.338 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.345 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.345 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.345 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.353 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.353 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.353 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.361 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.361 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.361 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.369 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129249663~6462086,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.369 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to HEAD diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.369 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129289827~5230086,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.376 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,7608321~12901254,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,67408128~7857911,82473537~389924,90842417~1915499,129249663~5270250,139193095~6833073,162734548~2602407,178949854~4791653,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,243273429~1227157,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~2189426,327957029~16346043,345304694~8095528,353805247~8189214,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:56.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:56 smithi099 bash[84576]: cluster 2024-09-13T14:42:55.536719+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-13T14:42:56.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:56 smithi099 bash[84576]: cluster 2024-09-13T14:42:55.536719+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-13T14:42:56.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:56 smithi099 bash[84576]: cluster 2024-09-13T14:42:56.199227+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-13T14:42:56.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:56 smithi099 bash[84576]: cluster 2024-09-13T14:42:56.199227+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-13T14:42:56.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:56 smithi099 bash[87147]: cluster 2024-09-13T14:42:55.536719+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-13T14:42:56.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:56 smithi099 bash[87147]: cluster 2024-09-13T14:42:55.536719+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-13T14:42:56.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:56 smithi099 bash[87147]: cluster 2024-09-13T14:42:56.199227+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-13T14:42:56.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:56 smithi099 bash[87147]: cluster 2024-09-13T14:42:56.199227+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-13T14:42:56.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:56 smithi177 bash[68594]: cluster 2024-09-13T14:42:55.536719+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-13T14:42:56.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:56 smithi177 bash[68594]: cluster 2024-09-13T14:42:55.536719+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-13T14:42:56.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:56 smithi177 bash[68594]: cluster 2024-09-13T14:42:56.199227+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-13T14:42:56.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:56 smithi177 bash[68594]: cluster 2024-09-13T14:42:56.199227+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-13T14:42:57.244 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:57.244 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:57.276 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:42:57.276 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:57.276 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:57.285 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:42:57.285 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:57.285 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:57.294 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:57.294 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:57.294 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:57.302 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:57.302 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:57.302 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:57.312 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:42:57.312 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.313 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.320 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.320 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.320 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:42:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:42:57] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.51.0" 2024-09-13T14:42:57.327 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.327 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.327 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.334 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.334 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.334 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.341 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.342 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.342 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.349 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.349 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.349 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.356 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.356 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.356 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.364 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.364 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.364 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129289827~6421922,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.371 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,82473537~389924,90842417~1915499,129249663~6462086,139193095~6833073,162734548~2602407,178949854~8573361,196783755~39519,197409751~530452,206286216~9371373,221004607~1059472,222718339~5152715,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:57.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:57 smithi099 bash[84576]: cluster 2024-09-13T14:42:56.376078+0000 mgr.y (mgr.26252) 494 : cluster [DBG] pgmap v871: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 1.4 MiB/s wr, 1.97k op/s; 24 MiB/s, 10 objects/s recovering 2024-09-13T14:42:57.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:57 smithi099 bash[84576]: cluster 2024-09-13T14:42:56.376078+0000 mgr.y (mgr.26252) 494 : cluster [DBG] pgmap v871: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 1.4 MiB/s wr, 1.97k op/s; 24 MiB/s, 10 objects/s recovering 2024-09-13T14:42:57.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:57 smithi099 bash[84576]: cluster 2024-09-13T14:42:57.210724+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-13T14:42:57.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:57 smithi099 bash[84576]: cluster 2024-09-13T14:42:57.210724+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-13T14:42:57.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:57 smithi099 bash[87147]: cluster 2024-09-13T14:42:56.376078+0000 mgr.y (mgr.26252) 494 : cluster [DBG] pgmap v871: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 1.4 MiB/s wr, 1.97k op/s; 24 MiB/s, 10 objects/s recovering 2024-09-13T14:42:57.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:57 smithi099 bash[87147]: cluster 2024-09-13T14:42:56.376078+0000 mgr.y (mgr.26252) 494 : cluster [DBG] pgmap v871: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 1.4 MiB/s wr, 1.97k op/s; 24 MiB/s, 10 objects/s recovering 2024-09-13T14:42:57.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:57 smithi099 bash[87147]: cluster 2024-09-13T14:42:57.210724+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-13T14:42:57.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:57 smithi099 bash[87147]: cluster 2024-09-13T14:42:57.210724+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-13T14:42:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:57 smithi177 bash[68594]: cluster 2024-09-13T14:42:56.376078+0000 mgr.y (mgr.26252) 494 : cluster [DBG] pgmap v871: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 1.4 MiB/s wr, 1.97k op/s; 24 MiB/s, 10 objects/s recovering 2024-09-13T14:42:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:57 smithi177 bash[68594]: cluster 2024-09-13T14:42:56.376078+0000 mgr.y (mgr.26252) 494 : cluster [DBG] pgmap v871: 105 pgs: 105 active+clean; 934 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 1.4 MiB/s wr, 1.97k op/s; 24 MiB/s, 10 objects/s recovering 2024-09-13T14:42:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:57 smithi177 bash[68594]: cluster 2024-09-13T14:42:57.210724+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-13T14:42:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:57 smithi177 bash[68594]: cluster 2024-09-13T14:42:57.210724+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-13T14:42:58.236 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:58.237 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:58.284 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:42:58.285 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:58.285 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:58.294 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:42:58.294 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:58.294 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:58.303 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:58.303 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:58.303 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:58.311 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:58.311 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:58.311 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:58.320 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:42:58.320 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.320 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.328 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.328 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.328 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.336 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.336 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.336 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.343 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.343 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.343 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.351 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.351 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.351 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.358 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.359 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.359 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.366 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.366 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.366 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:58.373 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,81203912~656330,82473537~389924,87193675~5564241,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.248 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:59.249 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:59.296 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:42:59.296 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:59.296 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:42:59.306 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:42:59.306 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:59.306 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:59.315 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:59.315 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:59.315 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:42:59.323 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:42:59.323 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:59.323 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:42:59.331 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:42:59.331 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.332 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.339 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.339 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.339 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.346 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.347 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.347 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.354 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.354 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.354 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.361 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.361 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.361 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.368 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.369 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.369 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~13121498,283377402~1311205,284903576~1106294,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.376 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~2825918,6870776~15886120,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~6884463,87193675~5564241,99104350~2644211,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,196783755~39519,197409751~530452,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~13121498,283257723~2752147,287173785~3975876,297590923~1695684,299549539~5670843,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:42:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:59 smithi099 bash[84576]: cluster 2024-09-13T14:42:58.213813+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-13T14:42:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:59 smithi099 bash[84576]: cluster 2024-09-13T14:42:58.213813+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-13T14:42:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:59 smithi099 bash[84576]: cluster 2024-09-13T14:42:58.376533+0000 mgr.y (mgr.26252) 495 : cluster [DBG] pgmap v874: 105 pgs: 105 active+clean; 913 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 138 KiB/s rd, 6.0 KiB/s wr, 1.50k op/s 2024-09-13T14:42:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:42:59 smithi099 bash[84576]: cluster 2024-09-13T14:42:58.376533+0000 mgr.y (mgr.26252) 495 : cluster [DBG] pgmap v874: 105 pgs: 105 active+clean; 913 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 138 KiB/s rd, 6.0 KiB/s wr, 1.50k op/s 2024-09-13T14:42:59.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:59 smithi099 bash[87147]: cluster 2024-09-13T14:42:58.213813+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-13T14:42:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:59 smithi099 bash[87147]: cluster 2024-09-13T14:42:58.213813+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-13T14:42:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:59 smithi099 bash[87147]: cluster 2024-09-13T14:42:58.376533+0000 mgr.y (mgr.26252) 495 : cluster [DBG] pgmap v874: 105 pgs: 105 active+clean; 913 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 138 KiB/s rd, 6.0 KiB/s wr, 1.50k op/s 2024-09-13T14:42:59.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:42:59 smithi099 bash[87147]: cluster 2024-09-13T14:42:58.376533+0000 mgr.y (mgr.26252) 495 : cluster [DBG] pgmap v874: 105 pgs: 105 active+clean; 913 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 138 KiB/s rd, 6.0 KiB/s wr, 1.50k op/s 2024-09-13T14:42:59.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:59 smithi177 bash[68594]: cluster 2024-09-13T14:42:58.213813+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-13T14:42:59.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:59 smithi177 bash[68594]: cluster 2024-09-13T14:42:58.213813+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-13T14:42:59.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:59 smithi177 bash[68594]: cluster 2024-09-13T14:42:58.376533+0000 mgr.y (mgr.26252) 495 : cluster [DBG] pgmap v874: 105 pgs: 105 active+clean; 913 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 138 KiB/s rd, 6.0 KiB/s wr, 1.50k op/s 2024-09-13T14:42:59.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:42:59 smithi177 bash[68594]: cluster 2024-09-13T14:42:58.376533+0000 mgr.y (mgr.26252) 495 : cluster [DBG] pgmap v874: 105 pgs: 105 active+clean; 913 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 138 KiB/s rd, 6.0 KiB/s wr, 1.50k op/s 2024-09-13T14:43:00.260 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:00.260 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:00.300 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:43:00.301 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:00.301 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:00.311 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:43:00.311 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:00.312 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:00.320 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:00.320 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:00.320 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:00.328 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:00.328 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:00.329 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:00.336 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:43:00.336 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.336 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.344 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.344 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.344 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.352 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.352 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.352 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.358 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.359 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.359 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.366 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.366 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.366 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,364542714~535737,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.373 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~2581660,37198352~5213903,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,176994513~13446551,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321062763~6431772,327957029~16346043,345304694~16689767,363131961~1946490,366861183~3878902,371897357~977757,373574339~19525852,399621917~2460564,410436271~3113586] 2024-09-13T14:43:00.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:00 smithi099 bash[84576]: cluster 2024-09-13T14:42:59.225150+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-13T14:43:00.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:00 smithi099 bash[84576]: cluster 2024-09-13T14:42:59.225150+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-13T14:43:00.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:00 smithi099 bash[87147]: cluster 2024-09-13T14:42:59.225150+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-13T14:43:00.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:00 smithi099 bash[87147]: cluster 2024-09-13T14:42:59.225150+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-13T14:43:00.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:00 smithi177 bash[68594]: cluster 2024-09-13T14:42:59.225150+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-13T14:43:00.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:00 smithi177 bash[68594]: cluster 2024-09-13T14:42:59.225150+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-13T14:43:00.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:43:00 smithi177 bash[61439]: ts=2024-09-13T14:43:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:01.243 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:01.243 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:01.276 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:43:01.276 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:01.276 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:01.284 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:43:01.284 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:01.284 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:01.292 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:01.292 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:01.292 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:01.299 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:01.299 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:01.299 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:01.306 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:43:01.306 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.306 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.313 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.313 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.313 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.320 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.320 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.320 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.328 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.328 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.328 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~853772,127749407~1500256,129289827~6421922,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,203924372~11733217,221004607~1059472,222718339~7067063,243273429~1227157,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.334 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [2195112~20561784,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~853772,127749407~7962342,139193095~6833073,162734548~2602407,165534750~3793518,175109128~15331936,194953710~5225570,201326592~14330997,221004607~1059472,222718339~7067063,243273429~1227157,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:01.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[84576]: cluster 2024-09-13T14:43:00.224154+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-13T14:43:01.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[84576]: cluster 2024-09-13T14:43:00.224154+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-13T14:43:01.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[84576]: cluster 2024-09-13T14:43:00.377006+0000 mgr.y (mgr.26252) 496 : cluster [DBG] pgmap v877: 105 pgs: 105 active+clean; 874 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 137 KiB/s rd, 7.0 KiB/s wr, 2.17k op/s 2024-09-13T14:43:01.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[84576]: cluster 2024-09-13T14:43:00.377006+0000 mgr.y (mgr.26252) 496 : cluster [DBG] pgmap v877: 105 pgs: 105 active+clean; 874 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 137 KiB/s rd, 7.0 KiB/s wr, 2.17k op/s 2024-09-13T14:43:01.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[84576]: cluster 2024-09-13T14:43:01.199748+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-13T14:43:01.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[84576]: cluster 2024-09-13T14:43:01.199748+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-13T14:43:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[87147]: cluster 2024-09-13T14:43:00.224154+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-13T14:43:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[87147]: cluster 2024-09-13T14:43:00.224154+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-13T14:43:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[87147]: cluster 2024-09-13T14:43:00.377006+0000 mgr.y (mgr.26252) 496 : cluster [DBG] pgmap v877: 105 pgs: 105 active+clean; 874 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 137 KiB/s rd, 7.0 KiB/s wr, 2.17k op/s 2024-09-13T14:43:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[87147]: cluster 2024-09-13T14:43:00.377006+0000 mgr.y (mgr.26252) 496 : cluster [DBG] pgmap v877: 105 pgs: 105 active+clean; 874 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 137 KiB/s rd, 7.0 KiB/s wr, 2.17k op/s 2024-09-13T14:43:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[87147]: cluster 2024-09-13T14:43:01.199748+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-13T14:43:01.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:01 smithi099 bash[87147]: cluster 2024-09-13T14:43:01.199748+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-13T14:43:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:01 smithi177 bash[68594]: cluster 2024-09-13T14:43:00.224154+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-13T14:43:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:01 smithi177 bash[68594]: cluster 2024-09-13T14:43:00.224154+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-13T14:43:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:01 smithi177 bash[68594]: cluster 2024-09-13T14:43:00.377006+0000 mgr.y (mgr.26252) 496 : cluster [DBG] pgmap v877: 105 pgs: 105 active+clean; 874 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 137 KiB/s rd, 7.0 KiB/s wr, 2.17k op/s 2024-09-13T14:43:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:01 smithi177 bash[68594]: cluster 2024-09-13T14:43:00.377006+0000 mgr.y (mgr.26252) 496 : cluster [DBG] pgmap v877: 105 pgs: 105 active+clean; 874 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 137 KiB/s rd, 7.0 KiB/s wr, 2.17k op/s 2024-09-13T14:43:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:01 smithi177 bash[68594]: cluster 2024-09-13T14:43:01.199748+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-13T14:43:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:01 smithi177 bash[68594]: cluster 2024-09-13T14:43:01.199748+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-13T14:43:02.271 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:02.272 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:02.330 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:43:02.330 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:02.330 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:02.340 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:43:02.340 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:02.340 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:02.348 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:02.349 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:02.349 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:02.357 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:02.357 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:02.357 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:02.365 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:43:02.365 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:02.365 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:02.374 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:02.374 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:02.374 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:02.381 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:02.381 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:02.381 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,283377402~1311205,284903576~1106294,287173785~3975876,294322084~10898298,313405246~6003536,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:02.389 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~8710148,42755322~9766283,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~9242632,165534750~3793518,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,283257723~2752147,287173785~3975876,293655082~11565300,313405246~6003536,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:03.299 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:03.299 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:03.339 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:43:03.340 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:03.340 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:03.348 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:43:03.348 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:03.348 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:03.356 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:03.356 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:03.356 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:03.363 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:03.363 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:03.363 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:03.370 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:43:03.370 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:03.370 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:03.377 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:03.377 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:03.377 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~1785751,264059082~15686768,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:03.384 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~1785751,264059082~15686768,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:03 smithi099 bash[84576]: cluster 2024-09-13T14:43:02.256058+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-13T14:43:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:03 smithi099 bash[84576]: cluster 2024-09-13T14:43:02.256058+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-13T14:43:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:03 smithi099 bash[84576]: cluster 2024-09-13T14:43:02.377581+0000 mgr.y (mgr.26252) 497 : cluster [DBG] pgmap v880: 105 pgs: 105 active+clean; 837 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 1.04k op/s 2024-09-13T14:43:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:03 smithi099 bash[84576]: cluster 2024-09-13T14:43:02.377581+0000 mgr.y (mgr.26252) 497 : cluster [DBG] pgmap v880: 105 pgs: 105 active+clean; 837 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 1.04k op/s 2024-09-13T14:43:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:03 smithi099 bash[87147]: cluster 2024-09-13T14:43:02.256058+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-13T14:43:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:03 smithi099 bash[87147]: cluster 2024-09-13T14:43:02.256058+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-13T14:43:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:03 smithi099 bash[87147]: cluster 2024-09-13T14:43:02.377581+0000 mgr.y (mgr.26252) 497 : cluster [DBG] pgmap v880: 105 pgs: 105 active+clean; 837 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 1.04k op/s 2024-09-13T14:43:03.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:03 smithi099 bash[87147]: cluster 2024-09-13T14:43:02.377581+0000 mgr.y (mgr.26252) 497 : cluster [DBG] pgmap v880: 105 pgs: 105 active+clean; 837 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 1.04k op/s 2024-09-13T14:43:03.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:03 smithi177 bash[68594]: cluster 2024-09-13T14:43:02.256058+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-13T14:43:03.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:03 smithi177 bash[68594]: cluster 2024-09-13T14:43:02.256058+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-13T14:43:03.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:03 smithi177 bash[68594]: cluster 2024-09-13T14:43:02.377581+0000 mgr.y (mgr.26252) 497 : cluster [DBG] pgmap v880: 105 pgs: 105 active+clean; 837 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 1.04k op/s 2024-09-13T14:43:03.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:03 smithi177 bash[68594]: cluster 2024-09-13T14:43:02.377581+0000 mgr.y (mgr.26252) 497 : cluster [DBG] pgmap v880: 105 pgs: 105 active+clean; 837 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 1.04k op/s 2024-09-13T14:43:04.307 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:04.307 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:04.342 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:43:04.342 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:04.342 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:04.350 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:43:04.350 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:04.350 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:04.358 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:04.359 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:04.359 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:04.367 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:04.367 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:04.367 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:04.375 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:43:04.375 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:04.376 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~15317969,96810451~4938110,105957581~4841492,117570724~7412468,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~23302505,345304694~17827267,364542714~535737,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:04.383 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~17260444,96810451~4938110,105957581~4841492,117570724~7412468,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~23302505,345304694~19773757,366123856~26976335,399621917~2460564,410436271~3113586] 2024-09-13T14:43:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:04 smithi099 bash[84576]: cluster 2024-09-13T14:43:03.266909+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-13T14:43:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:04 smithi099 bash[84576]: cluster 2024-09-13T14:43:03.266909+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-13T14:43:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:04 smithi099 bash[84576]: audit 2024-09-13T14:43:03.268674+0000 mon.a (mon.0) 1448 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:04 smithi099 bash[84576]: audit 2024-09-13T14:43:03.268674+0000 mon.a (mon.0) 1448 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:04 smithi099 bash[87147]: cluster 2024-09-13T14:43:03.266909+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-13T14:43:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:04 smithi099 bash[87147]: cluster 2024-09-13T14:43:03.266909+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-13T14:43:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:04 smithi099 bash[87147]: audit 2024-09-13T14:43:03.268674+0000 mon.a (mon.0) 1448 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:04 smithi099 bash[87147]: audit 2024-09-13T14:43:03.268674+0000 mon.a (mon.0) 1448 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:04 smithi177 bash[68594]: cluster 2024-09-13T14:43:03.266909+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-13T14:43:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:04 smithi177 bash[68594]: cluster 2024-09-13T14:43:03.266909+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-13T14:43:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:04 smithi177 bash[68594]: audit 2024-09-13T14:43:03.268674+0000 mon.a (mon.0) 1448 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:04 smithi177 bash[68594]: audit 2024-09-13T14:43:03.268674+0000 mon.a (mon.0) 1448 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:05.329 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:05.329 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:05.368 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:43:05.369 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:05.369 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:05.378 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:43:05.378 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:05.378 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:05.386 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:05.387 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:05.387 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:05.394 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:05.394 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:05.394 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~4841492,115764491~9218701,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~26976335,399621917~5183307,410436271~3113586] 2024-09-13T14:43:05.401 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,105957581~4841492,115764491~9218701,127749407~7962342,139193095~6833073,149400161~2594737,156094323~15666407,175109128~15331936,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~26976335,398458880~6346344,410436271~3113586] 2024-09-13T14:43:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:05 smithi099 bash[84576]: cluster 2024-09-13T14:43:04.272673+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-13T14:43:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:05 smithi099 bash[84576]: cluster 2024-09-13T14:43:04.272673+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-13T14:43:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:05 smithi099 bash[84576]: cluster 2024-09-13T14:43:04.378077+0000 mgr.y (mgr.26252) 498 : cluster [DBG] pgmap v883: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.31k op/s 2024-09-13T14:43:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:05 smithi099 bash[84576]: cluster 2024-09-13T14:43:04.378077+0000 mgr.y (mgr.26252) 498 : cluster [DBG] pgmap v883: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.31k op/s 2024-09-13T14:43:05.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:05 smithi099 bash[87147]: cluster 2024-09-13T14:43:04.272673+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-13T14:43:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:05 smithi099 bash[87147]: cluster 2024-09-13T14:43:04.272673+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-13T14:43:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:05 smithi099 bash[87147]: cluster 2024-09-13T14:43:04.378077+0000 mgr.y (mgr.26252) 498 : cluster [DBG] pgmap v883: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.31k op/s 2024-09-13T14:43:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:05 smithi099 bash[87147]: cluster 2024-09-13T14:43:04.378077+0000 mgr.y (mgr.26252) 498 : cluster [DBG] pgmap v883: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.31k op/s 2024-09-13T14:43:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:05 smithi177 bash[68594]: cluster 2024-09-13T14:43:04.272673+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-13T14:43:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:05 smithi177 bash[68594]: cluster 2024-09-13T14:43:04.272673+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-13T14:43:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:05 smithi177 bash[68594]: cluster 2024-09-13T14:43:04.378077+0000 mgr.y (mgr.26252) 498 : cluster [DBG] pgmap v883: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.31k op/s 2024-09-13T14:43:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:05 smithi177 bash[68594]: cluster 2024-09-13T14:43:04.378077+0000 mgr.y (mgr.26252) 498 : cluster [DBG] pgmap v883: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.31k op/s 2024-09-13T14:43:06.218 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:06.218 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:06.275 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:43:06.275 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:06.275 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:06.284 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:43:06.284 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:06.284 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:06.293 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:06.294 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:06.294 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~6421922,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6639063,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~8297412,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:06.301 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~7962342,139193095~6833073,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7241296,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~9082333,250331944~6999104,260623526~19122324,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:06 smithi099 bash[84576]: cluster 2024-09-13T14:43:05.295563+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-13T14:43:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:06 smithi099 bash[84576]: cluster 2024-09-13T14:43:05.295563+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-13T14:43:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:06 smithi099 bash[84576]: cluster 2024-09-13T14:43:06.202612+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-13T14:43:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:06 smithi099 bash[84576]: cluster 2024-09-13T14:43:06.202612+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-13T14:43:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:06 smithi099 bash[87147]: cluster 2024-09-13T14:43:05.295563+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-13T14:43:06.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:06 smithi099 bash[87147]: cluster 2024-09-13T14:43:05.295563+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-13T14:43:06.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:06 smithi099 bash[87147]: cluster 2024-09-13T14:43:06.202612+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-13T14:43:06.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:06 smithi099 bash[87147]: cluster 2024-09-13T14:43:06.202612+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-13T14:43:06.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:06 smithi177 bash[68594]: cluster 2024-09-13T14:43:05.295563+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-13T14:43:06.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:06 smithi177 bash[68594]: cluster 2024-09-13T14:43:05.295563+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-13T14:43:06.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:06 smithi177 bash[68594]: cluster 2024-09-13T14:43:06.202612+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-13T14:43:06.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:06 smithi177 bash[68594]: cluster 2024-09-13T14:43:06.202612+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-13T14:43:07.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:43:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:43:07] "GET /metrics HTTP/1.1" 200 36704 "" "Prometheus/2.51.0" 2024-09-13T14:43:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:07 smithi099 bash[87147]: cluster 2024-09-13T14:43:06.378708+0000 mgr.y (mgr.26252) 499 : cluster [DBG] pgmap v886: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 992 op/s 2024-09-13T14:43:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:07 smithi099 bash[87147]: cluster 2024-09-13T14:43:06.378708+0000 mgr.y (mgr.26252) 499 : cluster [DBG] pgmap v886: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 992 op/s 2024-09-13T14:43:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:07 smithi099 bash[84576]: cluster 2024-09-13T14:43:06.378708+0000 mgr.y (mgr.26252) 499 : cluster [DBG] pgmap v886: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 992 op/s 2024-09-13T14:43:07.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:07 smithi099 bash[84576]: cluster 2024-09-13T14:43:06.378708+0000 mgr.y (mgr.26252) 499 : cluster [DBG] pgmap v886: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 992 op/s 2024-09-13T14:43:07.347 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:07.347 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:07.373 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:43:07.374 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:07.374 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:07.383 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:43:07.383 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:07.383 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~18340726,96810451~4938110,105957581~19025611,127749407~1500256,129289827~16736341,149400161~2594737,152231961~2578327,156094323~34346741,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,410436271~3113586] 2024-09-13T14:43:07.394 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~20283201,96810451~4938110,104857600~20125592,127749407~18276761,149400161~2594737,152231961~2578327,156094323~34346741,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~8955015,321000567~44077884,366123856~31522735,398458880~6346344,410436271~3113586] 2024-09-13T14:43:07.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:07 smithi177 bash[68594]: cluster 2024-09-13T14:43:06.378708+0000 mgr.y (mgr.26252) 499 : cluster [DBG] pgmap v886: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 992 op/s 2024-09-13T14:43:07.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:07 smithi177 bash[68594]: cluster 2024-09-13T14:43:06.378708+0000 mgr.y (mgr.26252) 499 : cluster [DBG] pgmap v886: 105 pgs: 105 active+clean; 745 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 992 op/s 2024-09-13T14:43:08.343 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:08.343 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:08.379 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:43:08.379 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:08.379 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,77439947~26485320,105957581~19025611,127749407~1500256,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,221004607~1059472,222718339~7067063,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:08.388 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,64579445~10686594,75497472~28427795,104857600~20125592,127749407~18276761,149400161~41040903,192937984~7452113,201326592~14330997,221004607~1059472,222718339~7067063,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~6346344,405003943~8545914] 2024-09-13T14:43:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:08 smithi099 bash[84576]: cluster 2024-09-13T14:43:07.303708+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-13T14:43:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:08 smithi099 bash[84576]: cluster 2024-09-13T14:43:07.303708+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-13T14:43:08.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:08 smithi099 bash[87147]: cluster 2024-09-13T14:43:07.303708+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-13T14:43:08.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:08 smithi099 bash[87147]: cluster 2024-09-13T14:43:07.303708+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-13T14:43:08.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:08 smithi177 bash[68594]: cluster 2024-09-13T14:43:07.303708+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-13T14:43:08.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:08 smithi177 bash[68594]: cluster 2024-09-13T14:43:07.303708+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-13T14:43:09.354 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:09.354 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [160166~22596730,23836099~4227684,33702107~21180178,57087283~3131861,63605087~11660952,77439947~26485320,105957581~23292082,129289827~16736341,149400161~41040903,193540217~6849880,203924372~11733217,219834611~9950791,231958926~3416707,236203174~9483457,245777751~2963301,250358186~6972862,260623526~19122324,282281434~976289,283377402~1311205,284903576~1106294,287173785~6481297,294322084~10898298,310453767~8955015,321000567~42131394,364542714~535737,366123856~31522735,399621917~5183307,405003943~8545914] 2024-09-13T14:43:09.403 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~22756896,23836099~4227684,33702107~21180178,57087283~3131861,62914560~12351479,75497472~28427795,104857600~41168568,146800640~43640424,192937984~7452113,201326592~14330997,218103808~11681594,231958926~3416707,235418253~10268378,245777751~2963301,250331944~6999104,260046848~19699002,281018368~24202014,310453767~54624684,366123856~31522735,398458880~15090977] 2024-09-13T14:43:09.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:09 smithi177 bash[68594]: cluster 2024-09-13T14:43:08.307436+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-13T14:43:09.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:09 smithi177 bash[68594]: cluster 2024-09-13T14:43:08.307436+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-13T14:43:09.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:09 smithi177 bash[68594]: cluster 2024-09-13T14:43:08.379394+0000 mgr.y (mgr.26252) 500 : cluster [DBG] pgmap v889: 105 pgs: 105 active+clean; 647 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 67 KiB/s rd, 5.0 KiB/s wr, 457 op/s 2024-09-13T14:43:09.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:09 smithi177 bash[68594]: cluster 2024-09-13T14:43:08.379394+0000 mgr.y (mgr.26252) 500 : cluster [DBG] pgmap v889: 105 pgs: 105 active+clean; 647 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 67 KiB/s rd, 5.0 KiB/s wr, 457 op/s 2024-09-13T14:43:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:09 smithi099 bash[84576]: cluster 2024-09-13T14:43:08.307436+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-13T14:43:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:09 smithi099 bash[84576]: cluster 2024-09-13T14:43:08.307436+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-13T14:43:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:09 smithi099 bash[84576]: cluster 2024-09-13T14:43:08.379394+0000 mgr.y (mgr.26252) 500 : cluster [DBG] pgmap v889: 105 pgs: 105 active+clean; 647 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 67 KiB/s rd, 5.0 KiB/s wr, 457 op/s 2024-09-13T14:43:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:09 smithi099 bash[84576]: cluster 2024-09-13T14:43:08.379394+0000 mgr.y (mgr.26252) 500 : cluster [DBG] pgmap v889: 105 pgs: 105 active+clean; 647 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 67 KiB/s rd, 5.0 KiB/s wr, 457 op/s 2024-09-13T14:43:09.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:09 smithi099 bash[87147]: cluster 2024-09-13T14:43:08.307436+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-13T14:43:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:09 smithi099 bash[87147]: cluster 2024-09-13T14:43:08.307436+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-13T14:43:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:09 smithi099 bash[87147]: cluster 2024-09-13T14:43:08.379394+0000 mgr.y (mgr.26252) 500 : cluster [DBG] pgmap v889: 105 pgs: 105 active+clean; 647 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 67 KiB/s rd, 5.0 KiB/s wr, 457 op/s 2024-09-13T14:43:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:09 smithi099 bash[87147]: cluster 2024-09-13T14:43:08.379394+0000 mgr.y (mgr.26252) 500 : cluster [DBG] pgmap v889: 105 pgs: 105 active+clean; 647 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 67 KiB/s rd, 5.0 KiB/s wr, 457 op/s 2024-09-13T14:43:10.384 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterateStress (40953 ms) 2024-09-13T14:43:10.384 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterateRegression6926 2024-09-13T14:43:10.384 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:43:10.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:43:10 smithi177 bash[61439]: ts=2024-09-13T14:43:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:10.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:10 smithi177 bash[68594]: cluster 2024-09-13T14:43:09.335117+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-13T14:43:10.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:10 smithi177 bash[68594]: cluster 2024-09-13T14:43:09.335117+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-13T14:43:10.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:10 smithi099 bash[84576]: cluster 2024-09-13T14:43:09.335117+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-13T14:43:10.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:10 smithi099 bash[84576]: cluster 2024-09-13T14:43:09.335117+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-13T14:43:10.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:10 smithi099 bash[87147]: cluster 2024-09-13T14:43:09.335117+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-13T14:43:10.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:10 smithi099 bash[87147]: cluster 2024-09-13T14:43:09.335117+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-13T14:43:11.242 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~256 2024-09-13T14:43:11.253 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterateRegression6926 (869 ms) 2024-09-13T14:43:11.253 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParent 2024-09-13T14:43:11.253 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:43:11.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:11 smithi177 bash[68594]: cluster 2024-09-13T14:43:10.332342+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-13T14:43:11.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:11 smithi177 bash[68594]: cluster 2024-09-13T14:43:10.332342+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-13T14:43:11.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:11 smithi177 bash[68594]: cluster 2024-09-13T14:43:10.380056+0000 mgr.y (mgr.26252) 501 : cluster [DBG] pgmap v892: 105 pgs: 105 active+clean; 559 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 668 op/s 2024-09-13T14:43:11.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:11 smithi177 bash[68594]: cluster 2024-09-13T14:43:10.380056+0000 mgr.y (mgr.26252) 501 : cluster [DBG] pgmap v892: 105 pgs: 105 active+clean; 559 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 668 op/s 2024-09-13T14:43:11.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:11 smithi177 bash[68594]: cluster 2024-09-13T14:43:11.206445+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-13T14:43:11.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:11 smithi177 bash[68594]: cluster 2024-09-13T14:43:11.206445+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-13T14:43:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[84576]: cluster 2024-09-13T14:43:10.332342+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-13T14:43:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[84576]: cluster 2024-09-13T14:43:10.332342+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-13T14:43:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[84576]: cluster 2024-09-13T14:43:10.380056+0000 mgr.y (mgr.26252) 501 : cluster [DBG] pgmap v892: 105 pgs: 105 active+clean; 559 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 668 op/s 2024-09-13T14:43:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[84576]: cluster 2024-09-13T14:43:10.380056+0000 mgr.y (mgr.26252) 501 : cluster [DBG] pgmap v892: 105 pgs: 105 active+clean; 559 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 668 op/s 2024-09-13T14:43:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[84576]: cluster 2024-09-13T14:43:11.206445+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-13T14:43:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[84576]: cluster 2024-09-13T14:43:11.206445+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-13T14:43:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[87147]: cluster 2024-09-13T14:43:10.332342+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-13T14:43:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[87147]: cluster 2024-09-13T14:43:10.332342+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-13T14:43:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[87147]: cluster 2024-09-13T14:43:10.380056+0000 mgr.y (mgr.26252) 501 : cluster [DBG] pgmap v892: 105 pgs: 105 active+clean; 559 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 668 op/s 2024-09-13T14:43:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[87147]: cluster 2024-09-13T14:43:10.380056+0000 mgr.y (mgr.26252) 501 : cluster [DBG] pgmap v892: 105 pgs: 105 active+clean; 559 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 668 op/s 2024-09-13T14:43:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[87147]: cluster 2024-09-13T14:43:11.206445+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-13T14:43:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:11 smithi099 bash[87147]: cluster 2024-09-13T14:43:11.206445+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-13T14:43:12.523 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~4194304 2024-09-13T14:43:12.523 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 4194304~4194304 2024-09-13T14:43:12.524 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 8388608~4194304 2024-09-13T14:43:12.524 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 12582912~4194304 2024-09-13T14:43:12.524 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 16777216~4194304 2024-09-13T14:43:12.599 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~4194304 2024-09-13T14:43:12.600 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 4194304~4194304 2024-09-13T14:43:12.600 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 8388608~2097152 2024-09-13T14:43:12.600 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 16777216~4194304 2024-09-13T14:43:12.628 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterateParent (1375 ms) 2024-09-13T14:43:12.628 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterateIgnoreParent 2024-09-13T14:43:12.629 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:43:13.656 INFO:tasks.workunit.client.0.smithi099.stdout: wrote [1341559~100206,6177870~100289,6898556~93247,8437650~79905,8621206~85324,10851813~90807,11447149~132806,12330483~24791,15961077~54939] to clone 2024-09-13T14:43:13.657 INFO:tasks.workunit.client.0.smithi099.stdout: diff was [0~1441765,4194304~2797499,8388608~3966666,12582912~3433104] 2024-09-13T14:43:13.666 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterateIgnoreParent (1038 ms) 2024-09-13T14:43:13.666 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterateCallbackError 2024-09-13T14:43:13.666 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:43:13.723 INFO:tasks.workunit.client.0.smithi099.stdout: wrote [1300737~62054,3647778~101305,7440134~6102,8827482~97705,12673973~33625,12912812~70370,15248221~70307,15445871~60627,16838651~91850,18143153~84800] 2024-09-13T14:43:13.752 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterateCallbackError (86 ms) 2024-09-13T14:43:13.752 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParentDiscard 2024-09-13T14:43:13.753 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:43:13.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:13 smithi099 bash[84576]: cluster 2024-09-13T14:43:12.363365+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-13T14:43:13.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:13 smithi099 bash[84576]: cluster 2024-09-13T14:43:12.363365+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-13T14:43:13.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:13 smithi099 bash[84576]: cluster 2024-09-13T14:43:12.380992+0000 mgr.y (mgr.26252) 502 : cluster [DBG] pgmap v895: 105 pgs: 105 active+clean; 523 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 KiB/s wr, 257 op/s 2024-09-13T14:43:13.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:13 smithi099 bash[84576]: cluster 2024-09-13T14:43:12.380992+0000 mgr.y (mgr.26252) 502 : cluster [DBG] pgmap v895: 105 pgs: 105 active+clean; 523 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 KiB/s wr, 257 op/s 2024-09-13T14:43:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:13 smithi099 bash[87147]: cluster 2024-09-13T14:43:12.363365+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-13T14:43:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:13 smithi099 bash[87147]: cluster 2024-09-13T14:43:12.363365+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-13T14:43:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:13 smithi099 bash[87147]: cluster 2024-09-13T14:43:12.380992+0000 mgr.y (mgr.26252) 502 : cluster [DBG] pgmap v895: 105 pgs: 105 active+clean; 523 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 KiB/s wr, 257 op/s 2024-09-13T14:43:13.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:13 smithi099 bash[87147]: cluster 2024-09-13T14:43:12.380992+0000 mgr.y (mgr.26252) 502 : cluster [DBG] pgmap v895: 105 pgs: 105 active+clean; 523 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 KiB/s wr, 257 op/s 2024-09-13T14:43:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:13 smithi177 bash[68594]: cluster 2024-09-13T14:43:12.363365+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-13T14:43:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:13 smithi177 bash[68594]: cluster 2024-09-13T14:43:12.363365+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-13T14:43:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:13 smithi177 bash[68594]: cluster 2024-09-13T14:43:12.380992+0000 mgr.y (mgr.26252) 502 : cluster [DBG] pgmap v895: 105 pgs: 105 active+clean; 523 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 KiB/s wr, 257 op/s 2024-09-13T14:43:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:13 smithi177 bash[68594]: cluster 2024-09-13T14:43:12.380992+0000 mgr.y (mgr.26252) 502 : cluster [DBG] pgmap v895: 105 pgs: 105 active+clean; 523 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 KiB/s wr, 257 op/s 2024-09-13T14:43:14.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:14 smithi099 bash[84576]: cluster 2024-09-13T14:43:13.378521+0000 mon.a (mon.0) 1458 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-13T14:43:14.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:14 smithi099 bash[84576]: cluster 2024-09-13T14:43:13.378521+0000 mon.a (mon.0) 1458 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-13T14:43:14.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:14 smithi099 bash[87147]: cluster 2024-09-13T14:43:13.378521+0000 mon.a (mon.0) 1458 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-13T14:43:14.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:14 smithi099 bash[87147]: cluster 2024-09-13T14:43:13.378521+0000 mon.a (mon.0) 1458 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-13T14:43:14.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:14 smithi177 bash[68594]: cluster 2024-09-13T14:43:13.378521+0000 mon.a (mon.0) 1458 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-13T14:43:14.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:14 smithi177 bash[68594]: cluster 2024-09-13T14:43:13.378521+0000 mon.a (mon.0) 1458 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-13T14:43:15.632 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterateParentDiscard (1880 ms) 2024-09-13T14:43:15.632 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnalignedSmall 2024-09-13T14:43:15.632 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:43:15.738 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 5000005~1234 2024-09-13T14:43:15.747 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnalignedSmall (115 ms) 2024-09-13T14:43:15.747 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnaligned 2024-09-13T14:43:15.747 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:43:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:15 smithi099 bash[84576]: cluster 2024-09-13T14:43:14.382102+0000 mgr.y (mgr.26252) 503 : cluster [DBG] pgmap v897: 105 pgs: 105 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 6.0 MiB/s wr, 199 op/s 2024-09-13T14:43:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:15 smithi099 bash[84576]: cluster 2024-09-13T14:43:14.382102+0000 mgr.y (mgr.26252) 503 : cluster [DBG] pgmap v897: 105 pgs: 105 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 6.0 MiB/s wr, 199 op/s 2024-09-13T14:43:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:15 smithi099 bash[84576]: cluster 2024-09-13T14:43:14.385540+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-13T14:43:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:15 smithi099 bash[84576]: cluster 2024-09-13T14:43:14.385540+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-13T14:43:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:15 smithi099 bash[87147]: cluster 2024-09-13T14:43:14.382102+0000 mgr.y (mgr.26252) 503 : cluster [DBG] pgmap v897: 105 pgs: 105 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 6.0 MiB/s wr, 199 op/s 2024-09-13T14:43:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:15 smithi099 bash[87147]: cluster 2024-09-13T14:43:14.382102+0000 mgr.y (mgr.26252) 503 : cluster [DBG] pgmap v897: 105 pgs: 105 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 6.0 MiB/s wr, 199 op/s 2024-09-13T14:43:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:15 smithi099 bash[87147]: cluster 2024-09-13T14:43:14.385540+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-13T14:43:15.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:15 smithi099 bash[87147]: cluster 2024-09-13T14:43:14.385540+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-13T14:43:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:15 smithi177 bash[68594]: cluster 2024-09-13T14:43:14.382102+0000 mgr.y (mgr.26252) 503 : cluster [DBG] pgmap v897: 105 pgs: 105 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 6.0 MiB/s wr, 199 op/s 2024-09-13T14:43:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:15 smithi177 bash[68594]: cluster 2024-09-13T14:43:14.382102+0000 mgr.y (mgr.26252) 503 : cluster [DBG] pgmap v897: 105 pgs: 105 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 6.0 MiB/s wr, 199 op/s 2024-09-13T14:43:15.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:15 smithi177 bash[68594]: cluster 2024-09-13T14:43:14.385540+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-13T14:43:15.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:15 smithi177 bash[68594]: cluster 2024-09-13T14:43:14.385540+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-13T14:43:15.893 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 8376263~12345 2024-09-13T14:43:15.893 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 8388608~4194304 2024-09-13T14:43:15.893 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 12582912~54321 2024-09-13T14:43:15.906 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnaligned (159 ms) 2024-09-13T14:43:15.907 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStriping 2024-09-13T14:43:15.907 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:43:15.907 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/0.DiffIterateStriping (0 ms) 2024-09-13T14:43:16.212 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 11 tests from DiffIterateTest/0 (54543 ms total) 2024-09-13T14:43:16.212 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:43:16.212 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 11 tests from DiffIterateTest/1, where TypeParam = DiffIterateParams 2024-09-13T14:43:16.269 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterate 2024-09-13T14:43:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:16 smithi099 bash[84576]: cluster 2024-09-13T14:43:15.400140+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-13T14:43:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:16 smithi099 bash[84576]: cluster 2024-09-13T14:43:15.400140+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-13T14:43:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:16 smithi099 bash[84576]: cluster 2024-09-13T14:43:16.209611+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-13T14:43:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:16 smithi099 bash[84576]: cluster 2024-09-13T14:43:16.209611+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-13T14:43:16.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:16 smithi099 bash[87147]: cluster 2024-09-13T14:43:15.400140+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-13T14:43:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:16 smithi099 bash[87147]: cluster 2024-09-13T14:43:15.400140+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-13T14:43:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:16 smithi099 bash[87147]: cluster 2024-09-13T14:43:16.209611+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-13T14:43:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:16 smithi099 bash[87147]: cluster 2024-09-13T14:43:16.209611+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-13T14:43:16.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:16 smithi177 bash[68594]: cluster 2024-09-13T14:43:15.400140+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-13T14:43:16.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:16 smithi177 bash[68594]: cluster 2024-09-13T14:43:15.400140+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-13T14:43:16.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:16 smithi177 bash[68594]: cluster 2024-09-13T14:43:16.209611+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-13T14:43:16.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:16 smithi177 bash[68594]: cluster 2024-09-13T14:43:16.209611+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-13T14:43:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:43:17] "GET /metrics HTTP/1.1" 200 36704 "" "Prometheus/2.51.0" 2024-09-13T14:43:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[84576]: cluster 2024-09-13T14:43:16.382666+0000 mgr.y (mgr.26252) 504 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[84576]: cluster 2024-09-13T14:43:16.382666+0000 mgr.y (mgr.26252) 504 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[84576]: cluster 2024-09-13T14:43:17.217517+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-13T14:43:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[84576]: cluster 2024-09-13T14:43:17.217517+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-13T14:43:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[84576]: audit 2024-09-13T14:43:17.220477+0000 mon.a (mon.0) 1463 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:43:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[84576]: audit 2024-09-13T14:43:17.220477+0000 mon.a (mon.0) 1463 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:43:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[87147]: cluster 2024-09-13T14:43:16.382666+0000 mgr.y (mgr.26252) 504 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[87147]: cluster 2024-09-13T14:43:16.382666+0000 mgr.y (mgr.26252) 504 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[87147]: cluster 2024-09-13T14:43:17.217517+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-13T14:43:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[87147]: cluster 2024-09-13T14:43:17.217517+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-13T14:43:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[87147]: audit 2024-09-13T14:43:17.220477+0000 mon.a (mon.0) 1463 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:43:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:17 smithi099 bash[87147]: audit 2024-09-13T14:43:17.220477+0000 mon.a (mon.0) 1463 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:43:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:17 smithi177 bash[68594]: cluster 2024-09-13T14:43:16.382666+0000 mgr.y (mgr.26252) 504 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:17 smithi177 bash[68594]: cluster 2024-09-13T14:43:16.382666+0000 mgr.y (mgr.26252) 504 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:17 smithi177 bash[68594]: cluster 2024-09-13T14:43:17.217517+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-13T14:43:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:17 smithi177 bash[68594]: cluster 2024-09-13T14:43:17.217517+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-13T14:43:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:17 smithi177 bash[68594]: audit 2024-09-13T14:43:17.220477+0000 mon.a (mon.0) 1463 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:43:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:17 smithi177 bash[68594]: audit 2024-09-13T14:43:17.220477+0000 mon.a (mon.0) 1463 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T14:43:18.225 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:43:19.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[84576]: audit 2024-09-13T14:43:18.213614+0000 mon.a (mon.0) 1464 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:43:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[84576]: audit 2024-09-13T14:43:18.213614+0000 mon.a (mon.0) 1464 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:43:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[84576]: cluster 2024-09-13T14:43:18.219985+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-13T14:43:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[84576]: cluster 2024-09-13T14:43:18.219985+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-13T14:43:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[84576]: audit 2024-09-13T14:43:18.257169+0000 mon.a (mon.0) 1466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[84576]: audit 2024-09-13T14:43:18.257169+0000 mon.a (mon.0) 1466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[84576]: cluster 2024-09-13T14:43:18.383371+0000 mgr.y (mgr.26252) 505 : cluster [DBG] pgmap v904: 105 pgs: 14 creating+peering, 18 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[84576]: cluster 2024-09-13T14:43:18.383371+0000 mgr.y (mgr.26252) 505 : cluster [DBG] pgmap v904: 105 pgs: 14 creating+peering, 18 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[87147]: audit 2024-09-13T14:43:18.213614+0000 mon.a (mon.0) 1464 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:43:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[87147]: audit 2024-09-13T14:43:18.213614+0000 mon.a (mon.0) 1464 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:43:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[87147]: cluster 2024-09-13T14:43:18.219985+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-13T14:43:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[87147]: cluster 2024-09-13T14:43:18.219985+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-13T14:43:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[87147]: audit 2024-09-13T14:43:18.257169+0000 mon.a (mon.0) 1466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[87147]: audit 2024-09-13T14:43:18.257169+0000 mon.a (mon.0) 1466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[87147]: cluster 2024-09-13T14:43:18.383371+0000 mgr.y (mgr.26252) 505 : cluster [DBG] pgmap v904: 105 pgs: 14 creating+peering, 18 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:19 smithi099 bash[87147]: cluster 2024-09-13T14:43:18.383371+0000 mgr.y (mgr.26252) 505 : cluster [DBG] pgmap v904: 105 pgs: 14 creating+peering, 18 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:19 smithi177 bash[68594]: audit 2024-09-13T14:43:18.213614+0000 mon.a (mon.0) 1464 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:43:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:19 smithi177 bash[68594]: audit 2024-09-13T14:43:18.213614+0000 mon.a (mon.0) 1464 : audit [INF] from='client.? 172.21.15.99:0/2059157915' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi099-69110-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T14:43:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:19 smithi177 bash[68594]: cluster 2024-09-13T14:43:18.219985+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-13T14:43:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:19 smithi177 bash[68594]: cluster 2024-09-13T14:43:18.219985+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-13T14:43:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:19 smithi177 bash[68594]: audit 2024-09-13T14:43:18.257169+0000 mon.a (mon.0) 1466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:19 smithi177 bash[68594]: audit 2024-09-13T14:43:18.257169+0000 mon.a (mon.0) 1466 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:19 smithi177 bash[68594]: cluster 2024-09-13T14:43:18.383371+0000 mgr.y (mgr.26252) 505 : cluster [DBG] pgmap v904: 105 pgs: 14 creating+peering, 18 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:19 smithi177 bash[68594]: cluster 2024-09-13T14:43:18.383371+0000 mgr.y (mgr.26252) 505 : cluster [DBG] pgmap v904: 105 pgs: 14 creating+peering, 18 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:43:20.329 INFO:tasks.workunit.client.0.smithi099.stdout: wrote [1471843~72583,6609260~2968,11741807~38462,12817729~82464,15123128~35724,16144442~46846,16927369~107200,18904603~84881] 2024-09-13T14:43:20.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:20 smithi099 bash[84576]: cluster 2024-09-13T14:43:19.225149+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-13T14:43:20.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:20 smithi099 bash[84576]: cluster 2024-09-13T14:43:19.225149+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-13T14:43:20.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:20 smithi099 bash[87147]: cluster 2024-09-13T14:43:19.225149+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-13T14:43:20.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:20 smithi099 bash[87147]: cluster 2024-09-13T14:43:19.225149+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-13T14:43:20.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:20 smithi177 bash[68594]: cluster 2024-09-13T14:43:19.225149+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-13T14:43:20.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:20 smithi177 bash[68594]: cluster 2024-09-13T14:43:19.225149+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-13T14:43:20.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:43:20 smithi177 bash[61439]: ts=2024-09-13T14:43:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:21.289 INFO:tasks.workunit.client.0.smithi099.stdout: wrote [0~16777216] 2024-09-13T14:43:21.311 INFO:tasks.workunit.client.0.smithi099.stdout: diff was [0~16777216] 2024-09-13T14:43:21.320 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterate (5052 ms) 2024-09-13T14:43:21.320 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDiscard 2024-09-13T14:43:21.320 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:43:21.380 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~4194304 2024-09-13T14:43:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:21 smithi099 bash[84576]: cluster 2024-09-13T14:43:20.251976+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-13T14:43:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:21 smithi099 bash[84576]: cluster 2024-09-13T14:43:20.251976+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-13T14:43:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:21 smithi099 bash[84576]: cluster 2024-09-13T14:43:20.384070+0000 mgr.y (mgr.26252) 506 : cluster [DBG] pgmap v907: 105 pgs: 14 creating+peering, 6 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:43:21.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:21 smithi099 bash[84576]: cluster 2024-09-13T14:43:20.384070+0000 mgr.y (mgr.26252) 506 : cluster [DBG] pgmap v907: 105 pgs: 14 creating+peering, 6 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:43:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:21 smithi099 bash[87147]: cluster 2024-09-13T14:43:20.251976+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-13T14:43:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:21 smithi099 bash[87147]: cluster 2024-09-13T14:43:20.251976+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-13T14:43:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:21 smithi099 bash[87147]: cluster 2024-09-13T14:43:20.384070+0000 mgr.y (mgr.26252) 506 : cluster [DBG] pgmap v907: 105 pgs: 14 creating+peering, 6 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:43:21.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:21 smithi099 bash[87147]: cluster 2024-09-13T14:43:20.384070+0000 mgr.y (mgr.26252) 506 : cluster [DBG] pgmap v907: 105 pgs: 14 creating+peering, 6 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:43:21.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:21 smithi177 bash[68594]: cluster 2024-09-13T14:43:20.251976+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-13T14:43:21.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:21 smithi177 bash[68594]: cluster 2024-09-13T14:43:20.251976+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-13T14:43:21.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:21 smithi177 bash[68594]: cluster 2024-09-13T14:43:20.384070+0000 mgr.y (mgr.26252) 506 : cluster [DBG] pgmap v907: 105 pgs: 14 creating+peering, 6 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:43:21.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:21 smithi177 bash[68594]: cluster 2024-09-13T14:43:20.384070+0000 mgr.y (mgr.26252) 506 : cluster [DBG] pgmap v907: 105 pgs: 14 creating+peering, 6 unknown, 85 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:43:22.301 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~4194304 2024-09-13T14:43:22.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:22 smithi099 bash[84576]: cluster 2024-09-13T14:43:21.263488+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-13T14:43:22.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:22 smithi099 bash[84576]: cluster 2024-09-13T14:43:21.263488+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-13T14:43:22.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:22 smithi099 bash[87147]: cluster 2024-09-13T14:43:21.263488+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-13T14:43:22.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:22 smithi099 bash[87147]: cluster 2024-09-13T14:43:21.263488+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-13T14:43:22.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:22 smithi177 bash[68594]: cluster 2024-09-13T14:43:21.263488+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-13T14:43:22.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:22 smithi177 bash[68594]: cluster 2024-09-13T14:43:21.263488+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-13T14:43:23.325 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~4194304 2024-09-13T14:43:23.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:23 smithi099 bash[84576]: cluster 2024-09-13T14:43:22.263411+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-13T14:43:23.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:23 smithi099 bash[84576]: cluster 2024-09-13T14:43:22.263411+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-13T14:43:23.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:23 smithi099 bash[84576]: cluster 2024-09-13T14:43:22.384828+0000 mgr.y (mgr.26252) 507 : cluster [DBG] pgmap v910: 105 pgs: 12 creating+peering, 93 active+clean; 6.3 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 68 KiB/s wr, 24 op/s 2024-09-13T14:43:23.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:23 smithi099 bash[84576]: cluster 2024-09-13T14:43:22.384828+0000 mgr.y (mgr.26252) 507 : cluster [DBG] pgmap v910: 105 pgs: 12 creating+peering, 93 active+clean; 6.3 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 68 KiB/s wr, 24 op/s 2024-09-13T14:43:23.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:23 smithi099 bash[87147]: cluster 2024-09-13T14:43:22.263411+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-13T14:43:23.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:23 smithi099 bash[87147]: cluster 2024-09-13T14:43:22.263411+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-13T14:43:23.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:23 smithi099 bash[87147]: cluster 2024-09-13T14:43:22.384828+0000 mgr.y (mgr.26252) 507 : cluster [DBG] pgmap v910: 105 pgs: 12 creating+peering, 93 active+clean; 6.3 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 68 KiB/s wr, 24 op/s 2024-09-13T14:43:23.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:23 smithi099 bash[87147]: cluster 2024-09-13T14:43:22.384828+0000 mgr.y (mgr.26252) 507 : cluster [DBG] pgmap v910: 105 pgs: 12 creating+peering, 93 active+clean; 6.3 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 68 KiB/s wr, 24 op/s 2024-09-13T14:43:23.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:23 smithi177 bash[68594]: cluster 2024-09-13T14:43:22.263411+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-13T14:43:23.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:23 smithi177 bash[68594]: cluster 2024-09-13T14:43:22.263411+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-13T14:43:23.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:23 smithi177 bash[68594]: cluster 2024-09-13T14:43:22.384828+0000 mgr.y (mgr.26252) 507 : cluster [DBG] pgmap v910: 105 pgs: 12 creating+peering, 93 active+clean; 6.3 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 68 KiB/s wr, 24 op/s 2024-09-13T14:43:23.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:23 smithi177 bash[68594]: cluster 2024-09-13T14:43:22.384828+0000 mgr.y (mgr.26252) 507 : cluster [DBG] pgmap v910: 105 pgs: 12 creating+peering, 93 active+clean; 6.3 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 68 KiB/s wr, 24 op/s 2024-09-13T14:43:24.328 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~4194304 2024-09-13T14:43:24.331 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterateDiscard (3011 ms) 2024-09-13T14:43:24.332 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStress 2024-09-13T14:43:24.332 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:43:24.447 INFO:tasks.workunit.client.0.smithi099.stdout: i=0 exists [9183702~1175478,19576690~3218261,59019986~4636126,156127163~1562084,204007606~8149777,216867958~4269614,234443516~51028,235402319~7111213,254106442~1379248] wrote [9183702~1175478,19576690~3218261,59019986~4636126,156127163~1562084,204007606~8149777,216867958~4269614,234443516~51028,235402319~7111213,254106442~1379248] 2024-09-13T14:43:24.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:24 smithi099 bash[84576]: cluster 2024-09-13T14:43:23.286694+0000 mon.a (mon.0) 1471 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-13T14:43:24.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:24 smithi099 bash[84576]: cluster 2024-09-13T14:43:23.286694+0000 mon.a (mon.0) 1471 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-13T14:43:24.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:24 smithi099 bash[87147]: cluster 2024-09-13T14:43:23.286694+0000 mon.a (mon.0) 1471 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-13T14:43:24.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:24 smithi099 bash[87147]: cluster 2024-09-13T14:43:23.286694+0000 mon.a (mon.0) 1471 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-13T14:43:24.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:24 smithi177 bash[68594]: cluster 2024-09-13T14:43:23.286694+0000 mon.a (mon.0) 1471 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-13T14:43:24.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:24 smithi177 bash[68594]: cluster 2024-09-13T14:43:23.286694+0000 mon.a (mon.0) 1471 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-13T14:43:25.368 INFO:tasks.workunit.client.0.smithi099.stdout: i=1 exists [1240760~1503094,2954657~2502425,9183702~1175478,15813014~459096,19576690~3218261,48904902~201521,59019986~4636126,156127163~1562084,204007606~17129966,228560109~680722,232733252~461745,234443516~51028,235402319~7111213,254106442~1379248,277918089~5088372,291345447~1650458,338388621~7287356] wrote [1240760~1503094,2954657~2502425,15813014~459096,48904902~201521,210991008~7394149,228560109~680722,232733252~461745,277918089~5088372,291345447~1650458,338388621~7287356] 2024-09-13T14:43:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:25 smithi099 bash[84576]: cluster 2024-09-13T14:43:24.286356+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-13T14:43:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:25 smithi099 bash[84576]: cluster 2024-09-13T14:43:24.286356+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-13T14:43:25.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:25 smithi099 bash[84576]: cluster 2024-09-13T14:43:24.385606+0000 mgr.y (mgr.26252) 508 : cluster [DBG] pgmap v913: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 254 KiB/s wr, 89 op/s 2024-09-13T14:43:25.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:25 smithi099 bash[84576]: cluster 2024-09-13T14:43:24.385606+0000 mgr.y (mgr.26252) 508 : cluster [DBG] pgmap v913: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 254 KiB/s wr, 89 op/s 2024-09-13T14:43:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:25 smithi099 bash[87147]: cluster 2024-09-13T14:43:24.286356+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-13T14:43:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:25 smithi099 bash[87147]: cluster 2024-09-13T14:43:24.286356+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-13T14:43:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:25 smithi099 bash[87147]: cluster 2024-09-13T14:43:24.385606+0000 mgr.y (mgr.26252) 508 : cluster [DBG] pgmap v913: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 254 KiB/s wr, 89 op/s 2024-09-13T14:43:25.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:25 smithi099 bash[87147]: cluster 2024-09-13T14:43:24.385606+0000 mgr.y (mgr.26252) 508 : cluster [DBG] pgmap v913: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 254 KiB/s wr, 89 op/s 2024-09-13T14:43:25.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:25 smithi177 bash[68594]: cluster 2024-09-13T14:43:24.286356+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-13T14:43:25.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:25 smithi177 bash[68594]: cluster 2024-09-13T14:43:24.286356+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-13T14:43:25.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:25 smithi177 bash[68594]: cluster 2024-09-13T14:43:24.385606+0000 mgr.y (mgr.26252) 508 : cluster [DBG] pgmap v913: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 254 KiB/s wr, 89 op/s 2024-09-13T14:43:25.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:25 smithi177 bash[68594]: cluster 2024-09-13T14:43:24.385606+0000 mgr.y (mgr.26252) 508 : cluster [DBG] pgmap v913: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 254 KiB/s wr, 89 op/s 2024-09-13T14:43:26.314 INFO:tasks.workunit.client.0.smithi099.stdout: i=2 exists [1240760~1503094,2954657~2502425,9183702~1175478,15813014~459096,19576690~6979301,45541550~495906,48904902~201521,59019986~4636126,75580118~3136493,89303140~6009966,156127163~1562084,164131043~6007141,204007606~17129966,228560109~680722,232733252~461745,234443516~51028,235402319~7111213,254106442~1379248,275844644~1817217,277918089~5088372,291345447~1650458,338388621~7287356,372117514~6754920] wrote [21936850~4619141,45541550~495906,75580118~3136493,89303140~6009966,164131043~6007141,275844644~1817217,343976631~1132656,372117514~6754920] 2024-09-13T14:43:26.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:26 smithi099 bash[87147]: cluster 2024-09-13T14:43:25.312354+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-13T14:43:26.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:26 smithi099 bash[87147]: cluster 2024-09-13T14:43:25.312354+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-13T14:43:26.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:26 smithi099 bash[87147]: cluster 2024-09-13T14:43:26.211658+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-13T14:43:26.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:26 smithi099 bash[87147]: cluster 2024-09-13T14:43:26.211658+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-13T14:43:26.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:26 smithi099 bash[84576]: cluster 2024-09-13T14:43:25.312354+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-13T14:43:26.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:26 smithi099 bash[84576]: cluster 2024-09-13T14:43:25.312354+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-13T14:43:26.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:26 smithi099 bash[84576]: cluster 2024-09-13T14:43:26.211658+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-13T14:43:26.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:26 smithi099 bash[84576]: cluster 2024-09-13T14:43:26.211658+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-13T14:43:26.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:26 smithi177 bash[68594]: cluster 2024-09-13T14:43:25.312354+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-13T14:43:26.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:26 smithi177 bash[68594]: cluster 2024-09-13T14:43:25.312354+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-13T14:43:26.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:26 smithi177 bash[68594]: cluster 2024-09-13T14:43:26.211658+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-13T14:43:26.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:26 smithi177 bash[68594]: cluster 2024-09-13T14:43:26.211658+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-13T14:43:27.317 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:43:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:43:27] "GET /metrics HTTP/1.1" 200 36635 "" "Prometheus/2.51.0" 2024-09-13T14:43:27.424 INFO:tasks.workunit.client.0.smithi099.stdout: i=3 exists [1240760~1503094,2954657~2502425,9183702~1175478,15813014~459096,19576690~6979301,45541550~495906,48904902~201521,50976456~2703774,59019986~4636126,75580118~3136493,89303140~6009966,109879649~5800353,123285902~8029731,156127163~1562084,164131043~6007141,194670715~5416884,204007606~23044758,228560109~680722,232733252~461745,234443516~51028,235402319~7111213,251007057~4480060,271641823~736723,275844644~1817217,277918089~5088372,291345447~1650458,315174824~3113227,338388621~7287356,348559598~3112756,372117514~6754920,407371042~6381396] wrote [50976456~2703774,109879649~5800353,123285902~8029731,194670715~5416884,220294377~6757987,251007057~4480060,271641823~736723,315174824~3113227,348559598~3112756,407371042~6381396] 2024-09-13T14:43:27.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:27 smithi099 bash[84576]: cluster 2024-09-13T14:43:26.386326+0000 mgr.y (mgr.26252) 509 : cluster [DBG] pgmap v916: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 186 KiB/s wr, 64 op/s 2024-09-13T14:43:27.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:27 smithi099 bash[84576]: cluster 2024-09-13T14:43:26.386326+0000 mgr.y (mgr.26252) 509 : cluster [DBG] pgmap v916: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 186 KiB/s wr, 64 op/s 2024-09-13T14:43:27.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:27 smithi099 bash[87147]: cluster 2024-09-13T14:43:26.386326+0000 mgr.y (mgr.26252) 509 : cluster [DBG] pgmap v916: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 186 KiB/s wr, 64 op/s 2024-09-13T14:43:27.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:27 smithi099 bash[87147]: cluster 2024-09-13T14:43:26.386326+0000 mgr.y (mgr.26252) 509 : cluster [DBG] pgmap v916: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 186 KiB/s wr, 64 op/s 2024-09-13T14:43:27.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:27 smithi177 bash[68594]: cluster 2024-09-13T14:43:26.386326+0000 mgr.y (mgr.26252) 509 : cluster [DBG] pgmap v916: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 186 KiB/s wr, 64 op/s 2024-09-13T14:43:27.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:27 smithi177 bash[68594]: cluster 2024-09-13T14:43:26.386326+0000 mgr.y (mgr.26252) 509 : cluster [DBG] pgmap v916: 105 pgs: 105 active+clean; 15 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 186 KiB/s wr, 64 op/s 2024-09-13T14:43:28.432 INFO:tasks.workunit.client.0.smithi099.stdout: i=4 exists [1240760~1503094,2954657~2502425,9183702~1175478,15813014~459096,19576690~6979301,45541550~495906,48904902~201521,50976456~2703774,59019986~4636126,73023085~6910019,89303140~6009966,109879649~5800353,123285902~8029731,156127163~1562084,161889085~1563381,164131043~6007141,182058945~6424163,194670715~5416884,204007606~23044758,228560109~680722,232733252~461745,234443516~51028,235402319~7111213,251007057~4480060,271641823~736723,275844644~1817217,277918089~5088372,291345447~1650458,315174824~4884951,338388621~7287356,348559598~3112756,352489940~4479910,367105394~11767040,394389856~2378088,407371042~6381396] wrote [73023085~6910019,161889085~1563381,182058945~6424163,205613579~4538365,213822996~6254592,318151027~1908748,352489940~4479910,367105394~6460445,394389856~2378088] 2024-09-13T14:43:28.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:28 smithi099 bash[84576]: cluster 2024-09-13T14:43:27.328401+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-13T14:43:28.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:28 smithi099 bash[84576]: cluster 2024-09-13T14:43:27.328401+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-13T14:43:28.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:28 smithi099 bash[87147]: cluster 2024-09-13T14:43:27.328401+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-13T14:43:28.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:28 smithi099 bash[87147]: cluster 2024-09-13T14:43:27.328401+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-13T14:43:28.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:28 smithi177 bash[68594]: cluster 2024-09-13T14:43:27.328401+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-13T14:43:28.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:28 smithi177 bash[68594]: cluster 2024-09-13T14:43:27.328401+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-13T14:43:29.455 INFO:tasks.workunit.client.0.smithi099.stdout: i=5 exists [1240760~1503094,2954657~3424292,9183702~1175478,15813014~459096,19576690~6979301,45541550~495906,48904902~201521,50976456~2703774,59019986~4636126,73023085~6910019,89303140~6009966,97890717~4689748,109879649~5800353,123285902~8029731,138540093~6698869,156127163~3042713,161889085~1563381,164131043~6007141,182058945~6424163,194670715~7441555,204007606~26373727,232733252~461745,234443516~51028,235402319~7111213,251007057~4480060,271641823~736723,275844644~1817217,277918089~5088372,291345447~1650458,315174824~4884951,320206943~4753845,338388621~7287356,348559598~3112756,352489940~4479910,367105394~11767040,391771748~1017057,394389856~2378088,407371042~6381396] wrote [4967307~1411642,97890717~4689748,138540093~6698869,157003579~2166297,197217809~4894461,220664050~9717283,320206943~4753845,391771748~1017057] 2024-09-13T14:43:29.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:29 smithi177 bash[68594]: cluster 2024-09-13T14:43:28.337083+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-13T14:43:29.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:29 smithi177 bash[68594]: cluster 2024-09-13T14:43:28.337083+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-13T14:43:29.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:29 smithi177 bash[68594]: cluster 2024-09-13T14:43:28.386984+0000 mgr.y (mgr.26252) 510 : cluster [DBG] pgmap v919: 105 pgs: 105 active+clean; 96 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 16 MiB/s wr, 150 op/s 2024-09-13T14:43:29.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:29 smithi177 bash[68594]: cluster 2024-09-13T14:43:28.386984+0000 mgr.y (mgr.26252) 510 : cluster [DBG] pgmap v919: 105 pgs: 105 active+clean; 96 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 16 MiB/s wr, 150 op/s 2024-09-13T14:43:29.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:29 smithi177 bash[68594]: audit 2024-09-13T14:43:29.219676+0000 mon.a (mon.0) 1477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:29.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:29 smithi177 bash[68594]: audit 2024-09-13T14:43:29.219676+0000 mon.a (mon.0) 1477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[84576]: cluster 2024-09-13T14:43:28.337083+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-13T14:43:29.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[84576]: cluster 2024-09-13T14:43:28.337083+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-13T14:43:29.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[84576]: cluster 2024-09-13T14:43:28.386984+0000 mgr.y (mgr.26252) 510 : cluster [DBG] pgmap v919: 105 pgs: 105 active+clean; 96 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 16 MiB/s wr, 150 op/s 2024-09-13T14:43:29.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[84576]: cluster 2024-09-13T14:43:28.386984+0000 mgr.y (mgr.26252) 510 : cluster [DBG] pgmap v919: 105 pgs: 105 active+clean; 96 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 16 MiB/s wr, 150 op/s 2024-09-13T14:43:29.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[84576]: audit 2024-09-13T14:43:29.219676+0000 mon.a (mon.0) 1477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:29.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[84576]: audit 2024-09-13T14:43:29.219676+0000 mon.a (mon.0) 1477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[87147]: cluster 2024-09-13T14:43:28.337083+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-13T14:43:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[87147]: cluster 2024-09-13T14:43:28.337083+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-13T14:43:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[87147]: cluster 2024-09-13T14:43:28.386984+0000 mgr.y (mgr.26252) 510 : cluster [DBG] pgmap v919: 105 pgs: 105 active+clean; 96 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 16 MiB/s wr, 150 op/s 2024-09-13T14:43:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[87147]: cluster 2024-09-13T14:43:28.386984+0000 mgr.y (mgr.26252) 510 : cluster [DBG] pgmap v919: 105 pgs: 105 active+clean; 96 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 16 MiB/s wr, 150 op/s 2024-09-13T14:43:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[87147]: audit 2024-09-13T14:43:29.219676+0000 mon.a (mon.0) 1477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:29 smithi099 bash[87147]: audit 2024-09-13T14:43:29.219676+0000 mon.a (mon.0) 1477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:30.452 INFO:tasks.workunit.client.0.smithi099.stdout: i=6 exists [492851~203371,1240760~1503094,2954657~3424292,9183702~1175478,15813014~459096,19576690~11152703,45541550~495906,48904902~201521,50976456~2703774,59019986~4636126,73023085~6910019,89303140~6009966,97890717~6024456,109879649~5800353,123285902~8029731,138540093~6698869,155123369~8329097,164131043~13475332,182058945~6424163,194670715~7441555,204007606~26373727,232733252~461745,234443516~51028,235402319~7111213,251007057~4480060,256735477~4600650,271641823~736723,275844644~1817217,277918089~5088372,291345447~5501849,315174824~4884951,320206943~4753845,338388621~7287356,348559598~3112756,352489940~4479910,367105394~11767040,387823657~6044679,394389856~2378088,407371042~6381396] wrote [492851~203371,25484285~5245108,97947396~5967777,155123369~7388223,170053795~7552580,240248150~179187,256735477~4600650,291768558~5078738,387823657~6044679] 2024-09-13T14:43:30.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:30 smithi177 bash[68594]: cluster 2024-09-13T14:43:29.346950+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-13T14:43:30.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:30 smithi177 bash[68594]: cluster 2024-09-13T14:43:29.346950+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-13T14:43:30.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:43:30 smithi177 bash[61439]: ts=2024-09-13T14:43:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:30 smithi099 bash[84576]: cluster 2024-09-13T14:43:29.346950+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-13T14:43:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:30 smithi099 bash[84576]: cluster 2024-09-13T14:43:29.346950+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-13T14:43:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:30 smithi099 bash[87147]: cluster 2024-09-13T14:43:29.346950+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-13T14:43:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:30 smithi099 bash[87147]: cluster 2024-09-13T14:43:29.346950+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-13T14:43:31.479 INFO:tasks.workunit.client.0.smithi099.stdout: i=7 exists [492851~203371,1240760~1503094,2954657~3424292,9183702~1175478,15813014~459096,19576690~11152703,45541550~495906,48904902~201521,50976456~2703774,56371903~7284209,73023085~6910019,89303140~7715251,97890717~6024456,109879649~5800353,123285902~8029731,138540093~6698869,155123369~8329097,164131043~14401524,180363558~1048726,182058945~6424163,194670715~7441555,204007606~26373727,232733252~461745,234443516~51028,235402319~7111213,251007057~4480060,256735477~4600650,271641823~736723,275641179~2020682,277918089~8707449,291345447~5501849,315174824~4884951,320206943~4753845,338388621~7287356,348559598~3112756,352489940~4479910,367105394~11767040,387823657~6044679,394389856~2378088,407371042~6381396] wrote [23746446~4452080,56371903~3571437,90054067~6964324,165401983~5402322,172193274~6339293,180363558~1048726,275641179~1723343,278935981~7689557] 2024-09-13T14:43:31.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:31 smithi177 bash[68594]: cluster 2024-09-13T14:43:30.355692+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-13T14:43:31.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:31 smithi177 bash[68594]: cluster 2024-09-13T14:43:30.355692+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-13T14:43:31.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:31 smithi177 bash[68594]: cluster 2024-09-13T14:43:30.387874+0000 mgr.y (mgr.26252) 511 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 196 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 36 MiB/s wr, 174 op/s 2024-09-13T14:43:31.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:31 smithi177 bash[68594]: cluster 2024-09-13T14:43:30.387874+0000 mgr.y (mgr.26252) 511 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 196 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 36 MiB/s wr, 174 op/s 2024-09-13T14:43:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:31 smithi099 bash[84576]: cluster 2024-09-13T14:43:30.355692+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-13T14:43:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:31 smithi099 bash[84576]: cluster 2024-09-13T14:43:30.355692+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-13T14:43:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:31 smithi099 bash[84576]: cluster 2024-09-13T14:43:30.387874+0000 mgr.y (mgr.26252) 511 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 196 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 36 MiB/s wr, 174 op/s 2024-09-13T14:43:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:31 smithi099 bash[84576]: cluster 2024-09-13T14:43:30.387874+0000 mgr.y (mgr.26252) 511 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 196 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 36 MiB/s wr, 174 op/s 2024-09-13T14:43:31.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:31 smithi099 bash[87147]: cluster 2024-09-13T14:43:30.355692+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-13T14:43:31.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:31 smithi099 bash[87147]: cluster 2024-09-13T14:43:30.355692+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-13T14:43:31.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:31 smithi099 bash[87147]: cluster 2024-09-13T14:43:30.387874+0000 mgr.y (mgr.26252) 511 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 196 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 36 MiB/s wr, 174 op/s 2024-09-13T14:43:31.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:31 smithi099 bash[87147]: cluster 2024-09-13T14:43:30.387874+0000 mgr.y (mgr.26252) 511 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 196 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 36 MiB/s wr, 174 op/s 2024-09-13T14:43:32.477 INFO:tasks.workunit.client.0.smithi099.stdout: i=8 exists [492851~203371,1240760~1503094,2954657~3424292,9183702~1175478,15495149~3844694,19576690~11152703,33141988~3775478,44989342~1182837,48904902~201521,50976456~2703774,56035265~7696126,73023085~6910019,82506278~3779477,89303140~7715251,97890717~6024456,109879649~5800353,123285902~8029731,138540093~6698869,155123369~8329097,164131043~14401524,180363558~1048726,182058945~6424163,194670715~7441555,204007606~26373727,232733252~461745,234443516~51028,235402319~7111213,251007057~4480060,256735477~4600650,271641823~736723,274065303~12560235,291345447~5501849,315174824~4884951,320206943~4753845,338388621~7287356,348559598~3112756,352489940~4479910,363822651~15049783,387823657~6044679,394389856~2378088,407371042~6381396] wrote [15495149~3844694,33141988~3775478,44989342~1182837,56035265~7696126,75589267~3831459,82506278~3779477,219174975~305923,274065303~7237277,322656957~11717,363822651~6436667] 2024-09-13T14:43:32.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:32 smithi099 bash[84576]: cluster 2024-09-13T14:43:31.370513+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-13T14:43:32.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:32 smithi099 bash[84576]: cluster 2024-09-13T14:43:31.370513+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-13T14:43:32.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:32 smithi099 bash[87147]: cluster 2024-09-13T14:43:31.370513+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-13T14:43:32.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:32 smithi099 bash[87147]: cluster 2024-09-13T14:43:31.370513+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-13T14:43:32.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:32 smithi177 bash[68594]: cluster 2024-09-13T14:43:31.370513+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-13T14:43:32.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:32 smithi177 bash[68594]: cluster 2024-09-13T14:43:31.370513+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-13T14:43:33.504 INFO:tasks.workunit.client.0.smithi099.stdout: i=9 exists [492851~203371,1240760~1503094,2954657~3424292,9183702~1175478,15495149~3844694,19576690~11152703,33141988~3775478,37296596~1691836,44989342~8690888,54663189~9068202,73023085~6910019,82506278~4075640,89303140~7715251,97890717~6024456,106919506~8760496,119963058~11352575,138540093~6698869,155123369~8329097,164131043~14401524,180363558~1048726,182058945~6424163,194670715~7441555,204007606~26373727,232733252~461745,234443516~51028,235402319~7111213,251007057~10329070,271641823~736723,274065303~12560235,291345447~5501849,315174824~4884951,320206943~4753845,338388621~7287356,348559598~3112756,352489940~4479910,363822651~15049783,387823657~6044679,394389856~2378088,407371042~6381396] wrote [37296596~1691836,45043619~7217744,54663189~4605901,75945949~2610013,84233045~2348873,106919506~7484582,119963058~7602098,170767973~7136326,196405405~1918104,252240174~4549763] 2024-09-13T14:43:33.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[84576]: cluster 2024-09-13T14:43:32.371376+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-13T14:43:33.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[84576]: cluster 2024-09-13T14:43:32.371376+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-13T14:43:33.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[84576]: cluster 2024-09-13T14:43:32.388671+0000 mgr.y (mgr.26252) 512 : cluster [DBG] pgmap v925: 105 pgs: 105 active+clean; 225 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 26 MiB/s wr, 200 op/s 2024-09-13T14:43:33.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[84576]: cluster 2024-09-13T14:43:32.388671+0000 mgr.y (mgr.26252) 512 : cluster [DBG] pgmap v925: 105 pgs: 105 active+clean; 225 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 26 MiB/s wr, 200 op/s 2024-09-13T14:43:33.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[84576]: audit 2024-09-13T14:43:33.257858+0000 mon.a (mon.0) 1482 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:33.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[84576]: audit 2024-09-13T14:43:33.257858+0000 mon.a (mon.0) 1482 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:33.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[87147]: cluster 2024-09-13T14:43:32.371376+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-13T14:43:33.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[87147]: cluster 2024-09-13T14:43:32.371376+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-13T14:43:33.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[87147]: cluster 2024-09-13T14:43:32.388671+0000 mgr.y (mgr.26252) 512 : cluster [DBG] pgmap v925: 105 pgs: 105 active+clean; 225 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 26 MiB/s wr, 200 op/s 2024-09-13T14:43:33.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[87147]: cluster 2024-09-13T14:43:32.388671+0000 mgr.y (mgr.26252) 512 : cluster [DBG] pgmap v925: 105 pgs: 105 active+clean; 225 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 26 MiB/s wr, 200 op/s 2024-09-13T14:43:33.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[87147]: audit 2024-09-13T14:43:33.257858+0000 mon.a (mon.0) 1482 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:33.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:33 smithi099 bash[87147]: audit 2024-09-13T14:43:33.257858+0000 mon.a (mon.0) 1482 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:33.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:33 smithi177 bash[68594]: cluster 2024-09-13T14:43:32.371376+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-13T14:43:33.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:33 smithi177 bash[68594]: cluster 2024-09-13T14:43:32.371376+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-13T14:43:33.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:33 smithi177 bash[68594]: cluster 2024-09-13T14:43:32.388671+0000 mgr.y (mgr.26252) 512 : cluster [DBG] pgmap v925: 105 pgs: 105 active+clean; 225 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 26 MiB/s wr, 200 op/s 2024-09-13T14:43:33.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:33 smithi177 bash[68594]: cluster 2024-09-13T14:43:32.388671+0000 mgr.y (mgr.26252) 512 : cluster [DBG] pgmap v925: 105 pgs: 105 active+clean; 225 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 26 MiB/s wr, 200 op/s 2024-09-13T14:43:33.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:33 smithi177 bash[68594]: audit 2024-09-13T14:43:33.257858+0000 mon.a (mon.0) 1482 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:33.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:33 smithi177 bash[68594]: audit 2024-09-13T14:43:33.257858+0000 mon.a (mon.0) 1482 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:34.510 INFO:tasks.workunit.client.0.smithi099.stdout: i=10 exists [492851~203371,1240760~1503094,2954657~3424292,8970693~1671488,15495149~3844694,19576690~17340776,37296596~1691836,44989342~8690888,54663189~9068202,73023085~6910019,82506278~4075640,89303140~7715251,97890717~6024456,106919506~8760496,116732308~14583325,138540093~6698869,155123369~8329097,164131043~14401524,180363558~1048726,182058945~6424163,194670715~7441555,204007606~26373727,232733252~461745,234443516~51028,235402319~7111213,251007057~10329070,271641823~736723,274065303~15928875,291345447~5501849,304285207~6983663,315174824~4884951,320206943~4753845,338388621~7287356,348559598~3112756,352489940~4479910,363822651~15049783,380535642~6447324,387823657~6044679,394389856~2378088,407371042~6381396] wrote [8970693~1671488,29956987~4074297,116732308~3974741,196949929~1759324,206880176~6593843,284805280~5188898,304285207~6983663,368203909~5024213,380535642~6447324,394645657~1218335] 2024-09-13T14:43:34.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:34 smithi099 bash[84576]: cluster 2024-09-13T14:43:33.395607+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-13T14:43:34.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:34 smithi099 bash[84576]: cluster 2024-09-13T14:43:33.395607+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-13T14:43:34.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:34 smithi099 bash[87147]: cluster 2024-09-13T14:43:33.395607+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-13T14:43:34.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:34 smithi099 bash[87147]: cluster 2024-09-13T14:43:33.395607+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-13T14:43:34.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:34 smithi177 bash[68594]: cluster 2024-09-13T14:43:33.395607+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-13T14:43:34.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:34 smithi177 bash[68594]: cluster 2024-09-13T14:43:33.395607+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-13T14:43:35.578 INFO:tasks.workunit.client.0.smithi099.stdout: i=11 exists [492851~203371,1240760~1503094,2954657~3424292,8970693~1671488,13275589~23641877,37296596~1691836,44989342~8690888,54663189~9068202,73023085~6910019,82506278~4075640,89303140~14612033,106919506~8760496,116732308~14583325,138540093~6698869,155123369~8329097,164131043~16144674,180363558~1048726,182058945~12136234,194670715~7441555,204007606~26373727,231021199~11551573,251007057~10329070,271641823~736723,274065303~15928875,291345447~5501849,304285207~6983663,315174824~4884951,320206943~4753845,338388621~7287356,348559598~3112756,352489940~4479910,363822651~15049783,380535642~6447324,387823657~6044679,394389856~2378088,407371042~6381396] wrote [13275589~7768740,33060913~3537635,56078694~3944090,90955559~7808959,167447602~5085154,175217056~5058661,186678576~7516603,211768052~5125081,231021199~11551573] 2024-09-13T14:43:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:35 smithi099 bash[84576]: cluster 2024-09-13T14:43:34.389771+0000 mgr.y (mgr.26252) 513 : cluster [DBG] pgmap v927: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 49 MiB/s wr, 279 op/s 2024-09-13T14:43:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:35 smithi099 bash[84576]: cluster 2024-09-13T14:43:34.389771+0000 mgr.y (mgr.26252) 513 : cluster [DBG] pgmap v927: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 49 MiB/s wr, 279 op/s 2024-09-13T14:43:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:35 smithi099 bash[84576]: cluster 2024-09-13T14:43:34.395920+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-13T14:43:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:35 smithi099 bash[84576]: cluster 2024-09-13T14:43:34.395920+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-13T14:43:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:35 smithi099 bash[87147]: cluster 2024-09-13T14:43:34.389771+0000 mgr.y (mgr.26252) 513 : cluster [DBG] pgmap v927: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 49 MiB/s wr, 279 op/s 2024-09-13T14:43:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:35 smithi099 bash[87147]: cluster 2024-09-13T14:43:34.389771+0000 mgr.y (mgr.26252) 513 : cluster [DBG] pgmap v927: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 49 MiB/s wr, 279 op/s 2024-09-13T14:43:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:35 smithi099 bash[87147]: cluster 2024-09-13T14:43:34.395920+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-13T14:43:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:35 smithi099 bash[87147]: cluster 2024-09-13T14:43:34.395920+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-13T14:43:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:35 smithi177 bash[68594]: cluster 2024-09-13T14:43:34.389771+0000 mgr.y (mgr.26252) 513 : cluster [DBG] pgmap v927: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 49 MiB/s wr, 279 op/s 2024-09-13T14:43:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:35 smithi177 bash[68594]: cluster 2024-09-13T14:43:34.389771+0000 mgr.y (mgr.26252) 513 : cluster [DBG] pgmap v927: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 49 MiB/s wr, 279 op/s 2024-09-13T14:43:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:35 smithi177 bash[68594]: cluster 2024-09-13T14:43:34.395920+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-13T14:43:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:35 smithi177 bash[68594]: cluster 2024-09-13T14:43:34.395920+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-13T14:43:36.483 INFO:tasks.workunit.client.0.smithi099.stdout: i=12 exists [492851~203371,1240760~1503094,2954657~3424292,8970693~1671488,13275589~23879942,37296596~1691836,44989342~8690888,54663189~9068202,64155915~316756,73023085~8250660,82506278~4075640,89303140~14612033,106919506~8760496,116732308~14583325,138540093~6698869,155123369~8329097,164131043~16144674,180363558~1048726,182058945~12136234,194670715~7441555,204007606~26373727,231021199~11551573,247365269~2952846,251007057~10329070,271641823~736723,274065303~15928875,291345447~5501849,304285207~6983663,315174824~4884951,320206943~4753845,338388621~7287356,348559598~8410252,363822651~15049783,380535642~6447324,387823657~6044679,394389856~2378088,407371042~6381396] wrote [35367822~1787709,54946082~5388410,64155915~316756,79551301~1722444,102860555~240816,247365269~2952846,253899902~3894849,291584941~4260891,350681274~4184134,371993795~928341] 2024-09-13T14:43:36.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:36 smithi099 bash[84576]: cluster 2024-09-13T14:43:35.409459+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-13T14:43:36.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:36 smithi099 bash[84576]: cluster 2024-09-13T14:43:35.409459+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-13T14:43:36.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:36 smithi099 bash[84576]: audit 2024-09-13T14:43:35.988446+0000 mon.a (mon.0) 1486 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:36.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:36 smithi099 bash[84576]: audit 2024-09-13T14:43:35.988446+0000 mon.a (mon.0) 1486 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:36 smithi099 bash[87147]: cluster 2024-09-13T14:43:35.409459+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-13T14:43:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:36 smithi099 bash[87147]: cluster 2024-09-13T14:43:35.409459+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-13T14:43:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:36 smithi099 bash[87147]: audit 2024-09-13T14:43:35.988446+0000 mon.a (mon.0) 1486 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:36.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:36 smithi099 bash[87147]: audit 2024-09-13T14:43:35.988446+0000 mon.a (mon.0) 1486 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:36.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:36 smithi177 bash[68594]: cluster 2024-09-13T14:43:35.409459+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-13T14:43:36.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:36 smithi177 bash[68594]: cluster 2024-09-13T14:43:35.409459+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-13T14:43:36.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:36 smithi177 bash[68594]: audit 2024-09-13T14:43:35.988446+0000 mon.a (mon.0) 1486 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:36.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:36 smithi177 bash[68594]: audit 2024-09-13T14:43:35.988446+0000 mon.a (mon.0) 1486 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:43:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:43:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:43:37] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.51.0" 2024-09-13T14:43:37.582 INFO:tasks.workunit.client.0.smithi099.stdout: i=13 exists [492851~203371,1240760~1503094,2954657~5156697,8970693~1671488,13275589~23879942,37296596~1691836,44989342~8690888,54663189~13851495,73023085~8250660,82506278~4075640,89303140~14612033,106919506~8760496,116732308~14583325,136810411~8785681,155123369~8329097,164131043~30064136,194670715~7441555,204007606~26373727,231021199~15356162,247365269~2952846,251007057~10329070,271641823~736723,274065303~15928875,291345447~5501849,304285207~6983663,312953203~7158591,320206943~4753845,335541886~10134091,348559598~10530346,363822651~15049783,380535642~6447324,387823657~6044679,394389856~2378088,407371042~6381396] wrote [3828885~4282469,62524964~5989720,136810411~8785681,166018511~687765,175522544~7516975,241119398~5257963,312953203~7158591,335541886~6980903,352154206~6935738] 2024-09-13T14:43:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:37 smithi099 bash[84576]: cluster 2024-09-13T14:43:36.390540+0000 mgr.y (mgr.26252) 514 : cluster [DBG] pgmap v930: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 44 MiB/s wr, 105 op/s 2024-09-13T14:43:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:37 smithi099 bash[84576]: cluster 2024-09-13T14:43:36.390540+0000 mgr.y (mgr.26252) 514 : cluster [DBG] pgmap v930: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 44 MiB/s wr, 105 op/s 2024-09-13T14:43:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:37 smithi099 bash[84576]: cluster 2024-09-13T14:43:36.433263+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-13T14:43:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:37 smithi099 bash[84576]: cluster 2024-09-13T14:43:36.433263+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-13T14:43:37.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:37 smithi099 bash[87147]: cluster 2024-09-13T14:43:36.390540+0000 mgr.y (mgr.26252) 514 : cluster [DBG] pgmap v930: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 44 MiB/s wr, 105 op/s 2024-09-13T14:43:37.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:37 smithi099 bash[87147]: cluster 2024-09-13T14:43:36.390540+0000 mgr.y (mgr.26252) 514 : cluster [DBG] pgmap v930: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 44 MiB/s wr, 105 op/s 2024-09-13T14:43:37.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:37 smithi099 bash[87147]: cluster 2024-09-13T14:43:36.433263+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-13T14:43:37.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:37 smithi099 bash[87147]: cluster 2024-09-13T14:43:36.433263+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-13T14:43:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:37 smithi177 bash[68594]: cluster 2024-09-13T14:43:36.390540+0000 mgr.y (mgr.26252) 514 : cluster [DBG] pgmap v930: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 44 MiB/s wr, 105 op/s 2024-09-13T14:43:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:37 smithi177 bash[68594]: cluster 2024-09-13T14:43:36.390540+0000 mgr.y (mgr.26252) 514 : cluster [DBG] pgmap v930: 105 pgs: 105 active+clean; 414 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 44 MiB/s wr, 105 op/s 2024-09-13T14:43:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:37 smithi177 bash[68594]: cluster 2024-09-13T14:43:36.433263+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-13T14:43:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:37 smithi177 bash[68594]: cluster 2024-09-13T14:43:36.433263+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-13T14:43:38.574 INFO:tasks.workunit.client.0.smithi099.stdout: i=14 exists [492851~203371,1240760~1503094,2954657~5156697,8970693~1671488,13275589~23879942,37296596~1691836,42151267~11528963,54663189~14792248,73023085~8250660,82506278~4075640,89303140~14612033,106919506~8760496,116732308~14583325,136810411~15107618,155123369~8329097,164131043~30064136,194670715~7441555,204007606~26373727,231021199~15356162,247365269~2952846,251007057~13332767,271641823~736723,274065303~15928875,291345447~5501849,304285207~6983663,312953203~7158591,320206943~4753845,335541886~10134091,348559598~10530346,363822651~15049783,380535642~6447324,387823657~10645180,407371042~6381396] wrote [28057318~3828537,42151267~6466744,56710605~5759368,66280053~3175384,111550594~245490,145496573~6421456,190655304~1215364,232667735~1002032,257558328~6781496,393166678~5302159] 2024-09-13T14:43:38.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:38 smithi099 bash[84576]: cluster 2024-09-13T14:43:37.434268+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-13T14:43:38.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:38 smithi099 bash[84576]: cluster 2024-09-13T14:43:37.434268+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-13T14:43:38.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:38 smithi099 bash[87147]: cluster 2024-09-13T14:43:37.434268+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-13T14:43:38.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:38 smithi099 bash[87147]: cluster 2024-09-13T14:43:37.434268+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-13T14:43:38.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:38 smithi177 bash[68594]: cluster 2024-09-13T14:43:37.434268+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-13T14:43:38.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:38 smithi177 bash[68594]: cluster 2024-09-13T14:43:37.434268+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-13T14:43:39.602 INFO:tasks.workunit.client.0.smithi099.stdout: i=15 exists [492851~203371,1240760~1503094,2954657~34200874,37296596~1691836,42151267~11528963,54663189~14792248,73023085~8250660,82506278~4075640,89303140~14612033,106919506~24396127,136810411~15107618,155123369~8329097,164131043~30064136,194670715~7441555,204007606~26373727,231021199~15356162,247365269~2952846,251007057~13332767,271641823~736723,274065303~22781993,304285207~6983663,312953203~7158591,320206943~4753845,330218436~15457541,348559598~10530346,363822651~15049783,380535642~6447324,387823657~10645180,407371042~6381396] wrote [6308600~11319827,45444850~6668375,59145976~1806203,112521434~7205958,147170179~2145747,185168983~3571847,288588179~7052659,330218436~5937670] 2024-09-13T14:43:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:39 smithi099 bash[84576]: cluster 2024-09-13T14:43:38.391406+0000 mgr.y (mgr.26252) 515 : cluster [DBG] pgmap v933: 105 pgs: 105 active+clean; 545 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 KiB/s rd, 29 MiB/s wr, 322 op/s 2024-09-13T14:43:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:39 smithi099 bash[84576]: cluster 2024-09-13T14:43:38.391406+0000 mgr.y (mgr.26252) 515 : cluster [DBG] pgmap v933: 105 pgs: 105 active+clean; 545 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 KiB/s rd, 29 MiB/s wr, 322 op/s 2024-09-13T14:43:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:39 smithi099 bash[84576]: cluster 2024-09-13T14:43:38.454937+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-13T14:43:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:39 smithi099 bash[84576]: cluster 2024-09-13T14:43:38.454937+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-13T14:43:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:39 smithi099 bash[87147]: cluster 2024-09-13T14:43:38.391406+0000 mgr.y (mgr.26252) 515 : cluster [DBG] pgmap v933: 105 pgs: 105 active+clean; 545 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 KiB/s rd, 29 MiB/s wr, 322 op/s 2024-09-13T14:43:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:39 smithi099 bash[87147]: cluster 2024-09-13T14:43:38.391406+0000 mgr.y (mgr.26252) 515 : cluster [DBG] pgmap v933: 105 pgs: 105 active+clean; 545 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 KiB/s rd, 29 MiB/s wr, 322 op/s 2024-09-13T14:43:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:39 smithi099 bash[87147]: cluster 2024-09-13T14:43:38.454937+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-13T14:43:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:39 smithi099 bash[87147]: cluster 2024-09-13T14:43:38.454937+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-13T14:43:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:39 smithi177 bash[68594]: cluster 2024-09-13T14:43:38.391406+0000 mgr.y (mgr.26252) 515 : cluster [DBG] pgmap v933: 105 pgs: 105 active+clean; 545 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 KiB/s rd, 29 MiB/s wr, 322 op/s 2024-09-13T14:43:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:39 smithi177 bash[68594]: cluster 2024-09-13T14:43:38.391406+0000 mgr.y (mgr.26252) 515 : cluster [DBG] pgmap v933: 105 pgs: 105 active+clean; 545 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 KiB/s rd, 29 MiB/s wr, 322 op/s 2024-09-13T14:43:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:39 smithi177 bash[68594]: cluster 2024-09-13T14:43:38.454937+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-13T14:43:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:39 smithi177 bash[68594]: cluster 2024-09-13T14:43:38.454937+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-13T14:43:40.471 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:43:40 smithi177 bash[61439]: ts=2024-09-13T14:43:40.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:40.569 INFO:tasks.workunit.client.0.smithi099.stdout: i=16 exists [492851~203371,1240760~1503094,2954657~34200874,37296596~1691836,42151267~11528963,54663189~16867560,73023085~8250660,82506278~4075640,89303140~14612033,106919506~24396127,136012716~15905313,155123369~8329097,164131043~30064136,194670715~7441555,204007606~26373727,231021199~15356162,247365269~2952846,251007057~13332767,271641823~736723,274065303~22781993,304285207~6983663,312953203~7158591,320206943~4753845,330218436~15457541,348559598~10530346,363822651~15049783,380535642~6447324,387823657~10645180,407371042~6381396] wrote [8718652~6274896,19894821~4288364,51242942~1720472,54701002~6397348,69235285~2295464,136012716~6572912,165514054~3906960,290456459~198456,306910001~3327394,330487571~164614] 2024-09-13T14:43:40.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:40 smithi099 bash[84576]: cluster 2024-09-13T14:43:39.464917+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-13T14:43:40.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:40 smithi099 bash[84576]: cluster 2024-09-13T14:43:39.464917+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-13T14:43:40.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:40 smithi099 bash[87147]: cluster 2024-09-13T14:43:39.464917+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-13T14:43:40.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:40 smithi099 bash[87147]: cluster 2024-09-13T14:43:39.464917+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-13T14:43:40.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:40 smithi177 bash[68594]: cluster 2024-09-13T14:43:39.464917+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-13T14:43:40.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:40 smithi177 bash[68594]: cluster 2024-09-13T14:43:39.464917+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-13T14:43:41.557 INFO:tasks.workunit.client.0.smithi099.stdout: i=17 exists [492851~203371,1240760~1503094,2954657~34200874,37296596~1691836,42151267~11528963,54663189~16867560,73023085~8250660,82506278~4075640,89303140~14612033,106919506~26727178,136012716~15905313,155123369~8329097,164131043~30064136,194670715~7441555,204007606~26373727,230970728~15406633,247365269~2952846,251007057~13332767,271641823~736723,274065303~22781993,303430192~223399,304285207~6983663,312953203~7158591,320206943~4753845,330218436~15457541,348559598~10530346,363822651~15049783,380535642~6447324,387823657~10645180,407371042~6381396] wrote [19053164~6824656,58600976~1053906,77529503~2901030,97852535~5964088,130201816~3444868,172073273~7294011,230970728~4291615,303430192~223399,316271021~2238817] 2024-09-13T14:43:41.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:41 smithi099 bash[87147]: cluster 2024-09-13T14:43:40.392071+0000 mgr.y (mgr.26252) 516 : cluster [DBG] pgmap v936: 105 pgs: 105 active+clean; 639 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 204 KiB/s rd, 53 MiB/s wr, 335 op/s 2024-09-13T14:43:41.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:41 smithi099 bash[87147]: cluster 2024-09-13T14:43:40.392071+0000 mgr.y (mgr.26252) 516 : cluster [DBG] pgmap v936: 105 pgs: 105 active+clean; 639 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 204 KiB/s rd, 53 MiB/s wr, 335 op/s 2024-09-13T14:43:41.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:41 smithi099 bash[87147]: cluster 2024-09-13T14:43:40.482686+0000 mon.a (mon.0) 1491 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-13T14:43:41.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:41 smithi099 bash[87147]: cluster 2024-09-13T14:43:40.482686+0000 mon.a (mon.0) 1491 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-13T14:43:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:41 smithi099 bash[84576]: cluster 2024-09-13T14:43:40.392071+0000 mgr.y (mgr.26252) 516 : cluster [DBG] pgmap v936: 105 pgs: 105 active+clean; 639 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 204 KiB/s rd, 53 MiB/s wr, 335 op/s 2024-09-13T14:43:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:41 smithi099 bash[84576]: cluster 2024-09-13T14:43:40.392071+0000 mgr.y (mgr.26252) 516 : cluster [DBG] pgmap v936: 105 pgs: 105 active+clean; 639 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 204 KiB/s rd, 53 MiB/s wr, 335 op/s 2024-09-13T14:43:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:41 smithi099 bash[84576]: cluster 2024-09-13T14:43:40.482686+0000 mon.a (mon.0) 1491 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-13T14:43:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:41 smithi099 bash[84576]: cluster 2024-09-13T14:43:40.482686+0000 mon.a (mon.0) 1491 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-13T14:43:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:41 smithi177 bash[68594]: cluster 2024-09-13T14:43:40.392071+0000 mgr.y (mgr.26252) 516 : cluster [DBG] pgmap v936: 105 pgs: 105 active+clean; 639 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 204 KiB/s rd, 53 MiB/s wr, 335 op/s 2024-09-13T14:43:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:41 smithi177 bash[68594]: cluster 2024-09-13T14:43:40.392071+0000 mgr.y (mgr.26252) 516 : cluster [DBG] pgmap v936: 105 pgs: 105 active+clean; 639 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 204 KiB/s rd, 53 MiB/s wr, 335 op/s 2024-09-13T14:43:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:41 smithi177 bash[68594]: cluster 2024-09-13T14:43:40.482686+0000 mon.a (mon.0) 1491 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-13T14:43:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:41 smithi177 bash[68594]: cluster 2024-09-13T14:43:40.482686+0000 mon.a (mon.0) 1491 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-13T14:43:42.609 INFO:tasks.workunit.client.0.smithi099.stdout: i=18 exists [492851~203371,1240760~1503094,2954657~38077790,42151267~11528963,54663189~17100261,73023085~8250660,82506278~4075640,89303140~14612033,106919506~26727178,136012716~18419304,155123369~39071810,194670715~7441555,204007606~26373727,230970728~15406633,246618560~17721264,271641823~736723,274065303~22781993,303430192~223399,304285207~6983663,312953203~7158591,320206943~4753845,330218436~15457541,348559598~10530346,363822651~15049783,380535642~6447324,387823657~10645180,407371042~6381396] wrote [20156859~1617649,33464209~7568238,67580327~4183123,148494415~5937605,156234879~8158406,169177247~6942934,178994240~865589,209314833~6721836,246618560~5156517,314497929~4670854] 2024-09-13T14:43:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[87147]: cluster 2024-09-13T14:43:41.477088+0000 mon.a (mon.0) 1492 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-13T14:43:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[87147]: cluster 2024-09-13T14:43:41.477088+0000 mon.a (mon.0) 1492 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-13T14:43:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[87147]: audit 2024-09-13T14:43:41.872154+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[87147]: audit 2024-09-13T14:43:41.872154+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[87147]: audit 2024-09-13T14:43:41.879218+0000 mon.a (mon.0) 1494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[87147]: audit 2024-09-13T14:43:41.879218+0000 mon.a (mon.0) 1494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[87147]: audit 2024-09-13T14:43:42.138283+0000 mon.a (mon.0) 1495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[87147]: audit 2024-09-13T14:43:42.138283+0000 mon.a (mon.0) 1495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[87147]: audit 2024-09-13T14:43:42.147408+0000 mon.a (mon.0) 1496 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[87147]: audit 2024-09-13T14:43:42.147408+0000 mon.a (mon.0) 1496 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[84576]: cluster 2024-09-13T14:43:41.477088+0000 mon.a (mon.0) 1492 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-13T14:43:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[84576]: cluster 2024-09-13T14:43:41.477088+0000 mon.a (mon.0) 1492 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-13T14:43:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[84576]: audit 2024-09-13T14:43:41.872154+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[84576]: audit 2024-09-13T14:43:41.872154+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[84576]: audit 2024-09-13T14:43:41.879218+0000 mon.a (mon.0) 1494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[84576]: audit 2024-09-13T14:43:41.879218+0000 mon.a (mon.0) 1494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[84576]: audit 2024-09-13T14:43:42.138283+0000 mon.a (mon.0) 1495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[84576]: audit 2024-09-13T14:43:42.138283+0000 mon.a (mon.0) 1495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[84576]: audit 2024-09-13T14:43:42.147408+0000 mon.a (mon.0) 1496 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:42 smithi099 bash[84576]: audit 2024-09-13T14:43:42.147408+0000 mon.a (mon.0) 1496 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:42 smithi177 bash[68594]: cluster 2024-09-13T14:43:41.477088+0000 mon.a (mon.0) 1492 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-13T14:43:42.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:42 smithi177 bash[68594]: cluster 2024-09-13T14:43:41.477088+0000 mon.a (mon.0) 1492 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-13T14:43:42.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:42 smithi177 bash[68594]: audit 2024-09-13T14:43:41.872154+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:42 smithi177 bash[68594]: audit 2024-09-13T14:43:41.872154+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:42 smithi177 bash[68594]: audit 2024-09-13T14:43:41.879218+0000 mon.a (mon.0) 1494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:42 smithi177 bash[68594]: audit 2024-09-13T14:43:41.879218+0000 mon.a (mon.0) 1494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:42 smithi177 bash[68594]: audit 2024-09-13T14:43:42.138283+0000 mon.a (mon.0) 1495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:42 smithi177 bash[68594]: audit 2024-09-13T14:43:42.138283+0000 mon.a (mon.0) 1495 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:42 smithi177 bash[68594]: audit 2024-09-13T14:43:42.147408+0000 mon.a (mon.0) 1496 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:42 smithi177 bash[68594]: audit 2024-09-13T14:43:42.147408+0000 mon.a (mon.0) 1496 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:43.627 INFO:tasks.workunit.client.0.smithi099.stdout: i=19 exists [492851~203371,1240760~1503094,2954657~38077790,42151267~11528963,54663189~17100261,73023085~8250660,82506278~4075640,89303140~14612033,106919506~26727178,136012716~18419304,155123369~39071810,194670715~7441555,204007606~26373727,230970728~15406633,246618560~17721264,271641823~736723,274065303~25957237,303430192~223399,304285207~6983663,312953203~7158591,320206943~4753845,330218436~15457541,348559598~10530346,363822651~15049783,380535642~6447324,387823657~10645180,407371042~6381396] wrote [63411733~6503077,158686000~166567,168753796~7127958,182866210~5684951,216768547~4354465,293681478~6341062,391828815~5003356] 2024-09-13T14:43:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[84576]: cluster 2024-09-13T14:43:42.392758+0000 mgr.y (mgr.26252) 517 : cluster [DBG] pgmap v939: 105 pgs: 105 active+clean; 679 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 33 MiB/s wr, 211 op/s 2024-09-13T14:43:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[84576]: cluster 2024-09-13T14:43:42.392758+0000 mgr.y (mgr.26252) 517 : cluster [DBG] pgmap v939: 105 pgs: 105 active+clean; 679 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 33 MiB/s wr, 211 op/s 2024-09-13T14:43:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[84576]: cluster 2024-09-13T14:43:42.497047+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[84576]: cluster 2024-09-13T14:43:42.497047+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[84576]: audit 2024-09-13T14:43:42.825029+0000 mon.a (mon.0) 1498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[84576]: audit 2024-09-13T14:43:42.825029+0000 mon.a (mon.0) 1498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[84576]: audit 2024-09-13T14:43:42.826474+0000 mon.a (mon.0) 1499 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[84576]: audit 2024-09-13T14:43:42.826474+0000 mon.a (mon.0) 1499 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[84576]: audit 2024-09-13T14:43:42.836293+0000 mon.a (mon.0) 1500 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[84576]: audit 2024-09-13T14:43:42.836293+0000 mon.a (mon.0) 1500 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[87147]: cluster 2024-09-13T14:43:42.392758+0000 mgr.y (mgr.26252) 517 : cluster [DBG] pgmap v939: 105 pgs: 105 active+clean; 679 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 33 MiB/s wr, 211 op/s 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[87147]: cluster 2024-09-13T14:43:42.392758+0000 mgr.y (mgr.26252) 517 : cluster [DBG] pgmap v939: 105 pgs: 105 active+clean; 679 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 33 MiB/s wr, 211 op/s 2024-09-13T14:43:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[87147]: cluster 2024-09-13T14:43:42.497047+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-13T14:43:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[87147]: cluster 2024-09-13T14:43:42.497047+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-13T14:43:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[87147]: audit 2024-09-13T14:43:42.825029+0000 mon.a (mon.0) 1498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[87147]: audit 2024-09-13T14:43:42.825029+0000 mon.a (mon.0) 1498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[87147]: audit 2024-09-13T14:43:42.826474+0000 mon.a (mon.0) 1499 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[87147]: audit 2024-09-13T14:43:42.826474+0000 mon.a (mon.0) 1499 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[87147]: audit 2024-09-13T14:43:42.836293+0000 mon.a (mon.0) 1500 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:43.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:43 smithi099 bash[87147]: audit 2024-09-13T14:43:42.836293+0000 mon.a (mon.0) 1500 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:43 smithi177 bash[68594]: cluster 2024-09-13T14:43:42.392758+0000 mgr.y (mgr.26252) 517 : cluster [DBG] pgmap v939: 105 pgs: 105 active+clean; 679 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 33 MiB/s wr, 211 op/s 2024-09-13T14:43:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:43 smithi177 bash[68594]: cluster 2024-09-13T14:43:42.392758+0000 mgr.y (mgr.26252) 517 : cluster [DBG] pgmap v939: 105 pgs: 105 active+clean; 679 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 33 MiB/s wr, 211 op/s 2024-09-13T14:43:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:43 smithi177 bash[68594]: cluster 2024-09-13T14:43:42.497047+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-13T14:43:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:43 smithi177 bash[68594]: cluster 2024-09-13T14:43:42.497047+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-13T14:43:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:43 smithi177 bash[68594]: audit 2024-09-13T14:43:42.825029+0000 mon.a (mon.0) 1498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:43 smithi177 bash[68594]: audit 2024-09-13T14:43:42.825029+0000 mon.a (mon.0) 1498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:43 smithi177 bash[68594]: audit 2024-09-13T14:43:42.826474+0000 mon.a (mon.0) 1499 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:43 smithi177 bash[68594]: audit 2024-09-13T14:43:42.826474+0000 mon.a (mon.0) 1499 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:43 smithi177 bash[68594]: audit 2024-09-13T14:43:42.836293+0000 mon.a (mon.0) 1500 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:43 smithi177 bash[68594]: audit 2024-09-13T14:43:42.836293+0000 mon.a (mon.0) 1500 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:43:44.538 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 1 diff [0~4194304,12582912~4194304,46137344~4194304,209715200~8388608,226492416~8388608,276824064~8388608,289406976~4194304,335544320~8388608] 2024-09-13T14:43:44.538 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,46137344~4194304,209715200~8388608,226492416~8388608,276824064~8388608,289406976~4194304,335544320~8388608] 2024-09-13T14:43:44.564 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~4194304,46137344~4194304,209715200~12582912,226492416~8388608,276824064~8388608,289406976~4194304,335544320~12582912] 2024-09-13T14:43:44.565 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 2 diff [0~4194304,12582912~4194304,20971520~8388608,41943040~8388608,75497472~4194304,88080384~8388608,163577856~8388608,209715200~8388608,226492416~8388608,272629760~12582912,289406976~4194304,335544320~8388608,369098752~8388608] 2024-09-13T14:43:44.565 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,20971520~8388608,41943040~8388608,75497472~4194304,88080384~8388608,163577856~8388608,209715200~8388608,226492416~8388608,272629760~12582912,289406976~4194304,335544320~8388608,369098752~8388608] 2024-09-13T14:43:44.569 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~4194304,20971520~8388608,41943040~8388608,75497472~4194304,88080384~8388608,163577856~8388608,209715200~12582912,226492416~8388608,272629760~12582912,289406976~4194304,335544320~12582912,369098752~12582912] 2024-09-13T14:43:44.569 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 3 diff [0~4194304,12582912~4194304,20971520~8388608,41943040~12582912,75497472~4194304,88080384~8388608,109051904~8388608,121634816~8388608,163577856~8388608,192937984~8388608,209715200~25165824,247463936~8388608,268435456~16777216,289406976~4194304,314572800~4194304,335544320~8388608,348127232~4194304,369098752~8388608,406847488~8388608] 2024-09-13T14:43:44.569 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,20971520~8388608,41943040~12582912,75497472~4194304,88080384~8388608,109051904~8388608,121634816~8388608,163577856~8388608,192937984~8388608,209715200~25165824,247463936~8388608,268435456~16777216,289406976~4194304,314572800~4194304,335544320~8388608,348127232~4194304,369098752~8388608,406847488~8388608] 2024-09-13T14:43:44.575 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~4194304,20971520~8388608,41943040~12582912,75497472~4194304,88080384~8388608,109051904~8388608,121634816~12582912,163577856~8388608,192937984~8388608,209715200~25165824,247463936~8388608,268435456~16777216,289406976~4194304,314572800~4194304,335544320~16777216,369098752~12582912,406847488~8388608] 2024-09-13T14:43:44.575 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 4 diff [0~4194304,12582912~4194304,20971520~8388608,41943040~12582912,71303168~8388608,88080384~8388608,109051904~8388608,121634816~8388608,159383552~12582912,180355072~8388608,192937984~8388608,205520896~29360128,247463936~8388608,268435456~16777216,289406976~4194304,314572800~8388608,335544320~8388608,348127232~12582912,364904448~12582912,394264576~4194304,406847488~8388608] 2024-09-13T14:43:44.575 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,20971520~8388608,41943040~12582912,71303168~8388608,88080384~8388608,109051904~8388608,121634816~8388608,159383552~12582912,180355072~8388608,192937984~8388608,205520896~29360128,247463936~8388608,268435456~16777216,289406976~4194304,314572800~8388608,335544320~8388608,348127232~12582912,364904448~12582912,394264576~4194304,406847488~8388608] 2024-09-13T14:43:44.583 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~4194304,20971520~8388608,41943040~12582912,71303168~12582912,88080384~8388608,109051904~8388608,121634816~12582912,159383552~12582912,180355072~8388608,192937984~8388608,205520896~29360128,247463936~8388608,268435456~16777216,289406976~4194304,314572800~8388608,335544320~25165824,364904448~16777216,394264576~4194304,406847488~8388608] 2024-09-13T14:43:44.583 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 5 diff [0~4194304,12582912~4194304,20971520~8388608,41943040~12582912,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,155189248~16777216,180355072~8388608,192937984~8388608,205520896~29360128,247463936~8388608,268435456~16777216,289406976~4194304,314572800~12582912,335544320~8388608,348127232~12582912,364904448~12582912,390070272~8388608,406847488~8388608] 2024-09-13T14:43:44.583 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,20971520~8388608,41943040~12582912,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,155189248~16777216,180355072~8388608,192937984~8388608,205520896~29360128,247463936~8388608,268435456~16777216,289406976~4194304,314572800~12582912,335544320~8388608,348127232~12582912,364904448~12582912,390070272~8388608,406847488~8388608] 2024-09-13T14:43:44.591 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~4194304,20971520~8388608,41943040~12582912,71303168~12582912,88080384~16777216,109051904~8388608,121634816~12582912,138412032~8388608,155189248~16777216,180355072~8388608,192937984~41943040,247463936~8388608,268435456~16777216,289406976~4194304,314572800~12582912,335544320~25165824,364904448~16777216,390070272~8388608,406847488~8388608] 2024-09-13T14:43:44.591 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 6 diff [0~4194304,12582912~4194304,20971520~12582912,41943040~12582912,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~29360128,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,335544320~8388608,348127232~12582912,364904448~12582912,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.591 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,20971520~12582912,41943040~12582912,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~29360128,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,335544320~8388608,348127232~12582912,364904448~12582912,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.600 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~4194304,20971520~12582912,41943040~12582912,71303168~12582912,88080384~16777216,109051904~8388608,121634816~12582912,138412032~8388608,150994944~37748736,192937984~41943040,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,335544320~25165824,364904448~16777216,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.600 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 7 diff [0~4194304,12582912~4194304,20971520~12582912,41943040~16777216,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~29360128,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,335544320~8388608,348127232~12582912,364904448~12582912,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.600 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,20971520~12582912,41943040~16777216,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~29360128,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,335544320~8388608,348127232~12582912,364904448~12582912,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.610 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~4194304,20971520~12582912,41943040~20971520,71303168~12582912,88080384~16777216,109051904~8388608,121634816~12582912,138412032~8388608,150994944~37748736,192937984~41943040,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,335544320~25165824,364904448~16777216,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.610 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 8 diff [0~4194304,12582912~4194304,20971520~12582912,41943040~20971520,71303168~12582912,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~29360128,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,335544320~8388608,348127232~29360128,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.610 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,20971520~12582912,41943040~20971520,71303168~12582912,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~29360128,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,335544320~8388608,348127232~29360128,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.622 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~25165824,41943040~25165824,71303168~33554432,109051904~8388608,121634816~12582912,138412032~8388608,150994944~37748736,192937984~41943040,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,335544320~46137344,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.623 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 9 diff [0~4194304,12582912~4194304,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~29360128,239075328~4194304,247463936~12582912,268435456~16777216,289406976~8388608,314572800~12582912,335544320~8388608,348127232~29360128,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.623 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~29360128,239075328~4194304,247463936~12582912,268435456~16777216,289406976~8388608,314572800~12582912,335544320~8388608,348127232~29360128,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.636 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~54525952,71303168~62914560,138412032~8388608,150994944~37748736,192937984~41943040,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,335544320~46137344,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.636 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 10 diff [0~4194304,8388608~8388608,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~29360128,239075328~4194304,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,335544320~8388608,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.636 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,8388608~8388608,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~29360128,239075328~4194304,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,335544320~8388608,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.649 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~62914560,138412032~8388608,150994944~37748736,192937984~41943040,239075328~4194304,247463936~16777216,268435456~29360128,301989888~25165824,335544320~62914560,406847488~8388608] 2024-09-13T14:43:44.649 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 11 diff [0~4194304,8388608~29360128,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~37748736,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,335544320~8388608,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.650 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,8388608~29360128,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~37748736,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,335544320~8388608,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.663 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~62914560,138412032~8388608,150994944~92274688,247463936~16777216,268435456~29360128,301989888~25165824,335544320~62914560,406847488~8388608] 2024-09-13T14:43:44.663 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 12 diff [0~4194304,8388608~29360128,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~8388608,314572800~12582912,335544320~8388608,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.663 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,8388608~29360128,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~8388608,314572800~12582912,335544320~8388608,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.679 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~62914560,138412032~8388608,150994944~113246208,268435456~29360128,301989888~25165824,335544320~62914560,406847488~8388608] 2024-09-13T14:43:44.679 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 13 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~12582912,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~25165824,331350016~12582912,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.679 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~12582912,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~25165824,331350016~12582912,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.696 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~146800640,150994944~113246208,268435456~29360128,301989888~25165824,331350016~67108864,406847488~8388608] 2024-09-13T14:43:44.696 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 14 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~25165824,331350016~12582912,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.696 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~25165824,331350016~12582912,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.714 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~25165824,331350016~71303168,406847488~8388608] 2024-09-13T14:43:44.714 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 15 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~41943040,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.714 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~41943040,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.732 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~100663296,406847488~8388608] 2024-09-13T14:43:44.732 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 16 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~41943040,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.732 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~41943040,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.752 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~100663296,406847488~8388608] 2024-09-13T14:43:44.752 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 17 diff [0~37748736,41943040~41943040,88080384~71303168,163577856~37748736,205520896~58720256,268435456~29360128,301989888~41943040,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.752 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~71303168,163577856~37748736,205520896~58720256,268435456~29360128,301989888~41943040,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.771 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~100663296,406847488~8388608] 2024-09-13T14:43:44.772 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 18 diff [0~83886080,88080384~113246208,205520896~58720256,268435456~29360128,301989888~41943040,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.772 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~29360128,301989888~41943040,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.794 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~100663296,406847488~8388608] 2024-09-13T14:43:44.794 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to 19 diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.794 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:44.816 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:44.816 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 2 diff [20971520~8388608,41943040~4194304,75497472~4194304,88080384~8388608,163577856~8388608,272629760~4194304,343932928~4194304,369098752~8388608] 2024-09-13T14:43:44.816 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [20971520~8388608,41943040~4194304,75497472~4194304,88080384~8388608,163577856~8388608,272629760~4194304,343932928~4194304,369098752~8388608] 2024-09-13T14:43:44.819 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [20971520~8388608,41943040~4194304,75497472~4194304,88080384~8388608,163577856~8388608,272629760~8388608,343932928~4194304,369098752~12582912] 2024-09-13T14:43:44.819 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 3 diff [20971520~8388608,41943040~4194304,50331648~4194304,75497472~4194304,88080384~8388608,109051904~8388608,121634816~8388608,163577856~8388608,192937984~8388608,218103808~8388608,247463936~8388608,268435456~8388608,314572800~4194304,343932928~8388608,369098752~8388608,406847488~8388608] 2024-09-13T14:43:44.819 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [20971520~8388608,41943040~4194304,50331648~4194304,75497472~4194304,88080384~8388608,109051904~8388608,121634816~8388608,163577856~8388608,192937984~8388608,218103808~8388608,247463936~8388608,268435456~8388608,314572800~4194304,343932928~8388608,369098752~8388608,406847488~8388608] 2024-09-13T14:43:44.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:44 smithi099 bash[84576]: cluster 2024-09-13T14:43:43.506898+0000 mon.a (mon.0) 1501 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-13T14:43:44.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:44 smithi099 bash[84576]: cluster 2024-09-13T14:43:43.506898+0000 mon.a (mon.0) 1501 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-13T14:43:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:44 smithi099 bash[87147]: cluster 2024-09-13T14:43:43.506898+0000 mon.a (mon.0) 1501 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-13T14:43:44.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:44 smithi099 bash[87147]: cluster 2024-09-13T14:43:43.506898+0000 mon.a (mon.0) 1501 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-13T14:43:44.823 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [20971520~8388608,41943040~4194304,50331648~4194304,75497472~4194304,88080384~8388608,109051904~8388608,121634816~12582912,163577856~8388608,192937984~8388608,218103808~12582912,247463936~8388608,268435456~12582912,314572800~4194304,343932928~8388608,369098752~12582912,406847488~8388608] 2024-09-13T14:43:44.823 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 4 diff [20971520~8388608,41943040~4194304,50331648~4194304,71303168~8388608,88080384~8388608,109051904~8388608,121634816~8388608,159383552~12582912,180355072~8388608,192937984~8388608,205520896~20971520,247463936~8388608,268435456~8388608,314572800~8388608,343932928~16777216,364904448~12582912,394264576~4194304,406847488~8388608] 2024-09-13T14:43:44.823 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [20971520~8388608,41943040~4194304,50331648~4194304,71303168~8388608,88080384~8388608,109051904~8388608,121634816~8388608,159383552~12582912,180355072~8388608,192937984~8388608,205520896~20971520,247463936~8388608,268435456~8388608,314572800~8388608,343932928~16777216,364904448~12582912,394264576~4194304,406847488~8388608] 2024-09-13T14:43:44.828 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [20971520~8388608,41943040~4194304,50331648~4194304,71303168~12582912,88080384~8388608,109051904~8388608,121634816~12582912,159383552~12582912,180355072~8388608,192937984~8388608,205520896~25165824,247463936~8388608,268435456~12582912,314572800~8388608,343932928~16777216,364904448~16777216,394264576~4194304,406847488~8388608] 2024-09-13T14:43:44.829 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 5 diff [4194304~4194304,20971520~8388608,41943040~4194304,50331648~4194304,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,155189248~16777216,180355072~8388608,192937984~8388608,205520896~25165824,247463936~8388608,268435456~8388608,314572800~12582912,343932928~16777216,364904448~12582912,390070272~8388608,406847488~8388608] 2024-09-13T14:43:44.829 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~4194304,20971520~8388608,41943040~4194304,50331648~4194304,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,155189248~16777216,180355072~8388608,192937984~8388608,205520896~25165824,247463936~8388608,268435456~8388608,314572800~12582912,343932928~16777216,364904448~12582912,390070272~8388608,406847488~8388608] 2024-09-13T14:43:44.835 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~4194304,20971520~8388608,41943040~4194304,50331648~4194304,71303168~12582912,88080384~16777216,109051904~8388608,121634816~12582912,138412032~8388608,155189248~16777216,180355072~8388608,192937984~37748736,247463936~8388608,268435456~12582912,314572800~12582912,343932928~16777216,364904448~16777216,390070272~8388608,406847488~8388608] 2024-09-13T14:43:44.835 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 6 diff [0~8388608,20971520~12582912,41943040~4194304,50331648~4194304,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~8388608,289406976~8388608,314572800~12582912,343932928~16777216,364904448~12582912,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.835 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,20971520~12582912,41943040~4194304,50331648~4194304,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~8388608,289406976~8388608,314572800~12582912,343932928~16777216,364904448~12582912,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.842 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,20971520~12582912,41943040~4194304,50331648~4194304,71303168~12582912,88080384~16777216,109051904~8388608,121634816~12582912,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,247463936~16777216,268435456~12582912,289406976~8388608,314572800~12582912,343932928~16777216,364904448~16777216,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.842 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 7 diff [0~8388608,20971520~12582912,41943040~4194304,50331648~8388608,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,343932928~16777216,364904448~12582912,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.842 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,20971520~12582912,41943040~4194304,50331648~8388608,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,343932928~16777216,364904448~12582912,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.851 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,20971520~12582912,41943040~4194304,50331648~12582912,71303168~12582912,88080384~16777216,109051904~8388608,121634816~12582912,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,343932928~16777216,364904448~16777216,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.851 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 8 diff [0~8388608,12582912~4194304,20971520~12582912,41943040~4194304,50331648~12582912,71303168~12582912,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,343932928~33554432,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.852 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,12582912~4194304,20971520~12582912,41943040~4194304,50331648~12582912,71303168~12582912,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,343932928~33554432,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:44 smithi177 bash[68594]: cluster 2024-09-13T14:43:43.506898+0000 mon.a (mon.0) 1501 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-13T14:43:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:44 smithi177 bash[68594]: cluster 2024-09-13T14:43:43.506898+0000 mon.a (mon.0) 1501 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-13T14:43:44.861 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~25165824,41943040~25165824,71303168~33554432,109051904~8388608,121634816~12582912,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,343932928~37748736,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.862 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 9 diff [0~8388608,12582912~4194304,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~12582912,268435456~16777216,289406976~8388608,314572800~12582912,343932928~33554432,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.862 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,12582912~4194304,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~12582912,268435456~16777216,289406976~8388608,314572800~12582912,343932928~33554432,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.872 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~54525952,71303168~62914560,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,343932928~37748736,385875968~12582912,406847488~8388608] 2024-09-13T14:43:44.872 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 10 diff [0~16777216,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.872 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~16777216,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.884 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~62914560,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,247463936~16777216,268435456~29360128,301989888~25165824,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.885 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 11 diff [0~37748736,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~37748736,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.885 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~37748736,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.898 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~62914560,138412032~8388608,150994944~92274688,247463936~16777216,268435456~29360128,301989888~25165824,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.898 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 12 diff [0~37748736,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~8388608,314572800~12582912,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.898 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~8388608,314572800~12582912,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.912 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~62914560,138412032~8388608,150994944~113246208,268435456~29360128,301989888~25165824,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.912 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 13 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~12582912,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~25165824,331350016~8388608,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.912 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~12582912,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~25165824,331350016~8388608,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.928 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~146800640,150994944~113246208,268435456~29360128,301989888~25165824,331350016~67108864,406847488~8388608] 2024-09-13T14:43:44.928 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 14 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~25165824,331350016~8388608,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.928 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~25165824,331350016~8388608,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.944 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~25165824,331350016~71303168,406847488~8388608] 2024-09-13T14:43:44.944 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 15 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.945 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.962 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~100663296,406847488~8388608] 2024-09-13T14:43:44.962 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 16 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.962 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.981 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~100663296,406847488~8388608] 2024-09-13T14:43:44.981 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 17 diff [0~37748736,41943040~41943040,88080384~71303168,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,343932928~54525952,406847488~8388608] 2024-09-13T14:43:44.981 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~71303168,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,343932928~54525952,406847488~8388608] 2024-09-13T14:43:45.001 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~100663296,406847488~8388608] 2024-09-13T14:43:45.001 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 18 diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~29360128,301989888~37748736,343932928~54525952,406847488~8388608] 2024-09-13T14:43:45.001 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~29360128,301989888~37748736,343932928~54525952,406847488~8388608] 2024-09-13T14:43:45.021 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~100663296,406847488~8388608] 2024-09-13T14:43:45.022 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to 19 diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:45.022 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:45.042 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:45.043 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 3 diff [50331648~4194304,109051904~8388608,121634816~8388608,192937984~8388608,218103808~8388608,247463936~8388608,268435456~4194304,314572800~4194304,348127232~4194304,406847488~8388608] 2024-09-13T14:43:45.043 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [50331648~4194304,109051904~8388608,121634816~8388608,192937984~8388608,218103808~8388608,247463936~8388608,268435456~4194304,314572800~4194304,348127232~4194304,406847488~8388608] 2024-09-13T14:43:45.046 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [50331648~4194304,109051904~8388608,121634816~12582912,192937984~8388608,218103808~12582912,247463936~8388608,268435456~4194304,314572800~4194304,348127232~4194304,406847488~8388608] 2024-09-13T14:43:45.046 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 4 diff [50331648~4194304,71303168~8388608,109051904~8388608,121634816~8388608,159383552~4194304,180355072~8388608,192937984~8388608,205520896~20971520,247463936~8388608,268435456~4194304,314572800~8388608,348127232~12582912,364904448~8388608,394264576~4194304,406847488~8388608] 2024-09-13T14:43:45.046 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [50331648~4194304,71303168~8388608,109051904~8388608,121634816~8388608,159383552~4194304,180355072~8388608,192937984~8388608,205520896~20971520,247463936~8388608,268435456~4194304,314572800~8388608,348127232~12582912,364904448~8388608,394264576~4194304,406847488~8388608] 2024-09-13T14:43:45.050 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [50331648~4194304,71303168~12582912,109051904~8388608,121634816~12582912,159383552~4194304,180355072~8388608,192937984~8388608,205520896~25165824,247463936~8388608,268435456~4194304,314572800~8388608,348127232~12582912,364904448~12582912,394264576~4194304,406847488~8388608] 2024-09-13T14:43:45.050 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 5 diff [4194304~4194304,50331648~4194304,71303168~8388608,96468992~8388608,109051904~8388608,121634816~8388608,138412032~8388608,155189248~8388608,180355072~8388608,192937984~8388608,205520896~25165824,247463936~8388608,268435456~4194304,314572800~12582912,348127232~12582912,364904448~8388608,390070272~8388608,406847488~8388608] 2024-09-13T14:43:45.051 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~4194304,50331648~4194304,71303168~8388608,96468992~8388608,109051904~8388608,121634816~8388608,138412032~8388608,155189248~8388608,180355072~8388608,192937984~8388608,205520896~25165824,247463936~8388608,268435456~4194304,314572800~12582912,348127232~12582912,364904448~8388608,390070272~8388608,406847488~8388608] 2024-09-13T14:43:45.056 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~4194304,50331648~4194304,71303168~12582912,96468992~8388608,109051904~8388608,121634816~12582912,138412032~8388608,155189248~8388608,180355072~8388608,192937984~37748736,247463936~8388608,268435456~4194304,314572800~12582912,348127232~12582912,364904448~12582912,390070272~8388608,406847488~8388608] 2024-09-13T14:43:45.056 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 6 diff [0~8388608,25165824~8388608,50331648~4194304,71303168~8388608,96468992~8388608,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,167772160~8388608,180355072~8388608,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~4194304,289406976~8388608,314572800~12582912,348127232~12582912,364904448~8388608,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.056 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,25165824~8388608,50331648~4194304,71303168~8388608,96468992~8388608,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,167772160~8388608,180355072~8388608,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~4194304,289406976~8388608,314572800~12582912,348127232~12582912,364904448~8388608,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.062 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,25165824~8388608,50331648~4194304,71303168~12582912,96468992~8388608,109051904~8388608,121634816~12582912,138412032~8388608,150994944~12582912,167772160~20971520,192937984~37748736,239075328~4194304,247463936~16777216,268435456~4194304,289406976~8388608,314572800~12582912,348127232~12582912,364904448~12582912,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.062 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 7 diff [0~8388608,20971520~8388608,50331648~8388608,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,348127232~12582912,364904448~8388608,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.062 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,20971520~8388608,50331648~8388608,71303168~8388608,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,348127232~12582912,364904448~8388608,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.070 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,20971520~12582912,50331648~12582912,71303168~12582912,88080384~16777216,109051904~8388608,121634816~12582912,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,348127232~12582912,364904448~12582912,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.070 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 8 diff [0~8388608,12582912~4194304,20971520~12582912,41943040~4194304,50331648~12582912,71303168~12582912,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,348127232~25165824,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.070 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,12582912~4194304,20971520~12582912,41943040~4194304,50331648~12582912,71303168~12582912,88080384~16777216,109051904~8388608,121634816~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~16777216,268435456~16777216,289406976~8388608,314572800~12582912,348127232~25165824,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.079 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~25165824,41943040~25165824,71303168~33554432,109051904~8388608,121634816~12582912,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,348127232~29360128,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.079 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 9 diff [0~8388608,12582912~4194304,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~12582912,268435456~16777216,289406976~8388608,314572800~12582912,348127232~25165824,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.079 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,12582912~4194304,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~12582912,268435456~16777216,289406976~8388608,314572800~12582912,348127232~25165824,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.089 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~54525952,71303168~62914560,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,247463936~16777216,268435456~29360128,314572800~12582912,348127232~29360128,385875968~12582912,406847488~8388608] 2024-09-13T14:43:45.089 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 10 diff [0~16777216,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.090 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~16777216,20971520~16777216,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.100 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~62914560,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,247463936~16777216,268435456~29360128,301989888~25165824,348127232~50331648,406847488~8388608] 2024-09-13T14:43:45.100 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 11 diff [0~37748736,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~37748736,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.100 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~37748736,247463936~12582912,268435456~29360128,301989888~8388608,314572800~12582912,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.111 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~62914560,138412032~8388608,150994944~92274688,247463936~16777216,268435456~29360128,301989888~25165824,348127232~50331648,406847488~8388608] 2024-09-13T14:43:45.111 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 12 diff [0~37748736,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~8388608,314572800~12582912,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.111 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~25165824,71303168~12582912,88080384~41943040,138412032~8388608,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~8388608,314572800~12582912,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.124 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~62914560,138412032~8388608,150994944~113246208,268435456~29360128,301989888~25165824,348127232~50331648,406847488~8388608] 2024-09-13T14:43:45.124 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 13 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~12582912,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.125 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~12582912,150994944~8388608,163577856~37748736,205520896~54525952,268435456~29360128,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.139 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~146800640,150994944~113246208,268435456~29360128,301989888~25165824,331350016~12582912,348127232~50331648,406847488~8388608] 2024-09-13T14:43:45.139 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 14 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.139 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.154 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~25165824,331350016~12582912,348127232~54525952,406847488~8388608] 2024-09-13T14:43:45.155 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 15 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.155 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.169 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~41943040,348127232~54525952,406847488~8388608] 2024-09-13T14:43:45.169 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 16 diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.169 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~41943040,134217728~25165824,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.185 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~41943040,348127232~54525952,406847488~8388608] 2024-09-13T14:43:45.185 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 17 diff [0~37748736,41943040~41943040,88080384~71303168,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.186 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~71303168,163577856~37748736,205520896~58720256,268435456~29360128,301989888~37748736,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.202 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~41943040,348127232~54525952,406847488~8388608] 2024-09-13T14:43:45.202 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 18 diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~29360128,301989888~37748736,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.202 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~29360128,301989888~37748736,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.219 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~297795584,301989888~41943040,348127232~54525952,406847488~8388608] 2024-09-13T14:43:45.220 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to 19 diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.220 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:45.239 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:45.239 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 4 diff [71303168~8388608,159383552~4194304,180355072~8388608,205520896~12582912,314572800~4194304,352321536~8388608,364904448~8388608,394264576~4194304] 2024-09-13T14:43:45.240 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [71303168~8388608,159383552~4194304,180355072~8388608,205520896~12582912,314572800~4194304,352321536~8388608,364904448~8388608,394264576~4194304] 2024-09-13T14:43:45.243 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [71303168~12582912,159383552~4194304,180355072~8388608,205520896~16777216,314572800~8388608,352321536~8388608,364904448~12582912,394264576~4194304] 2024-09-13T14:43:45.244 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 5 diff [4194304~4194304,71303168~8388608,96468992~8388608,138412032~8388608,155189248~8388608,180355072~8388608,197132288~33554432,314572800~12582912,352321536~8388608,364904448~8388608,390070272~8388608] 2024-09-13T14:43:45.244 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~4194304,71303168~8388608,96468992~8388608,138412032~8388608,155189248~8388608,180355072~8388608,197132288~33554432,314572800~12582912,352321536~8388608,364904448~8388608,390070272~8388608] 2024-09-13T14:43:45.248 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~4194304,71303168~12582912,96468992~8388608,138412032~8388608,155189248~8388608,180355072~8388608,197132288~33554432,314572800~12582912,352321536~8388608,364904448~12582912,390070272~8388608] 2024-09-13T14:43:45.248 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 6 diff [0~8388608,25165824~8388608,71303168~8388608,96468992~8388608,138412032~8388608,150994944~8388608,167772160~8388608,180355072~8388608,197132288~33554432,239075328~4194304,255852544~8388608,289406976~8388608,314572800~12582912,352321536~8388608,364904448~8388608,385875968~12582912] 2024-09-13T14:43:45.248 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,25165824~8388608,71303168~8388608,96468992~8388608,138412032~8388608,150994944~8388608,167772160~8388608,180355072~8388608,197132288~33554432,239075328~4194304,255852544~8388608,289406976~8388608,314572800~12582912,352321536~8388608,364904448~8388608,385875968~12582912] 2024-09-13T14:43:45.253 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,25165824~8388608,71303168~12582912,96468992~8388608,138412032~8388608,150994944~12582912,167772160~20971520,197132288~33554432,239075328~4194304,255852544~8388608,289406976~8388608,314572800~12582912,352321536~8388608,364904448~12582912,385875968~12582912] 2024-09-13T14:43:45.253 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 7 diff [0~8388608,20971520~8388608,54525952~4194304,71303168~8388608,88080384~16777216,138412032~8388608,150994944~8388608,163577856~25165824,197132288~33554432,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,314572800~12582912,352321536~8388608,364904448~8388608,385875968~12582912] 2024-09-13T14:43:45.253 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,20971520~8388608,54525952~4194304,71303168~8388608,88080384~16777216,138412032~8388608,150994944~8388608,163577856~25165824,197132288~33554432,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,314572800~12582912,352321536~8388608,364904448~8388608,385875968~12582912] 2024-09-13T14:43:45.259 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,20971520~12582912,54525952~8388608,71303168~12582912,88080384~16777216,138412032~8388608,150994944~37748736,197132288~33554432,239075328~4194304,255852544~8388608,272629760~25165824,314572800~12582912,352321536~8388608,364904448~12582912,385875968~12582912] 2024-09-13T14:43:45.259 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 8 diff [0~8388608,12582912~4194304,20971520~12582912,41943040~4194304,54525952~8388608,71303168~12582912,88080384~16777216,138412032~8388608,150994944~8388608,163577856~25165824,197132288~33554432,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,314572800~12582912,352321536~20971520,385875968~12582912] 2024-09-13T14:43:45.259 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,12582912~4194304,20971520~12582912,41943040~4194304,54525952~8388608,71303168~12582912,88080384~16777216,138412032~8388608,150994944~8388608,163577856~25165824,197132288~33554432,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,314572800~12582912,352321536~20971520,385875968~12582912] 2024-09-13T14:43:45.266 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~25165824,41943040~8388608,54525952~12582912,71303168~33554432,138412032~8388608,150994944~37748736,197132288~33554432,239075328~4194304,255852544~8388608,272629760~25165824,314572800~12582912,352321536~25165824,385875968~12582912] 2024-09-13T14:43:45.266 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 9 diff [0~8388608,12582912~4194304,20971520~16777216,41943040~8388608,54525952~12582912,71303168~12582912,88080384~25165824,117440512~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,251658240~12582912,272629760~12582912,289406976~8388608,314572800~12582912,352321536~20971520,385875968~12582912] 2024-09-13T14:43:45.266 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,12582912~4194304,20971520~16777216,41943040~8388608,54525952~12582912,71303168~12582912,88080384~25165824,117440512~8388608,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,251658240~12582912,272629760~12582912,289406976~8388608,314572800~12582912,352321536~20971520,385875968~12582912] 2024-09-13T14:43:45.274 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~54525952,71303168~58720256,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,251658240~12582912,272629760~25165824,314572800~12582912,352321536~25165824,385875968~12582912] 2024-09-13T14:43:45.274 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 10 diff [0~16777216,20971520~16777216,41943040~8388608,54525952~12582912,71303168~12582912,88080384~37748736,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,251658240~12582912,272629760~25165824,301989888~8388608,314572800~12582912,352321536~20971520,377487360~20971520] 2024-09-13T14:43:45.274 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~16777216,20971520~16777216,41943040~8388608,54525952~12582912,71303168~12582912,88080384~37748736,138412032~8388608,150994944~8388608,163577856~25165824,192937984~8388608,205520896~25165824,239075328~4194304,251658240~12582912,272629760~25165824,301989888~8388608,314572800~12582912,352321536~20971520,377487360~20971520] 2024-09-13T14:43:45.282 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~58720256,138412032~8388608,150994944~37748736,192937984~37748736,239075328~4194304,251658240~12582912,272629760~25165824,301989888~25165824,352321536~46137344] 2024-09-13T14:43:45.282 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 11 diff [0~37748736,41943040~8388608,54525952~12582912,71303168~12582912,88080384~37748736,138412032~8388608,150994944~8388608,163577856~37748736,205520896~37748736,251658240~12582912,272629760~25165824,301989888~8388608,314572800~12582912,352321536~20971520,377487360~20971520] 2024-09-13T14:43:45.282 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~8388608,54525952~12582912,71303168~12582912,88080384~37748736,138412032~8388608,150994944~8388608,163577856~37748736,205520896~37748736,251658240~12582912,272629760~25165824,301989888~8388608,314572800~12582912,352321536~20971520,377487360~20971520] 2024-09-13T14:43:45.291 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~58720256,138412032~8388608,150994944~92274688,251658240~12582912,272629760~25165824,301989888~25165824,352321536~46137344] 2024-09-13T14:43:45.291 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 12 diff [0~37748736,41943040~8388608,54525952~12582912,71303168~12582912,88080384~37748736,138412032~8388608,150994944~8388608,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~8388608,314572800~12582912,348127232~8388608,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.291 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~8388608,54525952~12582912,71303168~12582912,88080384~37748736,138412032~8388608,150994944~8388608,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~8388608,314572800~12582912,348127232~8388608,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.302 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,71303168~58720256,138412032~8388608,150994944~113246208,272629760~25165824,301989888~25165824,348127232~50331648] 2024-09-13T14:43:45.302 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 13 diff [0~37748736,41943040~8388608,54525952~29360128,88080384~37748736,134217728~12582912,150994944~8388608,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.303 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~8388608,54525952~29360128,88080384~37748736,134217728~12582912,150994944~8388608,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.314 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~130023424,134217728~12582912,150994944~113246208,272629760~25165824,301989888~25165824,331350016~12582912,348127232~50331648] 2024-09-13T14:43:45.314 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 14 diff [0~37748736,41943040~41943040,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.314 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.327 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~130023424,134217728~134217728,272629760~25165824,301989888~25165824,331350016~12582912,348127232~54525952] 2024-09-13T14:43:45.327 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 15 diff [0~37748736,41943040~41943040,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.327 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.342 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~130023424,134217728~134217728,272629760~25165824,301989888~41943040,348127232~54525952] 2024-09-13T14:43:45.342 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 16 diff [0~37748736,41943040~41943040,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.342 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.356 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~130023424,134217728~134217728,272629760~25165824,301989888~41943040,348127232~54525952] 2024-09-13T14:43:45.357 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 17 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~29360128,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.357 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~29360128,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.373 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~25165824,301989888~41943040,348127232~54525952] 2024-09-13T14:43:45.373 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 18 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.373 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.393 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~25165824,301989888~41943040,348127232~54525952] 2024-09-13T14:43:45.393 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to 19 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.393 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.406 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:45.406 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 5 diff [4194304~4194304,96468992~8388608,138412032~8388608,155189248~4194304,197132288~8388608,218103808~12582912,318767104~8388608,390070272~4194304] 2024-09-13T14:43:45.406 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~4194304,96468992~8388608,138412032~8388608,155189248~4194304,197132288~8388608,218103808~12582912,318767104~8388608,390070272~4194304] 2024-09-13T14:43:45.409 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~4194304,96468992~8388608,138412032~8388608,155189248~4194304,197132288~8388608,218103808~12582912,318767104~8388608,390070272~4194304] 2024-09-13T14:43:45.410 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 6 diff [0~8388608,25165824~8388608,96468992~8388608,138412032~8388608,150994944~8388608,167772160~8388608,197132288~8388608,218103808~12582912,239075328~4194304,255852544~8388608,289406976~8388608,318767104~8388608,385875968~8388608] 2024-09-13T14:43:45.410 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,25165824~8388608,96468992~8388608,138412032~8388608,150994944~8388608,167772160~8388608,197132288~8388608,218103808~12582912,239075328~4194304,255852544~8388608,289406976~8388608,318767104~8388608,385875968~8388608] 2024-09-13T14:43:45.414 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,25165824~8388608,96468992~8388608,138412032~8388608,150994944~12582912,167772160~12582912,197132288~8388608,218103808~12582912,239075328~4194304,255852544~8388608,289406976~8388608,318767104~8388608,385875968~8388608] 2024-09-13T14:43:45.414 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 7 diff [0~8388608,20971520~8388608,54525952~4194304,88080384~16777216,138412032~8388608,150994944~8388608,163577856~20971520,197132288~8388608,218103808~12582912,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,318767104~8388608,385875968~8388608] 2024-09-13T14:43:45.414 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,20971520~8388608,54525952~4194304,88080384~16777216,138412032~8388608,150994944~8388608,163577856~20971520,197132288~8388608,218103808~12582912,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,318767104~8388608,385875968~8388608] 2024-09-13T14:43:45.420 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,20971520~12582912,54525952~8388608,88080384~16777216,138412032~8388608,150994944~33554432,197132288~8388608,218103808~12582912,239075328~4194304,255852544~8388608,272629760~25165824,318767104~8388608,385875968~8388608] 2024-09-13T14:43:45.420 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 8 diff [0~8388608,12582912~4194304,20971520~12582912,41943040~4194304,54525952~8388608,75497472~8388608,88080384~16777216,138412032~8388608,150994944~8388608,163577856~20971520,197132288~8388608,218103808~12582912,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,318767104~8388608,360710144~8388608,385875968~8388608] 2024-09-13T14:43:45.420 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,12582912~4194304,20971520~12582912,41943040~4194304,54525952~8388608,75497472~8388608,88080384~16777216,138412032~8388608,150994944~8388608,163577856~20971520,197132288~8388608,218103808~12582912,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,318767104~8388608,360710144~8388608,385875968~8388608] 2024-09-13T14:43:45.426 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~25165824,41943040~8388608,54525952~12582912,75497472~29360128,138412032~8388608,150994944~33554432,197132288~8388608,218103808~12582912,239075328~4194304,255852544~8388608,272629760~25165824,318767104~8388608,360710144~12582912,385875968~8388608] 2024-09-13T14:43:45.426 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 9 diff [0~8388608,12582912~4194304,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~25165824,117440512~8388608,138412032~8388608,150994944~8388608,163577856~20971520,192937984~8388608,218103808~12582912,239075328~4194304,251658240~12582912,272629760~12582912,289406976~8388608,318767104~8388608,360710144~8388608,385875968~8388608] 2024-09-13T14:43:45.426 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,12582912~4194304,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~25165824,117440512~8388608,138412032~8388608,150994944~8388608,163577856~20971520,192937984~8388608,218103808~12582912,239075328~4194304,251658240~12582912,272629760~12582912,289406976~8388608,318767104~8388608,360710144~8388608,385875968~8388608] 2024-09-13T14:43:45.434 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~54525952,75497472~54525952,138412032~8388608,150994944~33554432,192937984~12582912,218103808~12582912,239075328~4194304,251658240~12582912,272629760~25165824,318767104~8388608,360710144~12582912,385875968~8388608] 2024-09-13T14:43:45.434 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 10 diff [0~16777216,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,138412032~8388608,150994944~8388608,163577856~20971520,192937984~8388608,205520896~8388608,218103808~12582912,239075328~4194304,251658240~12582912,272629760~25165824,301989888~8388608,318767104~8388608,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.434 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~16777216,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,138412032~8388608,150994944~8388608,163577856~20971520,192937984~8388608,205520896~8388608,218103808~12582912,239075328~4194304,251658240~12582912,272629760~25165824,301989888~8388608,318767104~8388608,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.443 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,75497472~54525952,138412032~8388608,150994944~33554432,192937984~20971520,218103808~12582912,239075328~4194304,251658240~12582912,272629760~25165824,301989888~12582912,318767104~8388608,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.443 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 11 diff [0~37748736,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,138412032~8388608,150994944~8388608,163577856~37748736,205520896~37748736,251658240~12582912,272629760~25165824,301989888~8388608,318767104~8388608,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.443 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,138412032~8388608,150994944~8388608,163577856~37748736,205520896~37748736,251658240~12582912,272629760~25165824,301989888~8388608,318767104~8388608,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.453 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,75497472~54525952,138412032~8388608,150994944~92274688,251658240~12582912,272629760~25165824,301989888~12582912,318767104~8388608,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.453 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 12 diff [0~37748736,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,138412032~8388608,150994944~8388608,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~8388608,318767104~8388608,348127232~4194304,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.453 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,138412032~8388608,150994944~8388608,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~8388608,318767104~8388608,348127232~4194304,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.464 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~67108864,75497472~54525952,138412032~8388608,150994944~113246208,272629760~25165824,301989888~12582912,318767104~8388608,348127232~8388608,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.464 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 13 diff [0~37748736,41943040~8388608,54525952~16777216,75497472~8388608,88080384~37748736,134217728~12582912,150994944~8388608,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.464 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~8388608,54525952~16777216,75497472~8388608,88080384~37748736,134217728~12582912,150994944~8388608,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.476 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~12582912,150994944~113246208,272629760~25165824,301989888~25165824,331350016~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.476 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 14 diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.476 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~25165824,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.488 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~134217728,272629760~25165824,301989888~25165824,331350016~12582912,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.488 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 15 diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.489 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.502 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~134217728,272629760~25165824,301989888~41943040,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.502 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 16 diff [0~37748736,41943040~41943040,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.502 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,134217728~25165824,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.516 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~130023424,134217728~134217728,272629760~25165824,301989888~41943040,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.516 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 17 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~29360128,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.516 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~29360128,163577856~37748736,205520896~41943040,251658240~12582912,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.531 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~25165824,301989888~41943040,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.532 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 18 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.532 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~25165824,301989888~37748736,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.549 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~25165824,301989888~41943040,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.549 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to 19 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.549 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.566 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.566 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 6 diff [0~4194304,25165824~8388608,96468992~8388608,150994944~8388608,167772160~8388608,239075328~4194304,255852544~8388608,289406976~8388608,385875968~8388608] 2024-09-13T14:43:45.566 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,25165824~8388608,96468992~8388608,150994944~8388608,167772160~8388608,239075328~4194304,255852544~8388608,289406976~8388608,385875968~8388608] 2024-09-13T14:43:45.569 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~4194304,25165824~8388608,96468992~8388608,150994944~12582912,167772160~12582912,239075328~4194304,255852544~8388608,289406976~8388608,385875968~8388608] 2024-09-13T14:43:45.569 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 7 diff [0~4194304,20971520~8388608,54525952~4194304,88080384~16777216,150994944~8388608,163577856~20971520,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,385875968~8388608] 2024-09-13T14:43:45.569 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,20971520~8388608,54525952~4194304,88080384~16777216,150994944~8388608,163577856~20971520,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,385875968~8388608] 2024-09-13T14:43:45.573 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~4194304,20971520~12582912,54525952~8388608,88080384~16777216,150994944~33554432,239075328~4194304,255852544~8388608,272629760~25165824,385875968~8388608] 2024-09-13T14:43:45.574 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 8 diff [0~4194304,12582912~4194304,20971520~12582912,41943040~4194304,54525952~8388608,75497472~8388608,88080384~16777216,150994944~8388608,163577856~20971520,218103808~4194304,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,318767104~4194304,360710144~8388608,385875968~8388608] 2024-09-13T14:43:45.574 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,20971520~12582912,41943040~4194304,54525952~8388608,75497472~8388608,88080384~16777216,150994944~8388608,163577856~20971520,218103808~4194304,239075328~4194304,255852544~8388608,272629760~12582912,289406976~8388608,318767104~4194304,360710144~8388608,385875968~8388608] 2024-09-13T14:43:45.579 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~4194304,12582912~25165824,41943040~8388608,54525952~12582912,75497472~29360128,150994944~33554432,218103808~4194304,239075328~4194304,255852544~8388608,272629760~25165824,318767104~4194304,360710144~12582912,385875968~8388608] 2024-09-13T14:43:45.579 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 9 diff [0~4194304,12582912~4194304,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~25165824,117440512~8388608,150994944~8388608,163577856~20971520,192937984~4194304,218103808~4194304,239075328~4194304,251658240~12582912,272629760~12582912,289406976~8388608,318767104~4194304,360710144~8388608,385875968~8388608] 2024-09-13T14:43:45.579 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,12582912~4194304,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~25165824,117440512~8388608,150994944~8388608,163577856~20971520,192937984~4194304,218103808~4194304,239075328~4194304,251658240~12582912,272629760~12582912,289406976~8388608,318767104~4194304,360710144~8388608,385875968~8388608] 2024-09-13T14:43:45.585 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~4194304,12582912~54525952,75497472~54525952,150994944~33554432,192937984~8388608,218103808~4194304,239075328~4194304,251658240~12582912,272629760~25165824,318767104~4194304,360710144~12582912,385875968~8388608] 2024-09-13T14:43:45.586 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 10 diff [0~4194304,8388608~8388608,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,150994944~8388608,163577856~20971520,192937984~4194304,205520896~8388608,218103808~4194304,239075328~4194304,251658240~12582912,272629760~25165824,301989888~8388608,318767104~4194304,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.586 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,8388608~8388608,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,150994944~8388608,163577856~20971520,192937984~4194304,205520896~8388608,218103808~4194304,239075328~4194304,251658240~12582912,272629760~25165824,301989888~8388608,318767104~4194304,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.592 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~4194304,8388608~58720256,75497472~54525952,150994944~33554432,192937984~8388608,205520896~8388608,218103808~4194304,239075328~4194304,251658240~12582912,272629760~25165824,301989888~12582912,318767104~4194304,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.593 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 11 diff [0~4194304,8388608~29360128,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,150994944~8388608,163577856~33554432,205520896~16777216,230686720~12582912,251658240~12582912,272629760~25165824,301989888~8388608,318767104~4194304,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.593 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,8388608~29360128,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,150994944~8388608,163577856~33554432,205520896~16777216,230686720~12582912,251658240~12582912,272629760~25165824,301989888~8388608,318767104~4194304,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.601 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~4194304,8388608~58720256,75497472~54525952,150994944~50331648,205520896~16777216,230686720~12582912,251658240~12582912,272629760~25165824,301989888~12582912,318767104~4194304,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.601 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 12 diff [0~4194304,8388608~29360128,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,150994944~8388608,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~8388608,318767104~4194304,348127232~4194304,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.601 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~4194304,8388608~29360128,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,150994944~8388608,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~8388608,318767104~4194304,348127232~4194304,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.610 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~4194304,8388608~58720256,75497472~54525952,150994944~50331648,205520896~16777216,230686720~33554432,272629760~25165824,301989888~12582912,318767104~4194304,348127232~8388608,360710144~12582912,377487360~20971520] 2024-09-13T14:43:45.610 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 13 diff [0~37748736,41943040~8388608,54525952~16777216,75497472~8388608,88080384~37748736,134217728~12582912,150994944~8388608,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.610 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~8388608,54525952~16777216,75497472~8388608,88080384~37748736,134217728~12582912,150994944~8388608,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.621 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~12582912,150994944~50331648,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,331350016~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.621 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 14 diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~25165824,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.621 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~25165824,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.632 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~67108864,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,331350016~12582912,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.632 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 15 diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~25165824,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.632 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~25165824,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.645 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~67108864,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.645 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 16 diff [0~37748736,41943040~41943040,88080384~37748736,134217728~25165824,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.645 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,134217728~25165824,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.659 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~130023424,134217728~67108864,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.659 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 17 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~29360128,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.659 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~29360128,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.673 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.673 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 18 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.673 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.687 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.687 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to 19 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.688 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:45.704 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.704 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 7 diff [20971520~8388608,54525952~4194304,88080384~8388608,163577856~20971520,272629760~12582912] 2024-09-13T14:43:45.704 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [20971520~8388608,54525952~4194304,88080384~8388608,163577856~20971520,272629760~12582912] 2024-09-13T14:43:45.707 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [20971520~8388608,54525952~8388608,88080384~12582912,163577856~20971520,272629760~16777216] 2024-09-13T14:43:45.707 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 8 diff [12582912~4194304,20971520~12582912,41943040~4194304,54525952~8388608,75497472~8388608,88080384~8388608,163577856~20971520,218103808~4194304,272629760~12582912,318767104~4194304,360710144~8388608] 2024-09-13T14:43:45.707 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [12582912~4194304,20971520~12582912,41943040~4194304,54525952~8388608,75497472~8388608,88080384~8388608,163577856~20971520,218103808~4194304,272629760~12582912,318767104~4194304,360710144~8388608] 2024-09-13T14:43:45.711 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [12582912~25165824,41943040~8388608,54525952~12582912,75497472~25165824,163577856~20971520,218103808~4194304,272629760~16777216,318767104~4194304,360710144~12582912] 2024-09-13T14:43:45.711 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 9 diff [12582912~4194304,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~8388608,104857600~8388608,117440512~8388608,163577856~20971520,192937984~4194304,218103808~4194304,251658240~8388608,272629760~12582912,318767104~4194304,360710144~8388608] 2024-09-13T14:43:45.711 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [12582912~4194304,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~8388608,104857600~8388608,117440512~8388608,163577856~20971520,192937984~4194304,218103808~4194304,251658240~8388608,272629760~12582912,318767104~4194304,360710144~8388608] 2024-09-13T14:43:45.717 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [12582912~54525952,75497472~25165824,104857600~25165824,163577856~20971520,192937984~8388608,218103808~4194304,251658240~8388608,272629760~16777216,318767104~4194304,360710144~12582912] 2024-09-13T14:43:45.717 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 10 diff [8388608~8388608,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~8388608,104857600~20971520,163577856~20971520,192937984~4194304,205520896~8388608,218103808~4194304,251658240~8388608,272629760~16777216,301989888~8388608,318767104~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.717 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~8388608,20971520~16777216,41943040~8388608,54525952~12582912,75497472~8388608,88080384~8388608,104857600~20971520,163577856~20971520,192937984~4194304,205520896~8388608,218103808~4194304,251658240~8388608,272629760~16777216,301989888~8388608,318767104~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.723 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~58720256,75497472~25165824,104857600~25165824,163577856~20971520,192937984~8388608,205520896~8388608,218103808~4194304,251658240~8388608,272629760~20971520,301989888~12582912,318767104~4194304,360710144~12582912,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.723 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 11 diff [8388608~29360128,41943040~8388608,54525952~12582912,75497472~8388608,88080384~12582912,104857600~20971520,163577856~33554432,205520896~16777216,230686720~12582912,251658240~8388608,272629760~16777216,301989888~8388608,318767104~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.723 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~29360128,41943040~8388608,54525952~12582912,75497472~8388608,88080384~12582912,104857600~20971520,163577856~33554432,205520896~16777216,230686720~12582912,251658240~8388608,272629760~16777216,301989888~8388608,318767104~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.730 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~58720256,75497472~25165824,104857600~25165824,163577856~37748736,205520896~16777216,230686720~12582912,251658240~8388608,272629760~20971520,301989888~12582912,318767104~4194304,360710144~12582912,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.730 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 12 diff [8388608~29360128,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,163577856~33554432,205520896~16777216,230686720~16777216,251658240~8388608,272629760~25165824,301989888~8388608,318767104~4194304,348127232~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.730 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~29360128,41943040~8388608,54525952~12582912,75497472~8388608,88080384~37748736,163577856~33554432,205520896~16777216,230686720~16777216,251658240~8388608,272629760~25165824,301989888~8388608,318767104~4194304,348127232~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.739 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~58720256,75497472~54525952,163577856~37748736,205520896~16777216,230686720~29360128,272629760~25165824,301989888~12582912,318767104~4194304,348127232~8388608,360710144~12582912,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.739 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 13 diff [0~37748736,41943040~8388608,54525952~16777216,75497472~8388608,88080384~37748736,134217728~12582912,163577856~33554432,205520896~16777216,230686720~16777216,251658240~8388608,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.739 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~8388608,54525952~16777216,75497472~8388608,88080384~37748736,134217728~12582912,163577856~33554432,205520896~16777216,230686720~16777216,251658240~8388608,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.749 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~12582912,163577856~37748736,205520896~16777216,230686720~29360128,272629760~25165824,301989888~20971520,331350016~12582912,348127232~25165824,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.749 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 14 diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~16777216,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.749 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~16777216,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.760 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~20971520,163577856~37748736,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,331350016~12582912,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.760 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 15 diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~16777216,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.760 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~8388608,88080384~37748736,134217728~16777216,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.772 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~20971520,163577856~37748736,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.773 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 16 diff [0~37748736,41943040~41943040,88080384~37748736,134217728~16777216,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.773 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,134217728~16777216,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.785 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~130023424,134217728~20971520,163577856~37748736,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.785 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 17 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~20971520,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.786 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~20971520,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.799 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~155189248,163577856~37748736,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.799 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 18 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.799 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.813 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.813 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to 19 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.813 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:45 smithi099 bash[84576]: cluster 2024-09-13T14:43:44.393704+0000 mgr.y (mgr.26252) 518 : cluster [DBG] pgmap v942: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 50 MiB/s wr, 380 op/s 2024-09-13T14:43:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:45 smithi099 bash[84576]: cluster 2024-09-13T14:43:44.393704+0000 mgr.y (mgr.26252) 518 : cluster [DBG] pgmap v942: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 50 MiB/s wr, 380 op/s 2024-09-13T14:43:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:45 smithi099 bash[84576]: cluster 2024-09-13T14:43:44.520367+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-13T14:43:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:45 smithi099 bash[84576]: cluster 2024-09-13T14:43:44.520367+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-13T14:43:45.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:45 smithi099 bash[87147]: cluster 2024-09-13T14:43:44.393704+0000 mgr.y (mgr.26252) 518 : cluster [DBG] pgmap v942: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 50 MiB/s wr, 380 op/s 2024-09-13T14:43:45.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:45 smithi099 bash[87147]: cluster 2024-09-13T14:43:44.393704+0000 mgr.y (mgr.26252) 518 : cluster [DBG] pgmap v942: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 50 MiB/s wr, 380 op/s 2024-09-13T14:43:45.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:45 smithi099 bash[87147]: cluster 2024-09-13T14:43:44.520367+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-13T14:43:45.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:45 smithi099 bash[87147]: cluster 2024-09-13T14:43:44.520367+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-13T14:43:45.829 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.829 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 8 diff [12582912~4194304,29360128~4194304,41943040~4194304,54525952~8388608,75497472~8388608,218103808~4194304,272629760~8388608,318767104~4194304,360710144~8388608] 2024-09-13T14:43:45.829 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [12582912~4194304,29360128~4194304,41943040~4194304,54525952~8388608,75497472~8388608,218103808~4194304,272629760~8388608,318767104~4194304,360710144~8388608] 2024-09-13T14:43:45.832 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [12582912~8388608,29360128~8388608,41943040~8388608,54525952~12582912,75497472~12582912,218103808~4194304,272629760~12582912,318767104~4194304,360710144~12582912] 2024-09-13T14:43:45.833 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 9 diff [12582912~4194304,29360128~8388608,41943040~8388608,54525952~12582912,75497472~8388608,104857600~8388608,117440512~8388608,167772160~8388608,192937984~4194304,218103808~4194304,251658240~8388608,272629760~8388608,318767104~4194304,360710144~8388608] 2024-09-13T14:43:45.833 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [12582912~4194304,29360128~8388608,41943040~8388608,54525952~12582912,75497472~8388608,104857600~8388608,117440512~8388608,167772160~8388608,192937984~4194304,218103808~4194304,251658240~8388608,272629760~8388608,318767104~4194304,360710144~8388608] 2024-09-13T14:43:45.838 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [12582912~8388608,29360128~37748736,75497472~12582912,104857600~25165824,167772160~12582912,192937984~8388608,218103808~4194304,251658240~8388608,272629760~12582912,318767104~4194304,360710144~12582912] 2024-09-13T14:43:45.838 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 10 diff [8388608~8388608,29360128~8388608,41943040~8388608,54525952~12582912,75497472~8388608,104857600~20971520,167772160~8388608,192937984~4194304,205520896~8388608,218103808~4194304,251658240~8388608,272629760~16777216,301989888~8388608,318767104~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.838 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~8388608,29360128~8388608,41943040~8388608,54525952~12582912,75497472~8388608,104857600~20971520,167772160~8388608,192937984~4194304,205520896~8388608,218103808~4194304,251658240~8388608,272629760~16777216,301989888~8388608,318767104~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.843 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~12582912,29360128~37748736,75497472~12582912,104857600~25165824,167772160~12582912,192937984~8388608,205520896~8388608,218103808~4194304,251658240~8388608,272629760~20971520,301989888~12582912,318767104~4194304,360710144~12582912,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.843 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 11 diff [8388608~12582912,29360128~8388608,41943040~8388608,54525952~12582912,75497472~8388608,88080384~8388608,104857600~20971520,163577856~16777216,184549376~12582912,205520896~16777216,230686720~12582912,251658240~8388608,272629760~16777216,301989888~8388608,318767104~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.843 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~12582912,29360128~8388608,41943040~8388608,54525952~12582912,75497472~8388608,88080384~8388608,104857600~20971520,163577856~16777216,184549376~12582912,205520896~16777216,230686720~12582912,251658240~8388608,272629760~16777216,301989888~8388608,318767104~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.849 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~16777216,29360128~37748736,75497472~25165824,104857600~25165824,163577856~16777216,184549376~16777216,205520896~16777216,230686720~12582912,251658240~8388608,272629760~20971520,301989888~12582912,318767104~4194304,360710144~12582912,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.849 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 12 diff [8388608~12582912,29360128~8388608,41943040~8388608,54525952~12582912,75497472~8388608,88080384~8388608,100663296~25165824,163577856~16777216,184549376~12582912,205520896~16777216,230686720~16777216,251658240~8388608,272629760~25165824,301989888~8388608,318767104~4194304,348127232~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.849 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~12582912,29360128~8388608,41943040~8388608,54525952~12582912,75497472~8388608,88080384~8388608,100663296~25165824,163577856~16777216,184549376~12582912,205520896~16777216,230686720~16777216,251658240~8388608,272629760~25165824,301989888~8388608,318767104~4194304,348127232~4194304,360710144~12582912,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.856 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~16777216,29360128~37748736,75497472~54525952,163577856~16777216,184549376~16777216,205520896~16777216,230686720~29360128,272629760~25165824,301989888~12582912,318767104~4194304,348127232~8388608,360710144~12582912,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.857 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 13 diff [0~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~8388608,88080384~8388608,100663296~25165824,134217728~12582912,163577856~16777216,184549376~12582912,205520896~16777216,230686720~16777216,251658240~8388608,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.857 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~8388608,88080384~8388608,100663296~25165824,134217728~12582912,163577856~16777216,184549376~12582912,205520896~16777216,230686720~16777216,251658240~8388608,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:45 smithi177 bash[68594]: cluster 2024-09-13T14:43:44.393704+0000 mgr.y (mgr.26252) 518 : cluster [DBG] pgmap v942: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 50 MiB/s wr, 380 op/s 2024-09-13T14:43:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:45 smithi177 bash[68594]: cluster 2024-09-13T14:43:44.393704+0000 mgr.y (mgr.26252) 518 : cluster [DBG] pgmap v942: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 50 MiB/s wr, 380 op/s 2024-09-13T14:43:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:45 smithi177 bash[68594]: cluster 2024-09-13T14:43:44.520367+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-13T14:43:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:45 smithi177 bash[68594]: cluster 2024-09-13T14:43:44.520367+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-13T14:43:45.865 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~25165824,29360128~41943040,75497472~54525952,134217728~12582912,163577856~37748736,205520896~16777216,230686720~29360128,272629760~25165824,301989888~20971520,331350016~12582912,348127232~25165824,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.865 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 14 diff [0~20971520,25165824~12582912,41943040~29360128,75497472~8388608,88080384~8388608,100663296~25165824,134217728~16777216,163577856~16777216,184549376~12582912,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.865 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~12582912,41943040~29360128,75497472~8388608,88080384~8388608,100663296~25165824,134217728~16777216,163577856~16777216,184549376~12582912,205520896~16777216,230686720~16777216,251658240~12582912,272629760~25165824,301989888~20971520,331350016~8388608,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.874 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~20971520,163577856~37748736,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,331350016~12582912,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.874 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 15 diff [0~20971520,25165824~12582912,41943040~29360128,75497472~8388608,88080384~8388608,100663296~25165824,134217728~16777216,163577856~16777216,184549376~12582912,205520896~16777216,230686720~16777216,251658240~12582912,272629760~20971520,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.874 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~12582912,41943040~29360128,75497472~8388608,88080384~8388608,100663296~25165824,134217728~16777216,163577856~16777216,184549376~12582912,205520896~16777216,230686720~16777216,251658240~12582912,272629760~20971520,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.885 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~20971520,163577856~37748736,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.885 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 16 diff [0~20971520,25165824~12582912,41943040~41943040,88080384~8388608,100663296~25165824,134217728~16777216,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~20971520,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.885 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~12582912,41943040~41943040,88080384~8388608,100663296~25165824,134217728~16777216,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~20971520,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.896 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~130023424,134217728~20971520,163577856~37748736,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.896 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 17 diff [0~37748736,41943040~41943040,88080384~37748736,130023424~20971520,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~20971520,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.896 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~20971520,163577856~33554432,205520896~16777216,230686720~16777216,251658240~12582912,272629760~20971520,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.909 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~155189248,163577856~37748736,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.909 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 18 diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~20971520,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.909 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~20971520,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.924 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~25165824,301989888~20971520,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.924 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to 19 diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.924 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.938 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:45.938 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 9 diff [33554432~4194304,41943040~8388608,54525952~8388608,75497472~4194304,83886080~4194304,104857600~8388608,117440512~8388608,167772160~8388608,192937984~4194304,251658240~8388608] 2024-09-13T14:43:45.938 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [33554432~4194304,41943040~8388608,54525952~8388608,75497472~4194304,83886080~4194304,104857600~8388608,117440512~8388608,167772160~8388608,192937984~4194304,251658240~8388608] 2024-09-13T14:43:45.942 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [33554432~29360128,75497472~4194304,83886080~4194304,104857600~25165824,167772160~12582912,192937984~8388608,251658240~8388608] 2024-09-13T14:43:45.942 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 10 diff [8388608~4194304,29360128~8388608,41943040~8388608,54525952~8388608,75497472~4194304,83886080~4194304,104857600~20971520,167772160~8388608,192937984~4194304,205520896~8388608,251658240~8388608,281018368~8388608,301989888~8388608,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.942 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~4194304,29360128~8388608,41943040~8388608,54525952~8388608,75497472~4194304,83886080~4194304,104857600~20971520,167772160~8388608,192937984~4194304,205520896~8388608,251658240~8388608,281018368~8388608,301989888~8388608,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.947 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~4194304,29360128~33554432,75497472~4194304,83886080~4194304,104857600~25165824,167772160~12582912,192937984~8388608,205520896~8388608,251658240~8388608,281018368~12582912,301989888~12582912,364904448~8388608,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.947 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 11 diff [8388608~12582912,29360128~8388608,41943040~8388608,54525952~8388608,75497472~4194304,83886080~12582912,104857600~20971520,163577856~16777216,184549376~12582912,205520896~12582912,230686720~12582912,251658240~8388608,281018368~8388608,301989888~8388608,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.947 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~12582912,29360128~8388608,41943040~8388608,54525952~8388608,75497472~4194304,83886080~12582912,104857600~20971520,163577856~16777216,184549376~12582912,205520896~12582912,230686720~12582912,251658240~8388608,281018368~8388608,301989888~8388608,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.952 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~16777216,29360128~33554432,75497472~4194304,83886080~16777216,104857600~25165824,163577856~16777216,184549376~16777216,205520896~12582912,230686720~12582912,251658240~8388608,281018368~12582912,301989888~12582912,364904448~8388608,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.952 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 12 diff [8388608~12582912,29360128~8388608,41943040~8388608,54525952~12582912,75497472~4194304,83886080~12582912,100663296~25165824,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~8388608,281018368~16777216,301989888~8388608,348127232~4194304,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.952 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~12582912,29360128~8388608,41943040~8388608,54525952~12582912,75497472~4194304,83886080~12582912,100663296~25165824,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~8388608,281018368~16777216,301989888~8388608,348127232~4194304,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.959 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~16777216,29360128~37748736,75497472~54525952,163577856~16777216,184549376~16777216,205520896~12582912,230686720~29360128,281018368~16777216,301989888~12582912,348127232~8388608,364904448~8388608,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.960 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 13 diff [0~20971520,29360128~8388608,41943040~8388608,54525952~12582912,75497472~4194304,83886080~12582912,100663296~25165824,134217728~12582912,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~8388608,281018368~16777216,301989888~16777216,331350016~8388608,348127232~12582912,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.960 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,29360128~8388608,41943040~8388608,54525952~12582912,75497472~4194304,83886080~12582912,100663296~25165824,134217728~12582912,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~8388608,281018368~16777216,301989888~16777216,331350016~8388608,348127232~12582912,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:45.968 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~25165824,29360128~41943040,75497472~54525952,134217728~12582912,163577856~37748736,205520896~12582912,230686720~29360128,281018368~16777216,301989888~20971520,331350016~12582912,348127232~12582912,364904448~8388608,377487360~12582912,394264576~4194304] 2024-09-13T14:43:45.968 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 14 diff [0~20971520,25165824~12582912,41943040~25165824,75497472~4194304,83886080~12582912,100663296~25165824,134217728~16777216,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~12582912,281018368~16777216,301989888~16777216,331350016~8388608,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.968 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~12582912,41943040~25165824,75497472~4194304,83886080~12582912,100663296~25165824,134217728~16777216,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~12582912,281018368~16777216,301989888~16777216,331350016~8388608,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.977 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~20971520,163577856~37748736,205520896~12582912,230686720~37748736,281018368~16777216,301989888~20971520,331350016~12582912,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:45.977 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 15 diff [0~20971520,25165824~12582912,41943040~25165824,75497472~4194304,83886080~12582912,100663296~25165824,134217728~16777216,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.977 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~12582912,41943040~25165824,75497472~4194304,83886080~12582912,100663296~25165824,134217728~16777216,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.987 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~71303168,75497472~54525952,134217728~20971520,163577856~37748736,205520896~12582912,230686720~37748736,281018368~16777216,301989888~20971520,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:45.987 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 16 diff [0~20971520,25165824~12582912,41943040~29360128,75497472~4194304,83886080~12582912,100663296~25165824,134217728~16777216,163577856~33554432,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.988 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~12582912,41943040~29360128,75497472~4194304,83886080~12582912,100663296~25165824,134217728~16777216,163577856~33554432,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.999 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~130023424,134217728~20971520,163577856~37748736,205520896~12582912,230686720~37748736,281018368~16777216,301989888~20971520,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:45.999 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 17 diff [0~37748736,41943040~29360128,75497472~50331648,130023424~20971520,163577856~33554432,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:45.999 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~50331648,130023424~20971520,163577856~33554432,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.012 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~155189248,163577856~37748736,205520896~12582912,230686720~37748736,281018368~16777216,301989888~20971520,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.012 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 18 diff [0~71303168,75497472~50331648,130023424~67108864,205520896~12582912,230686720~33554432,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.012 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~12582912,230686720~33554432,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.025 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~12582912,230686720~37748736,281018368~16777216,301989888~20971520,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.025 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to 19 diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.025 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.039 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.039 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 10 diff [8388608~4194304,29360128~4194304,113246208~4194304,192937984~4194304,205520896~8388608,281018368~8388608,301989888~8388608,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:46.039 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~4194304,29360128~4194304,113246208~4194304,192937984~4194304,205520896~8388608,281018368~8388608,301989888~8388608,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:46.043 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~4194304,29360128~8388608,113246208~8388608,192937984~8388608,205520896~8388608,281018368~12582912,301989888~12582912,364904448~8388608,377487360~12582912,394264576~4194304] 2024-09-13T14:43:46.043 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 11 diff [8388608~12582912,29360128~8388608,54525952~4194304,88080384~8388608,113246208~4194304,163577856~16777216,184549376~12582912,205520896~12582912,230686720~12582912,281018368~8388608,301989888~8388608,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:46.043 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~12582912,29360128~8388608,54525952~4194304,88080384~8388608,113246208~4194304,163577856~16777216,184549376~12582912,205520896~12582912,230686720~12582912,281018368~8388608,301989888~8388608,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:46.047 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~16777216,29360128~8388608,54525952~8388608,88080384~12582912,113246208~8388608,163577856~16777216,184549376~16777216,205520896~12582912,230686720~12582912,281018368~12582912,301989888~12582912,364904448~8388608,377487360~12582912,394264576~4194304] 2024-09-13T14:43:46.047 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 12 diff [8388608~12582912,29360128~8388608,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,113246208~4194304,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~4194304,281018368~16777216,301989888~8388608,348127232~4194304,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:46.048 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~12582912,29360128~8388608,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,113246208~4194304,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~4194304,281018368~16777216,301989888~8388608,348127232~4194304,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:46.053 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~16777216,29360128~8388608,54525952~12582912,75497472~8388608,88080384~16777216,113246208~8388608,163577856~16777216,184549376~16777216,205520896~12582912,230686720~29360128,281018368~16777216,301989888~12582912,348127232~8388608,364904448~8388608,377487360~12582912,394264576~4194304] 2024-09-13T14:43:46.053 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 13 diff [0~20971520,29360128~8388608,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,113246208~4194304,134217728~12582912,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~4194304,281018368~16777216,301989888~16777216,331350016~8388608,348127232~12582912,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:46.053 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,29360128~8388608,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,113246208~4194304,134217728~12582912,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~4194304,281018368~16777216,301989888~16777216,331350016~8388608,348127232~12582912,364904448~8388608,377487360~8388608,394264576~4194304] 2024-09-13T14:43:46.059 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~25165824,29360128~8388608,54525952~16777216,75497472~8388608,88080384~16777216,113246208~8388608,134217728~12582912,163577856~37748736,205520896~12582912,230686720~29360128,281018368~16777216,301989888~20971520,331350016~12582912,348127232~12582912,364904448~8388608,377487360~12582912,394264576~4194304] 2024-09-13T14:43:46.060 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 14 diff [0~20971520,25165824~12582912,41943040~8388608,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,109051904~8388608,134217728~16777216,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~12582912,281018368~16777216,301989888~16777216,331350016~8388608,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.060 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~12582912,41943040~8388608,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,109051904~8388608,134217728~16777216,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~12582912,281018368~16777216,301989888~16777216,331350016~8388608,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.068 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~37748736,41943040~8388608,54525952~16777216,75497472~8388608,88080384~16777216,109051904~12582912,134217728~20971520,163577856~37748736,205520896~12582912,230686720~37748736,281018368~16777216,301989888~20971520,331350016~12582912,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.068 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 15 diff [0~20971520,25165824~12582912,41943040~25165824,75497472~4194304,88080384~8388608,100663296~4194304,109051904~8388608,134217728~16777216,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.068 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~12582912,41943040~25165824,75497472~4194304,88080384~8388608,100663296~4194304,109051904~8388608,134217728~16777216,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.077 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~37748736,41943040~29360128,75497472~8388608,88080384~16777216,109051904~12582912,134217728~20971520,163577856~37748736,205520896~12582912,230686720~37748736,281018368~16777216,301989888~20971520,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.077 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 16 diff [0~20971520,25165824~12582912,41943040~29360128,75497472~4194304,88080384~8388608,100663296~4194304,109051904~8388608,134217728~16777216,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.077 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~12582912,41943040~29360128,75497472~4194304,88080384~8388608,100663296~4194304,109051904~8388608,134217728~16777216,163577856~16777216,184549376~12582912,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.087 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~37748736,41943040~41943040,88080384~16777216,109051904~12582912,134217728~20971520,163577856~37748736,205520896~12582912,230686720~37748736,281018368~16777216,301989888~20971520,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.087 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 17 diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~20971520,163577856~33554432,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.087 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~20971520,163577856~33554432,205520896~12582912,230686720~16777216,251658240~12582912,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.098 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~37748736,41943040~41943040,88080384~16777216,109051904~12582912,130023424~25165824,163577856~37748736,205520896~12582912,230686720~37748736,281018368~16777216,301989888~20971520,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.098 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 18 diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~12582912,230686720~33554432,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.099 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~12582912,230686720~33554432,281018368~12582912,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.111 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~12582912,230686720~37748736,281018368~16777216,301989888~20971520,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.111 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to 19 diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.111 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.124 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.125 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 11 diff [12582912~8388608,29360128~4194304,54525952~4194304,88080384~8388608,163577856~16777216,184549376~8388608,209715200~8388608,230686720~12582912] 2024-09-13T14:43:46.125 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [12582912~8388608,29360128~4194304,54525952~4194304,88080384~8388608,163577856~16777216,184549376~8388608,209715200~8388608,230686720~12582912] 2024-09-13T14:43:46.128 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [12582912~12582912,29360128~8388608,54525952~8388608,88080384~12582912,163577856~16777216,184549376~12582912,209715200~8388608,230686720~12582912] 2024-09-13T14:43:46.128 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 12 diff [12582912~8388608,29360128~4194304,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,163577856~16777216,184549376~8388608,209715200~8388608,230686720~16777216,251658240~4194304,289406976~8388608,348127232~4194304,369098752~4194304] 2024-09-13T14:43:46.128 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [12582912~8388608,29360128~4194304,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,163577856~16777216,184549376~8388608,209715200~8388608,230686720~16777216,251658240~4194304,289406976~8388608,348127232~4194304,369098752~4194304] 2024-09-13T14:43:46.133 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [12582912~12582912,29360128~8388608,54525952~12582912,75497472~8388608,88080384~16777216,163577856~16777216,184549376~12582912,209715200~8388608,230686720~29360128,289406976~8388608,348127232~8388608,369098752~4194304] 2024-09-13T14:43:46.133 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 13 diff [0~8388608,12582912~8388608,29360128~4194304,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,134217728~12582912,163577856~16777216,184549376~8388608,209715200~8388608,230686720~16777216,251658240~4194304,289406976~8388608,310378496~8388608,331350016~8388608,348127232~12582912,369098752~4194304] 2024-09-13T14:43:46.133 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,12582912~8388608,29360128~4194304,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,134217728~12582912,163577856~16777216,184549376~8388608,209715200~8388608,230686720~16777216,251658240~4194304,289406976~8388608,310378496~8388608,331350016~8388608,348127232~12582912,369098752~4194304] 2024-09-13T14:43:46.138 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~12582912,29360128~8388608,54525952~16777216,75497472~8388608,88080384~16777216,134217728~12582912,163577856~33554432,209715200~8388608,230686720~29360128,289406976~8388608,310378496~12582912,331350016~12582912,348127232~12582912,369098752~4194304] 2024-09-13T14:43:46.138 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 14 diff [0~8388608,12582912~8388608,25165824~8388608,41943040~8388608,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,109051904~4194304,134217728~16777216,163577856~16777216,184549376~8388608,209715200~8388608,230686720~16777216,251658240~12582912,289406976~8388608,310378496~8388608,331350016~8388608,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.138 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,12582912~8388608,25165824~8388608,41943040~8388608,54525952~12582912,75497472~4194304,88080384~8388608,100663296~4194304,109051904~4194304,134217728~16777216,163577856~16777216,184549376~8388608,209715200~8388608,230686720~16777216,251658240~12582912,289406976~8388608,310378496~8388608,331350016~8388608,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.144 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,12582912~25165824,41943040~8388608,54525952~16777216,75497472~8388608,88080384~16777216,109051904~4194304,134217728~20971520,163577856~33554432,209715200~8388608,230686720~37748736,289406976~8388608,310378496~12582912,331350016~12582912,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.145 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 15 diff [0~20971520,25165824~8388608,41943040~25165824,75497472~4194304,88080384~8388608,100663296~4194304,109051904~8388608,134217728~16777216,163577856~16777216,184549376~12582912,209715200~8388608,230686720~16777216,251658240~12582912,285212672~8388608,310378496~8388608,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.145 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~8388608,41943040~25165824,75497472~4194304,88080384~8388608,100663296~4194304,109051904~8388608,134217728~16777216,163577856~16777216,184549376~12582912,209715200~8388608,230686720~16777216,251658240~12582912,285212672~8388608,310378496~8388608,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.153 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~37748736,41943040~29360128,75497472~8388608,88080384~16777216,109051904~12582912,134217728~20971520,163577856~33554432,209715200~8388608,230686720~37748736,285212672~12582912,310378496~12582912,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.153 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 16 diff [0~20971520,25165824~8388608,41943040~29360128,75497472~4194304,88080384~8388608,100663296~4194304,109051904~8388608,134217728~16777216,163577856~16777216,184549376~12582912,209715200~8388608,230686720~16777216,251658240~12582912,285212672~8388608,306184192~12582912,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.153 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~20971520,25165824~8388608,41943040~29360128,75497472~4194304,88080384~8388608,100663296~4194304,109051904~8388608,134217728~16777216,163577856~16777216,184549376~12582912,209715200~8388608,230686720~16777216,251658240~12582912,285212672~8388608,306184192~12582912,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.161 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~37748736,41943040~41943040,88080384~16777216,109051904~12582912,134217728~20971520,163577856~33554432,209715200~8388608,230686720~37748736,285212672~12582912,306184192~16777216,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.161 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 17 diff [0~33554432,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~20971520,163577856~33554432,209715200~8388608,230686720~16777216,251658240~12582912,285212672~8388608,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.161 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~33554432,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~20971520,163577856~33554432,209715200~8388608,230686720~16777216,251658240~12582912,285212672~8388608,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.173 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~37748736,41943040~41943040,88080384~16777216,109051904~12582912,130023424~25165824,163577856~33554432,209715200~8388608,230686720~37748736,285212672~12582912,301989888~20971520,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.173 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 18 diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~8388608,230686720~33554432,285212672~8388608,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.173 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~8388608,230686720~33554432,285212672~8388608,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.184 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~67108864,205520896~12582912,230686720~37748736,285212672~12582912,301989888~20971520,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.184 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to 19 diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.184 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.196 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~67108864,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.197 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 12 diff [33554432~4194304,54525952~12582912,75497472~4194304,100663296~4194304,243269632~4194304,251658240~4194304,289406976~8388608,348127232~4194304,369098752~4194304] 2024-09-13T14:43:46.197 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [33554432~4194304,54525952~12582912,75497472~4194304,100663296~4194304,243269632~4194304,251658240~4194304,289406976~8388608,348127232~4194304,369098752~4194304] 2024-09-13T14:43:46.201 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [33554432~4194304,54525952~12582912,75497472~8388608,100663296~4194304,243269632~16777216,289406976~8388608,348127232~8388608,369098752~4194304] 2024-09-13T14:43:46.201 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 13 diff [0~8388608,33554432~4194304,54525952~12582912,75497472~4194304,100663296~4194304,134217728~12582912,163577856~4194304,171966464~8388608,239075328~8388608,251658240~4194304,289406976~8388608,310378496~8388608,331350016~8388608,348127232~12582912,369098752~4194304] 2024-09-13T14:43:46.201 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,33554432~4194304,54525952~12582912,75497472~4194304,100663296~4194304,134217728~12582912,163577856~4194304,171966464~8388608,239075328~8388608,251658240~4194304,289406976~8388608,310378496~8388608,331350016~8388608,348127232~12582912,369098752~4194304] 2024-09-13T14:43:46.205 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,33554432~4194304,54525952~16777216,75497472~8388608,100663296~4194304,134217728~12582912,163577856~4194304,171966464~12582912,239075328~20971520,289406976~8388608,310378496~12582912,331350016~12582912,348127232~12582912,369098752~4194304] 2024-09-13T14:43:46.205 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 14 diff [0~8388608,25165824~4194304,33554432~4194304,41943040~8388608,54525952~12582912,75497472~4194304,100663296~4194304,109051904~4194304,134217728~16777216,163577856~4194304,171966464~8388608,188743680~4194304,230686720~4194304,239075328~8388608,251658240~12582912,289406976~8388608,310378496~8388608,331350016~8388608,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.205 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,25165824~4194304,33554432~4194304,41943040~8388608,54525952~12582912,75497472~4194304,100663296~4194304,109051904~4194304,134217728~16777216,163577856~4194304,171966464~8388608,188743680~4194304,230686720~4194304,239075328~8388608,251658240~12582912,289406976~8388608,310378496~8388608,331350016~8388608,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.211 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,25165824~12582912,41943040~8388608,54525952~16777216,75497472~8388608,100663296~4194304,109051904~4194304,134217728~20971520,163577856~4194304,171966464~12582912,188743680~4194304,230686720~4194304,239075328~29360128,289406976~8388608,310378496~12582912,331350016~12582912,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.212 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 15 diff [0~16777216,25165824~4194304,33554432~4194304,41943040~25165824,75497472~4194304,100663296~4194304,109051904~8388608,134217728~16777216,163577856~4194304,171966464~8388608,184549376~8388608,230686720~4194304,239075328~8388608,251658240~12582912,285212672~8388608,310378496~8388608,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.212 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~16777216,25165824~4194304,33554432~4194304,41943040~25165824,75497472~4194304,100663296~4194304,109051904~8388608,134217728~16777216,163577856~4194304,171966464~8388608,184549376~8388608,230686720~4194304,239075328~8388608,251658240~12582912,285212672~8388608,310378496~8388608,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.219 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~20971520,25165824~12582912,41943040~29360128,75497472~8388608,100663296~4194304,109051904~12582912,134217728~20971520,163577856~4194304,171966464~20971520,230686720~4194304,239075328~29360128,285212672~12582912,310378496~12582912,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.219 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 16 diff [0~29360128,33554432~4194304,41943040~29360128,75497472~4194304,100663296~4194304,109051904~8388608,134217728~16777216,163577856~4194304,171966464~8388608,184549376~8388608,230686720~4194304,239075328~8388608,251658240~12582912,285212672~8388608,306184192~12582912,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.220 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~29360128,33554432~4194304,41943040~29360128,75497472~4194304,100663296~4194304,109051904~8388608,134217728~16777216,163577856~4194304,171966464~8388608,184549376~8388608,230686720~4194304,239075328~8388608,251658240~12582912,285212672~8388608,306184192~12582912,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.228 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~37748736,41943040~41943040,100663296~4194304,109051904~12582912,134217728~20971520,163577856~29360128,230686720~4194304,239075328~29360128,285212672~12582912,306184192~16777216,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.228 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 17 diff [0~29360128,33554432~4194304,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~20971520,163577856~4194304,171966464~20971520,230686720~16777216,251658240~12582912,285212672~8388608,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.228 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~29360128,33554432~4194304,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~20971520,163577856~4194304,171966464~20971520,230686720~16777216,251658240~12582912,285212672~8388608,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.237 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~37748736,41943040~41943040,96468992~8388608,109051904~12582912,130023424~25165824,163577856~29360128,230686720~37748736,285212672~12582912,301989888~20971520,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.237 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 18 diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~8388608,230686720~33554432,285212672~8388608,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.237 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~8388608,230686720~33554432,285212672~8388608,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.248 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~12582912,230686720~37748736,285212672~12582912,301989888~20971520,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.248 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to 19 diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.248 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.259 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.259 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 13 diff [0~8388608,58720256~8388608,134217728~12582912,163577856~4194304,171966464~8388608,239075328~8388608,310378496~8388608,331350016~8388608,348127232~8388608] 2024-09-13T14:43:46.259 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,58720256~8388608,134217728~12582912,163577856~4194304,171966464~8388608,239075328~8388608,310378496~8388608,331350016~8388608,348127232~8388608] 2024-09-13T14:43:46.263 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,58720256~12582912,134217728~12582912,163577856~4194304,171966464~12582912,239075328~8388608,310378496~12582912,331350016~12582912,348127232~12582912] 2024-09-13T14:43:46.263 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 14 diff [0~8388608,25165824~4194304,41943040~8388608,54525952~12582912,109051904~4194304,134217728~16777216,163577856~4194304,171966464~8388608,188743680~4194304,230686720~4194304,239075328~8388608,255852544~8388608,310378496~8388608,331350016~8388608,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.263 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~8388608,25165824~4194304,41943040~8388608,54525952~12582912,109051904~4194304,134217728~16777216,163577856~4194304,171966464~8388608,188743680~4194304,230686720~4194304,239075328~8388608,255852544~8388608,310378496~8388608,331350016~8388608,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.268 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~8388608,25165824~8388608,41943040~8388608,54525952~16777216,109051904~4194304,134217728~20971520,163577856~4194304,171966464~12582912,188743680~4194304,230686720~4194304,239075328~8388608,255852544~12582912,310378496~12582912,331350016~12582912,348127232~12582912,390070272~12582912] 2024-09-13T14:43:46.268 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 15 diff [0~16777216,25165824~4194304,41943040~25165824,109051904~8388608,134217728~16777216,163577856~4194304,171966464~8388608,184549376~8388608,230686720~4194304,239075328~8388608,255852544~8388608,285212672~8388608,310378496~8388608,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.268 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~16777216,25165824~4194304,41943040~25165824,109051904~8388608,134217728~16777216,163577856~4194304,171966464~8388608,184549376~8388608,230686720~4194304,239075328~8388608,255852544~8388608,285212672~8388608,310378496~8388608,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.275 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~20971520,25165824~8388608,41943040~29360128,109051904~12582912,134217728~20971520,163577856~4194304,171966464~20971520,230686720~4194304,239075328~8388608,255852544~12582912,285212672~12582912,310378496~12582912,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:46.275 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 16 diff [0~29360128,41943040~29360128,109051904~8388608,134217728~16777216,163577856~4194304,171966464~8388608,184549376~8388608,230686720~4194304,239075328~8388608,255852544~8388608,285212672~8388608,306184192~12582912,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.275 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~29360128,41943040~29360128,109051904~8388608,134217728~16777216,163577856~4194304,171966464~8388608,184549376~8388608,230686720~4194304,239075328~8388608,255852544~8388608,285212672~8388608,306184192~12582912,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.282 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~33554432,41943040~33554432,109051904~12582912,134217728~20971520,163577856~29360128,230686720~4194304,239075328~8388608,255852544~12582912,285212672~12582912,306184192~16777216,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:46.282 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 17 diff [0~29360128,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~20971520,163577856~4194304,171966464~20971520,230686720~16777216,255852544~8388608,285212672~8388608,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.282 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~29360128,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~20971520,163577856~4194304,171966464~20971520,230686720~16777216,255852544~8388608,285212672~8388608,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.290 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~33554432,41943040~41943040,96468992~8388608,109051904~12582912,130023424~25165824,163577856~29360128,230686720~16777216,255852544~12582912,285212672~12582912,301989888~20971520,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:46.290 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 18 diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~8388608,230686720~20971520,255852544~8388608,285212672~8388608,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.290 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~8388608,230686720~20971520,255852544~8388608,285212672~8388608,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.297 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~12582912,230686720~37748736,285212672~12582912,301989888~20971520,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:46.297 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to 19 diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.297 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.304 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:46.305 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 14 diff [25165824~4194304,41943040~8388608,54525952~12582912,109051904~4194304,142606336~8388608,188743680~4194304,230686720~4194304,255852544~8388608,390070272~8388608] 2024-09-13T14:43:46.305 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [25165824~4194304,41943040~8388608,54525952~12582912,109051904~4194304,142606336~8388608,188743680~4194304,230686720~4194304,255852544~8388608,390070272~8388608] 2024-09-13T14:43:46.308 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [25165824~8388608,41943040~8388608,54525952~16777216,109051904~4194304,142606336~12582912,188743680~4194304,230686720~4194304,255852544~12582912,390070272~12582912] 2024-09-13T14:43:46.308 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 15 diff [4194304~12582912,25165824~4194304,41943040~25165824,109051904~8388608,142606336~8388608,184549376~8388608,230686720~4194304,255852544~8388608,285212672~8388608,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.308 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~12582912,25165824~4194304,41943040~25165824,109051904~8388608,142606336~8388608,184549376~8388608,230686720~4194304,255852544~8388608,285212672~8388608,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.313 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~16777216,25165824~8388608,41943040~29360128,109051904~12582912,142606336~12582912,184549376~8388608,230686720~4194304,255852544~12582912,285212672~12582912,327155712~12582912,390070272~12582912] 2024-09-13T14:43:46.313 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 16 diff [4194304~25165824,41943040~29360128,109051904~8388608,134217728~16777216,163577856~4194304,184549376~8388608,230686720~4194304,255852544~8388608,285212672~8388608,306184192~4194304,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.313 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~25165824,41943040~29360128,109051904~8388608,134217728~16777216,163577856~4194304,184549376~8388608,230686720~4194304,255852544~8388608,285212672~8388608,306184192~4194304,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.318 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~29360128,41943040~33554432,109051904~12582912,134217728~20971520,163577856~8388608,184549376~8388608,230686720~4194304,255852544~12582912,285212672~12582912,306184192~4194304,327155712~12582912,390070272~12582912] 2024-09-13T14:43:46.318 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 17 diff [4194304~25165824,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~20971520,163577856~4194304,171966464~8388608,184549376~8388608,230686720~8388608,255852544~8388608,285212672~8388608,301989888~8388608,314572800~4194304,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.318 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~25165824,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~20971520,163577856~4194304,171966464~8388608,184549376~8388608,230686720~8388608,255852544~8388608,285212672~8388608,301989888~8388608,314572800~4194304,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.323 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~29360128,41943040~41943040,96468992~8388608,109051904~12582912,130023424~25165824,163577856~16777216,184549376~8388608,230686720~8388608,255852544~12582912,285212672~12582912,301989888~8388608,314572800~4194304,327155712~12582912,390070272~12582912] 2024-09-13T14:43:46.324 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 18 diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~50331648,184549376~8388608,205520896~8388608,230686720~8388608,243269632~8388608,255852544~8388608,285212672~8388608,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.324 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~50331648,184549376~8388608,205520896~8388608,230686720~8388608,243269632~8388608,255852544~8388608,285212672~8388608,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.330 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~50331648,184549376~8388608,205520896~12582912,230686720~8388608,243269632~25165824,285212672~12582912,301989888~20971520,327155712~12582912,390070272~12582912] 2024-09-13T14:43:46.330 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to 19 diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.330 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.336 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~8388608,243269632~25165824,285212672~37748736,327155712~12582912,390070272~12582912] 2024-09-13T14:43:46.336 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to 15 diff [4194304~12582912,41943040~8388608,58720256~4194304,109051904~8388608,146800640~4194304,184549376~4194304,285212672~8388608,327155712~8388608] 2024-09-13T14:43:46.336 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~12582912,41943040~8388608,58720256~4194304,109051904~8388608,146800640~4194304,184549376~4194304,285212672~8388608,327155712~8388608] 2024-09-13T14:43:46.340 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~16777216,41943040~12582912,58720256~4194304,109051904~12582912,146800640~4194304,184549376~4194304,285212672~12582912,327155712~12582912] 2024-09-13T14:43:46.340 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to 16 diff [4194304~20971520,41943040~8388608,54525952~8388608,67108864~4194304,109051904~8388608,134217728~8388608,146800640~4194304,163577856~4194304,184549376~4194304,285212672~8388608,306184192~4194304,327155712~8388608] 2024-09-13T14:43:46.340 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~20971520,41943040~8388608,54525952~8388608,67108864~4194304,109051904~8388608,134217728~8388608,146800640~4194304,163577856~4194304,184549376~4194304,285212672~8388608,306184192~4194304,327155712~8388608] 2024-09-13T14:43:46.344 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~20971520,41943040~20971520,67108864~8388608,109051904~12582912,134217728~8388608,146800640~4194304,163577856~8388608,184549376~4194304,285212672~12582912,306184192~4194304,327155712~12582912] 2024-09-13T14:43:46.344 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to 17 diff [4194304~20971520,41943040~8388608,54525952~8388608,67108864~4194304,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~4194304,163577856~4194304,171966464~8388608,184549376~4194304,230686720~8388608,285212672~8388608,301989888~8388608,314572800~4194304,327155712~8388608] 2024-09-13T14:43:46.344 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~20971520,41943040~8388608,54525952~8388608,67108864~4194304,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~4194304,163577856~4194304,171966464~8388608,184549376~4194304,230686720~8388608,285212672~8388608,301989888~8388608,314572800~4194304,327155712~8388608] 2024-09-13T14:43:46.349 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~25165824,41943040~20971520,67108864~16777216,96468992~8388608,109051904~12582912,130023424~12582912,146800640~4194304,163577856~16777216,184549376~4194304,230686720~8388608,285212672~12582912,301989888~8388608,314572800~4194304,327155712~12582912] 2024-09-13T14:43:46.349 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to 18 diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~8388608,67108864~4194304,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~33554432,184549376~4194304,205520896~8388608,230686720~8388608,243269632~8388608,285212672~8388608,301989888~16777216,327155712~8388608] 2024-09-13T14:43:46.349 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~8388608,67108864~4194304,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~33554432,184549376~4194304,205520896~8388608,230686720~8388608,243269632~8388608,285212672~8388608,301989888~16777216,327155712~8388608] 2024-09-13T14:43:46.354 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~58720256,67108864~16777216,96468992~8388608,109051904~12582912,130023424~12582912,146800640~33554432,184549376~4194304,205520896~12582912,230686720~8388608,243269632~12582912,285212672~12582912,301989888~20971520,327155712~12582912] 2024-09-13T14:43:46.354 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to 19 diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.354 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.359 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,285212672~37748736,327155712~12582912,390070272~8388608] 2024-09-13T14:43:46.360 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to 16 diff [8388608~16777216,50331648~12582912,67108864~4194304,134217728~8388608,163577856~4194304,289406976~4194304,306184192~4194304,327155712~4194304] 2024-09-13T14:43:46.360 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~16777216,50331648~12582912,67108864~4194304,134217728~8388608,163577856~4194304,289406976~4194304,306184192~4194304,327155712~4194304] 2024-09-13T14:43:46.362 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~16777216,50331648~12582912,67108864~8388608,134217728~8388608,163577856~8388608,289406976~4194304,306184192~4194304,327155712~4194304] 2024-09-13T14:43:46.362 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to 17 diff [8388608~16777216,50331648~12582912,67108864~4194304,75497472~4194304,96468992~8388608,130023424~12582912,163577856~4194304,171966464~8388608,230686720~8388608,289406976~4194304,301989888~8388608,314572800~4194304,327155712~4194304] 2024-09-13T14:43:46.362 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~16777216,50331648~12582912,67108864~4194304,75497472~4194304,96468992~8388608,130023424~12582912,163577856~4194304,171966464~8388608,230686720~8388608,289406976~4194304,301989888~8388608,314572800~4194304,327155712~4194304] 2024-09-13T14:43:46.365 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~20971520,50331648~12582912,67108864~16777216,96468992~8388608,130023424~12582912,163577856~16777216,230686720~8388608,289406976~4194304,301989888~8388608,314572800~4194304,327155712~4194304] 2024-09-13T14:43:46.365 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to 18 diff [8388608~16777216,29360128~8388608,50331648~12582912,67108864~4194304,75497472~4194304,96468992~8388608,130023424~12582912,146800640~33554432,205520896~8388608,230686720~8388608,243269632~8388608,289406976~4194304,301989888~16777216,327155712~4194304] 2024-09-13T14:43:46.365 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~16777216,29360128~8388608,50331648~12582912,67108864~4194304,75497472~4194304,96468992~8388608,130023424~12582912,146800640~33554432,205520896~8388608,230686720~8388608,243269632~8388608,289406976~4194304,301989888~16777216,327155712~4194304] 2024-09-13T14:43:46.370 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~33554432,50331648~12582912,67108864~16777216,96468992~8388608,130023424~12582912,146800640~33554432,205520896~12582912,230686720~8388608,243269632~12582912,289406976~4194304,301989888~20971520,327155712~4194304] 2024-09-13T14:43:46.370 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to 19 diff [8388608~16777216,29360128~8388608,50331648~20971520,75497472~4194304,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,289406976~29360128,327155712~4194304,390070272~8388608] 2024-09-13T14:43:46.370 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~16777216,29360128~8388608,50331648~20971520,75497472~4194304,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,289406976~29360128,327155712~4194304,390070272~8388608] 2024-09-13T14:43:46.374 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~33554432,50331648~33554432,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,289406976~33554432,327155712~4194304,390070272~8388608] 2024-09-13T14:43:46.374 INFO:tasks.workunit.client.0.smithi099.stdout:from 16 to 17 diff [16777216~8388608,54525952~4194304,75497472~4194304,96468992~8388608,130023424~4194304,171966464~8388608,230686720~8388608,301989888~4194304,314572800~4194304] 2024-09-13T14:43:46.374 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [16777216~8388608,54525952~4194304,75497472~4194304,96468992~8388608,130023424~4194304,171966464~8388608,230686720~8388608,301989888~4194304,314572800~4194304] 2024-09-13T14:43:46.377 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [16777216~12582912,54525952~8388608,75497472~8388608,96468992~8388608,130023424~4194304,171966464~8388608,230686720~8388608,301989888~4194304,314572800~4194304] 2024-09-13T14:43:46.377 INFO:tasks.workunit.client.0.smithi099.stdout:from 16 to 18 diff [16777216~8388608,29360128~8388608,54525952~4194304,67108864~4194304,75497472~4194304,96468992~8388608,130023424~4194304,146800640~16777216,167772160~12582912,205520896~8388608,230686720~8388608,243269632~8388608,301989888~4194304,310378496~8388608] 2024-09-13T14:43:46.378 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [16777216~8388608,29360128~8388608,54525952~4194304,67108864~4194304,75497472~4194304,96468992~8388608,130023424~4194304,146800640~16777216,167772160~12582912,205520896~8388608,230686720~8388608,243269632~8388608,301989888~4194304,310378496~8388608] 2024-09-13T14:43:46.381 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [16777216~25165824,54525952~8388608,67108864~16777216,96468992~8388608,130023424~4194304,146800640~33554432,205520896~12582912,230686720~8388608,243269632~12582912,301989888~4194304,310378496~12582912] 2024-09-13T14:43:46.381 INFO:tasks.workunit.client.0.smithi099.stdout:from 16 to 19 diff [16777216~8388608,29360128~8388608,54525952~4194304,62914560~8388608,75497472~4194304,96468992~8388608,130023424~4194304,146800640~16777216,167772160~20971520,205520896~16777216,230686720~8388608,243269632~8388608,293601280~12582912,310378496~8388608,390070272~8388608] 2024-09-13T14:43:46.381 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [16777216~8388608,29360128~8388608,54525952~4194304,62914560~8388608,75497472~4194304,96468992~8388608,130023424~4194304,146800640~16777216,167772160~20971520,205520896~16777216,230686720~8388608,243269632~8388608,293601280~12582912,310378496~8388608,390070272~8388608] 2024-09-13T14:43:46.386 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [16777216~25165824,54525952~29360128,96468992~8388608,130023424~4194304,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,293601280~12582912,310378496~12582912,390070272~8388608] 2024-09-13T14:43:46.386 INFO:tasks.workunit.client.0.smithi099.stdout:from 17 to 18 diff [16777216~4194304,29360128~8388608,67108864~4194304,146800640~16777216,167772160~12582912,205520896~8388608,243269632~8388608,310378496~8388608] 2024-09-13T14:43:46.386 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [16777216~4194304,29360128~8388608,67108864~4194304,146800640~16777216,167772160~12582912,205520896~8388608,243269632~8388608,310378496~8388608] 2024-09-13T14:43:46.390 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [16777216~8388608,29360128~12582912,67108864~8388608,146800640~33554432,205520896~12582912,243269632~12582912,310378496~12582912] 2024-09-13T14:43:46.390 INFO:tasks.workunit.client.0.smithi099.stdout:from 17 to 19 diff [16777216~4194304,29360128~8388608,62914560~8388608,146800640~16777216,167772160~20971520,205520896~16777216,243269632~8388608,293601280~8388608,310378496~8388608,390070272~8388608] 2024-09-13T14:43:46.390 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [16777216~4194304,29360128~8388608,62914560~8388608,146800640~16777216,167772160~20971520,205520896~16777216,243269632~8388608,293601280~8388608,310378496~8388608,390070272~8388608] 2024-09-13T14:43:46.393 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [16777216~8388608,29360128~12582912,62914560~12582912,146800640~41943040,205520896~16777216,243269632~12582912,293601280~8388608,310378496~12582912,390070272~8388608] 2024-09-13T14:43:46.393 INFO:tasks.workunit.client.0.smithi099.stdout:from 18 to 19 diff [62914560~8388608,155189248~4194304,167772160~8388608,180355072~8388608,213909504~8388608,293601280~8388608,390070272~8388608] 2024-09-13T14:43:46.393 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [62914560~8388608,155189248~4194304,167772160~8388608,180355072~8388608,213909504~8388608,293601280~8388608,390070272~8388608] 2024-09-13T14:43:46.396 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [62914560~8388608,155189248~4194304,167772160~8388608,180355072~8388608,213909504~8388608,293601280~8388608,390070272~8388608] 2024-09-13T14:43:46.543 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:46.543 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:46.582 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:46.582 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:46.582 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:46.600 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:46.600 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:46.601 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:46.617 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:46.617 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:46.617 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:46.634 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:46.634 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:46.634 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:46.650 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:46.650 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:46.650 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:46.665 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:46.665 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.665 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.679 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:46.680 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.680 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.693 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:46.693 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.693 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.704 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.705 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.705 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:46.715 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:46.715 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.716 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.726 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~67108864,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.726 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.726 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:46.736 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:46.736 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.736 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:46.744 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:46.744 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to HEAD diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.744 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.752 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~8388608,243269632~25165824,285212672~37748736,327155712~12582912,390070272~12582912] 2024-09-13T14:43:46.752 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to HEAD diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.752 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:46.758 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,285212672~37748736,327155712~12582912,390070272~8388608] 2024-09-13T14:43:46.759 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to HEAD diff [8388608~16777216,29360128~8388608,50331648~20971520,75497472~4194304,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,289406976~29360128,327155712~4194304,390070272~8388608] 2024-09-13T14:43:46.759 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~16777216,29360128~8388608,50331648~20971520,75497472~4194304,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,289406976~29360128,327155712~4194304,390070272~8388608] 2024-09-13T14:43:46.763 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~33554432,50331648~33554432,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,289406976~33554432,327155712~4194304,390070272~8388608] 2024-09-13T14:43:46.763 INFO:tasks.workunit.client.0.smithi099.stdout:from 16 to HEAD diff [16777216~8388608,29360128~8388608,54525952~4194304,62914560~8388608,75497472~4194304,96468992~8388608,130023424~4194304,146800640~16777216,167772160~20971520,205520896~16777216,230686720~8388608,243269632~8388608,293601280~12582912,310378496~8388608,390070272~8388608] 2024-09-13T14:43:46.764 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [16777216~8388608,29360128~8388608,54525952~4194304,62914560~8388608,75497472~4194304,96468992~8388608,130023424~4194304,146800640~16777216,167772160~20971520,205520896~16777216,230686720~8388608,243269632~8388608,293601280~12582912,310378496~8388608,390070272~8388608] 2024-09-13T14:43:46.767 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [16777216~25165824,54525952~29360128,96468992~8388608,130023424~4194304,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,293601280~12582912,310378496~12582912,390070272~8388608] 2024-09-13T14:43:46.767 INFO:tasks.workunit.client.0.smithi099.stdout:from 17 to HEAD diff [16777216~4194304,29360128~8388608,62914560~8388608,146800640~16777216,167772160~20971520,205520896~16777216,243269632~8388608,293601280~8388608,310378496~8388608,390070272~8388608] 2024-09-13T14:43:46.768 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [16777216~4194304,29360128~8388608,62914560~8388608,146800640~16777216,167772160~20971520,205520896~16777216,243269632~8388608,293601280~8388608,310378496~8388608,390070272~8388608] 2024-09-13T14:43:46.770 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [16777216~8388608,29360128~12582912,62914560~12582912,146800640~41943040,205520896~16777216,243269632~12582912,293601280~8388608,310378496~12582912,390070272~8388608] 2024-09-13T14:43:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:43:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:43:47] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.51.0" 2024-09-13T14:43:47.562 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:47.562 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:47.592 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:47.592 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:47.592 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:47.621 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:47.622 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:47.622 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:47.637 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:47.638 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:47.638 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:47.653 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:47.653 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:47.653 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:47.667 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:47.667 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:47.667 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:47.681 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:47.681 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:47.681 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:47.694 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:47.694 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:47.694 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:47.706 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:47.706 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:47.706 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:47.717 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:47.717 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:47.717 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:47.727 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:47.727 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:47.727 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:47.736 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~67108864,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:47.736 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:47.737 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:47.744 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:47.744 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:47.745 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:47.752 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:47.752 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to HEAD diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:47.752 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:47.759 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~8388608,243269632~25165824,285212672~37748736,327155712~12582912,390070272~12582912] 2024-09-13T14:43:47.759 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to HEAD diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:47.759 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:47.764 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,285212672~37748736,327155712~12582912,390070272~8388608] 2024-09-13T14:43:47.764 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to HEAD diff [8388608~16777216,29360128~8388608,50331648~20971520,75497472~4194304,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,289406976~29360128,327155712~4194304,390070272~8388608] 2024-09-13T14:43:47.764 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~16777216,29360128~8388608,50331648~20971520,75497472~4194304,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,289406976~29360128,327155712~4194304,390070272~8388608] 2024-09-13T14:43:47.769 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~33554432,50331648~33554432,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,289406976~33554432,327155712~4194304,390070272~8388608] 2024-09-13T14:43:47.769 INFO:tasks.workunit.client.0.smithi099.stdout:from 16 to HEAD diff [16777216~8388608,29360128~8388608,54525952~4194304,62914560~8388608,75497472~4194304,96468992~8388608,130023424~4194304,146800640~16777216,167772160~20971520,205520896~16777216,230686720~8388608,243269632~8388608,293601280~12582912,310378496~8388608,390070272~8388608] 2024-09-13T14:43:47.769 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [16777216~8388608,29360128~8388608,54525952~4194304,62914560~8388608,75497472~4194304,96468992~8388608,130023424~4194304,146800640~16777216,167772160~20971520,205520896~16777216,230686720~8388608,243269632~8388608,293601280~12582912,310378496~8388608,390070272~8388608] 2024-09-13T14:43:47.772 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [16777216~25165824,54525952~29360128,96468992~8388608,130023424~4194304,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,293601280~12582912,310378496~12582912,390070272~8388608] 2024-09-13T14:43:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:47 smithi099 bash[84576]: cluster 2024-09-13T14:43:46.394412+0000 mgr.y (mgr.26252) 519 : cluster [DBG] pgmap v944: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 198 KiB/s rd, 40 MiB/s wr, 309 op/s 2024-09-13T14:43:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:47 smithi099 bash[84576]: cluster 2024-09-13T14:43:46.394412+0000 mgr.y (mgr.26252) 519 : cluster [DBG] pgmap v944: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 198 KiB/s rd, 40 MiB/s wr, 309 op/s 2024-09-13T14:43:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:47 smithi099 bash[84576]: cluster 2024-09-13T14:43:46.530719+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-13T14:43:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:47 smithi099 bash[84576]: cluster 2024-09-13T14:43:46.530719+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-13T14:43:47.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:47 smithi099 bash[87147]: cluster 2024-09-13T14:43:46.394412+0000 mgr.y (mgr.26252) 519 : cluster [DBG] pgmap v944: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 198 KiB/s rd, 40 MiB/s wr, 309 op/s 2024-09-13T14:43:47.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:47 smithi099 bash[87147]: cluster 2024-09-13T14:43:46.394412+0000 mgr.y (mgr.26252) 519 : cluster [DBG] pgmap v944: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 198 KiB/s rd, 40 MiB/s wr, 309 op/s 2024-09-13T14:43:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:47 smithi099 bash[87147]: cluster 2024-09-13T14:43:46.530719+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-13T14:43:47.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:47 smithi099 bash[87147]: cluster 2024-09-13T14:43:46.530719+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-13T14:43:47.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:47 smithi177 bash[68594]: cluster 2024-09-13T14:43:46.394412+0000 mgr.y (mgr.26252) 519 : cluster [DBG] pgmap v944: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 198 KiB/s rd, 40 MiB/s wr, 309 op/s 2024-09-13T14:43:47.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:47 smithi177 bash[68594]: cluster 2024-09-13T14:43:46.394412+0000 mgr.y (mgr.26252) 519 : cluster [DBG] pgmap v944: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 198 KiB/s rd, 40 MiB/s wr, 309 op/s 2024-09-13T14:43:47.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:47 smithi177 bash[68594]: cluster 2024-09-13T14:43:46.530719+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-13T14:43:47.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:47 smithi177 bash[68594]: cluster 2024-09-13T14:43:46.530719+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-13T14:43:48.607 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:48.607 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:48.638 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:48.638 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:48.638 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:48.653 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:48.653 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:48.653 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:48.668 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:48.668 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:48.668 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:48.683 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:48.683 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:48.683 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:48.697 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:48.697 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:48.697 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:48.712 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:48.712 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:48.712 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:48.725 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:48.725 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:48.725 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:48.737 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:48.737 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:48.738 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:48.748 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:48.748 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:48.749 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:48.758 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:48.758 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:48.758 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:48.766 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~67108864,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:48.766 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:48.767 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:48.774 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:48.774 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:48.774 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:48.781 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:48.781 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to HEAD diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:48.781 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:48.787 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~8388608,243269632~25165824,285212672~37748736,327155712~12582912,390070272~12582912] 2024-09-13T14:43:48.787 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to HEAD diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:48.787 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:48.792 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,285212672~37748736,327155712~12582912,390070272~8388608] 2024-09-13T14:43:48.792 INFO:tasks.workunit.client.0.smithi099.stdout:from 15 to HEAD diff [8388608~16777216,29360128~8388608,50331648~20971520,75497472~4194304,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,289406976~29360128,327155712~4194304,390070272~8388608] 2024-09-13T14:43:48.792 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [8388608~16777216,29360128~8388608,50331648~20971520,75497472~4194304,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,289406976~29360128,327155712~4194304,390070272~8388608] 2024-09-13T14:43:48.795 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [8388608~33554432,50331648~33554432,96468992~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,289406976~33554432,327155712~4194304,390070272~8388608] 2024-09-13T14:43:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:48 smithi099 bash[84576]: cluster 2024-09-13T14:43:47.532721+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-13T14:43:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:48 smithi099 bash[84576]: cluster 2024-09-13T14:43:47.532721+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-13T14:43:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:48 smithi099 bash[84576]: audit 2024-09-13T14:43:48.258672+0000 mon.a (mon.0) 1505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:48 smithi099 bash[84576]: audit 2024-09-13T14:43:48.258672+0000 mon.a (mon.0) 1505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:48 smithi099 bash[87147]: cluster 2024-09-13T14:43:47.532721+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-13T14:43:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:48 smithi099 bash[87147]: cluster 2024-09-13T14:43:47.532721+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-13T14:43:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:48 smithi099 bash[87147]: audit 2024-09-13T14:43:48.258672+0000 mon.a (mon.0) 1505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:48 smithi099 bash[87147]: audit 2024-09-13T14:43:48.258672+0000 mon.a (mon.0) 1505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:48.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:48 smithi177 bash[68594]: cluster 2024-09-13T14:43:47.532721+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-13T14:43:48.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:48 smithi177 bash[68594]: cluster 2024-09-13T14:43:47.532721+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-13T14:43:48.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:48 smithi177 bash[68594]: audit 2024-09-13T14:43:48.258672+0000 mon.a (mon.0) 1505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:48.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:48 smithi177 bash[68594]: audit 2024-09-13T14:43:48.258672+0000 mon.a (mon.0) 1505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:49.595 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:49.595 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:49.632 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:49.632 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:49.632 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:49.644 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:49.644 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:49.644 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:49.655 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:49.655 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:49.655 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:49.667 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:49.667 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:49.667 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:49.679 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:49.679 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:49.679 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:49.691 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:49.691 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:49.691 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:49.702 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:49.702 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:49.702 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:49.712 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:49.712 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:49.712 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:49.721 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:49.721 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:49.721 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:49.730 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:49.730 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:49.730 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:49.737 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~67108864,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:49.738 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:49.738 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:49.744 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:49.744 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:49.744 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:49.749 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:49.750 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to HEAD diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:49.750 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:49.754 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~8388608,243269632~25165824,285212672~37748736,327155712~12582912,390070272~12582912] 2024-09-13T14:43:49.754 INFO:tasks.workunit.client.0.smithi099.stdout:from 14 to HEAD diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:49.755 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~20971520,29360128~8388608,41943040~8388608,54525952~16777216,75497472~4194304,96468992~8388608,109051904~8388608,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:49.758 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~12582912,146800640~41943040,205520896~16777216,230686720~8388608,243269632~12582912,285212672~37748736,327155712~12582912,390070272~8388608] 2024-09-13T14:43:49.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:49 smithi099 bash[84576]: cluster 2024-09-13T14:43:48.395081+0000 mgr.y (mgr.26252) 520 : cluster [DBG] pgmap v947: 105 pgs: 105 active+clean; 883 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 302 KiB/s rd, 29 MiB/s wr, 538 op/s 2024-09-13T14:43:49.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:49 smithi099 bash[84576]: cluster 2024-09-13T14:43:48.395081+0000 mgr.y (mgr.26252) 520 : cluster [DBG] pgmap v947: 105 pgs: 105 active+clean; 883 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 302 KiB/s rd, 29 MiB/s wr, 538 op/s 2024-09-13T14:43:49.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:49 smithi099 bash[84576]: cluster 2024-09-13T14:43:48.570943+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-13T14:43:49.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:49 smithi099 bash[84576]: cluster 2024-09-13T14:43:48.570943+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-13T14:43:49.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:49 smithi099 bash[87147]: cluster 2024-09-13T14:43:48.395081+0000 mgr.y (mgr.26252) 520 : cluster [DBG] pgmap v947: 105 pgs: 105 active+clean; 883 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 302 KiB/s rd, 29 MiB/s wr, 538 op/s 2024-09-13T14:43:49.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:49 smithi099 bash[87147]: cluster 2024-09-13T14:43:48.395081+0000 mgr.y (mgr.26252) 520 : cluster [DBG] pgmap v947: 105 pgs: 105 active+clean; 883 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 302 KiB/s rd, 29 MiB/s wr, 538 op/s 2024-09-13T14:43:49.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:49 smithi099 bash[87147]: cluster 2024-09-13T14:43:48.570943+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-13T14:43:49.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:49 smithi099 bash[87147]: cluster 2024-09-13T14:43:48.570943+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-13T14:43:49.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:49 smithi177 bash[68594]: cluster 2024-09-13T14:43:48.395081+0000 mgr.y (mgr.26252) 520 : cluster [DBG] pgmap v947: 105 pgs: 105 active+clean; 883 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 302 KiB/s rd, 29 MiB/s wr, 538 op/s 2024-09-13T14:43:49.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:49 smithi177 bash[68594]: cluster 2024-09-13T14:43:48.395081+0000 mgr.y (mgr.26252) 520 : cluster [DBG] pgmap v947: 105 pgs: 105 active+clean; 883 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 302 KiB/s rd, 29 MiB/s wr, 538 op/s 2024-09-13T14:43:49.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:49 smithi177 bash[68594]: cluster 2024-09-13T14:43:48.570943+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-13T14:43:49.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:49 smithi177 bash[68594]: cluster 2024-09-13T14:43:48.570943+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-13T14:43:50.580 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:43:50 smithi177 bash[61439]: ts=2024-09-13T14:43:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:43:50.616 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:50.616 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:50.654 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:50.654 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:50.654 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:50.664 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:50.664 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:50.664 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:50.674 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:50.674 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:50.674 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:50.683 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:50.683 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:50.683 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:50.693 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:50.693 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:50.693 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:50.703 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:50.703 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:50.703 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:50.713 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:50.713 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:50.713 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:50.723 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:50.723 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:50.723 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:50.732 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:50.732 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:50.733 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:50.741 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:50.741 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:50.741 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:50.748 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~67108864,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:50.748 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:50.748 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:50.754 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:50.754 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:50.754 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:50.759 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:50.759 INFO:tasks.workunit.client.0.smithi099.stdout:from 13 to HEAD diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:50.759 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [4194304~33554432,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~8388608,243269632~8388608,255852544~8388608,285212672~12582912,301989888~16777216,327155712~8388608,390070272~8388608] 2024-09-13T14:43:50.763 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [4194304~79691776,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~8388608,243269632~25165824,285212672~37748736,327155712~12582912,390070272~12582912] 2024-09-13T14:43:50.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:50 smithi177 bash[68594]: cluster 2024-09-13T14:43:49.564888+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-13T14:43:50.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:50 smithi177 bash[68594]: cluster 2024-09-13T14:43:49.564888+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-13T14:43:51.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:50 smithi099 bash[84576]: cluster 2024-09-13T14:43:49.564888+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-13T14:43:51.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:50 smithi099 bash[84576]: cluster 2024-09-13T14:43:49.564888+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-13T14:43:51.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:50 smithi099 bash[87147]: cluster 2024-09-13T14:43:49.564888+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-13T14:43:51.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:50 smithi099 bash[87147]: cluster 2024-09-13T14:43:49.564888+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-13T14:43:51.257 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:51.257 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:51.293 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:51.293 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:51.293 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:51.304 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:51.304 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:51.305 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:51.313 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:51.313 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:51.313 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:51.321 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:51.321 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:51.322 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:51.331 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:51.331 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:51.331 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:51.338 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:51.338 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:51.338 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:51.346 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:51.346 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:51.347 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:51.355 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:51.355 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:51.355 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:51.361 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:51.362 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:51.362 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:51.367 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:51.367 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:51.367 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:51.372 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~67108864,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:51.372 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:51.372 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:51.376 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:51.376 INFO:tasks.workunit.client.0.smithi099.stdout:from 12 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:51.376 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~20971520,255852544~8388608,285212672~12582912,301989888~16777216,327155712~12582912,348127232~8388608,390070272~8388608] 2024-09-13T14:43:51.379 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,390070272~12582912] 2024-09-13T14:43:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:51 smithi177 bash[68594]: cluster 2024-09-13T14:43:50.395742+0000 mgr.y (mgr.26252) 521 : cluster [DBG] pgmap v950: 105 pgs: 105 active+clean; 871 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 695 KiB/s rd, 13 MiB/s wr, 1.30k op/s 2024-09-13T14:43:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:51 smithi177 bash[68594]: cluster 2024-09-13T14:43:50.395742+0000 mgr.y (mgr.26252) 521 : cluster [DBG] pgmap v950: 105 pgs: 105 active+clean; 871 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 695 KiB/s rd, 13 MiB/s wr, 1.30k op/s 2024-09-13T14:43:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:51 smithi177 bash[68594]: cluster 2024-09-13T14:43:50.591143+0000 mon.a (mon.0) 1508 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-13T14:43:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:51 smithi177 bash[68594]: cluster 2024-09-13T14:43:50.591143+0000 mon.a (mon.0) 1508 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-13T14:43:51.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:51 smithi177 bash[68594]: cluster 2024-09-13T14:43:51.219297+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-13T14:43:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:51 smithi177 bash[68594]: cluster 2024-09-13T14:43:51.219297+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-13T14:43:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[84576]: cluster 2024-09-13T14:43:50.395742+0000 mgr.y (mgr.26252) 521 : cluster [DBG] pgmap v950: 105 pgs: 105 active+clean; 871 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 695 KiB/s rd, 13 MiB/s wr, 1.30k op/s 2024-09-13T14:43:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[84576]: cluster 2024-09-13T14:43:50.395742+0000 mgr.y (mgr.26252) 521 : cluster [DBG] pgmap v950: 105 pgs: 105 active+clean; 871 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 695 KiB/s rd, 13 MiB/s wr, 1.30k op/s 2024-09-13T14:43:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[84576]: cluster 2024-09-13T14:43:50.591143+0000 mon.a (mon.0) 1508 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-13T14:43:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[84576]: cluster 2024-09-13T14:43:50.591143+0000 mon.a (mon.0) 1508 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-13T14:43:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[84576]: cluster 2024-09-13T14:43:51.219297+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-13T14:43:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[84576]: cluster 2024-09-13T14:43:51.219297+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-13T14:43:52.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[87147]: cluster 2024-09-13T14:43:50.395742+0000 mgr.y (mgr.26252) 521 : cluster [DBG] pgmap v950: 105 pgs: 105 active+clean; 871 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 695 KiB/s rd, 13 MiB/s wr, 1.30k op/s 2024-09-13T14:43:52.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[87147]: cluster 2024-09-13T14:43:50.395742+0000 mgr.y (mgr.26252) 521 : cluster [DBG] pgmap v950: 105 pgs: 105 active+clean; 871 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 695 KiB/s rd, 13 MiB/s wr, 1.30k op/s 2024-09-13T14:43:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[87147]: cluster 2024-09-13T14:43:50.591143+0000 mon.a (mon.0) 1508 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-13T14:43:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[87147]: cluster 2024-09-13T14:43:50.591143+0000 mon.a (mon.0) 1508 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-13T14:43:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[87147]: cluster 2024-09-13T14:43:51.219297+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-13T14:43:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:51 smithi099 bash[87147]: cluster 2024-09-13T14:43:51.219297+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-13T14:43:52.250 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:52.250 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:52.288 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:52.288 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:52.288 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:52.299 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:52.299 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:52.299 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:52.307 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:52.307 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:52.308 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:52.315 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:52.315 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:52.316 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:52.323 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:52.323 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:52.323 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:52.330 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:52.330 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:52.330 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:52.337 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:52.337 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:52.337 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:52.343 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:52.343 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:52.343 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:52.348 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:52.348 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:52.348 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:52.353 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:52.353 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:52.353 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:52.357 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~67108864,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:52.357 INFO:tasks.workunit.client.0.smithi099.stdout:from 11 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:52.358 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,96468992~8388608,109051904~8388608,130023424~62914560,205520896~16777216,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:52.360 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,96468992~8388608,109051904~12582912,130023424~62914560,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:53.262 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:53.262 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:53.297 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:53.297 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:53.297 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:53.306 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:53.306 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:53.306 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:53.314 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:53.314 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:53.314 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:53.321 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:53.321 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:53.322 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:53.327 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:53.327 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:53.327 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:53.333 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:53.333 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:53.333 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:53.338 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:53.338 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:53.338 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:53.343 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:53.343 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:53.343 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:53.348 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:53.348 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:53.348 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:53.351 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:53.351 INFO:tasks.workunit.client.0.smithi099.stdout:from 10 to HEAD diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:53.351 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~29360128,75497472~4194304,88080384~16777216,109051904~8388608,130023424~62914560,205520896~12582912,230686720~33554432,285212672~12582912,301989888~16777216,327155712~12582912,348127232~12582912,369098752~4194304,390070272~8388608] 2024-09-13T14:43:53.353 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~67108864,205520896~16777216,230686720~37748736,285212672~37748736,327155712~16777216,348127232~12582912,369098752~4194304,390070272~12582912] 2024-09-13T14:43:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:53 smithi099 bash[84576]: cluster 2024-09-13T14:43:52.221177+0000 mon.a (mon.0) 1510 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-13T14:43:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:53 smithi099 bash[84576]: cluster 2024-09-13T14:43:52.221177+0000 mon.a (mon.0) 1510 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-13T14:43:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:53 smithi099 bash[84576]: cluster 2024-09-13T14:43:52.396451+0000 mgr.y (mgr.26252) 522 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 858 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 384 KiB/s rd, 1.8 KiB/s wr, 768 op/s 2024-09-13T14:43:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:53 smithi099 bash[84576]: cluster 2024-09-13T14:43:52.396451+0000 mgr.y (mgr.26252) 522 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 858 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 384 KiB/s rd, 1.8 KiB/s wr, 768 op/s 2024-09-13T14:43:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:53 smithi099 bash[87147]: cluster 2024-09-13T14:43:52.221177+0000 mon.a (mon.0) 1510 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-13T14:43:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:53 smithi099 bash[87147]: cluster 2024-09-13T14:43:52.221177+0000 mon.a (mon.0) 1510 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-13T14:43:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:53 smithi099 bash[87147]: cluster 2024-09-13T14:43:52.396451+0000 mgr.y (mgr.26252) 522 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 858 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 384 KiB/s rd, 1.8 KiB/s wr, 768 op/s 2024-09-13T14:43:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:53 smithi099 bash[87147]: cluster 2024-09-13T14:43:52.396451+0000 mgr.y (mgr.26252) 522 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 858 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 384 KiB/s rd, 1.8 KiB/s wr, 768 op/s 2024-09-13T14:43:53.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:53 smithi177 bash[68594]: cluster 2024-09-13T14:43:52.221177+0000 mon.a (mon.0) 1510 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-13T14:43:53.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:53 smithi177 bash[68594]: cluster 2024-09-13T14:43:52.221177+0000 mon.a (mon.0) 1510 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-13T14:43:53.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:53 smithi177 bash[68594]: cluster 2024-09-13T14:43:52.396451+0000 mgr.y (mgr.26252) 522 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 858 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 384 KiB/s rd, 1.8 KiB/s wr, 768 op/s 2024-09-13T14:43:53.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:53 smithi177 bash[68594]: cluster 2024-09-13T14:43:52.396451+0000 mgr.y (mgr.26252) 522 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 858 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 384 KiB/s rd, 1.8 KiB/s wr, 768 op/s 2024-09-13T14:43:54.268 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:54.268 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:54.310 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:54.310 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:54.310 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:54.316 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:54.316 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:54.316 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:54.323 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:54.323 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:54.323 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:54.329 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:54.329 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:54.329 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:54.334 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:54.334 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:54.334 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:54.339 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:54.339 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:54.339 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:54.343 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:54.344 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:54.344 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:54.347 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:54.347 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:54.347 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:54.351 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:54.351 INFO:tasks.workunit.client.0.smithi099.stdout:from 9 to HEAD diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:54.351 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~4194304,88080384~16777216,109051904~8388608,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:54.353 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~83886080,88080384~16777216,109051904~12582912,130023424~71303168,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:54 smithi099 bash[84576]: cluster 2024-09-13T14:43:53.236316+0000 mon.a (mon.0) 1511 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-13T14:43:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:54 smithi099 bash[84576]: cluster 2024-09-13T14:43:53.236316+0000 mon.a (mon.0) 1511 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-13T14:43:54.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:54 smithi099 bash[87147]: cluster 2024-09-13T14:43:53.236316+0000 mon.a (mon.0) 1511 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-13T14:43:54.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:54 smithi099 bash[87147]: cluster 2024-09-13T14:43:53.236316+0000 mon.a (mon.0) 1511 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-13T14:43:54.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:54 smithi177 bash[68594]: cluster 2024-09-13T14:43:53.236316+0000 mon.a (mon.0) 1511 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-13T14:43:54.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:54 smithi177 bash[68594]: cluster 2024-09-13T14:43:53.236316+0000 mon.a (mon.0) 1511 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-13T14:43:55.313 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:55.313 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:55.337 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:55.337 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:55.337 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:55.344 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:55.344 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:55.344 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:55.358 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:55.358 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:55.358 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:55.358 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:55.358 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:55.358 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:55.362 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:55.362 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:55.363 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:55.367 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:55.367 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:55.367 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:55.371 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:55.371 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:55.371 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:55.375 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:55.375 INFO:tasks.workunit.client.0.smithi099.stdout:from 8 to HEAD diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:55.375 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~71303168,75497472~50331648,130023424~67108864,205520896~16777216,230686720~33554432,281018368~16777216,301989888~16777216,327155712~12582912,348127232~12582912,364904448~8388608,377487360~8388608,390070272~8388608] 2024-09-13T14:43:55.378 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,281018368~41943040,327155712~16777216,348127232~12582912,364904448~8388608,377487360~25165824] 2024-09-13T14:43:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:55 smithi099 bash[84576]: cluster 2024-09-13T14:43:54.234756+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-13T14:43:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:55 smithi099 bash[84576]: cluster 2024-09-13T14:43:54.234756+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-13T14:43:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:55 smithi099 bash[84576]: cluster 2024-09-13T14:43:54.397215+0000 mgr.y (mgr.26252) 523 : cluster [DBG] pgmap v957: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 356 KiB/s rd, 7.9 KiB/s wr, 608 op/s 2024-09-13T14:43:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:55 smithi099 bash[84576]: cluster 2024-09-13T14:43:54.397215+0000 mgr.y (mgr.26252) 523 : cluster [DBG] pgmap v957: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 356 KiB/s rd, 7.9 KiB/s wr, 608 op/s 2024-09-13T14:43:55.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:55 smithi099 bash[87147]: cluster 2024-09-13T14:43:54.234756+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-13T14:43:55.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:55 smithi099 bash[87147]: cluster 2024-09-13T14:43:54.234756+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-13T14:43:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:55 smithi099 bash[87147]: cluster 2024-09-13T14:43:54.397215+0000 mgr.y (mgr.26252) 523 : cluster [DBG] pgmap v957: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 356 KiB/s rd, 7.9 KiB/s wr, 608 op/s 2024-09-13T14:43:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:55 smithi099 bash[87147]: cluster 2024-09-13T14:43:54.397215+0000 mgr.y (mgr.26252) 523 : cluster [DBG] pgmap v957: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 356 KiB/s rd, 7.9 KiB/s wr, 608 op/s 2024-09-13T14:43:55.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:55 smithi177 bash[68594]: cluster 2024-09-13T14:43:54.234756+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-13T14:43:55.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:55 smithi177 bash[68594]: cluster 2024-09-13T14:43:54.234756+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-13T14:43:55.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:55 smithi177 bash[68594]: cluster 2024-09-13T14:43:54.397215+0000 mgr.y (mgr.26252) 523 : cluster [DBG] pgmap v957: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 356 KiB/s rd, 7.9 KiB/s wr, 608 op/s 2024-09-13T14:43:55.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:55 smithi177 bash[68594]: cluster 2024-09-13T14:43:54.397215+0000 mgr.y (mgr.26252) 523 : cluster [DBG] pgmap v957: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 356 KiB/s rd, 7.9 KiB/s wr, 608 op/s 2024-09-13T14:43:56.276 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:56.276 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:56.301 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:56.301 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:56.301 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:56.309 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:56.309 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:56.309 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:56.314 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:56.314 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:56.314 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:56.319 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:56.319 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:56.319 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:56.323 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:56.323 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:56.323 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:56.327 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:56.327 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:56.327 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:56.330 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:56.330 INFO:tasks.workunit.client.0.smithi099.stdout:from 7 to HEAD diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:56.330 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~25165824,301989888~20971520,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:56.333 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:56.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:56 smithi099 bash[84576]: cluster 2024-09-13T14:43:55.264553+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-13T14:43:56.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:56 smithi099 bash[84576]: cluster 2024-09-13T14:43:55.264553+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-13T14:43:56.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:56 smithi099 bash[84576]: cluster 2024-09-13T14:43:56.231872+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-13T14:43:56.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:56 smithi099 bash[84576]: cluster 2024-09-13T14:43:56.231872+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-13T14:43:56.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:56 smithi099 bash[87147]: cluster 2024-09-13T14:43:55.264553+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-13T14:43:56.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:56 smithi099 bash[87147]: cluster 2024-09-13T14:43:55.264553+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-13T14:43:56.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:56 smithi099 bash[87147]: cluster 2024-09-13T14:43:56.231872+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-13T14:43:56.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:56 smithi099 bash[87147]: cluster 2024-09-13T14:43:56.231872+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-13T14:43:56.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:56 smithi177 bash[68594]: cluster 2024-09-13T14:43:55.264553+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-13T14:43:56.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:56 smithi177 bash[68594]: cluster 2024-09-13T14:43:55.264553+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-13T14:43:56.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:56 smithi177 bash[68594]: cluster 2024-09-13T14:43:56.231872+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-13T14:43:56.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:56 smithi177 bash[68594]: cluster 2024-09-13T14:43:56.231872+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-13T14:43:57.298 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:57.298 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:57.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:43:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:43:57] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.51.0" 2024-09-13T14:43:57.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:57 smithi099 bash[84576]: cluster 2024-09-13T14:43:56.397989+0000 mgr.y (mgr.26252) 524 : cluster [DBG] pgmap v960: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 297 KiB/s rd, 6.0 KiB/s wr, 495 op/s 2024-09-13T14:43:57.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:57 smithi099 bash[84576]: cluster 2024-09-13T14:43:56.397989+0000 mgr.y (mgr.26252) 524 : cluster [DBG] pgmap v960: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 297 KiB/s rd, 6.0 KiB/s wr, 495 op/s 2024-09-13T14:43:57.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:57 smithi099 bash[87147]: cluster 2024-09-13T14:43:56.397989+0000 mgr.y (mgr.26252) 524 : cluster [DBG] pgmap v960: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 297 KiB/s rd, 6.0 KiB/s wr, 495 op/s 2024-09-13T14:43:57.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:57 smithi099 bash[87147]: cluster 2024-09-13T14:43:56.397989+0000 mgr.y (mgr.26252) 524 : cluster [DBG] pgmap v960: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 297 KiB/s rd, 6.0 KiB/s wr, 495 op/s 2024-09-13T14:43:57.332 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:57.332 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:57.333 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:57.349 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:57.349 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:57.349 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:57.357 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:57.357 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:57.357 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:57.362 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:57.362 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:57.362 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:57.365 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:57.365 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:57.365 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:57.368 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:57.369 INFO:tasks.workunit.client.0.smithi099.stdout:from 6 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:57.369 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~8388608,390070272~8388608] 2024-09-13T14:43:57.371 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:57.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:57 smithi177 bash[68594]: cluster 2024-09-13T14:43:56.397989+0000 mgr.y (mgr.26252) 524 : cluster [DBG] pgmap v960: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 297 KiB/s rd, 6.0 KiB/s wr, 495 op/s 2024-09-13T14:43:57.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:57 smithi177 bash[68594]: cluster 2024-09-13T14:43:56.397989+0000 mgr.y (mgr.26252) 524 : cluster [DBG] pgmap v960: 105 pgs: 105 active+clean; 794 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 297 KiB/s rd, 6.0 KiB/s wr, 495 op/s 2024-09-13T14:43:58.309 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:58.309 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:58.343 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:58.343 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:58.343 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:58.349 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:58.349 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:58.349 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:58.353 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:58.353 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:58.353 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:58.358 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:58.358 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:58.358 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:58.361 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:58.361 INFO:tasks.workunit.client.0.smithi099.stdout:from 5 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:58.361 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~67108864,205520896~16777216,230686720~33554432,272629760~50331648,327155712~12582912,348127232~25165824,377487360~20971520] 2024-09-13T14:43:58.363 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~201326592,205520896~16777216,230686720~37748736,272629760~50331648,327155712~16777216,348127232~25165824,377487360~25165824] 2024-09-13T14:43:58.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:58 smithi099 bash[84576]: cluster 2024-09-13T14:43:57.283364+0000 mon.a (mon.0) 1515 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-13T14:43:58.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:58 smithi099 bash[84576]: cluster 2024-09-13T14:43:57.283364+0000 mon.a (mon.0) 1515 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-13T14:43:58.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:58 smithi099 bash[87147]: cluster 2024-09-13T14:43:57.283364+0000 mon.a (mon.0) 1515 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-13T14:43:58.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:58 smithi099 bash[87147]: cluster 2024-09-13T14:43:57.283364+0000 mon.a (mon.0) 1515 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-13T14:43:58.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:58 smithi177 bash[68594]: cluster 2024-09-13T14:43:57.283364+0000 mon.a (mon.0) 1515 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-13T14:43:58.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:58 smithi177 bash[68594]: cluster 2024-09-13T14:43:57.283364+0000 mon.a (mon.0) 1515 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-13T14:43:59.341 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:59.342 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:43:59.363 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:59.363 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:59.363 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:43:59.368 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:43:59.368 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:59.368 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:43:59.373 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:43:59.373 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:59.373 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:59.376 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:43:59.376 INFO:tasks.workunit.client.0.smithi099.stdout:from 4 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:59.377 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:43:59.378 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~25165824,377487360~25165824] 2024-09-13T14:43:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:59 smithi099 bash[84576]: cluster 2024-09-13T14:43:58.284519+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-13T14:43:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:59 smithi099 bash[84576]: cluster 2024-09-13T14:43:58.284519+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-13T14:43:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:59 smithi099 bash[84576]: cluster 2024-09-13T14:43:58.398478+0000 mgr.y (mgr.26252) 525 : cluster [DBG] pgmap v963: 105 pgs: 105 active+clean; 713 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 130 KiB/s rd, 6.2 KiB/s wr, 201 op/s 2024-09-13T14:43:59.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:43:59 smithi099 bash[84576]: cluster 2024-09-13T14:43:58.398478+0000 mgr.y (mgr.26252) 525 : cluster [DBG] pgmap v963: 105 pgs: 105 active+clean; 713 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 130 KiB/s rd, 6.2 KiB/s wr, 201 op/s 2024-09-13T14:43:59.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:59 smithi099 bash[87147]: cluster 2024-09-13T14:43:58.284519+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-13T14:43:59.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:59 smithi099 bash[87147]: cluster 2024-09-13T14:43:58.284519+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-13T14:43:59.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:59 smithi099 bash[87147]: cluster 2024-09-13T14:43:58.398478+0000 mgr.y (mgr.26252) 525 : cluster [DBG] pgmap v963: 105 pgs: 105 active+clean; 713 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 130 KiB/s rd, 6.2 KiB/s wr, 201 op/s 2024-09-13T14:43:59.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:43:59 smithi099 bash[87147]: cluster 2024-09-13T14:43:58.398478+0000 mgr.y (mgr.26252) 525 : cluster [DBG] pgmap v963: 105 pgs: 105 active+clean; 713 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 130 KiB/s rd, 6.2 KiB/s wr, 201 op/s 2024-09-13T14:43:59.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:59 smithi177 bash[68594]: cluster 2024-09-13T14:43:58.284519+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-13T14:43:59.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:59 smithi177 bash[68594]: cluster 2024-09-13T14:43:58.284519+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-13T14:43:59.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:59 smithi177 bash[68594]: cluster 2024-09-13T14:43:58.398478+0000 mgr.y (mgr.26252) 525 : cluster [DBG] pgmap v963: 105 pgs: 105 active+clean; 713 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 130 KiB/s rd, 6.2 KiB/s wr, 201 op/s 2024-09-13T14:43:59.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:43:59 smithi177 bash[68594]: cluster 2024-09-13T14:43:58.398478+0000 mgr.y (mgr.26252) 525 : cluster [DBG] pgmap v963: 105 pgs: 105 active+clean; 713 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 130 KiB/s rd, 6.2 KiB/s wr, 201 op/s 2024-09-13T14:44:00.348 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:44:00.348 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:44:00.371 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:44:00.371 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:44:00.372 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:44:00.379 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:44:00.379 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:44:00.379 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:44:00.381 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:44:00.382 INFO:tasks.workunit.client.0.smithi099.stdout:from 3 to HEAD diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:44:00.382 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~37748736,130023424~71303168,205520896~58720256,272629760~67108864,348127232~25165824,377487360~20971520] 2024-09-13T14:44:00.383 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~268435456,272629760~71303168,348127232~54525952] 2024-09-13T14:44:00.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:00 smithi177 bash[68594]: cluster 2024-09-13T14:43:59.313050+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-13T14:44:00.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:00 smithi177 bash[68594]: cluster 2024-09-13T14:43:59.313050+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-13T14:44:00.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:44:00 smithi177 bash[61439]: ts=2024-09-13T14:44:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:00.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:00 smithi099 bash[84576]: cluster 2024-09-13T14:43:59.313050+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-13T14:44:00.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:00 smithi099 bash[84576]: cluster 2024-09-13T14:43:59.313050+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-13T14:44:00.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:00 smithi099 bash[87147]: cluster 2024-09-13T14:43:59.313050+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-13T14:44:00.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:00 smithi099 bash[87147]: cluster 2024-09-13T14:43:59.313050+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-13T14:44:01.283 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:44:01.283 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:44:01.308 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:44:01.308 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:44:01.308 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:44:01.310 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:44:01.310 INFO:tasks.workunit.client.0.smithi099.stdout:from 2 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:44:01.310 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,348127232~25165824,377487360~20971520,406847488~8388608] 2024-09-13T14:44:01.312 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~343932928,348127232~54525952,406847488~8388608] 2024-09-13T14:44:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:01 smithi177 bash[68594]: cluster 2024-09-13T14:44:00.316205+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-13T14:44:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:01 smithi177 bash[68594]: cluster 2024-09-13T14:44:00.316205+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-13T14:44:01.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:01 smithi177 bash[68594]: cluster 2024-09-13T14:44:00.399024+0000 mgr.y (mgr.26252) 526 : cluster [DBG] pgmap v966: 105 pgs: 105 active+clean; 651 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 169 KiB/s rd, 6.2 KiB/s wr, 279 op/s 2024-09-13T14:44:01.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:01 smithi177 bash[68594]: cluster 2024-09-13T14:44:00.399024+0000 mgr.y (mgr.26252) 526 : cluster [DBG] pgmap v966: 105 pgs: 105 active+clean; 651 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 169 KiB/s rd, 6.2 KiB/s wr, 279 op/s 2024-09-13T14:44:01.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:01 smithi177 bash[68594]: cluster 2024-09-13T14:44:01.235951+0000 mon.a (mon.0) 1519 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-13T14:44:01.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:01 smithi177 bash[68594]: cluster 2024-09-13T14:44:01.235951+0000 mon.a (mon.0) 1519 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-13T14:44:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[84576]: cluster 2024-09-13T14:44:00.316205+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-13T14:44:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[84576]: cluster 2024-09-13T14:44:00.316205+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-13T14:44:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[84576]: cluster 2024-09-13T14:44:00.399024+0000 mgr.y (mgr.26252) 526 : cluster [DBG] pgmap v966: 105 pgs: 105 active+clean; 651 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 169 KiB/s rd, 6.2 KiB/s wr, 279 op/s 2024-09-13T14:44:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[84576]: cluster 2024-09-13T14:44:00.399024+0000 mgr.y (mgr.26252) 526 : cluster [DBG] pgmap v966: 105 pgs: 105 active+clean; 651 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 169 KiB/s rd, 6.2 KiB/s wr, 279 op/s 2024-09-13T14:44:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[84576]: cluster 2024-09-13T14:44:01.235951+0000 mon.a (mon.0) 1519 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-13T14:44:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[84576]: cluster 2024-09-13T14:44:01.235951+0000 mon.a (mon.0) 1519 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-13T14:44:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[87147]: cluster 2024-09-13T14:44:00.316205+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-13T14:44:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[87147]: cluster 2024-09-13T14:44:00.316205+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-13T14:44:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[87147]: cluster 2024-09-13T14:44:00.399024+0000 mgr.y (mgr.26252) 526 : cluster [DBG] pgmap v966: 105 pgs: 105 active+clean; 651 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 169 KiB/s rd, 6.2 KiB/s wr, 279 op/s 2024-09-13T14:44:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[87147]: cluster 2024-09-13T14:44:00.399024+0000 mgr.y (mgr.26252) 526 : cluster [DBG] pgmap v966: 105 pgs: 105 active+clean; 651 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 169 KiB/s rd, 6.2 KiB/s wr, 279 op/s 2024-09-13T14:44:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[87147]: cluster 2024-09-13T14:44:01.235951+0000 mon.a (mon.0) 1519 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-13T14:44:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:01 smithi099 bash[87147]: cluster 2024-09-13T14:44:01.235951+0000 mon.a (mon.0) 1519 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-13T14:44:02.277 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:44:02.277 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:44:02.292 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:44:02.292 INFO:tasks.workunit.client.0.smithi099.stdout:from 1 to HEAD diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:44:02.292 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~37748736,41943040~41943040,88080384~113246208,205520896~58720256,268435456~71303168,343932928~54525952,406847488~8388608] 2024-09-13T14:44:02.295 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:44:03.274 INFO:tasks.workunit.client.0.smithi099.stdout:from 0 to HEAD diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:44:03.275 INFO:tasks.workunit.client.0.smithi099.stdout: limited diff [0~83886080,88080384~113246208,205520896~58720256,268435456~75497472,348127232~50331648,406847488~8388608] 2024-09-13T14:44:03.298 INFO:tasks.workunit.client.0.smithi099.stdout: actual was [0~402653184,406847488~8388608] 2024-09-13T14:44:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:03 smithi099 bash[84576]: cluster 2024-09-13T14:44:02.238441+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-13T14:44:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:03 smithi099 bash[84576]: cluster 2024-09-13T14:44:02.238441+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-13T14:44:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:03 smithi099 bash[84576]: cluster 2024-09-13T14:44:02.399720+0000 mgr.y (mgr.26252) 527 : cluster [DBG] pgmap v969: 105 pgs: 105 active+clean; 634 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 47 KiB/s rd, 767 B/s wr, 95 op/s 2024-09-13T14:44:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:03 smithi099 bash[84576]: cluster 2024-09-13T14:44:02.399720+0000 mgr.y (mgr.26252) 527 : cluster [DBG] pgmap v969: 105 pgs: 105 active+clean; 634 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 47 KiB/s rd, 767 B/s wr, 95 op/s 2024-09-13T14:44:03.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:03 smithi099 bash[87147]: cluster 2024-09-13T14:44:02.238441+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-13T14:44:03.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:03 smithi099 bash[87147]: cluster 2024-09-13T14:44:02.238441+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-13T14:44:03.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:03 smithi099 bash[87147]: cluster 2024-09-13T14:44:02.399720+0000 mgr.y (mgr.26252) 527 : cluster [DBG] pgmap v969: 105 pgs: 105 active+clean; 634 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 47 KiB/s rd, 767 B/s wr, 95 op/s 2024-09-13T14:44:03.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:03 smithi099 bash[87147]: cluster 2024-09-13T14:44:02.399720+0000 mgr.y (mgr.26252) 527 : cluster [DBG] pgmap v969: 105 pgs: 105 active+clean; 634 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 47 KiB/s rd, 767 B/s wr, 95 op/s 2024-09-13T14:44:03.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:03 smithi177 bash[68594]: cluster 2024-09-13T14:44:02.238441+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-13T14:44:03.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:03 smithi177 bash[68594]: cluster 2024-09-13T14:44:02.238441+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-13T14:44:03.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:03 smithi177 bash[68594]: cluster 2024-09-13T14:44:02.399720+0000 mgr.y (mgr.26252) 527 : cluster [DBG] pgmap v969: 105 pgs: 105 active+clean; 634 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 47 KiB/s rd, 767 B/s wr, 95 op/s 2024-09-13T14:44:03.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:03 smithi177 bash[68594]: cluster 2024-09-13T14:44:02.399720+0000 mgr.y (mgr.26252) 527 : cluster [DBG] pgmap v969: 105 pgs: 105 active+clean; 634 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 47 KiB/s rd, 767 B/s wr, 95 op/s 2024-09-13T14:44:04.316 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterateStress (39985 ms) 2024-09-13T14:44:04.316 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterateRegression6926 2024-09-13T14:44:04.316 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:44:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:04 smithi099 bash[84576]: cluster 2024-09-13T14:44:03.248617+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-13T14:44:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:04 smithi099 bash[84576]: cluster 2024-09-13T14:44:03.248617+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-13T14:44:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:04 smithi099 bash[84576]: audit 2024-09-13T14:44:03.266439+0000 mon.a (mon.0) 1522 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:04.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:04 smithi099 bash[84576]: audit 2024-09-13T14:44:03.266439+0000 mon.a (mon.0) 1522 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:04.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:04 smithi099 bash[87147]: cluster 2024-09-13T14:44:03.248617+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-13T14:44:04.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:04 smithi099 bash[87147]: cluster 2024-09-13T14:44:03.248617+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-13T14:44:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:04 smithi099 bash[87147]: audit 2024-09-13T14:44:03.266439+0000 mon.a (mon.0) 1522 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:04.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:04 smithi099 bash[87147]: audit 2024-09-13T14:44:03.266439+0000 mon.a (mon.0) 1522 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:04 smithi177 bash[68594]: cluster 2024-09-13T14:44:03.248617+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-13T14:44:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:04 smithi177 bash[68594]: cluster 2024-09-13T14:44:03.248617+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-13T14:44:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:04 smithi177 bash[68594]: audit 2024-09-13T14:44:03.266439+0000 mon.a (mon.0) 1522 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:04 smithi177 bash[68594]: audit 2024-09-13T14:44:03.266439+0000 mon.a (mon.0) 1522 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:05.302 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~4194304 2024-09-13T14:44:05.313 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterateRegression6926 (997 ms) 2024-09-13T14:44:05.313 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParent 2024-09-13T14:44:05.313 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:44:05.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:05 smithi099 bash[87147]: cluster 2024-09-13T14:44:04.265650+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-13T14:44:05.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:05 smithi099 bash[87147]: cluster 2024-09-13T14:44:04.265650+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-13T14:44:05.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:05 smithi099 bash[87147]: cluster 2024-09-13T14:44:04.400449+0000 mgr.y (mgr.26252) 528 : cluster [DBG] pgmap v972: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 87 KiB/s rd, 5.7 KiB/s wr, 135 op/s 2024-09-13T14:44:05.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:05 smithi099 bash[87147]: cluster 2024-09-13T14:44:04.400449+0000 mgr.y (mgr.26252) 528 : cluster [DBG] pgmap v972: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 87 KiB/s rd, 5.7 KiB/s wr, 135 op/s 2024-09-13T14:44:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:05 smithi099 bash[84576]: cluster 2024-09-13T14:44:04.265650+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-13T14:44:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:05 smithi099 bash[84576]: cluster 2024-09-13T14:44:04.265650+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-13T14:44:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:05 smithi099 bash[84576]: cluster 2024-09-13T14:44:04.400449+0000 mgr.y (mgr.26252) 528 : cluster [DBG] pgmap v972: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 87 KiB/s rd, 5.7 KiB/s wr, 135 op/s 2024-09-13T14:44:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:05 smithi099 bash[84576]: cluster 2024-09-13T14:44:04.400449+0000 mgr.y (mgr.26252) 528 : cluster [DBG] pgmap v972: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 87 KiB/s rd, 5.7 KiB/s wr, 135 op/s 2024-09-13T14:44:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:05 smithi177 bash[68594]: cluster 2024-09-13T14:44:04.265650+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-13T14:44:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:05 smithi177 bash[68594]: cluster 2024-09-13T14:44:04.265650+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-13T14:44:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:05 smithi177 bash[68594]: cluster 2024-09-13T14:44:04.400449+0000 mgr.y (mgr.26252) 528 : cluster [DBG] pgmap v972: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 87 KiB/s rd, 5.7 KiB/s wr, 135 op/s 2024-09-13T14:44:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:05 smithi177 bash[68594]: cluster 2024-09-13T14:44:04.400449+0000 mgr.y (mgr.26252) 528 : cluster [DBG] pgmap v972: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 87 KiB/s rd, 5.7 KiB/s wr, 135 op/s 2024-09-13T14:44:06.389 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~4194304 2024-09-13T14:44:06.389 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 4194304~4194304 2024-09-13T14:44:06.389 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 8388608~4194304 2024-09-13T14:44:06.389 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 12582912~4194304 2024-09-13T14:44:06.389 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 16777216~4194304 2024-09-13T14:44:06.460 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 0~4194304 2024-09-13T14:44:06.460 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 4194304~4194304 2024-09-13T14:44:06.460 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 8388608~4194304 2024-09-13T14:44:06.460 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 16777216~4194304 2024-09-13T14:44:06.491 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterateParent (1178 ms) 2024-09-13T14:44:06.491 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterateIgnoreParent 2024-09-13T14:44:06.491 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:44:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:06 smithi099 bash[84576]: cluster 2024-09-13T14:44:05.271979+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-13T14:44:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:06 smithi099 bash[84576]: cluster 2024-09-13T14:44:05.271979+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-13T14:44:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:06 smithi099 bash[84576]: cluster 2024-09-13T14:44:06.235551+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-13T14:44:06.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:06 smithi099 bash[84576]: cluster 2024-09-13T14:44:06.235551+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-13T14:44:06.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:06 smithi099 bash[87147]: cluster 2024-09-13T14:44:05.271979+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-13T14:44:06.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:06 smithi099 bash[87147]: cluster 2024-09-13T14:44:05.271979+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-13T14:44:06.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:06 smithi099 bash[87147]: cluster 2024-09-13T14:44:06.235551+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-13T14:44:06.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:06 smithi099 bash[87147]: cluster 2024-09-13T14:44:06.235551+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-13T14:44:06.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:06 smithi177 bash[68594]: cluster 2024-09-13T14:44:05.271979+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-13T14:44:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:06 smithi177 bash[68594]: cluster 2024-09-13T14:44:05.271979+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-13T14:44:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:06 smithi177 bash[68594]: cluster 2024-09-13T14:44:06.235551+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-13T14:44:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:06 smithi177 bash[68594]: cluster 2024-09-13T14:44:06.235551+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-13T14:44:07.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:07 smithi099 bash[84576]: cluster 2024-09-13T14:44:06.401354+0000 mgr.y (mgr.26252) 529 : cluster [DBG] pgmap v975: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 5.0 KiB/s wr, 118 op/s 2024-09-13T14:44:07.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:07 smithi099 bash[84576]: cluster 2024-09-13T14:44:06.401354+0000 mgr.y (mgr.26252) 529 : cluster [DBG] pgmap v975: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 5.0 KiB/s wr, 118 op/s 2024-09-13T14:44:07.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:44:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:44:07] "GET /metrics HTTP/1.1" 200 36704 "" "Prometheus/2.51.0" 2024-09-13T14:44:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:07 smithi099 bash[87147]: cluster 2024-09-13T14:44:06.401354+0000 mgr.y (mgr.26252) 529 : cluster [DBG] pgmap v975: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 5.0 KiB/s wr, 118 op/s 2024-09-13T14:44:07.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:07 smithi099 bash[87147]: cluster 2024-09-13T14:44:06.401354+0000 mgr.y (mgr.26252) 529 : cluster [DBG] pgmap v975: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 5.0 KiB/s wr, 118 op/s 2024-09-13T14:44:07.547 INFO:tasks.workunit.client.0.smithi099.stdout: wrote [0~12582912,16777216~4194304] to clone 2024-09-13T14:44:07.548 INFO:tasks.workunit.client.0.smithi099.stdout: diff was [0~12582912,16777216~4194304] 2024-09-13T14:44:07.558 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterateIgnoreParent (1067 ms) 2024-09-13T14:44:07.558 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterateCallbackError 2024-09-13T14:44:07.559 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:44:07.608 INFO:tasks.workunit.client.0.smithi099.stdout: wrote [1133536~8704,2746184~41466,9711560~85803,10050510~83827,10375969~76317,11449225~278,16046677~73697,16333153~79771,18104655~101504,19631016~47032] 2024-09-13T14:44:07.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:07 smithi177 bash[68594]: cluster 2024-09-13T14:44:06.401354+0000 mgr.y (mgr.26252) 529 : cluster [DBG] pgmap v975: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 5.0 KiB/s wr, 118 op/s 2024-09-13T14:44:07.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:07 smithi177 bash[68594]: cluster 2024-09-13T14:44:06.401354+0000 mgr.y (mgr.26252) 529 : cluster [DBG] pgmap v975: 105 pgs: 105 active+clean; 485 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 5.0 KiB/s wr, 118 op/s 2024-09-13T14:44:07.635 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterateCallbackError (77 ms) 2024-09-13T14:44:07.635 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParentDiscard 2024-09-13T14:44:07.635 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:44:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:08 smithi099 bash[84576]: cluster 2024-09-13T14:44:07.284736+0000 mon.a (mon.0) 1526 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-13T14:44:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:08 smithi099 bash[84576]: cluster 2024-09-13T14:44:07.284736+0000 mon.a (mon.0) 1526 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-13T14:44:08.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:08 smithi099 bash[87147]: cluster 2024-09-13T14:44:07.284736+0000 mon.a (mon.0) 1526 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-13T14:44:08.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:08 smithi099 bash[87147]: cluster 2024-09-13T14:44:07.284736+0000 mon.a (mon.0) 1526 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-13T14:44:08.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:08 smithi177 bash[68594]: cluster 2024-09-13T14:44:07.284736+0000 mon.a (mon.0) 1526 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-13T14:44:08.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:08 smithi177 bash[68594]: cluster 2024-09-13T14:44:07.284736+0000 mon.a (mon.0) 1526 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-13T14:44:09.539 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterateParentDiscard (1904 ms) 2024-09-13T14:44:09.539 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnalignedSmall 2024-09-13T14:44:09.539 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:44:09.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:09 smithi099 bash[84576]: cluster 2024-09-13T14:44:08.295761+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-13T14:44:09.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:09 smithi099 bash[84576]: cluster 2024-09-13T14:44:08.295761+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-13T14:44:09.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:09 smithi099 bash[84576]: cluster 2024-09-13T14:44:08.402100+0000 mgr.y (mgr.26252) 530 : cluster [DBG] pgmap v978: 105 pgs: 105 active+clean; 474 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.5 MiB/s wr, 128 op/s 2024-09-13T14:44:09.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:09 smithi099 bash[84576]: cluster 2024-09-13T14:44:08.402100+0000 mgr.y (mgr.26252) 530 : cluster [DBG] pgmap v978: 105 pgs: 105 active+clean; 474 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.5 MiB/s wr, 128 op/s 2024-09-13T14:44:09.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:09 smithi099 bash[87147]: cluster 2024-09-13T14:44:08.295761+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-13T14:44:09.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:09 smithi099 bash[87147]: cluster 2024-09-13T14:44:08.295761+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-13T14:44:09.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:09 smithi099 bash[87147]: cluster 2024-09-13T14:44:08.402100+0000 mgr.y (mgr.26252) 530 : cluster [DBG] pgmap v978: 105 pgs: 105 active+clean; 474 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.5 MiB/s wr, 128 op/s 2024-09-13T14:44:09.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:09 smithi099 bash[87147]: cluster 2024-09-13T14:44:08.402100+0000 mgr.y (mgr.26252) 530 : cluster [DBG] pgmap v978: 105 pgs: 105 active+clean; 474 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.5 MiB/s wr, 128 op/s 2024-09-13T14:44:09.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:09 smithi177 bash[68594]: cluster 2024-09-13T14:44:08.295761+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-13T14:44:09.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:09 smithi177 bash[68594]: cluster 2024-09-13T14:44:08.295761+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-13T14:44:09.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:09 smithi177 bash[68594]: cluster 2024-09-13T14:44:08.402100+0000 mgr.y (mgr.26252) 530 : cluster [DBG] pgmap v978: 105 pgs: 105 active+clean; 474 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.5 MiB/s wr, 128 op/s 2024-09-13T14:44:09.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:09 smithi177 bash[68594]: cluster 2024-09-13T14:44:08.402100+0000 mgr.y (mgr.26252) 530 : cluster [DBG] pgmap v978: 105 pgs: 105 active+clean; 474 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.5 MiB/s wr, 128 op/s 2024-09-13T14:44:09.635 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 5000005~1234 2024-09-13T14:44:09.643 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnalignedSmall (105 ms) 2024-09-13T14:44:09.644 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnaligned 2024-09-13T14:44:09.644 INFO:tasks.workunit.client.0.smithi099.stdout:using new format! 2024-09-13T14:44:09.785 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 8376263~12345 2024-09-13T14:44:09.785 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 8388608~4194304 2024-09-13T14:44:09.785 INFO:tasks.workunit.client.0.smithi099.stdout:iterate_cb 12582912~54321 2024-09-13T14:44:09.799 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnaligned (155 ms) 2024-09-13T14:44:09.799 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStriping 2024-09-13T14:44:09.799 INFO:tasks.workunit.client.0.smithi099.stdout:SKIPPING 2024-09-13T14:44:09.799 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] DiffIterateTest/1.DiffIterateStriping (0 ms) 2024-09-13T14:44:10.315 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 11 tests from DiffIterateTest/1 (53531 ms total) 2024-09-13T14:44:10.316 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:44:10.316 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] Global test environment tear-down 2024-09-13T14:44:10.316 INFO:tasks.workunit.client.0.smithi099.stdout:[==========] 406 tests from 15 test suites ran. (1049152 ms total) 2024-09-13T14:44:10.316 INFO:tasks.workunit.client.0.smithi099.stdout:[ PASSED ] 406 tests. 2024-09-13T14:44:10.316 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:44:10.316 INFO:tasks.workunit.client.0.smithi099.stdout: YOU HAVE 2 DISABLED TESTS 2024-09-13T14:44:10.316 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T14:44:10.354 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T14:44:10.354 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T14:44:10.367 INFO:tasks.workunit:Stopping ['rbd/test_librbd.sh'] on client.0... 2024-09-13T14:44:10.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T14:44:10.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:10 smithi099 bash[84576]: cluster 2024-09-13T14:44:09.308317+0000 mon.a (mon.0) 1528 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-13T14:44:10.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:10 smithi099 bash[84576]: cluster 2024-09-13T14:44:09.308317+0000 mon.a (mon.0) 1528 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-13T14:44:10.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:10 smithi099 bash[87147]: cluster 2024-09-13T14:44:09.308317+0000 mon.a (mon.0) 1528 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-13T14:44:10.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:10 smithi099 bash[87147]: cluster 2024-09-13T14:44:09.308317+0000 mon.a (mon.0) 1528 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-13T14:44:10.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:44:10 smithi177 bash[61439]: ts=2024-09-13T14:44:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:10.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:10 smithi177 bash[68594]: cluster 2024-09-13T14:44:09.308317+0000 mon.a (mon.0) 1528 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-13T14:44:10.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:10 smithi177 bash[68594]: cluster 2024-09-13T14:44:09.308317+0000 mon.a (mon.0) 1528 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-13T14:44:10.633 DEBUG:teuthology.parallel:result is None 2024-09-13T14:44:10.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:44:10.645 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:44:10.645 DEBUG:teuthology.orchestra.run.smithi099:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T14:44:10.729 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T14:44:10.729 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T14:44:10.729 INFO:teuthology.task.print:**** done rbd/test_librbd.sh 7-workload 2024-09-13T14:44:10.730 DEBUG:teuthology.parallel:result is None 2024-09-13T14:44:10.730 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:44:10.742 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-09-13T14:44:10.742 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-13T14:44:10.758 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:44:10.758 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:44:10.759 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:44:11.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:11 smithi177 bash[68594]: cluster 2024-09-13T14:44:10.308505+0000 mon.a (mon.0) 1529 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-13T14:44:11.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:11 smithi177 bash[68594]: cluster 2024-09-13T14:44:10.308505+0000 mon.a (mon.0) 1529 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-13T14:44:11.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:11 smithi177 bash[68594]: cluster 2024-09-13T14:44:10.402702+0000 mgr.y (mgr.26252) 531 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:11.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:11 smithi177 bash[68594]: cluster 2024-09-13T14:44:10.402702+0000 mgr.y (mgr.26252) 531 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:11 smithi099 bash[84576]: cluster 2024-09-13T14:44:10.308505+0000 mon.a (mon.0) 1529 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-13T14:44:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:11 smithi099 bash[84576]: cluster 2024-09-13T14:44:10.308505+0000 mon.a (mon.0) 1529 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-13T14:44:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:11 smithi099 bash[84576]: cluster 2024-09-13T14:44:10.402702+0000 mgr.y (mgr.26252) 531 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:11 smithi099 bash[84576]: cluster 2024-09-13T14:44:10.402702+0000 mgr.y (mgr.26252) 531 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:11.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:11 smithi099 bash[87147]: cluster 2024-09-13T14:44:10.308505+0000 mon.a (mon.0) 1529 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-13T14:44:11.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:11 smithi099 bash[87147]: cluster 2024-09-13T14:44:10.308505+0000 mon.a (mon.0) 1529 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-13T14:44:11.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:11 smithi099 bash[87147]: cluster 2024-09-13T14:44:10.402702+0000 mgr.y (mgr.26252) 531 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:11.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:11 smithi099 bash[87147]: cluster 2024-09-13T14:44:10.402702+0000 mgr.y (mgr.26252) 531 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:13.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:13 smithi099 bash[84576]: cluster 2024-09-13T14:44:12.403575+0000 mgr.y (mgr.26252) 532 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:13.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:13 smithi099 bash[84576]: cluster 2024-09-13T14:44:12.403575+0000 mgr.y (mgr.26252) 532 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:13.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:13 smithi099 bash[87147]: cluster 2024-09-13T14:44:12.403575+0000 mgr.y (mgr.26252) 532 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:13.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:13 smithi099 bash[87147]: cluster 2024-09-13T14:44:12.403575+0000 mgr.y (mgr.26252) 532 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:13 smithi177 bash[68594]: cluster 2024-09-13T14:44:12.403575+0000 mgr.y (mgr.26252) 532 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:13 smithi177 bash[68594]: cluster 2024-09-13T14:44:12.403575+0000 mgr.y (mgr.26252) 532 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:44:15.557 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:15.751 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:15 smithi099 bash[84576]: cluster 2024-09-13T14:44:14.404572+0000 mgr.y (mgr.26252) 533 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:15.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:15 smithi099 bash[84576]: cluster 2024-09-13T14:44:14.404572+0000 mgr.y (mgr.26252) 533 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:15.752 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:15 smithi099 bash[87147]: cluster 2024-09-13T14:44:14.404572+0000 mgr.y (mgr.26252) 533 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:15.752 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:15 smithi099 bash[87147]: cluster 2024-09-13T14:44:14.404572+0000 mgr.y (mgr.26252) 533 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:15 smithi177 bash[68594]: cluster 2024-09-13T14:44:14.404572+0000 mgr.y (mgr.26252) 533 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:15 smithi177 bash[68594]: cluster 2024-09-13T14:44:14.404572+0000 mgr.y (mgr.26252) 533 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:17.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:44:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:44:17] "GET /metrics HTTP/1.1" 200 36704 "" "Prometheus/2.51.0" 2024-09-13T14:44:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:17 smithi099 bash[84576]: cluster 2024-09-13T14:44:16.405188+0000 mgr.y (mgr.26252) 534 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:17 smithi099 bash[84576]: cluster 2024-09-13T14:44:16.405188+0000 mgr.y (mgr.26252) 534 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:17.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:17 smithi099 bash[87147]: cluster 2024-09-13T14:44:16.405188+0000 mgr.y (mgr.26252) 534 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:17.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:17 smithi099 bash[87147]: cluster 2024-09-13T14:44:16.405188+0000 mgr.y (mgr.26252) 534 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:17 smithi177 bash[68594]: cluster 2024-09-13T14:44:16.405188+0000 mgr.y (mgr.26252) 534 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:17 smithi177 bash[68594]: cluster 2024-09-13T14:44:16.405188+0000 mgr.y (mgr.26252) 534 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:18 smithi099 bash[87147]: audit 2024-09-13T14:44:18.265318+0000 mon.a (mon.0) 1530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:18 smithi099 bash[87147]: audit 2024-09-13T14:44:18.265318+0000 mon.a (mon.0) 1530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:18 smithi099 bash[84576]: audit 2024-09-13T14:44:18.265318+0000 mon.a (mon.0) 1530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:18 smithi099 bash[84576]: audit 2024-09-13T14:44:18.265318+0000 mon.a (mon.0) 1530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:18.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:18 smithi177 bash[68594]: audit 2024-09-13T14:44:18.265318+0000 mon.a (mon.0) 1530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:18.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:18 smithi177 bash[68594]: audit 2024-09-13T14:44:18.265318+0000 mon.a (mon.0) 1530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:19.770 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:19 smithi099 bash[84576]: cluster 2024-09-13T14:44:18.405805+0000 mgr.y (mgr.26252) 535 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:19.770 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:19 smithi099 bash[84576]: cluster 2024-09-13T14:44:18.405805+0000 mgr.y (mgr.26252) 535 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:19.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:19 smithi099 bash[87147]: cluster 2024-09-13T14:44:18.405805+0000 mgr.y (mgr.26252) 535 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:19.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:19 smithi099 bash[87147]: cluster 2024-09-13T14:44:18.405805+0000 mgr.y (mgr.26252) 535 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:19.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:19 smithi177 bash[68594]: cluster 2024-09-13T14:44:18.405805+0000 mgr.y (mgr.26252) 535 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:19.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:19 smithi177 bash[68594]: cluster 2024-09-13T14:44:18.405805+0000 mgr.y (mgr.26252) 535 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:20.167 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:44:20.489 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:44:20 smithi177 bash[61439]: ts=2024-09-13T14:44:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:20 smithi099 bash[84576]: audit 2024-09-13T14:44:20.165366+0000 mon.a (mon.0) 1531 : audit [DBG] from='client.? 172.21.15.99:0/739745565' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:44:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:20 smithi099 bash[84576]: audit 2024-09-13T14:44:20.165366+0000 mon.a (mon.0) 1531 : audit [DBG] from='client.? 172.21.15.99:0/739745565' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:44:20.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:20 smithi099 bash[87147]: audit 2024-09-13T14:44:20.165366+0000 mon.a (mon.0) 1531 : audit [DBG] from='client.? 172.21.15.99:0/739745565' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:44:20.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:20 smithi099 bash[87147]: audit 2024-09-13T14:44:20.165366+0000 mon.a (mon.0) 1531 : audit [DBG] from='client.? 172.21.15.99:0/739745565' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:44:20.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:20 smithi177 bash[68594]: audit 2024-09-13T14:44:20.165366+0000 mon.a (mon.0) 1531 : audit [DBG] from='client.? 172.21.15.99:0/739745565' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:44:20.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:20 smithi177 bash[68594]: audit 2024-09-13T14:44:20.165366+0000 mon.a (mon.0) 1531 : audit [DBG] from='client.? 172.21.15.99:0/739745565' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:44:21.111 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":39,"flags":0,"active_gid":26252,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":1762482294},{"type":"v1","addr":"172.21.15.99:6801","nonce":1762482294}]},"active_addr":"172.21.15.99:6801/1762482294","active_change":"2024-09-13T14:31:47.885582+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":25090,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.99:8443/","prometheus":"http://172.21.15.99:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":344,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":2966290862}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":3146366625}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":2775623165}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.99:0","nonce":1842329621}]}]} 2024-09-13T14:44:21.116 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:44:21.116 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:44:21.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:44:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:21 smithi099 bash[84576]: cluster 2024-09-13T14:44:20.406602+0000 mgr.y (mgr.26252) 536 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:21 smithi099 bash[84576]: cluster 2024-09-13T14:44:20.406602+0000 mgr.y (mgr.26252) 536 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:21.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:21 smithi099 bash[87147]: cluster 2024-09-13T14:44:20.406602+0000 mgr.y (mgr.26252) 536 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:21.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:21 smithi099 bash[87147]: cluster 2024-09-13T14:44:20.406602+0000 mgr.y (mgr.26252) 536 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:21.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:21 smithi177 bash[68594]: cluster 2024-09-13T14:44:20.406602+0000 mgr.y (mgr.26252) 536 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:21.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:21 smithi177 bash[68594]: cluster 2024-09-13T14:44:20.406602+0000 mgr.y (mgr.26252) 536 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:23.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:23 smithi099 bash[84576]: cluster 2024-09-13T14:44:22.407328+0000 mgr.y (mgr.26252) 537 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:23 smithi099 bash[84576]: cluster 2024-09-13T14:44:22.407328+0000 mgr.y (mgr.26252) 537 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:23.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:23 smithi099 bash[87147]: cluster 2024-09-13T14:44:22.407328+0000 mgr.y (mgr.26252) 537 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:23.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:23 smithi099 bash[87147]: cluster 2024-09-13T14:44:22.407328+0000 mgr.y (mgr.26252) 537 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:23.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:23 smithi177 bash[68594]: cluster 2024-09-13T14:44:22.407328+0000 mgr.y (mgr.26252) 537 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:23.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:23 smithi177 bash[68594]: cluster 2024-09-13T14:44:22.407328+0000 mgr.y (mgr.26252) 537 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:25.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:25 smithi099 bash[84576]: cluster 2024-09-13T14:44:24.408354+0000 mgr.y (mgr.26252) 538 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:25 smithi099 bash[84576]: cluster 2024-09-13T14:44:24.408354+0000 mgr.y (mgr.26252) 538 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:25.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:25 smithi099 bash[87147]: cluster 2024-09-13T14:44:24.408354+0000 mgr.y (mgr.26252) 538 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:25.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:25 smithi099 bash[87147]: cluster 2024-09-13T14:44:24.408354+0000 mgr.y (mgr.26252) 538 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:25.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:25 smithi177 bash[68594]: cluster 2024-09-13T14:44:24.408354+0000 mgr.y (mgr.26252) 538 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:25.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:25 smithi177 bash[68594]: cluster 2024-09-13T14:44:24.408354+0000 mgr.y (mgr.26252) 538 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:25.896 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:27.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:44:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:44:27] "GET /metrics HTTP/1.1" 200 34271 "" "Prometheus/2.51.0" 2024-09-13T14:44:27.671 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:27 smithi099 bash[84576]: cluster 2024-09-13T14:44:26.408979+0000 mgr.y (mgr.26252) 539 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:27.671 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:27 smithi099 bash[84576]: cluster 2024-09-13T14:44:26.408979+0000 mgr.y (mgr.26252) 539 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:27.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:27 smithi099 bash[87147]: cluster 2024-09-13T14:44:26.408979+0000 mgr.y (mgr.26252) 539 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:27.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:27 smithi099 bash[87147]: cluster 2024-09-13T14:44:26.408979+0000 mgr.y (mgr.26252) 539 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:27 smithi177 bash[68594]: cluster 2024-09-13T14:44:26.408979+0000 mgr.y (mgr.26252) 539 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:27 smithi177 bash[68594]: cluster 2024-09-13T14:44:26.408979+0000 mgr.y (mgr.26252) 539 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:29.796 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:44:29.796 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":951,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:10:21.363918+0000","modified":"2024-09-13T14:44:10.297966+0000","last_up_change":"2024-09-13T14:40:08.071422+0000","last_in_change":"2024-09-13T14:19:38.687766+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":40,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:15:37.290226+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:21:51.531836+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:24:15.786407+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:24:16.627611+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0624feaa-339b-4468-be53-582a1f04352c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":558,"up_from":575,"up_thru":899,"down_at":558,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6803","nonce":1813984480}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6805","nonce":1813984480}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6809","nonce":1813984480}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6807","nonce":1813984480}]},"public_addr":"172.21.15.99:6803/1813984480","cluster_addr":"172.21.15.99:6805/1813984480","heartbeat_back_addr":"172.21.15.99:6809/1813984480","heartbeat_front_addr":"172.21.15.99:6807/1813984480","state":["exists","up"]},{"osd":1,"uuid":"b7b63553-e49d-4636-b3e2-43b72556a1a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":638,"up_from":642,"up_thru":899,"down_at":639,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6811","nonce":804978824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6813","nonce":804978824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6817","nonce":804978824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6815","nonce":804978824}]},"public_addr":"172.21.15.99:6811/804978824","cluster_addr":"172.21.15.99:6813/804978824","heartbeat_back_addr":"172.21.15.99:6817/804978824","heartbeat_front_addr":"172.21.15.99:6815/804978824","state":["exists","up"]},{"osd":2,"uuid":"3b9eff6d-5f5c-4843-a1ff-815a902167ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":656,"up_from":672,"up_thru":899,"down_at":657,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6819","nonce":129558315}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6821","nonce":129558315}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6825","nonce":129558315}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6823","nonce":129558315}]},"public_addr":"172.21.15.99:6819/129558315","cluster_addr":"172.21.15.99:6821/129558315","heartbeat_back_addr":"172.21.15.99:6825/129558315","heartbeat_front_addr":"172.21.15.99:6823/129558315","state":["exists","up"]},{"osd":3,"uuid":"686c8d3c-9cab-4920-b02b-d243324f5808","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":689,"up_from":706,"up_thru":899,"down_at":690,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":876595003},{"type":"v1","addr":"172.21.15.99:6827","nonce":876595003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":876595003},{"type":"v1","addr":"172.21.15.99:6829","nonce":876595003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":876595003},{"type":"v1","addr":"172.21.15.99:6833","nonce":876595003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":876595003},{"type":"v1","addr":"172.21.15.99:6831","nonce":876595003}]},"public_addr":"172.21.15.99:6827/876595003","cluster_addr":"172.21.15.99:6829/876595003","heartbeat_back_addr":"172.21.15.99:6833/876595003","heartbeat_front_addr":"172.21.15.99:6831/876595003","state":["exists","up"]},{"osd":4,"uuid":"2d56712b-a6fb-4089-9583-850beed92b37","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":724,"up_from":739,"up_thru":899,"down_at":725,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6801","nonce":2135282964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6803","nonce":2135282964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6807","nonce":2135282964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6805","nonce":2135282964}]},"public_addr":"172.21.15.177:6801/2135282964","cluster_addr":"172.21.15.177:6803/2135282964","heartbeat_back_addr":"172.21.15.177:6807/2135282964","heartbeat_front_addr":"172.21.15.177:6805/2135282964","state":["exists","up"]},{"osd":5,"uuid":"414b880c-6534-495f-a54c-f8d28ba81c73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6809","nonce":947070587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6811","nonce":947070587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6815","nonce":947070587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6813","nonce":947070587}]},"public_addr":"172.21.15.177:6809/947070587","cluster_addr":"172.21.15.177:6811/947070587","heartbeat_back_addr":"172.21.15.177:6815/947070587","heartbeat_front_addr":"172.21.15.177:6813/947070587","state":["exists","up"]},{"osd":6,"uuid":"ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6817","nonce":3817692594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6819","nonce":3817692594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6823","nonce":3817692594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6821","nonce":3817692594}]},"public_addr":"172.21.15.177:6817/3817692594","cluster_addr":"172.21.15.177:6819/3817692594","heartbeat_back_addr":"172.21.15.177:6823/3817692594","heartbeat_front_addr":"172.21.15.177:6821/3817692594","state":["exists","up"]},{"osd":7,"uuid":"ad2c2977-fd6d-42f8-962e-907c2bf3204b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6825","nonce":1984676946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6827","nonce":1984676946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6831","nonce":1984676946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6829","nonce":1984676946}]},"public_addr":"172.21.15.177:6825/1984676946","cluster_addr":"172.21.15.177:6827/1984676946","heartbeat_back_addr":"172.21.15.177:6831/1984676946","heartbeat_front_addr":"172.21.15.177:6829/1984676946","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:46.390607+0000","dead_epoch":557},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:38.398637+0000","dead_epoch":638},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:15:33.420625+0000","dead_epoch":656},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:30.964755+0000","dead_epoch":689},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:22.513614+0000","dead_epoch":724},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:14.948720+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:19:10.431389+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:20:03.826389+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:0/3935330909":"2024-09-13T15:40:15.846422+0000","172.21.15.99:0/3556003775":"2024-09-13T15:35:26.139282+0000","172.21.15.99:0/1151643634":"2024-09-14T14:31:47.884997+0000","172.21.15.177:6834/4079938285":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/881490667":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/4242039269":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/316966644":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/2140252887":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/17074640":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/3817526904":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6835/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.177:6835/4079938285":"2024-09-14T14:30:38.246301+0000","172.21.15.99:6800/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6834/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.99:6801/1402010506":"2024-09-14T14:31:47.884997+0000","172.21.15.177:0/1330817275":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/1638158155":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/4129805635":"2024-09-14T14:11:18.714884+0000","172.21.15.177:0/3478790043":"2024-09-14T14:30:38.246301+0000","172.21.15.99:6801/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/888319381":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/181419264":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/3483428223":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6800/1402010506":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/795365709":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/1548831286":"2024-09-14T14:11:18.714884+0000","172.21.15.177:0/3423158096":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/3394219806":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/3192879133":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/1363241618":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/405712510":"2024-09-14T14:21:07.041302+0000","172.21.15.177:0/1523474123":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/313522629":"2024-09-13T15:35:27.165268+0000","172.21.15.99:6800/2887456408":"2024-09-14T14:29:03.998039+0000","172.21.15.99:6801/2887456408":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/4277277716":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6800/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:6800/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6801/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/2372940082":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6801/3775052239":"2024-09-14T14:21:07.041302+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":227,"length":1},{"begin":229,"length":1},{"begin":231,"length":1}]},{"pool":13,"snaps":[{"begin":7,"length":2}]},{"pool":15,"snaps":[{"begin":35,"length":3},{"begin":43,"length":2},{"begin":47,"length":1}]},{"pool":16,"snaps":[{"begin":22,"length":1}]},{"pool":17,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":8,"length":3}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:44:29.808 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:29 smithi099 bash[84576]: cluster 2024-09-13T14:44:28.409931+0000 mgr.y (mgr.26252) 540 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:29.808 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:29 smithi099 bash[84576]: cluster 2024-09-13T14:44:28.409931+0000 mgr.y (mgr.26252) 540 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:29.809 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:29 smithi099 bash[87147]: cluster 2024-09-13T14:44:28.409931+0000 mgr.y (mgr.26252) 540 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:29.809 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:29 smithi099 bash[87147]: cluster 2024-09-13T14:44:28.409931+0000 mgr.y (mgr.26252) 540 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:29 smithi177 bash[68594]: cluster 2024-09-13T14:44:28.409931+0000 mgr.y (mgr.26252) 540 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:29 smithi177 bash[68594]: cluster 2024-09-13T14:44:28.409931+0000 mgr.y (mgr.26252) 540 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:30.515 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:44:30 smithi177 bash[61439]: ts=2024-09-13T14:44:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:30.700 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:44:30.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:44:30.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:30 smithi099 bash[84576]: audit 2024-09-13T14:44:29.797000+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.99:0/2902621315' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:30 smithi099 bash[84576]: audit 2024-09-13T14:44:29.797000+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.99:0/2902621315' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:30 smithi099 bash[87147]: audit 2024-09-13T14:44:29.797000+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.99:0/2902621315' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:30.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:30 smithi099 bash[87147]: audit 2024-09-13T14:44:29.797000+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.99:0/2902621315' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:30.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:30 smithi177 bash[68594]: audit 2024-09-13T14:44:29.797000+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.99:0/2902621315' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:30.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:30 smithi177 bash[68594]: audit 2024-09-13T14:44:29.797000+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.99:0/2902621315' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:31.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:31 smithi099 bash[84576]: cluster 2024-09-13T14:44:30.410747+0000 mgr.y (mgr.26252) 541 : cluster [DBG] pgmap v991: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:31 smithi099 bash[84576]: cluster 2024-09-13T14:44:30.410747+0000 mgr.y (mgr.26252) 541 : cluster [DBG] pgmap v991: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:31.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:31 smithi099 bash[87147]: cluster 2024-09-13T14:44:30.410747+0000 mgr.y (mgr.26252) 541 : cluster [DBG] pgmap v991: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:31.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:31 smithi099 bash[87147]: cluster 2024-09-13T14:44:30.410747+0000 mgr.y (mgr.26252) 541 : cluster [DBG] pgmap v991: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:31 smithi177 bash[68594]: cluster 2024-09-13T14:44:30.410747+0000 mgr.y (mgr.26252) 541 : cluster [DBG] pgmap v991: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:31 smithi177 bash[68594]: cluster 2024-09-13T14:44:30.410747+0000 mgr.y (mgr.26252) 541 : cluster [DBG] pgmap v991: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:33.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:33 smithi099 bash[84576]: cluster 2024-09-13T14:44:32.411485+0000 mgr.y (mgr.26252) 542 : cluster [DBG] pgmap v992: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:33.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:33 smithi099 bash[84576]: cluster 2024-09-13T14:44:32.411485+0000 mgr.y (mgr.26252) 542 : cluster [DBG] pgmap v992: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:33.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:33 smithi099 bash[84576]: audit 2024-09-13T14:44:33.265978+0000 mon.a (mon.0) 1532 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:33.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:33 smithi099 bash[84576]: audit 2024-09-13T14:44:33.265978+0000 mon.a (mon.0) 1532 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:33 smithi099 bash[87147]: cluster 2024-09-13T14:44:32.411485+0000 mgr.y (mgr.26252) 542 : cluster [DBG] pgmap v992: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:33.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:33 smithi099 bash[87147]: cluster 2024-09-13T14:44:32.411485+0000 mgr.y (mgr.26252) 542 : cluster [DBG] pgmap v992: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:33.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:33 smithi099 bash[87147]: audit 2024-09-13T14:44:33.265978+0000 mon.a (mon.0) 1532 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:33.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:33 smithi099 bash[87147]: audit 2024-09-13T14:44:33.265978+0000 mon.a (mon.0) 1532 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:33.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:33 smithi177 bash[68594]: cluster 2024-09-13T14:44:32.411485+0000 mgr.y (mgr.26252) 542 : cluster [DBG] pgmap v992: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:33.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:33 smithi177 bash[68594]: cluster 2024-09-13T14:44:32.411485+0000 mgr.y (mgr.26252) 542 : cluster [DBG] pgmap v992: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:33 smithi177 bash[68594]: audit 2024-09-13T14:44:33.265978+0000 mon.a (mon.0) 1532 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:33 smithi177 bash[68594]: audit 2024-09-13T14:44:33.265978+0000 mon.a (mon.0) 1532 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:35.488 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:35 smithi099 bash[84576]: cluster 2024-09-13T14:44:34.412470+0000 mgr.y (mgr.26252) 543 : cluster [DBG] pgmap v993: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:35 smithi099 bash[84576]: cluster 2024-09-13T14:44:34.412470+0000 mgr.y (mgr.26252) 543 : cluster [DBG] pgmap v993: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:35 smithi099 bash[87147]: cluster 2024-09-13T14:44:34.412470+0000 mgr.y (mgr.26252) 543 : cluster [DBG] pgmap v993: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:35 smithi099 bash[87147]: cluster 2024-09-13T14:44:34.412470+0000 mgr.y (mgr.26252) 543 : cluster [DBG] pgmap v993: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:35.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:35 smithi177 bash[68594]: cluster 2024-09-13T14:44:34.412470+0000 mgr.y (mgr.26252) 543 : cluster [DBG] pgmap v993: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:35.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:35 smithi177 bash[68594]: cluster 2024-09-13T14:44:34.412470+0000 mgr.y (mgr.26252) 543 : cluster [DBG] pgmap v993: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:37.285 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:44:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:44:37] "GET /metrics HTTP/1.1" 200 34271 "" "Prometheus/2.51.0" 2024-09-13T14:44:37.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:37 smithi099 bash[84576]: cluster 2024-09-13T14:44:36.413000+0000 mgr.y (mgr.26252) 544 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:37.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:37 smithi099 bash[84576]: cluster 2024-09-13T14:44:36.413000+0000 mgr.y (mgr.26252) 544 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:37.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:37 smithi099 bash[87147]: cluster 2024-09-13T14:44:36.413000+0000 mgr.y (mgr.26252) 544 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:37.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:37 smithi099 bash[87147]: cluster 2024-09-13T14:44:36.413000+0000 mgr.y (mgr.26252) 544 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:37 smithi177 bash[68594]: cluster 2024-09-13T14:44:36.413000+0000 mgr.y (mgr.26252) 544 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:37 smithi177 bash[68594]: cluster 2024-09-13T14:44:36.413000+0000 mgr.y (mgr.26252) 544 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:39.490 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:44:39.490 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":951,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:10:21.363918+0000","modified":"2024-09-13T14:44:10.297966+0000","last_up_change":"2024-09-13T14:40:08.071422+0000","last_in_change":"2024-09-13T14:19:38.687766+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":40,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:15:37.290226+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:21:51.531836+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:24:15.786407+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:24:16.627611+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0624feaa-339b-4468-be53-582a1f04352c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":558,"up_from":575,"up_thru":899,"down_at":558,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6803","nonce":1813984480}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6805","nonce":1813984480}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6809","nonce":1813984480}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6807","nonce":1813984480}]},"public_addr":"172.21.15.99:6803/1813984480","cluster_addr":"172.21.15.99:6805/1813984480","heartbeat_back_addr":"172.21.15.99:6809/1813984480","heartbeat_front_addr":"172.21.15.99:6807/1813984480","state":["exists","up"]},{"osd":1,"uuid":"b7b63553-e49d-4636-b3e2-43b72556a1a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":638,"up_from":642,"up_thru":899,"down_at":639,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6811","nonce":804978824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6813","nonce":804978824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6817","nonce":804978824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6815","nonce":804978824}]},"public_addr":"172.21.15.99:6811/804978824","cluster_addr":"172.21.15.99:6813/804978824","heartbeat_back_addr":"172.21.15.99:6817/804978824","heartbeat_front_addr":"172.21.15.99:6815/804978824","state":["exists","up"]},{"osd":2,"uuid":"3b9eff6d-5f5c-4843-a1ff-815a902167ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":656,"up_from":672,"up_thru":899,"down_at":657,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6819","nonce":129558315}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6821","nonce":129558315}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6825","nonce":129558315}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6823","nonce":129558315}]},"public_addr":"172.21.15.99:6819/129558315","cluster_addr":"172.21.15.99:6821/129558315","heartbeat_back_addr":"172.21.15.99:6825/129558315","heartbeat_front_addr":"172.21.15.99:6823/129558315","state":["exists","up"]},{"osd":3,"uuid":"686c8d3c-9cab-4920-b02b-d243324f5808","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":689,"up_from":706,"up_thru":899,"down_at":690,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":876595003},{"type":"v1","addr":"172.21.15.99:6827","nonce":876595003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":876595003},{"type":"v1","addr":"172.21.15.99:6829","nonce":876595003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":876595003},{"type":"v1","addr":"172.21.15.99:6833","nonce":876595003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":876595003},{"type":"v1","addr":"172.21.15.99:6831","nonce":876595003}]},"public_addr":"172.21.15.99:6827/876595003","cluster_addr":"172.21.15.99:6829/876595003","heartbeat_back_addr":"172.21.15.99:6833/876595003","heartbeat_front_addr":"172.21.15.99:6831/876595003","state":["exists","up"]},{"osd":4,"uuid":"2d56712b-a6fb-4089-9583-850beed92b37","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":724,"up_from":739,"up_thru":899,"down_at":725,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6801","nonce":2135282964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6803","nonce":2135282964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6807","nonce":2135282964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6805","nonce":2135282964}]},"public_addr":"172.21.15.177:6801/2135282964","cluster_addr":"172.21.15.177:6803/2135282964","heartbeat_back_addr":"172.21.15.177:6807/2135282964","heartbeat_front_addr":"172.21.15.177:6805/2135282964","state":["exists","up"]},{"osd":5,"uuid":"414b880c-6534-495f-a54c-f8d28ba81c73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6809","nonce":947070587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6811","nonce":947070587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6815","nonce":947070587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6813","nonce":947070587}]},"public_addr":"172.21.15.177:6809/947070587","cluster_addr":"172.21.15.177:6811/947070587","heartbeat_back_addr":"172.21.15.177:6815/947070587","heartbeat_front_addr":"172.21.15.177:6813/947070587","state":["exists","up"]},{"osd":6,"uuid":"ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6817","nonce":3817692594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6819","nonce":3817692594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6823","nonce":3817692594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6821","nonce":3817692594}]},"public_addr":"172.21.15.177:6817/3817692594","cluster_addr":"172.21.15.177:6819/3817692594","heartbeat_back_addr":"172.21.15.177:6823/3817692594","heartbeat_front_addr":"172.21.15.177:6821/3817692594","state":["exists","up"]},{"osd":7,"uuid":"ad2c2977-fd6d-42f8-962e-907c2bf3204b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":899,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6825","nonce":1984676946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6827","nonce":1984676946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6831","nonce":1984676946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6829","nonce":1984676946}]},"public_addr":"172.21.15.177:6825/1984676946","cluster_addr":"172.21.15.177:6827/1984676946","heartbeat_back_addr":"172.21.15.177:6831/1984676946","heartbeat_front_addr":"172.21.15.177:6829/1984676946","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:46.390607+0000","dead_epoch":557},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:38.398637+0000","dead_epoch":638},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:15:33.420625+0000","dead_epoch":656},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:30.964755+0000","dead_epoch":689},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:22.513614+0000","dead_epoch":724},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:14.948720+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:19:10.431389+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:20:03.826389+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:0/3935330909":"2024-09-13T15:40:15.846422+0000","172.21.15.99:0/3556003775":"2024-09-13T15:35:26.139282+0000","172.21.15.99:0/1151643634":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6801/1402010506":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6800/1402010506":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/3817526904":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/3394219806":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/181419264":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/4129805635":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/1363241618":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/3192879133":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/3483428223":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/405712510":"2024-09-14T14:21:07.041302+0000","172.21.15.177:0/1523474123":"2024-09-14T14:30:38.246301+0000","172.21.15.99:6834/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.99:0/795365709":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/1548831286":"2024-09-14T14:11:18.714884+0000","172.21.15.177:0/3423158096":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/888319381":"2024-09-14T14:10:48.295312+0000","172.21.15.177:0/3478790043":"2024-09-14T14:30:38.246301+0000","172.21.15.99:6801/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.177:0/1330817275":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/1638158155":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/313522629":"2024-09-13T15:35:27.165268+0000","172.21.15.99:6800/2887456408":"2024-09-14T14:29:03.998039+0000","172.21.15.99:6801/2887456408":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/4277277716":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6800/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/17074640":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6835/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.99:6800/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.177:6834/4079938285":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/2372940082":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6801/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6801/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/316966644":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/2140252887":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/4242039269":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/881490667":"2024-09-14T14:10:48.295312+0000","172.21.15.99:6800/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.177:6835/4079938285":"2024-09-14T14:30:38.246301+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":227,"length":1},{"begin":229,"length":1},{"begin":231,"length":1}]},{"pool":13,"snaps":[{"begin":7,"length":2}]},{"pool":15,"snaps":[{"begin":35,"length":3},{"begin":43,"length":2},{"begin":47,"length":1}]},{"pool":16,"snaps":[{"begin":22,"length":1}]},{"pool":17,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":8,"length":3}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:44:39.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:39 smithi099 bash[84576]: cluster 2024-09-13T14:44:38.413876+0000 mgr.y (mgr.26252) 545 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:39.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:39 smithi099 bash[84576]: cluster 2024-09-13T14:44:38.413876+0000 mgr.y (mgr.26252) 545 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:39.753 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:39 smithi099 bash[84576]: audit 2024-09-13T14:44:39.491216+0000 mon.a (mon.0) 1533 : audit [DBG] from='client.? 172.21.15.99:0/721197847' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:39.753 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:39 smithi099 bash[84576]: audit 2024-09-13T14:44:39.491216+0000 mon.a (mon.0) 1533 : audit [DBG] from='client.? 172.21.15.99:0/721197847' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:39.753 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:39 smithi099 bash[87147]: cluster 2024-09-13T14:44:38.413876+0000 mgr.y (mgr.26252) 545 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:39.753 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:39 smithi099 bash[87147]: cluster 2024-09-13T14:44:38.413876+0000 mgr.y (mgr.26252) 545 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:39.753 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:39 smithi099 bash[87147]: audit 2024-09-13T14:44:39.491216+0000 mon.a (mon.0) 1533 : audit [DBG] from='client.? 172.21.15.99:0/721197847' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:39.753 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:39 smithi099 bash[87147]: audit 2024-09-13T14:44:39.491216+0000 mon.a (mon.0) 1533 : audit [DBG] from='client.? 172.21.15.99:0/721197847' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:39 smithi177 bash[68594]: cluster 2024-09-13T14:44:38.413876+0000 mgr.y (mgr.26252) 545 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:39 smithi177 bash[68594]: cluster 2024-09-13T14:44:38.413876+0000 mgr.y (mgr.26252) 545 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:39 smithi177 bash[68594]: audit 2024-09-13T14:44:39.491216+0000 mon.a (mon.0) 1533 : audit [DBG] from='client.? 172.21.15.99:0/721197847' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:39 smithi177 bash[68594]: audit 2024-09-13T14:44:39.491216+0000 mon.a (mon.0) 1533 : audit [DBG] from='client.? 172.21.15.99:0/721197847' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:44:40.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:44:40.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:44:40.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:44:40.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:44:40.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:44:40.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:44:40.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:44:40.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:44:40.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:44:40 smithi177 bash[61439]: ts=2024-09-13T14:44:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:41 smithi099 bash[84576]: cluster 2024-09-13T14:44:40.414548+0000 mgr.y (mgr.26252) 546 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:41 smithi099 bash[84576]: cluster 2024-09-13T14:44:40.414548+0000 mgr.y (mgr.26252) 546 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:41.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:41 smithi099 bash[87147]: cluster 2024-09-13T14:44:40.414548+0000 mgr.y (mgr.26252) 546 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:41.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:41 smithi099 bash[87147]: cluster 2024-09-13T14:44:40.414548+0000 mgr.y (mgr.26252) 546 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:41 smithi177 bash[68594]: cluster 2024-09-13T14:44:40.414548+0000 mgr.y (mgr.26252) 546 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:41 smithi177 bash[68594]: cluster 2024-09-13T14:44:40.414548+0000 mgr.y (mgr.26252) 546 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:43 smithi099 bash[84576]: cluster 2024-09-13T14:44:42.415287+0000 mgr.y (mgr.26252) 547 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:43 smithi099 bash[84576]: cluster 2024-09-13T14:44:42.415287+0000 mgr.y (mgr.26252) 547 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:43 smithi099 bash[84576]: audit 2024-09-13T14:44:42.965163+0000 mon.a (mon.0) 1534 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:43 smithi099 bash[84576]: audit 2024-09-13T14:44:42.965163+0000 mon.a (mon.0) 1534 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:43.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:43 smithi099 bash[87147]: cluster 2024-09-13T14:44:42.415287+0000 mgr.y (mgr.26252) 547 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:43 smithi099 bash[87147]: cluster 2024-09-13T14:44:42.415287+0000 mgr.y (mgr.26252) 547 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:43 smithi099 bash[87147]: audit 2024-09-13T14:44:42.965163+0000 mon.a (mon.0) 1534 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:43 smithi099 bash[87147]: audit 2024-09-13T14:44:42.965163+0000 mon.a (mon.0) 1534 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:43 smithi177 bash[68594]: cluster 2024-09-13T14:44:42.415287+0000 mgr.y (mgr.26252) 547 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:43 smithi177 bash[68594]: cluster 2024-09-13T14:44:42.415287+0000 mgr.y (mgr.26252) 547 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:43 smithi177 bash[68594]: audit 2024-09-13T14:44:42.965163+0000 mon.a (mon.0) 1534 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:43 smithi177 bash[68594]: audit 2024-09-13T14:44:42.965163+0000 mon.a (mon.0) 1534 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:45.235 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:45.235 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:45.235 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:45.238 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:45.239 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:45.239 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:45.241 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:45.242 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:45.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:45 smithi099 bash[84576]: cluster 2024-09-13T14:44:44.416289+0000 mgr.y (mgr.26252) 548 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:45.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:45 smithi099 bash[84576]: cluster 2024-09-13T14:44:44.416289+0000 mgr.y (mgr.26252) 548 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:45.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:45 smithi099 bash[87147]: cluster 2024-09-13T14:44:44.416289+0000 mgr.y (mgr.26252) 548 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:45.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:45 smithi099 bash[87147]: cluster 2024-09-13T14:44:44.416289+0000 mgr.y (mgr.26252) 548 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:45 smithi177 bash[68594]: cluster 2024-09-13T14:44:44.416289+0000 mgr.y (mgr.26252) 548 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:45 smithi177 bash[68594]: cluster 2024-09-13T14:44:44.416289+0000 mgr.y (mgr.26252) 548 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:47.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:44:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:44:47] "GET /metrics HTTP/1.1" 200 34271 "" "Prometheus/2.51.0" 2024-09-13T14:44:47.820 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:47 smithi099 bash[84576]: cluster 2024-09-13T14:44:46.416712+0000 mgr.y (mgr.26252) 549 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:47.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:47 smithi099 bash[84576]: cluster 2024-09-13T14:44:46.416712+0000 mgr.y (mgr.26252) 549 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:47.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:47 smithi099 bash[87147]: cluster 2024-09-13T14:44:46.416712+0000 mgr.y (mgr.26252) 549 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:47.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:47 smithi099 bash[87147]: cluster 2024-09-13T14:44:46.416712+0000 mgr.y (mgr.26252) 549 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:47.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:47 smithi177 bash[68594]: cluster 2024-09-13T14:44:46.416712+0000 mgr.y (mgr.26252) 549 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:47.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:47 smithi177 bash[68594]: cluster 2024-09-13T14:44:46.416712+0000 mgr.y (mgr.26252) 549 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[84576]: audit 2024-09-13T14:44:47.995734+0000 mon.a (mon.0) 1535 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[84576]: audit 2024-09-13T14:44:47.995734+0000 mon.a (mon.0) 1535 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[84576]: audit 2024-09-13T14:44:48.002116+0000 mon.a (mon.0) 1536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[84576]: audit 2024-09-13T14:44:48.002116+0000 mon.a (mon.0) 1536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[84576]: audit 2024-09-13T14:44:48.266279+0000 mon.a (mon.0) 1537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[84576]: audit 2024-09-13T14:44:48.266279+0000 mon.a (mon.0) 1537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[84576]: cluster 2024-09-13T14:44:48.417524+0000 mgr.y (mgr.26252) 550 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:49.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[84576]: cluster 2024-09-13T14:44:48.417524+0000 mgr.y (mgr.26252) 550 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[87147]: audit 2024-09-13T14:44:47.995734+0000 mon.a (mon.0) 1535 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[87147]: audit 2024-09-13T14:44:47.995734+0000 mon.a (mon.0) 1535 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[87147]: audit 2024-09-13T14:44:48.002116+0000 mon.a (mon.0) 1536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[87147]: audit 2024-09-13T14:44:48.002116+0000 mon.a (mon.0) 1536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[87147]: audit 2024-09-13T14:44:48.266279+0000 mon.a (mon.0) 1537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[87147]: audit 2024-09-13T14:44:48.266279+0000 mon.a (mon.0) 1537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[87147]: cluster 2024-09-13T14:44:48.417524+0000 mgr.y (mgr.26252) 550 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:49.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:49 smithi099 bash[87147]: cluster 2024-09-13T14:44:48.417524+0000 mgr.y (mgr.26252) 550 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:48 smithi177 bash[68594]: audit 2024-09-13T14:44:47.995734+0000 mon.a (mon.0) 1535 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:48 smithi177 bash[68594]: audit 2024-09-13T14:44:47.995734+0000 mon.a (mon.0) 1535 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:48 smithi177 bash[68594]: audit 2024-09-13T14:44:48.002116+0000 mon.a (mon.0) 1536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:48 smithi177 bash[68594]: audit 2024-09-13T14:44:48.002116+0000 mon.a (mon.0) 1536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:48 smithi177 bash[68594]: audit 2024-09-13T14:44:48.266279+0000 mon.a (mon.0) 1537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:48 smithi177 bash[68594]: audit 2024-09-13T14:44:48.266279+0000 mon.a (mon.0) 1537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:48 smithi177 bash[68594]: cluster 2024-09-13T14:44:48.417524+0000 mgr.y (mgr.26252) 550 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:48 smithi177 bash[68594]: cluster 2024-09-13T14:44:48.417524+0000 mgr.y (mgr.26252) 550 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:50.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[84576]: audit 2024-09-13T14:44:48.996685+0000 mon.a (mon.0) 1538 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[84576]: audit 2024-09-13T14:44:48.996685+0000 mon.a (mon.0) 1538 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[84576]: audit 2024-09-13T14:44:49.012747+0000 mon.a (mon.0) 1539 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[84576]: audit 2024-09-13T14:44:49.012747+0000 mon.a (mon.0) 1539 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[84576]: audit 2024-09-13T14:44:49.674851+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:50.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[84576]: audit 2024-09-13T14:44:49.674851+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:50.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[84576]: audit 2024-09-13T14:44:49.676394+0000 mon.a (mon.0) 1541 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:50.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[84576]: audit 2024-09-13T14:44:49.676394+0000 mon.a (mon.0) 1541 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:50.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[84576]: audit 2024-09-13T14:44:49.685787+0000 mon.a (mon.0) 1542 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[84576]: audit 2024-09-13T14:44:49.685787+0000 mon.a (mon.0) 1542 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[87147]: audit 2024-09-13T14:44:48.996685+0000 mon.a (mon.0) 1538 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[87147]: audit 2024-09-13T14:44:48.996685+0000 mon.a (mon.0) 1538 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[87147]: audit 2024-09-13T14:44:49.012747+0000 mon.a (mon.0) 1539 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[87147]: audit 2024-09-13T14:44:49.012747+0000 mon.a (mon.0) 1539 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[87147]: audit 2024-09-13T14:44:49.674851+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:50.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[87147]: audit 2024-09-13T14:44:49.674851+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:50.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[87147]: audit 2024-09-13T14:44:49.676394+0000 mon.a (mon.0) 1541 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:50.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[87147]: audit 2024-09-13T14:44:49.676394+0000 mon.a (mon.0) 1541 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:50.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[87147]: audit 2024-09-13T14:44:49.685787+0000 mon.a (mon.0) 1542 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:50 smithi099 bash[87147]: audit 2024-09-13T14:44:49.685787+0000 mon.a (mon.0) 1542 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[61439]: ts=2024-09-13T14:44:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:44:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[68594]: audit 2024-09-13T14:44:48.996685+0000 mon.a (mon.0) 1538 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[68594]: audit 2024-09-13T14:44:48.996685+0000 mon.a (mon.0) 1538 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[68594]: audit 2024-09-13T14:44:49.012747+0000 mon.a (mon.0) 1539 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[68594]: audit 2024-09-13T14:44:49.012747+0000 mon.a (mon.0) 1539 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[68594]: audit 2024-09-13T14:44:49.674851+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[68594]: audit 2024-09-13T14:44:49.674851+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[68594]: audit 2024-09-13T14:44:49.676394+0000 mon.a (mon.0) 1541 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[68594]: audit 2024-09-13T14:44:49.676394+0000 mon.a (mon.0) 1541 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[68594]: audit 2024-09-13T14:44:49.685787+0000 mon.a (mon.0) 1542 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:50.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:50 smithi177 bash[68594]: audit 2024-09-13T14:44:49.685787+0000 mon.a (mon.0) 1542 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:44:51.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:51 smithi177 bash[68594]: cluster 2024-09-13T14:44:50.418144+0000 mgr.y (mgr.26252) 551 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:51.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:51 smithi177 bash[68594]: cluster 2024-09-13T14:44:50.418144+0000 mgr.y (mgr.26252) 551 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:51.449 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:51 smithi099 bash[84576]: cluster 2024-09-13T14:44:50.418144+0000 mgr.y (mgr.26252) 551 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:51.450 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:51 smithi099 bash[84576]: cluster 2024-09-13T14:44:50.418144+0000 mgr.y (mgr.26252) 551 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:51.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:51 smithi099 bash[87147]: cluster 2024-09-13T14:44:50.418144+0000 mgr.y (mgr.26252) 551 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:51.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:51 smithi099 bash[87147]: cluster 2024-09-13T14:44:50.418144+0000 mgr.y (mgr.26252) 551 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:53 smithi099 bash[84576]: cluster 2024-09-13T14:44:52.418965+0000 mgr.y (mgr.26252) 552 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:53.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:53 smithi099 bash[84576]: cluster 2024-09-13T14:44:52.418965+0000 mgr.y (mgr.26252) 552 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:53 smithi099 bash[87147]: cluster 2024-09-13T14:44:52.418965+0000 mgr.y (mgr.26252) 552 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:53.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:53 smithi099 bash[87147]: cluster 2024-09-13T14:44:52.418965+0000 mgr.y (mgr.26252) 552 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:53.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:53 smithi177 bash[68594]: cluster 2024-09-13T14:44:52.418965+0000 mgr.y (mgr.26252) 552 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:53.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:53 smithi177 bash[68594]: cluster 2024-09-13T14:44:52.418965+0000 mgr.y (mgr.26252) 552 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:55.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:55 smithi099 bash[84576]: cluster 2024-09-13T14:44:54.419706+0000 mgr.y (mgr.26252) 553 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:55.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:55 smithi099 bash[84576]: cluster 2024-09-13T14:44:54.419706+0000 mgr.y (mgr.26252) 553 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:55.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:55 smithi099 bash[87147]: cluster 2024-09-13T14:44:54.419706+0000 mgr.y (mgr.26252) 553 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:55.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:55 smithi099 bash[87147]: cluster 2024-09-13T14:44:54.419706+0000 mgr.y (mgr.26252) 553 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:55.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:55 smithi177 bash[68594]: cluster 2024-09-13T14:44:54.419706+0000 mgr.y (mgr.26252) 553 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:55.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:55 smithi177 bash[68594]: cluster 2024-09-13T14:44:54.419706+0000 mgr.y (mgr.26252) 553 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:44:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:44:57] "GET /metrics HTTP/1.1" 200 34269 "" "Prometheus/2.51.0" 2024-09-13T14:44:57.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:57 smithi099 bash[84576]: cluster 2024-09-13T14:44:56.420283+0000 mgr.y (mgr.26252) 554 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:57.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:57 smithi099 bash[84576]: cluster 2024-09-13T14:44:56.420283+0000 mgr.y (mgr.26252) 554 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:57.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:57 smithi099 bash[87147]: cluster 2024-09-13T14:44:56.420283+0000 mgr.y (mgr.26252) 554 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:57.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:57 smithi099 bash[87147]: cluster 2024-09-13T14:44:56.420283+0000 mgr.y (mgr.26252) 554 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:57 smithi177 bash[68594]: cluster 2024-09-13T14:44:56.420283+0000 mgr.y (mgr.26252) 554 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:57 smithi177 bash[68594]: cluster 2024-09-13T14:44:56.420283+0000 mgr.y (mgr.26252) 554 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:59.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:59 smithi099 bash[84576]: cluster 2024-09-13T14:44:58.421245+0000 mgr.y (mgr.26252) 555 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:59.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:44:59 smithi099 bash[84576]: cluster 2024-09-13T14:44:58.421245+0000 mgr.y (mgr.26252) 555 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:59.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:59 smithi099 bash[87147]: cluster 2024-09-13T14:44:58.421245+0000 mgr.y (mgr.26252) 555 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:59.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:44:59 smithi099 bash[87147]: cluster 2024-09-13T14:44:58.421245+0000 mgr.y (mgr.26252) 555 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:59.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:59 smithi177 bash[68594]: cluster 2024-09-13T14:44:58.421245+0000 mgr.y (mgr.26252) 555 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:44:59.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:44:59 smithi177 bash[68594]: cluster 2024-09-13T14:44:58.421245+0000 mgr.y (mgr.26252) 555 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:00.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:45:00 smithi177 bash[61439]: ts=2024-09-13T14:45:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:01 smithi099 bash[84576]: cluster 2024-09-13T14:45:00.422232+0000 mgr.y (mgr.26252) 556 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:01 smithi099 bash[84576]: cluster 2024-09-13T14:45:00.422232+0000 mgr.y (mgr.26252) 556 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:01.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:01 smithi099 bash[87147]: cluster 2024-09-13T14:45:00.422232+0000 mgr.y (mgr.26252) 556 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:01.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:01 smithi099 bash[87147]: cluster 2024-09-13T14:45:00.422232+0000 mgr.y (mgr.26252) 556 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:01.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:01 smithi177 bash[68594]: cluster 2024-09-13T14:45:00.422232+0000 mgr.y (mgr.26252) 556 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:01.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:01 smithi177 bash[68594]: cluster 2024-09-13T14:45:00.422232+0000 mgr.y (mgr.26252) 556 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:03 smithi099 bash[84576]: cluster 2024-09-13T14:45:02.423157+0000 mgr.y (mgr.26252) 557 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:03 smithi099 bash[84576]: cluster 2024-09-13T14:45:02.423157+0000 mgr.y (mgr.26252) 557 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:03 smithi099 bash[84576]: audit 2024-09-13T14:45:03.266664+0000 mon.a (mon.0) 1543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:03 smithi099 bash[84576]: audit 2024-09-13T14:45:03.266664+0000 mon.a (mon.0) 1543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:03.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:03 smithi099 bash[87147]: cluster 2024-09-13T14:45:02.423157+0000 mgr.y (mgr.26252) 557 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:03 smithi099 bash[87147]: cluster 2024-09-13T14:45:02.423157+0000 mgr.y (mgr.26252) 557 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:03 smithi099 bash[87147]: audit 2024-09-13T14:45:03.266664+0000 mon.a (mon.0) 1543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:03 smithi099 bash[87147]: audit 2024-09-13T14:45:03.266664+0000 mon.a (mon.0) 1543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:03 smithi177 bash[68594]: cluster 2024-09-13T14:45:02.423157+0000 mgr.y (mgr.26252) 557 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:03 smithi177 bash[68594]: cluster 2024-09-13T14:45:02.423157+0000 mgr.y (mgr.26252) 557 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:03 smithi177 bash[68594]: audit 2024-09-13T14:45:03.266664+0000 mon.a (mon.0) 1543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:03 smithi177 bash[68594]: audit 2024-09-13T14:45:03.266664+0000 mon.a (mon.0) 1543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:04.577 INFO:teuthology.orchestra.run.smithi099.stdout:3032246911043 2024-09-13T14:45:04.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:45:05.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:05 smithi099 bash[84576]: cluster 2024-09-13T14:45:04.424214+0000 mgr.y (mgr.26252) 558 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:05.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:05 smithi099 bash[84576]: cluster 2024-09-13T14:45:04.424214+0000 mgr.y (mgr.26252) 558 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:05.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:05 smithi099 bash[87147]: cluster 2024-09-13T14:45:04.424214+0000 mgr.y (mgr.26252) 558 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:05.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:05 smithi099 bash[87147]: cluster 2024-09-13T14:45:04.424214+0000 mgr.y (mgr.26252) 558 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:05 smithi177 bash[68594]: cluster 2024-09-13T14:45:04.424214+0000 mgr.y (mgr.26252) 558 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:05.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:05 smithi177 bash[68594]: cluster 2024-09-13T14:45:04.424214+0000 mgr.y (mgr.26252) 558 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:07.066 INFO:teuthology.orchestra.run.smithi099.stdout:158913790276 2024-09-13T14:45:07.067 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:45:07.211 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:45:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:45:07] "GET /metrics HTTP/1.1" 200 34269 "" "Prometheus/2.51.0" 2024-09-13T14:45:07.479 INFO:teuthology.orchestra.run.smithi099.stdout:2886218022987 2024-09-13T14:45:07.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:45:07.713 INFO:teuthology.orchestra.run.smithi099.stdout:210453397808 2024-09-13T14:45:07.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:45:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:07 smithi099 bash[84576]: cluster 2024-09-13T14:45:06.424820+0000 mgr.y (mgr.26252) 559 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:07 smithi099 bash[84576]: cluster 2024-09-13T14:45:06.424820+0000 mgr.y (mgr.26252) 559 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:07 smithi099 bash[87147]: cluster 2024-09-13T14:45:06.424820+0000 mgr.y (mgr.26252) 559 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:07 smithi099 bash[87147]: cluster 2024-09-13T14:45:06.424820+0000 mgr.y (mgr.26252) 559 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:07.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:07 smithi177 bash[68594]: cluster 2024-09-13T14:45:06.424820+0000 mgr.y (mgr.26252) 559 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:07 smithi177 bash[68594]: cluster 2024-09-13T14:45:06.424820+0000 mgr.y (mgr.26252) 559 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:09 smithi099 bash[84576]: cluster 2024-09-13T14:45:08.425793+0000 mgr.y (mgr.26252) 560 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:09 smithi099 bash[84576]: cluster 2024-09-13T14:45:08.425793+0000 mgr.y (mgr.26252) 560 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:09.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:09 smithi099 bash[87147]: cluster 2024-09-13T14:45:08.425793+0000 mgr.y (mgr.26252) 560 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:09.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:09 smithi099 bash[87147]: cluster 2024-09-13T14:45:08.425793+0000 mgr.y (mgr.26252) 560 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:09.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:09 smithi177 bash[68594]: cluster 2024-09-13T14:45:08.425793+0000 mgr.y (mgr.26252) 560 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:09.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:09 smithi177 bash[68594]: cluster 2024-09-13T14:45:08.425793+0000 mgr.y (mgr.26252) 560 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:10.051 INFO:teuthology.orchestra.run.smithi099.stdout:3173980831805 2024-09-13T14:45:10.051 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:45:10.195 INFO:teuthology.orchestra.run.smithi099.stdout:2757369004117 2024-09-13T14:45:10.195 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:45:10.426 INFO:teuthology.orchestra.run.smithi099.stdout:2469606195310 2024-09-13T14:45:10.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:45:10.501 INFO:teuthology.orchestra.run.smithi099.stdout:184683594043 2024-09-13T14:45:10.502 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:45:10.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:45:10 smithi177 bash[61439]: ts=2024-09-13T14:45:10.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:11 smithi099 bash[84576]: cluster 2024-09-13T14:45:10.426340+0000 mgr.y (mgr.26252) 561 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:11 smithi099 bash[84576]: cluster 2024-09-13T14:45:10.426340+0000 mgr.y (mgr.26252) 561 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:11.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:11 smithi099 bash[87147]: cluster 2024-09-13T14:45:10.426340+0000 mgr.y (mgr.26252) 561 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:11.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:11 smithi099 bash[87147]: cluster 2024-09-13T14:45:10.426340+0000 mgr.y (mgr.26252) 561 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:11.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:11 smithi177 bash[68594]: cluster 2024-09-13T14:45:10.426340+0000 mgr.y (mgr.26252) 561 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:11.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:11 smithi177 bash[68594]: cluster 2024-09-13T14:45:10.426340+0000 mgr.y (mgr.26252) 561 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:13.304 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:13.304 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:13.304 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:13.304 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:13.361 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:45:12 smithi177 bash[60145]: logger=sqlstore.transactions t=2024-09-13T14:45:12.968536022Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:45:13.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:13 smithi099 bash[84576]: cluster 2024-09-13T14:45:12.427139+0000 mgr.y (mgr.26252) 562 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:13.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:13 smithi099 bash[84576]: cluster 2024-09-13T14:45:12.427139+0000 mgr.y (mgr.26252) 562 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:13.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:13 smithi099 bash[87147]: cluster 2024-09-13T14:45:12.427139+0000 mgr.y (mgr.26252) 562 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:13.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:13 smithi099 bash[87147]: cluster 2024-09-13T14:45:12.427139+0000 mgr.y (mgr.26252) 562 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:13 smithi177 bash[68594]: cluster 2024-09-13T14:45:12.427139+0000 mgr.y (mgr.26252) 562 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:13 smithi177 bash[68594]: cluster 2024-09-13T14:45:12.427139+0000 mgr.y (mgr.26252) 562 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:14.324 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:14.325 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:14.325 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:14.325 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:15 smithi099 bash[84576]: cluster 2024-09-13T14:45:14.428086+0000 mgr.y (mgr.26252) 563 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:15 smithi099 bash[84576]: cluster 2024-09-13T14:45:14.428086+0000 mgr.y (mgr.26252) 563 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:15.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:15 smithi099 bash[87147]: cluster 2024-09-13T14:45:14.428086+0000 mgr.y (mgr.26252) 563 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:15.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:15 smithi099 bash[87147]: cluster 2024-09-13T14:45:14.428086+0000 mgr.y (mgr.26252) 563 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:15 smithi177 bash[68594]: cluster 2024-09-13T14:45:14.428086+0000 mgr.y (mgr.26252) 563 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:15.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:15 smithi177 bash[68594]: cluster 2024-09-13T14:45:14.428086+0000 mgr.y (mgr.26252) 563 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:45:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:45:17] "GET /metrics HTTP/1.1" 200 34269 "" "Prometheus/2.51.0" 2024-09-13T14:45:17.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:17 smithi099 bash[87147]: cluster 2024-09-13T14:45:16.428484+0000 mgr.y (mgr.26252) 564 : cluster [DBG] pgmap v1014: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:17.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:17 smithi099 bash[87147]: cluster 2024-09-13T14:45:16.428484+0000 mgr.y (mgr.26252) 564 : cluster [DBG] pgmap v1014: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:17 smithi099 bash[84576]: cluster 2024-09-13T14:45:16.428484+0000 mgr.y (mgr.26252) 564 : cluster [DBG] pgmap v1014: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:17 smithi099 bash[84576]: cluster 2024-09-13T14:45:16.428484+0000 mgr.y (mgr.26252) 564 : cluster [DBG] pgmap v1014: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:17 smithi177 bash[68594]: cluster 2024-09-13T14:45:16.428484+0000 mgr.y (mgr.26252) 564 : cluster [DBG] pgmap v1014: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:17 smithi177 bash[68594]: cluster 2024-09-13T14:45:16.428484+0000 mgr.y (mgr.26252) 564 : cluster [DBG] pgmap v1014: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:18 smithi099 bash[84576]: audit 2024-09-13T14:45:18.266484+0000 mon.a (mon.0) 1544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:18 smithi099 bash[84576]: audit 2024-09-13T14:45:18.266484+0000 mon.a (mon.0) 1544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:18 smithi099 bash[87147]: audit 2024-09-13T14:45:18.266484+0000 mon.a (mon.0) 1544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:18 smithi099 bash[87147]: audit 2024-09-13T14:45:18.266484+0000 mon.a (mon.0) 1544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:18.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:18 smithi177 bash[68594]: audit 2024-09-13T14:45:18.266484+0000 mon.a (mon.0) 1544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:18.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:18 smithi177 bash[68594]: audit 2024-09-13T14:45:18.266484+0000 mon.a (mon.0) 1544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:19 smithi099 bash[84576]: cluster 2024-09-13T14:45:18.429013+0000 mgr.y (mgr.26252) 565 : cluster [DBG] pgmap v1015: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:19 smithi099 bash[84576]: cluster 2024-09-13T14:45:18.429013+0000 mgr.y (mgr.26252) 565 : cluster [DBG] pgmap v1015: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:19.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:19 smithi099 bash[87147]: cluster 2024-09-13T14:45:18.429013+0000 mgr.y (mgr.26252) 565 : cluster [DBG] pgmap v1015: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:19.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:19 smithi099 bash[87147]: cluster 2024-09-13T14:45:18.429013+0000 mgr.y (mgr.26252) 565 : cluster [DBG] pgmap v1015: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:19.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:19 smithi177 bash[68594]: cluster 2024-09-13T14:45:18.429013+0000 mgr.y (mgr.26252) 565 : cluster [DBG] pgmap v1015: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:19.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:19 smithi177 bash[68594]: cluster 2024-09-13T14:45:18.429013+0000 mgr.y (mgr.26252) 565 : cluster [DBG] pgmap v1015: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:20.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:45:20 smithi177 bash[61439]: ts=2024-09-13T14:45:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:21 smithi099 bash[84576]: cluster 2024-09-13T14:45:20.429839+0000 mgr.y (mgr.26252) 566 : cluster [DBG] pgmap v1016: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:21 smithi099 bash[84576]: cluster 2024-09-13T14:45:20.429839+0000 mgr.y (mgr.26252) 566 : cluster [DBG] pgmap v1016: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:21.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:21 smithi099 bash[87147]: cluster 2024-09-13T14:45:20.429839+0000 mgr.y (mgr.26252) 566 : cluster [DBG] pgmap v1016: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:21.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:21 smithi099 bash[87147]: cluster 2024-09-13T14:45:20.429839+0000 mgr.y (mgr.26252) 566 : cluster [DBG] pgmap v1016: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:21.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:21 smithi177 bash[68594]: cluster 2024-09-13T14:45:20.429839+0000 mgr.y (mgr.26252) 566 : cluster [DBG] pgmap v1016: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:21.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:21 smithi177 bash[68594]: cluster 2024-09-13T14:45:20.429839+0000 mgr.y (mgr.26252) 566 : cluster [DBG] pgmap v1016: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:23 smithi099 bash[84576]: cluster 2024-09-13T14:45:22.430415+0000 mgr.y (mgr.26252) 567 : cluster [DBG] pgmap v1017: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:23 smithi099 bash[84576]: cluster 2024-09-13T14:45:22.430415+0000 mgr.y (mgr.26252) 567 : cluster [DBG] pgmap v1017: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:23.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:23 smithi099 bash[87147]: cluster 2024-09-13T14:45:22.430415+0000 mgr.y (mgr.26252) 567 : cluster [DBG] pgmap v1017: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:23.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:23 smithi099 bash[87147]: cluster 2024-09-13T14:45:22.430415+0000 mgr.y (mgr.26252) 567 : cluster [DBG] pgmap v1017: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:23.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:23 smithi177 bash[68594]: cluster 2024-09-13T14:45:22.430415+0000 mgr.y (mgr.26252) 567 : cluster [DBG] pgmap v1017: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:23.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:23 smithi177 bash[68594]: cluster 2024-09-13T14:45:22.430415+0000 mgr.y (mgr.26252) 567 : cluster [DBG] pgmap v1017: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:25 smithi099 bash[84576]: cluster 2024-09-13T14:45:24.431254+0000 mgr.y (mgr.26252) 568 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:25.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:25 smithi099 bash[84576]: cluster 2024-09-13T14:45:24.431254+0000 mgr.y (mgr.26252) 568 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:25.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:25 smithi099 bash[87147]: cluster 2024-09-13T14:45:24.431254+0000 mgr.y (mgr.26252) 568 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:25.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:25 smithi099 bash[87147]: cluster 2024-09-13T14:45:24.431254+0000 mgr.y (mgr.26252) 568 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:25.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:25 smithi177 bash[68594]: cluster 2024-09-13T14:45:24.431254+0000 mgr.y (mgr.26252) 568 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:25 smithi177 bash[68594]: cluster 2024-09-13T14:45:24.431254+0000 mgr.y (mgr.26252) 568 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:27.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:45:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:45:27] "GET /metrics HTTP/1.1" 200 34269 "" "Prometheus/2.51.0" 2024-09-13T14:45:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:27 smithi099 bash[84576]: cluster 2024-09-13T14:45:26.431980+0000 mgr.y (mgr.26252) 569 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:27 smithi099 bash[84576]: cluster 2024-09-13T14:45:26.431980+0000 mgr.y (mgr.26252) 569 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:27.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:27 smithi099 bash[87147]: cluster 2024-09-13T14:45:26.431980+0000 mgr.y (mgr.26252) 569 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:27.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:27 smithi099 bash[87147]: cluster 2024-09-13T14:45:26.431980+0000 mgr.y (mgr.26252) 569 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:27 smithi177 bash[68594]: cluster 2024-09-13T14:45:26.431980+0000 mgr.y (mgr.26252) 569 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:27.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:27 smithi177 bash[68594]: cluster 2024-09-13T14:45:26.431980+0000 mgr.y (mgr.26252) 569 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:29.365 INFO:teuthology.orchestra.run.smithi099.stdout:2886218022992 2024-09-13T14:45:29.785 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:29 smithi099 bash[84576]: cluster 2024-09-13T14:45:28.432922+0000 mgr.y (mgr.26252) 570 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:29.785 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:29 smithi099 bash[84576]: cluster 2024-09-13T14:45:28.432922+0000 mgr.y (mgr.26252) 570 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:29.785 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:29 smithi099 bash[84576]: audit 2024-09-13T14:45:29.368888+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.99:0/71487147' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:45:29.785 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:29 smithi099 bash[84576]: audit 2024-09-13T14:45:29.368888+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.99:0/71487147' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:45:29.785 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:29 smithi099 bash[87147]: cluster 2024-09-13T14:45:28.432922+0000 mgr.y (mgr.26252) 570 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:29.785 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:29 smithi099 bash[87147]: cluster 2024-09-13T14:45:28.432922+0000 mgr.y (mgr.26252) 570 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:29.786 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:29 smithi099 bash[87147]: audit 2024-09-13T14:45:29.368888+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.99:0/71487147' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:45:29.786 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:29 smithi099 bash[87147]: audit 2024-09-13T14:45:29.368888+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.99:0/71487147' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:45:29.853 INFO:teuthology.orchestra.run.smithi099.stdout:210453397812 2024-09-13T14:45:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:29 smithi177 bash[68594]: cluster 2024-09-13T14:45:28.432922+0000 mgr.y (mgr.26252) 570 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:29.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:29 smithi177 bash[68594]: cluster 2024-09-13T14:45:28.432922+0000 mgr.y (mgr.26252) 570 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:29.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:29 smithi177 bash[68594]: audit 2024-09-13T14:45:29.368888+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.99:0/71487147' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:45:29.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:29 smithi177 bash[68594]: audit 2024-09-13T14:45:29.368888+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.99:0/71487147' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:45:30.075 INFO:teuthology.orchestra.run.smithi099.stdout:3032246911049 2024-09-13T14:45:30.421 INFO:teuthology.orchestra.run.smithi099.stdout:2757369004121 2024-09-13T14:45:30.563 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:45:30 smithi177 bash[61439]: ts=2024-09-13T14:45:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[84576]: audit 2024-09-13T14:45:29.856763+0000 mon.a (mon.0) 1545 : audit [DBG] from='client.? 172.21.15.99:0/4273283438' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:45:30.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[84576]: audit 2024-09-13T14:45:29.856763+0000 mon.a (mon.0) 1545 : audit [DBG] from='client.? 172.21.15.99:0/4273283438' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:45:30.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[84576]: audit 2024-09-13T14:45:30.078508+0000 mon.a (mon.0) 1546 : audit [DBG] from='client.? 172.21.15.99:0/901054990' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:45:30.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[84576]: audit 2024-09-13T14:45:30.078508+0000 mon.a (mon.0) 1546 : audit [DBG] from='client.? 172.21.15.99:0/901054990' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:45:30.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[84576]: audit 2024-09-13T14:45:30.424079+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.99:0/4162621339' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:45:30.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[84576]: audit 2024-09-13T14:45:30.424079+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.99:0/4162621339' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:45:30.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[87147]: audit 2024-09-13T14:45:29.856763+0000 mon.a (mon.0) 1545 : audit [DBG] from='client.? 172.21.15.99:0/4273283438' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:45:30.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[87147]: audit 2024-09-13T14:45:29.856763+0000 mon.a (mon.0) 1545 : audit [DBG] from='client.? 172.21.15.99:0/4273283438' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:45:30.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[87147]: audit 2024-09-13T14:45:30.078508+0000 mon.a (mon.0) 1546 : audit [DBG] from='client.? 172.21.15.99:0/901054990' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:45:30.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[87147]: audit 2024-09-13T14:45:30.078508+0000 mon.a (mon.0) 1546 : audit [DBG] from='client.? 172.21.15.99:0/901054990' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:45:30.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[87147]: audit 2024-09-13T14:45:30.424079+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.99:0/4162621339' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:45:30.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:30 smithi099 bash[87147]: audit 2024-09-13T14:45:30.424079+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.99:0/4162621339' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:45:30.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:30 smithi177 bash[68594]: audit 2024-09-13T14:45:29.856763+0000 mon.a (mon.0) 1545 : audit [DBG] from='client.? 172.21.15.99:0/4273283438' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:45:30.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:30 smithi177 bash[68594]: audit 2024-09-13T14:45:29.856763+0000 mon.a (mon.0) 1545 : audit [DBG] from='client.? 172.21.15.99:0/4273283438' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:45:30.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:30 smithi177 bash[68594]: audit 2024-09-13T14:45:30.078508+0000 mon.a (mon.0) 1546 : audit [DBG] from='client.? 172.21.15.99:0/901054990' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:45:30.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:30 smithi177 bash[68594]: audit 2024-09-13T14:45:30.078508+0000 mon.a (mon.0) 1546 : audit [DBG] from='client.? 172.21.15.99:0/901054990' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:45:30.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:30 smithi177 bash[68594]: audit 2024-09-13T14:45:30.424079+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.99:0/4162621339' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:45:30.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:30 smithi177 bash[68594]: audit 2024-09-13T14:45:30.424079+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.99:0/4162621339' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:45:31.719 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2886218022987 got 2886218022992 for osd.2 2024-09-13T14:45:31.719 DEBUG:teuthology.parallel:result is None 2024-09-13T14:45:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:31 smithi099 bash[84576]: cluster 2024-09-13T14:45:30.433495+0000 mgr.y (mgr.26252) 571 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:31 smithi099 bash[84576]: cluster 2024-09-13T14:45:30.433495+0000 mgr.y (mgr.26252) 571 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:31.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:31 smithi099 bash[87147]: cluster 2024-09-13T14:45:30.433495+0000 mgr.y (mgr.26252) 571 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:31.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:31 smithi099 bash[87147]: cluster 2024-09-13T14:45:30.433495+0000 mgr.y (mgr.26252) 571 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:31 smithi177 bash[68594]: cluster 2024-09-13T14:45:30.433495+0000 mgr.y (mgr.26252) 571 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:31.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:31 smithi177 bash[68594]: cluster 2024-09-13T14:45:30.433495+0000 mgr.y (mgr.26252) 571 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:32.131 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397808 got 210453397812 for osd.7 2024-09-13T14:45:32.131 DEBUG:teuthology.parallel:result is None 2024-09-13T14:45:32.582 INFO:teuthology.orchestra.run.smithi099.stdout:158913790282 2024-09-13T14:45:32.601 INFO:teuthology.orchestra.run.smithi099.stdout:3173980831811 2024-09-13T14:45:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[84576]: cluster 2024-09-13T14:45:32.433992+0000 mgr.y (mgr.26252) 572 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[84576]: cluster 2024-09-13T14:45:32.433992+0000 mgr.y (mgr.26252) 572 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[84576]: audit 2024-09-13T14:45:32.585793+0000 mon.a (mon.0) 1547 : audit [DBG] from='client.? 172.21.15.99:0/3912886121' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:45:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[84576]: audit 2024-09-13T14:45:32.585793+0000 mon.a (mon.0) 1547 : audit [DBG] from='client.? 172.21.15.99:0/3912886121' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:45:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[84576]: audit 2024-09-13T14:45:32.604956+0000 mon.a (mon.0) 1548 : audit [DBG] from='client.? 172.21.15.99:0/3053586688' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:45:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[84576]: audit 2024-09-13T14:45:32.604956+0000 mon.a (mon.0) 1548 : audit [DBG] from='client.? 172.21.15.99:0/3053586688' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:45:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[84576]: audit 2024-09-13T14:45:33.267370+0000 mon.a (mon.0) 1549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[84576]: audit 2024-09-13T14:45:33.267370+0000 mon.a (mon.0) 1549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:33.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[87147]: cluster 2024-09-13T14:45:32.433992+0000 mgr.y (mgr.26252) 572 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[87147]: cluster 2024-09-13T14:45:32.433992+0000 mgr.y (mgr.26252) 572 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[87147]: audit 2024-09-13T14:45:32.585793+0000 mon.a (mon.0) 1547 : audit [DBG] from='client.? 172.21.15.99:0/3912886121' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:45:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[87147]: audit 2024-09-13T14:45:32.585793+0000 mon.a (mon.0) 1547 : audit [DBG] from='client.? 172.21.15.99:0/3912886121' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:45:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[87147]: audit 2024-09-13T14:45:32.604956+0000 mon.a (mon.0) 1548 : audit [DBG] from='client.? 172.21.15.99:0/3053586688' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:45:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[87147]: audit 2024-09-13T14:45:32.604956+0000 mon.a (mon.0) 1548 : audit [DBG] from='client.? 172.21.15.99:0/3053586688' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:45:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[87147]: audit 2024-09-13T14:45:33.267370+0000 mon.a (mon.0) 1549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:33 smithi099 bash[87147]: audit 2024-09-13T14:45:33.267370+0000 mon.a (mon.0) 1549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:33 smithi177 bash[68594]: cluster 2024-09-13T14:45:32.433992+0000 mgr.y (mgr.26252) 572 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:33 smithi177 bash[68594]: cluster 2024-09-13T14:45:32.433992+0000 mgr.y (mgr.26252) 572 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:33 smithi177 bash[68594]: audit 2024-09-13T14:45:32.585793+0000 mon.a (mon.0) 1547 : audit [DBG] from='client.? 172.21.15.99:0/3912886121' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:45:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:33 smithi177 bash[68594]: audit 2024-09-13T14:45:32.585793+0000 mon.a (mon.0) 1547 : audit [DBG] from='client.? 172.21.15.99:0/3912886121' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:45:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:33 smithi177 bash[68594]: audit 2024-09-13T14:45:32.604956+0000 mon.a (mon.0) 1548 : audit [DBG] from='client.? 172.21.15.99:0/3053586688' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:45:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:33 smithi177 bash[68594]: audit 2024-09-13T14:45:32.604956+0000 mon.a (mon.0) 1548 : audit [DBG] from='client.? 172.21.15.99:0/3053586688' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:45:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:33 smithi177 bash[68594]: audit 2024-09-13T14:45:33.267370+0000 mon.a (mon.0) 1549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:33 smithi177 bash[68594]: audit 2024-09-13T14:45:33.267370+0000 mon.a (mon.0) 1549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:34.699 INFO:teuthology.orchestra.run.smithi099.stdout:184683594049 2024-09-13T14:45:35.014 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2757369004117 got 2757369004121 for osd.1 2024-09-13T14:45:35.014 DEBUG:teuthology.parallel:result is None 2024-09-13T14:45:35.017 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3032246911043 got 3032246911049 for osd.3 2024-09-13T14:45:35.017 DEBUG:teuthology.parallel:result is None 2024-09-13T14:45:35.066 INFO:teuthology.orchestra.run.smithi099.stdout:2469606195315 2024-09-13T14:45:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[84576]: cluster 2024-09-13T14:45:34.434567+0000 mgr.y (mgr.26252) 573 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[84576]: cluster 2024-09-13T14:45:34.434567+0000 mgr.y (mgr.26252) 573 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[84576]: audit 2024-09-13T14:45:34.702762+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.99:0/1964376235' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:45:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[84576]: audit 2024-09-13T14:45:34.702762+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.99:0/1964376235' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:45:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[84576]: audit 2024-09-13T14:45:35.069347+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.99:0/2152179221' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:45:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[84576]: audit 2024-09-13T14:45:35.069347+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.99:0/2152179221' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:45:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[87147]: cluster 2024-09-13T14:45:34.434567+0000 mgr.y (mgr.26252) 573 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[87147]: cluster 2024-09-13T14:45:34.434567+0000 mgr.y (mgr.26252) 573 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[87147]: audit 2024-09-13T14:45:34.702762+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.99:0/1964376235' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:45:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[87147]: audit 2024-09-13T14:45:34.702762+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.99:0/1964376235' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:45:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[87147]: audit 2024-09-13T14:45:35.069347+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.99:0/2152179221' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:45:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:35 smithi099 bash[87147]: audit 2024-09-13T14:45:35.069347+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.99:0/2152179221' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:45:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:35 smithi177 bash[68594]: cluster 2024-09-13T14:45:34.434567+0000 mgr.y (mgr.26252) 573 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:35 smithi177 bash[68594]: cluster 2024-09-13T14:45:34.434567+0000 mgr.y (mgr.26252) 573 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:35 smithi177 bash[68594]: audit 2024-09-13T14:45:34.702762+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.99:0/1964376235' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:45:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:35 smithi177 bash[68594]: audit 2024-09-13T14:45:34.702762+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.99:0/1964376235' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:45:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:35 smithi177 bash[68594]: audit 2024-09-13T14:45:35.069347+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.99:0/2152179221' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:45:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:35 smithi177 bash[68594]: audit 2024-09-13T14:45:35.069347+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.99:0/2152179221' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:45:36.678 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3173980831805 got 3173980831811 for osd.4 2024-09-13T14:45:36.678 DEBUG:teuthology.parallel:result is None 2024-09-13T14:45:37.021 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790276 got 158913790282 for osd.5 2024-09-13T14:45:37.022 DEBUG:teuthology.parallel:result is None 2024-09-13T14:45:37.280 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:45:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:45:37] "GET /metrics HTTP/1.1" 200 34268 "" "Prometheus/2.51.0" 2024-09-13T14:45:37.398 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683594043 got 184683594049 for osd.6 2024-09-13T14:45:37.399 DEBUG:teuthology.parallel:result is None 2024-09-13T14:45:37.500 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2469606195310 got 2469606195315 for osd.0 2024-09-13T14:45:37.501 DEBUG:teuthology.parallel:result is None 2024-09-13T14:45:37.501 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:45:37.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:45:37.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:37 smithi099 bash[84576]: cluster 2024-09-13T14:45:36.435118+0000 mgr.y (mgr.26252) 574 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:37.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:37 smithi099 bash[84576]: cluster 2024-09-13T14:45:36.435118+0000 mgr.y (mgr.26252) 574 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:37.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:37 smithi099 bash[87147]: cluster 2024-09-13T14:45:36.435118+0000 mgr.y (mgr.26252) 574 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:37.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:37 smithi099 bash[87147]: cluster 2024-09-13T14:45:36.435118+0000 mgr.y (mgr.26252) 574 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:37.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:37 smithi177 bash[68594]: cluster 2024-09-13T14:45:36.435118+0000 mgr.y (mgr.26252) 574 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:37.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:37 smithi177 bash[68594]: cluster 2024-09-13T14:45:36.435118+0000 mgr.y (mgr.26252) 574 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:39 smithi099 bash[84576]: cluster 2024-09-13T14:45:38.436064+0000 mgr.y (mgr.26252) 575 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:39 smithi099 bash[84576]: cluster 2024-09-13T14:45:38.436064+0000 mgr.y (mgr.26252) 575 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:39 smithi099 bash[87147]: cluster 2024-09-13T14:45:38.436064+0000 mgr.y (mgr.26252) 575 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:39 smithi099 bash[87147]: cluster 2024-09-13T14:45:38.436064+0000 mgr.y (mgr.26252) 575 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:39 smithi177 bash[68594]: cluster 2024-09-13T14:45:38.436064+0000 mgr.y (mgr.26252) 575 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:39 smithi177 bash[68594]: cluster 2024-09-13T14:45:38.436064+0000 mgr.y (mgr.26252) 575 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:40.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:45:40 smithi177 bash[61439]: ts=2024-09-13T14:45:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:41 smithi099 bash[84576]: cluster 2024-09-13T14:45:40.437012+0000 mgr.y (mgr.26252) 576 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:41.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:41 smithi099 bash[84576]: cluster 2024-09-13T14:45:40.437012+0000 mgr.y (mgr.26252) 576 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:41.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:41 smithi099 bash[87147]: cluster 2024-09-13T14:45:40.437012+0000 mgr.y (mgr.26252) 576 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:41.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:41 smithi099 bash[87147]: cluster 2024-09-13T14:45:40.437012+0000 mgr.y (mgr.26252) 576 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:41 smithi177 bash[68594]: cluster 2024-09-13T14:45:40.437012+0000 mgr.y (mgr.26252) 576 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:41 smithi177 bash[68594]: cluster 2024-09-13T14:45:40.437012+0000 mgr.y (mgr.26252) 576 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:42.286 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:43.767 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:43 smithi099 bash[84576]: cluster 2024-09-13T14:45:42.438034+0000 mgr.y (mgr.26252) 577 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:43.767 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:43 smithi099 bash[84576]: cluster 2024-09-13T14:45:42.438034+0000 mgr.y (mgr.26252) 577 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:43.768 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:43 smithi099 bash[87147]: cluster 2024-09-13T14:45:42.438034+0000 mgr.y (mgr.26252) 577 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:43.768 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:43 smithi099 bash[87147]: cluster 2024-09-13T14:45:42.438034+0000 mgr.y (mgr.26252) 577 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:43 smithi177 bash[68594]: cluster 2024-09-13T14:45:42.438034+0000 mgr.y (mgr.26252) 577 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:43 smithi177 bash[68594]: cluster 2024-09-13T14:45:42.438034+0000 mgr.y (mgr.26252) 577 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:45 smithi099 bash[84576]: cluster 2024-09-13T14:45:44.439112+0000 mgr.y (mgr.26252) 578 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:45.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:45 smithi099 bash[84576]: cluster 2024-09-13T14:45:44.439112+0000 mgr.y (mgr.26252) 578 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:45 smithi099 bash[87147]: cluster 2024-09-13T14:45:44.439112+0000 mgr.y (mgr.26252) 578 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:45.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:45 smithi099 bash[87147]: cluster 2024-09-13T14:45:44.439112+0000 mgr.y (mgr.26252) 578 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:45 smithi177 bash[68594]: cluster 2024-09-13T14:45:44.439112+0000 mgr.y (mgr.26252) 578 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:45 smithi177 bash[68594]: cluster 2024-09-13T14:45:44.439112+0000 mgr.y (mgr.26252) 578 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:46.358 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:45:46.359 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:45:47.279 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1028,"stamp":"2024-09-13T14:45:44.438413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":344,"num_read_kb":799,"num_write":212,"num_write_kb":1659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":164,"ondisk_log_size":164,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1480396,"kb_used_data":123044,"kb_used_omap":353,"kb_used_meta":1169694,"kb_avail":748480820,"statfs":{"total":767960285184,"available":766444359680,"internally_reserved":0,"allocated":125997056,"data_stored":79657613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":361898,"internal_metadata":1197767254},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004719"},"pg_stats":[{"pgid":"3.1f","version":"65'5","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313264+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T14:44:10.313264+0000","last_peered":"2024-09-13T14:44:10.313264+0000","last_clean":"2024-09-13T14:44:10.313264+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T14:44:10.313264+0000","last_undegraded":"2024-09-13T14:44:10.313264+0000","last_fullsized":"2024-09-13T14:44:10.313264+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1082,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.301691+0000","last_change":"2024-09-13T14:40:09.139213+0000","last_active":"2024-09-13T14:44:10.301691+0000","last_peered":"2024-09-13T14:44:10.301691+0000","last_clean":"2024-09-13T14:44:10.301691+0000","last_became_active":"2024-09-13T14:40:09.138480+0000","last_became_peered":"2024-09-13T14:40:09.138480+0000","last_unstale":"2024-09-13T14:44:10.301691+0000","last_undegraded":"2024-09-13T14:44:10.301691+0000","last_fullsized":"2024-09-13T14:44:10.301691+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:02:27.534835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1139,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724219+0000","last_change":"2024-09-13T14:39:34.403170+0000","last_active":"2024-09-13T14:44:10.724219+0000","last_peered":"2024-09-13T14:44:10.724219+0000","last_clean":"2024-09-13T14:44:10.724219+0000","last_became_active":"2024-09-13T14:39:34.389001+0000","last_became_peered":"2024-09-13T14:39:34.389001+0000","last_unstale":"2024-09-13T14:44:10.724219+0000","last_undegraded":"2024-09-13T14:44:10.724219+0000","last_fullsized":"2024-09-13T14:44:10.724219+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:38.542452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1152,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.722796+0000","last_change":"2024-09-13T14:39:34.316950+0000","last_active":"2024-09-13T14:44:10.722796+0000","last_peered":"2024-09-13T14:44:10.722796+0000","last_clean":"2024-09-13T14:44:10.722796+0000","last_became_active":"2024-09-13T14:39:34.316614+0000","last_became_peered":"2024-09-13T14:39:34.316614+0000","last_unstale":"2024-09-13T14:44:10.722796+0000","last_undegraded":"2024-09-13T14:44:10.722796+0000","last_fullsized":"2024-09-13T14:44:10.722796+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:20.664989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301239+0000","last_change":"2024-09-13T14:40:09.190077+0000","last_active":"2024-09-13T14:44:09.301239+0000","last_peered":"2024-09-13T14:44:09.301239+0000","last_clean":"2024-09-13T14:44:09.301239+0000","last_became_active":"2024-09-13T14:40:09.189717+0000","last_became_peered":"2024-09-13T14:40:09.189717+0000","last_unstale":"2024-09-13T14:44:09.301239+0000","last_undegraded":"2024-09-13T14:44:09.301239+0000","last_fullsized":"2024-09-13T14:44:09.301239+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:24.201026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303665+0000","last_change":"2024-09-13T14:40:09.219485+0000","last_active":"2024-09-13T14:44:10.303665+0000","last_peered":"2024-09-13T14:44:10.303665+0000","last_clean":"2024-09-13T14:44:10.303665+0000","last_became_active":"2024-09-13T14:40:09.212705+0000","last_became_peered":"2024-09-13T14:40:09.212705+0000","last_unstale":"2024-09-13T14:44:10.303665+0000","last_undegraded":"2024-09-13T14:44:10.303665+0000","last_fullsized":"2024-09-13T14:44:10.303665+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:02.378795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":925,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.302449+0000","last_change":"2024-09-13T14:40:09.192785+0000","last_active":"2024-09-13T14:44:09.302449+0000","last_peered":"2024-09-13T14:44:09.302449+0000","last_clean":"2024-09-13T14:44:09.302449+0000","last_became_active":"2024-09-13T14:40:09.192196+0000","last_became_peered":"2024-09-13T14:40:09.192196+0000","last_unstale":"2024-09-13T14:44:09.302449+0000","last_undegraded":"2024-09-13T14:44:09.302449+0000","last_fullsized":"2024-09-13T14:44:09.302449+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:52.737394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.305451+0000","last_change":"2024-09-13T14:40:09.220295+0000","last_active":"2024-09-13T14:44:10.305451+0000","last_peered":"2024-09-13T14:44:10.305451+0000","last_clean":"2024-09-13T14:44:10.305451+0000","last_became_active":"2024-09-13T14:40:09.209986+0000","last_became_peered":"2024-09-13T14:40:09.209986+0000","last_unstale":"2024-09-13T14:44:10.305451+0000","last_undegraded":"2024-09-13T14:44:10.305451+0000","last_fullsized":"2024-09-13T14:44:10.305451+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:57.153128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1282,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.311627+0000","last_change":"2024-09-13T14:40:09.194263+0000","last_active":"2024-09-13T14:44:10.311627+0000","last_peered":"2024-09-13T14:44:10.311627+0000","last_clean":"2024-09-13T14:44:10.311627+0000","last_became_active":"2024-09-13T14:40:09.192631+0000","last_became_peered":"2024-09-13T14:40:09.192631+0000","last_unstale":"2024-09-13T14:44:10.311627+0000","last_undegraded":"2024-09-13T14:44:10.311627+0000","last_fullsized":"2024-09-13T14:44:10.311627+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:41:31.185137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1186,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.740194+0000","last_change":"2024-09-13T14:39:34.358767+0000","last_active":"2024-09-13T14:44:10.740194+0000","last_peered":"2024-09-13T14:44:10.740194+0000","last_clean":"2024-09-13T14:44:10.740194+0000","last_became_active":"2024-09-13T14:39:34.358244+0000","last_became_peered":"2024-09-13T14:39:34.358244+0000","last_unstale":"2024-09-13T14:44:10.740194+0000","last_undegraded":"2024-09-13T14:44:10.740194+0000","last_fullsized":"2024-09-13T14:44:10.740194+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:39.392859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":1132,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303234+0000","last_change":"2024-09-13T14:40:09.189390+0000","last_active":"2024-09-13T14:44:10.303234+0000","last_peered":"2024-09-13T14:44:10.303234+0000","last_clean":"2024-09-13T14:44:10.303234+0000","last_became_active":"2024-09-13T14:40:09.189042+0000","last_became_peered":"2024-09-13T14:40:09.189042+0000","last_unstale":"2024-09-13T14:44:10.303234+0000","last_undegraded":"2024-09-13T14:44:10.303234+0000","last_fullsized":"2024-09-13T14:44:10.303234+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:05.081276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1158,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724166+0000","last_change":"2024-09-13T14:40:09.227622+0000","last_active":"2024-09-13T14:44:10.724166+0000","last_peered":"2024-09-13T14:44:10.724166+0000","last_clean":"2024-09-13T14:44:10.724166+0000","last_became_active":"2024-09-13T14:40:09.226971+0000","last_became_peered":"2024-09-13T14:40:09.226971+0000","last_unstale":"2024-09-13T14:44:10.724166+0000","last_undegraded":"2024-09-13T14:44:10.724166+0000","last_fullsized":"2024-09-13T14:44:10.724166+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:48:21.955396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.730771+0000","last_change":"2024-09-13T14:40:09.173906+0000","last_active":"2024-09-13T14:44:10.730771+0000","last_peered":"2024-09-13T14:44:10.730771+0000","last_clean":"2024-09-13T14:44:10.730771+0000","last_became_active":"2024-09-13T14:40:09.173303+0000","last_became_peered":"2024-09-13T14:40:09.173303+0000","last_unstale":"2024-09-13T14:44:10.730771+0000","last_undegraded":"2024-09-13T14:44:10.730771+0000","last_fullsized":"2024-09-13T14:44:10.730771+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:34:11.061883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314602+0000","last_change":"2024-09-13T14:39:34.362206+0000","last_active":"2024-09-13T14:44:10.314602+0000","last_peered":"2024-09-13T14:44:10.314602+0000","last_clean":"2024-09-13T14:44:10.314602+0000","last_became_active":"2024-09-13T14:39:34.361932+0000","last_became_peered":"2024-09-13T14:39:34.361932+0000","last_unstale":"2024-09-13T14:44:10.314602+0000","last_undegraded":"2024-09-13T14:44:10.314602+0000","last_fullsized":"2024-09-13T14:44:10.314602+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:00.449274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1165,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723274+0000","last_change":"2024-09-13T14:39:34.403479+0000","last_active":"2024-09-13T14:44:10.723274+0000","last_peered":"2024-09-13T14:44:10.723274+0000","last_clean":"2024-09-13T14:44:10.723274+0000","last_became_active":"2024-09-13T14:39:34.386530+0000","last_became_peered":"2024-09-13T14:39:34.386530+0000","last_unstale":"2024-09-13T14:44:10.723274+0000","last_undegraded":"2024-09-13T14:44:10.723274+0000","last_fullsized":"2024-09-13T14:44:10.723274+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:29.559892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487821+0000","last_change":"2024-09-13T14:38:11.206109+0000","last_active":"2024-09-13T14:44:09.487821+0000","last_peered":"2024-09-13T14:44:09.487821+0000","last_clean":"2024-09-13T14:44:09.487821+0000","last_became_active":"2024-09-13T14:38:11.205546+0000","last_became_peered":"2024-09-13T14:38:11.205546+0000","last_unstale":"2024-09-13T14:44:09.487821+0000","last_undegraded":"2024-09-13T14:44:09.487821+0000","last_fullsized":"2024-09-13T14:44:09.487821+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:33.834490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1145,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.722999+0000","last_change":"2024-09-13T14:39:34.368100+0000","last_active":"2024-09-13T14:44:10.722999+0000","last_peered":"2024-09-13T14:44:10.722999+0000","last_clean":"2024-09-13T14:44:10.722999+0000","last_became_active":"2024-09-13T14:39:34.366992+0000","last_became_peered":"2024-09-13T14:39:34.366992+0000","last_unstale":"2024-09-13T14:44:10.722999+0000","last_undegraded":"2024-09-13T14:44:10.722999+0000","last_fullsized":"2024-09-13T14:44:10.722999+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:54.220100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1276,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313166+0000","last_change":"2024-09-13T14:39:34.318266+0000","last_active":"2024-09-13T14:44:10.313166+0000","last_peered":"2024-09-13T14:44:10.313166+0000","last_clean":"2024-09-13T14:44:10.313166+0000","last_became_active":"2024-09-13T14:39:34.315973+0000","last_became_peered":"2024-09-13T14:39:34.315973+0000","last_unstale":"2024-09-13T14:44:10.313166+0000","last_undegraded":"2024-09-13T14:44:10.313166+0000","last_fullsized":"2024-09-13T14:44:10.313166+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:15.270282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1124,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723081+0000","last_change":"2024-09-13T14:39:34.338378+0000","last_active":"2024-09-13T14:44:10.723081+0000","last_peered":"2024-09-13T14:44:10.723081+0000","last_clean":"2024-09-13T14:44:10.723081+0000","last_became_active":"2024-09-13T14:39:34.338005+0000","last_became_peered":"2024-09-13T14:39:34.338005+0000","last_unstale":"2024-09-13T14:44:10.723081+0000","last_undegraded":"2024-09-13T14:44:10.723081+0000","last_fullsized":"2024-09-13T14:44:10.723081+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:29.120379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":905,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301800+0000","last_change":"2024-09-13T14:38:11.231165+0000","last_active":"2024-09-13T14:44:09.301800+0000","last_peered":"2024-09-13T14:44:09.301800+0000","last_clean":"2024-09-13T14:44:09.301800+0000","last_became_active":"2024-09-13T14:38:11.230759+0000","last_became_peered":"2024-09-13T14:38:11.230759+0000","last_unstale":"2024-09-13T14:44:09.301800+0000","last_undegraded":"2024-09-13T14:44:09.301800+0000","last_fullsized":"2024-09-13T14:44:09.301800+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:34.222184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":930,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345752+0000","last_change":"2024-09-13T14:40:09.191682+0000","last_active":"2024-09-13T14:44:09.345752+0000","last_peered":"2024-09-13T14:44:09.345752+0000","last_clean":"2024-09-13T14:44:09.345752+0000","last_became_active":"2024-09-13T14:40:09.184791+0000","last_became_peered":"2024-09-13T14:40:09.184791+0000","last_unstale":"2024-09-13T14:44:09.345752+0000","last_undegraded":"2024-09-13T14:44:09.345752+0000","last_fullsized":"2024-09-13T14:44:09.345752+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:56.463093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.733304+0000","last_change":"2024-09-13T14:38:59.370056+0000","last_active":"2024-09-13T14:44:10.733304+0000","last_peered":"2024-09-13T14:44:10.733304+0000","last_clean":"2024-09-13T14:44:10.733304+0000","last_became_active":"2024-09-13T14:38:59.369705+0000","last_became_peered":"2024-09-13T14:38:59.369705+0000","last_unstale":"2024-09-13T14:44:10.733304+0000","last_undegraded":"2024-09-13T14:44:10.733304+0000","last_fullsized":"2024-09-13T14:44:10.733304+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:59.898050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":1125,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303667+0000","last_change":"2024-09-13T14:40:09.180133+0000","last_active":"2024-09-13T14:44:10.303667+0000","last_peered":"2024-09-13T14:44:10.303667+0000","last_clean":"2024-09-13T14:44:10.303667+0000","last_became_active":"2024-09-13T14:40:09.179808+0000","last_became_peered":"2024-09-13T14:40:09.179808+0000","last_unstale":"2024-09-13T14:44:10.303667+0000","last_undegraded":"2024-09-13T14:44:10.303667+0000","last_fullsized":"2024-09-13T14:44:10.303667+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:11.823441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1122,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.306210+0000","last_change":"2024-09-13T14:40:09.190329+0000","last_active":"2024-09-13T14:44:10.306210+0000","last_peered":"2024-09-13T14:44:10.306210+0000","last_clean":"2024-09-13T14:44:10.306210+0000","last_became_active":"2024-09-13T14:40:09.189399+0000","last_became_peered":"2024-09-13T14:40:09.189399+0000","last_unstale":"2024-09-13T14:44:10.306210+0000","last_undegraded":"2024-09-13T14:44:10.306210+0000","last_fullsized":"2024-09-13T14:44:10.306210+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:36.338067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1099,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.304546+0000","last_change":"2024-09-13T14:40:09.202450+0000","last_active":"2024-09-13T14:44:10.304546+0000","last_peered":"2024-09-13T14:44:10.304546+0000","last_clean":"2024-09-13T14:44:10.304546+0000","last_became_active":"2024-09-13T14:40:09.201651+0000","last_became_peered":"2024-09-13T14:40:09.201651+0000","last_unstale":"2024-09-13T14:44:10.304546+0000","last_undegraded":"2024-09-13T14:44:10.304546+0000","last_fullsized":"2024-09-13T14:44:10.304546+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:12.544442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":921,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487923+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T14:44:09.487923+0000","last_peered":"2024-09-13T14:44:09.487923+0000","last_clean":"2024-09-13T14:44:09.487923+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T14:44:09.487923+0000","last_undegraded":"2024-09-13T14:44:09.487923+0000","last_fullsized":"2024-09-13T14:44:09.487923+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1293,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.315290+0000","last_change":"2024-09-13T14:40:09.187749+0000","last_active":"2024-09-13T14:44:10.315290+0000","last_peered":"2024-09-13T14:44:10.315290+0000","last_clean":"2024-09-13T14:44:10.315290+0000","last_became_active":"2024-09-13T14:40:09.187007+0000","last_became_peered":"2024-09-13T14:40:09.187007+0000","last_unstale":"2024-09-13T14:44:10.315290+0000","last_undegraded":"2024-09-13T14:44:10.315290+0000","last_fullsized":"2024-09-13T14:44:10.315290+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:08.019463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1122,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723614+0000","last_change":"2024-09-13T14:39:34.349945+0000","last_active":"2024-09-13T14:44:10.723614+0000","last_peered":"2024-09-13T14:44:10.723614+0000","last_clean":"2024-09-13T14:44:10.723614+0000","last_became_active":"2024-09-13T14:39:34.346730+0000","last_became_peered":"2024-09-13T14:39:34.346730+0000","last_unstale":"2024-09-13T14:44:10.723614+0000","last_undegraded":"2024-09-13T14:44:10.723614+0000","last_fullsized":"2024-09-13T14:44:10.723614+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:48.854603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1234,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729597+0000","last_change":"2024-09-13T14:40:09.177544+0000","last_active":"2024-09-13T14:44:10.729597+0000","last_peered":"2024-09-13T14:44:10.729597+0000","last_clean":"2024-09-13T14:44:10.729597+0000","last_became_active":"2024-09-13T14:40:09.177252+0000","last_became_peered":"2024-09-13T14:40:09.177252+0000","last_unstale":"2024-09-13T14:44:10.729597+0000","last_undegraded":"2024-09-13T14:44:10.729597+0000","last_fullsized":"2024-09-13T14:44:10.729597+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:45:50.539379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1125,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724802+0000","last_change":"2024-09-13T14:39:34.401432+0000","last_active":"2024-09-13T14:44:10.724802+0000","last_peered":"2024-09-13T14:44:10.724802+0000","last_clean":"2024-09-13T14:44:10.724802+0000","last_became_active":"2024-09-13T14:39:34.395458+0000","last_became_peered":"2024-09-13T14:39:34.395458+0000","last_unstale":"2024-09-13T14:44:10.724802+0000","last_undegraded":"2024-09-13T14:44:10.724802+0000","last_fullsized":"2024-09-13T14:44:10.724802+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:17.007388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1266,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.311205+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T14:44:10.311205+0000","last_peered":"2024-09-13T14:44:10.311205+0000","last_clean":"2024-09-13T14:44:10.311205+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T14:44:10.311205+0000","last_undegraded":"2024-09-13T14:44:10.311205+0000","last_fullsized":"2024-09-13T14:44:10.311205+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":937,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345711+0000","last_change":"2024-09-13T14:40:09.185516+0000","last_active":"2024-09-13T14:44:09.345711+0000","last_peered":"2024-09-13T14:44:09.345711+0000","last_clean":"2024-09-13T14:44:09.345711+0000","last_became_active":"2024-09-13T14:40:09.184652+0000","last_became_peered":"2024-09-13T14:40:09.184652+0000","last_unstale":"2024-09-13T14:44:09.345711+0000","last_undegraded":"2024-09-13T14:44:09.345711+0000","last_fullsized":"2024-09-13T14:44:09.345711+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:48.492776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729090+0000","last_change":"2024-09-13T14:38:59.328518+0000","last_active":"2024-09-13T14:44:10.729090+0000","last_peered":"2024-09-13T14:44:10.729090+0000","last_clean":"2024-09-13T14:44:10.729090+0000","last_became_active":"2024-09-13T14:38:59.326010+0000","last_became_peered":"2024-09-13T14:38:59.326010+0000","last_unstale":"2024-09-13T14:44:10.729090+0000","last_undegraded":"2024-09-13T14:44:10.729090+0000","last_fullsized":"2024-09-13T14:44:10.729090+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:01.198043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.733416+0000","last_change":"2024-09-13T14:40:09.160901+0000","last_active":"2024-09-13T14:44:10.733416+0000","last_peered":"2024-09-13T14:44:10.733416+0000","last_clean":"2024-09-13T14:44:10.733416+0000","last_became_active":"2024-09-13T14:40:09.160278+0000","last_became_peered":"2024-09-13T14:40:09.160278+0000","last_unstale":"2024-09-13T14:44:10.733416+0000","last_undegraded":"2024-09-13T14:44:10.733416+0000","last_fullsized":"2024-09-13T14:44:10.733416+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:48:56.238297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.728934+0000","last_change":"2024-09-13T14:38:11.234481+0000","last_active":"2024-09-13T14:44:10.728934+0000","last_peered":"2024-09-13T14:44:10.728934+0000","last_clean":"2024-09-13T14:44:10.728934+0000","last_became_active":"2024-09-13T14:38:11.232868+0000","last_became_peered":"2024-09-13T14:38:11.232868+0000","last_unstale":"2024-09-13T14:44:10.728934+0000","last_undegraded":"2024-09-13T14:44:10.728934+0000","last_fullsized":"2024-09-13T14:44:10.728934+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:06:01.704192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":922,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301454+0000","last_change":"2024-09-13T14:39:34.321388+0000","last_active":"2024-09-13T14:44:09.301454+0000","last_peered":"2024-09-13T14:44:09.301454+0000","last_clean":"2024-09-13T14:44:09.301454+0000","last_became_active":"2024-09-13T14:39:34.321025+0000","last_became_peered":"2024-09-13T14:39:34.321025+0000","last_unstale":"2024-09-13T14:44:09.301454+0000","last_undegraded":"2024-09-13T14:44:09.301454+0000","last_fullsized":"2024-09-13T14:44:09.301454+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:52.593825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729257+0000","last_change":"2024-09-13T14:39:34.378359+0000","last_active":"2024-09-13T14:44:10.729257+0000","last_peered":"2024-09-13T14:44:10.729257+0000","last_clean":"2024-09-13T14:44:10.729257+0000","last_became_active":"2024-09-13T14:39:34.377967+0000","last_became_peered":"2024-09-13T14:39:34.377967+0000","last_unstale":"2024-09-13T14:44:10.729257+0000","last_undegraded":"2024-09-13T14:44:10.729257+0000","last_fullsized":"2024-09-13T14:44:10.729257+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:37.645170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1098,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.308015+0000","last_change":"2024-09-13T14:40:09.222631+0000","last_active":"2024-09-13T14:44:10.308015+0000","last_peered":"2024-09-13T14:44:10.308015+0000","last_clean":"2024-09-13T14:44:10.308015+0000","last_became_active":"2024-09-13T14:40:09.211300+0000","last_became_peered":"2024-09-13T14:40:09.211300+0000","last_unstale":"2024-09-13T14:44:10.308015+0000","last_undegraded":"2024-09-13T14:44:10.308015+0000","last_fullsized":"2024-09-13T14:44:10.308015+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:56.913336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":919,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.299732+0000","last_change":"2024-09-13T14:38:11.232633+0000","last_active":"2024-09-13T14:44:09.299732+0000","last_peered":"2024-09-13T14:44:09.299732+0000","last_clean":"2024-09-13T14:44:09.299732+0000","last_became_active":"2024-09-13T14:38:11.230948+0000","last_became_peered":"2024-09-13T14:38:11.230948+0000","last_unstale":"2024-09-13T14:44:09.299732+0000","last_undegraded":"2024-09-13T14:44:09.299732+0000","last_fullsized":"2024-09-13T14:44:09.299732+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:38.665278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.736730+0000","last_change":"2024-09-13T14:38:59.402033+0000","last_active":"2024-09-13T14:44:10.736730+0000","last_peered":"2024-09-13T14:44:10.736730+0000","last_clean":"2024-09-13T14:44:10.736730+0000","last_became_active":"2024-09-13T14:38:59.401826+0000","last_became_peered":"2024-09-13T14:38:59.401826+0000","last_unstale":"2024-09-13T14:44:10.736730+0000","last_undegraded":"2024-09-13T14:44:10.736730+0000","last_fullsized":"2024-09-13T14:44:10.736730+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:17.785114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.727845+0000","last_change":"2024-09-13T14:38:59.329634+0000","last_active":"2024-09-13T14:44:10.727845+0000","last_peered":"2024-09-13T14:44:10.727845+0000","last_clean":"2024-09-13T14:44:10.727845+0000","last_became_active":"2024-09-13T14:38:59.328813+0000","last_became_peered":"2024-09-13T14:38:59.328813+0000","last_unstale":"2024-09-13T14:44:10.727845+0000","last_undegraded":"2024-09-13T14:44:10.727845+0000","last_fullsized":"2024-09-13T14:44:10.727845+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:35.653113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1175,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.737951+0000","last_change":"2024-09-13T14:39:34.322533+0000","last_active":"2024-09-13T14:44:10.737951+0000","last_peered":"2024-09-13T14:44:10.737951+0000","last_clean":"2024-09-13T14:44:10.737951+0000","last_became_active":"2024-09-13T14:39:34.322070+0000","last_became_peered":"2024-09-13T14:39:34.322070+0000","last_unstale":"2024-09-13T14:44:10.737951+0000","last_undegraded":"2024-09-13T14:44:10.737951+0000","last_fullsized":"2024-09-13T14:44:10.737951+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:43.746420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314327+0000","last_change":"2024-09-13T14:38:59.338775+0000","last_active":"2024-09-13T14:44:10.314327+0000","last_peered":"2024-09-13T14:44:10.314327+0000","last_clean":"2024-09-13T14:44:10.314327+0000","last_became_active":"2024-09-13T14:38:59.338265+0000","last_became_peered":"2024-09-13T14:38:59.338265+0000","last_unstale":"2024-09-13T14:44:10.314327+0000","last_undegraded":"2024-09-13T14:44:10.314327+0000","last_fullsized":"2024-09-13T14:44:10.314327+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:48.255980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313649+0000","last_change":"2024-09-13T14:40:09.188286+0000","last_active":"2024-09-13T14:44:10.313649+0000","last_peered":"2024-09-13T14:44:10.313649+0000","last_clean":"2024-09-13T14:44:10.313649+0000","last_became_active":"2024-09-13T14:40:09.186778+0000","last_became_peered":"2024-09-13T14:40:09.186778+0000","last_unstale":"2024-09-13T14:44:10.313649+0000","last_undegraded":"2024-09-13T14:44:10.313649+0000","last_fullsized":"2024-09-13T14:44:10.313649+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:44:24.275819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":937,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345163+0000","last_change":"2024-09-13T14:38:11.178130+0000","last_active":"2024-09-13T14:44:09.345163+0000","last_peered":"2024-09-13T14:44:09.345163+0000","last_clean":"2024-09-13T14:44:09.345163+0000","last_became_active":"2024-09-13T14:38:11.177484+0000","last_became_peered":"2024-09-13T14:38:11.177484+0000","last_unstale":"2024-09-13T14:44:09.345163+0000","last_undegraded":"2024-09-13T14:44:09.345163+0000","last_fullsized":"2024-09-13T14:44:09.345163+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:18:24.776012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487609+0000","last_change":"2024-09-13T14:40:09.183716+0000","last_active":"2024-09-13T14:44:09.487609+0000","last_peered":"2024-09-13T14:44:09.487609+0000","last_clean":"2024-09-13T14:44:09.487609+0000","last_became_active":"2024-09-13T14:40:09.183420+0000","last_became_peered":"2024-09-13T14:40:09.183420+0000","last_unstale":"2024-09-13T14:44:09.487609+0000","last_undegraded":"2024-09-13T14:44:09.487609+0000","last_fullsized":"2024-09-13T14:44:09.487609+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:03.929337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1123,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723729+0000","last_change":"2024-09-13T14:39:34.356129+0000","last_active":"2024-09-13T14:44:10.723729+0000","last_peered":"2024-09-13T14:44:10.723729+0000","last_clean":"2024-09-13T14:44:10.723729+0000","last_became_active":"2024-09-13T14:39:34.355288+0000","last_became_peered":"2024-09-13T14:39:34.355288+0000","last_unstale":"2024-09-13T14:44:10.723729+0000","last_undegraded":"2024-09-13T14:44:10.723729+0000","last_fullsized":"2024-09-13T14:44:10.723729+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.725003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":918,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.300306+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T14:44:09.300306+0000","last_peered":"2024-09-13T14:44:09.300306+0000","last_clean":"2024-09-13T14:44:09.300306+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T14:44:09.300306+0000","last_undegraded":"2024-09-13T14:44:09.300306+0000","last_fullsized":"2024-09-13T14:44:09.300306+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1025,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345613+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T14:44:09.345613+0000","last_peered":"2024-09-13T14:44:09.345613+0000","last_clean":"2024-09-13T14:44:09.345613+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T14:44:09.345613+0000","last_undegraded":"2024-09-13T14:44:09.345613+0000","last_fullsized":"2024-09-13T14:44:09.345613+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.728956+0000","last_change":"2024-09-13T14:40:09.174566+0000","last_active":"2024-09-13T14:44:10.728956+0000","last_peered":"2024-09-13T14:44:10.728956+0000","last_clean":"2024-09-13T14:44:10.728956+0000","last_became_active":"2024-09-13T14:40:09.173698+0000","last_became_peered":"2024-09-13T14:40:09.173698+0000","last_unstale":"2024-09-13T14:44:10.728956+0000","last_undegraded":"2024-09-13T14:44:10.728956+0000","last_fullsized":"2024-09-13T14:44:10.728956+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:22.408868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1148,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723763+0000","last_change":"2024-09-13T14:39:34.403656+0000","last_active":"2024-09-13T14:44:10.723763+0000","last_peered":"2024-09-13T14:44:10.723763+0000","last_clean":"2024-09-13T14:44:10.723763+0000","last_became_active":"2024-09-13T14:39:34.386890+0000","last_became_peered":"2024-09-13T14:39:34.386890+0000","last_unstale":"2024-09-13T14:44:10.723763+0000","last_undegraded":"2024-09-13T14:44:10.723763+0000","last_fullsized":"2024-09-13T14:44:10.723763+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:24.563109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1082,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.307149+0000","last_change":"2024-09-13T14:40:09.182955+0000","last_active":"2024-09-13T14:44:10.307149+0000","last_peered":"2024-09-13T14:44:10.307149+0000","last_clean":"2024-09-13T14:44:10.307149+0000","last_became_active":"2024-09-13T14:40:09.182702+0000","last_became_peered":"2024-09-13T14:40:09.182702+0000","last_unstale":"2024-09-13T14:44:10.307149+0000","last_undegraded":"2024-09-13T14:44:10.307149+0000","last_fullsized":"2024-09-13T14:44:10.307149+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:00:15.207705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1113,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303286+0000","last_change":"2024-09-13T14:40:09.185241+0000","last_active":"2024-09-13T14:44:10.303286+0000","last_peered":"2024-09-13T14:44:10.303286+0000","last_clean":"2024-09-13T14:44:10.303286+0000","last_became_active":"2024-09-13T14:40:09.184259+0000","last_became_peered":"2024-09-13T14:40:09.184259+0000","last_unstale":"2024-09-13T14:44:10.303286+0000","last_undegraded":"2024-09-13T14:44:10.303286+0000","last_fullsized":"2024-09-13T14:44:10.303286+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:27:17.762953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1129,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.304287+0000","last_change":"2024-09-13T14:40:09.200853+0000","last_active":"2024-09-13T14:44:10.304287+0000","last_peered":"2024-09-13T14:44:10.304287+0000","last_clean":"2024-09-13T14:44:10.304287+0000","last_became_active":"2024-09-13T14:40:09.195505+0000","last_became_peered":"2024-09-13T14:40:09.195505+0000","last_unstale":"2024-09-13T14:44:10.304287+0000","last_undegraded":"2024-09-13T14:44:10.304287+0000","last_fullsized":"2024-09-13T14:44:10.304287+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:28.009085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":926,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.486881+0000","last_change":"2024-09-13T14:40:09.163144+0000","last_active":"2024-09-13T14:44:09.486881+0000","last_peered":"2024-09-13T14:44:09.486881+0000","last_clean":"2024-09-13T14:44:09.486881+0000","last_became_active":"2024-09-13T14:40:09.162863+0000","last_became_peered":"2024-09-13T14:40:09.162863+0000","last_unstale":"2024-09-13T14:44:09.486881+0000","last_undegraded":"2024-09-13T14:44:09.486881+0000","last_fullsized":"2024-09-13T14:44:09.486881+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:48.398889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1104,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303932+0000","last_change":"2024-09-13T14:40:09.223911+0000","last_active":"2024-09-13T14:44:10.303932+0000","last_peered":"2024-09-13T14:44:10.303932+0000","last_clean":"2024-09-13T14:44:10.303932+0000","last_became_active":"2024-09-13T14:40:09.215480+0000","last_became_peered":"2024-09-13T14:40:09.215480+0000","last_unstale":"2024-09-13T14:44:10.303932+0000","last_undegraded":"2024-09-13T14:44:10.303932+0000","last_fullsized":"2024-09-13T14:44:10.303932+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.613398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1137,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723407+0000","last_change":"2024-09-13T14:40:09.208549+0000","last_active":"2024-09-13T14:44:10.723407+0000","last_peered":"2024-09-13T14:44:10.723407+0000","last_clean":"2024-09-13T14:44:10.723407+0000","last_became_active":"2024-09-13T14:40:09.208174+0000","last_became_peered":"2024-09-13T14:40:09.208174+0000","last_unstale":"2024-09-13T14:44:10.723407+0000","last_undegraded":"2024-09-13T14:44:10.723407+0000","last_fullsized":"2024-09-13T14:44:10.723407+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:18.971929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.311474+0000","last_change":"2024-09-13T14:40:09.192856+0000","last_active":"2024-09-13T14:44:10.311474+0000","last_peered":"2024-09-13T14:44:10.311474+0000","last_clean":"2024-09-13T14:44:10.311474+0000","last_became_active":"2024-09-13T14:40:09.186733+0000","last_became_peered":"2024-09-13T14:40:09.186733+0000","last_unstale":"2024-09-13T14:44:10.311474+0000","last_undegraded":"2024-09-13T14:44:10.311474+0000","last_fullsized":"2024-09-13T14:44:10.311474+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:55.336998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":905,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.302232+0000","last_change":"2024-09-13T14:39:34.393673+0000","last_active":"2024-09-13T14:44:09.302232+0000","last_peered":"2024-09-13T14:44:09.302232+0000","last_clean":"2024-09-13T14:44:09.302232+0000","last_became_active":"2024-09-13T14:39:34.393186+0000","last_became_peered":"2024-09-13T14:39:34.393186+0000","last_unstale":"2024-09-13T14:44:09.302232+0000","last_undegraded":"2024-09-13T14:44:09.302232+0000","last_fullsized":"2024-09-13T14:44:09.302232+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:23.130084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.486735+0000","last_change":"2024-09-13T14:38:11.219076+0000","last_active":"2024-09-13T14:44:09.486735+0000","last_peered":"2024-09-13T14:44:09.486735+0000","last_clean":"2024-09-13T14:44:09.486735+0000","last_became_active":"2024-09-13T14:38:11.217912+0000","last_became_peered":"2024-09-13T14:38:11.217912+0000","last_unstale":"2024-09-13T14:44:09.486735+0000","last_undegraded":"2024-09-13T14:44:09.486735+0000","last_fullsized":"2024-09-13T14:44:09.486735+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:32.673534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":893,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.344340+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:44:09.344340+0000","last_peered":"2024-09-13T14:44:09.344340+0000","last_clean":"2024-09-13T14:44:09.344340+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:44:09.344340+0000","last_undegraded":"2024-09-13T14:44:09.344340+0000","last_fullsized":"2024-09-13T14:44:09.344340+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.302845+0000","last_change":"2024-09-13T14:40:09.211963+0000","last_active":"2024-09-13T14:44:10.302845+0000","last_peered":"2024-09-13T14:44:10.302845+0000","last_clean":"2024-09-13T14:44:10.302845+0000","last_became_active":"2024-09-13T14:40:09.205704+0000","last_became_peered":"2024-09-13T14:40:09.205704+0000","last_unstale":"2024-09-13T14:44:10.302845+0000","last_undegraded":"2024-09-13T14:44:10.302845+0000","last_fullsized":"2024-09-13T14:44:10.302845+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:18.704369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":928,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345857+0000","last_change":"2024-09-13T14:40:09.193142+0000","last_active":"2024-09-13T14:44:09.345857+0000","last_peered":"2024-09-13T14:44:09.345857+0000","last_clean":"2024-09-13T14:44:09.345857+0000","last_became_active":"2024-09-13T14:40:09.191806+0000","last_became_peered":"2024-09-13T14:40:09.191806+0000","last_unstale":"2024-09-13T14:44:09.345857+0000","last_undegraded":"2024-09-13T14:44:09.345857+0000","last_fullsized":"2024-09-13T14:44:09.345857+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:41:20.972768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":887,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.299536+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:44:09.299536+0000","last_peered":"2024-09-13T14:44:09.299536+0000","last_clean":"2024-09-13T14:44:09.299536+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:44:09.299536+0000","last_undegraded":"2024-09-13T14:44:09.299536+0000","last_fullsized":"2024-09-13T14:44:09.299536+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":927,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345967+0000","last_change":"2024-09-13T14:40:09.191167+0000","last_active":"2024-09-13T14:44:09.345967+0000","last_peered":"2024-09-13T14:44:09.345967+0000","last_clean":"2024-09-13T14:44:09.345967+0000","last_became_active":"2024-09-13T14:40:09.185797+0000","last_became_peered":"2024-09-13T14:40:09.185797+0000","last_unstale":"2024-09-13T14:44:09.345967+0000","last_undegraded":"2024-09-13T14:44:09.345967+0000","last_fullsized":"2024-09-13T14:44:09.345967+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:52.157413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":921,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.485897+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T14:44:09.485897+0000","last_peered":"2024-09-13T14:44:09.485897+0000","last_clean":"2024-09-13T14:44:09.485897+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T14:44:09.485897+0000","last_undegraded":"2024-09-13T14:44:09.485897+0000","last_fullsized":"2024-09-13T14:44:09.485897+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724498+0000","last_change":"2024-09-13T14:39:34.395859+0000","last_active":"2024-09-13T14:44:10.724498+0000","last_peered":"2024-09-13T14:44:10.724498+0000","last_clean":"2024-09-13T14:44:10.724498+0000","last_became_active":"2024-09-13T14:39:34.389481+0000","last_became_peered":"2024-09-13T14:39:34.389481+0000","last_unstale":"2024-09-13T14:44:10.724498+0000","last_undegraded":"2024-09-13T14:44:10.724498+0000","last_fullsized":"2024-09-13T14:44:10.724498+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.724694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.344937+0000","last_change":"2024-09-13T14:40:09.173284+0000","last_active":"2024-09-13T14:44:09.344937+0000","last_peered":"2024-09-13T14:44:09.344937+0000","last_clean":"2024-09-13T14:44:09.344937+0000","last_became_active":"2024-09-13T14:40:09.171846+0000","last_became_peered":"2024-09-13T14:40:09.171846+0000","last_unstale":"2024-09-13T14:44:09.344937+0000","last_undegraded":"2024-09-13T14:44:09.344937+0000","last_fullsized":"2024-09-13T14:44:09.344937+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:51:35.460440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1276,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314748+0000","last_change":"2024-09-13T14:39:34.362694+0000","last_active":"2024-09-13T14:44:10.314748+0000","last_peered":"2024-09-13T14:44:10.314748+0000","last_clean":"2024-09-13T14:44:10.314748+0000","last_became_active":"2024-09-13T14:39:34.362435+0000","last_became_peered":"2024-09-13T14:39:34.362435+0000","last_unstale":"2024-09-13T14:44:10.314748+0000","last_undegraded":"2024-09-13T14:44:10.314748+0000","last_fullsized":"2024-09-13T14:44:10.314748+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:33.350328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314791+0000","last_change":"2024-09-13T14:39:34.360941+0000","last_active":"2024-09-13T14:44:10.314791+0000","last_peered":"2024-09-13T14:44:10.314791+0000","last_clean":"2024-09-13T14:44:10.314791+0000","last_became_active":"2024-09-13T14:39:34.360383+0000","last_became_peered":"2024-09-13T14:39:34.360383+0000","last_unstale":"2024-09-13T14:44:10.314791+0000","last_undegraded":"2024-09-13T14:44:10.314791+0000","last_fullsized":"2024-09-13T14:44:10.314791+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:51.344296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":904,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301001+0000","last_change":"2024-09-13T14:39:34.380634+0000","last_active":"2024-09-13T14:44:09.301001+0000","last_peered":"2024-09-13T14:44:09.301001+0000","last_clean":"2024-09-13T14:44:09.301001+0000","last_became_active":"2024-09-13T14:39:34.379785+0000","last_became_peered":"2024-09-13T14:39:34.379785+0000","last_unstale":"2024-09-13T14:44:09.301001+0000","last_undegraded":"2024-09-13T14:44:09.301001+0000","last_fullsized":"2024-09-13T14:44:09.301001+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:02.436193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":929,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345354+0000","last_change":"2024-09-13T14:40:09.191394+0000","last_active":"2024-09-13T14:44:09.345354+0000","last_peered":"2024-09-13T14:44:09.345354+0000","last_clean":"2024-09-13T14:44:09.345354+0000","last_became_active":"2024-09-13T14:40:09.184697+0000","last_became_peered":"2024-09-13T14:40:09.184697+0000","last_unstale":"2024-09-13T14:44:09.345354+0000","last_undegraded":"2024-09-13T14:44:09.345354+0000","last_fullsized":"2024-09-13T14:44:09.345354+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:31.632367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724494+0000","last_change":"2024-09-13T14:39:34.401131+0000","last_active":"2024-09-13T14:44:10.724494+0000","last_peered":"2024-09-13T14:44:10.724494+0000","last_clean":"2024-09-13T14:44:10.724494+0000","last_became_active":"2024-09-13T14:39:34.393396+0000","last_became_peered":"2024-09-13T14:39:34.393396+0000","last_unstale":"2024-09-13T14:44:10.724494+0000","last_undegraded":"2024-09-13T14:44:10.724494+0000","last_fullsized":"2024-09-13T14:44:10.724494+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:35.475048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397816,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325592,"kb_used_data":15760,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93419560,"statfs":{"total":95995035648,"available":95661629440,"internally_reserved":0,"allocated":16138240,"data_stored":10329470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594051,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":345488,"kb_used_data":15744,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93399664,"statfs":{"total":95995035648,"available":95641255936,"internally_reserved":0,"allocated":16121856,"data_stored":10328713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790285,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362036,"kb_used_data":15160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93383116,"statfs":{"total":95995035648,"available":95624310784,"internally_reserved":0,"allocated":15523840,"data_stored":9734478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980831813,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":79212,"kb_used_data":15176,"kb_used_omap":76,"kb_used_meta":50099,"kb_avail":93665940,"statfs":{"total":95995035648,"available":95913922560,"internally_reserved":0,"allocated":15540224,"data_stored":9739044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":51302208},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911052,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":69504,"kb_used_data":15156,"kb_used_omap":77,"kb_used_meta":54258,"kb_avail":93675648,"statfs":{"total":95995035648,"available":95923863552,"internally_reserved":0,"allocated":15519744,"data_stored":9734252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":55561013},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218022995,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":51460,"kb_used_data":15160,"kb_used_omap":71,"kb_used_meta":36216,"kb_avail":93693692,"statfs":{"total":95995035648,"available":95942340608,"internally_reserved":0,"allocated":15523840,"data_stored":9734774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":37085856},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004125,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":117824,"kb_used_data":15152,"kb_used_omap":66,"kb_used_meta":62397,"kb_avail":93627328,"statfs":{"total":95995035648,"available":95874383872,"internally_reserved":0,"allocated":15515648,"data_stored":9733874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":63894698},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195317,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":129280,"kb_used_data":15736,"kb_used_omap":56,"kb_used_meta":49735,"kb_avail":93615872,"statfs":{"total":95995035648,"available":95862652928,"internally_reserved":0,"allocated":16113664,"data_stored":10323008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":50928783},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:45:47.283 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:45:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:45:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:45:47] "GET /metrics HTTP/1.1" 200 34268 "" "Prometheus/2.51.0" 2024-09-13T14:45:47.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:47 smithi099 bash[84576]: audit 2024-09-13T14:45:46.355373+0000 mgr.y (mgr.26252) 579 : audit [DBG] from='client.45077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:47.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:47 smithi099 bash[84576]: audit 2024-09-13T14:45:46.355373+0000 mgr.y (mgr.26252) 579 : audit [DBG] from='client.45077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:47.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:47 smithi099 bash[84576]: cluster 2024-09-13T14:45:46.439522+0000 mgr.y (mgr.26252) 580 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:47.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:47 smithi099 bash[84576]: cluster 2024-09-13T14:45:46.439522+0000 mgr.y (mgr.26252) 580 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:47.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:47 smithi099 bash[87147]: audit 2024-09-13T14:45:46.355373+0000 mgr.y (mgr.26252) 579 : audit [DBG] from='client.45077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:47.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:47 smithi099 bash[87147]: audit 2024-09-13T14:45:46.355373+0000 mgr.y (mgr.26252) 579 : audit [DBG] from='client.45077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:47.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:47 smithi099 bash[87147]: cluster 2024-09-13T14:45:46.439522+0000 mgr.y (mgr.26252) 580 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:47.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:47 smithi099 bash[87147]: cluster 2024-09-13T14:45:46.439522+0000 mgr.y (mgr.26252) 580 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:47.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:47 smithi177 bash[68594]: audit 2024-09-13T14:45:46.355373+0000 mgr.y (mgr.26252) 579 : audit [DBG] from='client.45077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:47.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:47 smithi177 bash[68594]: audit 2024-09-13T14:45:46.355373+0000 mgr.y (mgr.26252) 579 : audit [DBG] from='client.45077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:47.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:47 smithi177 bash[68594]: cluster 2024-09-13T14:45:46.439522+0000 mgr.y (mgr.26252) 580 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:47.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:47 smithi177 bash[68594]: cluster 2024-09-13T14:45:46.439522+0000 mgr.y (mgr.26252) 580 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:48 smithi099 bash[84576]: audit 2024-09-13T14:45:48.268281+0000 mon.a (mon.0) 1552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:48.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:48 smithi099 bash[84576]: audit 2024-09-13T14:45:48.268281+0000 mon.a (mon.0) 1552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:48 smithi099 bash[87147]: audit 2024-09-13T14:45:48.268281+0000 mon.a (mon.0) 1552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:48.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:48 smithi099 bash[87147]: audit 2024-09-13T14:45:48.268281+0000 mon.a (mon.0) 1552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:48.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:48 smithi177 bash[68594]: audit 2024-09-13T14:45:48.268281+0000 mon.a (mon.0) 1552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:48.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:48 smithi177 bash[68594]: audit 2024-09-13T14:45:48.268281+0000 mon.a (mon.0) 1552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:49.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:49 smithi099 bash[87147]: cluster 2024-09-13T14:45:48.440385+0000 mgr.y (mgr.26252) 581 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:49.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:49 smithi099 bash[87147]: cluster 2024-09-13T14:45:48.440385+0000 mgr.y (mgr.26252) 581 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:49 smithi099 bash[84576]: cluster 2024-09-13T14:45:48.440385+0000 mgr.y (mgr.26252) 581 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:49.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:49 smithi099 bash[84576]: cluster 2024-09-13T14:45:48.440385+0000 mgr.y (mgr.26252) 581 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:49.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:49 smithi177 bash[68594]: cluster 2024-09-13T14:45:48.440385+0000 mgr.y (mgr.26252) 581 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:49.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:49 smithi177 bash[68594]: cluster 2024-09-13T14:45:48.440385+0000 mgr.y (mgr.26252) 581 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:50.521 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:45:50 smithi177 bash[61439]: ts=2024-09-13T14:45:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:45:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:50 smithi099 bash[84576]: audit 2024-09-13T14:45:49.781526+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:50.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:50 smithi099 bash[84576]: audit 2024-09-13T14:45:49.781526+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:50.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:50 smithi099 bash[87147]: audit 2024-09-13T14:45:49.781526+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:50.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:50 smithi099 bash[87147]: audit 2024-09-13T14:45:49.781526+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:50.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:50 smithi177 bash[68594]: audit 2024-09-13T14:45:49.781526+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:50.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:50 smithi177 bash[68594]: audit 2024-09-13T14:45:49.781526+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:51.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:51 smithi099 bash[84576]: cluster 2024-09-13T14:45:50.441188+0000 mgr.y (mgr.26252) 582 : cluster [DBG] pgmap v1031: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:51.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:51 smithi099 bash[84576]: cluster 2024-09-13T14:45:50.441188+0000 mgr.y (mgr.26252) 582 : cluster [DBG] pgmap v1031: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:51.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:51 smithi099 bash[87147]: cluster 2024-09-13T14:45:50.441188+0000 mgr.y (mgr.26252) 582 : cluster [DBG] pgmap v1031: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:51.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:51 smithi099 bash[87147]: cluster 2024-09-13T14:45:50.441188+0000 mgr.y (mgr.26252) 582 : cluster [DBG] pgmap v1031: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:51 smithi177 bash[68594]: cluster 2024-09-13T14:45:50.441188+0000 mgr.y (mgr.26252) 582 : cluster [DBG] pgmap v1031: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:51 smithi177 bash[68594]: cluster 2024-09-13T14:45:50.441188+0000 mgr.y (mgr.26252) 582 : cluster [DBG] pgmap v1031: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:52.074 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:53.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:53 smithi099 bash[84576]: cluster 2024-09-13T14:45:52.442036+0000 mgr.y (mgr.26252) 583 : cluster [DBG] pgmap v1032: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:53.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:53 smithi099 bash[84576]: cluster 2024-09-13T14:45:52.442036+0000 mgr.y (mgr.26252) 583 : cluster [DBG] pgmap v1032: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:53.819 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:53 smithi099 bash[87147]: cluster 2024-09-13T14:45:52.442036+0000 mgr.y (mgr.26252) 583 : cluster [DBG] pgmap v1032: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:53.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:53 smithi099 bash[87147]: cluster 2024-09-13T14:45:52.442036+0000 mgr.y (mgr.26252) 583 : cluster [DBG] pgmap v1032: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:53.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:53 smithi177 bash[68594]: cluster 2024-09-13T14:45:52.442036+0000 mgr.y (mgr.26252) 583 : cluster [DBG] pgmap v1032: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:53.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:53 smithi177 bash[68594]: cluster 2024-09-13T14:45:52.442036+0000 mgr.y (mgr.26252) 583 : cluster [DBG] pgmap v1032: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:55.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:55 smithi099 bash[84576]: cluster 2024-09-13T14:45:54.442831+0000 mgr.y (mgr.26252) 584 : cluster [DBG] pgmap v1033: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:55.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:55 smithi099 bash[84576]: cluster 2024-09-13T14:45:54.442831+0000 mgr.y (mgr.26252) 584 : cluster [DBG] pgmap v1033: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:55.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:55 smithi099 bash[87147]: cluster 2024-09-13T14:45:54.442831+0000 mgr.y (mgr.26252) 584 : cluster [DBG] pgmap v1033: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:55.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:55 smithi099 bash[87147]: cluster 2024-09-13T14:45:54.442831+0000 mgr.y (mgr.26252) 584 : cluster [DBG] pgmap v1033: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:55.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:55 smithi177 bash[68594]: cluster 2024-09-13T14:45:54.442831+0000 mgr.y (mgr.26252) 584 : cluster [DBG] pgmap v1033: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:55.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:55 smithi177 bash[68594]: cluster 2024-09-13T14:45:54.442831+0000 mgr.y (mgr.26252) 584 : cluster [DBG] pgmap v1033: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:56.024 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:45:56.025 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:45:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:55.582083+0000 mon.a (mon.0) 1554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:55.582083+0000 mon.a (mon.0) 1554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:55.587754+0000 mon.a (mon.0) 1555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:55.587754+0000 mon.a (mon.0) 1555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:55.829662+0000 mon.a (mon.0) 1556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:55.829662+0000 mon.a (mon.0) 1556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:55.835242+0000 mon.a (mon.0) 1557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:55.835242+0000 mon.a (mon.0) 1557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:56.444665+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:56.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:56.444665+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:56.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:56.446148+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:56.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:56.446148+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:56.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:56.457341+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:56 smithi177 bash[68594]: audit 2024-09-13T14:45:56.457341+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.942 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:55.582083+0000 mon.a (mon.0) 1554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.942 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:55.582083+0000 mon.a (mon.0) 1554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.942 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:55.587754+0000 mon.a (mon.0) 1555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.942 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:55.587754+0000 mon.a (mon.0) 1555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.942 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:55.829662+0000 mon.a (mon.0) 1556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.942 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:55.829662+0000 mon.a (mon.0) 1556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.942 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:55.835242+0000 mon.a (mon.0) 1557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:55.835242+0000 mon.a (mon.0) 1557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:56.444665+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:56.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:56.444665+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:56.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:56.446148+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:56.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:56.446148+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:56.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:56.457341+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.943 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[84576]: audit 2024-09-13T14:45:56.457341+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.943 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:55.582083+0000 mon.a (mon.0) 1554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.943 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:55.582083+0000 mon.a (mon.0) 1554 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:55.587754+0000 mon.a (mon.0) 1555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:55.587754+0000 mon.a (mon.0) 1555 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:55.829662+0000 mon.a (mon.0) 1556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:55.829662+0000 mon.a (mon.0) 1556 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:55.835242+0000 mon.a (mon.0) 1557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:55.835242+0000 mon.a (mon.0) 1557 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:56.444665+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:56.444665+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:56.446148+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:56.446148+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:56.457341+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:56 smithi099 bash[87147]: audit 2024-09-13T14:45:56.457341+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:45:56.945 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1033,"stamp":"2024-09-13T14:45:54.442302+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":344,"num_read_kb":799,"num_write":212,"num_write_kb":1659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":164,"ondisk_log_size":164,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1480396,"kb_used_data":123044,"kb_used_omap":353,"kb_used_meta":1169694,"kb_avail":748480820,"statfs":{"total":767960285184,"available":766444359680,"internally_reserved":0,"allocated":125997056,"data_stored":79657613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":361898,"internal_metadata":1197767254},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004938"},"pg_stats":[{"pgid":"3.1f","version":"65'5","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313264+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T14:44:10.313264+0000","last_peered":"2024-09-13T14:44:10.313264+0000","last_clean":"2024-09-13T14:44:10.313264+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T14:44:10.313264+0000","last_undegraded":"2024-09-13T14:44:10.313264+0000","last_fullsized":"2024-09-13T14:44:10.313264+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1082,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.301691+0000","last_change":"2024-09-13T14:40:09.139213+0000","last_active":"2024-09-13T14:44:10.301691+0000","last_peered":"2024-09-13T14:44:10.301691+0000","last_clean":"2024-09-13T14:44:10.301691+0000","last_became_active":"2024-09-13T14:40:09.138480+0000","last_became_peered":"2024-09-13T14:40:09.138480+0000","last_unstale":"2024-09-13T14:44:10.301691+0000","last_undegraded":"2024-09-13T14:44:10.301691+0000","last_fullsized":"2024-09-13T14:44:10.301691+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:02:27.534835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1139,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724219+0000","last_change":"2024-09-13T14:39:34.403170+0000","last_active":"2024-09-13T14:44:10.724219+0000","last_peered":"2024-09-13T14:44:10.724219+0000","last_clean":"2024-09-13T14:44:10.724219+0000","last_became_active":"2024-09-13T14:39:34.389001+0000","last_became_peered":"2024-09-13T14:39:34.389001+0000","last_unstale":"2024-09-13T14:44:10.724219+0000","last_undegraded":"2024-09-13T14:44:10.724219+0000","last_fullsized":"2024-09-13T14:44:10.724219+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:38.542452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1152,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.722796+0000","last_change":"2024-09-13T14:39:34.316950+0000","last_active":"2024-09-13T14:44:10.722796+0000","last_peered":"2024-09-13T14:44:10.722796+0000","last_clean":"2024-09-13T14:44:10.722796+0000","last_became_active":"2024-09-13T14:39:34.316614+0000","last_became_peered":"2024-09-13T14:39:34.316614+0000","last_unstale":"2024-09-13T14:44:10.722796+0000","last_undegraded":"2024-09-13T14:44:10.722796+0000","last_fullsized":"2024-09-13T14:44:10.722796+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:20.664989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301239+0000","last_change":"2024-09-13T14:40:09.190077+0000","last_active":"2024-09-13T14:44:09.301239+0000","last_peered":"2024-09-13T14:44:09.301239+0000","last_clean":"2024-09-13T14:44:09.301239+0000","last_became_active":"2024-09-13T14:40:09.189717+0000","last_became_peered":"2024-09-13T14:40:09.189717+0000","last_unstale":"2024-09-13T14:44:09.301239+0000","last_undegraded":"2024-09-13T14:44:09.301239+0000","last_fullsized":"2024-09-13T14:44:09.301239+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:24.201026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303665+0000","last_change":"2024-09-13T14:40:09.219485+0000","last_active":"2024-09-13T14:44:10.303665+0000","last_peered":"2024-09-13T14:44:10.303665+0000","last_clean":"2024-09-13T14:44:10.303665+0000","last_became_active":"2024-09-13T14:40:09.212705+0000","last_became_peered":"2024-09-13T14:40:09.212705+0000","last_unstale":"2024-09-13T14:44:10.303665+0000","last_undegraded":"2024-09-13T14:44:10.303665+0000","last_fullsized":"2024-09-13T14:44:10.303665+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:02.378795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":925,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.302449+0000","last_change":"2024-09-13T14:40:09.192785+0000","last_active":"2024-09-13T14:44:09.302449+0000","last_peered":"2024-09-13T14:44:09.302449+0000","last_clean":"2024-09-13T14:44:09.302449+0000","last_became_active":"2024-09-13T14:40:09.192196+0000","last_became_peered":"2024-09-13T14:40:09.192196+0000","last_unstale":"2024-09-13T14:44:09.302449+0000","last_undegraded":"2024-09-13T14:44:09.302449+0000","last_fullsized":"2024-09-13T14:44:09.302449+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:52.737394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.305451+0000","last_change":"2024-09-13T14:40:09.220295+0000","last_active":"2024-09-13T14:44:10.305451+0000","last_peered":"2024-09-13T14:44:10.305451+0000","last_clean":"2024-09-13T14:44:10.305451+0000","last_became_active":"2024-09-13T14:40:09.209986+0000","last_became_peered":"2024-09-13T14:40:09.209986+0000","last_unstale":"2024-09-13T14:44:10.305451+0000","last_undegraded":"2024-09-13T14:44:10.305451+0000","last_fullsized":"2024-09-13T14:44:10.305451+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:57.153128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1282,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.311627+0000","last_change":"2024-09-13T14:40:09.194263+0000","last_active":"2024-09-13T14:44:10.311627+0000","last_peered":"2024-09-13T14:44:10.311627+0000","last_clean":"2024-09-13T14:44:10.311627+0000","last_became_active":"2024-09-13T14:40:09.192631+0000","last_became_peered":"2024-09-13T14:40:09.192631+0000","last_unstale":"2024-09-13T14:44:10.311627+0000","last_undegraded":"2024-09-13T14:44:10.311627+0000","last_fullsized":"2024-09-13T14:44:10.311627+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:41:31.185137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1186,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.740194+0000","last_change":"2024-09-13T14:39:34.358767+0000","last_active":"2024-09-13T14:44:10.740194+0000","last_peered":"2024-09-13T14:44:10.740194+0000","last_clean":"2024-09-13T14:44:10.740194+0000","last_became_active":"2024-09-13T14:39:34.358244+0000","last_became_peered":"2024-09-13T14:39:34.358244+0000","last_unstale":"2024-09-13T14:44:10.740194+0000","last_undegraded":"2024-09-13T14:44:10.740194+0000","last_fullsized":"2024-09-13T14:44:10.740194+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:39.392859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":1132,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303234+0000","last_change":"2024-09-13T14:40:09.189390+0000","last_active":"2024-09-13T14:44:10.303234+0000","last_peered":"2024-09-13T14:44:10.303234+0000","last_clean":"2024-09-13T14:44:10.303234+0000","last_became_active":"2024-09-13T14:40:09.189042+0000","last_became_peered":"2024-09-13T14:40:09.189042+0000","last_unstale":"2024-09-13T14:44:10.303234+0000","last_undegraded":"2024-09-13T14:44:10.303234+0000","last_fullsized":"2024-09-13T14:44:10.303234+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:05.081276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1158,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724166+0000","last_change":"2024-09-13T14:40:09.227622+0000","last_active":"2024-09-13T14:44:10.724166+0000","last_peered":"2024-09-13T14:44:10.724166+0000","last_clean":"2024-09-13T14:44:10.724166+0000","last_became_active":"2024-09-13T14:40:09.226971+0000","last_became_peered":"2024-09-13T14:40:09.226971+0000","last_unstale":"2024-09-13T14:44:10.724166+0000","last_undegraded":"2024-09-13T14:44:10.724166+0000","last_fullsized":"2024-09-13T14:44:10.724166+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:48:21.955396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.730771+0000","last_change":"2024-09-13T14:40:09.173906+0000","last_active":"2024-09-13T14:44:10.730771+0000","last_peered":"2024-09-13T14:44:10.730771+0000","last_clean":"2024-09-13T14:44:10.730771+0000","last_became_active":"2024-09-13T14:40:09.173303+0000","last_became_peered":"2024-09-13T14:40:09.173303+0000","last_unstale":"2024-09-13T14:44:10.730771+0000","last_undegraded":"2024-09-13T14:44:10.730771+0000","last_fullsized":"2024-09-13T14:44:10.730771+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:34:11.061883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314602+0000","last_change":"2024-09-13T14:39:34.362206+0000","last_active":"2024-09-13T14:44:10.314602+0000","last_peered":"2024-09-13T14:44:10.314602+0000","last_clean":"2024-09-13T14:44:10.314602+0000","last_became_active":"2024-09-13T14:39:34.361932+0000","last_became_peered":"2024-09-13T14:39:34.361932+0000","last_unstale":"2024-09-13T14:44:10.314602+0000","last_undegraded":"2024-09-13T14:44:10.314602+0000","last_fullsized":"2024-09-13T14:44:10.314602+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:00.449274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1165,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723274+0000","last_change":"2024-09-13T14:39:34.403479+0000","last_active":"2024-09-13T14:44:10.723274+0000","last_peered":"2024-09-13T14:44:10.723274+0000","last_clean":"2024-09-13T14:44:10.723274+0000","last_became_active":"2024-09-13T14:39:34.386530+0000","last_became_peered":"2024-09-13T14:39:34.386530+0000","last_unstale":"2024-09-13T14:44:10.723274+0000","last_undegraded":"2024-09-13T14:44:10.723274+0000","last_fullsized":"2024-09-13T14:44:10.723274+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:29.559892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487821+0000","last_change":"2024-09-13T14:38:11.206109+0000","last_active":"2024-09-13T14:44:09.487821+0000","last_peered":"2024-09-13T14:44:09.487821+0000","last_clean":"2024-09-13T14:44:09.487821+0000","last_became_active":"2024-09-13T14:38:11.205546+0000","last_became_peered":"2024-09-13T14:38:11.205546+0000","last_unstale":"2024-09-13T14:44:09.487821+0000","last_undegraded":"2024-09-13T14:44:09.487821+0000","last_fullsized":"2024-09-13T14:44:09.487821+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:33.834490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1145,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.722999+0000","last_change":"2024-09-13T14:39:34.368100+0000","last_active":"2024-09-13T14:44:10.722999+0000","last_peered":"2024-09-13T14:44:10.722999+0000","last_clean":"2024-09-13T14:44:10.722999+0000","last_became_active":"2024-09-13T14:39:34.366992+0000","last_became_peered":"2024-09-13T14:39:34.366992+0000","last_unstale":"2024-09-13T14:44:10.722999+0000","last_undegraded":"2024-09-13T14:44:10.722999+0000","last_fullsized":"2024-09-13T14:44:10.722999+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:54.220100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313289+0000","last_change":"2024-09-13T14:39:34.318266+0000","last_active":"2024-09-13T14:44:10.313289+0000","last_peered":"2024-09-13T14:44:10.313289+0000","last_clean":"2024-09-13T14:44:10.313289+0000","last_became_active":"2024-09-13T14:39:34.315973+0000","last_became_peered":"2024-09-13T14:39:34.315973+0000","last_unstale":"2024-09-13T14:44:10.313289+0000","last_undegraded":"2024-09-13T14:44:10.313289+0000","last_fullsized":"2024-09-13T14:44:10.313289+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:15.270282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1124,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723081+0000","last_change":"2024-09-13T14:39:34.338378+0000","last_active":"2024-09-13T14:44:10.723081+0000","last_peered":"2024-09-13T14:44:10.723081+0000","last_clean":"2024-09-13T14:44:10.723081+0000","last_became_active":"2024-09-13T14:39:34.338005+0000","last_became_peered":"2024-09-13T14:39:34.338005+0000","last_unstale":"2024-09-13T14:44:10.723081+0000","last_undegraded":"2024-09-13T14:44:10.723081+0000","last_fullsized":"2024-09-13T14:44:10.723081+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:29.120379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":905,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301800+0000","last_change":"2024-09-13T14:38:11.231165+0000","last_active":"2024-09-13T14:44:09.301800+0000","last_peered":"2024-09-13T14:44:09.301800+0000","last_clean":"2024-09-13T14:44:09.301800+0000","last_became_active":"2024-09-13T14:38:11.230759+0000","last_became_peered":"2024-09-13T14:38:11.230759+0000","last_unstale":"2024-09-13T14:44:09.301800+0000","last_undegraded":"2024-09-13T14:44:09.301800+0000","last_fullsized":"2024-09-13T14:44:09.301800+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:34.222184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":930,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345752+0000","last_change":"2024-09-13T14:40:09.191682+0000","last_active":"2024-09-13T14:44:09.345752+0000","last_peered":"2024-09-13T14:44:09.345752+0000","last_clean":"2024-09-13T14:44:09.345752+0000","last_became_active":"2024-09-13T14:40:09.184791+0000","last_became_peered":"2024-09-13T14:40:09.184791+0000","last_unstale":"2024-09-13T14:44:09.345752+0000","last_undegraded":"2024-09-13T14:44:09.345752+0000","last_fullsized":"2024-09-13T14:44:09.345752+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:56.463093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.733304+0000","last_change":"2024-09-13T14:38:59.370056+0000","last_active":"2024-09-13T14:44:10.733304+0000","last_peered":"2024-09-13T14:44:10.733304+0000","last_clean":"2024-09-13T14:44:10.733304+0000","last_became_active":"2024-09-13T14:38:59.369705+0000","last_became_peered":"2024-09-13T14:38:59.369705+0000","last_unstale":"2024-09-13T14:44:10.733304+0000","last_undegraded":"2024-09-13T14:44:10.733304+0000","last_fullsized":"2024-09-13T14:44:10.733304+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:59.898050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":1125,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303667+0000","last_change":"2024-09-13T14:40:09.180133+0000","last_active":"2024-09-13T14:44:10.303667+0000","last_peered":"2024-09-13T14:44:10.303667+0000","last_clean":"2024-09-13T14:44:10.303667+0000","last_became_active":"2024-09-13T14:40:09.179808+0000","last_became_peered":"2024-09-13T14:40:09.179808+0000","last_unstale":"2024-09-13T14:44:10.303667+0000","last_undegraded":"2024-09-13T14:44:10.303667+0000","last_fullsized":"2024-09-13T14:44:10.303667+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:11.823441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1122,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.306210+0000","last_change":"2024-09-13T14:40:09.190329+0000","last_active":"2024-09-13T14:44:10.306210+0000","last_peered":"2024-09-13T14:44:10.306210+0000","last_clean":"2024-09-13T14:44:10.306210+0000","last_became_active":"2024-09-13T14:40:09.189399+0000","last_became_peered":"2024-09-13T14:40:09.189399+0000","last_unstale":"2024-09-13T14:44:10.306210+0000","last_undegraded":"2024-09-13T14:44:10.306210+0000","last_fullsized":"2024-09-13T14:44:10.306210+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:36.338067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1099,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.304546+0000","last_change":"2024-09-13T14:40:09.202450+0000","last_active":"2024-09-13T14:44:10.304546+0000","last_peered":"2024-09-13T14:44:10.304546+0000","last_clean":"2024-09-13T14:44:10.304546+0000","last_became_active":"2024-09-13T14:40:09.201651+0000","last_became_peered":"2024-09-13T14:40:09.201651+0000","last_unstale":"2024-09-13T14:44:10.304546+0000","last_undegraded":"2024-09-13T14:44:10.304546+0000","last_fullsized":"2024-09-13T14:44:10.304546+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:12.544442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":921,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487923+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T14:44:09.487923+0000","last_peered":"2024-09-13T14:44:09.487923+0000","last_clean":"2024-09-13T14:44:09.487923+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T14:44:09.487923+0000","last_undegraded":"2024-09-13T14:44:09.487923+0000","last_fullsized":"2024-09-13T14:44:09.487923+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1293,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.315290+0000","last_change":"2024-09-13T14:40:09.187749+0000","last_active":"2024-09-13T14:44:10.315290+0000","last_peered":"2024-09-13T14:44:10.315290+0000","last_clean":"2024-09-13T14:44:10.315290+0000","last_became_active":"2024-09-13T14:40:09.187007+0000","last_became_peered":"2024-09-13T14:40:09.187007+0000","last_unstale":"2024-09-13T14:44:10.315290+0000","last_undegraded":"2024-09-13T14:44:10.315290+0000","last_fullsized":"2024-09-13T14:44:10.315290+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:08.019463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1122,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723614+0000","last_change":"2024-09-13T14:39:34.349945+0000","last_active":"2024-09-13T14:44:10.723614+0000","last_peered":"2024-09-13T14:44:10.723614+0000","last_clean":"2024-09-13T14:44:10.723614+0000","last_became_active":"2024-09-13T14:39:34.346730+0000","last_became_peered":"2024-09-13T14:39:34.346730+0000","last_unstale":"2024-09-13T14:44:10.723614+0000","last_undegraded":"2024-09-13T14:44:10.723614+0000","last_fullsized":"2024-09-13T14:44:10.723614+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:48.854603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1234,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729597+0000","last_change":"2024-09-13T14:40:09.177544+0000","last_active":"2024-09-13T14:44:10.729597+0000","last_peered":"2024-09-13T14:44:10.729597+0000","last_clean":"2024-09-13T14:44:10.729597+0000","last_became_active":"2024-09-13T14:40:09.177252+0000","last_became_peered":"2024-09-13T14:40:09.177252+0000","last_unstale":"2024-09-13T14:44:10.729597+0000","last_undegraded":"2024-09-13T14:44:10.729597+0000","last_fullsized":"2024-09-13T14:44:10.729597+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:45:50.539379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1125,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724802+0000","last_change":"2024-09-13T14:39:34.401432+0000","last_active":"2024-09-13T14:44:10.724802+0000","last_peered":"2024-09-13T14:44:10.724802+0000","last_clean":"2024-09-13T14:44:10.724802+0000","last_became_active":"2024-09-13T14:39:34.395458+0000","last_became_peered":"2024-09-13T14:39:34.395458+0000","last_unstale":"2024-09-13T14:44:10.724802+0000","last_undegraded":"2024-09-13T14:44:10.724802+0000","last_fullsized":"2024-09-13T14:44:10.724802+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:17.007388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1266,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.311205+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T14:44:10.311205+0000","last_peered":"2024-09-13T14:44:10.311205+0000","last_clean":"2024-09-13T14:44:10.311205+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T14:44:10.311205+0000","last_undegraded":"2024-09-13T14:44:10.311205+0000","last_fullsized":"2024-09-13T14:44:10.311205+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":937,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345711+0000","last_change":"2024-09-13T14:40:09.185516+0000","last_active":"2024-09-13T14:44:09.345711+0000","last_peered":"2024-09-13T14:44:09.345711+0000","last_clean":"2024-09-13T14:44:09.345711+0000","last_became_active":"2024-09-13T14:40:09.184652+0000","last_became_peered":"2024-09-13T14:40:09.184652+0000","last_unstale":"2024-09-13T14:44:09.345711+0000","last_undegraded":"2024-09-13T14:44:09.345711+0000","last_fullsized":"2024-09-13T14:44:09.345711+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:48.492776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729090+0000","last_change":"2024-09-13T14:38:59.328518+0000","last_active":"2024-09-13T14:44:10.729090+0000","last_peered":"2024-09-13T14:44:10.729090+0000","last_clean":"2024-09-13T14:44:10.729090+0000","last_became_active":"2024-09-13T14:38:59.326010+0000","last_became_peered":"2024-09-13T14:38:59.326010+0000","last_unstale":"2024-09-13T14:44:10.729090+0000","last_undegraded":"2024-09-13T14:44:10.729090+0000","last_fullsized":"2024-09-13T14:44:10.729090+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:01.198043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.733416+0000","last_change":"2024-09-13T14:40:09.160901+0000","last_active":"2024-09-13T14:44:10.733416+0000","last_peered":"2024-09-13T14:44:10.733416+0000","last_clean":"2024-09-13T14:44:10.733416+0000","last_became_active":"2024-09-13T14:40:09.160278+0000","last_became_peered":"2024-09-13T14:40:09.160278+0000","last_unstale":"2024-09-13T14:44:10.733416+0000","last_undegraded":"2024-09-13T14:44:10.733416+0000","last_fullsized":"2024-09-13T14:44:10.733416+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:48:56.238297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.728934+0000","last_change":"2024-09-13T14:38:11.234481+0000","last_active":"2024-09-13T14:44:10.728934+0000","last_peered":"2024-09-13T14:44:10.728934+0000","last_clean":"2024-09-13T14:44:10.728934+0000","last_became_active":"2024-09-13T14:38:11.232868+0000","last_became_peered":"2024-09-13T14:38:11.232868+0000","last_unstale":"2024-09-13T14:44:10.728934+0000","last_undegraded":"2024-09-13T14:44:10.728934+0000","last_fullsized":"2024-09-13T14:44:10.728934+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:06:01.704192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":922,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301454+0000","last_change":"2024-09-13T14:39:34.321388+0000","last_active":"2024-09-13T14:44:09.301454+0000","last_peered":"2024-09-13T14:44:09.301454+0000","last_clean":"2024-09-13T14:44:09.301454+0000","last_became_active":"2024-09-13T14:39:34.321025+0000","last_became_peered":"2024-09-13T14:39:34.321025+0000","last_unstale":"2024-09-13T14:44:09.301454+0000","last_undegraded":"2024-09-13T14:44:09.301454+0000","last_fullsized":"2024-09-13T14:44:09.301454+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:52.593825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729257+0000","last_change":"2024-09-13T14:39:34.378359+0000","last_active":"2024-09-13T14:44:10.729257+0000","last_peered":"2024-09-13T14:44:10.729257+0000","last_clean":"2024-09-13T14:44:10.729257+0000","last_became_active":"2024-09-13T14:39:34.377967+0000","last_became_peered":"2024-09-13T14:39:34.377967+0000","last_unstale":"2024-09-13T14:44:10.729257+0000","last_undegraded":"2024-09-13T14:44:10.729257+0000","last_fullsized":"2024-09-13T14:44:10.729257+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:37.645170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1098,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.308015+0000","last_change":"2024-09-13T14:40:09.222631+0000","last_active":"2024-09-13T14:44:10.308015+0000","last_peered":"2024-09-13T14:44:10.308015+0000","last_clean":"2024-09-13T14:44:10.308015+0000","last_became_active":"2024-09-13T14:40:09.211300+0000","last_became_peered":"2024-09-13T14:40:09.211300+0000","last_unstale":"2024-09-13T14:44:10.308015+0000","last_undegraded":"2024-09-13T14:44:10.308015+0000","last_fullsized":"2024-09-13T14:44:10.308015+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:56.913336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":919,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.299732+0000","last_change":"2024-09-13T14:38:11.232633+0000","last_active":"2024-09-13T14:44:09.299732+0000","last_peered":"2024-09-13T14:44:09.299732+0000","last_clean":"2024-09-13T14:44:09.299732+0000","last_became_active":"2024-09-13T14:38:11.230948+0000","last_became_peered":"2024-09-13T14:38:11.230948+0000","last_unstale":"2024-09-13T14:44:09.299732+0000","last_undegraded":"2024-09-13T14:44:09.299732+0000","last_fullsized":"2024-09-13T14:44:09.299732+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:38.665278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.736730+0000","last_change":"2024-09-13T14:38:59.402033+0000","last_active":"2024-09-13T14:44:10.736730+0000","last_peered":"2024-09-13T14:44:10.736730+0000","last_clean":"2024-09-13T14:44:10.736730+0000","last_became_active":"2024-09-13T14:38:59.401826+0000","last_became_peered":"2024-09-13T14:38:59.401826+0000","last_unstale":"2024-09-13T14:44:10.736730+0000","last_undegraded":"2024-09-13T14:44:10.736730+0000","last_fullsized":"2024-09-13T14:44:10.736730+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:17.785114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.727845+0000","last_change":"2024-09-13T14:38:59.329634+0000","last_active":"2024-09-13T14:44:10.727845+0000","last_peered":"2024-09-13T14:44:10.727845+0000","last_clean":"2024-09-13T14:44:10.727845+0000","last_became_active":"2024-09-13T14:38:59.328813+0000","last_became_peered":"2024-09-13T14:38:59.328813+0000","last_unstale":"2024-09-13T14:44:10.727845+0000","last_undegraded":"2024-09-13T14:44:10.727845+0000","last_fullsized":"2024-09-13T14:44:10.727845+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:35.653113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1175,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.737951+0000","last_change":"2024-09-13T14:39:34.322533+0000","last_active":"2024-09-13T14:44:10.737951+0000","last_peered":"2024-09-13T14:44:10.737951+0000","last_clean":"2024-09-13T14:44:10.737951+0000","last_became_active":"2024-09-13T14:39:34.322070+0000","last_became_peered":"2024-09-13T14:39:34.322070+0000","last_unstale":"2024-09-13T14:44:10.737951+0000","last_undegraded":"2024-09-13T14:44:10.737951+0000","last_fullsized":"2024-09-13T14:44:10.737951+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:43.746420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314327+0000","last_change":"2024-09-13T14:38:59.338775+0000","last_active":"2024-09-13T14:44:10.314327+0000","last_peered":"2024-09-13T14:44:10.314327+0000","last_clean":"2024-09-13T14:44:10.314327+0000","last_became_active":"2024-09-13T14:38:59.338265+0000","last_became_peered":"2024-09-13T14:38:59.338265+0000","last_unstale":"2024-09-13T14:44:10.314327+0000","last_undegraded":"2024-09-13T14:44:10.314327+0000","last_fullsized":"2024-09-13T14:44:10.314327+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:48.255980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313649+0000","last_change":"2024-09-13T14:40:09.188286+0000","last_active":"2024-09-13T14:44:10.313649+0000","last_peered":"2024-09-13T14:44:10.313649+0000","last_clean":"2024-09-13T14:44:10.313649+0000","last_became_active":"2024-09-13T14:40:09.186778+0000","last_became_peered":"2024-09-13T14:40:09.186778+0000","last_unstale":"2024-09-13T14:44:10.313649+0000","last_undegraded":"2024-09-13T14:44:10.313649+0000","last_fullsized":"2024-09-13T14:44:10.313649+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:44:24.275819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":937,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345163+0000","last_change":"2024-09-13T14:38:11.178130+0000","last_active":"2024-09-13T14:44:09.345163+0000","last_peered":"2024-09-13T14:44:09.345163+0000","last_clean":"2024-09-13T14:44:09.345163+0000","last_became_active":"2024-09-13T14:38:11.177484+0000","last_became_peered":"2024-09-13T14:38:11.177484+0000","last_unstale":"2024-09-13T14:44:09.345163+0000","last_undegraded":"2024-09-13T14:44:09.345163+0000","last_fullsized":"2024-09-13T14:44:09.345163+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:18:24.776012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487609+0000","last_change":"2024-09-13T14:40:09.183716+0000","last_active":"2024-09-13T14:44:09.487609+0000","last_peered":"2024-09-13T14:44:09.487609+0000","last_clean":"2024-09-13T14:44:09.487609+0000","last_became_active":"2024-09-13T14:40:09.183420+0000","last_became_peered":"2024-09-13T14:40:09.183420+0000","last_unstale":"2024-09-13T14:44:09.487609+0000","last_undegraded":"2024-09-13T14:44:09.487609+0000","last_fullsized":"2024-09-13T14:44:09.487609+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:03.929337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1124,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723784+0000","last_change":"2024-09-13T14:39:34.356129+0000","last_active":"2024-09-13T14:44:10.723784+0000","last_peered":"2024-09-13T14:44:10.723784+0000","last_clean":"2024-09-13T14:44:10.723784+0000","last_became_active":"2024-09-13T14:39:34.355288+0000","last_became_peered":"2024-09-13T14:39:34.355288+0000","last_unstale":"2024-09-13T14:44:10.723784+0000","last_undegraded":"2024-09-13T14:44:10.723784+0000","last_fullsized":"2024-09-13T14:44:10.723784+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.725003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":918,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.300306+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T14:44:09.300306+0000","last_peered":"2024-09-13T14:44:09.300306+0000","last_clean":"2024-09-13T14:44:09.300306+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T14:44:09.300306+0000","last_undegraded":"2024-09-13T14:44:09.300306+0000","last_fullsized":"2024-09-13T14:44:09.300306+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1025,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345613+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T14:44:09.345613+0000","last_peered":"2024-09-13T14:44:09.345613+0000","last_clean":"2024-09-13T14:44:09.345613+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T14:44:09.345613+0000","last_undegraded":"2024-09-13T14:44:09.345613+0000","last_fullsized":"2024-09-13T14:44:09.345613+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.728956+0000","last_change":"2024-09-13T14:40:09.174566+0000","last_active":"2024-09-13T14:44:10.728956+0000","last_peered":"2024-09-13T14:44:10.728956+0000","last_clean":"2024-09-13T14:44:10.728956+0000","last_became_active":"2024-09-13T14:40:09.173698+0000","last_became_peered":"2024-09-13T14:40:09.173698+0000","last_unstale":"2024-09-13T14:44:10.728956+0000","last_undegraded":"2024-09-13T14:44:10.728956+0000","last_fullsized":"2024-09-13T14:44:10.728956+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:22.408868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1148,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723763+0000","last_change":"2024-09-13T14:39:34.403656+0000","last_active":"2024-09-13T14:44:10.723763+0000","last_peered":"2024-09-13T14:44:10.723763+0000","last_clean":"2024-09-13T14:44:10.723763+0000","last_became_active":"2024-09-13T14:39:34.386890+0000","last_became_peered":"2024-09-13T14:39:34.386890+0000","last_unstale":"2024-09-13T14:44:10.723763+0000","last_undegraded":"2024-09-13T14:44:10.723763+0000","last_fullsized":"2024-09-13T14:44:10.723763+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:24.563109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1082,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.307149+0000","last_change":"2024-09-13T14:40:09.182955+0000","last_active":"2024-09-13T14:44:10.307149+0000","last_peered":"2024-09-13T14:44:10.307149+0000","last_clean":"2024-09-13T14:44:10.307149+0000","last_became_active":"2024-09-13T14:40:09.182702+0000","last_became_peered":"2024-09-13T14:40:09.182702+0000","last_unstale":"2024-09-13T14:44:10.307149+0000","last_undegraded":"2024-09-13T14:44:10.307149+0000","last_fullsized":"2024-09-13T14:44:10.307149+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:00:15.207705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1113,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303286+0000","last_change":"2024-09-13T14:40:09.185241+0000","last_active":"2024-09-13T14:44:10.303286+0000","last_peered":"2024-09-13T14:44:10.303286+0000","last_clean":"2024-09-13T14:44:10.303286+0000","last_became_active":"2024-09-13T14:40:09.184259+0000","last_became_peered":"2024-09-13T14:40:09.184259+0000","last_unstale":"2024-09-13T14:44:10.303286+0000","last_undegraded":"2024-09-13T14:44:10.303286+0000","last_fullsized":"2024-09-13T14:44:10.303286+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:27:17.762953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1129,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.304287+0000","last_change":"2024-09-13T14:40:09.200853+0000","last_active":"2024-09-13T14:44:10.304287+0000","last_peered":"2024-09-13T14:44:10.304287+0000","last_clean":"2024-09-13T14:44:10.304287+0000","last_became_active":"2024-09-13T14:40:09.195505+0000","last_became_peered":"2024-09-13T14:40:09.195505+0000","last_unstale":"2024-09-13T14:44:10.304287+0000","last_undegraded":"2024-09-13T14:44:10.304287+0000","last_fullsized":"2024-09-13T14:44:10.304287+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:28.009085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":926,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.486881+0000","last_change":"2024-09-13T14:40:09.163144+0000","last_active":"2024-09-13T14:44:09.486881+0000","last_peered":"2024-09-13T14:44:09.486881+0000","last_clean":"2024-09-13T14:44:09.486881+0000","last_became_active":"2024-09-13T14:40:09.162863+0000","last_became_peered":"2024-09-13T14:40:09.162863+0000","last_unstale":"2024-09-13T14:44:09.486881+0000","last_undegraded":"2024-09-13T14:44:09.486881+0000","last_fullsized":"2024-09-13T14:44:09.486881+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:48.398889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1104,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303932+0000","last_change":"2024-09-13T14:40:09.223911+0000","last_active":"2024-09-13T14:44:10.303932+0000","last_peered":"2024-09-13T14:44:10.303932+0000","last_clean":"2024-09-13T14:44:10.303932+0000","last_became_active":"2024-09-13T14:40:09.215480+0000","last_became_peered":"2024-09-13T14:40:09.215480+0000","last_unstale":"2024-09-13T14:44:10.303932+0000","last_undegraded":"2024-09-13T14:44:10.303932+0000","last_fullsized":"2024-09-13T14:44:10.303932+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.613398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1137,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723407+0000","last_change":"2024-09-13T14:40:09.208549+0000","last_active":"2024-09-13T14:44:10.723407+0000","last_peered":"2024-09-13T14:44:10.723407+0000","last_clean":"2024-09-13T14:44:10.723407+0000","last_became_active":"2024-09-13T14:40:09.208174+0000","last_became_peered":"2024-09-13T14:40:09.208174+0000","last_unstale":"2024-09-13T14:44:10.723407+0000","last_undegraded":"2024-09-13T14:44:10.723407+0000","last_fullsized":"2024-09-13T14:44:10.723407+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:18.971929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.311474+0000","last_change":"2024-09-13T14:40:09.192856+0000","last_active":"2024-09-13T14:44:10.311474+0000","last_peered":"2024-09-13T14:44:10.311474+0000","last_clean":"2024-09-13T14:44:10.311474+0000","last_became_active":"2024-09-13T14:40:09.186733+0000","last_became_peered":"2024-09-13T14:40:09.186733+0000","last_unstale":"2024-09-13T14:44:10.311474+0000","last_undegraded":"2024-09-13T14:44:10.311474+0000","last_fullsized":"2024-09-13T14:44:10.311474+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:55.336998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":905,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.302232+0000","last_change":"2024-09-13T14:39:34.393673+0000","last_active":"2024-09-13T14:44:09.302232+0000","last_peered":"2024-09-13T14:44:09.302232+0000","last_clean":"2024-09-13T14:44:09.302232+0000","last_became_active":"2024-09-13T14:39:34.393186+0000","last_became_peered":"2024-09-13T14:39:34.393186+0000","last_unstale":"2024-09-13T14:44:09.302232+0000","last_undegraded":"2024-09-13T14:44:09.302232+0000","last_fullsized":"2024-09-13T14:44:09.302232+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:23.130084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.486735+0000","last_change":"2024-09-13T14:38:11.219076+0000","last_active":"2024-09-13T14:44:09.486735+0000","last_peered":"2024-09-13T14:44:09.486735+0000","last_clean":"2024-09-13T14:44:09.486735+0000","last_became_active":"2024-09-13T14:38:11.217912+0000","last_became_peered":"2024-09-13T14:38:11.217912+0000","last_unstale":"2024-09-13T14:44:09.486735+0000","last_undegraded":"2024-09-13T14:44:09.486735+0000","last_fullsized":"2024-09-13T14:44:09.486735+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:32.673534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":893,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.344340+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:44:09.344340+0000","last_peered":"2024-09-13T14:44:09.344340+0000","last_clean":"2024-09-13T14:44:09.344340+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:44:09.344340+0000","last_undegraded":"2024-09-13T14:44:09.344340+0000","last_fullsized":"2024-09-13T14:44:09.344340+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.302845+0000","last_change":"2024-09-13T14:40:09.211963+0000","last_active":"2024-09-13T14:44:10.302845+0000","last_peered":"2024-09-13T14:44:10.302845+0000","last_clean":"2024-09-13T14:44:10.302845+0000","last_became_active":"2024-09-13T14:40:09.205704+0000","last_became_peered":"2024-09-13T14:40:09.205704+0000","last_unstale":"2024-09-13T14:44:10.302845+0000","last_undegraded":"2024-09-13T14:44:10.302845+0000","last_fullsized":"2024-09-13T14:44:10.302845+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:18.704369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":928,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345857+0000","last_change":"2024-09-13T14:40:09.193142+0000","last_active":"2024-09-13T14:44:09.345857+0000","last_peered":"2024-09-13T14:44:09.345857+0000","last_clean":"2024-09-13T14:44:09.345857+0000","last_became_active":"2024-09-13T14:40:09.191806+0000","last_became_peered":"2024-09-13T14:40:09.191806+0000","last_unstale":"2024-09-13T14:44:09.345857+0000","last_undegraded":"2024-09-13T14:44:09.345857+0000","last_fullsized":"2024-09-13T14:44:09.345857+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:41:20.972768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":887,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.299536+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:44:09.299536+0000","last_peered":"2024-09-13T14:44:09.299536+0000","last_clean":"2024-09-13T14:44:09.299536+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:44:09.299536+0000","last_undegraded":"2024-09-13T14:44:09.299536+0000","last_fullsized":"2024-09-13T14:44:09.299536+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":927,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345967+0000","last_change":"2024-09-13T14:40:09.191167+0000","last_active":"2024-09-13T14:44:09.345967+0000","last_peered":"2024-09-13T14:44:09.345967+0000","last_clean":"2024-09-13T14:44:09.345967+0000","last_became_active":"2024-09-13T14:40:09.185797+0000","last_became_peered":"2024-09-13T14:40:09.185797+0000","last_unstale":"2024-09-13T14:44:09.345967+0000","last_undegraded":"2024-09-13T14:44:09.345967+0000","last_fullsized":"2024-09-13T14:44:09.345967+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:52.157413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":921,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.485897+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T14:44:09.485897+0000","last_peered":"2024-09-13T14:44:09.485897+0000","last_clean":"2024-09-13T14:44:09.485897+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T14:44:09.485897+0000","last_undegraded":"2024-09-13T14:44:09.485897+0000","last_fullsized":"2024-09-13T14:44:09.485897+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724498+0000","last_change":"2024-09-13T14:39:34.395859+0000","last_active":"2024-09-13T14:44:10.724498+0000","last_peered":"2024-09-13T14:44:10.724498+0000","last_clean":"2024-09-13T14:44:10.724498+0000","last_became_active":"2024-09-13T14:39:34.389481+0000","last_became_peered":"2024-09-13T14:39:34.389481+0000","last_unstale":"2024-09-13T14:44:10.724498+0000","last_undegraded":"2024-09-13T14:44:10.724498+0000","last_fullsized":"2024-09-13T14:44:10.724498+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.724694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.344937+0000","last_change":"2024-09-13T14:40:09.173284+0000","last_active":"2024-09-13T14:44:09.344937+0000","last_peered":"2024-09-13T14:44:09.344937+0000","last_clean":"2024-09-13T14:44:09.344937+0000","last_became_active":"2024-09-13T14:40:09.171846+0000","last_became_peered":"2024-09-13T14:40:09.171846+0000","last_unstale":"2024-09-13T14:44:09.344937+0000","last_undegraded":"2024-09-13T14:44:09.344937+0000","last_fullsized":"2024-09-13T14:44:09.344937+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:51:35.460440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314998+0000","last_change":"2024-09-13T14:39:34.362694+0000","last_active":"2024-09-13T14:44:10.314998+0000","last_peered":"2024-09-13T14:44:10.314998+0000","last_clean":"2024-09-13T14:44:10.314998+0000","last_became_active":"2024-09-13T14:39:34.362435+0000","last_became_peered":"2024-09-13T14:39:34.362435+0000","last_unstale":"2024-09-13T14:44:10.314998+0000","last_undegraded":"2024-09-13T14:44:10.314998+0000","last_fullsized":"2024-09-13T14:44:10.314998+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:33.350328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314791+0000","last_change":"2024-09-13T14:39:34.360941+0000","last_active":"2024-09-13T14:44:10.314791+0000","last_peered":"2024-09-13T14:44:10.314791+0000","last_clean":"2024-09-13T14:44:10.314791+0000","last_became_active":"2024-09-13T14:39:34.360383+0000","last_became_peered":"2024-09-13T14:39:34.360383+0000","last_unstale":"2024-09-13T14:44:10.314791+0000","last_undegraded":"2024-09-13T14:44:10.314791+0000","last_fullsized":"2024-09-13T14:44:10.314791+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:51.344296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":904,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301001+0000","last_change":"2024-09-13T14:39:34.380634+0000","last_active":"2024-09-13T14:44:09.301001+0000","last_peered":"2024-09-13T14:44:09.301001+0000","last_clean":"2024-09-13T14:44:09.301001+0000","last_became_active":"2024-09-13T14:39:34.379785+0000","last_became_peered":"2024-09-13T14:39:34.379785+0000","last_unstale":"2024-09-13T14:44:09.301001+0000","last_undegraded":"2024-09-13T14:44:09.301001+0000","last_fullsized":"2024-09-13T14:44:09.301001+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:02.436193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":929,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345354+0000","last_change":"2024-09-13T14:40:09.191394+0000","last_active":"2024-09-13T14:44:09.345354+0000","last_peered":"2024-09-13T14:44:09.345354+0000","last_clean":"2024-09-13T14:44:09.345354+0000","last_became_active":"2024-09-13T14:40:09.184697+0000","last_became_peered":"2024-09-13T14:40:09.184697+0000","last_unstale":"2024-09-13T14:44:09.345354+0000","last_undegraded":"2024-09-13T14:44:09.345354+0000","last_fullsized":"2024-09-13T14:44:09.345354+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:31.632367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724494+0000","last_change":"2024-09-13T14:39:34.401131+0000","last_active":"2024-09-13T14:44:10.724494+0000","last_peered":"2024-09-13T14:44:10.724494+0000","last_clean":"2024-09-13T14:44:10.724494+0000","last_became_active":"2024-09-13T14:39:34.393396+0000","last_became_peered":"2024-09-13T14:39:34.393396+0000","last_unstale":"2024-09-13T14:44:10.724494+0000","last_undegraded":"2024-09-13T14:44:10.724494+0000","last_fullsized":"2024-09-13T14:44:10.724494+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:35.475048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397818,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325592,"kb_used_data":15760,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93419560,"statfs":{"total":95995035648,"available":95661629440,"internally_reserved":0,"allocated":16138240,"data_stored":10329470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594053,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":345488,"kb_used_data":15744,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93399664,"statfs":{"total":95995035648,"available":95641255936,"internally_reserved":0,"allocated":16121856,"data_stored":10328713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790287,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362036,"kb_used_data":15160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93383116,"statfs":{"total":95995035648,"available":95624310784,"internally_reserved":0,"allocated":15523840,"data_stored":9734478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980831815,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":79212,"kb_used_data":15176,"kb_used_omap":76,"kb_used_meta":50099,"kb_avail":93665940,"statfs":{"total":95995035648,"available":95913922560,"internally_reserved":0,"allocated":15540224,"data_stored":9739044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":51302208},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911054,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":69504,"kb_used_data":15156,"kb_used_omap":77,"kb_used_meta":54258,"kb_avail":93675648,"statfs":{"total":95995035648,"available":95923863552,"internally_reserved":0,"allocated":15519744,"data_stored":9734252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":55561013},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218022997,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":51460,"kb_used_data":15160,"kb_used_omap":71,"kb_used_meta":36216,"kb_avail":93693692,"statfs":{"total":95995035648,"available":95942340608,"internally_reserved":0,"allocated":15523840,"data_stored":9734774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":37085856},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004127,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":117824,"kb_used_data":15152,"kb_used_omap":66,"kb_used_meta":62397,"kb_avail":93627328,"statfs":{"total":95995035648,"available":95874383872,"internally_reserved":0,"allocated":15515648,"data_stored":9733874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":63894698},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195319,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":129280,"kb_used_data":15736,"kb_used_omap":56,"kb_used_meta":49735,"kb_avail":93615872,"statfs":{"total":95995035648,"available":95862652928,"internally_reserved":0,"allocated":16113664,"data_stored":10323008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":50928783},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:45:56.949 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:45:56.949 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:45:56.949 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:45:56.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:45:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:45:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:45:57] "GET /metrics HTTP/1.1" 200 34268 "" "Prometheus/2.51.0" 2024-09-13T14:45:58.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:57 smithi177 bash[68594]: audit 2024-09-13T14:45:56.021155+0000 mgr.y (mgr.26252) 585 : audit [DBG] from='client.45083 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:58.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:57 smithi177 bash[68594]: audit 2024-09-13T14:45:56.021155+0000 mgr.y (mgr.26252) 585 : audit [DBG] from='client.45083 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:58.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:57 smithi177 bash[68594]: cluster 2024-09-13T14:45:56.443427+0000 mgr.y (mgr.26252) 586 : cluster [DBG] pgmap v1034: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:58.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:57 smithi177 bash[68594]: cluster 2024-09-13T14:45:56.443427+0000 mgr.y (mgr.26252) 586 : cluster [DBG] pgmap v1034: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:58.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:57 smithi099 bash[84576]: audit 2024-09-13T14:45:56.021155+0000 mgr.y (mgr.26252) 585 : audit [DBG] from='client.45083 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:58.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:57 smithi099 bash[84576]: audit 2024-09-13T14:45:56.021155+0000 mgr.y (mgr.26252) 585 : audit [DBG] from='client.45083 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:58.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:57 smithi099 bash[84576]: cluster 2024-09-13T14:45:56.443427+0000 mgr.y (mgr.26252) 586 : cluster [DBG] pgmap v1034: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:58.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:57 smithi099 bash[84576]: cluster 2024-09-13T14:45:56.443427+0000 mgr.y (mgr.26252) 586 : cluster [DBG] pgmap v1034: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:58.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:57 smithi099 bash[87147]: audit 2024-09-13T14:45:56.021155+0000 mgr.y (mgr.26252) 585 : audit [DBG] from='client.45083 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:58.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:57 smithi099 bash[87147]: audit 2024-09-13T14:45:56.021155+0000 mgr.y (mgr.26252) 585 : audit [DBG] from='client.45083 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:58.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:57 smithi099 bash[87147]: cluster 2024-09-13T14:45:56.443427+0000 mgr.y (mgr.26252) 586 : cluster [DBG] pgmap v1034: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:45:58.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:57 smithi099 bash[87147]: cluster 2024-09-13T14:45:56.443427+0000 mgr.y (mgr.26252) 586 : cluster [DBG] pgmap v1034: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:00.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:59 smithi177 bash[68594]: cluster 2024-09-13T14:45:58.444348+0000 mgr.y (mgr.26252) 587 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:00.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:45:59 smithi177 bash[68594]: cluster 2024-09-13T14:45:58.444348+0000 mgr.y (mgr.26252) 587 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:59 smithi099 bash[84576]: cluster 2024-09-13T14:45:58.444348+0000 mgr.y (mgr.26252) 587 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:00.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:45:59 smithi099 bash[84576]: cluster 2024-09-13T14:45:58.444348+0000 mgr.y (mgr.26252) 587 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:00.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:59 smithi099 bash[87147]: cluster 2024-09-13T14:45:58.444348+0000 mgr.y (mgr.26252) 587 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:00.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:45:59 smithi099 bash[87147]: cluster 2024-09-13T14:45:58.444348+0000 mgr.y (mgr.26252) 587 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:00.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:46:00 smithi177 bash[61439]: ts=2024-09-13T14:46:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:01.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:00 smithi099 bash[84576]: cluster 2024-09-13T14:46:00.445304+0000 mgr.y (mgr.26252) 588 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:01.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:00 smithi099 bash[84576]: cluster 2024-09-13T14:46:00.445304+0000 mgr.y (mgr.26252) 588 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:01.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:00 smithi099 bash[87147]: cluster 2024-09-13T14:46:00.445304+0000 mgr.y (mgr.26252) 588 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:01.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:00 smithi099 bash[87147]: cluster 2024-09-13T14:46:00.445304+0000 mgr.y (mgr.26252) 588 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:01.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:00 smithi177 bash[68594]: cluster 2024-09-13T14:46:00.445304+0000 mgr.y (mgr.26252) 588 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:01.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:00 smithi177 bash[68594]: cluster 2024-09-13T14:46:00.445304+0000 mgr.y (mgr.26252) 588 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:01.761 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:03 smithi099 bash[84576]: cluster 2024-09-13T14:46:02.446262+0000 mgr.y (mgr.26252) 589 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:03 smithi099 bash[84576]: cluster 2024-09-13T14:46:02.446262+0000 mgr.y (mgr.26252) 589 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:03 smithi099 bash[84576]: audit 2024-09-13T14:46:03.267826+0000 mon.a (mon.0) 1561 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:03 smithi099 bash[84576]: audit 2024-09-13T14:46:03.267826+0000 mon.a (mon.0) 1561 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:03 smithi099 bash[87147]: cluster 2024-09-13T14:46:02.446262+0000 mgr.y (mgr.26252) 589 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:03 smithi099 bash[87147]: cluster 2024-09-13T14:46:02.446262+0000 mgr.y (mgr.26252) 589 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:03 smithi099 bash[87147]: audit 2024-09-13T14:46:03.267826+0000 mon.a (mon.0) 1561 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:03 smithi099 bash[87147]: audit 2024-09-13T14:46:03.267826+0000 mon.a (mon.0) 1561 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:03 smithi177 bash[68594]: cluster 2024-09-13T14:46:02.446262+0000 mgr.y (mgr.26252) 589 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:03 smithi177 bash[68594]: cluster 2024-09-13T14:46:02.446262+0000 mgr.y (mgr.26252) 589 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:03 smithi177 bash[68594]: audit 2024-09-13T14:46:03.267826+0000 mon.a (mon.0) 1561 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:03 smithi177 bash[68594]: audit 2024-09-13T14:46:03.267826+0000 mon.a (mon.0) 1561 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:05.720 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:46:05.721 INFO:teuthology.orchestra.run.smithi099.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:46:05.732 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:05 smithi099 bash[84576]: cluster 2024-09-13T14:46:04.447266+0000 mgr.y (mgr.26252) 590 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:05.732 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:05 smithi099 bash[84576]: cluster 2024-09-13T14:46:04.447266+0000 mgr.y (mgr.26252) 590 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:05.732 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:05 smithi099 bash[87147]: cluster 2024-09-13T14:46:04.447266+0000 mgr.y (mgr.26252) 590 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:05.732 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:05 smithi099 bash[87147]: cluster 2024-09-13T14:46:04.447266+0000 mgr.y (mgr.26252) 590 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:05.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:05 smithi177 bash[68594]: cluster 2024-09-13T14:46:04.447266+0000 mgr.y (mgr.26252) 590 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:05.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:05 smithi177 bash[68594]: cluster 2024-09-13T14:46:04.447266+0000 mgr.y (mgr.26252) 590 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:06.538 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:46:06.538 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:46:06.549 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-09-13T14:46:06.549 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T14:46:06.562 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T14:46:06.562 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:46:06.562 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-09-13T14:46:06.564 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-13T14:46:06.564 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'wip-aclamk-cbt-combined'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}}, 'workunit': {'branch': 'wip-aclamk-cbt-combined', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}} 2024-09-13T14:46:06.564 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-13T14:46:06.564 INFO:tasks.thrashosds:Beginning thrashosds... 2024-09-13T14:46:06.565 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:46:06.565 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:46:06.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:06 smithi099 bash[84576]: audit 2024-09-13T14:46:05.723628+0000 mon.a (mon.0) 1562 : audit [DBG] from='client.? 172.21.15.99:0/2673843035' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:46:06.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:06 smithi099 bash[84576]: audit 2024-09-13T14:46:05.723628+0000 mon.a (mon.0) 1562 : audit [DBG] from='client.? 172.21.15.99:0/2673843035' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:46:06.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:06 smithi099 bash[87147]: audit 2024-09-13T14:46:05.723628+0000 mon.a (mon.0) 1562 : audit [DBG] from='client.? 172.21.15.99:0/2673843035' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:46:06.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:06 smithi099 bash[87147]: audit 2024-09-13T14:46:05.723628+0000 mon.a (mon.0) 1562 : audit [DBG] from='client.? 172.21.15.99:0/2673843035' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:46:06.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:06 smithi177 bash[68594]: audit 2024-09-13T14:46:05.723628+0000 mon.a (mon.0) 1562 : audit [DBG] from='client.? 172.21.15.99:0/2673843035' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:46:06.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:06 smithi177 bash[68594]: audit 2024-09-13T14:46:05.723628+0000 mon.a (mon.0) 1562 : audit [DBG] from='client.? 172.21.15.99:0/2673843035' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:46:07.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:46:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:46:07] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.51.0" 2024-09-13T14:46:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:07 smithi099 bash[84576]: cluster 2024-09-13T14:46:06.447960+0000 mgr.y (mgr.26252) 591 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:07 smithi099 bash[84576]: cluster 2024-09-13T14:46:06.447960+0000 mgr.y (mgr.26252) 591 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:07.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:07 smithi099 bash[87147]: cluster 2024-09-13T14:46:06.447960+0000 mgr.y (mgr.26252) 591 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:07 smithi099 bash[87147]: cluster 2024-09-13T14:46:06.447960+0000 mgr.y (mgr.26252) 591 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:07.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:07 smithi177 bash[68594]: cluster 2024-09-13T14:46:06.447960+0000 mgr.y (mgr.26252) 591 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:07 smithi177 bash[68594]: cluster 2024-09-13T14:46:06.447960+0000 mgr.y (mgr.26252) 591 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:09 smithi099 bash[84576]: cluster 2024-09-13T14:46:08.449013+0000 mgr.y (mgr.26252) 592 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:09 smithi099 bash[84576]: cluster 2024-09-13T14:46:08.449013+0000 mgr.y (mgr.26252) 592 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:09.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:09 smithi099 bash[87147]: cluster 2024-09-13T14:46:08.449013+0000 mgr.y (mgr.26252) 592 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:09.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:09 smithi099 bash[87147]: cluster 2024-09-13T14:46:08.449013+0000 mgr.y (mgr.26252) 592 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:09.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:09 smithi177 bash[68594]: cluster 2024-09-13T14:46:08.449013+0000 mgr.y (mgr.26252) 592 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:09.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:09 smithi177 bash[68594]: cluster 2024-09-13T14:46:08.449013+0000 mgr.y (mgr.26252) 592 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:10.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:46:10 smithi177 bash[61439]: ts=2024-09-13T14:46:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:11.345 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:11 smithi099 bash[84576]: cluster 2024-09-13T14:46:10.449986+0000 mgr.y (mgr.26252) 593 : cluster [DBG] pgmap v1041: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:11.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:11 smithi099 bash[84576]: cluster 2024-09-13T14:46:10.449986+0000 mgr.y (mgr.26252) 593 : cluster [DBG] pgmap v1041: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:11 smithi099 bash[87147]: cluster 2024-09-13T14:46:10.449986+0000 mgr.y (mgr.26252) 593 : cluster [DBG] pgmap v1041: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:11 smithi099 bash[87147]: cluster 2024-09-13T14:46:10.449986+0000 mgr.y (mgr.26252) 593 : cluster [DBG] pgmap v1041: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:11.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:11 smithi177 bash[68594]: cluster 2024-09-13T14:46:10.449986+0000 mgr.y (mgr.26252) 593 : cluster [DBG] pgmap v1041: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:11.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:11 smithi177 bash[68594]: cluster 2024-09-13T14:46:10.449986+0000 mgr.y (mgr.26252) 593 : cluster [DBG] pgmap v1041: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:13.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:13 smithi099 bash[87147]: cluster 2024-09-13T14:46:12.450887+0000 mgr.y (mgr.26252) 594 : cluster [DBG] pgmap v1042: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:13.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:13 smithi099 bash[87147]: cluster 2024-09-13T14:46:12.450887+0000 mgr.y (mgr.26252) 594 : cluster [DBG] pgmap v1042: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:13 smithi099 bash[84576]: cluster 2024-09-13T14:46:12.450887+0000 mgr.y (mgr.26252) 594 : cluster [DBG] pgmap v1042: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:13.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:13 smithi099 bash[84576]: cluster 2024-09-13T14:46:12.450887+0000 mgr.y (mgr.26252) 594 : cluster [DBG] pgmap v1042: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:13 smithi177 bash[68594]: cluster 2024-09-13T14:46:12.450887+0000 mgr.y (mgr.26252) 594 : cluster [DBG] pgmap v1042: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:13 smithi177 bash[68594]: cluster 2024-09-13T14:46:12.450887+0000 mgr.y (mgr.26252) 594 : cluster [DBG] pgmap v1042: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:15.560 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:46:15.560 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:46:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:15 smithi099 bash[84576]: cluster 2024-09-13T14:46:14.451930+0000 mgr.y (mgr.26252) 595 : cluster [DBG] pgmap v1043: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:15 smithi099 bash[84576]: cluster 2024-09-13T14:46:14.451930+0000 mgr.y (mgr.26252) 595 : cluster [DBG] pgmap v1043: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:15.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:15 smithi099 bash[87147]: cluster 2024-09-13T14:46:14.451930+0000 mgr.y (mgr.26252) 595 : cluster [DBG] pgmap v1043: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:15.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:15 smithi099 bash[87147]: cluster 2024-09-13T14:46:14.451930+0000 mgr.y (mgr.26252) 595 : cluster [DBG] pgmap v1043: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:15 smithi177 bash[68594]: cluster 2024-09-13T14:46:14.451930+0000 mgr.y (mgr.26252) 595 : cluster [DBG] pgmap v1043: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:15 smithi177 bash[68594]: cluster 2024-09-13T14:46:14.451930+0000 mgr.y (mgr.26252) 595 : cluster [DBG] pgmap v1043: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:16.480 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1043,"stamp":"2024-09-13T14:46:14.451222+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":344,"num_read_kb":799,"num_write":212,"num_write_kb":1659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":164,"ondisk_log_size":164,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1480396,"kb_used_data":123044,"kb_used_omap":353,"kb_used_meta":1169694,"kb_avail":748480820,"statfs":{"total":767960285184,"available":766444359680,"internally_reserved":0,"allocated":125997056,"data_stored":79657613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":361898,"internal_metadata":1197767254},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005572"},"pg_stats":[{"pgid":"3.1f","version":"65'5","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313264+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T14:44:10.313264+0000","last_peered":"2024-09-13T14:44:10.313264+0000","last_clean":"2024-09-13T14:44:10.313264+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T14:44:10.313264+0000","last_undegraded":"2024-09-13T14:44:10.313264+0000","last_fullsized":"2024-09-13T14:44:10.313264+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1082,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.301691+0000","last_change":"2024-09-13T14:40:09.139213+0000","last_active":"2024-09-13T14:44:10.301691+0000","last_peered":"2024-09-13T14:44:10.301691+0000","last_clean":"2024-09-13T14:44:10.301691+0000","last_became_active":"2024-09-13T14:40:09.138480+0000","last_became_peered":"2024-09-13T14:40:09.138480+0000","last_unstale":"2024-09-13T14:44:10.301691+0000","last_undegraded":"2024-09-13T14:44:10.301691+0000","last_fullsized":"2024-09-13T14:44:10.301691+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:02:27.534835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1139,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724219+0000","last_change":"2024-09-13T14:39:34.403170+0000","last_active":"2024-09-13T14:44:10.724219+0000","last_peered":"2024-09-13T14:44:10.724219+0000","last_clean":"2024-09-13T14:44:10.724219+0000","last_became_active":"2024-09-13T14:39:34.389001+0000","last_became_peered":"2024-09-13T14:39:34.389001+0000","last_unstale":"2024-09-13T14:44:10.724219+0000","last_undegraded":"2024-09-13T14:44:10.724219+0000","last_fullsized":"2024-09-13T14:44:10.724219+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:38.542452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1152,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.722796+0000","last_change":"2024-09-13T14:39:34.316950+0000","last_active":"2024-09-13T14:44:10.722796+0000","last_peered":"2024-09-13T14:44:10.722796+0000","last_clean":"2024-09-13T14:44:10.722796+0000","last_became_active":"2024-09-13T14:39:34.316614+0000","last_became_peered":"2024-09-13T14:39:34.316614+0000","last_unstale":"2024-09-13T14:44:10.722796+0000","last_undegraded":"2024-09-13T14:44:10.722796+0000","last_fullsized":"2024-09-13T14:44:10.722796+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:20.664989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301239+0000","last_change":"2024-09-13T14:40:09.190077+0000","last_active":"2024-09-13T14:44:09.301239+0000","last_peered":"2024-09-13T14:44:09.301239+0000","last_clean":"2024-09-13T14:44:09.301239+0000","last_became_active":"2024-09-13T14:40:09.189717+0000","last_became_peered":"2024-09-13T14:40:09.189717+0000","last_unstale":"2024-09-13T14:44:09.301239+0000","last_undegraded":"2024-09-13T14:44:09.301239+0000","last_fullsized":"2024-09-13T14:44:09.301239+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:24.201026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303665+0000","last_change":"2024-09-13T14:40:09.219485+0000","last_active":"2024-09-13T14:44:10.303665+0000","last_peered":"2024-09-13T14:44:10.303665+0000","last_clean":"2024-09-13T14:44:10.303665+0000","last_became_active":"2024-09-13T14:40:09.212705+0000","last_became_peered":"2024-09-13T14:40:09.212705+0000","last_unstale":"2024-09-13T14:44:10.303665+0000","last_undegraded":"2024-09-13T14:44:10.303665+0000","last_fullsized":"2024-09-13T14:44:10.303665+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:02.378795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":925,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.302449+0000","last_change":"2024-09-13T14:40:09.192785+0000","last_active":"2024-09-13T14:44:09.302449+0000","last_peered":"2024-09-13T14:44:09.302449+0000","last_clean":"2024-09-13T14:44:09.302449+0000","last_became_active":"2024-09-13T14:40:09.192196+0000","last_became_peered":"2024-09-13T14:40:09.192196+0000","last_unstale":"2024-09-13T14:44:09.302449+0000","last_undegraded":"2024-09-13T14:44:09.302449+0000","last_fullsized":"2024-09-13T14:44:09.302449+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:52.737394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.305451+0000","last_change":"2024-09-13T14:40:09.220295+0000","last_active":"2024-09-13T14:44:10.305451+0000","last_peered":"2024-09-13T14:44:10.305451+0000","last_clean":"2024-09-13T14:44:10.305451+0000","last_became_active":"2024-09-13T14:40:09.209986+0000","last_became_peered":"2024-09-13T14:40:09.209986+0000","last_unstale":"2024-09-13T14:44:10.305451+0000","last_undegraded":"2024-09-13T14:44:10.305451+0000","last_fullsized":"2024-09-13T14:44:10.305451+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:57.153128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1283,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.312105+0000","last_change":"2024-09-13T14:40:09.194263+0000","last_active":"2024-09-13T14:44:10.312105+0000","last_peered":"2024-09-13T14:44:10.312105+0000","last_clean":"2024-09-13T14:44:10.312105+0000","last_became_active":"2024-09-13T14:40:09.192631+0000","last_became_peered":"2024-09-13T14:40:09.192631+0000","last_unstale":"2024-09-13T14:44:10.312105+0000","last_undegraded":"2024-09-13T14:44:10.312105+0000","last_fullsized":"2024-09-13T14:44:10.312105+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:41:31.185137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1186,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.740194+0000","last_change":"2024-09-13T14:39:34.358767+0000","last_active":"2024-09-13T14:44:10.740194+0000","last_peered":"2024-09-13T14:44:10.740194+0000","last_clean":"2024-09-13T14:44:10.740194+0000","last_became_active":"2024-09-13T14:39:34.358244+0000","last_became_peered":"2024-09-13T14:39:34.358244+0000","last_unstale":"2024-09-13T14:44:10.740194+0000","last_undegraded":"2024-09-13T14:44:10.740194+0000","last_fullsized":"2024-09-13T14:44:10.740194+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:39.392859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":1132,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303234+0000","last_change":"2024-09-13T14:40:09.189390+0000","last_active":"2024-09-13T14:44:10.303234+0000","last_peered":"2024-09-13T14:44:10.303234+0000","last_clean":"2024-09-13T14:44:10.303234+0000","last_became_active":"2024-09-13T14:40:09.189042+0000","last_became_peered":"2024-09-13T14:40:09.189042+0000","last_unstale":"2024-09-13T14:44:10.303234+0000","last_undegraded":"2024-09-13T14:44:10.303234+0000","last_fullsized":"2024-09-13T14:44:10.303234+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:05.081276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1158,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724166+0000","last_change":"2024-09-13T14:40:09.227622+0000","last_active":"2024-09-13T14:44:10.724166+0000","last_peered":"2024-09-13T14:44:10.724166+0000","last_clean":"2024-09-13T14:44:10.724166+0000","last_became_active":"2024-09-13T14:40:09.226971+0000","last_became_peered":"2024-09-13T14:40:09.226971+0000","last_unstale":"2024-09-13T14:44:10.724166+0000","last_undegraded":"2024-09-13T14:44:10.724166+0000","last_fullsized":"2024-09-13T14:44:10.724166+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:48:21.955396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.730771+0000","last_change":"2024-09-13T14:40:09.173906+0000","last_active":"2024-09-13T14:44:10.730771+0000","last_peered":"2024-09-13T14:44:10.730771+0000","last_clean":"2024-09-13T14:44:10.730771+0000","last_became_active":"2024-09-13T14:40:09.173303+0000","last_became_peered":"2024-09-13T14:40:09.173303+0000","last_unstale":"2024-09-13T14:44:10.730771+0000","last_undegraded":"2024-09-13T14:44:10.730771+0000","last_fullsized":"2024-09-13T14:44:10.730771+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:34:11.061883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1278,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.477091+0000","last_change":"2024-09-13T14:39:34.362206+0000","last_active":"2024-09-13T14:44:12.477091+0000","last_peered":"2024-09-13T14:44:12.477091+0000","last_clean":"2024-09-13T14:44:12.477091+0000","last_became_active":"2024-09-13T14:39:34.361932+0000","last_became_peered":"2024-09-13T14:39:34.361932+0000","last_unstale":"2024-09-13T14:44:12.477091+0000","last_undegraded":"2024-09-13T14:44:12.477091+0000","last_fullsized":"2024-09-13T14:44:12.477091+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:00.449274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1165,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723274+0000","last_change":"2024-09-13T14:39:34.403479+0000","last_active":"2024-09-13T14:44:10.723274+0000","last_peered":"2024-09-13T14:44:10.723274+0000","last_clean":"2024-09-13T14:44:10.723274+0000","last_became_active":"2024-09-13T14:39:34.386530+0000","last_became_peered":"2024-09-13T14:39:34.386530+0000","last_unstale":"2024-09-13T14:44:10.723274+0000","last_undegraded":"2024-09-13T14:44:10.723274+0000","last_fullsized":"2024-09-13T14:44:10.723274+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:29.559892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487821+0000","last_change":"2024-09-13T14:38:11.206109+0000","last_active":"2024-09-13T14:44:09.487821+0000","last_peered":"2024-09-13T14:44:09.487821+0000","last_clean":"2024-09-13T14:44:09.487821+0000","last_became_active":"2024-09-13T14:38:11.205546+0000","last_became_peered":"2024-09-13T14:38:11.205546+0000","last_unstale":"2024-09-13T14:44:09.487821+0000","last_undegraded":"2024-09-13T14:44:09.487821+0000","last_fullsized":"2024-09-13T14:44:09.487821+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:33.834490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1145,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.722999+0000","last_change":"2024-09-13T14:39:34.368100+0000","last_active":"2024-09-13T14:44:10.722999+0000","last_peered":"2024-09-13T14:44:10.722999+0000","last_clean":"2024-09-13T14:44:10.722999+0000","last_became_active":"2024-09-13T14:39:34.366992+0000","last_became_peered":"2024-09-13T14:39:34.366992+0000","last_unstale":"2024-09-13T14:44:10.722999+0000","last_undegraded":"2024-09-13T14:44:10.722999+0000","last_fullsized":"2024-09-13T14:44:10.722999+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:54.220100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313289+0000","last_change":"2024-09-13T14:39:34.318266+0000","last_active":"2024-09-13T14:44:10.313289+0000","last_peered":"2024-09-13T14:44:10.313289+0000","last_clean":"2024-09-13T14:44:10.313289+0000","last_became_active":"2024-09-13T14:39:34.315973+0000","last_became_peered":"2024-09-13T14:39:34.315973+0000","last_unstale":"2024-09-13T14:44:10.313289+0000","last_undegraded":"2024-09-13T14:44:10.313289+0000","last_fullsized":"2024-09-13T14:44:10.313289+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:15.270282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1124,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723081+0000","last_change":"2024-09-13T14:39:34.338378+0000","last_active":"2024-09-13T14:44:10.723081+0000","last_peered":"2024-09-13T14:44:10.723081+0000","last_clean":"2024-09-13T14:44:10.723081+0000","last_became_active":"2024-09-13T14:39:34.338005+0000","last_became_peered":"2024-09-13T14:39:34.338005+0000","last_unstale":"2024-09-13T14:44:10.723081+0000","last_undegraded":"2024-09-13T14:44:10.723081+0000","last_fullsized":"2024-09-13T14:44:10.723081+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:29.120379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":905,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301800+0000","last_change":"2024-09-13T14:38:11.231165+0000","last_active":"2024-09-13T14:44:09.301800+0000","last_peered":"2024-09-13T14:44:09.301800+0000","last_clean":"2024-09-13T14:44:09.301800+0000","last_became_active":"2024-09-13T14:38:11.230759+0000","last_became_peered":"2024-09-13T14:38:11.230759+0000","last_unstale":"2024-09-13T14:44:09.301800+0000","last_undegraded":"2024-09-13T14:44:09.301800+0000","last_fullsized":"2024-09-13T14:44:09.301800+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:34.222184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":930,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345752+0000","last_change":"2024-09-13T14:40:09.191682+0000","last_active":"2024-09-13T14:44:09.345752+0000","last_peered":"2024-09-13T14:44:09.345752+0000","last_clean":"2024-09-13T14:44:09.345752+0000","last_became_active":"2024-09-13T14:40:09.184791+0000","last_became_peered":"2024-09-13T14:40:09.184791+0000","last_unstale":"2024-09-13T14:44:09.345752+0000","last_undegraded":"2024-09-13T14:44:09.345752+0000","last_fullsized":"2024-09-13T14:44:09.345752+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:56.463093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.733304+0000","last_change":"2024-09-13T14:38:59.370056+0000","last_active":"2024-09-13T14:44:10.733304+0000","last_peered":"2024-09-13T14:44:10.733304+0000","last_clean":"2024-09-13T14:44:10.733304+0000","last_became_active":"2024-09-13T14:38:59.369705+0000","last_became_peered":"2024-09-13T14:38:59.369705+0000","last_unstale":"2024-09-13T14:44:10.733304+0000","last_undegraded":"2024-09-13T14:44:10.733304+0000","last_fullsized":"2024-09-13T14:44:10.733304+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:59.898050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":1125,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303667+0000","last_change":"2024-09-13T14:40:09.180133+0000","last_active":"2024-09-13T14:44:10.303667+0000","last_peered":"2024-09-13T14:44:10.303667+0000","last_clean":"2024-09-13T14:44:10.303667+0000","last_became_active":"2024-09-13T14:40:09.179808+0000","last_became_peered":"2024-09-13T14:40:09.179808+0000","last_unstale":"2024-09-13T14:44:10.303667+0000","last_undegraded":"2024-09-13T14:44:10.303667+0000","last_fullsized":"2024-09-13T14:44:10.303667+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:11.823441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1122,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.306210+0000","last_change":"2024-09-13T14:40:09.190329+0000","last_active":"2024-09-13T14:44:10.306210+0000","last_peered":"2024-09-13T14:44:10.306210+0000","last_clean":"2024-09-13T14:44:10.306210+0000","last_became_active":"2024-09-13T14:40:09.189399+0000","last_became_peered":"2024-09-13T14:40:09.189399+0000","last_unstale":"2024-09-13T14:44:10.306210+0000","last_undegraded":"2024-09-13T14:44:10.306210+0000","last_fullsized":"2024-09-13T14:44:10.306210+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:36.338067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1099,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.304546+0000","last_change":"2024-09-13T14:40:09.202450+0000","last_active":"2024-09-13T14:44:10.304546+0000","last_peered":"2024-09-13T14:44:10.304546+0000","last_clean":"2024-09-13T14:44:10.304546+0000","last_became_active":"2024-09-13T14:40:09.201651+0000","last_became_peered":"2024-09-13T14:40:09.201651+0000","last_unstale":"2024-09-13T14:44:10.304546+0000","last_undegraded":"2024-09-13T14:44:10.304546+0000","last_fullsized":"2024-09-13T14:44:10.304546+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:12.544442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":921,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487923+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T14:44:09.487923+0000","last_peered":"2024-09-13T14:44:09.487923+0000","last_clean":"2024-09-13T14:44:09.487923+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T14:44:09.487923+0000","last_undegraded":"2024-09-13T14:44:09.487923+0000","last_fullsized":"2024-09-13T14:44:09.487923+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1293,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.315290+0000","last_change":"2024-09-13T14:40:09.187749+0000","last_active":"2024-09-13T14:44:10.315290+0000","last_peered":"2024-09-13T14:44:10.315290+0000","last_clean":"2024-09-13T14:44:10.315290+0000","last_became_active":"2024-09-13T14:40:09.187007+0000","last_became_peered":"2024-09-13T14:40:09.187007+0000","last_unstale":"2024-09-13T14:44:10.315290+0000","last_undegraded":"2024-09-13T14:44:10.315290+0000","last_fullsized":"2024-09-13T14:44:10.315290+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:08.019463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1122,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723614+0000","last_change":"2024-09-13T14:39:34.349945+0000","last_active":"2024-09-13T14:44:10.723614+0000","last_peered":"2024-09-13T14:44:10.723614+0000","last_clean":"2024-09-13T14:44:10.723614+0000","last_became_active":"2024-09-13T14:39:34.346730+0000","last_became_peered":"2024-09-13T14:39:34.346730+0000","last_unstale":"2024-09-13T14:44:10.723614+0000","last_undegraded":"2024-09-13T14:44:10.723614+0000","last_fullsized":"2024-09-13T14:44:10.723614+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:48.854603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1234,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729597+0000","last_change":"2024-09-13T14:40:09.177544+0000","last_active":"2024-09-13T14:44:10.729597+0000","last_peered":"2024-09-13T14:44:10.729597+0000","last_clean":"2024-09-13T14:44:10.729597+0000","last_became_active":"2024-09-13T14:40:09.177252+0000","last_became_peered":"2024-09-13T14:40:09.177252+0000","last_unstale":"2024-09-13T14:44:10.729597+0000","last_undegraded":"2024-09-13T14:44:10.729597+0000","last_fullsized":"2024-09-13T14:44:10.729597+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:45:50.539379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1126,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724938+0000","last_change":"2024-09-13T14:39:34.401432+0000","last_active":"2024-09-13T14:44:10.724938+0000","last_peered":"2024-09-13T14:44:10.724938+0000","last_clean":"2024-09-13T14:44:10.724938+0000","last_became_active":"2024-09-13T14:39:34.395458+0000","last_became_peered":"2024-09-13T14:39:34.395458+0000","last_unstale":"2024-09-13T14:44:10.724938+0000","last_undegraded":"2024-09-13T14:44:10.724938+0000","last_fullsized":"2024-09-13T14:44:10.724938+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:17.007388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1266,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.311205+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T14:44:10.311205+0000","last_peered":"2024-09-13T14:44:10.311205+0000","last_clean":"2024-09-13T14:44:10.311205+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T14:44:10.311205+0000","last_undegraded":"2024-09-13T14:44:10.311205+0000","last_fullsized":"2024-09-13T14:44:10.311205+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":937,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345711+0000","last_change":"2024-09-13T14:40:09.185516+0000","last_active":"2024-09-13T14:44:09.345711+0000","last_peered":"2024-09-13T14:44:09.345711+0000","last_clean":"2024-09-13T14:44:09.345711+0000","last_became_active":"2024-09-13T14:40:09.184652+0000","last_became_peered":"2024-09-13T14:40:09.184652+0000","last_unstale":"2024-09-13T14:44:09.345711+0000","last_undegraded":"2024-09-13T14:44:09.345711+0000","last_fullsized":"2024-09-13T14:44:09.345711+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:48.492776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729090+0000","last_change":"2024-09-13T14:38:59.328518+0000","last_active":"2024-09-13T14:44:10.729090+0000","last_peered":"2024-09-13T14:44:10.729090+0000","last_clean":"2024-09-13T14:44:10.729090+0000","last_became_active":"2024-09-13T14:38:59.326010+0000","last_became_peered":"2024-09-13T14:38:59.326010+0000","last_unstale":"2024-09-13T14:44:10.729090+0000","last_undegraded":"2024-09-13T14:44:10.729090+0000","last_fullsized":"2024-09-13T14:44:10.729090+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:01.198043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.733416+0000","last_change":"2024-09-13T14:40:09.160901+0000","last_active":"2024-09-13T14:44:10.733416+0000","last_peered":"2024-09-13T14:44:10.733416+0000","last_clean":"2024-09-13T14:44:10.733416+0000","last_became_active":"2024-09-13T14:40:09.160278+0000","last_became_peered":"2024-09-13T14:40:09.160278+0000","last_unstale":"2024-09-13T14:44:10.733416+0000","last_undegraded":"2024-09-13T14:44:10.733416+0000","last_fullsized":"2024-09-13T14:44:10.733416+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:48:56.238297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.728934+0000","last_change":"2024-09-13T14:38:11.234481+0000","last_active":"2024-09-13T14:44:10.728934+0000","last_peered":"2024-09-13T14:44:10.728934+0000","last_clean":"2024-09-13T14:44:10.728934+0000","last_became_active":"2024-09-13T14:38:11.232868+0000","last_became_peered":"2024-09-13T14:38:11.232868+0000","last_unstale":"2024-09-13T14:44:10.728934+0000","last_undegraded":"2024-09-13T14:44:10.728934+0000","last_fullsized":"2024-09-13T14:44:10.728934+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:06:01.704192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":922,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301454+0000","last_change":"2024-09-13T14:39:34.321388+0000","last_active":"2024-09-13T14:44:09.301454+0000","last_peered":"2024-09-13T14:44:09.301454+0000","last_clean":"2024-09-13T14:44:09.301454+0000","last_became_active":"2024-09-13T14:39:34.321025+0000","last_became_peered":"2024-09-13T14:39:34.321025+0000","last_unstale":"2024-09-13T14:44:09.301454+0000","last_undegraded":"2024-09-13T14:44:09.301454+0000","last_fullsized":"2024-09-13T14:44:09.301454+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:52.593825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729257+0000","last_change":"2024-09-13T14:39:34.378359+0000","last_active":"2024-09-13T14:44:10.729257+0000","last_peered":"2024-09-13T14:44:10.729257+0000","last_clean":"2024-09-13T14:44:10.729257+0000","last_became_active":"2024-09-13T14:39:34.377967+0000","last_became_peered":"2024-09-13T14:39:34.377967+0000","last_unstale":"2024-09-13T14:44:10.729257+0000","last_undegraded":"2024-09-13T14:44:10.729257+0000","last_fullsized":"2024-09-13T14:44:10.729257+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:37.645170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1098,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.308015+0000","last_change":"2024-09-13T14:40:09.222631+0000","last_active":"2024-09-13T14:44:10.308015+0000","last_peered":"2024-09-13T14:44:10.308015+0000","last_clean":"2024-09-13T14:44:10.308015+0000","last_became_active":"2024-09-13T14:40:09.211300+0000","last_became_peered":"2024-09-13T14:40:09.211300+0000","last_unstale":"2024-09-13T14:44:10.308015+0000","last_undegraded":"2024-09-13T14:44:10.308015+0000","last_fullsized":"2024-09-13T14:44:10.308015+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:56.913336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":919,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.299732+0000","last_change":"2024-09-13T14:38:11.232633+0000","last_active":"2024-09-13T14:44:09.299732+0000","last_peered":"2024-09-13T14:44:09.299732+0000","last_clean":"2024-09-13T14:44:09.299732+0000","last_became_active":"2024-09-13T14:38:11.230948+0000","last_became_peered":"2024-09-13T14:38:11.230948+0000","last_unstale":"2024-09-13T14:44:09.299732+0000","last_undegraded":"2024-09-13T14:44:09.299732+0000","last_fullsized":"2024-09-13T14:44:09.299732+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:38.665278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.736730+0000","last_change":"2024-09-13T14:38:59.402033+0000","last_active":"2024-09-13T14:44:10.736730+0000","last_peered":"2024-09-13T14:44:10.736730+0000","last_clean":"2024-09-13T14:44:10.736730+0000","last_became_active":"2024-09-13T14:38:59.401826+0000","last_became_peered":"2024-09-13T14:38:59.401826+0000","last_unstale":"2024-09-13T14:44:10.736730+0000","last_undegraded":"2024-09-13T14:44:10.736730+0000","last_fullsized":"2024-09-13T14:44:10.736730+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:17.785114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1225,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.727900+0000","last_change":"2024-09-13T14:38:59.329634+0000","last_active":"2024-09-13T14:44:10.727900+0000","last_peered":"2024-09-13T14:44:10.727900+0000","last_clean":"2024-09-13T14:44:10.727900+0000","last_became_active":"2024-09-13T14:38:59.328813+0000","last_became_peered":"2024-09-13T14:38:59.328813+0000","last_unstale":"2024-09-13T14:44:10.727900+0000","last_undegraded":"2024-09-13T14:44:10.727900+0000","last_fullsized":"2024-09-13T14:44:10.727900+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:35.653113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1175,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.737951+0000","last_change":"2024-09-13T14:39:34.322533+0000","last_active":"2024-09-13T14:44:10.737951+0000","last_peered":"2024-09-13T14:44:10.737951+0000","last_clean":"2024-09-13T14:44:10.737951+0000","last_became_active":"2024-09-13T14:39:34.322070+0000","last_became_peered":"2024-09-13T14:39:34.322070+0000","last_unstale":"2024-09-13T14:44:10.737951+0000","last_undegraded":"2024-09-13T14:44:10.737951+0000","last_fullsized":"2024-09-13T14:44:10.737951+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:43.746420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314327+0000","last_change":"2024-09-13T14:38:59.338775+0000","last_active":"2024-09-13T14:44:10.314327+0000","last_peered":"2024-09-13T14:44:10.314327+0000","last_clean":"2024-09-13T14:44:10.314327+0000","last_became_active":"2024-09-13T14:38:59.338265+0000","last_became_peered":"2024-09-13T14:38:59.338265+0000","last_unstale":"2024-09-13T14:44:10.314327+0000","last_undegraded":"2024-09-13T14:44:10.314327+0000","last_fullsized":"2024-09-13T14:44:10.314327+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:48.255980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313649+0000","last_change":"2024-09-13T14:40:09.188286+0000","last_active":"2024-09-13T14:44:10.313649+0000","last_peered":"2024-09-13T14:44:10.313649+0000","last_clean":"2024-09-13T14:44:10.313649+0000","last_became_active":"2024-09-13T14:40:09.186778+0000","last_became_peered":"2024-09-13T14:40:09.186778+0000","last_unstale":"2024-09-13T14:44:10.313649+0000","last_undegraded":"2024-09-13T14:44:10.313649+0000","last_fullsized":"2024-09-13T14:44:10.313649+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:44:24.275819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":937,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345163+0000","last_change":"2024-09-13T14:38:11.178130+0000","last_active":"2024-09-13T14:44:09.345163+0000","last_peered":"2024-09-13T14:44:09.345163+0000","last_clean":"2024-09-13T14:44:09.345163+0000","last_became_active":"2024-09-13T14:38:11.177484+0000","last_became_peered":"2024-09-13T14:38:11.177484+0000","last_unstale":"2024-09-13T14:44:09.345163+0000","last_undegraded":"2024-09-13T14:44:09.345163+0000","last_fullsized":"2024-09-13T14:44:09.345163+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:18:24.776012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487609+0000","last_change":"2024-09-13T14:40:09.183716+0000","last_active":"2024-09-13T14:44:09.487609+0000","last_peered":"2024-09-13T14:44:09.487609+0000","last_clean":"2024-09-13T14:44:09.487609+0000","last_became_active":"2024-09-13T14:40:09.183420+0000","last_became_peered":"2024-09-13T14:40:09.183420+0000","last_unstale":"2024-09-13T14:44:09.487609+0000","last_undegraded":"2024-09-13T14:44:09.487609+0000","last_fullsized":"2024-09-13T14:44:09.487609+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:03.929337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1124,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723784+0000","last_change":"2024-09-13T14:39:34.356129+0000","last_active":"2024-09-13T14:44:10.723784+0000","last_peered":"2024-09-13T14:44:10.723784+0000","last_clean":"2024-09-13T14:44:10.723784+0000","last_became_active":"2024-09-13T14:39:34.355288+0000","last_became_peered":"2024-09-13T14:39:34.355288+0000","last_unstale":"2024-09-13T14:44:10.723784+0000","last_undegraded":"2024-09-13T14:44:10.723784+0000","last_fullsized":"2024-09-13T14:44:10.723784+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.725003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":918,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.300306+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T14:44:09.300306+0000","last_peered":"2024-09-13T14:44:09.300306+0000","last_clean":"2024-09-13T14:44:09.300306+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T14:44:09.300306+0000","last_undegraded":"2024-09-13T14:44:09.300306+0000","last_fullsized":"2024-09-13T14:44:09.300306+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1025,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345613+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T14:44:09.345613+0000","last_peered":"2024-09-13T14:44:09.345613+0000","last_clean":"2024-09-13T14:44:09.345613+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T14:44:09.345613+0000","last_undegraded":"2024-09-13T14:44:09.345613+0000","last_fullsized":"2024-09-13T14:44:09.345613+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.728956+0000","last_change":"2024-09-13T14:40:09.174566+0000","last_active":"2024-09-13T14:44:10.728956+0000","last_peered":"2024-09-13T14:44:10.728956+0000","last_clean":"2024-09-13T14:44:10.728956+0000","last_became_active":"2024-09-13T14:40:09.173698+0000","last_became_peered":"2024-09-13T14:40:09.173698+0000","last_unstale":"2024-09-13T14:44:10.728956+0000","last_undegraded":"2024-09-13T14:44:10.728956+0000","last_fullsized":"2024-09-13T14:44:10.728956+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:22.408868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1149,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723899+0000","last_change":"2024-09-13T14:39:34.403656+0000","last_active":"2024-09-13T14:44:10.723899+0000","last_peered":"2024-09-13T14:44:10.723899+0000","last_clean":"2024-09-13T14:44:10.723899+0000","last_became_active":"2024-09-13T14:39:34.386890+0000","last_became_peered":"2024-09-13T14:39:34.386890+0000","last_unstale":"2024-09-13T14:44:10.723899+0000","last_undegraded":"2024-09-13T14:44:10.723899+0000","last_fullsized":"2024-09-13T14:44:10.723899+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:24.563109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1082,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.307149+0000","last_change":"2024-09-13T14:40:09.182955+0000","last_active":"2024-09-13T14:44:10.307149+0000","last_peered":"2024-09-13T14:44:10.307149+0000","last_clean":"2024-09-13T14:44:10.307149+0000","last_became_active":"2024-09-13T14:40:09.182702+0000","last_became_peered":"2024-09-13T14:40:09.182702+0000","last_unstale":"2024-09-13T14:44:10.307149+0000","last_undegraded":"2024-09-13T14:44:10.307149+0000","last_fullsized":"2024-09-13T14:44:10.307149+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:00:15.207705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1113,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303286+0000","last_change":"2024-09-13T14:40:09.185241+0000","last_active":"2024-09-13T14:44:10.303286+0000","last_peered":"2024-09-13T14:44:10.303286+0000","last_clean":"2024-09-13T14:44:10.303286+0000","last_became_active":"2024-09-13T14:40:09.184259+0000","last_became_peered":"2024-09-13T14:40:09.184259+0000","last_unstale":"2024-09-13T14:44:10.303286+0000","last_undegraded":"2024-09-13T14:44:10.303286+0000","last_fullsized":"2024-09-13T14:44:10.303286+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:27:17.762953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1129,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.304287+0000","last_change":"2024-09-13T14:40:09.200853+0000","last_active":"2024-09-13T14:44:10.304287+0000","last_peered":"2024-09-13T14:44:10.304287+0000","last_clean":"2024-09-13T14:44:10.304287+0000","last_became_active":"2024-09-13T14:40:09.195505+0000","last_became_peered":"2024-09-13T14:40:09.195505+0000","last_unstale":"2024-09-13T14:44:10.304287+0000","last_undegraded":"2024-09-13T14:44:10.304287+0000","last_fullsized":"2024-09-13T14:44:10.304287+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:28.009085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":926,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.486881+0000","last_change":"2024-09-13T14:40:09.163144+0000","last_active":"2024-09-13T14:44:09.486881+0000","last_peered":"2024-09-13T14:44:09.486881+0000","last_clean":"2024-09-13T14:44:09.486881+0000","last_became_active":"2024-09-13T14:40:09.162863+0000","last_became_peered":"2024-09-13T14:40:09.162863+0000","last_unstale":"2024-09-13T14:44:09.486881+0000","last_undegraded":"2024-09-13T14:44:09.486881+0000","last_fullsized":"2024-09-13T14:44:09.486881+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:48.398889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1104,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303932+0000","last_change":"2024-09-13T14:40:09.223911+0000","last_active":"2024-09-13T14:44:10.303932+0000","last_peered":"2024-09-13T14:44:10.303932+0000","last_clean":"2024-09-13T14:44:10.303932+0000","last_became_active":"2024-09-13T14:40:09.215480+0000","last_became_peered":"2024-09-13T14:40:09.215480+0000","last_unstale":"2024-09-13T14:44:10.303932+0000","last_undegraded":"2024-09-13T14:44:10.303932+0000","last_fullsized":"2024-09-13T14:44:10.303932+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.613398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1137,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723407+0000","last_change":"2024-09-13T14:40:09.208549+0000","last_active":"2024-09-13T14:44:10.723407+0000","last_peered":"2024-09-13T14:44:10.723407+0000","last_clean":"2024-09-13T14:44:10.723407+0000","last_became_active":"2024-09-13T14:40:09.208174+0000","last_became_peered":"2024-09-13T14:40:09.208174+0000","last_unstale":"2024-09-13T14:44:10.723407+0000","last_undegraded":"2024-09-13T14:44:10.723407+0000","last_fullsized":"2024-09-13T14:44:10.723407+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:18.971929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.311474+0000","last_change":"2024-09-13T14:40:09.192856+0000","last_active":"2024-09-13T14:44:10.311474+0000","last_peered":"2024-09-13T14:44:10.311474+0000","last_clean":"2024-09-13T14:44:10.311474+0000","last_became_active":"2024-09-13T14:40:09.186733+0000","last_became_peered":"2024-09-13T14:40:09.186733+0000","last_unstale":"2024-09-13T14:44:10.311474+0000","last_undegraded":"2024-09-13T14:44:10.311474+0000","last_fullsized":"2024-09-13T14:44:10.311474+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:55.336998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":905,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.302232+0000","last_change":"2024-09-13T14:39:34.393673+0000","last_active":"2024-09-13T14:44:09.302232+0000","last_peered":"2024-09-13T14:44:09.302232+0000","last_clean":"2024-09-13T14:44:09.302232+0000","last_became_active":"2024-09-13T14:39:34.393186+0000","last_became_peered":"2024-09-13T14:39:34.393186+0000","last_unstale":"2024-09-13T14:44:09.302232+0000","last_undegraded":"2024-09-13T14:44:09.302232+0000","last_fullsized":"2024-09-13T14:44:09.302232+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:23.130084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.486735+0000","last_change":"2024-09-13T14:38:11.219076+0000","last_active":"2024-09-13T14:44:09.486735+0000","last_peered":"2024-09-13T14:44:09.486735+0000","last_clean":"2024-09-13T14:44:09.486735+0000","last_became_active":"2024-09-13T14:38:11.217912+0000","last_became_peered":"2024-09-13T14:38:11.217912+0000","last_unstale":"2024-09-13T14:44:09.486735+0000","last_undegraded":"2024-09-13T14:44:09.486735+0000","last_fullsized":"2024-09-13T14:44:09.486735+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:32.673534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":893,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.344340+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:44:09.344340+0000","last_peered":"2024-09-13T14:44:09.344340+0000","last_clean":"2024-09-13T14:44:09.344340+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:44:09.344340+0000","last_undegraded":"2024-09-13T14:44:09.344340+0000","last_fullsized":"2024-09-13T14:44:09.344340+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.302845+0000","last_change":"2024-09-13T14:40:09.211963+0000","last_active":"2024-09-13T14:44:10.302845+0000","last_peered":"2024-09-13T14:44:10.302845+0000","last_clean":"2024-09-13T14:44:10.302845+0000","last_became_active":"2024-09-13T14:40:09.205704+0000","last_became_peered":"2024-09-13T14:40:09.205704+0000","last_unstale":"2024-09-13T14:44:10.302845+0000","last_undegraded":"2024-09-13T14:44:10.302845+0000","last_fullsized":"2024-09-13T14:44:10.302845+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:18.704369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":928,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345857+0000","last_change":"2024-09-13T14:40:09.193142+0000","last_active":"2024-09-13T14:44:09.345857+0000","last_peered":"2024-09-13T14:44:09.345857+0000","last_clean":"2024-09-13T14:44:09.345857+0000","last_became_active":"2024-09-13T14:40:09.191806+0000","last_became_peered":"2024-09-13T14:40:09.191806+0000","last_unstale":"2024-09-13T14:44:09.345857+0000","last_undegraded":"2024-09-13T14:44:09.345857+0000","last_fullsized":"2024-09-13T14:44:09.345857+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:41:20.972768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":887,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.299536+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:44:09.299536+0000","last_peered":"2024-09-13T14:44:09.299536+0000","last_clean":"2024-09-13T14:44:09.299536+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:44:09.299536+0000","last_undegraded":"2024-09-13T14:44:09.299536+0000","last_fullsized":"2024-09-13T14:44:09.299536+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":927,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345967+0000","last_change":"2024-09-13T14:40:09.191167+0000","last_active":"2024-09-13T14:44:09.345967+0000","last_peered":"2024-09-13T14:44:09.345967+0000","last_clean":"2024-09-13T14:44:09.345967+0000","last_became_active":"2024-09-13T14:40:09.185797+0000","last_became_peered":"2024-09-13T14:40:09.185797+0000","last_unstale":"2024-09-13T14:44:09.345967+0000","last_undegraded":"2024-09-13T14:44:09.345967+0000","last_fullsized":"2024-09-13T14:44:09.345967+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:52.157413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":921,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.485897+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T14:44:09.485897+0000","last_peered":"2024-09-13T14:44:09.485897+0000","last_clean":"2024-09-13T14:44:09.485897+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T14:44:09.485897+0000","last_undegraded":"2024-09-13T14:44:09.485897+0000","last_fullsized":"2024-09-13T14:44:09.485897+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724498+0000","last_change":"2024-09-13T14:39:34.395859+0000","last_active":"2024-09-13T14:44:10.724498+0000","last_peered":"2024-09-13T14:44:10.724498+0000","last_clean":"2024-09-13T14:44:10.724498+0000","last_became_active":"2024-09-13T14:39:34.389481+0000","last_became_peered":"2024-09-13T14:39:34.389481+0000","last_unstale":"2024-09-13T14:44:10.724498+0000","last_undegraded":"2024-09-13T14:44:10.724498+0000","last_fullsized":"2024-09-13T14:44:10.724498+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.724694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.344937+0000","last_change":"2024-09-13T14:40:09.173284+0000","last_active":"2024-09-13T14:44:09.344937+0000","last_peered":"2024-09-13T14:44:09.344937+0000","last_clean":"2024-09-13T14:44:09.344937+0000","last_became_active":"2024-09-13T14:40:09.171846+0000","last_became_peered":"2024-09-13T14:40:09.171846+0000","last_unstale":"2024-09-13T14:44:09.344937+0000","last_undegraded":"2024-09-13T14:44:09.344937+0000","last_fullsized":"2024-09-13T14:44:09.344937+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:51:35.460440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314998+0000","last_change":"2024-09-13T14:39:34.362694+0000","last_active":"2024-09-13T14:44:10.314998+0000","last_peered":"2024-09-13T14:44:10.314998+0000","last_clean":"2024-09-13T14:44:10.314998+0000","last_became_active":"2024-09-13T14:39:34.362435+0000","last_became_peered":"2024-09-13T14:39:34.362435+0000","last_unstale":"2024-09-13T14:44:10.314998+0000","last_undegraded":"2024-09-13T14:44:10.314998+0000","last_fullsized":"2024-09-13T14:44:10.314998+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:33.350328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314791+0000","last_change":"2024-09-13T14:39:34.360941+0000","last_active":"2024-09-13T14:44:10.314791+0000","last_peered":"2024-09-13T14:44:10.314791+0000","last_clean":"2024-09-13T14:44:10.314791+0000","last_became_active":"2024-09-13T14:39:34.360383+0000","last_became_peered":"2024-09-13T14:39:34.360383+0000","last_unstale":"2024-09-13T14:44:10.314791+0000","last_undegraded":"2024-09-13T14:44:10.314791+0000","last_fullsized":"2024-09-13T14:44:10.314791+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:51.344296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":904,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301001+0000","last_change":"2024-09-13T14:39:34.380634+0000","last_active":"2024-09-13T14:44:09.301001+0000","last_peered":"2024-09-13T14:44:09.301001+0000","last_clean":"2024-09-13T14:44:09.301001+0000","last_became_active":"2024-09-13T14:39:34.379785+0000","last_became_peered":"2024-09-13T14:39:34.379785+0000","last_unstale":"2024-09-13T14:44:09.301001+0000","last_undegraded":"2024-09-13T14:44:09.301001+0000","last_fullsized":"2024-09-13T14:44:09.301001+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:02.436193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":929,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345354+0000","last_change":"2024-09-13T14:40:09.191394+0000","last_active":"2024-09-13T14:44:09.345354+0000","last_peered":"2024-09-13T14:44:09.345354+0000","last_clean":"2024-09-13T14:44:09.345354+0000","last_became_active":"2024-09-13T14:40:09.184697+0000","last_became_peered":"2024-09-13T14:40:09.184697+0000","last_unstale":"2024-09-13T14:44:09.345354+0000","last_undegraded":"2024-09-13T14:44:09.345354+0000","last_fullsized":"2024-09-13T14:44:09.345354+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:31.632367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724494+0000","last_change":"2024-09-13T14:39:34.401131+0000","last_active":"2024-09-13T14:44:10.724494+0000","last_peered":"2024-09-13T14:44:10.724494+0000","last_clean":"2024-09-13T14:44:10.724494+0000","last_became_active":"2024-09-13T14:39:34.393396+0000","last_became_peered":"2024-09-13T14:39:34.393396+0000","last_unstale":"2024-09-13T14:44:10.724494+0000","last_undegraded":"2024-09-13T14:44:10.724494+0000","last_fullsized":"2024-09-13T14:44:10.724494+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:35.475048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397822,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325592,"kb_used_data":15760,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93419560,"statfs":{"total":95995035648,"available":95661629440,"internally_reserved":0,"allocated":16138240,"data_stored":10329470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594057,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":345488,"kb_used_data":15744,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93399664,"statfs":{"total":95995035648,"available":95641255936,"internally_reserved":0,"allocated":16121856,"data_stored":10328713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790291,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362036,"kb_used_data":15160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93383116,"statfs":{"total":95995035648,"available":95624310784,"internally_reserved":0,"allocated":15523840,"data_stored":9734478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980831819,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":79212,"kb_used_data":15176,"kb_used_omap":76,"kb_used_meta":50099,"kb_avail":93665940,"statfs":{"total":95995035648,"available":95913922560,"internally_reserved":0,"allocated":15540224,"data_stored":9739044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":51302208},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911058,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":69504,"kb_used_data":15156,"kb_used_omap":77,"kb_used_meta":54258,"kb_avail":93675648,"statfs":{"total":95995035648,"available":95923863552,"internally_reserved":0,"allocated":15519744,"data_stored":9734252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":55561013},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023001,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":51460,"kb_used_data":15160,"kb_used_omap":71,"kb_used_meta":36216,"kb_avail":93693692,"statfs":{"total":95995035648,"available":95942340608,"internally_reserved":0,"allocated":15523840,"data_stored":9734774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":37085856},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004131,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":117824,"kb_used_data":15152,"kb_used_omap":66,"kb_used_meta":62397,"kb_avail":93627328,"statfs":{"total":95995035648,"available":95874383872,"internally_reserved":0,"allocated":15515648,"data_stored":9733874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":63894698},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195323,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":129280,"kb_used_data":15736,"kb_used_omap":56,"kb_used_meta":49735,"kb_avail":93615872,"statfs":{"total":95995035648,"available":95862652928,"internally_reserved":0,"allocated":16113664,"data_stored":10323008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":50928783},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:46:16.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:46:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:16 smithi099 bash[84576]: audit 2024-09-13T14:46:15.555325+0000 mgr.y (mgr.26252) 596 : audit [DBG] from='client.35199 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:16 smithi099 bash[84576]: audit 2024-09-13T14:46:15.555325+0000 mgr.y (mgr.26252) 596 : audit [DBG] from='client.35199 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:16.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:16 smithi099 bash[87147]: audit 2024-09-13T14:46:15.555325+0000 mgr.y (mgr.26252) 596 : audit [DBG] from='client.35199 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:16.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:16 smithi099 bash[87147]: audit 2024-09-13T14:46:15.555325+0000 mgr.y (mgr.26252) 596 : audit [DBG] from='client.35199 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:16.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:16 smithi177 bash[68594]: audit 2024-09-13T14:46:15.555325+0000 mgr.y (mgr.26252) 596 : audit [DBG] from='client.35199 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:16.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:16 smithi177 bash[68594]: audit 2024-09-13T14:46:15.555325+0000 mgr.y (mgr.26252) 596 : audit [DBG] from='client.35199 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:46:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:46:17] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.51.0" 2024-09-13T14:46:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:17 smithi099 bash[84576]: cluster 2024-09-13T14:46:16.452706+0000 mgr.y (mgr.26252) 597 : cluster [DBG] pgmap v1044: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:17 smithi099 bash[84576]: cluster 2024-09-13T14:46:16.452706+0000 mgr.y (mgr.26252) 597 : cluster [DBG] pgmap v1044: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:17 smithi099 bash[87147]: cluster 2024-09-13T14:46:16.452706+0000 mgr.y (mgr.26252) 597 : cluster [DBG] pgmap v1044: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:17 smithi099 bash[87147]: cluster 2024-09-13T14:46:16.452706+0000 mgr.y (mgr.26252) 597 : cluster [DBG] pgmap v1044: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:17 smithi177 bash[68594]: cluster 2024-09-13T14:46:16.452706+0000 mgr.y (mgr.26252) 597 : cluster [DBG] pgmap v1044: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:17 smithi177 bash[68594]: cluster 2024-09-13T14:46:16.452706+0000 mgr.y (mgr.26252) 597 : cluster [DBG] pgmap v1044: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:18 smithi099 bash[84576]: audit 2024-09-13T14:46:18.268693+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:18 smithi099 bash[84576]: audit 2024-09-13T14:46:18.268693+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:18.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:18 smithi099 bash[87147]: audit 2024-09-13T14:46:18.268693+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:18 smithi099 bash[87147]: audit 2024-09-13T14:46:18.268693+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:18 smithi177 bash[68594]: audit 2024-09-13T14:46:18.268693+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:18 smithi177 bash[68594]: audit 2024-09-13T14:46:18.268693+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:19 smithi099 bash[84576]: cluster 2024-09-13T14:46:18.453659+0000 mgr.y (mgr.26252) 598 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:19 smithi099 bash[84576]: cluster 2024-09-13T14:46:18.453659+0000 mgr.y (mgr.26252) 598 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:19.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:19 smithi099 bash[87147]: cluster 2024-09-13T14:46:18.453659+0000 mgr.y (mgr.26252) 598 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:19.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:19 smithi099 bash[87147]: cluster 2024-09-13T14:46:18.453659+0000 mgr.y (mgr.26252) 598 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:19.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:19 smithi177 bash[68594]: cluster 2024-09-13T14:46:18.453659+0000 mgr.y (mgr.26252) 598 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:19.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:19 smithi177 bash[68594]: cluster 2024-09-13T14:46:18.453659+0000 mgr.y (mgr.26252) 598 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:20.619 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:46:20 smithi177 bash[61439]: ts=2024-09-13T14:46:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:21.274 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:21 smithi099 bash[84576]: cluster 2024-09-13T14:46:20.454538+0000 mgr.y (mgr.26252) 599 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:21 smithi099 bash[84576]: cluster 2024-09-13T14:46:20.454538+0000 mgr.y (mgr.26252) 599 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:21 smithi099 bash[87147]: cluster 2024-09-13T14:46:20.454538+0000 mgr.y (mgr.26252) 599 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:21 smithi099 bash[87147]: cluster 2024-09-13T14:46:20.454538+0000 mgr.y (mgr.26252) 599 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:21.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:21 smithi177 bash[68594]: cluster 2024-09-13T14:46:20.454538+0000 mgr.y (mgr.26252) 599 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:21.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:21 smithi177 bash[68594]: cluster 2024-09-13T14:46:20.454538+0000 mgr.y (mgr.26252) 599 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:23.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:23 smithi099 bash[87147]: cluster 2024-09-13T14:46:22.455414+0000 mgr.y (mgr.26252) 600 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:23.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:23 smithi099 bash[87147]: cluster 2024-09-13T14:46:22.455414+0000 mgr.y (mgr.26252) 600 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:23 smithi099 bash[84576]: cluster 2024-09-13T14:46:22.455414+0000 mgr.y (mgr.26252) 600 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:23 smithi099 bash[84576]: cluster 2024-09-13T14:46:22.455414+0000 mgr.y (mgr.26252) 600 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:23.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:23 smithi177 bash[68594]: cluster 2024-09-13T14:46:22.455414+0000 mgr.y (mgr.26252) 600 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:23.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:23 smithi177 bash[68594]: cluster 2024-09-13T14:46:22.455414+0000 mgr.y (mgr.26252) 600 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:25.511 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:46:25.512 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:46:25.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:25 smithi099 bash[84576]: cluster 2024-09-13T14:46:24.456413+0000 mgr.y (mgr.26252) 601 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:25.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:25 smithi099 bash[84576]: cluster 2024-09-13T14:46:24.456413+0000 mgr.y (mgr.26252) 601 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:25.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:25 smithi099 bash[87147]: cluster 2024-09-13T14:46:24.456413+0000 mgr.y (mgr.26252) 601 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:25.820 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:25 smithi099 bash[87147]: cluster 2024-09-13T14:46:24.456413+0000 mgr.y (mgr.26252) 601 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:25 smithi177 bash[68594]: cluster 2024-09-13T14:46:24.456413+0000 mgr.y (mgr.26252) 601 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:25 smithi177 bash[68594]: cluster 2024-09-13T14:46:24.456413+0000 mgr.y (mgr.26252) 601 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:26.464 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1048,"stamp":"2024-09-13T14:46:24.455715+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":344,"num_read_kb":799,"num_write":212,"num_write_kb":1659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":164,"ondisk_log_size":164,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1480396,"kb_used_data":123044,"kb_used_omap":353,"kb_used_meta":1169694,"kb_avail":748480820,"statfs":{"total":767960285184,"available":766444359680,"internally_reserved":0,"allocated":125997056,"data_stored":79657613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":361898,"internal_metadata":1197767254},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005420"},"pg_stats":[{"pgid":"3.1f","version":"65'5","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313264+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T14:44:10.313264+0000","last_peered":"2024-09-13T14:44:10.313264+0000","last_clean":"2024-09-13T14:44:10.313264+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T14:44:10.313264+0000","last_undegraded":"2024-09-13T14:44:10.313264+0000","last_fullsized":"2024-09-13T14:44:10.313264+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1082,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.301691+0000","last_change":"2024-09-13T14:40:09.139213+0000","last_active":"2024-09-13T14:44:10.301691+0000","last_peered":"2024-09-13T14:44:10.301691+0000","last_clean":"2024-09-13T14:44:10.301691+0000","last_became_active":"2024-09-13T14:40:09.138480+0000","last_became_peered":"2024-09-13T14:40:09.138480+0000","last_unstale":"2024-09-13T14:44:10.301691+0000","last_undegraded":"2024-09-13T14:44:10.301691+0000","last_fullsized":"2024-09-13T14:44:10.301691+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:02:27.534835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1139,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724219+0000","last_change":"2024-09-13T14:39:34.403170+0000","last_active":"2024-09-13T14:44:10.724219+0000","last_peered":"2024-09-13T14:44:10.724219+0000","last_clean":"2024-09-13T14:44:10.724219+0000","last_became_active":"2024-09-13T14:39:34.389001+0000","last_became_peered":"2024-09-13T14:39:34.389001+0000","last_unstale":"2024-09-13T14:44:10.724219+0000","last_undegraded":"2024-09-13T14:44:10.724219+0000","last_fullsized":"2024-09-13T14:44:10.724219+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:38.542452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1152,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.722796+0000","last_change":"2024-09-13T14:39:34.316950+0000","last_active":"2024-09-13T14:44:10.722796+0000","last_peered":"2024-09-13T14:44:10.722796+0000","last_clean":"2024-09-13T14:44:10.722796+0000","last_became_active":"2024-09-13T14:39:34.316614+0000","last_became_peered":"2024-09-13T14:39:34.316614+0000","last_unstale":"2024-09-13T14:44:10.722796+0000","last_undegraded":"2024-09-13T14:44:10.722796+0000","last_fullsized":"2024-09-13T14:44:10.722796+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:35:20.664989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301239+0000","last_change":"2024-09-13T14:40:09.190077+0000","last_active":"2024-09-13T14:44:09.301239+0000","last_peered":"2024-09-13T14:44:09.301239+0000","last_clean":"2024-09-13T14:44:09.301239+0000","last_became_active":"2024-09-13T14:40:09.189717+0000","last_became_peered":"2024-09-13T14:40:09.189717+0000","last_unstale":"2024-09-13T14:44:09.301239+0000","last_undegraded":"2024-09-13T14:44:09.301239+0000","last_fullsized":"2024-09-13T14:44:09.301239+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:24.201026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303665+0000","last_change":"2024-09-13T14:40:09.219485+0000","last_active":"2024-09-13T14:44:10.303665+0000","last_peered":"2024-09-13T14:44:10.303665+0000","last_clean":"2024-09-13T14:44:10.303665+0000","last_became_active":"2024-09-13T14:40:09.212705+0000","last_became_peered":"2024-09-13T14:40:09.212705+0000","last_unstale":"2024-09-13T14:44:10.303665+0000","last_undegraded":"2024-09-13T14:44:10.303665+0000","last_fullsized":"2024-09-13T14:44:10.303665+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:02.378795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":925,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.302449+0000","last_change":"2024-09-13T14:40:09.192785+0000","last_active":"2024-09-13T14:44:09.302449+0000","last_peered":"2024-09-13T14:44:09.302449+0000","last_clean":"2024-09-13T14:44:09.302449+0000","last_became_active":"2024-09-13T14:40:09.192196+0000","last_became_peered":"2024-09-13T14:40:09.192196+0000","last_unstale":"2024-09-13T14:44:09.302449+0000","last_undegraded":"2024-09-13T14:44:09.302449+0000","last_fullsized":"2024-09-13T14:44:09.302449+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:52.737394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.305451+0000","last_change":"2024-09-13T14:40:09.220295+0000","last_active":"2024-09-13T14:44:10.305451+0000","last_peered":"2024-09-13T14:44:10.305451+0000","last_clean":"2024-09-13T14:44:10.305451+0000","last_became_active":"2024-09-13T14:40:09.209986+0000","last_became_peered":"2024-09-13T14:40:09.209986+0000","last_unstale":"2024-09-13T14:44:10.305451+0000","last_undegraded":"2024-09-13T14:44:10.305451+0000","last_fullsized":"2024-09-13T14:44:10.305451+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:57.153128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1283,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.312105+0000","last_change":"2024-09-13T14:40:09.194263+0000","last_active":"2024-09-13T14:44:10.312105+0000","last_peered":"2024-09-13T14:44:10.312105+0000","last_clean":"2024-09-13T14:44:10.312105+0000","last_became_active":"2024-09-13T14:40:09.192631+0000","last_became_peered":"2024-09-13T14:40:09.192631+0000","last_unstale":"2024-09-13T14:44:10.312105+0000","last_undegraded":"2024-09-13T14:44:10.312105+0000","last_fullsized":"2024-09-13T14:44:10.312105+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:41:31.185137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1186,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.740194+0000","last_change":"2024-09-13T14:39:34.358767+0000","last_active":"2024-09-13T14:44:10.740194+0000","last_peered":"2024-09-13T14:44:10.740194+0000","last_clean":"2024-09-13T14:44:10.740194+0000","last_became_active":"2024-09-13T14:39:34.358244+0000","last_became_peered":"2024-09-13T14:39:34.358244+0000","last_unstale":"2024-09-13T14:44:10.740194+0000","last_undegraded":"2024-09-13T14:44:10.740194+0000","last_fullsized":"2024-09-13T14:44:10.740194+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:39.392859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":1132,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303234+0000","last_change":"2024-09-13T14:40:09.189390+0000","last_active":"2024-09-13T14:44:10.303234+0000","last_peered":"2024-09-13T14:44:10.303234+0000","last_clean":"2024-09-13T14:44:10.303234+0000","last_became_active":"2024-09-13T14:40:09.189042+0000","last_became_peered":"2024-09-13T14:40:09.189042+0000","last_unstale":"2024-09-13T14:44:10.303234+0000","last_undegraded":"2024-09-13T14:44:10.303234+0000","last_fullsized":"2024-09-13T14:44:10.303234+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:05.081276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1158,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724166+0000","last_change":"2024-09-13T14:40:09.227622+0000","last_active":"2024-09-13T14:44:10.724166+0000","last_peered":"2024-09-13T14:44:10.724166+0000","last_clean":"2024-09-13T14:44:10.724166+0000","last_became_active":"2024-09-13T14:40:09.226971+0000","last_became_peered":"2024-09-13T14:40:09.226971+0000","last_unstale":"2024-09-13T14:44:10.724166+0000","last_undegraded":"2024-09-13T14:44:10.724166+0000","last_fullsized":"2024-09-13T14:44:10.724166+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:48:21.955396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.730771+0000","last_change":"2024-09-13T14:40:09.173906+0000","last_active":"2024-09-13T14:44:10.730771+0000","last_peered":"2024-09-13T14:44:10.730771+0000","last_clean":"2024-09-13T14:44:10.730771+0000","last_became_active":"2024-09-13T14:40:09.173303+0000","last_became_peered":"2024-09-13T14:40:09.173303+0000","last_unstale":"2024-09-13T14:44:10.730771+0000","last_undegraded":"2024-09-13T14:44:10.730771+0000","last_fullsized":"2024-09-13T14:44:10.730771+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:34:11.061883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1278,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:12.477091+0000","last_change":"2024-09-13T14:39:34.362206+0000","last_active":"2024-09-13T14:44:12.477091+0000","last_peered":"2024-09-13T14:44:12.477091+0000","last_clean":"2024-09-13T14:44:12.477091+0000","last_became_active":"2024-09-13T14:39:34.361932+0000","last_became_peered":"2024-09-13T14:39:34.361932+0000","last_unstale":"2024-09-13T14:44:12.477091+0000","last_undegraded":"2024-09-13T14:44:12.477091+0000","last_fullsized":"2024-09-13T14:44:12.477091+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:00.449274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1165,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723274+0000","last_change":"2024-09-13T14:39:34.403479+0000","last_active":"2024-09-13T14:44:10.723274+0000","last_peered":"2024-09-13T14:44:10.723274+0000","last_clean":"2024-09-13T14:44:10.723274+0000","last_became_active":"2024-09-13T14:39:34.386530+0000","last_became_peered":"2024-09-13T14:39:34.386530+0000","last_unstale":"2024-09-13T14:44:10.723274+0000","last_undegraded":"2024-09-13T14:44:10.723274+0000","last_fullsized":"2024-09-13T14:44:10.723274+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:29.559892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487821+0000","last_change":"2024-09-13T14:38:11.206109+0000","last_active":"2024-09-13T14:44:09.487821+0000","last_peered":"2024-09-13T14:44:09.487821+0000","last_clean":"2024-09-13T14:44:09.487821+0000","last_became_active":"2024-09-13T14:38:11.205546+0000","last_became_peered":"2024-09-13T14:38:11.205546+0000","last_unstale":"2024-09-13T14:44:09.487821+0000","last_undegraded":"2024-09-13T14:44:09.487821+0000","last_fullsized":"2024-09-13T14:44:09.487821+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:33.834490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1145,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.722999+0000","last_change":"2024-09-13T14:39:34.368100+0000","last_active":"2024-09-13T14:44:10.722999+0000","last_peered":"2024-09-13T14:44:10.722999+0000","last_clean":"2024-09-13T14:44:10.722999+0000","last_became_active":"2024-09-13T14:39:34.366992+0000","last_became_peered":"2024-09-13T14:39:34.366992+0000","last_unstale":"2024-09-13T14:44:10.722999+0000","last_undegraded":"2024-09-13T14:44:10.722999+0000","last_fullsized":"2024-09-13T14:44:10.722999+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:48:54.220100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313289+0000","last_change":"2024-09-13T14:39:34.318266+0000","last_active":"2024-09-13T14:44:10.313289+0000","last_peered":"2024-09-13T14:44:10.313289+0000","last_clean":"2024-09-13T14:44:10.313289+0000","last_became_active":"2024-09-13T14:39:34.315973+0000","last_became_peered":"2024-09-13T14:39:34.315973+0000","last_unstale":"2024-09-13T14:44:10.313289+0000","last_undegraded":"2024-09-13T14:44:10.313289+0000","last_fullsized":"2024-09-13T14:44:10.313289+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:15.270282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1124,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723081+0000","last_change":"2024-09-13T14:39:34.338378+0000","last_active":"2024-09-13T14:44:10.723081+0000","last_peered":"2024-09-13T14:44:10.723081+0000","last_clean":"2024-09-13T14:44:10.723081+0000","last_became_active":"2024-09-13T14:39:34.338005+0000","last_became_peered":"2024-09-13T14:39:34.338005+0000","last_unstale":"2024-09-13T14:44:10.723081+0000","last_undegraded":"2024-09-13T14:44:10.723081+0000","last_fullsized":"2024-09-13T14:44:10.723081+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:29.120379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":905,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301800+0000","last_change":"2024-09-13T14:38:11.231165+0000","last_active":"2024-09-13T14:44:09.301800+0000","last_peered":"2024-09-13T14:44:09.301800+0000","last_clean":"2024-09-13T14:44:09.301800+0000","last_became_active":"2024-09-13T14:38:11.230759+0000","last_became_peered":"2024-09-13T14:38:11.230759+0000","last_unstale":"2024-09-13T14:44:09.301800+0000","last_undegraded":"2024-09-13T14:44:09.301800+0000","last_fullsized":"2024-09-13T14:44:09.301800+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:34.222184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":930,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345752+0000","last_change":"2024-09-13T14:40:09.191682+0000","last_active":"2024-09-13T14:44:09.345752+0000","last_peered":"2024-09-13T14:44:09.345752+0000","last_clean":"2024-09-13T14:44:09.345752+0000","last_became_active":"2024-09-13T14:40:09.184791+0000","last_became_peered":"2024-09-13T14:40:09.184791+0000","last_unstale":"2024-09-13T14:44:09.345752+0000","last_undegraded":"2024-09-13T14:44:09.345752+0000","last_fullsized":"2024-09-13T14:44:09.345752+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:56.463093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.733304+0000","last_change":"2024-09-13T14:38:59.370056+0000","last_active":"2024-09-13T14:44:10.733304+0000","last_peered":"2024-09-13T14:44:10.733304+0000","last_clean":"2024-09-13T14:44:10.733304+0000","last_became_active":"2024-09-13T14:38:59.369705+0000","last_became_peered":"2024-09-13T14:38:59.369705+0000","last_unstale":"2024-09-13T14:44:10.733304+0000","last_undegraded":"2024-09-13T14:44:10.733304+0000","last_fullsized":"2024-09-13T14:44:10.733304+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:59.898050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":1125,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303667+0000","last_change":"2024-09-13T14:40:09.180133+0000","last_active":"2024-09-13T14:44:10.303667+0000","last_peered":"2024-09-13T14:44:10.303667+0000","last_clean":"2024-09-13T14:44:10.303667+0000","last_became_active":"2024-09-13T14:40:09.179808+0000","last_became_peered":"2024-09-13T14:40:09.179808+0000","last_unstale":"2024-09-13T14:44:10.303667+0000","last_undegraded":"2024-09-13T14:44:10.303667+0000","last_fullsized":"2024-09-13T14:44:10.303667+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:11.823441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1122,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.306210+0000","last_change":"2024-09-13T14:40:09.190329+0000","last_active":"2024-09-13T14:44:10.306210+0000","last_peered":"2024-09-13T14:44:10.306210+0000","last_clean":"2024-09-13T14:44:10.306210+0000","last_became_active":"2024-09-13T14:40:09.189399+0000","last_became_peered":"2024-09-13T14:40:09.189399+0000","last_unstale":"2024-09-13T14:44:10.306210+0000","last_undegraded":"2024-09-13T14:44:10.306210+0000","last_fullsized":"2024-09-13T14:44:10.306210+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:36.338067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1100,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.727471+0000","last_change":"2024-09-13T14:40:09.202450+0000","last_active":"2024-09-13T14:44:16.727471+0000","last_peered":"2024-09-13T14:44:16.727471+0000","last_clean":"2024-09-13T14:44:16.727471+0000","last_became_active":"2024-09-13T14:40:09.201651+0000","last_became_peered":"2024-09-13T14:40:09.201651+0000","last_unstale":"2024-09-13T14:44:16.727471+0000","last_undegraded":"2024-09-13T14:44:16.727471+0000","last_fullsized":"2024-09-13T14:44:16.727471+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:12.544442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":921,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487923+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T14:44:09.487923+0000","last_peered":"2024-09-13T14:44:09.487923+0000","last_clean":"2024-09-13T14:44:09.487923+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T14:44:09.487923+0000","last_undegraded":"2024-09-13T14:44:09.487923+0000","last_fullsized":"2024-09-13T14:44:09.487923+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1293,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.315290+0000","last_change":"2024-09-13T14:40:09.187749+0000","last_active":"2024-09-13T14:44:10.315290+0000","last_peered":"2024-09-13T14:44:10.315290+0000","last_clean":"2024-09-13T14:44:10.315290+0000","last_became_active":"2024-09-13T14:40:09.187007+0000","last_became_peered":"2024-09-13T14:40:09.187007+0000","last_unstale":"2024-09-13T14:44:10.315290+0000","last_undegraded":"2024-09-13T14:44:10.315290+0000","last_fullsized":"2024-09-13T14:44:10.315290+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:08.019463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1122,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723614+0000","last_change":"2024-09-13T14:39:34.349945+0000","last_active":"2024-09-13T14:44:10.723614+0000","last_peered":"2024-09-13T14:44:10.723614+0000","last_clean":"2024-09-13T14:44:10.723614+0000","last_became_active":"2024-09-13T14:39:34.346730+0000","last_became_peered":"2024-09-13T14:39:34.346730+0000","last_unstale":"2024-09-13T14:44:10.723614+0000","last_undegraded":"2024-09-13T14:44:10.723614+0000","last_fullsized":"2024-09-13T14:44:10.723614+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:48.854603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1234,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729597+0000","last_change":"2024-09-13T14:40:09.177544+0000","last_active":"2024-09-13T14:44:10.729597+0000","last_peered":"2024-09-13T14:44:10.729597+0000","last_clean":"2024-09-13T14:44:10.729597+0000","last_became_active":"2024-09-13T14:40:09.177252+0000","last_became_peered":"2024-09-13T14:40:09.177252+0000","last_unstale":"2024-09-13T14:44:10.729597+0000","last_undegraded":"2024-09-13T14:44:10.729597+0000","last_fullsized":"2024-09-13T14:44:10.729597+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:45:50.539379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1126,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724938+0000","last_change":"2024-09-13T14:39:34.401432+0000","last_active":"2024-09-13T14:44:10.724938+0000","last_peered":"2024-09-13T14:44:10.724938+0000","last_clean":"2024-09-13T14:44:10.724938+0000","last_became_active":"2024-09-13T14:39:34.395458+0000","last_became_peered":"2024-09-13T14:39:34.395458+0000","last_unstale":"2024-09-13T14:44:10.724938+0000","last_undegraded":"2024-09-13T14:44:10.724938+0000","last_fullsized":"2024-09-13T14:44:10.724938+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:17.007388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1266,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.311205+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T14:44:10.311205+0000","last_peered":"2024-09-13T14:44:10.311205+0000","last_clean":"2024-09-13T14:44:10.311205+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T14:44:10.311205+0000","last_undegraded":"2024-09-13T14:44:10.311205+0000","last_fullsized":"2024-09-13T14:44:10.311205+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":937,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345711+0000","last_change":"2024-09-13T14:40:09.185516+0000","last_active":"2024-09-13T14:44:09.345711+0000","last_peered":"2024-09-13T14:44:09.345711+0000","last_clean":"2024-09-13T14:44:09.345711+0000","last_became_active":"2024-09-13T14:40:09.184652+0000","last_became_peered":"2024-09-13T14:40:09.184652+0000","last_unstale":"2024-09-13T14:44:09.345711+0000","last_undegraded":"2024-09-13T14:44:09.345711+0000","last_fullsized":"2024-09-13T14:44:09.345711+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:48.492776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729090+0000","last_change":"2024-09-13T14:38:59.328518+0000","last_active":"2024-09-13T14:44:10.729090+0000","last_peered":"2024-09-13T14:44:10.729090+0000","last_clean":"2024-09-13T14:44:10.729090+0000","last_became_active":"2024-09-13T14:38:59.326010+0000","last_became_peered":"2024-09-13T14:38:59.326010+0000","last_unstale":"2024-09-13T14:44:10.729090+0000","last_undegraded":"2024-09-13T14:44:10.729090+0000","last_fullsized":"2024-09-13T14:44:10.729090+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:01.198043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.733416+0000","last_change":"2024-09-13T14:40:09.160901+0000","last_active":"2024-09-13T14:44:10.733416+0000","last_peered":"2024-09-13T14:44:10.733416+0000","last_clean":"2024-09-13T14:44:10.733416+0000","last_became_active":"2024-09-13T14:40:09.160278+0000","last_became_peered":"2024-09-13T14:40:09.160278+0000","last_unstale":"2024-09-13T14:44:10.733416+0000","last_undegraded":"2024-09-13T14:44:10.733416+0000","last_fullsized":"2024-09-13T14:44:10.733416+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:48:56.238297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1224,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.728934+0000","last_change":"2024-09-13T14:38:11.234481+0000","last_active":"2024-09-13T14:44:10.728934+0000","last_peered":"2024-09-13T14:44:10.728934+0000","last_clean":"2024-09-13T14:44:10.728934+0000","last_became_active":"2024-09-13T14:38:11.232868+0000","last_became_peered":"2024-09-13T14:38:11.232868+0000","last_unstale":"2024-09-13T14:44:10.728934+0000","last_undegraded":"2024-09-13T14:44:10.728934+0000","last_fullsized":"2024-09-13T14:44:10.728934+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:06:01.704192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":922,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301454+0000","last_change":"2024-09-13T14:39:34.321388+0000","last_active":"2024-09-13T14:44:09.301454+0000","last_peered":"2024-09-13T14:44:09.301454+0000","last_clean":"2024-09-13T14:44:09.301454+0000","last_became_active":"2024-09-13T14:39:34.321025+0000","last_became_peered":"2024-09-13T14:39:34.321025+0000","last_unstale":"2024-09-13T14:44:09.301454+0000","last_undegraded":"2024-09-13T14:44:09.301454+0000","last_fullsized":"2024-09-13T14:44:09.301454+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:52.593825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.729257+0000","last_change":"2024-09-13T14:39:34.378359+0000","last_active":"2024-09-13T14:44:10.729257+0000","last_peered":"2024-09-13T14:44:10.729257+0000","last_clean":"2024-09-13T14:44:10.729257+0000","last_became_active":"2024-09-13T14:39:34.377967+0000","last_became_peered":"2024-09-13T14:39:34.377967+0000","last_unstale":"2024-09-13T14:44:10.729257+0000","last_undegraded":"2024-09-13T14:44:10.729257+0000","last_fullsized":"2024-09-13T14:44:10.729257+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:37.645170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1099,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:16.727560+0000","last_change":"2024-09-13T14:40:09.222631+0000","last_active":"2024-09-13T14:44:16.727560+0000","last_peered":"2024-09-13T14:44:16.727560+0000","last_clean":"2024-09-13T14:44:16.727560+0000","last_became_active":"2024-09-13T14:40:09.211300+0000","last_became_peered":"2024-09-13T14:40:09.211300+0000","last_unstale":"2024-09-13T14:44:16.727560+0000","last_undegraded":"2024-09-13T14:44:16.727560+0000","last_fullsized":"2024-09-13T14:44:16.727560+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:56.913336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":919,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.299732+0000","last_change":"2024-09-13T14:38:11.232633+0000","last_active":"2024-09-13T14:44:09.299732+0000","last_peered":"2024-09-13T14:44:09.299732+0000","last_clean":"2024-09-13T14:44:09.299732+0000","last_became_active":"2024-09-13T14:38:11.230948+0000","last_became_peered":"2024-09-13T14:38:11.230948+0000","last_unstale":"2024-09-13T14:44:09.299732+0000","last_undegraded":"2024-09-13T14:44:09.299732+0000","last_fullsized":"2024-09-13T14:44:09.299732+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:38.665278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.736730+0000","last_change":"2024-09-13T14:38:59.402033+0000","last_active":"2024-09-13T14:44:10.736730+0000","last_peered":"2024-09-13T14:44:10.736730+0000","last_clean":"2024-09-13T14:44:10.736730+0000","last_became_active":"2024-09-13T14:38:59.401826+0000","last_became_peered":"2024-09-13T14:38:59.401826+0000","last_unstale":"2024-09-13T14:44:10.736730+0000","last_undegraded":"2024-09-13T14:44:10.736730+0000","last_fullsized":"2024-09-13T14:44:10.736730+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:21:17.785114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1225,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.727900+0000","last_change":"2024-09-13T14:38:59.329634+0000","last_active":"2024-09-13T14:44:10.727900+0000","last_peered":"2024-09-13T14:44:10.727900+0000","last_clean":"2024-09-13T14:44:10.727900+0000","last_became_active":"2024-09-13T14:38:59.328813+0000","last_became_peered":"2024-09-13T14:38:59.328813+0000","last_unstale":"2024-09-13T14:44:10.727900+0000","last_undegraded":"2024-09-13T14:44:10.727900+0000","last_fullsized":"2024-09-13T14:44:10.727900+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:35.653113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1175,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.737951+0000","last_change":"2024-09-13T14:39:34.322533+0000","last_active":"2024-09-13T14:44:10.737951+0000","last_peered":"2024-09-13T14:44:10.737951+0000","last_clean":"2024-09-13T14:44:10.737951+0000","last_became_active":"2024-09-13T14:39:34.322070+0000","last_became_peered":"2024-09-13T14:39:34.322070+0000","last_unstale":"2024-09-13T14:44:10.737951+0000","last_undegraded":"2024-09-13T14:44:10.737951+0000","last_fullsized":"2024-09-13T14:44:10.737951+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:43.746420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314327+0000","last_change":"2024-09-13T14:38:59.338775+0000","last_active":"2024-09-13T14:44:10.314327+0000","last_peered":"2024-09-13T14:44:10.314327+0000","last_clean":"2024-09-13T14:44:10.314327+0000","last_became_active":"2024-09-13T14:38:59.338265+0000","last_became_peered":"2024-09-13T14:38:59.338265+0000","last_unstale":"2024-09-13T14:44:10.314327+0000","last_undegraded":"2024-09-13T14:44:10.314327+0000","last_fullsized":"2024-09-13T14:44:10.314327+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:48.255980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.313649+0000","last_change":"2024-09-13T14:40:09.188286+0000","last_active":"2024-09-13T14:44:10.313649+0000","last_peered":"2024-09-13T14:44:10.313649+0000","last_clean":"2024-09-13T14:44:10.313649+0000","last_became_active":"2024-09-13T14:40:09.186778+0000","last_became_peered":"2024-09-13T14:40:09.186778+0000","last_unstale":"2024-09-13T14:44:10.313649+0000","last_undegraded":"2024-09-13T14:44:10.313649+0000","last_fullsized":"2024-09-13T14:44:10.313649+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:44:24.275819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":937,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345163+0000","last_change":"2024-09-13T14:38:11.178130+0000","last_active":"2024-09-13T14:44:09.345163+0000","last_peered":"2024-09-13T14:44:09.345163+0000","last_clean":"2024-09-13T14:44:09.345163+0000","last_became_active":"2024-09-13T14:38:11.177484+0000","last_became_peered":"2024-09-13T14:38:11.177484+0000","last_unstale":"2024-09-13T14:44:09.345163+0000","last_undegraded":"2024-09-13T14:44:09.345163+0000","last_fullsized":"2024-09-13T14:44:09.345163+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:18:24.776012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.487609+0000","last_change":"2024-09-13T14:40:09.183716+0000","last_active":"2024-09-13T14:44:09.487609+0000","last_peered":"2024-09-13T14:44:09.487609+0000","last_clean":"2024-09-13T14:44:09.487609+0000","last_became_active":"2024-09-13T14:40:09.183420+0000","last_became_peered":"2024-09-13T14:40:09.183420+0000","last_unstale":"2024-09-13T14:44:09.487609+0000","last_undegraded":"2024-09-13T14:44:09.487609+0000","last_fullsized":"2024-09-13T14:44:09.487609+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:03.929337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1124,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723784+0000","last_change":"2024-09-13T14:39:34.356129+0000","last_active":"2024-09-13T14:44:10.723784+0000","last_peered":"2024-09-13T14:44:10.723784+0000","last_clean":"2024-09-13T14:44:10.723784+0000","last_became_active":"2024-09-13T14:39:34.355288+0000","last_became_peered":"2024-09-13T14:39:34.355288+0000","last_unstale":"2024-09-13T14:44:10.723784+0000","last_undegraded":"2024-09-13T14:44:10.723784+0000","last_fullsized":"2024-09-13T14:44:10.723784+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.725003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":918,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.300306+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T14:44:09.300306+0000","last_peered":"2024-09-13T14:44:09.300306+0000","last_clean":"2024-09-13T14:44:09.300306+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T14:44:09.300306+0000","last_undegraded":"2024-09-13T14:44:09.300306+0000","last_fullsized":"2024-09-13T14:44:09.300306+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1025,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345613+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T14:44:09.345613+0000","last_peered":"2024-09-13T14:44:09.345613+0000","last_clean":"2024-09-13T14:44:09.345613+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T14:44:09.345613+0000","last_undegraded":"2024-09-13T14:44:09.345613+0000","last_fullsized":"2024-09-13T14:44:09.345613+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1235,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.728956+0000","last_change":"2024-09-13T14:40:09.174566+0000","last_active":"2024-09-13T14:44:10.728956+0000","last_peered":"2024-09-13T14:44:10.728956+0000","last_clean":"2024-09-13T14:44:10.728956+0000","last_became_active":"2024-09-13T14:40:09.173698+0000","last_became_peered":"2024-09-13T14:40:09.173698+0000","last_unstale":"2024-09-13T14:44:10.728956+0000","last_undegraded":"2024-09-13T14:44:10.728956+0000","last_fullsized":"2024-09-13T14:44:10.728956+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:22.408868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1149,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723899+0000","last_change":"2024-09-13T14:39:34.403656+0000","last_active":"2024-09-13T14:44:10.723899+0000","last_peered":"2024-09-13T14:44:10.723899+0000","last_clean":"2024-09-13T14:44:10.723899+0000","last_became_active":"2024-09-13T14:39:34.386890+0000","last_became_peered":"2024-09-13T14:39:34.386890+0000","last_unstale":"2024-09-13T14:44:10.723899+0000","last_undegraded":"2024-09-13T14:44:10.723899+0000","last_fullsized":"2024-09-13T14:44:10.723899+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:24.563109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1082,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.307149+0000","last_change":"2024-09-13T14:40:09.182955+0000","last_active":"2024-09-13T14:44:10.307149+0000","last_peered":"2024-09-13T14:44:10.307149+0000","last_clean":"2024-09-13T14:44:10.307149+0000","last_became_active":"2024-09-13T14:40:09.182702+0000","last_became_peered":"2024-09-13T14:40:09.182702+0000","last_unstale":"2024-09-13T14:44:10.307149+0000","last_undegraded":"2024-09-13T14:44:10.307149+0000","last_fullsized":"2024-09-13T14:44:10.307149+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:00:15.207705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1113,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303286+0000","last_change":"2024-09-13T14:40:09.185241+0000","last_active":"2024-09-13T14:44:10.303286+0000","last_peered":"2024-09-13T14:44:10.303286+0000","last_clean":"2024-09-13T14:44:10.303286+0000","last_became_active":"2024-09-13T14:40:09.184259+0000","last_became_peered":"2024-09-13T14:40:09.184259+0000","last_unstale":"2024-09-13T14:44:10.303286+0000","last_undegraded":"2024-09-13T14:44:10.303286+0000","last_fullsized":"2024-09-13T14:44:10.303286+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:27:17.762953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1129,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.304287+0000","last_change":"2024-09-13T14:40:09.200853+0000","last_active":"2024-09-13T14:44:10.304287+0000","last_peered":"2024-09-13T14:44:10.304287+0000","last_clean":"2024-09-13T14:44:10.304287+0000","last_became_active":"2024-09-13T14:40:09.195505+0000","last_became_peered":"2024-09-13T14:40:09.195505+0000","last_unstale":"2024-09-13T14:44:10.304287+0000","last_undegraded":"2024-09-13T14:44:10.304287+0000","last_fullsized":"2024-09-13T14:44:10.304287+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:28.009085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":926,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.486881+0000","last_change":"2024-09-13T14:40:09.163144+0000","last_active":"2024-09-13T14:44:09.486881+0000","last_peered":"2024-09-13T14:44:09.486881+0000","last_clean":"2024-09-13T14:44:09.486881+0000","last_became_active":"2024-09-13T14:40:09.162863+0000","last_became_peered":"2024-09-13T14:40:09.162863+0000","last_unstale":"2024-09-13T14:44:09.486881+0000","last_undegraded":"2024-09-13T14:44:09.486881+0000","last_fullsized":"2024-09-13T14:44:09.486881+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:48.398889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1104,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.303932+0000","last_change":"2024-09-13T14:40:09.223911+0000","last_active":"2024-09-13T14:44:10.303932+0000","last_peered":"2024-09-13T14:44:10.303932+0000","last_clean":"2024-09-13T14:44:10.303932+0000","last_became_active":"2024-09-13T14:40:09.215480+0000","last_became_peered":"2024-09-13T14:40:09.215480+0000","last_unstale":"2024-09-13T14:44:10.303932+0000","last_undegraded":"2024-09-13T14:44:10.303932+0000","last_fullsized":"2024-09-13T14:44:10.303932+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.613398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1138,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.723558+0000","last_change":"2024-09-13T14:40:09.208549+0000","last_active":"2024-09-13T14:44:10.723558+0000","last_peered":"2024-09-13T14:44:10.723558+0000","last_clean":"2024-09-13T14:44:10.723558+0000","last_became_active":"2024-09-13T14:40:09.208174+0000","last_became_peered":"2024-09-13T14:40:09.208174+0000","last_unstale":"2024-09-13T14:44:10.723558+0000","last_undegraded":"2024-09-13T14:44:10.723558+0000","last_fullsized":"2024-09-13T14:44:10.723558+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:18.971929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1288,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.311474+0000","last_change":"2024-09-13T14:40:09.192856+0000","last_active":"2024-09-13T14:44:10.311474+0000","last_peered":"2024-09-13T14:44:10.311474+0000","last_clean":"2024-09-13T14:44:10.311474+0000","last_became_active":"2024-09-13T14:40:09.186733+0000","last_became_peered":"2024-09-13T14:40:09.186733+0000","last_unstale":"2024-09-13T14:44:10.311474+0000","last_undegraded":"2024-09-13T14:44:10.311474+0000","last_fullsized":"2024-09-13T14:44:10.311474+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:55.336998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":905,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.302232+0000","last_change":"2024-09-13T14:39:34.393673+0000","last_active":"2024-09-13T14:44:09.302232+0000","last_peered":"2024-09-13T14:44:09.302232+0000","last_clean":"2024-09-13T14:44:09.302232+0000","last_became_active":"2024-09-13T14:39:34.393186+0000","last_became_peered":"2024-09-13T14:39:34.393186+0000","last_unstale":"2024-09-13T14:44:09.302232+0000","last_undegraded":"2024-09-13T14:44:09.302232+0000","last_fullsized":"2024-09-13T14:44:09.302232+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:23.130084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.486735+0000","last_change":"2024-09-13T14:38:11.219076+0000","last_active":"2024-09-13T14:44:09.486735+0000","last_peered":"2024-09-13T14:44:09.486735+0000","last_clean":"2024-09-13T14:44:09.486735+0000","last_became_active":"2024-09-13T14:38:11.217912+0000","last_became_peered":"2024-09-13T14:38:11.217912+0000","last_unstale":"2024-09-13T14:44:09.486735+0000","last_undegraded":"2024-09-13T14:44:09.486735+0000","last_fullsized":"2024-09-13T14:44:09.486735+0000","mapping_epoch":642,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:32.673534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":893,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.344340+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:44:09.344340+0000","last_peered":"2024-09-13T14:44:09.344340+0000","last_clean":"2024-09-13T14:44:09.344340+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:44:09.344340+0000","last_undegraded":"2024-09-13T14:44:09.344340+0000","last_fullsized":"2024-09-13T14:44:09.344340+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1121,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.302845+0000","last_change":"2024-09-13T14:40:09.211963+0000","last_active":"2024-09-13T14:44:10.302845+0000","last_peered":"2024-09-13T14:44:10.302845+0000","last_clean":"2024-09-13T14:44:10.302845+0000","last_became_active":"2024-09-13T14:40:09.205704+0000","last_became_peered":"2024-09-13T14:40:09.205704+0000","last_unstale":"2024-09-13T14:44:10.302845+0000","last_undegraded":"2024-09-13T14:44:10.302845+0000","last_fullsized":"2024-09-13T14:44:10.302845+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:18.704369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":928,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345857+0000","last_change":"2024-09-13T14:40:09.193142+0000","last_active":"2024-09-13T14:44:09.345857+0000","last_peered":"2024-09-13T14:44:09.345857+0000","last_clean":"2024-09-13T14:44:09.345857+0000","last_became_active":"2024-09-13T14:40:09.191806+0000","last_became_peered":"2024-09-13T14:40:09.191806+0000","last_unstale":"2024-09-13T14:44:09.345857+0000","last_undegraded":"2024-09-13T14:44:09.345857+0000","last_fullsized":"2024-09-13T14:44:09.345857+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:41:20.972768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":887,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.299536+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:44:09.299536+0000","last_peered":"2024-09-13T14:44:09.299536+0000","last_clean":"2024-09-13T14:44:09.299536+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:44:09.299536+0000","last_undegraded":"2024-09-13T14:44:09.299536+0000","last_fullsized":"2024-09-13T14:44:09.299536+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":927,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345967+0000","last_change":"2024-09-13T14:40:09.191167+0000","last_active":"2024-09-13T14:44:09.345967+0000","last_peered":"2024-09-13T14:44:09.345967+0000","last_clean":"2024-09-13T14:44:09.345967+0000","last_became_active":"2024-09-13T14:40:09.185797+0000","last_became_peered":"2024-09-13T14:40:09.185797+0000","last_unstale":"2024-09-13T14:44:09.345967+0000","last_undegraded":"2024-09-13T14:44:09.345967+0000","last_fullsized":"2024-09-13T14:44:09.345967+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:52.157413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":921,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.485897+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T14:44:09.485897+0000","last_peered":"2024-09-13T14:44:09.485897+0000","last_clean":"2024-09-13T14:44:09.485897+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T14:44:09.485897+0000","last_undegraded":"2024-09-13T14:44:09.485897+0000","last_fullsized":"2024-09-13T14:44:09.485897+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724498+0000","last_change":"2024-09-13T14:39:34.395859+0000","last_active":"2024-09-13T14:44:10.724498+0000","last_peered":"2024-09-13T14:44:10.724498+0000","last_clean":"2024-09-13T14:44:10.724498+0000","last_became_active":"2024-09-13T14:39:34.389481+0000","last_became_peered":"2024-09-13T14:39:34.389481+0000","last_unstale":"2024-09-13T14:44:10.724498+0000","last_undegraded":"2024-09-13T14:44:10.724498+0000","last_fullsized":"2024-09-13T14:44:10.724498+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.724694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":908,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.344937+0000","last_change":"2024-09-13T14:40:09.173284+0000","last_active":"2024-09-13T14:44:09.344937+0000","last_peered":"2024-09-13T14:44:09.344937+0000","last_clean":"2024-09-13T14:44:09.344937+0000","last_became_active":"2024-09-13T14:40:09.171846+0000","last_became_peered":"2024-09-13T14:40:09.171846+0000","last_unstale":"2024-09-13T14:44:09.344937+0000","last_undegraded":"2024-09-13T14:44:09.344937+0000","last_fullsized":"2024-09-13T14:44:09.344937+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:51:35.460440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314998+0000","last_change":"2024-09-13T14:39:34.362694+0000","last_active":"2024-09-13T14:44:10.314998+0000","last_peered":"2024-09-13T14:44:10.314998+0000","last_clean":"2024-09-13T14:44:10.314998+0000","last_became_active":"2024-09-13T14:39:34.362435+0000","last_became_peered":"2024-09-13T14:39:34.362435+0000","last_unstale":"2024-09-13T14:44:10.314998+0000","last_undegraded":"2024-09-13T14:44:10.314998+0000","last_fullsized":"2024-09-13T14:44:10.314998+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:33.350328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1277,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.314791+0000","last_change":"2024-09-13T14:39:34.360941+0000","last_active":"2024-09-13T14:44:10.314791+0000","last_peered":"2024-09-13T14:44:10.314791+0000","last_clean":"2024-09-13T14:44:10.314791+0000","last_became_active":"2024-09-13T14:39:34.360383+0000","last_became_peered":"2024-09-13T14:39:34.360383+0000","last_unstale":"2024-09-13T14:44:10.314791+0000","last_undegraded":"2024-09-13T14:44:10.314791+0000","last_fullsized":"2024-09-13T14:44:10.314791+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:51.344296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":904,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.301001+0000","last_change":"2024-09-13T14:39:34.380634+0000","last_active":"2024-09-13T14:44:09.301001+0000","last_peered":"2024-09-13T14:44:09.301001+0000","last_clean":"2024-09-13T14:44:09.301001+0000","last_became_active":"2024-09-13T14:39:34.379785+0000","last_became_peered":"2024-09-13T14:39:34.379785+0000","last_unstale":"2024-09-13T14:44:09.301001+0000","last_undegraded":"2024-09-13T14:44:09.301001+0000","last_fullsized":"2024-09-13T14:44:09.301001+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:02.436193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":929,"reported_epoch":950,"state":"active+clean","last_fresh":"2024-09-13T14:44:09.345354+0000","last_change":"2024-09-13T14:40:09.191394+0000","last_active":"2024-09-13T14:44:09.345354+0000","last_peered":"2024-09-13T14:44:09.345354+0000","last_clean":"2024-09-13T14:44:09.345354+0000","last_became_active":"2024-09-13T14:40:09.184697+0000","last_became_peered":"2024-09-13T14:40:09.184697+0000","last_unstale":"2024-09-13T14:44:09.345354+0000","last_undegraded":"2024-09-13T14:44:09.345354+0000","last_fullsized":"2024-09-13T14:44:09.345354+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:31.632367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1147,"reported_epoch":951,"state":"active+clean","last_fresh":"2024-09-13T14:44:10.724494+0000","last_change":"2024-09-13T14:39:34.401131+0000","last_active":"2024-09-13T14:44:10.724494+0000","last_peered":"2024-09-13T14:44:10.724494+0000","last_clean":"2024-09-13T14:44:10.724494+0000","last_became_active":"2024-09-13T14:39:34.393396+0000","last_became_peered":"2024-09-13T14:39:34.393396+0000","last_unstale":"2024-09-13T14:44:10.724494+0000","last_undegraded":"2024-09-13T14:44:10.724494+0000","last_fullsized":"2024-09-13T14:44:10.724494+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:35.475048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397824,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325592,"kb_used_data":15760,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93419560,"statfs":{"total":95995035648,"available":95661629440,"internally_reserved":0,"allocated":16138240,"data_stored":10329470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594059,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":345488,"kb_used_data":15744,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93399664,"statfs":{"total":95995035648,"available":95641255936,"internally_reserved":0,"allocated":16121856,"data_stored":10328713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790293,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362036,"kb_used_data":15160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93383116,"statfs":{"total":95995035648,"available":95624310784,"internally_reserved":0,"allocated":15523840,"data_stored":9734478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980831821,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":79212,"kb_used_data":15176,"kb_used_omap":76,"kb_used_meta":50099,"kb_avail":93665940,"statfs":{"total":95995035648,"available":95913922560,"internally_reserved":0,"allocated":15540224,"data_stored":9739044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":51302208},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911060,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":69504,"kb_used_data":15156,"kb_used_omap":77,"kb_used_meta":54258,"kb_avail":93675648,"statfs":{"total":95995035648,"available":95923863552,"internally_reserved":0,"allocated":15519744,"data_stored":9734252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":55561013},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023003,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":51460,"kb_used_data":15160,"kb_used_omap":71,"kb_used_meta":36216,"kb_avail":93693692,"statfs":{"total":95995035648,"available":95942340608,"internally_reserved":0,"allocated":15523840,"data_stored":9734774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":37085856},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004133,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":117824,"kb_used_data":15152,"kb_used_omap":66,"kb_used_meta":62397,"kb_avail":93627328,"statfs":{"total":95995035648,"available":95874383872,"internally_reserved":0,"allocated":15515648,"data_stored":9733874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":63894698},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195325,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":129280,"kb_used_data":15736,"kb_used_omap":56,"kb_used_meta":49735,"kb_avail":93615872,"statfs":{"total":95995035648,"available":95862652928,"internally_reserved":0,"allocated":16113664,"data_stored":10323008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":50928783},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:46:26.466 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:46:26.467 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd dump 2024-09-13T14:46:26.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:26 smithi099 bash[84576]: audit 2024-09-13T14:46:25.502472+0000 mgr.y (mgr.26252) 602 : audit [DBG] from='client.45101 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:26.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:26 smithi099 bash[84576]: audit 2024-09-13T14:46:25.502472+0000 mgr.y (mgr.26252) 602 : audit [DBG] from='client.45101 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:26 smithi099 bash[87147]: audit 2024-09-13T14:46:25.502472+0000 mgr.y (mgr.26252) 602 : audit [DBG] from='client.45101 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:26.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:26 smithi099 bash[87147]: audit 2024-09-13T14:46:25.502472+0000 mgr.y (mgr.26252) 602 : audit [DBG] from='client.45101 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:26.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:26 smithi177 bash[68594]: audit 2024-09-13T14:46:25.502472+0000 mgr.y (mgr.26252) 602 : audit [DBG] from='client.45101 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:26.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:26 smithi177 bash[68594]: audit 2024-09-13T14:46:25.502472+0000 mgr.y (mgr.26252) 602 : audit [DBG] from='client.45101 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:27.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:46:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:46:27] "GET /metrics HTTP/1.1" 200 34272 "" "Prometheus/2.51.0" 2024-09-13T14:46:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:27 smithi099 bash[84576]: cluster 2024-09-13T14:46:26.457089+0000 mgr.y (mgr.26252) 603 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:27 smithi099 bash[84576]: cluster 2024-09-13T14:46:26.457089+0000 mgr.y (mgr.26252) 603 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:27 smithi099 bash[87147]: cluster 2024-09-13T14:46:26.457089+0000 mgr.y (mgr.26252) 603 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:27 smithi099 bash[87147]: cluster 2024-09-13T14:46:26.457089+0000 mgr.y (mgr.26252) 603 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:27 smithi177 bash[68594]: cluster 2024-09-13T14:46:26.457089+0000 mgr.y (mgr.26252) 603 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:27.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:27 smithi177 bash[68594]: cluster 2024-09-13T14:46:26.457089+0000 mgr.y (mgr.26252) 603 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:29.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:29 smithi177 bash[68594]: cluster 2024-09-13T14:46:28.458048+0000 mgr.y (mgr.26252) 604 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:29.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:29 smithi177 bash[68594]: cluster 2024-09-13T14:46:28.458048+0000 mgr.y (mgr.26252) 604 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:30.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:29 smithi099 bash[84576]: cluster 2024-09-13T14:46:28.458048+0000 mgr.y (mgr.26252) 604 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:30.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:29 smithi099 bash[84576]: cluster 2024-09-13T14:46:28.458048+0000 mgr.y (mgr.26252) 604 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:30.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:29 smithi099 bash[87147]: cluster 2024-09-13T14:46:28.458048+0000 mgr.y (mgr.26252) 604 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:30.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:29 smithi099 bash[87147]: cluster 2024-09-13T14:46:28.458048+0000 mgr.y (mgr.26252) 604 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:30.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:46:30 smithi177 bash[61439]: ts=2024-09-13T14:46:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:31.258 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:31.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:31 smithi099 bash[87147]: cluster 2024-09-13T14:46:30.458918+0000 mgr.y (mgr.26252) 605 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:31.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:31 smithi099 bash[87147]: cluster 2024-09-13T14:46:30.458918+0000 mgr.y (mgr.26252) 605 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:31 smithi099 bash[84576]: cluster 2024-09-13T14:46:30.458918+0000 mgr.y (mgr.26252) 605 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:31 smithi099 bash[84576]: cluster 2024-09-13T14:46:30.458918+0000 mgr.y (mgr.26252) 605 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:31.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:31 smithi177 bash[68594]: cluster 2024-09-13T14:46:30.458918+0000 mgr.y (mgr.26252) 605 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:31.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:31 smithi177 bash[68594]: cluster 2024-09-13T14:46:30.458918+0000 mgr.y (mgr.26252) 605 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:33 smithi177 bash[68594]: cluster 2024-09-13T14:46:32.459738+0000 mgr.y (mgr.26252) 606 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:33 smithi177 bash[68594]: cluster 2024-09-13T14:46:32.459738+0000 mgr.y (mgr.26252) 606 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:33 smithi177 bash[68594]: audit 2024-09-13T14:46:33.269240+0000 mon.a (mon.0) 1564 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:33 smithi177 bash[68594]: audit 2024-09-13T14:46:33.269240+0000 mon.a (mon.0) 1564 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:33.875 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:33 smithi099 bash[84576]: cluster 2024-09-13T14:46:32.459738+0000 mgr.y (mgr.26252) 606 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:33.875 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:33 smithi099 bash[84576]: cluster 2024-09-13T14:46:32.459738+0000 mgr.y (mgr.26252) 606 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:33.876 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:33 smithi099 bash[84576]: audit 2024-09-13T14:46:33.269240+0000 mon.a (mon.0) 1564 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:33.876 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:33 smithi099 bash[84576]: audit 2024-09-13T14:46:33.269240+0000 mon.a (mon.0) 1564 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:33.876 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:33 smithi099 bash[87147]: cluster 2024-09-13T14:46:32.459738+0000 mgr.y (mgr.26252) 606 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:33.876 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:33 smithi099 bash[87147]: cluster 2024-09-13T14:46:32.459738+0000 mgr.y (mgr.26252) 606 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:33.876 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:33 smithi099 bash[87147]: audit 2024-09-13T14:46:33.269240+0000 mon.a (mon.0) 1564 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:33.876 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:33 smithi099 bash[87147]: audit 2024-09-13T14:46:33.269240+0000 mon.a (mon.0) 1564 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:35.471 INFO:teuthology.orchestra.run.smithi099.stdout:epoch 951 2024-09-13T14:46:35.471 INFO:teuthology.orchestra.run.smithi099.stdout:fsid af9d2c22-71d9-11ef-bceb-c7b262605968 2024-09-13T14:46:35.471 INFO:teuthology.orchestra.run.smithi099.stdout:created 2024-09-13T14:10:21.363918+0000 2024-09-13T14:46:35.471 INFO:teuthology.orchestra.run.smithi099.stdout:modified 2024-09-13T14:44:10.297966+0000 2024-09-13T14:46:35.471 INFO:teuthology.orchestra.run.smithi099.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-09-13T14:46:35.471 INFO:teuthology.orchestra.run.smithi099.stdout:crush_version 23 2024-09-13T14:46:35.471 INFO:teuthology.orchestra.run.smithi099.stdout:full_ratio 0.95 2024-09-13T14:46:35.471 INFO:teuthology.orchestra.run.smithi099.stdout:backfillfull_ratio 0.9 2024-09-13T14:46:35.471 INFO:teuthology.orchestra.run.smithi099.stdout:nearfull_ratio 0.85 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:require_min_compat_client luminous 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:min_compat_client luminous 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:require_osd_release quincy 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:stretch_mode_enabled false 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 58 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 63 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 64 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:max_osd 8 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0 up in weight 1 up_from 575 up_thru 899 down_at 558 last_clean_interval [8,558) [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480] [v2:172.21.15.99:6804/1813984480,v1:172.21.15.99:6805/1813984480] exists,up 0624feaa-339b-4468-be53-582a1f04352c 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1 up in weight 1 up_from 642 up_thru 899 down_at 639 last_clean_interval [13,638) [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824] [v2:172.21.15.99:6812/804978824,v1:172.21.15.99:6813/804978824] exists,up b7b63553-e49d-4636-b3e2-43b72556a1a6 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2 up in weight 1 up_from 672 up_thru 899 down_at 657 last_clean_interval [18,656) [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315] [v2:172.21.15.99:6820/129558315,v1:172.21.15.99:6821/129558315] exists,up 3b9eff6d-5f5c-4843-a1ff-815a902167ec 2024-09-13T14:46:35.472 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3 up in weight 1 up_from 706 up_thru 899 down_at 690 last_clean_interval [26,689) [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003] [v2:172.21.15.99:6828/876595003,v1:172.21.15.99:6829/876595003] exists,up 686c8d3c-9cab-4920-b02b-d243324f5808 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4 up in weight 1 up_from 739 up_thru 899 down_at 725 last_clean_interval [31,724) [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964] [v2:172.21.15.177:6802/2135282964,v1:172.21.15.177:6803/2135282964] exists,up 2d56712b-a6fb-4089-9583-850beed92b37 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5 up in weight 1 up_from 37 up_thru 899 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587] [v2:172.21.15.177:6810/947070587,v1:172.21.15.177:6811/947070587] exists,up 414b880c-6534-495f-a54c-f8d28ba81c73 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6 up in weight 1 up_from 43 up_thru 899 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594] [v2:172.21.15.177:6818/3817692594,v1:172.21.15.177:6819/3817692594] exists,up ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7 up in weight 1 up_from 49 up_thru 899 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946] [v2:172.21.15.177:6826/1984676946,v1:172.21.15.177:6827/1984676946] exists,up ad2c2977-fd6d-42f8-962e-907c2bf3204b 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 4.7 [1,2] 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 4.10 [1,2] 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:pg_upmap_items 4.13 [1,2] 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/3935330909 expires 2024-09-13T15:40:15.846422+0000 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/3556003775 expires 2024-09-13T15:35:26.139282+0000 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/1151643634 expires 2024-09-14T14:31:47.884997+0000 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6801/1402010506 expires 2024-09-14T14:31:47.884997+0000 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6800/1402010506 expires 2024-09-14T14:31:47.884997+0000 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/3817526904 expires 2024-09-14T14:31:47.884997+0000 2024-09-13T14:46:35.473 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/3394219806 expires 2024-09-14T14:31:47.884997+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/181419264 expires 2024-09-14T14:31:47.884997+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/4129805635 expires 2024-09-14T14:11:18.714884+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/1363241618 expires 2024-09-14T14:21:07.041302+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/3192879133 expires 2024-09-14T14:11:18.714884+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/3483428223 expires 2024-09-14T14:21:07.041302+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/405712510 expires 2024-09-14T14:21:07.041302+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.177:0/1523474123 expires 2024-09-14T14:30:38.246301+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6834/3867979721 expires 2024-09-14T14:24:26.174022+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/795365709 expires 2024-09-14T14:10:48.295312+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/1548831286 expires 2024-09-14T14:11:18.714884+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.177:0/3423158096 expires 2024-09-14T14:30:38.246301+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/888319381 expires 2024-09-14T14:10:48.295312+0000 2024-09-13T14:46:35.474 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.177:0/3478790043 expires 2024-09-14T14:30:38.246301+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6801/2671898016 expires 2024-09-14T14:10:48.295312+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.177:0/1330817275 expires 2024-09-14T14:30:38.246301+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/1638158155 expires 2024-09-14T14:29:03.998039+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/313522629 expires 2024-09-13T15:35:27.165268+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6800/2887456408 expires 2024-09-14T14:29:03.998039+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6801/2887456408 expires 2024-09-14T14:29:03.998039+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/4277277716 expires 2024-09-14T14:31:47.884997+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6800/2671898016 expires 2024-09-14T14:10:48.295312+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/17074640 expires 2024-09-14T14:21:07.041302+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6835/3867979721 expires 2024-09-14T14:24:26.174022+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6800/3775052239 expires 2024-09-14T14:21:07.041302+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.177:6834/4079938285 expires 2024-09-14T14:30:38.246301+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/2372940082 expires 2024-09-14T14:31:47.884997+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6801/3838091368 expires 2024-09-14T14:11:18.714884+0000 2024-09-13T14:46:35.475 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6801/3775052239 expires 2024-09-14T14:21:07.041302+0000 2024-09-13T14:46:35.476 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/316966644 expires 2024-09-14T14:29:03.998039+0000 2024-09-13T14:46:35.476 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/2140252887 expires 2024-09-14T14:29:03.998039+0000 2024-09-13T14:46:35.476 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/4242039269 expires 2024-09-14T14:29:03.998039+0000 2024-09-13T14:46:35.476 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:0/881490667 expires 2024-09-14T14:10:48.295312+0000 2024-09-13T14:46:35.476 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.99:6800/3838091368 expires 2024-09-14T14:11:18.714884+0000 2024-09-13T14:46:35.476 INFO:teuthology.orchestra.run.smithi099.stdout:blocklist 172.21.15.177:6835/4079938285 expires 2024-09-14T14:30:38.246301+0000 2024-09-13T14:46:35.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:35 smithi099 bash[84576]: cluster 2024-09-13T14:46:34.460789+0000 mgr.y (mgr.26252) 607 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:35.776 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:35 smithi099 bash[84576]: cluster 2024-09-13T14:46:34.460789+0000 mgr.y (mgr.26252) 607 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:35.776 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:35 smithi099 bash[84576]: audit 2024-09-13T14:46:35.472724+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.99:0/761659367' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:46:35.776 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:35 smithi099 bash[84576]: audit 2024-09-13T14:46:35.472724+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.99:0/761659367' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:46:35.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:35 smithi099 bash[87147]: cluster 2024-09-13T14:46:34.460789+0000 mgr.y (mgr.26252) 607 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:35.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:35 smithi099 bash[87147]: cluster 2024-09-13T14:46:34.460789+0000 mgr.y (mgr.26252) 607 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:35.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:35 smithi099 bash[87147]: audit 2024-09-13T14:46:35.472724+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.99:0/761659367' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:46:35.776 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:35 smithi099 bash[87147]: audit 2024-09-13T14:46:35.472724+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.99:0/761659367' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:46:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:35 smithi177 bash[68594]: cluster 2024-09-13T14:46:34.460789+0000 mgr.y (mgr.26252) 607 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:35 smithi177 bash[68594]: cluster 2024-09-13T14:46:34.460789+0000 mgr.y (mgr.26252) 607 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:35 smithi177 bash[68594]: audit 2024-09-13T14:46:35.472724+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.99:0/761659367' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:46:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:35 smithi177 bash[68594]: audit 2024-09-13T14:46:35.472724+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.99:0/761659367' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:46:36.388 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 575 up_thru 899 down_at 558 last_clean_interval [8,558) [v2:172.21.15.99:6802/1813984480,v1:172.21.15.99:6803/1813984480] [v2:172.21.15.99:6804/1813984480,v1:172.21.15.99:6805/1813984480] exists,up 0624feaa-339b-4468-be53-582a1f04352c', 'osd.1 up in weight 1 up_from 642 up_thru 899 down_at 639 last_clean_interval [13,638) [v2:172.21.15.99:6810/804978824,v1:172.21.15.99:6811/804978824] [v2:172.21.15.99:6812/804978824,v1:172.21.15.99:6813/804978824] exists,up b7b63553-e49d-4636-b3e2-43b72556a1a6', 'osd.2 up in weight 1 up_from 672 up_thru 899 down_at 657 last_clean_interval [18,656) [v2:172.21.15.99:6818/129558315,v1:172.21.15.99:6819/129558315] [v2:172.21.15.99:6820/129558315,v1:172.21.15.99:6821/129558315] exists,up 3b9eff6d-5f5c-4843-a1ff-815a902167ec', 'osd.3 up in weight 1 up_from 706 up_thru 899 down_at 690 last_clean_interval [26,689) [v2:172.21.15.99:6826/876595003,v1:172.21.15.99:6827/876595003] [v2:172.21.15.99:6828/876595003,v1:172.21.15.99:6829/876595003] exists,up 686c8d3c-9cab-4920-b02b-d243324f5808', 'osd.4 up in weight 1 up_from 739 up_thru 899 down_at 725 last_clean_interval [31,724) [v2:172.21.15.177:6800/2135282964,v1:172.21.15.177:6801/2135282964] [v2:172.21.15.177:6802/2135282964,v1:172.21.15.177:6803/2135282964] exists,up 2d56712b-a6fb-4089-9583-850beed92b37', 'osd.5 up in weight 1 up_from 37 up_thru 899 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6808/947070587,v1:172.21.15.177:6809/947070587] [v2:172.21.15.177:6810/947070587,v1:172.21.15.177:6811/947070587] exists,up 414b880c-6534-495f-a54c-f8d28ba81c73', 'osd.6 up in weight 1 up_from 43 up_thru 899 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6816/3817692594,v1:172.21.15.177:6817/3817692594] [v2:172.21.15.177:6818/3817692594,v1:172.21.15.177:6819/3817692594] exists,up ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b', 'osd.7 up in weight 1 up_from 49 up_thru 899 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6824/1984676946,v1:172.21.15.177:6825/1984676946] [v2:172.21.15.177:6826/1984676946,v1:172.21.15.177:6827/1984676946] exists,up ad2c2977-fd6d-42f8-962e-907c2bf3204b'] 2024-09-13T14:46:36.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon mon.a config show 2024-09-13T14:46:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:46:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:46:37] "GET /metrics HTTP/1.1" 200 34272 "" "Prometheus/2.51.0" 2024-09-13T14:46:37.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:37 smithi177 bash[68594]: cluster 2024-09-13T14:46:36.461381+0000 mgr.y (mgr.26252) 608 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:37.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:37 smithi177 bash[68594]: cluster 2024-09-13T14:46:36.461381+0000 mgr.y (mgr.26252) 608 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:38.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:37 smithi099 bash[84576]: cluster 2024-09-13T14:46:36.461381+0000 mgr.y (mgr.26252) 608 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:38.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:37 smithi099 bash[84576]: cluster 2024-09-13T14:46:36.461381+0000 mgr.y (mgr.26252) 608 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:38.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:37 smithi099 bash[87147]: cluster 2024-09-13T14:46:36.461381+0000 mgr.y (mgr.26252) 608 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:38.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:37 smithi099 bash[87147]: cluster 2024-09-13T14:46:36.461381+0000 mgr.y (mgr.26252) 608 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:39 smithi177 bash[68594]: cluster 2024-09-13T14:46:38.462334+0000 mgr.y (mgr.26252) 609 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:39 smithi177 bash[68594]: cluster 2024-09-13T14:46:38.462334+0000 mgr.y (mgr.26252) 609 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:40.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:39 smithi099 bash[84576]: cluster 2024-09-13T14:46:38.462334+0000 mgr.y (mgr.26252) 609 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:40.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:39 smithi099 bash[84576]: cluster 2024-09-13T14:46:38.462334+0000 mgr.y (mgr.26252) 609 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:40.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:39 smithi099 bash[87147]: cluster 2024-09-13T14:46:38.462334+0000 mgr.y (mgr.26252) 609 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:40.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:39 smithi099 bash[87147]: cluster 2024-09-13T14:46:38.462334+0000 mgr.y (mgr.26252) 609 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:40.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:46:40 smithi177 bash[61439]: ts=2024-09-13T14:46:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:41.178 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:41 smithi177 bash[68594]: cluster 2024-09-13T14:46:40.463196+0000 mgr.y (mgr.26252) 610 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:41 smithi177 bash[68594]: cluster 2024-09-13T14:46:40.463196+0000 mgr.y (mgr.26252) 610 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:41 smithi099 bash[84576]: cluster 2024-09-13T14:46:40.463196+0000 mgr.y (mgr.26252) 610 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:41 smithi099 bash[84576]: cluster 2024-09-13T14:46:40.463196+0000 mgr.y (mgr.26252) 610 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:41 smithi099 bash[87147]: cluster 2024-09-13T14:46:40.463196+0000 mgr.y (mgr.26252) 610 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:41 smithi099 bash[87147]: cluster 2024-09-13T14:46:40.463196+0000 mgr.y (mgr.26252) 610 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:43 smithi177 bash[68594]: cluster 2024-09-13T14:46:42.464017+0000 mgr.y (mgr.26252) 611 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:43 smithi177 bash[68594]: cluster 2024-09-13T14:46:42.464017+0000 mgr.y (mgr.26252) 611 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:44.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:43 smithi099 bash[84576]: cluster 2024-09-13T14:46:42.464017+0000 mgr.y (mgr.26252) 611 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:44.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:43 smithi099 bash[84576]: cluster 2024-09-13T14:46:42.464017+0000 mgr.y (mgr.26252) 611 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:44.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:43 smithi099 bash[87147]: cluster 2024-09-13T14:46:42.464017+0000 mgr.y (mgr.26252) 611 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:44.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:43 smithi099 bash[87147]: cluster 2024-09-13T14:46:42.464017+0000 mgr.y (mgr.26252) 611 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:45.027 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:46:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "name": "mon.a", 2024-09-13T14:46:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "cluster": "ceph", 2024-09-13T14:46:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-09-13T14:46:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "admin_socket_mode": "", 2024-09-13T14:46:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-09-13T14:46:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "auth_client_required": "cephx, none", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "auth_cluster_required": "cephx", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "auth_debug": "true", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "auth_service_required": "cephx", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "auth_service_ticket_ttl": "240.000000", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "auth_supported": "", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_aio": "true", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_aio_max_queue_depth": "1024", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_aio_poll_ms": "250", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_aio_reap_max": "16", 2024-09-13T14:46:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_async_discard_threads": "0", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_block_size": "4096", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_debug_aio": "false", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_debug_inflight_ios": "false", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_enable_discard": "false", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_flock_retry": "3", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_flock_retry_interval": "0.100000", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_inject_crash": "0", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_inject_crash_flush_delay": "2", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_ioring": "false", 2024-09-13T14:46:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_ioring_hipri": "false", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_ioring_sqthread_poll": "false", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_read_buffer_alignment": "4096", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bdev_type": "", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_alloc_size": "1048576", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_allocator": "hybrid", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_buffered_io": "true", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_check_for_zeros": "false", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_check_volume_selector_often": "false", 2024-09-13T14:46:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-09-13T14:46:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_compact_log_sync": "false", 2024-09-13T14:46:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-09-13T14:46:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-09-13T14:46:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_log_compact_min_size": "16777216", 2024-09-13T14:46:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_log_replay_check_allocations": "true", 2024-09-13T14:46:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_max_log_runway": "4194304", 2024-09-13T14:46:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_max_prefetch": "1048576", 2024-09-13T14:46:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_min_flush_size": "524288", 2024-09-13T14:46:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_min_log_runway": "1048576", 2024-09-13T14:46:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_replay_recovery": "false", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_shared_alloc_size": "65536", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluefs_sync_write": "false", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_allocation_from_file": "true", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_allocator": "hybrid", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_async_db_compaction": "true", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-09-13T14:46:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_bdev_label_multi": "true", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_bdev_label_require_all": "true", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_blobid_prealloc": "10240", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_block_create": "true", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_block_db_create": "false", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_block_db_path": "", 2024-09-13T14:46:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_block_db_size": "0", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_block_path": "", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_block_preallocate_file": "false", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_block_size": "107374182400", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_block_wal_create": "false", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_block_wal_path": "", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_block_wal_size": "100663296", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_bluefs": "true", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_bluefs_env_mirror": "false", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-09-13T14:46:45.034 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_autotune": "true", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_size": "0", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-09-13T14:46:45.035 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cache_type": "2q", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_clone_cow": "true", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_compression_algorithm": "snappy", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_compression_max_blob_size": "0", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_compression_min_blob_size": "0", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_compression_mode": "none", 2024-09-13T14:46:45.036 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_csum_type": "crc32c", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_enforce_settings": "default", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_freelist": "false", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_fsck_abort": "false", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_inject_read_err": "false", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_legacy_omap": "false", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_small_allocations": "0", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_default_buffered_read": "true", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_default_buffered_write": "false", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_deferred_batch_ops": "0", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-09-13T14:46:45.038 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_elastic_shared_blobs": "true", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fail_eio": "false", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_on_mkfs": "true", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-09-13T14:46:45.039 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_on_mount": "false", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_on_umount": "false", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_ignore_data_csum": "false", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-09-13T14:46:45.040 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_kvbackend": "rocksdb", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_log_op_age": "5.000000", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_max_alloc_size": "0", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_max_blob_size": "0", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_max_defer_interval": "3.000000", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_max_deferred_txc": "32", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_min_alloc_size": "0", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-09-13T14:46:45.041 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_nid_prealloc": "1024", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_prefer_deferred_size": "0", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_qfsck_on_mount": "true", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_retry_disk_reads": "3", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_rocksdb_cf": "true", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_rocksdb_options_annex": "", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-09-13T14:46:45.042 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_spdk_coremask": "0x1", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_spdk_io_sleep": "5", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_spdk_max_io_completion": "0", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_spdk_mem": "512", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_sync_submit_transaction": "false", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_throttle_bytes": "67108864", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_throttle_cost_per_io": "0", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_tracing": "false", 2024-09-13T14:46:45.043 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_volume_selection_reserved": "0", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_write_v2": "false", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_write_v2_random": "false", 2024-09-13T14:46:45.044 INFO:teuthology.orchestra.run.smithi099.stdout: "bluestore_zero_block_detection": "false", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephfs_mirror_action_update_interval": "2", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephfs_mirror_mount_timeout": "10", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-09-13T14:46:45.045 INFO:teuthology.orchestra.run.smithi099.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "cephx_cluster_require_signatures": "false", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "cephx_cluster_require_version": "2", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "cephx_require_signatures": "false", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "cephx_require_version": "2", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "cephx_service_require_signatures": "false", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "cephx_service_require_version": "2", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "cephx_sign_messages": "true", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "chdir": "", 2024-09-13T14:46:45.046 INFO:teuthology.orchestra.run.smithi099.stdout: "client_acl_type": "", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_asio_thread_count": "2", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_cache_mid": "0.750000", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_cache_size": "16384", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_caps_release_delay": "5", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_check_pool_perm": "true", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_collect_and_send_global_metrics": "false", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_debug_force_sync_read": "false", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_debug_getattr_caps": "false", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_debug_inject_features": "", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_debug_inject_tick_delay": "0", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_die_on_failed_remount": "false", 2024-09-13T14:46:45.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_dirsize_rbytes": "true", 2024-09-13T14:46:45.048 INFO:teuthology.orchestra.run.smithi099.stdout: "client_force_lazyio": "false", 2024-09-13T14:46:45.048 INFO:teuthology.orchestra.run.smithi099.stdout: "client_fs": "", 2024-09-13T14:46:45.048 INFO:teuthology.orchestra.run.smithi099.stdout: "client_inject_fixed_oldest_tid": "false", 2024-09-13T14:46:45.048 INFO:teuthology.orchestra.run.smithi099.stdout: "client_inject_release_failure": "false", 2024-09-13T14:46:45.048 INFO:teuthology.orchestra.run.smithi099.stdout: "client_max_inline_size": "4096", 2024-09-13T14:46:45.048 INFO:teuthology.orchestra.run.smithi099.stdout: "client_max_retries_on_remount_failure": "5", 2024-09-13T14:46:45.048 INFO:teuthology.orchestra.run.smithi099.stdout: "client_mds_namespace": "", 2024-09-13T14:46:45.048 INFO:teuthology.orchestra.run.smithi099.stdout: "client_metadata": "", 2024-09-13T14:46:45.048 INFO:teuthology.orchestra.run.smithi099.stdout: "client_mount_gid": "-1", 2024-09-13T14:46:45.048 INFO:teuthology.orchestra.run.smithi099.stdout: "client_mount_timeout": "300", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_mount_uid": "-1", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_mountpoint": "/", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_notify_timeout": "10", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_oc": "true", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_oc_max_dirty": "104857600", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_oc_max_dirty_age": "5.000000", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_oc_max_objects": "1000", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_oc_size": "209715200", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_oc_target_dirty": "8388608", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_permissions": "true", 2024-09-13T14:46:45.049 INFO:teuthology.orchestra.run.smithi099.stdout: "client_quota": "true", 2024-09-13T14:46:45.050 INFO:teuthology.orchestra.run.smithi099.stdout: "client_quota_df": "true", 2024-09-13T14:46:45.050 INFO:teuthology.orchestra.run.smithi099.stdout: "client_readahead_max_bytes": "0", 2024-09-13T14:46:45.050 INFO:teuthology.orchestra.run.smithi099.stdout: "client_readahead_max_periods": "4", 2024-09-13T14:46:45.050 INFO:teuthology.orchestra.run.smithi099.stdout: "client_readahead_min": "131072", 2024-09-13T14:46:45.050 INFO:teuthology.orchestra.run.smithi099.stdout: "client_reconnect_stale": "false", 2024-09-13T14:46:45.050 INFO:teuthology.orchestra.run.smithi099.stdout: "client_shutdown_timeout": "30", 2024-09-13T14:46:45.050 INFO:teuthology.orchestra.run.smithi099.stdout: "client_snapdir": ".snap", 2024-09-13T14:46:45.050 INFO:teuthology.orchestra.run.smithi099.stdout: "client_tick_interval": "1", 2024-09-13T14:46:45.050 INFO:teuthology.orchestra.run.smithi099.stdout: "client_trace": "", 2024-09-13T14:46:45.050 INFO:teuthology.orchestra.run.smithi099.stdout: "client_try_dentry_invalidate": "false", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "client_use_faked_inos": "false", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "client_use_random_mds": "false", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "clog_to_graylog": "false", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "clog_to_graylog_port": "12201", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "clog_to_monitors": "default=true", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "clog_to_syslog": "false", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "clog_to_syslog_level": "info", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "cluster_addr": "-", 2024-09-13T14:46:45.051 INFO:teuthology.orchestra.run.smithi099.stdout: "cluster_network": "", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "cluster_network_interface": "", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "compressor_zlib_isal": "false", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "compressor_zlib_level": "5", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "compressor_zlib_winsize": "-15", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "compressor_zstd_level": "1", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "crimson_alien_op_num_threads": "6", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "crimson_alien_thread_cpu_cores": "", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "crimson_osd_obc_lru_size": "512", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-09-13T14:46:45.052 INFO:teuthology.orchestra.run.smithi099.stdout: "crimson_osd_stat_interval": "0", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "crimson_seastar_cpu_cores": "", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "crimson_seastar_num_threads": "0", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_location": "", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_location_hook": "", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "crush_location_hook_timeout": "10", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "daemonize": "false", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "daos_pool": "tank", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "dbstore_db_name_prefix": "dbstore", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_alienstore": "0/5", 2024-09-13T14:46:45.053 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_allow_any_pool_priority": "false", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_asok": "1/5", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_asok_assert_abort": "false", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_asserts_on_shutdown": "true", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_auth": "1/5", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_bdev": "1/3", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_bluefs": "1/5", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_bluestore": "1/5", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_buffer": "0/1", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_ceph_dedup": "0/5", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_ceph_exporter": "1/5", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_cephfs_mirror": "0/5", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_cephsqlite": "0/5", 2024-09-13T14:46:45.054 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_client": "0/5", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_compressor": "1/5", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_context": "0/1", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_crimson_interrupt": "0/5", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_crush": "1/1", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_crypto": "1/5", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_cyanstore": "0/5", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_deliberately_leak_memory": "false", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_disable_randomized_ping": "false", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_dpdk": "1/5", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_eventtrace": "1/5", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_filer": "0/1", 2024-09-13T14:46:45.055 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_filestore": "1/3", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_finisher": "1/1", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_fuse": "1/5", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_heartbeat_testing_span": "0", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_heartbeatmap": "1/5", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_immutable_obj_cache": "0/5", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_javaclient": "1/5", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_journal": "1/3", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_journaler": "0/5", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_kstore": "1/5", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_lockdep": "0/1", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mclock": "1/5", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mds": "1/5", 2024-09-13T14:46:45.056 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mds_balancer": "1/5", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mds_locker": "1/5", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mds_log": "1/5", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mds_log_expire": "1/5", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mds_migrator": "1/5", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mds_quiesce": "3/5", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_memstore": "1/5", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mgr": "2/5", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mgrc": "1/5", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_mon": "20/20", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_monc": "0/10", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_ms": "1/1", 2024-09-13T14:46:45.057 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_none": "0/5", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_objclass": "0/5", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_objectcacher": "0/5", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_objecter": "0/1", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_optracker": "0/5", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_osd": "1/5", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_paxos": "20/20", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_perfcounter": "1/5", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_prioritycache": "1/5", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rados": "0/5", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rbd": "0/5", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rbd_mirror": "0/5", 2024-09-13T14:46:45.058 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rbd_pwl": "0/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rbd_replay": "0/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_refs": "0/0", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_reserver": "1/1", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rgw": "1/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rgw_access": "1/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rgw_datacache": "1/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rgw_dbstore": "1/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rgw_flight": "1/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rgw_lifecycle": "1/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rgw_notification": "1/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rgw_sync": "1/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_rocksdb": "4/5", 2024-09-13T14:46:45.059 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_backref": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_cache": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_cleaner": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_device": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_epm": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_journal": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_lba": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_odata": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_omap": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_onode": "0/5", 2024-09-13T14:46:45.060 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_t": "0/5", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_seastore_tm": "0/5", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_striper": "0/1", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_test": "0/5", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_throttle": "1/1", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_timer": "0/1", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_tp": "0/5", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "debug_trace": "1/5", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "device_failure_prediction_mode": "none", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "err_to_graylog": "false", 2024-09-13T14:46:45.061 INFO:teuthology.orchestra.run.smithi099.stdout: "err_to_journald": "false", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "err_to_stderr": "true", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "err_to_syslog": "false", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event_tracing": "false", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "exporter_addr": "0.0.0.0", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "exporter_cert_file": "", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "exporter_http_port": "9926", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "exporter_key_file": "", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "exporter_prio_limit": "5", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "exporter_sort_metrics": "true", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "exporter_stats_period": "5", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "fake_statfs_for_testing": "0", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "fatal_signal_handlers": "true", 2024-09-13T14:46:45.062 INFO:teuthology.orchestra.run.smithi099.stdout: "filer_max_purge_ops": "10", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filer_max_truncate_ops": "128", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_apply_finisher_threads": "1", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_blackhole": "false", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_btrfs_clone_range": "true", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_btrfs_snap": "true", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_caller_concurrency": "10", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_collect_device_partition_information": "true", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_commit_timeout": "600.000000", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_debug_inject_read_err": "false", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_debug_omap_check": "false", 2024-09-13T14:46:45.063 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_debug_verify_split": "false", 2024-09-13T14:46:45.064 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_dump_file": "", 2024-09-13T14:46:45.064 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-09-13T14:46:45.064 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-09-13T14:46:45.064 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_fadvise": "true", 2024-09-13T14:46:45.064 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_fail_eio": "true", 2024-09-13T14:46:45.064 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_fd_cache_shards": "16", 2024-09-13T14:46:45.064 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_fd_cache_size": "128", 2024-09-13T14:46:45.064 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_fiemap": "false", 2024-09-13T14:46:45.064 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_fiemap_threshold": "4096", 2024-09-13T14:46:45.064 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_index_retry_probability": "0.000000", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_inject_stall": "0", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_journal_parallel": "false", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_journal_trailing": "false", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_journal_writeahead": "false", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_kill_at": "0", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_inline_xattr_size": "0", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-09-13T14:46:45.065 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_inline_xattrs": "0", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_inline_xattrs_other": "2", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_sync_interval": "5.000000", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_xattr_value_size": "0", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_merge_threshold": "-10", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_min_sync_interval": "0.010000", 2024-09-13T14:46:45.066 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_odsync_write": "false", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_omap_backend": "rocksdb", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_omap_backend_path": "", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_omap_header_cache_size": "1024", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_ondisk_finisher_threads": "1", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_op_thread_timeout": "60", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_op_threads": "2", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_punch_hole": "false", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-09-13T14:46:45.067 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-09-13T14:46:45.068 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-09-13T14:46:45.068 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-09-13T14:46:45.068 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_queue_max_bytes": "104857600", 2024-09-13T14:46:45.068 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-09-13T14:46:45.068 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-09-13T14:46:45.068 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-09-13T14:46:45.068 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_queue_max_ops": "50", 2024-09-13T14:46:45.068 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-09-13T14:46:45.068 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_seek_data_hole": "false", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_sloppy_crc": "false", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_splice": "false", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_split_multiple": "2", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_split_rand_factor": "20", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_update_to": "1000", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-09-13T14:46:45.069 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_enable": "true", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_xfs_extsize": "false", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "filestore_zfs_snap": "false", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "fio_dir": "/tmp/fio", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "fsid": "af9d2c22-71d9-11ef-bceb-c7b262605968", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_allow_other": "true", 2024-09-13T14:46:45.070 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_atomic_o_trunc": "true", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_big_writes": "true", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_debug": "false", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_default_permissions": "false", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_disable_pagecache": "false", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_max_write": "0", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_multithreaded": "true", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_require_active_mds": "true", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_set_user_groups": "true", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_splice_move": "true", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_splice_read": "true", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_splice_write": "true", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_syncfs_on_mksnap": "true", 2024-09-13T14:46:45.071 INFO:teuthology.orchestra.run.smithi099.stdout: "fuse_use_invalidate_cb": "true", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "gss_target_name": "ceph", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "heartbeat_file": "", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "heartbeat_inject_failure": "0", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "heartbeat_interval": "5", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "host": "smithi099", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_max_size": "1073741824", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-09-13T14:46:45.072 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "immutable_object_cache_watermark": "0.900000", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "inject_early_sigterm": "false", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "jaeger_agent_port": "6799", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "jaeger_tracing_enable": "false", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_aio": "true", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_align_min_size": "65536", 2024-09-13T14:46:45.073 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_block_align": "true", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_block_size": "4096", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_dio": "true", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_discard": "false", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_force_aio": "false", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_ignore_corruption": "false", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_max_write_bytes": "10485760", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_max_write_entries": "100", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_replay_from": "0", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_throttle_high_multiple": "0.000000", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_throttle_max_multiple": "0.000000", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_write_header_frequency": "0", 2024-09-13T14:46:45.074 INFO:teuthology.orchestra.run.smithi099.stdout: "journal_zero_on_create": "false", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "journaler_prefetch_periods": "10", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "journaler_prezero_periods": "5", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "journaler_write_head_interval": "15", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "key": "", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "keyfile": "", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_backend": "rocksdb", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_default_stripe_size": "65536", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_fsck_on_mount": "false", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_fsck_on_mount_deep": "true", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_max_bytes": "67108864", 2024-09-13T14:46:45.075 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_max_ops": "512", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_nid_prealloc": "1024", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_onode_map_size": "1024", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_sync_submit_transaction": "false", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "kstore_sync_transaction": "false", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "librados_thread_count": "2", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "lockdep": "false", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "lockdep_force_backtrace": "false", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "log_coarse_timestamps": "true", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "log_flush_on_exit": "false", 2024-09-13T14:46:45.076 INFO:teuthology.orchestra.run.smithi099.stdout: "log_graylog_host": "127.0.0.1", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "log_graylog_port": "12201", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "log_max_new": "1000", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "log_max_recent": "10000", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "log_stderr_prefix": "debug ", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "log_stop_at_utilization": "0.970000", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_file": "true", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_graylog": "false", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_journald": "false", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_stderr": "false", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "log_to_syslog": "false", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "max_rotating_auth_attempts": "10", 2024-09-13T14:46:45.077 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_action_on_write_error": "1", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_alternate_name_max": "8192", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_asio_thread_count": "2", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_export_pin": "true", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_fragment_dirs": "true", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_fragment_interval": "5", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_fragment_size_max": "100000", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_idle_threshold": "0.000000", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_interval": "10", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_max": "-1", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_max_until": "-1", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_merge_size": "50", 2024-09-13T14:46:45.078 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_midchunk": "0.300000", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_min_rebalance": "0.100000", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_min_start": "0.200000", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_minchunk": "0.001000", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_mode": "0", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_need_max": "1.200000", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_need_min": "0.800000", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_overload_epochs": "2", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_sample_interval": "3.000000", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_split_bits": "3", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_split_rd": "25000.000000", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_split_size": "10000", 2024-09-13T14:46:45.079 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_split_wr": "10000.000000", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_target_decay": "10.000000", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_beacon_grace": "15.000000", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_beacon_interval": "4.000000", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_beacon_mon_down_grace": "60", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_memory_limit": "4294967296", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_mid": "0.700000", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_quiesce_delay": "0", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_quiesce_sleep": "200", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_quiesce_splitauth": "true", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_quiesce_threshold": "524288", 2024-09-13T14:46:45.080 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_release_free_interval": "10", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_reservation": "0.050000", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_trim_interval": "1", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cache_trim_threshold": "262144", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_client_delegate_inos_pct": "50", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_client_prealloc_inos": "1000", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_connect_bootstrapping": "false", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_damage_table_max_entries": "10000", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-09-13T14:46:45.081 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_debug_auth_pins": "false", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_debug_frag": "false", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_debug_scatterstat": "false", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_debug_subtrees": "false", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_decay_halflife": "5.000000", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_default_dir_hash": "2", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_defer_session_stale": "true", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_deny_all_reconnect": "false", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_dir_keys_per_op": "16384", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_dir_max_commit_size": "10", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_dir_max_entries": "0", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_dir_prefetch": "true", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_dirstat_min_interval": "1.000000", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_dump_cache_after_rejoin": "false", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_dump_cache_on_map": "false", 2024-09-13T14:46:45.082 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_dump_cache_threshold_file": "0", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_early_reply": "true", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_enable_op_tracker": "true", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_enforce_unique_name": "true", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_export_ephemeral_distributed": "true", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_export_ephemeral_random": "true", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_forward_all_requests_to_auth": "false", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-09-13T14:46:45.083 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_health_cache_threshold": "1.500000", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_health_summarize_threshold": "10", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_heartbeat_grace": "15.000000", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_heartbeat_reset_grace": "1000", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_inject_health_dummy": "false", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_inject_migrator_session_race": "false", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_join_fs": "", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_journal_format": "1", 2024-09-13T14:46:45.084 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_create_at": "0", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_dirfrag_at": "0", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_export_at": "0", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_import_at": "0", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_journal_at": "0", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_journal_expire_at": "0", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_journal_replay_at": "0", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_link_at": "0", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_mdstable_at": "0", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_openc_at": "0", 2024-09-13T14:46:45.085 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_rename_at": "0", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_kill_shutdown_at": "0", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_event_large_threshold": "524288", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_events_per_segment": "1024", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_major_segment_event_ratio": "12", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_max_events": "-1", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_max_segments": "128", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_pause": "false", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_segment_size": "0", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_skip_corrupt_events": "false", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_skip_unbounded_events": "false", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_trim_threshold": "128", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-09-13T14:46:45.086 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_log_warn_factor": "2.000000", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_max_caps_per_client": "1048576", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_max_completed_flushes": "100000", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_max_completed_requests": "100000", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_max_export_size": "20971520", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_max_file_recover": "32", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_max_purge_files": "64", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_max_purge_ops": "8192", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_max_snaps_per_dir": "100", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_metrics_update_interval": "2", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_min_caps_per_client": "100", 2024-09-13T14:46:45.087 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_min_caps_working_set": "10000", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_numa_node": "-1", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_op_complaint_time": "30.000000", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_op_history_duration": "600", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_op_history_size": "20", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_op_history_slow_op_size": "20", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_op_history_slow_op_threshold": "10", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_op_log_threshold": "5", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_ping_grace": "15", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_ping_interval": "5", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-09-13T14:46:45.088 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_recall_max_caps": "30000", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_recall_max_decay_threshold": "131072", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_recall_warning_threshold": "262144", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_reconnect_timeout": "45.000000", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_replay_interval": "1.000000", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_root_ino_gid": "0", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_root_ino_uid": "0", 2024-09-13T14:46:45.089 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_session_blocklist_on_evict": "true", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_session_blocklist_on_timeout": "true", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_session_metadata_threshold": "16777216", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_shutdown_check": "0", 2024-09-13T14:46:45.090 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_skip_ino": "0", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_sleep_rank_change": "0.000000", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_snap_max_uid": "4294967294", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_snap_min_uid": "0", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_snap_rstat": "false", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_standby_replay_damaged": "false", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_symlink_recovery": "true", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_task_status_update_interval": "2.000000", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_thrash_exports": "0", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_thrash_fragments": "0", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_tick_interval": "5.000000", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_valgrind_exit": "false", 2024-09-13T14:46:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_verify_backtrace": "1", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_verify_scatter": "false", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_wipe_ino_prealloc": "false", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "mds_wipe_sessions": "false", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "mempool_debug": "false", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "memstore_debug_omit_block_device_write": "false", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "memstore_device_bytes": "1073741824", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "memstore_page_set": "false", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "memstore_page_size": "65536", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_client_bytes": "134217728", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_client_messages": "512", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_connect_retry_interval": "1.000000", 2024-09-13T14:46:45.092 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_disabled_modules": "", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_enable_op_tracker": "true", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_max_pg_creating": "1024", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_max_pg_num_change": "128", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_mds_bytes": "134217728", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_mds_messages": "128", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_mon_bytes": "134217728", 2024-09-13T14:46:45.093 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_mon_messages": "128", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_num_op_tracker_shard": "32", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_op_complaint_time": "30.000000", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_op_history_duration": "600", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_op_history_size": "20", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_op_history_slow_op_size": "20", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_op_log_threshold": "5", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_osd_bytes": "536870912", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_osd_messages": "8192", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_pool": "true", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_service_beacon_grace": "60.000000", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_standby_modules": "true", 2024-09-13T14:46:45.094 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_stats_period": "5", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_stats_threshold": "5", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_tick_period": "2", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_accept_timeout_factor": "2.000000", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_allow_pool_delete": "true", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_allow_pool_size_one": "false", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_auth_validate_all_caps": "true", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_bytes": "104857600", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_directed_command_retry": "2", 2024-09-13T14:46:45.095 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_hunt_interval": "3.000000", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_hunt_parallel": "3", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_log_interval": "1.000000", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_ping_interval": "10.000000", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_ping_timeout": "30.000000", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_client_target_rank": "-1", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_clock_drift_allowed": "1.000000", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-09-13T14:46:45.096 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cluster_log_level": "debug", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cluster_log_to_file": "true", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cluster_log_to_graylog": "false", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cluster_log_to_journald": "false", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cluster_log_to_stderr": "true", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_compact_on_bootstrap": "false", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_compact_on_start": "false", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_compact_on_trim": "true", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_con_tracker_persist_interval": "10", 2024-09-13T14:46:45.097 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_con_tracker_score_halflife": "43200", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_config_key_max_entry_size": "65536", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_cpu_threads": "4", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_crush_min_required_version": "hammer", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_daemon_bytes": "419430400", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_data_avail_crit": "5", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_data_avail_warn": "5", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_data_size_warn": "16106127360", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_block_osdmap_trim": "false", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_dump_json": "false", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-09-13T14:46:45.098 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_dump_transactions": "false", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_extra_checks": "false", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_no_require_reef": "false", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_no_require_squid": "false", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_delta_reset_interval": "10.000000", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_dns_srv_name": "ceph-mon", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_down_mkfs_grace": "60", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_down_uptime_grace": "60", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_election_default_strategy": "3", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_election_timeout": "5.000000", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-09-13T14:46:45.099 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_elector_ping_divisor": "2", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_elector_ping_timeout": "2.000000", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_enable_op_tracker": "true", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_fake_pool_delete": "false", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_force_quorum_join": "false", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_fsmap_prune_threshold": "300", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_globalid_prealloc": "10000", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_health_detail_to_clog": "true", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_health_log_update_period": "5", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_health_max_detail": "50", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_health_to_clog": "true", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_health_to_clog_interval": "600", 2024-09-13T14:46:45.100 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_host": "[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] [v2:172.21.15.99:3301/0,v1:172.21.15.99:6790/0]", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_host_override": "", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_initial_members": "", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_keyvaluedb": "rocksdb", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_lease": "5.000000", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_log_full_interval": "50", 2024-09-13T14:46:45.101 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_log_max": "10000", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_log_max_summary": "50", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_max_log_entries_per_event": "4096", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_max_log_epochs": "500", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_max_mdsmap_epochs": "500", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_max_mgrmap_epochs": "500", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_max_nvmeof_epochs": "500", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_max_osd": "10000", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_max_pg_per_osd": "10000", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_max_pool_pg_num": "65536", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-09-13T14:46:45.102 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_mds_force_trim_to": "0", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_mds_skip_sanity": "true", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_memory_autotune": "true", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_memory_target": "2147483648", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_mgr_beacon_grace": "30", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_mgr_digest_period": "5", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_mgr_inactive_grace": "60", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_mgr_mkfs_grace": "240", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_min_osdmap_epochs": "500", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_op_complaint_time": "30", 2024-09-13T14:46:45.103 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_op_history_duration": "600", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_op_history_size": "20", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_op_history_slow_op_size": "20", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_op_history_slow_op_threshold": "10", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_op_log_threshold": "5", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_auto_mark_in": "false", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_auto_mark_new_in": "true", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_cache_size": "500", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_cache_size_min": "134217728", 2024-09-13T14:46:45.104 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_crush_smoke_test": "true", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_destroyed_out_interval": "600", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_down_out_interval": "600", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_force_trim_to": "0", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_full_ratio": "0.950000", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_laggy_halflife": "3600", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_laggy_max_interval": "300", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_laggy_weight": "0.300000", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_max_initial_pgs": "1024", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_min_down_reporters": "2", 2024-09-13T14:46:45.105 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_prime_pg_temp": "true", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_report_timeout": "900", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_warn_num_repaired": "10", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osd_warn_op_age": "32.000000", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osdmap_full_prune_interval": "10", 2024-09-13T14:46:45.106 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osdmap_full_prune_min": "10000", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_pg_stuck_threshold": "60", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_pg_warn_min_objects": "10000", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_pg_warn_min_per_osd": "0", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_pool_quota_crit_threshold": "0", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_pool_quota_warn_threshold": "0", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_probe_timeout": "2.000000", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_reweight_max_change": "0.050000", 2024-09-13T14:46:45.107 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_reweight_max_osds": "4", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_scrub_interval": "86400", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_scrub_max_keys": "100", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_scrub_timeout": "300", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_session_timeout": "300", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_smart_report_timeout": "5", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_stat_smooth_intervals": "6", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-09-13T14:46:45.108 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_stretch_pool_min_size": "2", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_stretch_pool_size": "4", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_subscribe_interval": "86400.000000", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_sync_debug": "false", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_sync_max_payload_keys": "2000", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_sync_max_payload_size": "1048576", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_sync_provider_kill_at": "0", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_sync_requester_kill_at": "0", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_sync_timeout": "60.000000", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_target_pg_per_osd": "100", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_tick_interval": "5", 2024-09-13T14:46:45.109 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_timecheck_interval": "300.000000", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_use_min_delay_socket": "false", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_older_version_delay": "604800", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_filestore_osds": "true", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_misplaced": "false", 2024-09-13T14:46:45.110 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-09-13T14:46:45.111 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_older_version": "true", 2024-09-13T14:46:45.111 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-09-13T14:46:45.111 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_pool_no_app": "true", 2024-09-13T14:46:45.111 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-09-13T14:46:45.111 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-09-13T14:46:45.111 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-09-13T14:46:45.111 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-09-13T14:46:45.111 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_on_too_few_osds": "false", 2024-09-13T14:46:45.111 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-09-13T14:46:45.111 INFO:teuthology.orchestra.run.smithi099.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "monmap": "", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "motr_tracing_enabled": "false", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_op_threads": "3", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_buffer_size": "131072", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_cm": "false", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_device_name": "", 2024-09-13T14:46:45.112 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_dscp": "96", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_gid_idx": "0", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_local_gid": "", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_polling_us": "1000", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_port_num": "1", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_roce_ver": "1", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_send_buffers": "1024", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_sl": "3", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_support_srq": "true", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_rdma_type": "ib", 2024-09-13T14:46:45.113 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_async_reap_threshold": "5", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_bind_before_connect": "false", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_bind_ipv4": "true", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_bind_ipv6": "false", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_bind_msgr1": "true", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_bind_msgr2": "true", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_bind_port_max": "7568", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_bind_port_min": "6800", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_bind_prefer_ipv4": "false", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_bind_retry_count": "3", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_bind_retry_delay": "5", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_blackhole_client": "false", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_blackhole_mds": "false", 2024-09-13T14:46:45.114 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_blackhole_mgr": "false", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_blackhole_mon": "false", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_blackhole_osd": "false", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_client_mode": "crc secure", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_cluster_mode": "crc secure", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_cluster_type": "", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_compress_secure": "false", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_connection_idle_timeout": "900", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_connection_ready_timeout": "10", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_crc_data": "true", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_crc_header": "true", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_die_on_bad_msg": "false", 2024-09-13T14:46:45.115 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_die_on_bug": "true", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_die_on_old_message": "true", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_die_on_skipped_message": "false", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_die_on_unhandled_msg": "false", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_coremask": "0xF", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_devs_allowlist": "", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_enable_tso": "true", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_hugepages": "", 2024-09-13T14:46:45.116 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_hw_flow_control": "true", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_lro": "true", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_memory_channel": "4", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_pmd": "", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_port_id": "0", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dump_corrupt_message_level": "1", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_dump_on_send": "false", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_initial_backoff": "0.200000", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_inject_delay_max": "1.000000", 2024-09-13T14:46:45.117 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_inject_delay_probability": "0.000000", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_inject_delay_type": "", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_inject_internal_delays": "0.000000", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_inject_network_congestion": "0", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_inject_socket_failures": "0", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_learn_addr_from_peer": "true", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_max_accept_failures": "4", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_max_backoff": "15.000000", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_mon_client_mode": "secure crc", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_mon_cluster_mode": "secure crc", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_mon_service_mode": "secure crc", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_osd_compress_min_size": "1024", 2024-09-13T14:46:45.118 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_osd_compress_mode": "none", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_osd_compression_algorithm": "snappy", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_pq_min_cost": "65536", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_public_type": "", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_service_mode": "crc secure", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_tcp_listen_backlog": "512", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_tcp_nodelay": "true", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_tcp_rcvbuf": "0", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_time_events_min_wait_interval": "1000", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ms_type": "async+posix", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "no_config_file": "false", 2024-09-13T14:46:45.119 INFO:teuthology.orchestra.run.smithi099.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "nvmeof_mon_client_tick_period": "2", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "objecter_completion_locks_per_session": "32", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "objecter_debug_inject_relock_delay": "false", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "objecter_inflight_op_bytes": "104857600", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "objecter_inflight_ops": "1024", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "objecter_inject_no_watch_ping": "false", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "objecter_tick_interval": "5.000000", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "objecter_timeout": "10.000000", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "objectstore_blackhole": "false", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "openssl_engine_opts": "", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_agent_delay_time": "5.000000", 2024-09-13T14:46:45.120 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_agent_hist_halflife": "1000", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_agent_max_low_ops": "2", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_agent_max_ops": "4", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_agent_quantize_effort": "0.100000", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_agent_slop": "0.020000", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_allow_recovery_below_min_size": "true", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_asio_thread_count": "2", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_async_recovery_min_cost": "100", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_auto_mark_unfound_lost": "false", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_backfill_retry_interval": "30.000000", 2024-09-13T14:46:45.121 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_backfill_scan_max": "512", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_backfill_scan_min": "64", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_backoff_on_degraded": "false", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_backoff_on_peering": "false", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_backoff_on_unfound": "true", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_beacon_report_interval": "300", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_bench_duration": "30", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_bench_max_block_size": "67108864", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_bench_small_size_max_iops": "100", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_blkin_trace_all": "false", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_blocked_scrub_grace_period": "120", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-09-13T14:46:45.122 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_check_for_log_corruption": "false", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_class_update_on_start": "true", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_client_message_cap": "256", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_client_message_size_cap": "524288000", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_client_op_priority": "63", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_client_watch_timeout": "30", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_command_max_records": "256", 2024-09-13T14:46:45.123 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_compact_on_start": "false", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_crush_chooseleaf_type": "1", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_crush_initial_weight": "-1.000000", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_crush_update_on_start": "true", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_crush_update_weight_set": "true", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_drop_ping_duration": "0", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_feed_pullee": "-1", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-09-13T14:46:45.124 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_misdirected_ops": "false", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_no_acting_change": "false", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_no_purge_strays": "false", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_op_order": "false", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_pg_log_writeout": "false", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_pretend_recovery_active": "false", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_shutdown": "false", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_trim_objects": "false", 2024-09-13T14:46:45.125 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_verify_cached_snaps": "false", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_verify_missing_on_start": "false", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_verify_snaps": "false", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_deep_scrub_keys": "1024", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_deep_scrub_stride": "524288", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_default_data_pool_replay_window": "45", 2024-09-13T14:46:45.126 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_default_notify_timeout": "30", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_delete_sleep": "0.000000", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_discard_disconnected_ops": "true", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_ec_partial_reads": "true", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_enable_op_tracker": "true", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_extblkdev_plugins": "vdo", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-09-13T14:46:45.127 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_fast_info": "true", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_fast_shutdown": "true", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_fast_shutdown_timeout": "15", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_function_tracing": "false", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_heartbeat_grace": "20", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_heartbeat_interval": "6", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_heartbeat_min_peers": "10", 2024-09-13T14:46:45.128 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_heartbeat_min_size": "2000", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_heartbeat_stale": "600", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_hit_set_max_size": "100000", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_hit_set_min_size": "1000", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_journal_flush_on_shutdown": "true", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_journal_size": "5120", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_kill_backfill_at": "0", 2024-09-13T14:46:45.129 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_loop_before_reset_tphandle": "64", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_map_cache_size": "50", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_map_dedup": "true", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_map_message_max": "40", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_map_message_max_bytes": "10485760", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_map_share_max_epochs": "40", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_attr_name_len": "100", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_attr_size": "0", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_backfills": "1", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_markdown_count": "5", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_markdown_period": "600", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_object_name_len": "2048", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_object_namespace_len": "256", 2024-09-13T14:46:45.130 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_object_size": "134217728", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_omap_entries_per_request": "1024", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_pg_blocked_by": "16", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_pg_log_entries": "10000", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_pgls": "1024", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_push_cost": "8388608", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_push_objects": "10", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_scrubs": "3", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_trimming_pgs": "2", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_write_op_reply_len": "64", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_max_write_size": "90", 2024-09-13T14:46:45.131 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_override_recovery_settings": "false", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_profile": "balanced", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-09-13T14:46:45.132 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mclock_skip_benchmark": "false", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_memory_base": "805306368", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_memory_cache_min": "134217728", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-09-13T14:46:45.133 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_memory_target": "4294967296", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_memory_target_autotune": "false", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_min_pg_log_entries": "250", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_min_recovery_priority": "0", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mon_heartbeat_interval": "30", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mon_report_interval": "5", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mon_report_max_in_flight": "2", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_num_cache_shards": "32", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_num_op_tracker_shard": "32", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_numa_auto_affinity": "true", 2024-09-13T14:46:45.134 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_numa_node": "-1", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_numa_prefer_iface": "true", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_objecter_finishers": "1", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_objectstore": "bluestore", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_objectstore_fuse": "false", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_objectstore_ideal_list_max": "64", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_objectstore_tracing": "false", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_complaint_time": "30.000000", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_history_duration": "600", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_history_size": "20", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_history_slow_op_size": "20", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_log_threshold": "5", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_num_shards": "0", 2024-09-13T14:46:45.135 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_num_shards_hdd": "5", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_num_shards_ssd": "8", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_num_threads_per_shard": "0", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_pq_min_cost": "65536", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_queue": "mclock_scheduler", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_queue_cut_off": "high", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_thread_suicide_timeout": "150", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_op_thread_timeout": "15", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_open_classes_on_start": "true", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_os_flags": "0", 2024-09-13T14:46:45.136 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_peering_op_priority": "255", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_delete_cost": "1048576", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_delete_priority": "5", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_log_dups_tracked": "3000", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_log_trim_max": "10000", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_log_trim_min": "100", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_object_context_cache_count": "64", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-09-13T14:46:45.137 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_crimson": "false", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_crush_rule": "-1", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_ec_fast_read": "false", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_flag_bulk": "false", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_flag_nodelete": "false", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_flags": "0", 2024-09-13T14:46:45.138 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_min_size": "0", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_pg_num": "32", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_pgp_num": "0", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_read_ratio": "70", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_size": "3", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_default_type": "replicated", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_pool_use_gmt_hitset": "true", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_push_per_object_cost": "1000", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_read_ec_check_for_errors": "false", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recover_clone_overlap": "true", 2024-09-13T14:46:45.139 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recover_clone_overlap_limit": "10", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_cost": "20971520", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_delay_start": "0.000000", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_max_active": "0", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_max_active_hdd": "3", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_max_active_ssd": "10", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_max_chunk": "8388608", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_max_single_start": "1", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_op_priority": "3", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_op_warn_multiple": "16", 2024-09-13T14:46:45.140 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_priority": "5", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_retry_interval": "30.000000", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_sleep": "0.000000", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_repair_during_recovery": "false", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_requested_scrub_priority": "120", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_rollback_to_cluster_snap": "", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_auto_repair": "false", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-09-13T14:46:45.141 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_begin_hour": "0", 2024-09-13T14:46:45.142 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_begin_week_day": "0", 2024-09-13T14:46:45.142 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_chunk_max": "25", 2024-09-13T14:46:45.142 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_chunk_min": "5", 2024-09-13T14:46:45.142 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_cost": "52428800", 2024-09-13T14:46:45.142 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-09-13T14:46:45.142 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_during_recovery": "false", 2024-09-13T14:46:45.142 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_end_hour": "0", 2024-09-13T14:46:45.142 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_end_week_day": "0", 2024-09-13T14:46:45.142 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_event_cost": "4096", 2024-09-13T14:46:45.142 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_invalid_stats": "true", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_load_threshold": "0.500000", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_max_interval": "604800.000000", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_max_preemptions": "5", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_min_interval": "86400.000000", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_priority": "5", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_scrub_sleep": "0.000000", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_shutdown_pgref_assert": "false", 2024-09-13T14:46:45.143 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_skip_data_digest": "false", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_smart_report_timeout": "5", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_snap_trim_cost": "1048576", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_snap_trim_priority": "5", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_snap_trim_sleep": "0.000000", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_stats_update_period_scrubbing": "15", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-09-13T14:46:45.144 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_target_transaction_size": "30", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tier_default_cache_mode": "writeback", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_tracing": "false", 2024-09-13T14:46:45.145 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_use_stale_snap": "false", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "osdc_blkin_trace_all": "false", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_kill_at": "0", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_max_join_drift": "10", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_min": "500", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_min_wait": "0.050000", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_propose_interval": "1.000000", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_service_trim_max": "500", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_service_trim_max_multiplier": "20", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_service_trim_min": "250", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_stash_full_interval": "25", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_trim_max": "500", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "paxos_trim_min": "250", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "perf": "true", 2024-09-13T14:46:45.146 INFO:teuthology.orchestra.run.smithi099.stdout: "pid_file": "", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "public_addr": "-", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "public_addrv": "", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "public_bind_addr": "-", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "public_network": "172.21.15.254/32,172.21.0.2/32,172.21.0.0/20,172.21.0.1/32", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "public_network_interface": "", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "qat_compressor_busy_polling": "false", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "qat_compressor_enabled": "false", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "qat_compressor_session_max_number": "256", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "rados_mon_op_timeout": "0", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "rados_osd_op_timeout": "0", 2024-09-13T14:46:45.147 INFO:teuthology.orchestra.run.smithi099.stdout: "rados_tracing": "false", 2024-09-13T14:46:45.148 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_atime_update_interval": "60", 2024-09-13T14:46:45.148 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-09-13T14:46:45.148 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_balance_parent_reads": "false", 2024-09-13T14:46:45.148 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_balance_snap_reads": "false", 2024-09-13T14:46:45.148 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_blkin_trace_all": "false", 2024-09-13T14:46:45.148 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_blocklist_expire_seconds": "0", 2024-09-13T14:46:45.148 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_blocklist_on_break_lock": "true", 2024-09-13T14:46:45.148 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_cache": "true", 2024-09-13T14:46:45.148 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_cache_block_writes_upfront": "false", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_cache_max_dirty": "25165824", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_cache_max_dirty_object": "0", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_cache_policy": "writearound", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_cache_size": "33554432", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_cache_target_dirty": "16777216", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_clone_copy_on_read": "false", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_compression_hint": "none", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_concurrent_management_ops": "10", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_config_pool_override_update_timestamp": "1726238421", 2024-09-13T14:46:45.149 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_default_clone_format": "auto", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_default_data_pool": "", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_default_features": "61", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_default_format": "2", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_default_map_options": "", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_default_order": "22", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_default_pool": "rbd", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_default_stripe_count": "0", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_default_stripe_unit": "0", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_disable_zero_copy_writes": "true", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_discard_granularity_bytes": "65536", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_enable_alloc_hint": "true", 2024-09-13T14:46:45.150 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-09-13T14:46:45.151 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_io_scheduler": "simple", 2024-09-13T14:46:45.151 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-09-13T14:46:45.151 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_commit_age": "5.000000", 2024-09-13T14:46:45.151 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-09-13T14:46:45.151 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-09-13T14:46:45.151 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-09-13T14:46:45.151 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-09-13T14:46:45.151 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_object_flush_interval": "0", 2024-09-13T14:46:45.151 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-09-13T14:46:45.151 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_order": "24", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_pool": "", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_journal_splay_width": "4", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_localize_parent_reads": "false", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_localize_snap_reads": "false", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-09-13T14:46:45.152 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_image_policy_type": "simple", 2024-09-13T14:46:45.153 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-09-13T14:46:45.153 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-09-13T14:46:45.153 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-09-13T14:46:45.153 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-09-13T14:46:45.153 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-09-13T14:46:45.153 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-09-13T14:46:45.153 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-09-13T14:46:45.153 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_memory_autotune": "true", 2024-09-13T14:46:45.153 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_memory_base": "805306368", 2024-09-13T14:46:45.153 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_memory_target": "4294967296", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirroring_delete_delay": "0", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirroring_replay_delay": "0", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-09-13T14:46:45.154 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_move_to_trash_on_remove": "false", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_mtime_update_interval": "60", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_non_blocking_aio": "true", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_op_thread_timeout": "60", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_op_threads": "1", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_parent_cache_enabled": "false", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_persistent_cache_mode": "disabled", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_persistent_cache_path": "/tmp", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_persistent_cache_size": "1073741824", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_plugins": "", 2024-09-13T14:46:45.155 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_bps_burst": "0", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_bps_limit": "0", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_exclude_ops": "0", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_iops_burst": "0", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_iops_limit": "0", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_read_bps_burst": "0", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_read_bps_limit": "0", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_read_iops_burst": "0", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-09-13T14:46:45.156 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_read_iops_limit": "0", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_schedule_tick_min": "50", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_write_bps_burst": "0", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_write_bps_limit": "0", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_write_iops_burst": "0", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_qos_write_iops_limit": "0", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_quiesce_notification_attempts": "10", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_read_from_replica_policy": "default", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_readahead_max_bytes": "524288", 2024-09-13T14:46:45.157 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_readahead_trigger_requests": "10", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_request_timed_out_seconds": "30", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_skip_partial_discard": "true", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_tracing": "false", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_validate_names": "true", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rbd_validate_pool": "true", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "restapi_base_url": "", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "restapi_log_level": "", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_account_default_quota_max_size": "-1", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_acl_grants_max_num": "100", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_admin_entry": "admin", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-09-13T14:46:45.158 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_asio_assert_yielding": "false", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_backend_store": "rados", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_barbican_url": "", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_beast_enable_async": "true", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_bucket_counters_cache": "false", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_bucket_index_max_aio": "128", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_bucket_quota_ttl": "600", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_cache_enabled": "true", 2024-09-13T14:46:45.159 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_cache_expiry_interval": "900", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_cache_lru_size": "25000", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_config_store": "rados", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_content_length_compat": "false", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_copy_obj_progress": "true", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_cors_rules_max_num": "100", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_cross_domain_policy": "", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_default_encryption_key": "", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_kmip_addr": "", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_kmip_ca_path": "", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_kmip_client_cert": "", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_kmip_client_key": "", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-09-13T14:46:45.160 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_kmip_password": "", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_kmip_username": "", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_require_ssl": "true", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-09-13T14:46:45.161 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_suppress_logs": "true", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_vault_addr": "", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_vault_auth": "token", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_vault_namespace": "", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_vault_prefix": "", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-09-13T14:46:45.162 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_vault_token_file": "", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_curl_buffersize": "524288", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_curl_low_speed_limit": "1024", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_curl_low_speed_time": "300", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_curl_tcp_keepalive": "0", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d3n_l1_fadvise": "4", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-09-13T14:46:45.163 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d4n_l1_fadvise": "4", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-09-13T14:46:45.164 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_data_log_changes_size": "1000", 2024-09-13T14:46:45.165 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_data_log_num_shards": "128", 2024-09-13T14:46:45.165 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-09-13T14:46:45.165 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_data_log_window": "30", 2024-09-13T14:46:45.165 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_data_notify_interval_msec": "0", 2024-09-13T14:46:45.165 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_data_sync_poll_interval": "20", 2024-09-13T14:46:45.165 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_data_sync_spawn_window": "20", 2024-09-13T14:46:45.165 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-09-13T14:46:45.165 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_default_data_log_backing": "fifo", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_default_region_info_oid": "default.region", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_defer_to_bucket_acls": "", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_disable_s3select": "false", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_data_res": "500.000000", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dns_name": "", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dns_s3website_name": "", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dynamic_resharding": "true", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-09-13T14:46:45.167 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_enable_gc_threads": "true", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_enable_lc_threads": "true", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_enable_mdsearch": "true", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_enable_ops_log": "false", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_enable_quota_threads": "true", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_enable_static_website": "false", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_enable_usage_log": "false", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_enforce_swift_acls": "true", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_exit_timeout_secs": "120", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_expose_bucket": "false", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_extended_http_attrs": "", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_filter": "none", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-09-13T14:46:45.168 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_frontends": "beast port=7480", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_gc_max_concurrent_io": "10", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_gc_max_deferred": "50", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_gc_max_objs": "32", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_gc_max_queue_size": "134213632", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_gc_max_trim_chunk": "16", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_gc_obj_min_wait": "7200", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_gc_processor_max_time": "3600", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_gc_processor_period": "3600", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_get_obj_window_size": "16777216", 2024-09-13T14:46:45.169 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_healthcheck_disabling_path": "", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ignore_get_invalid_range": "false", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_init_timeout": "300", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_kafka_connection_idle": "300", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_kafka_message_timeout": "5000", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_kafka_sleep_timeout": "10", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-09-13T14:46:45.170 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_admin_domain": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_admin_password": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_admin_password_path": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_admin_project": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_admin_tenant": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_admin_token": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_admin_token_path": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_admin_user": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_barbican_domain": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_barbican_password": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_barbican_project": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_barbican_tenant": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_barbican_user": "", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-09-13T14:46:45.171 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_implicit_tenants": "false", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_service_token_enabled": "false", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_token_cache_size": "10000", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_url": "", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_keystone_verify_ssl": "true", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_lc_debug_interval": "-1", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_lc_lock_max_time": "90", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_lc_max_objs": "32", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_lc_max_rules": "1000", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_lc_max_worker": "3", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_lc_max_wp_worker": "3", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_lc_thread_delay": "0", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ldap_dnattr": "uid", 2024-09-13T14:46:45.172 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ldap_searchfilter": "", 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ldap_uri": "", 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_lfuda_sync_frequency": "60", 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_log_http_headers": "", 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_log_nonexistent_bucket": "false", 2024-09-13T14:46:45.173 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-09-13T14:46:45.174 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_log_object_name_utc": "false", 2024-09-13T14:46:45.174 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-09-13T14:46:45.174 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-09-13T14:46:45.174 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_attr_name_len": "0", 2024-09-13T14:46:45.174 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_attr_size": "0", 2024-09-13T14:46:45.174 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_attrs_num_in_req": "0", 2024-09-13T14:46:45.174 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_chunk_size": "4194304", 2024-09-13T14:46:45.174 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_concurrent_requests": "1024", 2024-09-13T14:46:45.174 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-09-13T14:46:45.174 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_dynamic_shards": "1999", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_listing_results": "1000", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_notify_retries": "10", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_objs_per_shard": "100000", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_put_param_size": "1048576", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_put_size": "5368709120", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_max_slo_entries": "1000", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_md_log_max_shards": "64", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_md_notify_interval_msec": "200", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_meta_sync_poll_interval": "20", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_meta_sync_spawn_window": "20", 2024-09-13T14:46:45.175 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_mp_lock_max_time": "600", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_multipart_min_part_size": "5242880", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_fhcache_size": "2017", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_lru_lanes": "5", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_max_gc": "300", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-09-13T14:46:45.176 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_run_gc_threads": "false", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_run_lc_threads": "false", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_run_quota_threads": "false", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_run_sync_thread": "false", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_num_async_rados_threads": "32", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_num_control_oids": "8", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_numa_node": "-1", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_obj_stripe_size": "4194304", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-09-13T14:46:45.177 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_objexp_chunk_size": "100", 2024-09-13T14:46:45.178 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_objexp_gc_interval": "600", 2024-09-13T14:46:45.178 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_objexp_hints_num_shards": "127", 2024-09-13T14:46:45.178 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-09-13T14:46:45.178 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-09-13T14:46:45.178 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_op_thread_timeout": "600", 2024-09-13T14:46:45.178 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_op_tracing": "false", 2024-09-13T14:46:45.178 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_opa_token": "", 2024-09-13T14:46:45.178 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_opa_url": "", 2024-09-13T14:46:45.178 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_opa_verify_ssl": "true", 2024-09-13T14:46:45.178 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ops_log_rados": "false", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_ops_log_socket_path": "", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_period_push_interval": "2.000000", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_period_push_interval_max": "30.000000", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_period_root_pool": ".rgw.root", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_posix_cache_lanes": "3", 2024-09-13T14:46:45.179 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_posix_cache_max_buckets": "100", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_posix_cache_partitions": "3", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_print_continue": "true", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_print_prohibited_content_length": "false", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_rados_tracing": "false", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_realm": "", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_realm_id": "", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-09-13T14:46:45.180 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_region": "", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_region_root_pool": ".rgw.root", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_relaxed_region_enforcement": "false", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_request_uri": "", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_reshard_batch_size": "64", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_reshard_debug_interval": "-1", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_reshard_max_aio": "128", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_reshard_num_logs": "16", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_reshard_thread_interval": "600", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_resolve_cname": "false", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_rest_getusage_op_compat": "false", 2024-09-13T14:46:45.181 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_run_sync_thread": "true", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_s3_auth_use_keystone": "false", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_s3_auth_use_ldap": "false", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_s3_auth_use_rados": "true", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_s3_auth_use_sts": "false", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_s3_success_create_obj_status": "0", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_scheduler_type": "throttler", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_script_uri": "", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_service_provider_name": "", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sts_client_id": "", 2024-09-13T14:46:45.182 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sts_client_secret": "", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sts_entry": "sts", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sts_key": "sts", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sts_max_session_duration": "43200", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sts_min_session_duration": "900", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sts_token_introspection_url": "", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_account_in_url": "false", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_auth_entry": "auth", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_auth_url": "", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_custom_header": "", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_enforce_content_length": "false", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_need_stats": "true", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_tenant_name": "", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_token_expiration": "86400", 2024-09-13T14:46:45.183 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_url": "", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_url_prefix": "swift", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_swift_versioning_enabled": "false", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_lease_period": "120", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_log_trim_interval": "1200", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_obj_etag_verify": "false", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_trace_history_size": "4096", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-09-13T14:46:45.184 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_thread_pool_size": "512", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_topic_persistency_max_retries": "0", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_topic_require_publish_policy": "false", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_torrent_comment": "", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_torrent_createby": "", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_torrent_encoding": "", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_torrent_flag": "false", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_torrent_max_size": "5368709120", 2024-09-13T14:46:45.185 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_torrent_origin": "", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_torrent_sha_unit": "524288", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_torrent_tracker": "", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_trust_forwarded_https": "false", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_usage_log_tick_interval": "30", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_usage_max_shards": "32", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_usage_max_user_shards": "1", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_use_opa_authz": "false", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_counters_cache": "false", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_counters_cache_size": "10000", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-09-13T14:46:45.186 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_default_quota_max_size": "-1", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_max_buckets": "1000", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_policies_max_num": "100", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_quota_sync_interval": "86400", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_user_unique_email": "true", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_verify_ssl": "true", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_website_routing_rules_max_num": "50", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_zone": "", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_zone_id": "", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-09-13T14:46:45.187 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_zonegroup": "", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_zonegroup_id": "", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rgwlc_auto_session_clear": "true", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rgwlc_skip_bucket_step": "false", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_block_size": "4096", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_cache_shard_bits": "4", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_cache_size": "536870912", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_cache_type": "binned_lru", 2024-09-13T14:46:45.188 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_collect_compaction_stats": "false", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_collect_extended_stats": "false", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_collect_memory_stats": "false", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_index_type": "binary_search", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_log_to_ceph_log": "true", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_metadata_block_size": "4096", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_partition_filters": "false", 2024-09-13T14:46:45.189 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_perf": "false", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "rotating_keys_renewal_timeout": "10", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "run_dir": "/var/run/ceph", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_block_create": "true", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_cache_lru_size": "67108864", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_cbjournal_size": "5368709120", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_data_delta_based_overwrite": "0", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_default_max_object_size": "16777216", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_device_size": "53687091200", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-09-13T14:46:45.190 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_full_integrity_check": "false", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_journal_batch_capacity": "16", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_journal_iodepth_limit": "5", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_main_device_type": "SSD", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_max_concurrent_transactions": "8", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_max_data_allocation_size": "32768", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-09-13T14:46:45.191 INFO:teuthology.orchestra.run.smithi099.stdout: "seastore_segment_size": "67108864", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "set_keepcaps": "false", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "setgroup": "ceph", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "setuser": "ceph", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "setuser_match_path": "", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "target_max_misplaced_ratio": "0.050000", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "thp": "false", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "threadpool_default_timeout": "60", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "threadpool_empty_queue_max_wait": "2", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "throttler_perf_counter": "true", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "tmp_dir": "/tmp", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX", 2024-09-13T14:46:45.192 INFO:teuthology.orchestra.run.smithi099.stdout: "uadk_compressor_enabled": "false", 2024-09-13T14:46:45.193 INFO:teuthology.orchestra.run.smithi099.stdout: "uadk_wd_sync_ctx_num": "2" 2024-09-13T14:46:45.193 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:46:45.876 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:45 smithi099 bash[87147]: cluster 2024-09-13T14:46:44.464938+0000 mgr.y (mgr.26252) 612 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:45.876 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:45 smithi099 bash[87147]: cluster 2024-09-13T14:46:44.464938+0000 mgr.y (mgr.26252) 612 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:45.876 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:45 smithi099 bash[84576]: cluster 2024-09-13T14:46:44.464938+0000 mgr.y (mgr.26252) 612 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:45.876 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:45 smithi099 bash[84576]: cluster 2024-09-13T14:46:44.464938+0000 mgr.y (mgr.26252) 612 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:45.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-09-13T14:46:46.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:45 smithi177 bash[68594]: cluster 2024-09-13T14:46:44.464938+0000 mgr.y (mgr.26252) 612 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:46.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:45 smithi177 bash[68594]: cluster 2024-09-13T14:46:44.464938+0000 mgr.y (mgr.26252) 612 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:46:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:46:47] "GET /metrics HTTP/1.1" 200 34272 "" "Prometheus/2.51.0" 2024-09-13T14:46:48.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:47 smithi099 bash[84576]: cluster 2024-09-13T14:46:46.465578+0000 mgr.y (mgr.26252) 613 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:48.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:47 smithi099 bash[84576]: cluster 2024-09-13T14:46:46.465578+0000 mgr.y (mgr.26252) 613 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:48.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:47 smithi099 bash[87147]: cluster 2024-09-13T14:46:46.465578+0000 mgr.y (mgr.26252) 613 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:48.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:47 smithi099 bash[87147]: cluster 2024-09-13T14:46:46.465578+0000 mgr.y (mgr.26252) 613 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:48.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:47 smithi177 bash[68594]: cluster 2024-09-13T14:46:46.465578+0000 mgr.y (mgr.26252) 613 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:48.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:47 smithi177 bash[68594]: cluster 2024-09-13T14:46:46.465578+0000 mgr.y (mgr.26252) 613 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:49.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:48 smithi099 bash[84576]: audit 2024-09-13T14:46:48.269714+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:49.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:48 smithi099 bash[84576]: audit 2024-09-13T14:46:48.269714+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:49.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:48 smithi099 bash[87147]: audit 2024-09-13T14:46:48.269714+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:49.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:48 smithi099 bash[87147]: audit 2024-09-13T14:46:48.269714+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:49.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:48 smithi177 bash[68594]: audit 2024-09-13T14:46:48.269714+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:49.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:48 smithi177 bash[68594]: audit 2024-09-13T14:46:48.269714+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:50.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:49 smithi099 bash[84576]: cluster 2024-09-13T14:46:48.466676+0000 mgr.y (mgr.26252) 614 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:50.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:49 smithi099 bash[84576]: cluster 2024-09-13T14:46:48.466676+0000 mgr.y (mgr.26252) 614 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:50.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:49 smithi099 bash[87147]: cluster 2024-09-13T14:46:48.466676+0000 mgr.y (mgr.26252) 614 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:50.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:49 smithi099 bash[87147]: cluster 2024-09-13T14:46:48.466676+0000 mgr.y (mgr.26252) 614 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:50.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:49 smithi177 bash[68594]: cluster 2024-09-13T14:46:48.466676+0000 mgr.y (mgr.26252) 614 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:50.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:49 smithi177 bash[68594]: cluster 2024-09-13T14:46:48.466676+0000 mgr.y (mgr.26252) 614 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:50.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:46:50 smithi177 bash[61439]: ts=2024-09-13T14:46:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:46:50.654 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:52.059 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:51 smithi099 bash[84576]: cluster 2024-09-13T14:46:50.467645+0000 mgr.y (mgr.26252) 615 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:52.059 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:51 smithi099 bash[84576]: cluster 2024-09-13T14:46:50.467645+0000 mgr.y (mgr.26252) 615 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:52.059 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:51 smithi099 bash[87147]: cluster 2024-09-13T14:46:50.467645+0000 mgr.y (mgr.26252) 615 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:52.059 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:51 smithi099 bash[87147]: cluster 2024-09-13T14:46:50.467645+0000 mgr.y (mgr.26252) 615 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:52.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:51 smithi177 bash[68594]: cluster 2024-09-13T14:46:50.467645+0000 mgr.y (mgr.26252) 615 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:52.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:51 smithi177 bash[68594]: cluster 2024-09-13T14:46:50.467645+0000 mgr.y (mgr.26252) 615 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:54.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:53 smithi099 bash[84576]: cluster 2024-09-13T14:46:52.468624+0000 mgr.y (mgr.26252) 616 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:54.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:53 smithi099 bash[84576]: cluster 2024-09-13T14:46:52.468624+0000 mgr.y (mgr.26252) 616 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:54.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:53 smithi099 bash[87147]: cluster 2024-09-13T14:46:52.468624+0000 mgr.y (mgr.26252) 616 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:54.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:53 smithi099 bash[87147]: cluster 2024-09-13T14:46:52.468624+0000 mgr.y (mgr.26252) 616 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:54.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:53 smithi177 bash[68594]: cluster 2024-09-13T14:46:52.468624+0000 mgr.y (mgr.26252) 616 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:54.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:53 smithi177 bash[68594]: cluster 2024-09-13T14:46:52.468624+0000 mgr.y (mgr.26252) 616 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:54.370 INFO:teuthology.orchestra.run.smithi099.stdout:mon.a: {} 2024-09-13T14:46:54.370 INFO:teuthology.orchestra.run.smithi099.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:46:54.383 INFO:teuthology.orchestra.run.smithi099.stdout:mon.b: {} 2024-09-13T14:46:54.383 INFO:teuthology.orchestra.run.smithi099.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:46:54.392 INFO:teuthology.orchestra.run.smithi099.stdout:mon.c: {} 2024-09-13T14:46:54.393 INFO:teuthology.orchestra.run.smithi099.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:46:55.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:54 smithi099 bash[84576]: audit 2024-09-13T14:46:54.362705+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.99:0/2605047480' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:46:55.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:54 smithi099 bash[84576]: audit 2024-09-13T14:46:54.362705+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.99:0/2605047480' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:46:55.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:54 smithi099 bash[87147]: audit 2024-09-13T14:46:54.362705+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.99:0/2605047480' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:46:55.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:54 smithi099 bash[87147]: audit 2024-09-13T14:46:54.362705+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.99:0/2605047480' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:46:55.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:54 smithi177 bash[68594]: audit 2024-09-13T14:46:54.362705+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.99:0/2605047480' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:46:55.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:54 smithi177 bash[68594]: audit 2024-09-13T14:46:54.362705+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.99:0/2605047480' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:46:55.296 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-09-13T14:46:55.297 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-13T14:46:55.298 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:46:55.299 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:46:55.300 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-13T14:46:55.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-09-13T14:46:55.300 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-09-13T14:46:55.300 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:46:55.301 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:46:55.301 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:46:55.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd out 4 2024-09-13T14:46:55.301 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-09-13T14:46:55.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:55.302 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-09-13T14:46:55.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:46:55.302 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-09-13T14:46:55.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:46:55.303 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-09-13T14:46:55.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:46:55.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:55.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:55.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:55.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:55.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:55.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:55.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:55.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:55.691 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:55.714 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:55.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:55.817 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:55.828 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:55 smithi177 bash[68594]: cluster 2024-09-13T14:46:54.469315+0000 mgr.y (mgr.26252) 617 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:55.829 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:55 smithi177 bash[68594]: cluster 2024-09-13T14:46:54.469315+0000 mgr.y (mgr.26252) 617 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:55.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:55.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:55.948 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:55.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:55 smithi099 bash[84576]: cluster 2024-09-13T14:46:54.469315+0000 mgr.y (mgr.26252) 617 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:55.960 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:55 smithi099 bash[84576]: cluster 2024-09-13T14:46:54.469315+0000 mgr.y (mgr.26252) 617 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:55.960 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:55 smithi099 bash[87147]: cluster 2024-09-13T14:46:54.469315+0000 mgr.y (mgr.26252) 617 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:55.960 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:55 smithi099 bash[87147]: cluster 2024-09-13T14:46:54.469315+0000 mgr.y (mgr.26252) 617 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:55.973 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:55.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:56.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:56.103 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:56.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:56.206 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:56.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:56.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:56.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:56.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:56.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:56.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:56.494 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:56.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:56.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:56.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:56.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:56.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:56.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:56.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:56.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:56.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:56 smithi177 bash[68594]: audit 2024-09-13T14:46:56.553616+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:56.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:56 smithi177 bash[68594]: audit 2024-09-13T14:46:56.553616+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:56.887 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:56.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:56.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:57.000 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:56 smithi099 bash[84576]: audit 2024-09-13T14:46:56.553616+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:57.000 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:56 smithi099 bash[84576]: audit 2024-09-13T14:46:56.553616+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:57.000 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:56 smithi099 bash[87147]: audit 2024-09-13T14:46:56.553616+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:57.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:56 smithi099 bash[87147]: audit 2024-09-13T14:46:56.553616+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:46:57.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:57.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:57.134 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:57.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:57.236 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:57.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:57.260 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:46:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:46:57] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.51.0" 2024-09-13T14:46:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:57.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:57.384 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:57.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:57.513 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:57.616 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:57.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:57.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:57.771 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:57.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:57 smithi177 bash[68594]: cluster 2024-09-13T14:46:56.469921+0000 mgr.y (mgr.26252) 618 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:57.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:57 smithi177 bash[68594]: cluster 2024-09-13T14:46:56.469921+0000 mgr.y (mgr.26252) 618 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:57.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:57.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:58.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:58.015 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:57 smithi099 bash[84576]: cluster 2024-09-13T14:46:56.469921+0000 mgr.y (mgr.26252) 618 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:58.015 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:57 smithi099 bash[84576]: cluster 2024-09-13T14:46:56.469921+0000 mgr.y (mgr.26252) 618 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:58.016 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:57 smithi099 bash[87147]: cluster 2024-09-13T14:46:56.469921+0000 mgr.y (mgr.26252) 618 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:58.016 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:57 smithi099 bash[87147]: cluster 2024-09-13T14:46:56.469921+0000 mgr.y (mgr.26252) 618 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:58.031 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:58.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:58.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:58.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:58.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:58.290 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:58.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:58.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:58.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:58.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:58.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:58.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:58.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:58.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:58.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:58.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:58.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:58.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:58.909 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:58.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:58.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:59.038 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:59.064 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:59.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:59.168 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:59.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:59.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:59.299 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:59.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:59.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:59.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:59.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:59.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:59.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:59.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:59.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:59.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:59.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:59.848 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:59.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:59.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:59 smithi177 bash[68594]: cluster 2024-09-13T14:46:58.470884+0000 mgr.y (mgr.26252) 619 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:59.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:46:59 smithi177 bash[68594]: cluster 2024-09-13T14:46:58.470884+0000 mgr.y (mgr.26252) 619 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:59.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:59.962 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:59 smithi099 bash[84576]: cluster 2024-09-13T14:46:58.470884+0000 mgr.y (mgr.26252) 619 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:59.963 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:46:59 smithi099 bash[84576]: cluster 2024-09-13T14:46:58.470884+0000 mgr.y (mgr.26252) 619 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:59.963 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:59 smithi099 bash[87147]: cluster 2024-09-13T14:46:58.470884+0000 mgr.y (mgr.26252) 619 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:59.963 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:46:59 smithi099 bash[87147]: cluster 2024-09-13T14:46:58.470884+0000 mgr.y (mgr.26252) 619 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:46:59.978 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:59.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:00.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:00.086 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:00.087 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:00.088 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:00.088 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:00.091 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:00.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:00.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:00.212 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:00.237 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:00.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:00.339 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:00.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:47:00 smithi177 bash[61439]: ts=2024-09-13T14:47:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:00.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:00.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:00.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:00.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:00.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:00.600 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:00.622 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:00.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:00.725 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:00.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:00.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:00.856 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:00.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:00.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:00.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:01.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:01.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:01.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:01.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:01.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:01.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:01.276 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:01.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:01.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.408 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:01.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:01.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:01.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:01.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:01.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:01.885 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.903 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:01.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:02.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:02.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:01 smithi177 bash[68594]: cluster 2024-09-13T14:47:00.471836+0000 mgr.y (mgr.26252) 620 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:02.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:01 smithi177 bash[68594]: cluster 2024-09-13T14:47:00.471836+0000 mgr.y (mgr.26252) 620 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:02.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:02.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:02.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:01 smithi099 bash[84576]: cluster 2024-09-13T14:47:00.471836+0000 mgr.y (mgr.26252) 620 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:02.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:01 smithi099 bash[84576]: cluster 2024-09-13T14:47:00.471836+0000 mgr.y (mgr.26252) 620 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:02.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:01 smithi099 bash[87147]: cluster 2024-09-13T14:47:00.471836+0000 mgr.y (mgr.26252) 620 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:02.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:01 smithi099 bash[87147]: cluster 2024-09-13T14:47:00.471836+0000 mgr.y (mgr.26252) 620 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:02.129 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:02.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:02.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:02.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.393 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:02.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:02.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:02.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:02.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:02.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:02.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:02.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:02.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:02.893 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[84576]: audit 2024-09-13T14:47:01.800245+0000 mon.a (mon.0) 1568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.893 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[84576]: audit 2024-09-13T14:47:01.800245+0000 mon.a (mon.0) 1568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.893 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[84576]: audit 2024-09-13T14:47:01.804645+0000 mon.a (mon.0) 1569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.893 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[84576]: audit 2024-09-13T14:47:01.804645+0000 mon.a (mon.0) 1569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[84576]: audit 2024-09-13T14:47:02.566075+0000 mon.a (mon.0) 1570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[84576]: audit 2024-09-13T14:47:02.566075+0000 mon.a (mon.0) 1570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[84576]: audit 2024-09-13T14:47:02.576244+0000 mon.a (mon.0) 1571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[84576]: audit 2024-09-13T14:47:02.576244+0000 mon.a (mon.0) 1571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[87147]: audit 2024-09-13T14:47:01.800245+0000 mon.a (mon.0) 1568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[87147]: audit 2024-09-13T14:47:01.800245+0000 mon.a (mon.0) 1568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[87147]: audit 2024-09-13T14:47:01.804645+0000 mon.a (mon.0) 1569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[87147]: audit 2024-09-13T14:47:01.804645+0000 mon.a (mon.0) 1569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[87147]: audit 2024-09-13T14:47:02.566075+0000 mon.a (mon.0) 1570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[87147]: audit 2024-09-13T14:47:02.566075+0000 mon.a (mon.0) 1570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.894 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[87147]: audit 2024-09-13T14:47:02.576244+0000 mon.a (mon.0) 1571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.895 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:02 smithi099 bash[87147]: audit 2024-09-13T14:47:02.576244+0000 mon.a (mon.0) 1571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:02.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:02.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:03.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:03.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:03.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:02 smithi177 bash[68594]: audit 2024-09-13T14:47:01.800245+0000 mon.a (mon.0) 1568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:03.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:02 smithi177 bash[68594]: audit 2024-09-13T14:47:01.800245+0000 mon.a (mon.0) 1568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:03.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:02 smithi177 bash[68594]: audit 2024-09-13T14:47:01.804645+0000 mon.a (mon.0) 1569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:03.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:02 smithi177 bash[68594]: audit 2024-09-13T14:47:01.804645+0000 mon.a (mon.0) 1569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:03.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:02 smithi177 bash[68594]: audit 2024-09-13T14:47:02.566075+0000 mon.a (mon.0) 1570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:03.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:02 smithi177 bash[68594]: audit 2024-09-13T14:47:02.566075+0000 mon.a (mon.0) 1570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:03.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:02 smithi177 bash[68594]: audit 2024-09-13T14:47:02.576244+0000 mon.a (mon.0) 1571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:03.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:02 smithi177 bash[68594]: audit 2024-09-13T14:47:02.576244+0000 mon.a (mon.0) 1571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:03.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:03.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:03.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:03.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:03.272 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:03.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:03.375 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:03.396 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:03.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:03.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:03.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:03.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:03.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:03.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:03.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:03.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:03.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:03.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:03.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:03.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:04.005 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:04.032 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:04.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:04.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[84576]: cluster 2024-09-13T14:47:02.472650+0000 mgr.y (mgr.26252) 621 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:04.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[84576]: cluster 2024-09-13T14:47:02.472650+0000 mgr.y (mgr.26252) 621 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:04.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[84576]: audit 2024-09-13T14:47:03.204244+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:04.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[84576]: audit 2024-09-13T14:47:03.204244+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:04.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[84576]: audit 2024-09-13T14:47:03.205992+0000 mon.a (mon.0) 1573 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:04.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[84576]: audit 2024-09-13T14:47:03.205992+0000 mon.a (mon.0) 1573 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:04.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[84576]: audit 2024-09-13T14:47:03.214199+0000 mon.a (mon.0) 1574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:04.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[84576]: audit 2024-09-13T14:47:03.214199+0000 mon.a (mon.0) 1574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:04.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[84576]: audit 2024-09-13T14:47:03.269337+0000 mon.a (mon.0) 1575 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:04.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[84576]: audit 2024-09-13T14:47:03.269337+0000 mon.a (mon.0) 1575 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:04.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[87147]: cluster 2024-09-13T14:47:02.472650+0000 mgr.y (mgr.26252) 621 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:04.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[87147]: cluster 2024-09-13T14:47:02.472650+0000 mgr.y (mgr.26252) 621 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:04.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[87147]: audit 2024-09-13T14:47:03.204244+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[87147]: audit 2024-09-13T14:47:03.204244+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[87147]: audit 2024-09-13T14:47:03.205992+0000 mon.a (mon.0) 1573 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[87147]: audit 2024-09-13T14:47:03.205992+0000 mon.a (mon.0) 1573 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[87147]: audit 2024-09-13T14:47:03.214199+0000 mon.a (mon.0) 1574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[87147]: audit 2024-09-13T14:47:03.214199+0000 mon.a (mon.0) 1574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[87147]: audit 2024-09-13T14:47:03.269337+0000 mon.a (mon.0) 1575 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:03 smithi099 bash[87147]: audit 2024-09-13T14:47:03.269337+0000 mon.a (mon.0) 1575 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:04.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:03 smithi177 bash[68594]: cluster 2024-09-13T14:47:02.472650+0000 mgr.y (mgr.26252) 621 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:04.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:03 smithi177 bash[68594]: cluster 2024-09-13T14:47:02.472650+0000 mgr.y (mgr.26252) 621 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:04.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:03 smithi177 bash[68594]: audit 2024-09-13T14:47:03.204244+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:04.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:03 smithi177 bash[68594]: audit 2024-09-13T14:47:03.204244+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:04.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:03 smithi177 bash[68594]: audit 2024-09-13T14:47:03.205992+0000 mon.a (mon.0) 1573 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:04.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:03 smithi177 bash[68594]: audit 2024-09-13T14:47:03.205992+0000 mon.a (mon.0) 1573 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:04.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:03 smithi177 bash[68594]: audit 2024-09-13T14:47:03.214199+0000 mon.a (mon.0) 1574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:04.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:03 smithi177 bash[68594]: audit 2024-09-13T14:47:03.214199+0000 mon.a (mon.0) 1574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:04.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:03 smithi177 bash[68594]: audit 2024-09-13T14:47:03.269337+0000 mon.a (mon.0) 1575 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:04.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:03 smithi177 bash[68594]: audit 2024-09-13T14:47:03.269337+0000 mon.a (mon.0) 1575 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:04.135 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:04.159 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:04.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:04.262 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:04.287 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:04.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:04.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:04.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:04.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:04.548 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:04.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:04.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:04.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:04.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:04.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:04.812 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:04.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:04.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:04.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:04.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:05.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:05.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:05.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:05.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:05.200 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:05.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:05.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:05.331 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:05.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:05.434 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:05.457 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:05.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:05.560 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:05.587 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:05.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:05.690 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:05.715 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:05.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:05.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:05.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:05.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:05.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:05.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:05.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:06.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:05 smithi099 bash[84576]: cluster 2024-09-13T14:47:04.473662+0000 mgr.y (mgr.26252) 622 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:06.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:05 smithi099 bash[84576]: cluster 2024-09-13T14:47:04.473662+0000 mgr.y (mgr.26252) 622 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:06.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:05 smithi099 bash[87147]: cluster 2024-09-13T14:47:04.473662+0000 mgr.y (mgr.26252) 622 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:06.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:05 smithi099 bash[87147]: cluster 2024-09-13T14:47:04.473662+0000 mgr.y (mgr.26252) 622 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:06.080 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:06.107 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:06.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:06.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:05 smithi177 bash[68594]: cluster 2024-09-13T14:47:04.473662+0000 mgr.y (mgr.26252) 622 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:06.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:05 smithi177 bash[68594]: cluster 2024-09-13T14:47:04.473662+0000 mgr.y (mgr.26252) 622 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:06.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:06.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:06.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:06.342 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:06.370 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:06.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:06.473 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:06.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:06.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:06.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:06.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:06.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:06.757 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:06.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:06.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:06.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:06.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:06.991 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:07.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:07.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:07.121 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:07.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:07.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:07.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:07.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:07.280 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:47:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:47:07] "GET /metrics HTTP/1.1" 200 34271 "" "Prometheus/2.51.0" 2024-09-13T14:47:07.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:07.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:07.409 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:07.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:07.512 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:07.540 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:07.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:07.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:07.672 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:07.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:07.775 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:07.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:07.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:07.914 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:07 smithi099 bash[84576]: cluster 2024-09-13T14:47:06.474279+0000 mgr.y (mgr.26252) 623 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:07.914 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:07 smithi099 bash[84576]: cluster 2024-09-13T14:47:06.474279+0000 mgr.y (mgr.26252) 623 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:07.915 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:07 smithi099 bash[87147]: cluster 2024-09-13T14:47:06.474279+0000 mgr.y (mgr.26252) 623 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:07.915 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:07 smithi099 bash[87147]: cluster 2024-09-13T14:47:06.474279+0000 mgr.y (mgr.26252) 623 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:07.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:07.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:08.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:08.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:07 smithi177 bash[68594]: cluster 2024-09-13T14:47:06.474279+0000 mgr.y (mgr.26252) 623 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:08.047 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:07 smithi177 bash[68594]: cluster 2024-09-13T14:47:06.474279+0000 mgr.y (mgr.26252) 623 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:08.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:08.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:08.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:08.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:08.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:08.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:08.323 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:08.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:08.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:08.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:08.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:08.556 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:08.583 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:08.686 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:08.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:08.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:08.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:08.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:08.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:08.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:08.975 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:08.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:09.078 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:09.104 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:09.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:09.207 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:09.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:09.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:09.337 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:09.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:09.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:09.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:09.495 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:09.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:09.599 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:09.626 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:09.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:09.728 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:09.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:09.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:09.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:09.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:09 smithi177 bash[68594]: cluster 2024-09-13T14:47:08.475176+0000 mgr.y (mgr.26252) 624 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:09.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:09 smithi177 bash[68594]: cluster 2024-09-13T14:47:08.475176+0000 mgr.y (mgr.26252) 624 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:09.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:09.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:09.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:10.012 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:09 smithi099 bash[84576]: cluster 2024-09-13T14:47:08.475176+0000 mgr.y (mgr.26252) 624 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:10.012 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:09 smithi099 bash[84576]: cluster 2024-09-13T14:47:08.475176+0000 mgr.y (mgr.26252) 624 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:10.012 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:09 smithi099 bash[87147]: cluster 2024-09-13T14:47:08.475176+0000 mgr.y (mgr.26252) 624 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:10.012 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:09 smithi099 bash[87147]: cluster 2024-09-13T14:47:08.475176+0000 mgr.y (mgr.26252) 624 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:10.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:10.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:10.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:10.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:10.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:10.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:10.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:10.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:47:10 smithi177 bash[61439]: ts=2024-09-13T14:47:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:10.374 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.400 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:10.504 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.532 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:10.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.660 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:10.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:10.789 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:10.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:10.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:10.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:11.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:11.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:11.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:11.154 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:11.181 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:11.284 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:11.311 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:11.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:11.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:11.441 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:11.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:11.544 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:11.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:11.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:11.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:11.700 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:11.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:11.803 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:11.830 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:11.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:11.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:11.944 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:11 smithi177 bash[68594]: cluster 2024-09-13T14:47:10.476121+0000 mgr.y (mgr.26252) 625 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:11.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:11 smithi177 bash[68594]: cluster 2024-09-13T14:47:10.476121+0000 mgr.y (mgr.26252) 625 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:11.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:11.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:12.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:12.086 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:12.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:12.132 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:11 smithi099 bash[84576]: cluster 2024-09-13T14:47:10.476121+0000 mgr.y (mgr.26252) 625 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:12.132 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:11 smithi099 bash[84576]: cluster 2024-09-13T14:47:10.476121+0000 mgr.y (mgr.26252) 625 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:12.132 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:11 smithi099 bash[87147]: cluster 2024-09-13T14:47:10.476121+0000 mgr.y (mgr.26252) 625 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:12.132 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:11 smithi099 bash[87147]: cluster 2024-09-13T14:47:10.476121+0000 mgr.y (mgr.26252) 625 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:12.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:12.217 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:12.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:12.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:12.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:12.448 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:12.474 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:12.577 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:12.605 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:12.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:12.735 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:12.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:12.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:12.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:12.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:12.968 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:12.989 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:12.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:13.079 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:47:13.079 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:13.093 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:13.096 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:47:13.097 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:13.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:13.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:13.114 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:13.114 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:47:13.132 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:47:13.132 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:13.153 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:47:13.154 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:13.174 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:47:13.174 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:13.194 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:47:13.194 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:13.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:13.214 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:47:13.214 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:13.241 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:13.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:13.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:13.369 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:13.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:13.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:13.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:13.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:13.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:13.632 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:13.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:13.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:13.760 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:13.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:13.863 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:13.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:13.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:13.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:14.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:13 smithi099 bash[84576]: cluster 2024-09-13T14:47:12.476758+0000 mgr.y (mgr.26252) 626 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:14.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:13 smithi099 bash[84576]: cluster 2024-09-13T14:47:12.476758+0000 mgr.y (mgr.26252) 626 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:14.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:13 smithi099 bash[84576]: audit 2024-09-13T14:47:13.063015+0000 mon.a (mon.0) 1576 : audit [DBG] from='client.? 172.21.15.99:0/1032779346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:14.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:13 smithi099 bash[84576]: audit 2024-09-13T14:47:13.063015+0000 mon.a (mon.0) 1576 : audit [DBG] from='client.? 172.21.15.99:0/1032779346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:14.005 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:13 smithi099 bash[87147]: cluster 2024-09-13T14:47:12.476758+0000 mgr.y (mgr.26252) 626 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:14.005 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:13 smithi099 bash[87147]: cluster 2024-09-13T14:47:12.476758+0000 mgr.y (mgr.26252) 626 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:14.005 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:13 smithi099 bash[87147]: audit 2024-09-13T14:47:13.063015+0000 mon.a (mon.0) 1576 : audit [DBG] from='client.? 172.21.15.99:0/1032779346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:14.005 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:13 smithi099 bash[87147]: audit 2024-09-13T14:47:13.063015+0000 mon.a (mon.0) 1576 : audit [DBG] from='client.? 172.21.15.99:0/1032779346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:14.020 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:14.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:13 smithi177 bash[68594]: cluster 2024-09-13T14:47:12.476758+0000 mgr.y (mgr.26252) 626 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:14.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:13 smithi177 bash[68594]: cluster 2024-09-13T14:47:12.476758+0000 mgr.y (mgr.26252) 626 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:14.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:13 smithi177 bash[68594]: audit 2024-09-13T14:47:13.063015+0000 mon.a (mon.0) 1576 : audit [DBG] from='client.? 172.21.15.99:0/1032779346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:14.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:13 smithi177 bash[68594]: audit 2024-09-13T14:47:13.063015+0000 mon.a (mon.0) 1576 : audit [DBG] from='client.? 172.21.15.99:0/1032779346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:14.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:14.150 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:14.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:14.253 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:14.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:14.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:14.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:14.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:14.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:14.542 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:14.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:14.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:14.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:14.768 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:14.788 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:14.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:14.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:14.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:14.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:15.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:15.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:15.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:15.148 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:15.175 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:15.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:15.267 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:14 smithi099 bash[84576]: debug 2024-09-13T14:47:14.903+0000 7f0688bd2640 -1 mon.a@0(leader).osd e951 definitely_dead 0 2024-09-13T14:47:15.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:15.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:15.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:15.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:15.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:15.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:15.441 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:15.441 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:15.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:15.558 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:15.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:15.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:15.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:15.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:15.791 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:15.816 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:15.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:15.837 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T14:47:15.847 INFO:teuthology.orchestra.run.smithi099.stderr:marked out osd.4. 2024-09-13T14:47:15.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:15.940 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:15.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:16.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:16.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:15 smithi177 bash[68594]: cluster 2024-09-13T14:47:14.477634+0000 mgr.y (mgr.26252) 627 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:16.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:15 smithi177 bash[68594]: cluster 2024-09-13T14:47:14.477634+0000 mgr.y (mgr.26252) 627 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:16.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:15 smithi177 bash[68594]: audit 2024-09-13T14:47:14.907209+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.99:0/4071443358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:15 smithi177 bash[68594]: audit 2024-09-13T14:47:14.907209+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.99:0/4071443358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:15 smithi177 bash[68594]: audit 2024-09-13T14:47:14.907867+0000 mon.a (mon.0) 1577 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:15 smithi177 bash[68594]: audit 2024-09-13T14:47:14.907867+0000 mon.a (mon.0) 1577 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:15 smithi177 bash[68594]: cluster 2024-09-13T14:47:14.908031+0000 mon.a (mon.0) 1578 : cluster [INF] Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:47:16.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:15 smithi177 bash[68594]: cluster 2024-09-13T14:47:14.908031+0000 mon.a (mon.0) 1578 : cluster [INF] Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:47:16.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:15 smithi177 bash[68594]: audit 2024-09-13T14:47:15.811642+0000 mon.a (mon.0) 1579 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:16.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:15 smithi177 bash[68594]: audit 2024-09-13T14:47:15.811642+0000 mon.a (mon.0) 1579 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:16.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:16.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:16.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:16.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[87147]: cluster 2024-09-13T14:47:14.477634+0000 mgr.y (mgr.26252) 627 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:16.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[87147]: cluster 2024-09-13T14:47:14.477634+0000 mgr.y (mgr.26252) 627 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:16.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[87147]: audit 2024-09-13T14:47:14.907209+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.99:0/4071443358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[87147]: audit 2024-09-13T14:47:14.907209+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.99:0/4071443358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[87147]: audit 2024-09-13T14:47:14.907867+0000 mon.a (mon.0) 1577 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[87147]: audit 2024-09-13T14:47:14.907867+0000 mon.a (mon.0) 1577 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[87147]: cluster 2024-09-13T14:47:14.908031+0000 mon.a (mon.0) 1578 : cluster [INF] Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:47:16.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[87147]: cluster 2024-09-13T14:47:14.908031+0000 mon.a (mon.0) 1578 : cluster [INF] Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:47:16.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[87147]: audit 2024-09-13T14:47:15.811642+0000 mon.a (mon.0) 1579 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[87147]: audit 2024-09-13T14:47:15.811642+0000 mon.a (mon.0) 1579 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[84576]: cluster 2024-09-13T14:47:14.477634+0000 mgr.y (mgr.26252) 627 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[84576]: cluster 2024-09-13T14:47:14.477634+0000 mgr.y (mgr.26252) 627 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[84576]: audit 2024-09-13T14:47:14.907209+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.99:0/4071443358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[84576]: audit 2024-09-13T14:47:14.907209+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.99:0/4071443358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[84576]: audit 2024-09-13T14:47:14.907867+0000 mon.a (mon.0) 1577 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[84576]: audit 2024-09-13T14:47:14.907867+0000 mon.a (mon.0) 1577 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[84576]: cluster 2024-09-13T14:47:14.908031+0000 mon.a (mon.0) 1578 : cluster [INF] Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[84576]: cluster 2024-09-13T14:47:14.908031+0000 mon.a (mon.0) 1578 : cluster [INF] Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[84576]: audit 2024-09-13T14:47:15.811642+0000 mon.a (mon.0) 1579 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:16.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:15 smithi099 bash[84576]: audit 2024-09-13T14:47:15.811642+0000 mon.a (mon.0) 1579 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:16.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:16.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:16.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:16.328 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:16.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:16.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:16.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:16.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:16.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:16.588 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:16.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:16.691 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:16.718 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:16.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:16.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:16.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:16.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:16.865 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_0' created 2024-09-13T14:47:16.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:16.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:16.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:17.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:17.086 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:47:17] "GET /metrics HTTP/1.1" 200 34271 "" "Prometheus/2.51.0" 2024-09-13T14:47:17.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: cluster 2024-09-13T14:47:15.831775+0000 mon.a (mon.0) 1580 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:17.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: cluster 2024-09-13T14:47:15.831775+0000 mon.a (mon.0) 1580 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:17.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: audit 2024-09-13T14:47:15.840706+0000 mon.a (mon.0) 1581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:47:17.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: audit 2024-09-13T14:47:15.840706+0000 mon.a (mon.0) 1581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:47:17.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: audit 2024-09-13T14:47:15.840770+0000 mon.a (mon.0) 1582 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:17.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: audit 2024-09-13T14:47:15.840770+0000 mon.a (mon.0) 1582 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:17.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: cluster 2024-09-13T14:47:15.843650+0000 mon.a (mon.0) 1583 : cluster [DBG] osdmap e952: 8 total, 8 up, 7 in 2024-09-13T14:47:17.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: cluster 2024-09-13T14:47:15.843650+0000 mon.a (mon.0) 1583 : cluster [DBG] osdmap e952: 8 total, 8 up, 7 in 2024-09-13T14:47:17.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: audit 2024-09-13T14:47:16.034699+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.99:0/1994115905' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: audit 2024-09-13T14:47:16.034699+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.99:0/1994115905' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: audit 2024-09-13T14:47:16.035154+0000 mon.a (mon.0) 1584 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[87147]: audit 2024-09-13T14:47:16.035154+0000 mon.a (mon.0) 1584 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: cluster 2024-09-13T14:47:15.831775+0000 mon.a (mon.0) 1580 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: cluster 2024-09-13T14:47:15.831775+0000 mon.a (mon.0) 1580 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: audit 2024-09-13T14:47:15.840706+0000 mon.a (mon.0) 1581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: audit 2024-09-13T14:47:15.840706+0000 mon.a (mon.0) 1581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: audit 2024-09-13T14:47:15.840770+0000 mon.a (mon.0) 1582 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: audit 2024-09-13T14:47:15.840770+0000 mon.a (mon.0) 1582 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: cluster 2024-09-13T14:47:15.843650+0000 mon.a (mon.0) 1583 : cluster [DBG] osdmap e952: 8 total, 8 up, 7 in 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: cluster 2024-09-13T14:47:15.843650+0000 mon.a (mon.0) 1583 : cluster [DBG] osdmap e952: 8 total, 8 up, 7 in 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: audit 2024-09-13T14:47:16.034699+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.99:0/1994115905' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: audit 2024-09-13T14:47:16.034699+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.99:0/1994115905' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: audit 2024-09-13T14:47:16.035154+0000 mon.a (mon.0) 1584 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.089 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:16 smithi099 bash[84576]: audit 2024-09-13T14:47:16.035154+0000 mon.a (mon.0) 1584 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:17.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: cluster 2024-09-13T14:47:15.831775+0000 mon.a (mon.0) 1580 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: cluster 2024-09-13T14:47:15.831775+0000 mon.a (mon.0) 1580 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: audit 2024-09-13T14:47:15.840706+0000 mon.a (mon.0) 1581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:47:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: audit 2024-09-13T14:47:15.840706+0000 mon.a (mon.0) 1581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:47:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: audit 2024-09-13T14:47:15.840770+0000 mon.a (mon.0) 1582 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: audit 2024-09-13T14:47:15.840770+0000 mon.a (mon.0) 1582 : audit [INF] from='client.? 172.21.15.99:0/2690629725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: cluster 2024-09-13T14:47:15.843650+0000 mon.a (mon.0) 1583 : cluster [DBG] osdmap e952: 8 total, 8 up, 7 in 2024-09-13T14:47:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: cluster 2024-09-13T14:47:15.843650+0000 mon.a (mon.0) 1583 : cluster [DBG] osdmap e952: 8 total, 8 up, 7 in 2024-09-13T14:47:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: audit 2024-09-13T14:47:16.034699+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.99:0/1994115905' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: audit 2024-09-13T14:47:16.034699+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.99:0/1994115905' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: audit 2024-09-13T14:47:16.035154+0000 mon.a (mon.0) 1584 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:16 smithi177 bash[68594]: audit 2024-09-13T14:47:16.035154+0000 mon.a (mon.0) 1584 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-13T14:47:17.205 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:17.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:17.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:17.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:17.364 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:17.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:17.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:47:17.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:17.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:17.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:17.627 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:17.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:17.731 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:17.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:17.833 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:47:17.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:17.896 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:17.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:17.999 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:18.008 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[84576]: cluster 2024-09-13T14:47:16.478323+0000 mgr.y (mgr.26252) 628 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:47:18.008 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[84576]: cluster 2024-09-13T14:47:16.478323+0000 mgr.y (mgr.26252) 628 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:47:18.008 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[84576]: audit 2024-09-13T14:47:16.848460+0000 mon.a (mon.0) 1585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:47:18.008 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[84576]: audit 2024-09-13T14:47:16.848460+0000 mon.a (mon.0) 1585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:47:18.008 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[84576]: cluster 2024-09-13T14:47:16.856712+0000 mon.a (mon.0) 1586 : cluster [DBG] osdmap e953: 8 total, 8 up, 7 in 2024-09-13T14:47:18.008 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[84576]: cluster 2024-09-13T14:47:16.856712+0000 mon.a (mon.0) 1586 : cluster [DBG] osdmap e953: 8 total, 8 up, 7 in 2024-09-13T14:47:18.008 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[87147]: cluster 2024-09-13T14:47:16.478323+0000 mgr.y (mgr.26252) 628 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:47:18.008 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[87147]: cluster 2024-09-13T14:47:16.478323+0000 mgr.y (mgr.26252) 628 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:47:18.009 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[87147]: audit 2024-09-13T14:47:16.848460+0000 mon.a (mon.0) 1585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:47:18.009 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[87147]: audit 2024-09-13T14:47:16.848460+0000 mon.a (mon.0) 1585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:47:18.009 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[87147]: cluster 2024-09-13T14:47:16.856712+0000 mon.a (mon.0) 1586 : cluster [DBG] osdmap e953: 8 total, 8 up, 7 in 2024-09-13T14:47:18.009 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:17 smithi099 bash[87147]: cluster 2024-09-13T14:47:16.856712+0000 mon.a (mon.0) 1586 : cluster [DBG] osdmap e953: 8 total, 8 up, 7 in 2024-09-13T14:47:18.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:18.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:18.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:17 smithi177 bash[68594]: cluster 2024-09-13T14:47:16.478323+0000 mgr.y (mgr.26252) 628 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:47:18.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:17 smithi177 bash[68594]: cluster 2024-09-13T14:47:16.478323+0000 mgr.y (mgr.26252) 628 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:47:18.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:17 smithi177 bash[68594]: audit 2024-09-13T14:47:16.848460+0000 mon.a (mon.0) 1585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:47:18.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:17 smithi177 bash[68594]: audit 2024-09-13T14:47:16.848460+0000 mon.a (mon.0) 1585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:47:18.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:17 smithi177 bash[68594]: cluster 2024-09-13T14:47:16.856712+0000 mon.a (mon.0) 1586 : cluster [DBG] osdmap e953: 8 total, 8 up, 7 in 2024-09-13T14:47:18.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:17 smithi177 bash[68594]: cluster 2024-09-13T14:47:16.856712+0000 mon.a (mon.0) 1586 : cluster [DBG] osdmap e953: 8 total, 8 up, 7 in 2024-09-13T14:47:18.122 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:18.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:18.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:18.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:18.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:18.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:18.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:18.515 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:18.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:18.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:18.672 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:18.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:18.774 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:18.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:18.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:18.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:18.913 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[84576]: cluster 2024-09-13T14:47:17.854447+0000 mon.a (mon.0) 1587 : cluster [DBG] osdmap e954: 8 total, 8 up, 7 in 2024-09-13T14:47:18.913 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[84576]: cluster 2024-09-13T14:47:17.854447+0000 mon.a (mon.0) 1587 : cluster [DBG] osdmap e954: 8 total, 8 up, 7 in 2024-09-13T14:47:18.914 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[84576]: audit 2024-09-13T14:47:18.276911+0000 mon.a (mon.0) 1588 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:18.914 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[84576]: audit 2024-09-13T14:47:18.276911+0000 mon.a (mon.0) 1588 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:18.914 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[84576]: audit 2024-09-13T14:47:18.278692+0000 mon.a (mon.0) 1589 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:18.914 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[84576]: audit 2024-09-13T14:47:18.278692+0000 mon.a (mon.0) 1589 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:18.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[87147]: cluster 2024-09-13T14:47:17.854447+0000 mon.a (mon.0) 1587 : cluster [DBG] osdmap e954: 8 total, 8 up, 7 in 2024-09-13T14:47:18.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[87147]: cluster 2024-09-13T14:47:17.854447+0000 mon.a (mon.0) 1587 : cluster [DBG] osdmap e954: 8 total, 8 up, 7 in 2024-09-13T14:47:18.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[87147]: audit 2024-09-13T14:47:18.276911+0000 mon.a (mon.0) 1588 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:18.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[87147]: audit 2024-09-13T14:47:18.276911+0000 mon.a (mon.0) 1588 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:18.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[87147]: audit 2024-09-13T14:47:18.278692+0000 mon.a (mon.0) 1589 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:18.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:18 smithi099 bash[87147]: audit 2024-09-13T14:47:18.278692+0000 mon.a (mon.0) 1589 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:18.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:19.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:19.059 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:19.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:19.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:19.186 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:19.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:19.289 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:19.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:18 smithi177 bash[68594]: cluster 2024-09-13T14:47:17.854447+0000 mon.a (mon.0) 1587 : cluster [DBG] osdmap e954: 8 total, 8 up, 7 in 2024-09-13T14:47:19.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:18 smithi177 bash[68594]: cluster 2024-09-13T14:47:17.854447+0000 mon.a (mon.0) 1587 : cluster [DBG] osdmap e954: 8 total, 8 up, 7 in 2024-09-13T14:47:19.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:18 smithi177 bash[68594]: audit 2024-09-13T14:47:18.276911+0000 mon.a (mon.0) 1588 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:19.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:18 smithi177 bash[68594]: audit 2024-09-13T14:47:18.276911+0000 mon.a (mon.0) 1588 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:19.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:18 smithi177 bash[68594]: audit 2024-09-13T14:47:18.278692+0000 mon.a (mon.0) 1589 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:19.301 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:18 smithi177 bash[68594]: audit 2024-09-13T14:47:18.278692+0000 mon.a (mon.0) 1589 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:19.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:19.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:19.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:19.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:19.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:19.550 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:19.576 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:19.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:19.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-13T14:47:19.680 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:19.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:19.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:19.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:19.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:19.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:19.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:19 smithi177 bash[68594]: cluster 2024-09-13T14:47:18.479013+0000 mgr.y (mgr.26252) 629 : cluster [DBG] pgmap v1078: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:47:19.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:19 smithi177 bash[68594]: cluster 2024-09-13T14:47:18.479013+0000 mgr.y (mgr.26252) 629 : cluster [DBG] pgmap v1078: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:47:19.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:19 smithi177 bash[68594]: cluster 2024-09-13T14:47:18.875457+0000 mon.a (mon.0) 1590 : cluster [DBG] osdmap e955: 8 total, 8 up, 7 in 2024-09-13T14:47:19.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:19 smithi177 bash[68594]: cluster 2024-09-13T14:47:18.875457+0000 mon.a (mon.0) 1590 : cluster [DBG] osdmap e955: 8 total, 8 up, 7 in 2024-09-13T14:47:19.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:19 smithi177 bash[68594]: cluster 2024-09-13T14:47:19.272329+0000 mon.a (mon.0) 1591 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:47:19.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:19 smithi177 bash[68594]: cluster 2024-09-13T14:47:19.272329+0000 mon.a (mon.0) 1591 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:47:19.962 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:19.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:20.065 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:20.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:20.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:20.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:20.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[84576]: cluster 2024-09-13T14:47:18.479013+0000 mgr.y (mgr.26252) 629 : cluster [DBG] pgmap v1078: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:47:20.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[84576]: cluster 2024-09-13T14:47:18.479013+0000 mgr.y (mgr.26252) 629 : cluster [DBG] pgmap v1078: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:47:20.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[84576]: cluster 2024-09-13T14:47:18.875457+0000 mon.a (mon.0) 1590 : cluster [DBG] osdmap e955: 8 total, 8 up, 7 in 2024-09-13T14:47:20.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[84576]: cluster 2024-09-13T14:47:18.875457+0000 mon.a (mon.0) 1590 : cluster [DBG] osdmap e955: 8 total, 8 up, 7 in 2024-09-13T14:47:20.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[84576]: cluster 2024-09-13T14:47:19.272329+0000 mon.a (mon.0) 1591 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:47:20.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[84576]: cluster 2024-09-13T14:47:19.272329+0000 mon.a (mon.0) 1591 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:47:20.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[87147]: cluster 2024-09-13T14:47:18.479013+0000 mgr.y (mgr.26252) 629 : cluster [DBG] pgmap v1078: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:47:20.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[87147]: cluster 2024-09-13T14:47:18.479013+0000 mgr.y (mgr.26252) 629 : cluster [DBG] pgmap v1078: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:47:20.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[87147]: cluster 2024-09-13T14:47:18.875457+0000 mon.a (mon.0) 1590 : cluster [DBG] osdmap e955: 8 total, 8 up, 7 in 2024-09-13T14:47:20.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[87147]: cluster 2024-09-13T14:47:18.875457+0000 mon.a (mon.0) 1590 : cluster [DBG] osdmap e955: 8 total, 8 up, 7 in 2024-09-13T14:47:20.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[87147]: cluster 2024-09-13T14:47:19.272329+0000 mon.a (mon.0) 1591 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:47:20.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:19 smithi099 bash[87147]: cluster 2024-09-13T14:47:19.272329+0000 mon.a (mon.0) 1591 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:47:20.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:20.358 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:20.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:20.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:20.488 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:20.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:20.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:20.601 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:47:20 smithi177 bash[61439]: ts=2024-09-13T14:47:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:20.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:20.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:20.720 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:20.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:20.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:20.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:20.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:20.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:20.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:21.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:21.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:21.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:20 smithi099 bash[84576]: cluster 2024-09-13T14:47:20.480015+0000 mgr.y (mgr.26252) 630 : cluster [DBG] pgmap v1080: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 1.0 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-13T14:47:21.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:20 smithi099 bash[84576]: cluster 2024-09-13T14:47:20.480015+0000 mgr.y (mgr.26252) 630 : cluster [DBG] pgmap v1080: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 1.0 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-13T14:47:21.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:20 smithi099 bash[87147]: cluster 2024-09-13T14:47:20.480015+0000 mgr.y (mgr.26252) 630 : cluster [DBG] pgmap v1080: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 1.0 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-13T14:47:21.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:20 smithi099 bash[87147]: cluster 2024-09-13T14:47:20.480015+0000 mgr.y (mgr.26252) 630 : cluster [DBG] pgmap v1080: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 1.0 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-13T14:47:21.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:21.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:21.143 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:20 smithi177 bash[68594]: cluster 2024-09-13T14:47:20.480015+0000 mgr.y (mgr.26252) 630 : cluster [DBG] pgmap v1080: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 1.0 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-13T14:47:21.143 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:20 smithi177 bash[68594]: cluster 2024-09-13T14:47:20.480015+0000 mgr.y (mgr.26252) 630 : cluster [DBG] pgmap v1080: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 1.0 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-13T14:47:21.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:21.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:21.270 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:21.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:21.373 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:21.400 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:21.465 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:47:21.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:21.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:21.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:21.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:21.655 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:21.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:21.758 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:21.786 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:21.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:21.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:21.917 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:22.021 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:22.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:22.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:22.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:22.179 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:22.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:22.187 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:22.189 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:22.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:22.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:22.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:22.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:22.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:22.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:22.541 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:22.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:22.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:22.700 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:22.803 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:22.831 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:22.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:22.934 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:22.961 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:23.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:23.092 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:23.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:23.195 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:23.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:23.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:23.454 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.477 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:23.580 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:23.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:23.735 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:23 smithi099 bash[84576]: cluster 2024-09-13T14:47:22.480900+0000 mgr.y (mgr.26252) 631 : cluster [DBG] pgmap v1081: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 814 B/s, 1 keys/s, 2 objects/s recovering 2024-09-13T14:47:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:23 smithi099 bash[84576]: cluster 2024-09-13T14:47:22.480900+0000 mgr.y (mgr.26252) 631 : cluster [DBG] pgmap v1081: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 814 B/s, 1 keys/s, 2 objects/s recovering 2024-09-13T14:47:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:23 smithi099 bash[87147]: cluster 2024-09-13T14:47:22.480900+0000 mgr.y (mgr.26252) 631 : cluster [DBG] pgmap v1081: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 814 B/s, 1 keys/s, 2 objects/s recovering 2024-09-13T14:47:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:23 smithi099 bash[87147]: cluster 2024-09-13T14:47:22.480900+0000 mgr.y (mgr.26252) 631 : cluster [DBG] pgmap v1081: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 814 B/s, 1 keys/s, 2 objects/s recovering 2024-09-13T14:47:23.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:23.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:23 smithi177 bash[68594]: cluster 2024-09-13T14:47:22.480900+0000 mgr.y (mgr.26252) 631 : cluster [DBG] pgmap v1081: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 814 B/s, 1 keys/s, 2 objects/s recovering 2024-09-13T14:47:23.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:23 smithi177 bash[68594]: cluster 2024-09-13T14:47:22.480900+0000 mgr.y (mgr.26252) 631 : cluster [DBG] pgmap v1081: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 814 B/s, 1 keys/s, 2 objects/s recovering 2024-09-13T14:47:23.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:23.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:23.968 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:23.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:23.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:24.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:24.119 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:24.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:24.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:24.229 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:24.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:24.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:24.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:24.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:24.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:24.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:24.502 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:24.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:24.599 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:47:24.599 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:47:24.599 INFO:tasks.thrashosds.thrasher:Adding osd 4 2024-09-13T14:47:24.599 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd in 4 2024-09-13T14:47:24.605 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:24.632 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:24.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:24.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:24.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:24.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:24.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:24.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:24.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:24.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:25.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:25.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:25.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:25.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:25.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:25.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:25.284 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:25.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:25.386 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:25.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:25.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:25.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:25.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:25.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:25.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:25.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:25.779 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:25.806 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:25 smithi177 bash[68594]: cluster 2024-09-13T14:47:24.481679+0000 mgr.y (mgr.26252) 632 : cluster [DBG] pgmap v1082: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 640 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:25 smithi177 bash[68594]: cluster 2024-09-13T14:47:24.481679+0000 mgr.y (mgr.26252) 632 : cluster [DBG] pgmap v1082: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 640 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:25.893 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:25 smithi099 bash[87147]: cluster 2024-09-13T14:47:24.481679+0000 mgr.y (mgr.26252) 632 : cluster [DBG] pgmap v1082: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 640 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:25.894 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:25 smithi099 bash[87147]: cluster 2024-09-13T14:47:24.481679+0000 mgr.y (mgr.26252) 632 : cluster [DBG] pgmap v1082: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 640 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:25.894 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:25 smithi099 bash[84576]: cluster 2024-09-13T14:47:24.481679+0000 mgr.y (mgr.26252) 632 : cluster [DBG] pgmap v1082: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 640 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:25.894 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:25 smithi099 bash[84576]: cluster 2024-09-13T14:47:24.481679+0000 mgr.y (mgr.26252) 632 : cluster [DBG] pgmap v1082: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 640 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:25.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:25.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:25.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:26.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:26.067 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:26.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:26.170 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:26.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:26.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:26.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:26.330 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:26.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:26.433 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:26.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:26.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:26.562 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:26.592 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:26.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:26.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:26.722 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:26.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:26.849 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:26.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:26.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:26.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:27.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:27.109 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:27.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:27.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:27.240 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:27.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:27.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:27.371 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:27.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:27.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:27.484 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:47:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:47:27] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-13T14:47:27.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:27.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:27.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:27.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:27.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:27.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:27.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:27 smithi099 bash[84576]: cluster 2024-09-13T14:47:26.482296+0000 mgr.y (mgr.26252) 633 : cluster [DBG] pgmap v1083: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 566 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:27 smithi099 bash[84576]: cluster 2024-09-13T14:47:26.482296+0000 mgr.y (mgr.26252) 633 : cluster [DBG] pgmap v1083: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 566 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:27 smithi099 bash[87147]: cluster 2024-09-13T14:47:26.482296+0000 mgr.y (mgr.26252) 633 : cluster [DBG] pgmap v1083: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 566 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:27 smithi099 bash[87147]: cluster 2024-09-13T14:47:26.482296+0000 mgr.y (mgr.26252) 633 : cluster [DBG] pgmap v1083: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 566 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:27.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:27 smithi177 bash[68594]: cluster 2024-09-13T14:47:26.482296+0000 mgr.y (mgr.26252) 633 : cluster [DBG] pgmap v1083: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 566 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:27 smithi177 bash[68594]: cluster 2024-09-13T14:47:26.482296+0000 mgr.y (mgr.26252) 633 : cluster [DBG] pgmap v1083: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 566 B/s, 1 keys/s, 1 objects/s recovering 2024-09-13T14:47:27.882 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:27.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:27.953 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:27.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:28.004 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:28.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:28.134 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:28.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:28.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:28.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:28.368 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:28.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:28.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:28.523 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:28.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:28.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:28.756 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:28.780 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:28.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:28.883 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:28.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:28.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:29.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:29.041 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:29.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:29.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:29.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:29.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:29.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:29.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:29.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:29.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:29.436 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:29.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:29.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:29.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:29.670 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:29.680 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:29 smithi177 bash[68594]: cluster 2024-09-13T14:47:28.483083+0000 mgr.y (mgr.26252) 634 : cluster [DBG] pgmap v1084: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 488 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:29.680 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:29 smithi177 bash[68594]: cluster 2024-09-13T14:47:28.483083+0000 mgr.y (mgr.26252) 634 : cluster [DBG] pgmap v1084: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 488 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:29.681 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:29 smithi177 bash[68594]: audit 2024-09-13T14:47:29.274506+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:29.681 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:29 smithi177 bash[68594]: audit 2024-09-13T14:47:29.274506+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:29.695 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:29.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:29.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:29.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:29.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:29 smithi099 bash[84576]: cluster 2024-09-13T14:47:28.483083+0000 mgr.y (mgr.26252) 634 : cluster [DBG] pgmap v1084: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 488 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:29.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:29 smithi099 bash[84576]: cluster 2024-09-13T14:47:28.483083+0000 mgr.y (mgr.26252) 634 : cluster [DBG] pgmap v1084: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 488 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:29.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:29 smithi099 bash[84576]: audit 2024-09-13T14:47:29.274506+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:29.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:29 smithi099 bash[84576]: audit 2024-09-13T14:47:29.274506+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:29.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:29 smithi099 bash[87147]: cluster 2024-09-13T14:47:28.483083+0000 mgr.y (mgr.26252) 634 : cluster [DBG] pgmap v1084: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 488 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:29.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:29 smithi099 bash[87147]: cluster 2024-09-13T14:47:28.483083+0000 mgr.y (mgr.26252) 634 : cluster [DBG] pgmap v1084: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 488 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:29.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:29 smithi099 bash[87147]: audit 2024-09-13T14:47:29.274506+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:29.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:29 smithi099 bash[87147]: audit 2024-09-13T14:47:29.274506+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:29.927 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:29.956 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:30.059 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:30.084 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:30.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:30.187 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:30.209 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:30.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:30.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:30.324 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:47:30 smithi177 bash[61439]: ts=2024-09-13T14:47:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:30.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:30.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:30.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:30.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:30.526 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:30.573 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:30.597 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:30.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:30.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:30.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:30.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:30.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:30.852 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:30.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:30.956 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:30.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:30.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:31.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:31.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:31.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:31.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:31.240 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:31.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:31.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:31.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:31.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:31.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:31.500 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:31.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:31.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:31.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:31 smithi177 bash[68594]: cluster 2024-09-13T14:47:30.484124+0000 mgr.y (mgr.26252) 635 : cluster [DBG] pgmap v1085: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 421 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:31.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:31 smithi177 bash[68594]: cluster 2024-09-13T14:47:30.484124+0000 mgr.y (mgr.26252) 635 : cluster [DBG] pgmap v1085: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 421 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:31.631 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:31.632 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:31 smithi099 bash[84576]: cluster 2024-09-13T14:47:30.484124+0000 mgr.y (mgr.26252) 635 : cluster [DBG] pgmap v1085: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 421 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:31.632 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:31 smithi099 bash[84576]: cluster 2024-09-13T14:47:30.484124+0000 mgr.y (mgr.26252) 635 : cluster [DBG] pgmap v1085: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 421 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:31.632 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:31 smithi099 bash[87147]: cluster 2024-09-13T14:47:30.484124+0000 mgr.y (mgr.26252) 635 : cluster [DBG] pgmap v1085: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 421 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:31.633 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:31 smithi099 bash[87147]: cluster 2024-09-13T14:47:30.484124+0000 mgr.y (mgr.26252) 635 : cluster [DBG] pgmap v1085: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 421 B/s, 0 keys/s, 1 objects/s recovering 2024-09-13T14:47:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:31.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:31.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:31.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:31.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:31.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:31.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:31.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:32.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:32.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:32.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:32.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:32.254 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:32.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:32.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:32.411 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:32.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:32.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:32.540 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:32.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:32.643 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:32.670 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:32.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:32.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:32.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:32.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:32.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:32.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:33.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:33.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:33.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:33.165 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:33.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:33.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:33.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:33.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:33.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:33.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:33.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:33.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:33.556 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:33.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:33 smithi177 bash[68594]: cluster 2024-09-13T14:47:32.484656+0000 mgr.y (mgr.26252) 636 : cluster [DBG] pgmap v1086: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:33.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:33 smithi177 bash[68594]: cluster 2024-09-13T14:47:32.484656+0000 mgr.y (mgr.26252) 636 : cluster [DBG] pgmap v1086: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:33.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:33 smithi177 bash[68594]: audit 2024-09-13T14:47:33.277716+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:33.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:33 smithi177 bash[68594]: audit 2024-09-13T14:47:33.277716+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:33.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:33 smithi177 bash[68594]: audit 2024-09-13T14:47:33.279857+0000 mon.a (mon.0) 1594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:33.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:33 smithi177 bash[68594]: audit 2024-09-13T14:47:33.279857+0000 mon.a (mon.0) 1594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:33.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:33.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:33.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:33.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:33.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:33.819 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[84576]: cluster 2024-09-13T14:47:32.484656+0000 mgr.y (mgr.26252) 636 : cluster [DBG] pgmap v1086: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:33.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[84576]: cluster 2024-09-13T14:47:32.484656+0000 mgr.y (mgr.26252) 636 : cluster [DBG] pgmap v1086: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[84576]: audit 2024-09-13T14:47:33.277716+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[84576]: audit 2024-09-13T14:47:33.277716+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[84576]: audit 2024-09-13T14:47:33.279857+0000 mon.a (mon.0) 1594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[84576]: audit 2024-09-13T14:47:33.279857+0000 mon.a (mon.0) 1594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[87147]: cluster 2024-09-13T14:47:32.484656+0000 mgr.y (mgr.26252) 636 : cluster [DBG] pgmap v1086: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[87147]: cluster 2024-09-13T14:47:32.484656+0000 mgr.y (mgr.26252) 636 : cluster [DBG] pgmap v1086: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[87147]: audit 2024-09-13T14:47:33.277716+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[87147]: audit 2024-09-13T14:47:33.277716+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[87147]: audit 2024-09-13T14:47:33.279857+0000 mon.a (mon.0) 1594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:33.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:33 smithi099 bash[87147]: audit 2024-09-13T14:47:33.279857+0000 mon.a (mon.0) 1594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:33.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:33.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:33.950 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:33.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:33.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:34.078 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:34.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:34.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:34.205 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:34.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:34.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:34.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:34.360 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:34.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:34.463 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:34.491 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:34.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:34.594 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:34.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:34.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:34.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:34.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:34.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:34.880 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:34.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:35.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:35.133 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:35.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:35.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:35.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:35.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:35.342 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:35.342 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:35.368 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:35.394 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:35.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:35.497 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:35.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:35.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:35.543 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:47:35.543 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:35.561 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:47:35.562 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:35.577 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:47:35.578 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:35.596 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:47:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:35.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:35 smithi099 bash[84576]: cluster 2024-09-13T14:47:34.485449+0000 mgr.y (mgr.26252) 637 : cluster [DBG] pgmap v1087: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:35.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:35 smithi099 bash[84576]: cluster 2024-09-13T14:47:34.485449+0000 mgr.y (mgr.26252) 637 : cluster [DBG] pgmap v1087: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:35.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:35 smithi099 bash[84576]: audit 2024-09-13T14:47:35.528919+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.99:0/1156159047' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:35.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:35 smithi099 bash[84576]: audit 2024-09-13T14:47:35.528919+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.99:0/1156159047' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:35.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:35 smithi099 bash[87147]: cluster 2024-09-13T14:47:34.485449+0000 mgr.y (mgr.26252) 637 : cluster [DBG] pgmap v1087: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:35.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:35 smithi099 bash[87147]: cluster 2024-09-13T14:47:34.485449+0000 mgr.y (mgr.26252) 637 : cluster [DBG] pgmap v1087: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:35.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:35 smithi099 bash[87147]: audit 2024-09-13T14:47:35.528919+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.99:0/1156159047' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:35.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:35 smithi099 bash[87147]: audit 2024-09-13T14:47:35.528919+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.99:0/1156159047' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:35.617 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:47:35.617 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:35.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:35.642 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:47:35.642 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:35.661 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:47:35.661 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:35.681 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:47:35.681 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:35.739 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:35.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:35.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:35 smithi177 bash[68594]: cluster 2024-09-13T14:47:34.485449+0000 mgr.y (mgr.26252) 637 : cluster [DBG] pgmap v1087: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:35 smithi177 bash[68594]: cluster 2024-09-13T14:47:34.485449+0000 mgr.y (mgr.26252) 637 : cluster [DBG] pgmap v1087: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:35 smithi177 bash[68594]: audit 2024-09-13T14:47:35.528919+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.99:0/1156159047' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:35 smithi177 bash[68594]: audit 2024-09-13T14:47:35.528919+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.99:0/1156159047' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:35.868 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.889 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:35.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:36.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:36.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:36.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:36.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:36.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:36.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:36.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:36.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:36.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:36.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:36.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:36.509 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:36.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:36.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:36.641 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:36.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:36.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:36.771 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:36.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:36.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:36.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:36.928 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:36.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:37.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:37.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:37.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:37.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:37.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:37.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:37.319 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:37.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:37.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:37.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:37.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:37.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:37.558 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:47:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:47:37] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.51.0" 2024-09-13T14:47:37.579 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:37.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:37.682 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:37.709 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:37.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:37.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:37 smithi099 bash[84576]: cluster 2024-09-13T14:47:36.486177+0000 mgr.y (mgr.26252) 638 : cluster [DBG] pgmap v1088: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:37.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:37 smithi099 bash[84576]: cluster 2024-09-13T14:47:36.486177+0000 mgr.y (mgr.26252) 638 : cluster [DBG] pgmap v1088: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:37.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:37 smithi099 bash[87147]: cluster 2024-09-13T14:47:36.486177+0000 mgr.y (mgr.26252) 638 : cluster [DBG] pgmap v1088: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:37.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:37 smithi099 bash[87147]: cluster 2024-09-13T14:47:36.486177+0000 mgr.y (mgr.26252) 638 : cluster [DBG] pgmap v1088: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:37.835 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:37.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:37 smithi177 bash[68594]: cluster 2024-09-13T14:47:36.486177+0000 mgr.y (mgr.26252) 638 : cluster [DBG] pgmap v1088: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:37 smithi177 bash[68594]: cluster 2024-09-13T14:47:36.486177+0000 mgr.y (mgr.26252) 638 : cluster [DBG] pgmap v1088: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:37.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:37.965 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:37.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:38.068 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:38.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:38.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:38.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:38.225 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:38.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:38.328 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:38.354 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:38.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:38.457 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:38.485 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:38.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:38.579 INFO:teuthology.orchestra.run.smithi099.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-13T14:47:38.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:38.617 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:38.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:38.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:38.730 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:38 smithi177 bash[68594]: audit 2024-09-13T14:47:38.176441+0000 mon.c (mon.2) 50 : audit [INF] from='client.? 172.21.15.99:0/837132075' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:38.730 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:38 smithi177 bash[68594]: audit 2024-09-13T14:47:38.176441+0000 mon.c (mon.2) 50 : audit [INF] from='client.? 172.21.15.99:0/837132075' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:38.730 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:38 smithi177 bash[68594]: audit 2024-09-13T14:47:38.177147+0000 mon.a (mon.0) 1595 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:38.730 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:38 smithi177 bash[68594]: audit 2024-09-13T14:47:38.177147+0000 mon.a (mon.0) 1595 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:38.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:38.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:38.847 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:38.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:38.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:38.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:39.004 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:39.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:38 smithi099 bash[87147]: audit 2024-09-13T14:47:38.176441+0000 mon.c (mon.2) 50 : audit [INF] from='client.? 172.21.15.99:0/837132075' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:39.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:38 smithi099 bash[87147]: audit 2024-09-13T14:47:38.176441+0000 mon.c (mon.2) 50 : audit [INF] from='client.? 172.21.15.99:0/837132075' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:39.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:38 smithi099 bash[87147]: audit 2024-09-13T14:47:38.177147+0000 mon.a (mon.0) 1595 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:39.044 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:38 smithi099 bash[87147]: audit 2024-09-13T14:47:38.177147+0000 mon.a (mon.0) 1595 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:39.044 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:38 smithi099 bash[84576]: audit 2024-09-13T14:47:38.176441+0000 mon.c (mon.2) 50 : audit [INF] from='client.? 172.21.15.99:0/837132075' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:39.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:38 smithi099 bash[84576]: audit 2024-09-13T14:47:38.176441+0000 mon.c (mon.2) 50 : audit [INF] from='client.? 172.21.15.99:0/837132075' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:39.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:38 smithi099 bash[84576]: audit 2024-09-13T14:47:38.177147+0000 mon.a (mon.0) 1595 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:39.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:38 smithi099 bash[84576]: audit 2024-09-13T14:47:38.177147+0000 mon.a (mon.0) 1595 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:47:39.107 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:39.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:39.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:39.238 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:39.264 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:39.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:39.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:39.393 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:39.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:39.524 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:39.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:39.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:39.651 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:39.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:39.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:39.765 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[84576]: cluster 2024-09-13T14:47:38.487127+0000 mgr.y (mgr.26252) 639 : cluster [DBG] pgmap v1089: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:39.765 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[84576]: cluster 2024-09-13T14:47:38.487127+0000 mgr.y (mgr.26252) 639 : cluster [DBG] pgmap v1089: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:39.765 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[84576]: audit 2024-09-13T14:47:38.573821+0000 mon.a (mon.0) 1596 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:47:39.765 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[84576]: audit 2024-09-13T14:47:38.573821+0000 mon.a (mon.0) 1596 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:47:39.765 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[84576]: cluster 2024-09-13T14:47:38.581655+0000 mon.a (mon.0) 1597 : cluster [DBG] osdmap e956: 8 total, 8 up, 7 in 2024-09-13T14:47:39.765 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[84576]: cluster 2024-09-13T14:47:38.581655+0000 mon.a (mon.0) 1597 : cluster [DBG] osdmap e956: 8 total, 8 up, 7 in 2024-09-13T14:47:39.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[87147]: cluster 2024-09-13T14:47:38.487127+0000 mgr.y (mgr.26252) 639 : cluster [DBG] pgmap v1089: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:39.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[87147]: cluster 2024-09-13T14:47:38.487127+0000 mgr.y (mgr.26252) 639 : cluster [DBG] pgmap v1089: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:39.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[87147]: audit 2024-09-13T14:47:38.573821+0000 mon.a (mon.0) 1596 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:47:39.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[87147]: audit 2024-09-13T14:47:38.573821+0000 mon.a (mon.0) 1596 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:47:39.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[87147]: cluster 2024-09-13T14:47:38.581655+0000 mon.a (mon.0) 1597 : cluster [DBG] osdmap e956: 8 total, 8 up, 7 in 2024-09-13T14:47:39.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:39 smithi099 bash[87147]: cluster 2024-09-13T14:47:38.581655+0000 mon.a (mon.0) 1597 : cluster [DBG] osdmap e956: 8 total, 8 up, 7 in 2024-09-13T14:47:39.780 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:39 smithi177 bash[68594]: cluster 2024-09-13T14:47:38.487127+0000 mgr.y (mgr.26252) 639 : cluster [DBG] pgmap v1089: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:39 smithi177 bash[68594]: cluster 2024-09-13T14:47:38.487127+0000 mgr.y (mgr.26252) 639 : cluster [DBG] pgmap v1089: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:39 smithi177 bash[68594]: audit 2024-09-13T14:47:38.573821+0000 mon.a (mon.0) 1596 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:47:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:39 smithi177 bash[68594]: audit 2024-09-13T14:47:38.573821+0000 mon.a (mon.0) 1596 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:47:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:39 smithi177 bash[68594]: cluster 2024-09-13T14:47:38.581655+0000 mon.a (mon.0) 1597 : cluster [DBG] osdmap e956: 8 total, 8 up, 7 in 2024-09-13T14:47:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:39 smithi177 bash[68594]: cluster 2024-09-13T14:47:38.581655+0000 mon.a (mon.0) 1597 : cluster [DBG] osdmap e956: 8 total, 8 up, 7 in 2024-09-13T14:47:39.883 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:39.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:39.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:40.011 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:40.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:40.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:40.140 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:40.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:40.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:40.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:47:40.269 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:40.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:40.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:40.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:40.420 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:40.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:40.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:40.534 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:47:40 smithi177 bash[61439]: ts=2024-09-13T14:47:40.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:40.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:40.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:40.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:40.663 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:40 smithi099 bash[84576]: cluster 2024-09-13T14:47:40.575483+0000 mon.a (mon.0) 1598 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:47:40.663 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:40 smithi099 bash[84576]: cluster 2024-09-13T14:47:40.575483+0000 mon.a (mon.0) 1598 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:47:40.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:40 smithi099 bash[87147]: cluster 2024-09-13T14:47:40.575483+0000 mon.a (mon.0) 1598 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:47:40.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:40 smithi099 bash[87147]: cluster 2024-09-13T14:47:40.575483+0000 mon.a (mon.0) 1598 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:47:40.679 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:40.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:40.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:40.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:40.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:40.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:40 smithi177 bash[68594]: cluster 2024-09-13T14:47:40.575483+0000 mon.a (mon.0) 1598 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:47:40.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:40 smithi177 bash[68594]: cluster 2024-09-13T14:47:40.575483+0000 mon.a (mon.0) 1598 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:47:40.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:40.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:41.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:47:41.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:41.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:41.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:41.177 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:41.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:41.308 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:41.336 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:41.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:41.439 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:41.465 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:41.568 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:41.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:41.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:41.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:41.709 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:41 smithi177 bash[68594]: cluster 2024-09-13T14:47:40.487914+0000 mgr.y (mgr.26252) 640 : cluster [DBG] pgmap v1091: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:41.709 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:41 smithi177 bash[68594]: cluster 2024-09-13T14:47:40.487914+0000 mgr.y (mgr.26252) 640 : cluster [DBG] pgmap v1091: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:41.725 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:41.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:41.736 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:41 smithi099 bash[84576]: cluster 2024-09-13T14:47:40.487914+0000 mgr.y (mgr.26252) 640 : cluster [DBG] pgmap v1091: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:41.736 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:41 smithi099 bash[84576]: cluster 2024-09-13T14:47:40.487914+0000 mgr.y (mgr.26252) 640 : cluster [DBG] pgmap v1091: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:41.737 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:41 smithi099 bash[87147]: cluster 2024-09-13T14:47:40.487914+0000 mgr.y (mgr.26252) 640 : cluster [DBG] pgmap v1091: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:41.737 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:41 smithi099 bash[87147]: cluster 2024-09-13T14:47:40.487914+0000 mgr.y (mgr.26252) 640 : cluster [DBG] pgmap v1091: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:41.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:41.855 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:41.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:41.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:41.983 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:42.086 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:42.114 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:42.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:42.217 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:42.244 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:42.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:42.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:42.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:42.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:42.487 DEBUG:teuthology.orchestra.run.smithi099:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-09-13T14:47:42.489 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:47:42.489 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:47:42.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:42.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:42.531 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:hints = 1 2024-09-13T14:47:42.531 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:47:42.532 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object prefix: benchmark_data_smithi099_126887 2024-09-13T14:47:42.532 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:47:42.532 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:47:42.609 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:42.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:42.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:42.740 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:42.763 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:42.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:42.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:42.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:42.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:42.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:43.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:43.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:43.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:43.152 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:43.255 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:43.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:43.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:43.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:43.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:43.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:43.532 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 1 16 2276 2260 141.235 141.25 0.00230175 0.00527524 2024-09-13T14:47:43.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:43.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:43.637 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:43.648 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:43 smithi099 bash[84576]: cluster 2024-09-13T14:47:42.488700+0000 mgr.y (mgr.26252) 641 : cluster [DBG] pgmap v1092: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:43.648 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:43 smithi099 bash[84576]: cluster 2024-09-13T14:47:42.488700+0000 mgr.y (mgr.26252) 641 : cluster [DBG] pgmap v1092: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:43.648 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:43 smithi099 bash[87147]: cluster 2024-09-13T14:47:42.488700+0000 mgr.y (mgr.26252) 641 : cluster [DBG] pgmap v1092: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:43.648 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:43 smithi099 bash[87147]: cluster 2024-09-13T14:47:42.488700+0000 mgr.y (mgr.26252) 641 : cluster [DBG] pgmap v1092: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:43.670 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:43.773 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:43.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:43.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:43 smithi177 bash[68594]: cluster 2024-09-13T14:47:42.488700+0000 mgr.y (mgr.26252) 641 : cluster [DBG] pgmap v1092: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:43 smithi177 bash[68594]: cluster 2024-09-13T14:47:42.488700+0000 mgr.y (mgr.26252) 641 : cluster [DBG] pgmap v1092: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-13T14:47:43.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:43.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:43.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:44.034 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:44.062 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:44.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:44.166 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:44.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:44.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:44.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:44.321 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:44.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:44.425 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:44.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:44.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:44.532 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 2 16 4988 4972 155.359 169.5 0.00184801 0.00642429 2024-09-13T14:47:44.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:44.581 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:44.659 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T14:47:44.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:44.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:44.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:44.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:44.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:44 smithi177 bash[68594]: audit 2024-09-13T14:47:44.419008+0000 mon.b (mon.1) 5 : audit [INF] from='client.? 172.21.15.99:0/775888469' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:44 smithi177 bash[68594]: audit 2024-09-13T14:47:44.419008+0000 mon.b (mon.1) 5 : audit [INF] from='client.? 172.21.15.99:0/775888469' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:44 smithi177 bash[68594]: audit 2024-09-13T14:47:44.421899+0000 mon.a (mon.0) 1599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:44 smithi177 bash[68594]: audit 2024-09-13T14:47:44.421899+0000 mon.a (mon.0) 1599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.840 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:44.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:44.943 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:44.953 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:44 smithi099 bash[84576]: audit 2024-09-13T14:47:44.419008+0000 mon.b (mon.1) 5 : audit [INF] from='client.? 172.21.15.99:0/775888469' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.953 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:44 smithi099 bash[84576]: audit 2024-09-13T14:47:44.419008+0000 mon.b (mon.1) 5 : audit [INF] from='client.? 172.21.15.99:0/775888469' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.953 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:44 smithi099 bash[84576]: audit 2024-09-13T14:47:44.421899+0000 mon.a (mon.0) 1599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.953 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:44 smithi099 bash[84576]: audit 2024-09-13T14:47:44.421899+0000 mon.a (mon.0) 1599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.953 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:44 smithi099 bash[87147]: audit 2024-09-13T14:47:44.419008+0000 mon.b (mon.1) 5 : audit [INF] from='client.? 172.21.15.99:0/775888469' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.953 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:44 smithi099 bash[87147]: audit 2024-09-13T14:47:44.419008+0000 mon.b (mon.1) 5 : audit [INF] from='client.? 172.21.15.99:0/775888469' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.953 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:44 smithi099 bash[87147]: audit 2024-09-13T14:47:44.421899+0000 mon.a (mon.0) 1599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.953 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:44 smithi099 bash[87147]: audit 2024-09-13T14:47:44.421899+0000 mon.a (mon.0) 1599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:44.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:45.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:45.104 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:45.207 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:45.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:45.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:45.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:45.355 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:45.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:45.480 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:45.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:45.532 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 3 16 7547 7531 156.88 159.938 0.00675494 0.0063663 2024-09-13T14:47:45.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: debug 2024-09-13T14:47:45.251+0000 7f0688bd2640 -1 mon.a@0(leader).osd e957 definitely_dead 0 2024-09-13T14:47:45.583 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:45.608 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:45.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:45.654 INFO:teuthology.orchestra.run.smithi099.stderr:marked in osd.4. 2024-09-13T14:47:45.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:45.728 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:45.831 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:45.854 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:45.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:45 smithi177 bash[68594]: cluster 2024-09-13T14:47:44.489387+0000 mgr.y (mgr.26252) 642 : cluster [DBG] pgmap v1093: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 11 MiB/s wr, 176 op/s 2024-09-13T14:47:45.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:45 smithi177 bash[68594]: cluster 2024-09-13T14:47:44.489387+0000 mgr.y (mgr.26252) 642 : cluster [DBG] pgmap v1093: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 11 MiB/s wr, 176 op/s 2024-09-13T14:47:45.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:45 smithi177 bash[68594]: cluster 2024-09-13T14:47:44.638509+0000 mon.a (mon.0) 1600 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:45.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:45 smithi177 bash[68594]: cluster 2024-09-13T14:47:44.638509+0000 mon.a (mon.0) 1600 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:45 smithi177 bash[68594]: audit 2024-09-13T14:47:44.645622+0000 mon.a (mon.0) 1601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:45 smithi177 bash[68594]: audit 2024-09-13T14:47:44.645622+0000 mon.a (mon.0) 1601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:45 smithi177 bash[68594]: cluster 2024-09-13T14:47:44.654115+0000 mon.a (mon.0) 1602 : cluster [DBG] osdmap e957: 8 total, 8 up, 7 in 2024-09-13T14:47:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:45 smithi177 bash[68594]: cluster 2024-09-13T14:47:44.654115+0000 mon.a (mon.0) 1602 : cluster [DBG] osdmap e957: 8 total, 8 up, 7 in 2024-09-13T14:47:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:45 smithi177 bash[68594]: audit 2024-09-13T14:47:45.255144+0000 mon.a (mon.0) 1603 : audit [DBG] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:47:45.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:45 smithi177 bash[68594]: audit 2024-09-13T14:47:45.255144+0000 mon.a (mon.0) 1603 : audit [DBG] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:47:45.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:45.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:45.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: cluster 2024-09-13T14:47:44.489387+0000 mgr.y (mgr.26252) 642 : cluster [DBG] pgmap v1093: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 11 MiB/s wr, 176 op/s 2024-09-13T14:47:45.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: cluster 2024-09-13T14:47:44.489387+0000 mgr.y (mgr.26252) 642 : cluster [DBG] pgmap v1093: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 11 MiB/s wr, 176 op/s 2024-09-13T14:47:45.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: cluster 2024-09-13T14:47:44.638509+0000 mon.a (mon.0) 1600 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:45.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: cluster 2024-09-13T14:47:44.638509+0000 mon.a (mon.0) 1600 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:45.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: audit 2024-09-13T14:47:44.645622+0000 mon.a (mon.0) 1601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:45.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: audit 2024-09-13T14:47:44.645622+0000 mon.a (mon.0) 1601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:45.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: cluster 2024-09-13T14:47:44.654115+0000 mon.a (mon.0) 1602 : cluster [DBG] osdmap e957: 8 total, 8 up, 7 in 2024-09-13T14:47:45.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: cluster 2024-09-13T14:47:44.654115+0000 mon.a (mon.0) 1602 : cluster [DBG] osdmap e957: 8 total, 8 up, 7 in 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: audit 2024-09-13T14:47:45.255144+0000 mon.a (mon.0) 1603 : audit [DBG] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[84576]: audit 2024-09-13T14:47:45.255144+0000 mon.a (mon.0) 1603 : audit [DBG] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[87147]: cluster 2024-09-13T14:47:44.489387+0000 mgr.y (mgr.26252) 642 : cluster [DBG] pgmap v1093: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 11 MiB/s wr, 176 op/s 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[87147]: cluster 2024-09-13T14:47:44.489387+0000 mgr.y (mgr.26252) 642 : cluster [DBG] pgmap v1093: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 11 MiB/s wr, 176 op/s 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[87147]: cluster 2024-09-13T14:47:44.638509+0000 mon.a (mon.0) 1600 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[87147]: cluster 2024-09-13T14:47:44.638509+0000 mon.a (mon.0) 1600 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[87147]: audit 2024-09-13T14:47:44.645622+0000 mon.a (mon.0) 1601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[87147]: audit 2024-09-13T14:47:44.645622+0000 mon.a (mon.0) 1601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[87147]: cluster 2024-09-13T14:47:44.654115+0000 mon.a (mon.0) 1602 : cluster [DBG] osdmap e957: 8 total, 8 up, 7 in 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[87147]: cluster 2024-09-13T14:47:44.654115+0000 mon.a (mon.0) 1602 : cluster [DBG] osdmap e957: 8 total, 8 up, 7 in 2024-09-13T14:47:45.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[87147]: audit 2024-09-13T14:47:45.255144+0000 mon.a (mon.0) 1603 : audit [DBG] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:47:45.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:45 smithi099 bash[87147]: audit 2024-09-13T14:47:45.255144+0000 mon.a (mon.0) 1603 : audit [DBG] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:47:45.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:45.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:46.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:46.112 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:46.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:46.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:46.241 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:46.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:46.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:46.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:46.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:46.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:46.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:46.532 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 4 16 7929 7913 123.627 23.875 0.00151106 0.00635858 2024-09-13T14:47:46.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:46.622 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:46.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:46.725 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:46.736 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:46 smithi177 bash[68594]: audit 2024-09-13T14:47:45.656991+0000 mon.a (mon.0) 1604 : audit [INF] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:47:46.736 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:46 smithi177 bash[68594]: audit 2024-09-13T14:47:45.656991+0000 mon.a (mon.0) 1604 : audit [INF] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:47:46.736 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:46 smithi177 bash[68594]: cluster 2024-09-13T14:47:45.662445+0000 mon.a (mon.0) 1605 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-09-13T14:47:46.736 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:46 smithi177 bash[68594]: cluster 2024-09-13T14:47:45.662445+0000 mon.a (mon.0) 1605 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-09-13T14:47:46.736 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:46 smithi177 bash[68594]: cluster 2024-09-13T14:47:46.311557+0000 mon.a (mon.0) 1606 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-09-13T14:47:46.737 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:46 smithi177 bash[68594]: cluster 2024-09-13T14:47:46.311557+0000 mon.a (mon.0) 1606 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-09-13T14:47:46.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:46.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:46.855 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:46.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:46.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:46.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.012 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:47.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[84576]: audit 2024-09-13T14:47:45.656991+0000 mon.a (mon.0) 1604 : audit [INF] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:47:47.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[84576]: audit 2024-09-13T14:47:45.656991+0000 mon.a (mon.0) 1604 : audit [INF] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:47:47.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[84576]: cluster 2024-09-13T14:47:45.662445+0000 mon.a (mon.0) 1605 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-09-13T14:47:47.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[84576]: cluster 2024-09-13T14:47:45.662445+0000 mon.a (mon.0) 1605 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-09-13T14:47:47.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[84576]: cluster 2024-09-13T14:47:46.311557+0000 mon.a (mon.0) 1606 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-09-13T14:47:47.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[84576]: cluster 2024-09-13T14:47:46.311557+0000 mon.a (mon.0) 1606 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-09-13T14:47:47.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[87147]: audit 2024-09-13T14:47:45.656991+0000 mon.a (mon.0) 1604 : audit [INF] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:47:47.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[87147]: audit 2024-09-13T14:47:45.656991+0000 mon.a (mon.0) 1604 : audit [INF] from='client.? 172.21.15.99:0/808752154' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:47:47.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[87147]: cluster 2024-09-13T14:47:45.662445+0000 mon.a (mon.0) 1605 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-09-13T14:47:47.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[87147]: cluster 2024-09-13T14:47:45.662445+0000 mon.a (mon.0) 1605 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-09-13T14:47:47.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[87147]: cluster 2024-09-13T14:47:46.311557+0000 mon.a (mon.0) 1606 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-09-13T14:47:47.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:46 smithi099 bash[87147]: cluster 2024-09-13T14:47:46.311557+0000 mon.a (mon.0) 1606 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-09-13T14:47:47.115 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:47.141 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:47.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:47.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:47.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:47.348 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:47:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:47:47] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.51.0" 2024-09-13T14:47:47.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:47.409 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:47.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:47.512 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.532 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 5 16 8294 8278 103.464 22.8125 0.00796275 0.00965494 2024-09-13T14:47:47.540 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:47.643 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.674 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:47.779 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:47.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:47.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:47 smithi099 bash[84576]: cluster 2024-09-13T14:47:46.489996+0000 mgr.y (mgr.26252) 643 : cluster [DBG] pgmap v1097: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 18 MiB/s wr, 293 op/s 2024-09-13T14:47:47.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:47 smithi099 bash[84576]: cluster 2024-09-13T14:47:46.489996+0000 mgr.y (mgr.26252) 643 : cluster [DBG] pgmap v1097: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 18 MiB/s wr, 293 op/s 2024-09-13T14:47:47.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:47 smithi099 bash[84576]: cluster 2024-09-13T14:47:47.319823+0000 mon.a (mon.0) 1607 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-09-13T14:47:47.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:47 smithi099 bash[84576]: cluster 2024-09-13T14:47:47.319823+0000 mon.a (mon.0) 1607 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-09-13T14:47:47.805 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:47 smithi099 bash[87147]: cluster 2024-09-13T14:47:46.489996+0000 mgr.y (mgr.26252) 643 : cluster [DBG] pgmap v1097: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 18 MiB/s wr, 293 op/s 2024-09-13T14:47:47.805 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:47 smithi099 bash[87147]: cluster 2024-09-13T14:47:46.489996+0000 mgr.y (mgr.26252) 643 : cluster [DBG] pgmap v1097: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 18 MiB/s wr, 293 op/s 2024-09-13T14:47:47.805 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:47 smithi099 bash[87147]: cluster 2024-09-13T14:47:47.319823+0000 mon.a (mon.0) 1607 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-09-13T14:47:47.805 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:47 smithi099 bash[87147]: cluster 2024-09-13T14:47:47.319823+0000 mon.a (mon.0) 1607 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-09-13T14:47:47.902 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:47 smithi177 bash[68594]: cluster 2024-09-13T14:47:46.489996+0000 mgr.y (mgr.26252) 643 : cluster [DBG] pgmap v1097: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 18 MiB/s wr, 293 op/s 2024-09-13T14:47:47.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:47 smithi177 bash[68594]: cluster 2024-09-13T14:47:46.489996+0000 mgr.y (mgr.26252) 643 : cluster [DBG] pgmap v1097: 89 pgs: 89 active+clean; 111 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 18 MiB/s wr, 293 op/s 2024-09-13T14:47:47.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:47 smithi177 bash[68594]: cluster 2024-09-13T14:47:47.319823+0000 mon.a (mon.0) 1607 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-09-13T14:47:47.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:47 smithi177 bash[68594]: cluster 2024-09-13T14:47:47.319823+0000 mon.a (mon.0) 1607 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-09-13T14:47:47.937 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:48.041 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:48.068 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:48.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:48.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:48.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:48.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:48.320 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:48.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:48.425 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:48.443 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:48.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:48.533 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 6 16 9904 9888 102.989 100.625 0.00736424 0.00970151 2024-09-13T14:47:48.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:48.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:48.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:48.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:48.697 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:48.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:48.801 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:48.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:48.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:48.929 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:48.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:48.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:49.060 INFO:tasks.thrashosds.thrasher:Added osd 4 2024-09-13T14:47:49.062 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:49.089 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:49.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:49.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:49.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:49.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:49.323 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:49.356 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:49.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[84576]: audit 2024-09-13T14:47:48.277652+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:49.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[84576]: audit 2024-09-13T14:47:48.277652+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:49.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[84576]: audit 2024-09-13T14:47:48.279015+0000 mon.a (mon.0) 1609 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:49.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[84576]: audit 2024-09-13T14:47:48.279015+0000 mon.a (mon.0) 1609 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:49.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[84576]: cluster 2024-09-13T14:47:48.490936+0000 mgr.y (mgr.26252) 644 : cluster [DBG] pgmap v1099: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 397 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 1479/19116 objects degraded (7.737%); 499/19116 objects misplaced (2.610%); 0 B/s, 0 objects/s recovering 2024-09-13T14:47:49.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[84576]: cluster 2024-09-13T14:47:48.490936+0000 mgr.y (mgr.26252) 644 : cluster [DBG] pgmap v1099: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 397 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 1479/19116 objects degraded (7.737%); 499/19116 objects misplaced (2.610%); 0 B/s, 0 objects/s recovering 2024-09-13T14:47:49.357 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[87147]: audit 2024-09-13T14:47:48.277652+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:49.357 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[87147]: audit 2024-09-13T14:47:48.277652+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:49.357 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[87147]: audit 2024-09-13T14:47:48.279015+0000 mon.a (mon.0) 1609 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:49.358 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[87147]: audit 2024-09-13T14:47:48.279015+0000 mon.a (mon.0) 1609 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:49.358 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[87147]: cluster 2024-09-13T14:47:48.490936+0000 mgr.y (mgr.26252) 644 : cluster [DBG] pgmap v1099: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 397 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 1479/19116 objects degraded (7.737%); 499/19116 objects misplaced (2.610%); 0 B/s, 0 objects/s recovering 2024-09-13T14:47:49.358 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:49 smithi099 bash[87147]: cluster 2024-09-13T14:47:48.490936+0000 mgr.y (mgr.26252) 644 : cluster [DBG] pgmap v1099: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 397 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 1479/19116 objects degraded (7.737%); 499/19116 objects misplaced (2.610%); 0 B/s, 0 objects/s recovering 2024-09-13T14:47:49.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:49.459 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:49.472 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:49.472 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:49.480 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:49.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:49.533 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 7 16 12025 12009 107.212 132.562 0.0014928 0.00909053 2024-09-13T14:47:49.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:49.602 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:49 smithi177 bash[68594]: audit 2024-09-13T14:47:48.277652+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:49 smithi177 bash[68594]: audit 2024-09-13T14:47:48.277652+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:49 smithi177 bash[68594]: audit 2024-09-13T14:47:48.279015+0000 mon.a (mon.0) 1609 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:49 smithi177 bash[68594]: audit 2024-09-13T14:47:48.279015+0000 mon.a (mon.0) 1609 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:49 smithi177 bash[68594]: cluster 2024-09-13T14:47:48.490936+0000 mgr.y (mgr.26252) 644 : cluster [DBG] pgmap v1099: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 397 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 1479/19116 objects degraded (7.737%); 499/19116 objects misplaced (2.610%); 0 B/s, 0 objects/s recovering 2024-09-13T14:47:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:49 smithi177 bash[68594]: cluster 2024-09-13T14:47:48.490936+0000 mgr.y (mgr.26252) 644 : cluster [DBG] pgmap v1099: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 397 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 1479/19116 objects degraded (7.737%); 499/19116 objects misplaced (2.610%); 0 B/s, 0 objects/s recovering 2024-09-13T14:47:49.705 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:49.725 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:49.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:49.828 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:49.852 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:49.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:49.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:49.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:49.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:50.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:50.104 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:50.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:50.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:50.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:50.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:50.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:50.365 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:50.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:50.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:50.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:50.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:50.533 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 8 16 12428 12412 96.9585 25.1875 0.00530274 0.0103059 2024-09-13T14:47:50.601 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:50.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:47:50 smithi177 bash[61439]: ts=2024-09-13T14:47:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:47:50.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:50 smithi177 bash[68594]: cluster 2024-09-13T14:47:49.273202+0000 mon.a (mon.0) 1610 : cluster [WRN] Health check failed: Degraded data redundancy: 1479/19116 objects degraded (7.737%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:50.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:50 smithi177 bash[68594]: cluster 2024-09-13T14:47:49.273202+0000 mon.a (mon.0) 1610 : cluster [WRN] Health check failed: Degraded data redundancy: 1479/19116 objects degraded (7.737%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:50.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:50 smithi177 bash[68594]: cluster 2024-09-13T14:47:49.298084+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-09-13T14:47:50.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:50 smithi177 bash[68594]: cluster 2024-09-13T14:47:49.298084+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-09-13T14:47:50.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:50.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:50.734 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:50.764 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:50.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:50.815 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:50 smithi099 bash[84576]: cluster 2024-09-13T14:47:49.273202+0000 mon.a (mon.0) 1610 : cluster [WRN] Health check failed: Degraded data redundancy: 1479/19116 objects degraded (7.737%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:50.815 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:50 smithi099 bash[84576]: cluster 2024-09-13T14:47:49.273202+0000 mon.a (mon.0) 1610 : cluster [WRN] Health check failed: Degraded data redundancy: 1479/19116 objects degraded (7.737%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:50.815 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:50 smithi099 bash[84576]: cluster 2024-09-13T14:47:49.298084+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-09-13T14:47:50.815 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:50 smithi099 bash[84576]: cluster 2024-09-13T14:47:49.298084+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-09-13T14:47:50.815 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:50 smithi099 bash[87147]: cluster 2024-09-13T14:47:49.273202+0000 mon.a (mon.0) 1610 : cluster [WRN] Health check failed: Degraded data redundancy: 1479/19116 objects degraded (7.737%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:50.815 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:50 smithi099 bash[87147]: cluster 2024-09-13T14:47:49.273202+0000 mon.a (mon.0) 1610 : cluster [WRN] Health check failed: Degraded data redundancy: 1479/19116 objects degraded (7.737%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:50.816 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:50 smithi099 bash[87147]: cluster 2024-09-13T14:47:49.298084+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-09-13T14:47:50.816 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:50 smithi099 bash[87147]: cluster 2024-09-13T14:47:49.298084+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-09-13T14:47:50.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:47:50.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:50.922 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:50.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:51.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:51.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:51.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:51.156 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:51.174 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:51.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:51.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:51.301 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:51.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:51.407 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:51.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:51.533 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 9 16 13546 13530 93.948 69.875 0.00175829 0.0101306 2024-09-13T14:47:51.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:51.547 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:51 smithi099 bash[84576]: cluster 2024-09-13T14:47:50.310217+0000 mon.a (mon.0) 1612 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2024-09-13T14:47:51.548 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:51 smithi099 bash[84576]: cluster 2024-09-13T14:47:50.310217+0000 mon.a (mon.0) 1612 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2024-09-13T14:47:51.548 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:51 smithi099 bash[84576]: cluster 2024-09-13T14:47:50.493958+0000 mgr.y (mgr.26252) 645 : cluster [DBG] pgmap v1102: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 577 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 1977/27735 objects degraded (7.128%); 521/27735 objects misplaced (1.878%); 7.0 MiB/s, 2 keys/s, 114 objects/s recovering 2024-09-13T14:47:51.548 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:51 smithi099 bash[84576]: cluster 2024-09-13T14:47:50.493958+0000 mgr.y (mgr.26252) 645 : cluster [DBG] pgmap v1102: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 577 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 1977/27735 objects degraded (7.128%); 521/27735 objects misplaced (1.878%); 7.0 MiB/s, 2 keys/s, 114 objects/s recovering 2024-09-13T14:47:51.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:51 smithi099 bash[87147]: cluster 2024-09-13T14:47:50.310217+0000 mon.a (mon.0) 1612 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2024-09-13T14:47:51.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:51 smithi099 bash[87147]: cluster 2024-09-13T14:47:50.310217+0000 mon.a (mon.0) 1612 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2024-09-13T14:47:51.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:51 smithi099 bash[87147]: cluster 2024-09-13T14:47:50.493958+0000 mgr.y (mgr.26252) 645 : cluster [DBG] pgmap v1102: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 577 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 1977/27735 objects degraded (7.128%); 521/27735 objects misplaced (1.878%); 7.0 MiB/s, 2 keys/s, 114 objects/s recovering 2024-09-13T14:47:51.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:51 smithi099 bash[87147]: cluster 2024-09-13T14:47:50.493958+0000 mgr.y (mgr.26252) 645 : cluster [DBG] pgmap v1102: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 577 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 1977/27735 objects degraded (7.128%); 521/27735 objects misplaced (1.878%); 7.0 MiB/s, 2 keys/s, 114 objects/s recovering 2024-09-13T14:47:51.568 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:51.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:51 smithi177 bash[68594]: cluster 2024-09-13T14:47:50.310217+0000 mon.a (mon.0) 1612 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2024-09-13T14:47:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:51 smithi177 bash[68594]: cluster 2024-09-13T14:47:50.310217+0000 mon.a (mon.0) 1612 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2024-09-13T14:47:51.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:51 smithi177 bash[68594]: cluster 2024-09-13T14:47:50.493958+0000 mgr.y (mgr.26252) 645 : cluster [DBG] pgmap v1102: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 577 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 1977/27735 objects degraded (7.128%); 521/27735 objects misplaced (1.878%); 7.0 MiB/s, 2 keys/s, 114 objects/s recovering 2024-09-13T14:47:51.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:51 smithi177 bash[68594]: cluster 2024-09-13T14:47:50.493958+0000 mgr.y (mgr.26252) 645 : cluster [DBG] pgmap v1102: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 577 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 1977/27735 objects degraded (7.128%); 521/27735 objects misplaced (1.878%); 7.0 MiB/s, 2 keys/s, 114 objects/s recovering 2024-09-13T14:47:51.670 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:51.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:51.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:51.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:51.820 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:51.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:51.923 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:51.947 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:51.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:52.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:52.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:52.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:52.180 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:52.207 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:52.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:52.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:52.340 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:52.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:52.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:52.476 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:52.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:52.533 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 10 16 14180 14164 88.5153 39.625 0.00582115 0.0112887 2024-09-13T14:47:52.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:52.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:52 smithi099 bash[84576]: cluster 2024-09-13T14:47:51.307838+0000 mon.a (mon.0) 1613 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2024-09-13T14:47:52.593 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:52 smithi099 bash[84576]: cluster 2024-09-13T14:47:51.307838+0000 mon.a (mon.0) 1613 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2024-09-13T14:47:52.593 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:52 smithi099 bash[87147]: cluster 2024-09-13T14:47:51.307838+0000 mon.a (mon.0) 1613 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2024-09-13T14:47:52.593 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:52 smithi099 bash[87147]: cluster 2024-09-13T14:47:51.307838+0000 mon.a (mon.0) 1613 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2024-09-13T14:47:52.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:52.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:52 smithi177 bash[68594]: cluster 2024-09-13T14:47:51.307838+0000 mon.a (mon.0) 1613 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2024-09-13T14:47:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:52 smithi177 bash[68594]: cluster 2024-09-13T14:47:51.307838+0000 mon.a (mon.0) 1613 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2024-09-13T14:47:52.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:52.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:52.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:52.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:52.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:52.969 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:52.995 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:52.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:53.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:53.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:53.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:53.244 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:53.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:53.372 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:53.476 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:53.486 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:53 smithi099 bash[84576]: cluster 2024-09-13T14:47:52.324451+0000 mon.a (mon.0) 1614 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2024-09-13T14:47:53.486 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:53 smithi099 bash[84576]: cluster 2024-09-13T14:47:52.324451+0000 mon.a (mon.0) 1614 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2024-09-13T14:47:53.486 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:53 smithi099 bash[84576]: cluster 2024-09-13T14:47:52.494832+0000 mgr.y (mgr.26252) 646 : cluster [DBG] pgmap v1105: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 84 active+clean; 716 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 1507/34395 objects degraded (4.381%); 521/34395 objects misplaced (1.515%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2024-09-13T14:47:53.486 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:53 smithi099 bash[84576]: cluster 2024-09-13T14:47:52.494832+0000 mgr.y (mgr.26252) 646 : cluster [DBG] pgmap v1105: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 84 active+clean; 716 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 1507/34395 objects degraded (4.381%); 521/34395 objects misplaced (1.515%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2024-09-13T14:47:53.486 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:53 smithi099 bash[87147]: cluster 2024-09-13T14:47:52.324451+0000 mon.a (mon.0) 1614 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2024-09-13T14:47:53.486 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:53 smithi099 bash[87147]: cluster 2024-09-13T14:47:52.324451+0000 mon.a (mon.0) 1614 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2024-09-13T14:47:53.486 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:53 smithi099 bash[87147]: cluster 2024-09-13T14:47:52.494832+0000 mgr.y (mgr.26252) 646 : cluster [DBG] pgmap v1105: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 84 active+clean; 716 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 1507/34395 objects degraded (4.381%); 521/34395 objects misplaced (1.515%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2024-09-13T14:47:53.487 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:53 smithi099 bash[87147]: cluster 2024-09-13T14:47:52.494832+0000 mgr.y (mgr.26252) 646 : cluster [DBG] pgmap v1105: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 84 active+clean; 716 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 1507/34395 objects degraded (4.381%); 521/34395 objects misplaced (1.515%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2024-09-13T14:47:53.500 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:53.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:53.533 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 11 16 16182 16166 91.8424 125.125 0.00334716 0.0108795 2024-09-13T14:47:53.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:53.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:53 smithi177 bash[68594]: cluster 2024-09-13T14:47:52.324451+0000 mon.a (mon.0) 1614 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2024-09-13T14:47:53.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:53 smithi177 bash[68594]: cluster 2024-09-13T14:47:52.324451+0000 mon.a (mon.0) 1614 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2024-09-13T14:47:53.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:53 smithi177 bash[68594]: cluster 2024-09-13T14:47:52.494832+0000 mgr.y (mgr.26252) 646 : cluster [DBG] pgmap v1105: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 84 active+clean; 716 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 1507/34395 objects degraded (4.381%); 521/34395 objects misplaced (1.515%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2024-09-13T14:47:53.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:53 smithi177 bash[68594]: cluster 2024-09-13T14:47:52.494832+0000 mgr.y (mgr.26252) 646 : cluster [DBG] pgmap v1105: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 84 active+clean; 716 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 1507/34395 objects degraded (4.381%); 521/34395 objects misplaced (1.515%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2024-09-13T14:47:53.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:53.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:53.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:53.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:53.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:53.863 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:53.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:53.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:53.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:54.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:54.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:54.060 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:47:54.061 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:47:54.061 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-13T14:47:54.061 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:47:54.061 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:47:54.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-13T14:47:54.121 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:54.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:54.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:54.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:54.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:54.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:54.377 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:54.400 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:54.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:54.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:54.530 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:54.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:54.533 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 12 16 18218 18202 94.7921 127.25 0.0031465 0.0105249 2024-09-13T14:47:54.633 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:54.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:54.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:54.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:54.875 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:54.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:54.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:54.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:55.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:55.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:55.118 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:55.137 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:55.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:55.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:55.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:55.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:55.362 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:55.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:55.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:55.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:55.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:55.533 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 13 16 18376 18360 88.26 9.875 0.00850106 0.0113234 2024-09-13T14:47:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:55.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:55.648 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:55 smithi177 bash[68594]: cluster 2024-09-13T14:47:54.454143+0000 mon.a (mon.0) 1615 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2024-09-13T14:47:55.649 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:55 smithi177 bash[68594]: cluster 2024-09-13T14:47:54.454143+0000 mon.a (mon.0) 1615 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2024-09-13T14:47:55.649 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:55 smithi177 bash[68594]: cluster 2024-09-13T14:47:54.495282+0000 mgr.y (mgr.26252) 647 : cluster [DBG] pgmap v1107: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:55.649 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:55 smithi177 bash[68594]: cluster 2024-09-13T14:47:54.495282+0000 mgr.y (mgr.26252) 647 : cluster [DBG] pgmap v1107: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:55.661 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:55.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:55.764 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:55.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:55 smithi099 bash[84576]: cluster 2024-09-13T14:47:54.454143+0000 mon.a (mon.0) 1615 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2024-09-13T14:47:55.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:55 smithi099 bash[84576]: cluster 2024-09-13T14:47:54.454143+0000 mon.a (mon.0) 1615 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2024-09-13T14:47:55.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:55 smithi099 bash[84576]: cluster 2024-09-13T14:47:54.495282+0000 mgr.y (mgr.26252) 647 : cluster [DBG] pgmap v1107: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:55.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:55 smithi099 bash[84576]: cluster 2024-09-13T14:47:54.495282+0000 mgr.y (mgr.26252) 647 : cluster [DBG] pgmap v1107: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:55.774 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:55 smithi099 bash[87147]: cluster 2024-09-13T14:47:54.454143+0000 mon.a (mon.0) 1615 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2024-09-13T14:47:55.774 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:55 smithi099 bash[87147]: cluster 2024-09-13T14:47:54.454143+0000 mon.a (mon.0) 1615 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2024-09-13T14:47:55.775 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:55 smithi099 bash[87147]: cluster 2024-09-13T14:47:54.495282+0000 mgr.y (mgr.26252) 647 : cluster [DBG] pgmap v1107: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:55.775 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:55 smithi099 bash[87147]: cluster 2024-09-13T14:47:54.495282+0000 mgr.y (mgr.26252) 647 : cluster [DBG] pgmap v1107: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:55.797 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:55.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:55.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:55.928 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:56.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:56.061 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:56.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:56.164 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:56.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:56.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:56.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:56.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:56.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:56.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:56.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:56.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:56.533 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 14 16 20627 20611 92.0039 140.688 0.00571125 0.0108426 2024-09-13T14:47:56.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:56.571 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:56.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:56.675 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:56.683 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:56 smithi177 bash[68594]: cluster 2024-09-13T14:47:55.447481+0000 mon.a (mon.0) 1616 : cluster [WRN] Health check update: Degraded data redundancy: 1507/45483 objects degraded (3.313%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:56.683 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:56 smithi177 bash[68594]: cluster 2024-09-13T14:47:55.447481+0000 mon.a (mon.0) 1616 : cluster [WRN] Health check update: Degraded data redundancy: 1507/45483 objects degraded (3.313%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:56.683 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:56 smithi177 bash[68594]: cluster 2024-09-13T14:47:55.468979+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2024-09-13T14:47:56.683 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:56 smithi177 bash[68594]: cluster 2024-09-13T14:47:55.468979+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2024-09-13T14:47:56.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:56.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:56.796 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:56.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:56 smithi099 bash[84576]: cluster 2024-09-13T14:47:55.447481+0000 mon.a (mon.0) 1616 : cluster [WRN] Health check update: Degraded data redundancy: 1507/45483 objects degraded (3.313%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:56.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:56 smithi099 bash[84576]: cluster 2024-09-13T14:47:55.447481+0000 mon.a (mon.0) 1616 : cluster [WRN] Health check update: Degraded data redundancy: 1507/45483 objects degraded (3.313%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:56.807 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:56 smithi099 bash[84576]: cluster 2024-09-13T14:47:55.468979+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2024-09-13T14:47:56.808 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:56 smithi099 bash[84576]: cluster 2024-09-13T14:47:55.468979+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2024-09-13T14:47:56.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:56 smithi099 bash[87147]: cluster 2024-09-13T14:47:55.447481+0000 mon.a (mon.0) 1616 : cluster [WRN] Health check update: Degraded data redundancy: 1507/45483 objects degraded (3.313%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:56.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:56 smithi099 bash[87147]: cluster 2024-09-13T14:47:55.447481+0000 mon.a (mon.0) 1616 : cluster [WRN] Health check update: Degraded data redundancy: 1507/45483 objects degraded (3.313%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:56.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:56 smithi099 bash[87147]: cluster 2024-09-13T14:47:55.468979+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2024-09-13T14:47:56.808 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:56 smithi099 bash[87147]: cluster 2024-09-13T14:47:55.468979+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2024-09-13T14:47:56.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:56.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:56.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:56.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:56.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:57.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:57.063 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:47:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:47:57] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.51.0" 2024-09-13T14:47:57.078 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:57.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:57.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:57.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:57.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:57.311 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:57.340 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:57.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:57.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:57.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:57.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:57.534 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 15 15 20807 20792 86.6244 11.3125 0.00977851 0.0115343 2024-09-13T14:47:57.581 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:57.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:57.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:57.722 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:57.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:57 smithi099 bash[84576]: cluster 2024-09-13T14:47:56.465532+0000 mon.a (mon.0) 1618 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2024-09-13T14:47:57.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:57 smithi099 bash[84576]: cluster 2024-09-13T14:47:56.465532+0000 mon.a (mon.0) 1618 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2024-09-13T14:47:57.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:57 smithi099 bash[84576]: cluster 2024-09-13T14:47:56.495687+0000 mgr.y (mgr.26252) 648 : cluster [DBG] pgmap v1110: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:57.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:57 smithi099 bash[84576]: cluster 2024-09-13T14:47:56.495687+0000 mgr.y (mgr.26252) 648 : cluster [DBG] pgmap v1110: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:57.731 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:57 smithi099 bash[87147]: cluster 2024-09-13T14:47:56.465532+0000 mon.a (mon.0) 1618 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2024-09-13T14:47:57.731 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:57 smithi099 bash[87147]: cluster 2024-09-13T14:47:56.465532+0000 mon.a (mon.0) 1618 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2024-09-13T14:47:57.731 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:57 smithi099 bash[87147]: cluster 2024-09-13T14:47:56.495687+0000 mgr.y (mgr.26252) 648 : cluster [DBG] pgmap v1110: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:57.731 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:57 smithi099 bash[87147]: cluster 2024-09-13T14:47:56.495687+0000 mgr.y (mgr.26252) 648 : cluster [DBG] pgmap v1110: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:57.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:57.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:57.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:57.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:57 smithi177 bash[68594]: cluster 2024-09-13T14:47:56.465532+0000 mon.a (mon.0) 1618 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2024-09-13T14:47:57.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:57 smithi177 bash[68594]: cluster 2024-09-13T14:47:56.465532+0000 mon.a (mon.0) 1618 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2024-09-13T14:47:57.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:57 smithi177 bash[68594]: cluster 2024-09-13T14:47:56.495687+0000 mgr.y (mgr.26252) 648 : cluster [DBG] pgmap v1110: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:57.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:57 smithi177 bash[68594]: cluster 2024-09-13T14:47:56.495687+0000 mgr.y (mgr.26252) 648 : cluster [DBG] pgmap v1110: 89 pgs: 1 active+recovery_wait, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 947 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 1507/45483 objects degraded (3.313%); 274/45483 objects misplaced (0.602%); 11 MiB/s, 171 objects/s recovering 2024-09-13T14:47:57.867 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:57.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:58.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:58.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:58.105 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:58.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:58.235 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:58.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:58.370 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:58.399 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:58.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:58.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:58.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:58.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:58.534 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 16 16 22720 22704 88.6783 119.5 0.0111032 0.0112664 2024-09-13T14:47:58.634 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:58.655 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:58.759 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:58.766 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:58 smithi099 bash[84576]: cluster 2024-09-13T14:47:57.491879+0000 mon.a (mon.0) 1619 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2024-09-13T14:47:58.767 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:58 smithi099 bash[84576]: cluster 2024-09-13T14:47:57.491879+0000 mon.a (mon.0) 1619 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2024-09-13T14:47:58.767 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:58 smithi099 bash[87147]: cluster 2024-09-13T14:47:57.491879+0000 mon.a (mon.0) 1619 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2024-09-13T14:47:58.767 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:58 smithi099 bash[87147]: cluster 2024-09-13T14:47:57.491879+0000 mon.a (mon.0) 1619 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2024-09-13T14:47:58.777 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:58.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:58.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:58 smithi177 bash[68594]: cluster 2024-09-13T14:47:57.491879+0000 mon.a (mon.0) 1619 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2024-09-13T14:47:58.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:58 smithi177 bash[68594]: cluster 2024-09-13T14:47:57.491879+0000 mon.a (mon.0) 1619 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2024-09-13T14:47:58.880 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:58.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:58.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:59.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:59.027 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:59.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:59.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:59.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:59.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:59.260 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:59.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:59.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:59.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:59.452 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:59.496 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[84576]: cluster 2024-09-13T14:47:58.486372+0000 mon.a (mon.0) 1620 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2024-09-13T14:47:59.497 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[84576]: cluster 2024-09-13T14:47:58.486372+0000 mon.a (mon.0) 1620 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2024-09-13T14:47:59.497 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[84576]: cluster 2024-09-13T14:47:58.496052+0000 mgr.y (mgr.26252) 649 : cluster [DBG] pgmap v1113: 89 pgs: 1 active+recovery_wait, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 1011/58200 objects degraded (1.737%); 12 MiB/s, 192 objects/s recovering 2024-09-13T14:47:59.497 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[84576]: cluster 2024-09-13T14:47:58.496052+0000 mgr.y (mgr.26252) 649 : cluster [DBG] pgmap v1113: 89 pgs: 1 active+recovery_wait, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 1011/58200 objects degraded (1.737%); 12 MiB/s, 192 objects/s recovering 2024-09-13T14:47:59.497 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[84576]: audit 2024-09-13T14:47:59.292644+0000 mon.a (mon.0) 1621 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:59.497 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[84576]: audit 2024-09-13T14:47:59.292644+0000 mon.a (mon.0) 1621 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:59.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.534 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 17 16 22993 22977 84.4656 17.0625 0.00345215 0.011834 2024-09-13T14:47:59.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:59.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:59 smithi177 bash[68594]: cluster 2024-09-13T14:47:58.486372+0000 mon.a (mon.0) 1620 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2024-09-13T14:47:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:59 smithi177 bash[68594]: cluster 2024-09-13T14:47:58.486372+0000 mon.a (mon.0) 1620 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2024-09-13T14:47:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:59 smithi177 bash[68594]: cluster 2024-09-13T14:47:58.496052+0000 mgr.y (mgr.26252) 649 : cluster [DBG] pgmap v1113: 89 pgs: 1 active+recovery_wait, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 1011/58200 objects degraded (1.737%); 12 MiB/s, 192 objects/s recovering 2024-09-13T14:47:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:59 smithi177 bash[68594]: cluster 2024-09-13T14:47:58.496052+0000 mgr.y (mgr.26252) 649 : cluster [DBG] pgmap v1113: 89 pgs: 1 active+recovery_wait, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 1011/58200 objects degraded (1.737%); 12 MiB/s, 192 objects/s recovering 2024-09-13T14:47:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:59 smithi177 bash[68594]: audit 2024-09-13T14:47:59.292644+0000 mon.a (mon.0) 1621 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:59 smithi177 bash[68594]: audit 2024-09-13T14:47:59.292644+0000 mon.a (mon.0) 1621 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:59 smithi177 bash[68594]: cluster 2024-09-13T14:47:59.487167+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2024-09-13T14:47:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:47:59 smithi177 bash[68594]: cluster 2024-09-13T14:47:59.487167+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2024-09-13T14:47:59.667 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:59.770 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:59.783 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[84576]: cluster 2024-09-13T14:47:59.487167+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2024-09-13T14:47:59.783 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[84576]: cluster 2024-09-13T14:47:59.487167+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2024-09-13T14:47:59.783 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[87147]: cluster 2024-09-13T14:47:58.486372+0000 mon.a (mon.0) 1620 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2024-09-13T14:47:59.784 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[87147]: cluster 2024-09-13T14:47:58.486372+0000 mon.a (mon.0) 1620 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2024-09-13T14:47:59.784 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[87147]: cluster 2024-09-13T14:47:58.496052+0000 mgr.y (mgr.26252) 649 : cluster [DBG] pgmap v1113: 89 pgs: 1 active+recovery_wait, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 1011/58200 objects degraded (1.737%); 12 MiB/s, 192 objects/s recovering 2024-09-13T14:47:59.784 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[87147]: cluster 2024-09-13T14:47:58.496052+0000 mgr.y (mgr.26252) 649 : cluster [DBG] pgmap v1113: 89 pgs: 1 active+recovery_wait, 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 1011/58200 objects degraded (1.737%); 12 MiB/s, 192 objects/s recovering 2024-09-13T14:47:59.784 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[87147]: audit 2024-09-13T14:47:59.292644+0000 mon.a (mon.0) 1621 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:59.784 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[87147]: audit 2024-09-13T14:47:59.292644+0000 mon.a (mon.0) 1621 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:47:59.785 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[87147]: cluster 2024-09-13T14:47:59.487167+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2024-09-13T14:47:59.785 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:47:59 smithi099 bash[87147]: cluster 2024-09-13T14:47:59.487167+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2024-09-13T14:47:59.798 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:59.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:59.902 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:59.932 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:59.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:00.037 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:00.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:00.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:00.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:00.201 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:00.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:00.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:00.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:00.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:00.465 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:00.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:00.468 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:48:00 smithi177 bash[61439]: ts=2024-09-13T14:48:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:00.534 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 18 16 24972 24956 86.6439 123.688 0.00409987 0.0115367 2024-09-13T14:48:00.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:00.578 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:00 smithi177 bash[68594]: cluster 2024-09-13T14:48:00.498209+0000 mon.a (mon.0) 1623 : cluster [WRN] Health check update: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:00.578 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:00 smithi177 bash[68594]: cluster 2024-09-13T14:48:00.498209+0000 mon.a (mon.0) 1623 : cluster [WRN] Health check update: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:00.598 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:00.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:00.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:00.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:00.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:00.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:00 smithi099 bash[84576]: cluster 2024-09-13T14:48:00.498209+0000 mon.a (mon.0) 1623 : cluster [WRN] Health check update: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:00.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:00 smithi099 bash[84576]: cluster 2024-09-13T14:48:00.498209+0000 mon.a (mon.0) 1623 : cluster [WRN] Health check update: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:00.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:00 smithi099 bash[87147]: cluster 2024-09-13T14:48:00.498209+0000 mon.a (mon.0) 1623 : cluster [WRN] Health check update: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:00.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:00 smithi099 bash[87147]: cluster 2024-09-13T14:48:00.498209+0000 mon.a (mon.0) 1623 : cluster [WRN] Health check update: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:00.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:00.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:00.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:00.974 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:00.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:01.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:01.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:01.122 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:01.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:01.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:01.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:01.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:01.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:01.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:01.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:01.483 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:01.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:01.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:01.534 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 19 16 26343 26327 86.5932 85.6875 0.00170229 0.0114474 2024-09-13T14:48:01.604 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:01.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:01 smithi177 bash[68594]: cluster 2024-09-13T14:48:00.496423+0000 mgr.y (mgr.26252) 650 : cluster [DBG] pgmap v1115: 89 pgs: 1 peering, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 513/64869 objects degraded (0.791%); 20 MiB/s, 314 objects/s recovering 2024-09-13T14:48:01.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:01 smithi177 bash[68594]: cluster 2024-09-13T14:48:00.496423+0000 mgr.y (mgr.26252) 650 : cluster [DBG] pgmap v1115: 89 pgs: 1 peering, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 513/64869 objects degraded (0.791%); 20 MiB/s, 314 objects/s recovering 2024-09-13T14:48:01.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:01 smithi177 bash[68594]: cluster 2024-09-13T14:48:01.308837+0000 mon.a (mon.0) 1624 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2024-09-13T14:48:01.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:01 smithi177 bash[68594]: cluster 2024-09-13T14:48:01.308837+0000 mon.a (mon.0) 1624 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2024-09-13T14:48:01.627 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:01.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:01.729 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:01.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:01 smithi099 bash[84576]: cluster 2024-09-13T14:48:00.496423+0000 mgr.y (mgr.26252) 650 : cluster [DBG] pgmap v1115: 89 pgs: 1 peering, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 513/64869 objects degraded (0.791%); 20 MiB/s, 314 objects/s recovering 2024-09-13T14:48:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:01 smithi099 bash[84576]: cluster 2024-09-13T14:48:00.496423+0000 mgr.y (mgr.26252) 650 : cluster [DBG] pgmap v1115: 89 pgs: 1 peering, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 513/64869 objects degraded (0.791%); 20 MiB/s, 314 objects/s recovering 2024-09-13T14:48:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:01 smithi099 bash[84576]: cluster 2024-09-13T14:48:01.308837+0000 mon.a (mon.0) 1624 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2024-09-13T14:48:01.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:01 smithi099 bash[84576]: cluster 2024-09-13T14:48:01.308837+0000 mon.a (mon.0) 1624 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2024-09-13T14:48:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:01 smithi099 bash[87147]: cluster 2024-09-13T14:48:00.496423+0000 mgr.y (mgr.26252) 650 : cluster [DBG] pgmap v1115: 89 pgs: 1 peering, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 513/64869 objects degraded (0.791%); 20 MiB/s, 314 objects/s recovering 2024-09-13T14:48:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:01 smithi099 bash[87147]: cluster 2024-09-13T14:48:00.496423+0000 mgr.y (mgr.26252) 650 : cluster [DBG] pgmap v1115: 89 pgs: 1 peering, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 513/64869 objects degraded (0.791%); 20 MiB/s, 314 objects/s recovering 2024-09-13T14:48:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:01 smithi099 bash[87147]: cluster 2024-09-13T14:48:01.308837+0000 mon.a (mon.0) 1624 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2024-09-13T14:48:01.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:01 smithi099 bash[87147]: cluster 2024-09-13T14:48:01.308837+0000 mon.a (mon.0) 1624 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2024-09-13T14:48:01.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:01.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:01.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:01.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.007 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:02.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:02.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:02.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:02.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:02.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:02.373 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:02.410 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:02.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:02.515 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:02.538 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T14:48:02.537837+0000 min lat: 0.00109707 max lat: 1.73119 avg lat: 0.0119872 2024-09-13T14:48:02.538 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:48:02.538 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 20 16 26695 26679 83.3634 22 0.0129789 0.0119872 2024-09-13T14:48:02.547 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:02.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:02.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:02.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:02.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:02.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:02.945 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:03.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:03.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:03.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:03.180 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:03.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:03.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:03.341 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:03.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:03.348 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[87147]: cluster 2024-09-13T14:48:02.314454+0000 mon.a (mon.0) 1625 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2024-09-13T14:48:03.348 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[87147]: cluster 2024-09-13T14:48:02.314454+0000 mon.a (mon.0) 1625 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2024-09-13T14:48:03.348 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[87147]: cluster 2024-09-13T14:48:02.496880+0000 mgr.y (mgr.26252) 651 : cluster [DBG] pgmap v1118: 89 pgs: 1 creating+peering, 1 peering, 87 active+clean; 1.5 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 16 MiB/s, 252 objects/s recovering 2024-09-13T14:48:03.348 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[87147]: cluster 2024-09-13T14:48:02.496880+0000 mgr.y (mgr.26252) 651 : cluster [DBG] pgmap v1118: 89 pgs: 1 creating+peering, 1 peering, 87 active+clean; 1.5 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 16 MiB/s, 252 objects/s recovering 2024-09-13T14:48:03.349 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[87147]: audit 2024-09-13T14:48:03.275516+0000 mon.a (mon.0) 1626 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:03.349 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[87147]: audit 2024-09-13T14:48:03.275516+0000 mon.a (mon.0) 1626 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:03.349 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[87147]: audit 2024-09-13T14:48:03.276731+0000 mon.a (mon.0) 1627 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:03.349 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[87147]: audit 2024-09-13T14:48:03.276731+0000 mon.a (mon.0) 1627 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:03.349 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[87147]: audit 2024-09-13T14:48:03.304639+0000 mon.a (mon.0) 1628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:03.349 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[87147]: audit 2024-09-13T14:48:03.304639+0000 mon.a (mon.0) 1628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:03.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[84576]: cluster 2024-09-13T14:48:02.314454+0000 mon.a (mon.0) 1625 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2024-09-13T14:48:03.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[84576]: cluster 2024-09-13T14:48:02.314454+0000 mon.a (mon.0) 1625 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2024-09-13T14:48:03.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[84576]: cluster 2024-09-13T14:48:02.496880+0000 mgr.y (mgr.26252) 651 : cluster [DBG] pgmap v1118: 89 pgs: 1 creating+peering, 1 peering, 87 active+clean; 1.5 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 16 MiB/s, 252 objects/s recovering 2024-09-13T14:48:03.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[84576]: cluster 2024-09-13T14:48:02.496880+0000 mgr.y (mgr.26252) 651 : cluster [DBG] pgmap v1118: 89 pgs: 1 creating+peering, 1 peering, 87 active+clean; 1.5 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 16 MiB/s, 252 objects/s recovering 2024-09-13T14:48:03.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[84576]: audit 2024-09-13T14:48:03.275516+0000 mon.a (mon.0) 1626 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:03.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[84576]: audit 2024-09-13T14:48:03.275516+0000 mon.a (mon.0) 1626 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:03.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[84576]: audit 2024-09-13T14:48:03.276731+0000 mon.a (mon.0) 1627 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:03.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[84576]: audit 2024-09-13T14:48:03.276731+0000 mon.a (mon.0) 1627 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:03.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[84576]: audit 2024-09-13T14:48:03.304639+0000 mon.a (mon.0) 1628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:03.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:03 smithi099 bash[84576]: audit 2024-09-13T14:48:03.304639+0000 mon.a (mon.0) 1628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:03.449 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:03.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:03.534 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 21 16 28148 28132 83.7178 90.8125 0.00707895 0.0119387 2024-09-13T14:48:03.580 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:03.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:03.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:03 smithi177 bash[68594]: cluster 2024-09-13T14:48:02.314454+0000 mon.a (mon.0) 1625 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2024-09-13T14:48:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:03 smithi177 bash[68594]: cluster 2024-09-13T14:48:02.314454+0000 mon.a (mon.0) 1625 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2024-09-13T14:48:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:03 smithi177 bash[68594]: cluster 2024-09-13T14:48:02.496880+0000 mgr.y (mgr.26252) 651 : cluster [DBG] pgmap v1118: 89 pgs: 1 creating+peering, 1 peering, 87 active+clean; 1.5 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 16 MiB/s, 252 objects/s recovering 2024-09-13T14:48:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:03 smithi177 bash[68594]: cluster 2024-09-13T14:48:02.496880+0000 mgr.y (mgr.26252) 651 : cluster [DBG] pgmap v1118: 89 pgs: 1 creating+peering, 1 peering, 87 active+clean; 1.5 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 16 MiB/s, 252 objects/s recovering 2024-09-13T14:48:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:03 smithi177 bash[68594]: audit 2024-09-13T14:48:03.275516+0000 mon.a (mon.0) 1626 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:03 smithi177 bash[68594]: audit 2024-09-13T14:48:03.275516+0000 mon.a (mon.0) 1626 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:03 smithi177 bash[68594]: audit 2024-09-13T14:48:03.276731+0000 mon.a (mon.0) 1627 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:03 smithi177 bash[68594]: audit 2024-09-13T14:48:03.276731+0000 mon.a (mon.0) 1627 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:03 smithi177 bash[68594]: audit 2024-09-13T14:48:03.304639+0000 mon.a (mon.0) 1628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:03 smithi177 bash[68594]: audit 2024-09-13T14:48:03.304639+0000 mon.a (mon.0) 1628 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:03.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:03.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:03.742 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:03.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:03.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:03.872 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:03.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:03.978 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:04.001 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:04.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:04.105 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:04.126 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:04.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:04.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:04.260 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:04.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:04.364 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:04.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:04.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:04.506 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:04 smithi099 bash[84576]: cluster 2024-09-13T14:48:03.308188+0000 mon.a (mon.0) 1629 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded) 2024-09-13T14:48:04.506 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:04 smithi099 bash[84576]: cluster 2024-09-13T14:48:03.308188+0000 mon.a (mon.0) 1629 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded) 2024-09-13T14:48:04.507 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:04 smithi099 bash[87147]: cluster 2024-09-13T14:48:03.308188+0000 mon.a (mon.0) 1629 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded) 2024-09-13T14:48:04.507 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:04 smithi099 bash[87147]: cluster 2024-09-13T14:48:03.308188+0000 mon.a (mon.0) 1629 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded) 2024-09-13T14:48:04.519 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:04.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:04.534 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 22 16 29594 29578 84.02 90.375 0.00499636 0.0118968 2024-09-13T14:48:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:04 smithi177 bash[68594]: cluster 2024-09-13T14:48:03.308188+0000 mon.a (mon.0) 1629 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded) 2024-09-13T14:48:04.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:04 smithi177 bash[68594]: cluster 2024-09-13T14:48:03.308188+0000 mon.a (mon.0) 1629 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 513/64869 objects degraded (0.791%), 1 pg degraded) 2024-09-13T14:48:04.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:04.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.649 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:04.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:04.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:04.768 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:04.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:04.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:04.901 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:04.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:05.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:05.041 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:05.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:05.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:05.173 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:05.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:05.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:05.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:05.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:05.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:05.427 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:05.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:05.529 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:05.534 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 23 16 31146 31130 84.584 97 0.00715304 0.0118164 2024-09-13T14:48:05.539 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:05 smithi099 bash[84576]: cluster 2024-09-13T14:48:04.497784+0000 mgr.y (mgr.26252) 652 : cluster [DBG] pgmap v1119: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 11 MiB/s, 168 objects/s recovering 2024-09-13T14:48:05.539 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:05 smithi099 bash[84576]: cluster 2024-09-13T14:48:04.497784+0000 mgr.y (mgr.26252) 652 : cluster [DBG] pgmap v1119: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 11 MiB/s, 168 objects/s recovering 2024-09-13T14:48:05.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:05 smithi099 bash[87147]: cluster 2024-09-13T14:48:04.497784+0000 mgr.y (mgr.26252) 652 : cluster [DBG] pgmap v1119: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 11 MiB/s, 168 objects/s recovering 2024-09-13T14:48:05.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:05 smithi099 bash[87147]: cluster 2024-09-13T14:48:04.497784+0000 mgr.y (mgr.26252) 652 : cluster [DBG] pgmap v1119: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 11 MiB/s, 168 objects/s recovering 2024-09-13T14:48:05.555 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:05.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:05 smithi177 bash[68594]: cluster 2024-09-13T14:48:04.497784+0000 mgr.y (mgr.26252) 652 : cluster [DBG] pgmap v1119: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 11 MiB/s, 168 objects/s recovering 2024-09-13T14:48:05.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:05 smithi177 bash[68594]: cluster 2024-09-13T14:48:04.497784+0000 mgr.y (mgr.26252) 652 : cluster [DBG] pgmap v1119: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 11 MiB/s, 168 objects/s recovering 2024-09-13T14:48:05.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:05.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.693 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:05.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:05.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:05.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:05.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:05.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:05.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:05.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:06.052 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:06.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:06.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:06.178 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:48:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T14:48:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T14:48:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T14:48:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7890 41.0 41:0618b954:::benchmark_data_smithi099_126887_object7889:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314348+0000", 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.860959898000001, 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.073313586, 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7890 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314348+0000", 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314348+0000", 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314350+0000", 2024-09-13T14:48:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4739999999999999e-06 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314409+0000", 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8826999999999998e-05 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314428+0000", 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8879e-05 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314442+0000", 2024-09-13T14:48:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4144999999999999e-05 2024-09-13T14:48:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314608+0000", 2024-09-13T14:48:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000165737 2024-09-13T14:48:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:48:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314627+0000", 2024-09-13T14:48:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9366000000000001e-05 2024-09-13T14:48:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.387501+0000", 2024-09-13T14:48:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0728740640000001 2024-09-13T14:48:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.387661+0000", 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000160094 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7911 41.a 41:50dfa993:::benchmark_data_smithi099_126887_object7910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314466+0000", 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.860841194999999, 2024-09-13T14:48:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0920385480000001, 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7911 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314466+0000", 2024-09-13T14:48:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314466+0000", 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314468+0000", 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1200000000000001e-06 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314505+0000", 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7172000000000003e-05 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314519+0000", 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3875e-05 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314528+0000", 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0580000000000004e-06 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.325392+0000", 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010863892999999999 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.325405+0000", 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3390000000000001e-05 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.395721+0000", 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.070316327 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.396103+0000", 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000381985 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.396584+0000", 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00048042400000000002 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.399219+0000", 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026351489999999998 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402020+0000", 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028009150000000002 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.405979+0000", 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039590570000000002 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.406038+0000", 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8906999999999997e-05 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.406505+0000", 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00046727599999999998 2024-09-13T14:48:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7914 41.a 41:52fc18a2:::benchmark_data_smithi099_126887_object7913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314554+0000", 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.860754065999998, 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.092421273, 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7914 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314554+0000", 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314554+0000", 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314555+0000", 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7999999999999993e-07 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314599+0000", 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4251e-05 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314611+0000", 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2007999999999999e-05 2024-09-13T14:48:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314621+0000", 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0375e-05 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327110+0000", 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012489109 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327122+0000", 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1985e-05 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.399315+0000", 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0721924110000001 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.399578+0000", 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000263255 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:48:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.399875+0000", 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000297175 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402094+0000", 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022189380000000002 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.403605+0000", 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00151092 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.406576+0000", 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029710460000000002 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.406651+0000", 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5186999999999997e-05 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.406975+0000", 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00032363299999999998 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7924 41.a 41:5d619088:::benchmark_data_smithi099_126887_object7923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314639+0000", 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.860668339, 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0982459849999999, 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7924 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314639+0000", 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314639+0000", 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314640+0000", 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.015e-06 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314677+0000", 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6748999999999997e-05 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314689+0000", 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2133000000000001e-05 2024-09-13T14:48:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314698+0000", 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8449999999999992e-06 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327158+0000", 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012460404 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327169+0000", 2024-09-13T14:48:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0023e-05 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.401249+0000", 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0740800580000001 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.401395+0000", 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014684799999999999 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:48:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.401566+0000", 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000170457 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.403524+0000", 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019586009999999999 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.407054+0000", 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003529131 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.412300+0000", 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052460839999999998 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.412491+0000", 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019173899999999999 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.412885+0000", 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000393898 2024-09-13T14:48:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7925 41.a 41:50f18a2a:::benchmark_data_smithi099_126887_object7924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314765+0000", 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.860542597999999, 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0986208369999999, 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7925 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314765+0000", 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314765+0000", 2024-09-13T14:48:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314766+0000", 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.128e-06 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314806+0000", 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9544e-05 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314820+0000", 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3998e-05 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314829+0000", 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0499999999999997e-06 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327205+0000", 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012376687000000001 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327215+0000", 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5179999999999993e-06 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402126+0000", 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.074911476 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402298+0000", 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000171891 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402506+0000", 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020722000000000001 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.408228+0000", 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005722703 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.408293+0000", 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4697999999999996e-05 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.412947+0000", 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046542809999999997 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.413012+0000", 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4727999999999994e-05 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.413386+0000", 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000373915 2024-09-13T14:48:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13528 41.d 41:b2f0624e:::benchmark_data_smithi099_126887_object13527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e962)", 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.103363+0000", 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.071944512, 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45605169299999998, 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13528 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103363+0000", 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103363+0000", 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103364+0000", 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2699999999999999e-06 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103391+0000", 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6205000000000001e-05 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103403+0000", 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2425e-05 2024-09-13T14:48:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103411+0000", 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9149999999999996e-06 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103552+0000", 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014133200000000001 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103723+0000", 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017070599999999999 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106181+0000", 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024575119999999998 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107504+0000", 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001323219 2024-09-13T14:48:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108459+0000", 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00095574500000000003 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559286+0000", 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.450826424 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559360+0000", 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4437999999999994e-05 2024-09-13T14:48:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559415+0000", 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4502000000000001e-05 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13531 41.0 41:09568b39:::benchmark_data_smithi099_126887_object13530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e962)", 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.104399+0000", 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.070909167, 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45930618299999998, 2024-09-13T14:48:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13531 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104399+0000", 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104399+0000", 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104400+0000", 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.319e-06 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104438+0000", 2024-09-13T14:48:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7682000000000003e-05 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104450+0000", 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2646000000000001e-05 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104457+0000", 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0249999999999997e-06 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104501+0000", 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3707e-05 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104607+0000", 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010582999999999999 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5,7", 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104731+0000", 2024-09-13T14:48:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012387100000000001 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106377+0000", 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001646214 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107174+0000", 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00079693799999999996 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108037+0000", 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00086317000000000002 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563163+0000", 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45512561299999998 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563228+0000", 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5668999999999998e-05 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563705+0000", 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00047649900000000002 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13534 41.d 41:b4692011:::benchmark_data_smithi099_126887_object13533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e962)", 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.104475+0000", 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.070832977, 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45571292200000002, 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13534 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104475+0000", 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104475+0000", 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104475+0000", 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0100000000000005e-07 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104499+0000", 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3255999999999999e-05 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104508+0000", 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9110000000000005e-06 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104514+0000", 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.361e-06 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106804+0000", 2024-09-13T14:48:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022900260000000001 2024-09-13T14:48:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106884+0000", 2024-09-13T14:48:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9691999999999995e-05 2024-09-13T14:48:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106976+0000", 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2046000000000006e-05 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558992+0000", 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45201617500000002 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559729+0000", 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00073704599999999999 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559988+0000", 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025961000000000002 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560118+0000", 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012939099999999999 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560188+0000", 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9807000000000004e-05 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13540 41.9 41:9360d5c8:::benchmark_data_smithi099_126887_object13539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e962)", 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.106714+0000", 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.068593648, 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45411328200000001, 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13540 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106714+0000", 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106714+0000", 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106716+0000", 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.677e-06 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106751+0000", 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5389000000000003e-05 2024-09-13T14:48:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106762+0000", 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1367e-05 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106992+0000", 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000229583 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107142+0000", 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000150078 2024-09-13T14:48:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107253+0000", 2024-09-13T14:48:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011100299999999999 2024-09-13T14:48:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:48:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107385+0000", 2024-09-13T14:48:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000131839 2024-09-13T14:48:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560667+0000", 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45328227100000001 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560712+0000", 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4416999999999998e-05 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560736+0000", 2024-09-13T14:48:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4258999999999999e-05 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560788+0000", 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2485000000000003e-05 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560827+0000", 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8914e-05 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:13538 41.c e962/953 41:305179fc:::benchmark_data_smithi099_126887_object13537:head v 962'852, mlcod=962'850)", 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.107328+0000", 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.067980152000001, 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45097309600000002, 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107328+0000", 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107328+0000", 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107328+0000", 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.13e-07 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107351+0000", 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2756999999999999e-05 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107359+0000", 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4090000000000003e-06 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107366+0000", 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2709999999999998e-06 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107936+0000", 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00057086699999999997 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107948+0000", 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1112e-05 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108933+0000", 2024-09-13T14:48:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00098501600000000002 2024-09-13T14:48:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558301+0000", 2024-09-13T14:48:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.44936805099999999 2024-09-13T14:48:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.35268.0:13531 41.0 e962/959 ondisk, result = 0)", 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.108299+0000", 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.067008962999999, 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45542017800000001, 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108299+0000", 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108299+0000", 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108299+0000", 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0900000000000001e-07 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108302+0000", 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8150000000000002e-06 2024-09-13T14:48:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108313+0000", 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0533000000000001e-05 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108319+0000", 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1519999999999997e-06 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563146+0000", 2024-09-13T14:48:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.454827537 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563155+0000", 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9029999999999993e-06 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563719+0000", 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00056362899999999995 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:13542 41.7 e962/953 41:e044e2cf:::benchmark_data_smithi099_126887_object13541:head v 962'875, mlcod=962'872)", 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.108483+0000", 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.066824437999999, 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45160520900000001, 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108483+0000", 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108483+0000", 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108484+0000", 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2900000000000003e-07 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108531+0000", 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7268999999999998e-05 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108537+0000", 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7450000000000001e-06 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108540+0000", 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5159999999999999e-06 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108582+0000", 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1177999999999998e-05 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108600+0000", 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8212000000000001e-05 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560050+0000", 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45145017900000001 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560088+0000", 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8481e-05 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.35268.0:13540 41.9 e962/953 ondisk, result = 0)", 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.108590+0000", 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.066717751000001, 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.452089888, 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108590+0000", 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108590+0000", 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108591+0000", 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8800000000000002e-07 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108595+0000", 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0210000000000001e-06 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108604+0000", 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4329999999999993e-06 2024-09-13T14:48:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108610+0000", 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0360000000000003e-06 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560651+0000", 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45204074599999999 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560661+0000", 2024-09-13T14:48:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0307e-05 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560680+0000", 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8657000000000001e-05 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.35268.0:13534 41.d e962/953 ondisk, result = 0)", 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.108637+0000", 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.066670651000001, 2024-09-13T14:48:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45155849999999997, 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108637+0000", 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108637+0000", 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108638+0000", 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5400000000000001e-07 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108641+0000", 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.625e-06 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108651+0000", 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6719999999999999e-06 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108657+0000", 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8490000000000002e-06 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559973+0000", 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45131629600000001 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559982+0000", 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.524e-06 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560196+0000", 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021388000000000001 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:13541 41.c e962/953 41:3de424e3:::benchmark_data_smithi099_126887_object13540:head v 962'853, mlcod=962'850)", 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.108765+0000", 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.066543054, 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45157181899999999, 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108765+0000", 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108765+0000", 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108765+0000", 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9800000000000003e-07 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108790+0000", 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4769e-05 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558374+0000", 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.44958437800000001 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558386+0000", 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1822e-05 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558458+0000", 2024-09-13T14:48:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1435999999999998e-05 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558485+0000", 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7705999999999999e-05 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560297+0000", 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018116829999999999 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560336+0000", 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9427e-05 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:28836 41.d 41:bbf0e1c8:::benchmark_data_smithi099_126887_object28835:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e972)", 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:03.997929+0000", 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 2.1773782939999999, 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.18131926800000001, 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 28836 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.997929+0000", 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.997929+0000", 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.997930+0000", 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6499999999999999e-07 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.997957+0000", 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7429000000000001e-05 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.997968+0000", 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0910000000000001e-05 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.997976+0000", 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2910000000000002e-06 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.001201+0000", 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032249779999999999 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.001399+0000", 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019799100000000001 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.001796+0000", 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00039742599999999999 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009361+0000", 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075646000000000003 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.012753+0000", 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033921659999999998 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.178943+0000", 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.16619008199999999 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.179116+0000", 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017305599999999999 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.179249+0000", 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132674 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:28841 41.d 41:be5dffdf:::benchmark_data_smithi099_126887_object28840:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e972)", 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:03.997987+0000", 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 2.1773209360000001, 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.19276228000000001, 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 28841 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.997987+0000", 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.997987+0000", 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.997987+0000", 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3900000000000005e-07 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998040+0000", 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2389000000000003e-05 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998048+0000", 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6889999999999993e-06 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998054+0000", 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6269999999999997e-06 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.003829+0000", 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057752289999999998 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.004027+0000", 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019741600000000001 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.005164+0000", 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011369710000000001 2024-09-13T14:48:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009422+0000", 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0042582679999999999 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.015544+0000", 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006122181 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.190355+0000", 2024-09-13T14:48:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.174810785 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.190629+0000", 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00027402399999999999 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.190749+0000", 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012016200000000001 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:28842 41.d 41:bd7e03ba:::benchmark_data_smithi099_126887_object28841:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e972)", 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:03.998063+0000", 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 2.1772451290000001, 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.19311845599999999, 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 28842 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998063+0000", 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998063+0000", 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998063+0000", 2024-09-13T14:48:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6899999999999997e-07 2024-09-13T14:48:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998110+0000", 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7228999999999998e-05 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998118+0000", 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6820000000000003e-06 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998123+0000", 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0889999999999999e-06 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009470+0000", 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011346459 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009603+0000", 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013335500000000001 2024-09-13T14:48:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009754+0000", 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015111399999999999 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.012671+0000", 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029166299999999999 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.015613+0000", 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029424099999999999 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.190879+0000", 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.17526604600000001 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.191055+0000", 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017573600000000001 2024-09-13T14:48:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.191181+0000", 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012603700000000001 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:28843 41.d 41:bab42194:::benchmark_data_smithi099_126887_object28842:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e972)", 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:03.998130+0000", 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 2.1771773250000002, 2024-09-13T14:48:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.19524170199999999, 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 28843 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998130+0000", 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998130+0000", 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998131+0000", 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5999999999999999e-07 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998179+0000", 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8158999999999997e-05 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998186+0000", 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.764e-06 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:03.998191+0000", 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1889999999999998e-06 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.010848+0000", 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012657376999999999 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.011023+0000", 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017479600000000001 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.011325+0000", 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000301926 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.013750+0000", 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024252850000000001 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.015678+0000", 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001927855 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.192851+0000", 2024-09-13T14:48:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.177172988 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.193213+0000", 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00036192899999999998 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.193372+0000", 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015897400000000001 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:28862 41.d 41:be1b71c6:::benchmark_data_smithi099_126887_object28861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e972)", 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:04.009523+0000", 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 2.1657848820000001, 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.188446427, 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 28862 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009523+0000", 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009523+0000", 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009525+0000", 2024-09-13T14:48:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0099999999999998e-06 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009781+0000", 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000256445 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009799+0000", 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7654000000000001e-05 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:48:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.009811+0000", 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2244e-05 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.013809+0000", 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003997741 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.014009+0000", 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000200581 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:48:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.014253+0000", 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00024373299999999999 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.015731+0000", 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001477695 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.016021+0000", 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00029029700000000002 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.196924+0000", 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.18090236100000001 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.197847+0000", 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00092312000000000004 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:04.197969+0000", 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012254599999999999 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:48:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:48:06.301 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:48:06.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:06.327 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:06.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:06.430 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:06.465 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:06.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:06.492 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:48:06.492 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:06.534 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 24 16 33007 32991 85.9056 116.312 0.00778547 0.0116347 2024-09-13T14:48:06.547 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:48:06.547 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:06.572 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:06.573 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:06.573 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:48:06.597 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:06.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:06.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:06 smithi177 bash[68594]: audit 2024-09-13T14:48:06.445149+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.99:0/1579989497' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:06.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:06 smithi177 bash[68594]: audit 2024-09-13T14:48:06.445149+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.99:0/1579989497' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:06.623 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:48:06.623 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:06.656 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:48:06.656 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:06.694 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:48:06.694 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:06.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:06.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:06 smithi099 bash[87147]: audit 2024-09-13T14:48:06.445149+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.99:0/1579989497' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:06.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:06 smithi099 bash[87147]: audit 2024-09-13T14:48:06.445149+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.99:0/1579989497' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:06.710 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:06 smithi099 bash[84576]: audit 2024-09-13T14:48:06.445149+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.99:0/1579989497' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:06.710 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:06 smithi099 bash[84576]: audit 2024-09-13T14:48:06.445149+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.99:0/1579989497' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:06.728 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:06.730 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:48:06.730 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:06.756 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:48:06.757 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:06.832 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:06.851 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:06.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:06.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:06.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:06.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:07.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:07.097 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:07.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:07.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:07.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:07.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:07.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:07.361 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:07.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:07.465 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:07.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:07.535 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 25 16 33843 33827 84.5592 52.25 0.00261602 0.0117617 2024-09-13T14:48:07.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:07.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:07.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:07.623 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:07 smithi099 bash[84576]: cluster 2024-09-13T14:48:06.498227+0000 mgr.y (mgr.26252) 653 : cluster [DBG] pgmap v1120: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:07.623 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:48:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:48:07] "GET /metrics HTTP/1.1" 200 36762 "" "Prometheus/2.51.0" 2024-09-13T14:48:07.624 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:07 smithi099 bash[87147]: cluster 2024-09-13T14:48:06.498227+0000 mgr.y (mgr.26252) 653 : cluster [DBG] pgmap v1120: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:07.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:07.753 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:07.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:07.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:07.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:07 smithi177 bash[68594]: cluster 2024-09-13T14:48:06.498227+0000 mgr.y (mgr.26252) 653 : cluster [DBG] pgmap v1120: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:07.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:07 smithi177 bash[68594]: cluster 2024-09-13T14:48:06.498227+0000 mgr.y (mgr.26252) 653 : cluster [DBG] pgmap v1120: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:07 smithi099 bash[84576]: cluster 2024-09-13T14:48:06.498227+0000 mgr.y (mgr.26252) 653 : cluster [DBG] pgmap v1120: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:07.869 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:07 smithi099 bash[87147]: cluster 2024-09-13T14:48:06.498227+0000 mgr.y (mgr.26252) 653 : cluster [DBG] pgmap v1120: 89 pgs: 1 creating+peering, 88 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:07.885 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:07.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:08.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:08.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:08.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:08.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:08.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:08.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:08.274 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:08.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:08.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.409 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:08.516 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.534 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 26 16 35592 35576 85.5108 109.312 0.00339521 0.0116867 2024-09-13T14:48:08.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:08.645 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.674 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:08.778 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:08.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:08.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:08.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:08.935 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:08.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:09.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:09.074 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:09.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:09.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:09.211 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:09.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:09.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:09.341 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:09.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:09.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:09.471 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:09.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:09.535 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 27 16 37240 37224 86.1582 103 0.00774271 0.0116009 2024-09-13T14:48:09.573 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:09.582 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:09 smithi177 bash[68594]: cluster 2024-09-13T14:48:08.498853+0000 mgr.y (mgr.26252) 654 : cluster [DBG] pgmap v1121: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 4.0 MiB/s, 64 objects/s recovering 2024-09-13T14:48:09.582 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:09 smithi177 bash[68594]: cluster 2024-09-13T14:48:08.498853+0000 mgr.y (mgr.26252) 654 : cluster [DBG] pgmap v1121: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 4.0 MiB/s, 64 objects/s recovering 2024-09-13T14:48:09.583 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:09 smithi177 bash[68594]: audit 2024-09-13T14:48:09.309511+0000 mon.a (mon.0) 1630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:09.583 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:09 smithi177 bash[68594]: audit 2024-09-13T14:48:09.309511+0000 mon.a (mon.0) 1630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:09.594 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:09.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:09.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:09.706 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:09 smithi099 bash[84576]: cluster 2024-09-13T14:48:08.498853+0000 mgr.y (mgr.26252) 654 : cluster [DBG] pgmap v1121: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 4.0 MiB/s, 64 objects/s recovering 2024-09-13T14:48:09.706 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:09 smithi099 bash[84576]: cluster 2024-09-13T14:48:08.498853+0000 mgr.y (mgr.26252) 654 : cluster [DBG] pgmap v1121: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 4.0 MiB/s, 64 objects/s recovering 2024-09-13T14:48:09.706 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:09 smithi099 bash[84576]: audit 2024-09-13T14:48:09.309511+0000 mon.a (mon.0) 1630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:09.706 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:09 smithi099 bash[84576]: audit 2024-09-13T14:48:09.309511+0000 mon.a (mon.0) 1630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:09.707 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:09 smithi099 bash[87147]: cluster 2024-09-13T14:48:08.498853+0000 mgr.y (mgr.26252) 654 : cluster [DBG] pgmap v1121: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 4.0 MiB/s, 64 objects/s recovering 2024-09-13T14:48:09.707 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:09 smithi099 bash[87147]: cluster 2024-09-13T14:48:08.498853+0000 mgr.y (mgr.26252) 654 : cluster [DBG] pgmap v1121: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 4.0 MiB/s, 64 objects/s recovering 2024-09-13T14:48:09.707 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:09 smithi099 bash[87147]: audit 2024-09-13T14:48:09.309511+0000 mon.a (mon.0) 1630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:09.707 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:09 smithi099 bash[87147]: audit 2024-09-13T14:48:09.309511+0000 mon.a (mon.0) 1630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:09.724 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:09.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:09.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:09.860 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:09.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:09.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:10.003 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:10.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:10.108 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:10.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:10.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:10.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:10.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:10.375 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:10.385 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:48:10 smithi177 bash[61439]: ts=2024-09-13T14:48:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:10.396 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:10.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:10.499 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:10.519 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:10.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:10.535 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 28 16 38400 38384 85.6701 72.5 0.00318987 0.011575 2024-09-13T14:48:10.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:48:10.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:10.673 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:10.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:10.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:10.805 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:10.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:10.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:10.938 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:10.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:11.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.068 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:11.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:10 smithi177 bash[68594]: audit 2024-09-13T14:48:09.902872+0000 mon.a (mon.0) 1631 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:10 smithi177 bash[68594]: audit 2024-09-13T14:48:09.902872+0000 mon.a (mon.0) 1631 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:10 smithi177 bash[68594]: audit 2024-09-13T14:48:09.915567+0000 mon.a (mon.0) 1632 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:10 smithi177 bash[68594]: audit 2024-09-13T14:48:09.915567+0000 mon.a (mon.0) 1632 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:10 smithi177 bash[68594]: cluster 2024-09-13T14:48:10.499848+0000 mgr.y (mgr.26252) 655 : cluster [DBG] pgmap v1122: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:48:11.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:10 smithi177 bash[68594]: cluster 2024-09-13T14:48:10.499848+0000 mgr.y (mgr.26252) 655 : cluster [DBG] pgmap v1122: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:48:11.197 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:11.300 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:11.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:10 smithi099 bash[84576]: audit 2024-09-13T14:48:09.902872+0000 mon.a (mon.0) 1631 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:10 smithi099 bash[84576]: audit 2024-09-13T14:48:09.902872+0000 mon.a (mon.0) 1631 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:10 smithi099 bash[84576]: audit 2024-09-13T14:48:09.915567+0000 mon.a (mon.0) 1632 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:10 smithi099 bash[84576]: audit 2024-09-13T14:48:09.915567+0000 mon.a (mon.0) 1632 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:10 smithi099 bash[84576]: cluster 2024-09-13T14:48:10.499848+0000 mgr.y (mgr.26252) 655 : cluster [DBG] pgmap v1122: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:48:11.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:10 smithi099 bash[84576]: cluster 2024-09-13T14:48:10.499848+0000 mgr.y (mgr.26252) 655 : cluster [DBG] pgmap v1122: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:48:11.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:10 smithi099 bash[87147]: audit 2024-09-13T14:48:09.902872+0000 mon.a (mon.0) 1631 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:10 smithi099 bash[87147]: audit 2024-09-13T14:48:09.902872+0000 mon.a (mon.0) 1631 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:11 smithi099 bash[87147]: audit 2024-09-13T14:48:09.915567+0000 mon.a (mon.0) 1632 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:11 smithi099 bash[87147]: audit 2024-09-13T14:48:09.915567+0000 mon.a (mon.0) 1632 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:11.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:11 smithi099 bash[87147]: cluster 2024-09-13T14:48:10.499848+0000 mgr.y (mgr.26252) 655 : cluster [DBG] pgmap v1122: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:48:11.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:11 smithi099 bash[87147]: cluster 2024-09-13T14:48:10.499848+0000 mgr.y (mgr.26252) 655 : cluster [DBG] pgmap v1122: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:48:11.338 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:11.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:11.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:11.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:11.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:11.535 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 29 16 39707 39691 85.5325 81.6875 0.0137413 0.0116861 2024-09-13T14:48:11.562 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:11.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:11.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:11.690 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:11.718 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:11.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:11.822 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:11.854 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:11.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:11.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:11.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:12.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:12.119 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:12.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:12.222 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:12.248 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:12.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:12.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:12.382 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:12.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:12.486 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:12.511 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:12.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:12.535 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 30 16 41416 41400 86.2414 106.812 0.00772941 0.0115903 2024-09-13T14:48:12.614 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:12.636 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:12.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:12.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:12.764 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:12.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:12.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:12.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:12.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:12.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:13.017 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:13.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:13.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:13.137 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:13.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:13.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:13.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:13.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:13.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:13.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:13.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:13.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:13.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:13.535 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 31 16 42540 42524 85.7254 70.25 0.0143602 0.0116596 2024-09-13T14:48:13.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:13.653 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:13.654 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:13 smithi099 bash[84576]: cluster 2024-09-13T14:48:12.500432+0000 mgr.y (mgr.26252) 656 : cluster [DBG] pgmap v1123: 89 pgs: 89 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 6.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:13.655 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:13 smithi099 bash[84576]: cluster 2024-09-13T14:48:12.500432+0000 mgr.y (mgr.26252) 656 : cluster [DBG] pgmap v1123: 89 pgs: 89 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 6.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:13.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:13.656 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:13 smithi099 bash[87147]: cluster 2024-09-13T14:48:12.500432+0000 mgr.y (mgr.26252) 656 : cluster [DBG] pgmap v1123: 89 pgs: 89 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 6.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:13.656 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:13 smithi099 bash[87147]: cluster 2024-09-13T14:48:12.500432+0000 mgr.y (mgr.26252) 656 : cluster [DBG] pgmap v1123: 89 pgs: 89 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 6.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:13.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:13.780 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:13.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:13 smithi177 bash[68594]: cluster 2024-09-13T14:48:12.500432+0000 mgr.y (mgr.26252) 656 : cluster [DBG] pgmap v1123: 89 pgs: 89 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 6.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:13 smithi177 bash[68594]: cluster 2024-09-13T14:48:12.500432+0000 mgr.y (mgr.26252) 656 : cluster [DBG] pgmap v1123: 89 pgs: 89 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 6.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:13.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:13.908 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:13.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:14.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:14.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:14.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:14.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:14.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:14.271 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.293 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:14.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:14.420 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:14.522 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:14.535 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 32 16 43664 43648 85.2415 70.25 0.119414 0.0116538 2024-09-13T14:48:14.547 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:14.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:14.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:14.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:14.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:14.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:14.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:14.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:15.043 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:15.065 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:15.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:15.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:15.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:48:15.313 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:15.348 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:15.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.472 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:15.535 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 33 16 44682 44666 84.5863 63.625 0.00399587 0.0117902 2024-09-13T14:48:15.575 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.596 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:15.699 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:15.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:15.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:15.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:15 smithi177 bash[68594]: cluster 2024-09-13T14:48:14.501478+0000 mgr.y (mgr.26252) 657 : cluster [DBG] pgmap v1124: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:15 smithi177 bash[68594]: cluster 2024-09-13T14:48:14.501478+0000 mgr.y (mgr.26252) 657 : cluster [DBG] pgmap v1124: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:15.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:16.005 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:16.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:16.064 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:15 smithi099 bash[87147]: cluster 2024-09-13T14:48:14.501478+0000 mgr.y (mgr.26252) 657 : cluster [DBG] pgmap v1124: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:16.064 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:15 smithi099 bash[87147]: cluster 2024-09-13T14:48:14.501478+0000 mgr.y (mgr.26252) 657 : cluster [DBG] pgmap v1124: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:16.066 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:15 smithi099 bash[84576]: cluster 2024-09-13T14:48:14.501478+0000 mgr.y (mgr.26252) 657 : cluster [DBG] pgmap v1124: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:16.066 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:15 smithi099 bash[84576]: cluster 2024-09-13T14:48:14.501478+0000 mgr.y (mgr.26252) 657 : cluster [DBG] pgmap v1124: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:48:16.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:16.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:16.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:16.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:16.298 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:16.400 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:16.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:16.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:16.529 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:16.535 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 34 16 45726 45710 84.0174 65.25 0.00674989 0.0118959 2024-09-13T14:48:16.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:16.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:16.659 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:16.669 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:16.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:16 smithi177 bash[68594]: audit 2024-09-13T14:48:15.635254+0000 mon.a (mon.0) 1633 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:16 smithi177 bash[68594]: audit 2024-09-13T14:48:15.635254+0000 mon.a (mon.0) 1633 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:16 smithi177 bash[68594]: audit 2024-09-13T14:48:15.640089+0000 mon.a (mon.0) 1634 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:16 smithi177 bash[68594]: audit 2024-09-13T14:48:15.640089+0000 mon.a (mon.0) 1634 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:16 smithi177 bash[68594]: audit 2024-09-13T14:48:16.511597+0000 mon.a (mon.0) 1635 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:16.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:16 smithi177 bash[68594]: audit 2024-09-13T14:48:16.511597+0000 mon.a (mon.0) 1635 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:16.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:16 smithi177 bash[68594]: audit 2024-09-13T14:48:16.512822+0000 mon.a (mon.0) 1636 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:16.670 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:16 smithi177 bash[68594]: audit 2024-09-13T14:48:16.512822+0000 mon.a (mon.0) 1636 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:16.670 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:16 smithi177 bash[68594]: audit 2024-09-13T14:48:16.522864+0000 mon.a (mon.0) 1637 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.670 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:16 smithi177 bash[68594]: audit 2024-09-13T14:48:16.522864+0000 mon.a (mon.0) 1637 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:16.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:16.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:16.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:16.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:16.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:16.937 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:48:16.937 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:48:16.937 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:48:16.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[84576]: audit 2024-09-13T14:48:15.635254+0000 mon.a (mon.0) 1633 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[84576]: audit 2024-09-13T14:48:15.635254+0000 mon.a (mon.0) 1633 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[84576]: audit 2024-09-13T14:48:15.640089+0000 mon.a (mon.0) 1634 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[84576]: audit 2024-09-13T14:48:15.640089+0000 mon.a (mon.0) 1634 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[84576]: audit 2024-09-13T14:48:16.511597+0000 mon.a (mon.0) 1635 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[84576]: audit 2024-09-13T14:48:16.511597+0000 mon.a (mon.0) 1635 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[84576]: audit 2024-09-13T14:48:16.512822+0000 mon.a (mon.0) 1636 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[84576]: audit 2024-09-13T14:48:16.512822+0000 mon.a (mon.0) 1636 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[84576]: audit 2024-09-13T14:48:16.522864+0000 mon.a (mon.0) 1637 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[84576]: audit 2024-09-13T14:48:16.522864+0000 mon.a (mon.0) 1637 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[87147]: audit 2024-09-13T14:48:15.635254+0000 mon.a (mon.0) 1633 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[87147]: audit 2024-09-13T14:48:15.635254+0000 mon.a (mon.0) 1633 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[87147]: audit 2024-09-13T14:48:15.640089+0000 mon.a (mon.0) 1634 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[87147]: audit 2024-09-13T14:48:15.640089+0000 mon.a (mon.0) 1634 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[87147]: audit 2024-09-13T14:48:16.511597+0000 mon.a (mon.0) 1635 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:16.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[87147]: audit 2024-09-13T14:48:16.511597+0000 mon.a (mon.0) 1635 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:48:16.953 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[87147]: audit 2024-09-13T14:48:16.512822+0000 mon.a (mon.0) 1636 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:16.953 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[87147]: audit 2024-09-13T14:48:16.512822+0000 mon.a (mon.0) 1636 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:48:16.953 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[87147]: audit 2024-09-13T14:48:16.522864+0000 mon.a (mon.0) 1637 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.953 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:16 smithi099 bash[87147]: audit 2024-09-13T14:48:16.522864+0000 mon.a (mon.0) 1637 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:16.954 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:16.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:17.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:17.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:17.213 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:48:17] "GET /metrics HTTP/1.1" 200 36762 "" "Prometheus/2.51.0" 2024-09-13T14:48:17.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:17.314 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:17.339 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:17.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:17.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:17.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:17.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:17.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 35 16 47261 47245 84.3578 95.9375 0.0101403 0.011849 2024-09-13T14:48:17.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:17.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:17.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:17.650 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T14:48:17.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:17.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:17.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:17.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[84576]: cluster 2024-09-13T14:48:16.501850+0000 mgr.y (mgr.26252) 658 : cluster [DBG] pgmap v1125: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[84576]: cluster 2024-09-13T14:48:16.501850+0000 mgr.y (mgr.26252) 658 : cluster [DBG] pgmap v1125: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[84576]: audit 2024-09-13T14:48:16.683902+0000 mon.c (mon.2) 52 : audit [INF] from='client.? 172.21.15.99:0/3101744393' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[84576]: audit 2024-09-13T14:48:16.683902+0000 mon.c (mon.2) 52 : audit [INF] from='client.? 172.21.15.99:0/3101744393' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[84576]: audit 2024-09-13T14:48:16.684882+0000 mon.a (mon.0) 1638 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[84576]: audit 2024-09-13T14:48:16.684882+0000 mon.a (mon.0) 1638 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[87147]: cluster 2024-09-13T14:48:16.501850+0000 mgr.y (mgr.26252) 658 : cluster [DBG] pgmap v1125: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[87147]: cluster 2024-09-13T14:48:16.501850+0000 mgr.y (mgr.26252) 658 : cluster [DBG] pgmap v1125: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[87147]: audit 2024-09-13T14:48:16.683902+0000 mon.c (mon.2) 52 : audit [INF] from='client.? 172.21.15.99:0/3101744393' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[87147]: audit 2024-09-13T14:48:16.683902+0000 mon.c (mon.2) 52 : audit [INF] from='client.? 172.21.15.99:0/3101744393' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[87147]: audit 2024-09-13T14:48:16.684882+0000 mon.a (mon.0) 1638 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:17 smithi099 bash[87147]: audit 2024-09-13T14:48:16.684882+0000 mon.a (mon.0) 1638 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.839 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.840 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:17 smithi177 bash[68594]: cluster 2024-09-13T14:48:16.501850+0000 mgr.y (mgr.26252) 658 : cluster [DBG] pgmap v1125: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:17.841 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:17 smithi177 bash[68594]: cluster 2024-09-13T14:48:16.501850+0000 mgr.y (mgr.26252) 658 : cluster [DBG] pgmap v1125: 89 pgs: 89 active+clean; 2.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:17.841 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:17 smithi177 bash[68594]: audit 2024-09-13T14:48:16.683902+0000 mon.c (mon.2) 52 : audit [INF] from='client.? 172.21.15.99:0/3101744393' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.841 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:17 smithi177 bash[68594]: audit 2024-09-13T14:48:16.683902+0000 mon.c (mon.2) 52 : audit [INF] from='client.? 172.21.15.99:0/3101744393' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.841 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:17 smithi177 bash[68594]: audit 2024-09-13T14:48:16.684882+0000 mon.a (mon.0) 1638 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.841 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:17 smithi177 bash[68594]: audit 2024-09-13T14:48:16.684882+0000 mon.a (mon.0) 1638 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:48:17.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:17.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:17.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:18.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:18.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:18.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:18.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:18.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:18.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:18.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:18.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:18.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:18.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:18.488 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:18.535 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 36 16 48348 48332 83.9015 67.9375 0.0114061 0.0119136 2024-09-13T14:48:18.593 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:18.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:18.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:18.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:18.732 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:18 smithi177 bash[68594]: cluster 2024-09-13T14:48:17.642947+0000 mon.a (mon.0) 1639 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:18.732 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:18 smithi177 bash[68594]: cluster 2024-09-13T14:48:17.642947+0000 mon.a (mon.0) 1639 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:18.732 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:18 smithi177 bash[68594]: audit 2024-09-13T14:48:17.644981+0000 mon.a (mon.0) 1640 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:18.732 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:18 smithi177 bash[68594]: audit 2024-09-13T14:48:17.644981+0000 mon.a (mon.0) 1640 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:18.732 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:18 smithi177 bash[68594]: cluster 2024-09-13T14:48:17.653082+0000 mon.a (mon.0) 1641 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2024-09-13T14:48:18.732 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:18 smithi177 bash[68594]: cluster 2024-09-13T14:48:17.653082+0000 mon.a (mon.0) 1641 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2024-09-13T14:48:18.732 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:18 smithi177 bash[68594]: audit 2024-09-13T14:48:18.275526+0000 mon.a (mon.0) 1642 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:18.732 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:18 smithi177 bash[68594]: audit 2024-09-13T14:48:18.275526+0000 mon.a (mon.0) 1642 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:18.732 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:18 smithi177 bash[68594]: audit 2024-09-13T14:48:18.276630+0000 mon.a (mon.0) 1643 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:18.732 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:18 smithi177 bash[68594]: audit 2024-09-13T14:48:18.276630+0000 mon.a (mon.0) 1643 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:18.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:18.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:18.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:18.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:18.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:18.986 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:18.997 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[84576]: cluster 2024-09-13T14:48:17.642947+0000 mon.a (mon.0) 1639 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:18.997 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[84576]: cluster 2024-09-13T14:48:17.642947+0000 mon.a (mon.0) 1639 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:18.997 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[84576]: audit 2024-09-13T14:48:17.644981+0000 mon.a (mon.0) 1640 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:18.997 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[84576]: audit 2024-09-13T14:48:17.644981+0000 mon.a (mon.0) 1640 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:18.997 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[84576]: cluster 2024-09-13T14:48:17.653082+0000 mon.a (mon.0) 1641 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[84576]: cluster 2024-09-13T14:48:17.653082+0000 mon.a (mon.0) 1641 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[84576]: audit 2024-09-13T14:48:18.275526+0000 mon.a (mon.0) 1642 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[84576]: audit 2024-09-13T14:48:18.275526+0000 mon.a (mon.0) 1642 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[84576]: audit 2024-09-13T14:48:18.276630+0000 mon.a (mon.0) 1643 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[84576]: audit 2024-09-13T14:48:18.276630+0000 mon.a (mon.0) 1643 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[87147]: cluster 2024-09-13T14:48:17.642947+0000 mon.a (mon.0) 1639 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[87147]: cluster 2024-09-13T14:48:17.642947+0000 mon.a (mon.0) 1639 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[87147]: audit 2024-09-13T14:48:17.644981+0000 mon.a (mon.0) 1640 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[87147]: audit 2024-09-13T14:48:17.644981+0000 mon.a (mon.0) 1640 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[87147]: cluster 2024-09-13T14:48:17.653082+0000 mon.a (mon.0) 1641 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2024-09-13T14:48:18.998 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[87147]: cluster 2024-09-13T14:48:17.653082+0000 mon.a (mon.0) 1641 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2024-09-13T14:48:18.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[87147]: audit 2024-09-13T14:48:18.275526+0000 mon.a (mon.0) 1642 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:18.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[87147]: audit 2024-09-13T14:48:18.275526+0000 mon.a (mon.0) 1642 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:18.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[87147]: audit 2024-09-13T14:48:18.276630+0000 mon.a (mon.0) 1643 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:18.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:18 smithi099 bash[87147]: audit 2024-09-13T14:48:18.276630+0000 mon.a (mon.0) 1643 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:19.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:19.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:19.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:19.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:19.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:19.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:19.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:19.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:19.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:19.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:19.497 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:19.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:19.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:19.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 37 16 49695 49679 83.909 84.1875 0.00679247 0.0119112 2024-09-13T14:48:19.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:19.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:19.761 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:19.784 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:19.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:19.787 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:19 smithi177 bash[68594]: cluster 2024-09-13T14:48:18.502356+0000 mgr.y (mgr.26252) 659 : cluster [DBG] pgmap v1127: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:48:19.787 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:19 smithi177 bash[68594]: cluster 2024-09-13T14:48:18.502356+0000 mgr.y (mgr.26252) 659 : cluster [DBG] pgmap v1127: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:48:19.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:19.912 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:19.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:19.920 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:19 smithi099 bash[84576]: cluster 2024-09-13T14:48:18.502356+0000 mgr.y (mgr.26252) 659 : cluster [DBG] pgmap v1127: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:48:19.920 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:19 smithi099 bash[84576]: cluster 2024-09-13T14:48:18.502356+0000 mgr.y (mgr.26252) 659 : cluster [DBG] pgmap v1127: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:48:19.921 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:19 smithi099 bash[87147]: cluster 2024-09-13T14:48:18.502356+0000 mgr.y (mgr.26252) 659 : cluster [DBG] pgmap v1127: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:48:19.921 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:19 smithi099 bash[87147]: cluster 2024-09-13T14:48:18.502356+0000 mgr.y (mgr.26252) 659 : cluster [DBG] pgmap v1127: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:48:20.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:20.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:20.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:20.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:20.172 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:20.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:20.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:20.301 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:20.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:20.404 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:20.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:20.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:20.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:20.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 38 16 50757 50741 83.4474 66.375 0.00531665 0.0119788 2024-09-13T14:48:20.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:20.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:20.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:48:20 smithi177 bash[61439]: ts=2024-09-13T14:48:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:20.657 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:20.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:20.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:20.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:20.817 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:20.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:20.921 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:20.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:20.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:21.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:21.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:21.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:21.180 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:21.199 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:21.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:21.302 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:21.341 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:21.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:21.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:21.468 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:21.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:21.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 39 16 52120 52104 83.4918 85.1875 0.00725618 0.011972 2024-09-13T14:48:21.573 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:21.596 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:21.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:21.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:21.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:21.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:21.810 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:21 smithi099 bash[84576]: cluster 2024-09-13T14:48:20.503225+0000 mgr.y (mgr.26252) 660 : cluster [DBG] pgmap v1128: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:48:21.811 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:21 smithi099 bash[84576]: cluster 2024-09-13T14:48:20.503225+0000 mgr.y (mgr.26252) 660 : cluster [DBG] pgmap v1128: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:48:21.811 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:21 smithi099 bash[87147]: cluster 2024-09-13T14:48:20.503225+0000 mgr.y (mgr.26252) 660 : cluster [DBG] pgmap v1128: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:48:21.811 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:21 smithi099 bash[87147]: cluster 2024-09-13T14:48:20.503225+0000 mgr.y (mgr.26252) 660 : cluster [DBG] pgmap v1128: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:48:21.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:21.865 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:21.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:21.969 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:21.992 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:22.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:22.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:21 smithi177 bash[68594]: cluster 2024-09-13T14:48:20.503225+0000 mgr.y (mgr.26252) 660 : cluster [DBG] pgmap v1128: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:48:22.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:21 smithi177 bash[68594]: cluster 2024-09-13T14:48:20.503225+0000 mgr.y (mgr.26252) 660 : cluster [DBG] pgmap v1128: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:48:22.169 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:22.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:22.273 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:22.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:22.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:22.408 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:22.440 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:22.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:22.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T14:48:22.539734+0000 min lat: 0.00109707 max lat: 1.73119 avg lat: 0.0119334 2024-09-13T14:48:22.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:48:22.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 40 16 53621 53605 83.7496 93.8125 0.00372184 0.0119334 2024-09-13T14:48:22.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:22.575 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:22.678 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:22.705 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:22.810 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:22.838 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:22.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:22.943 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:22.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:22.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:23.063 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:23.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:23.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:23.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:23.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:23.344 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:23.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:23.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:23.474 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:23.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:23.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 41 16 54298 54282 82.7388 42.3125 0.00452774 0.0120817 2024-09-13T14:48:23.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:23.616 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:23.720 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:23.750 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:23.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:23.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:48:23.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:23.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:23 smithi099 bash[84576]: cluster 2024-09-13T14:48:22.503752+0000 mgr.y (mgr.26252) 661 : cluster [DBG] pgmap v1129: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:48:23.869 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:23 smithi099 bash[84576]: cluster 2024-09-13T14:48:22.503752+0000 mgr.y (mgr.26252) 661 : cluster [DBG] pgmap v1129: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:48:23.870 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:23 smithi099 bash[87147]: cluster 2024-09-13T14:48:22.503752+0000 mgr.y (mgr.26252) 661 : cluster [DBG] pgmap v1129: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:48:23.870 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:23 smithi099 bash[87147]: cluster 2024-09-13T14:48:22.503752+0000 mgr.y (mgr.26252) 661 : cluster [DBG] pgmap v1129: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:48:23.902 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:23.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:24.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:24.036 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:24.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:24.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:23 smithi177 bash[68594]: cluster 2024-09-13T14:48:22.503752+0000 mgr.y (mgr.26252) 661 : cluster [DBG] pgmap v1129: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:48:24.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:23 smithi177 bash[68594]: cluster 2024-09-13T14:48:22.503752+0000 mgr.y (mgr.26252) 661 : cluster [DBG] pgmap v1129: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:48:24.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:24.160 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:24.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:24.265 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:24.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:24.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:24.324 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:24.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:24.427 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:24.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:24.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:24.537 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 42 16 55552 55536 82.6348 78.375 0.0103478 0.0120959 2024-09-13T14:48:24.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:24.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:24.671 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:24.693 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:24.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:24.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:24.835 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:24.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:24.941 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:24.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:25.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:25.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.096 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:25.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:25.197 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:25.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.354 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:25.458 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:25.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 43 16 56788 56772 82.5094 77.25 0.00516475 0.0121154 2024-09-13T14:48:25.589 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:25.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.730 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:25 smithi099 bash[87147]: cluster 2024-09-13T14:48:24.504294+0000 mgr.y (mgr.26252) 662 : cluster [DBG] pgmap v1130: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:25.730 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:25 smithi099 bash[87147]: cluster 2024-09-13T14:48:24.504294+0000 mgr.y (mgr.26252) 662 : cluster [DBG] pgmap v1130: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:25.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:25 smithi099 bash[84576]: cluster 2024-09-13T14:48:24.504294+0000 mgr.y (mgr.26252) 662 : cluster [DBG] pgmap v1130: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:25.730 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:25 smithi099 bash[84576]: cluster 2024-09-13T14:48:24.504294+0000 mgr.y (mgr.26252) 662 : cluster [DBG] pgmap v1130: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:25.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:25.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.872 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:25.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:26.004 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:26.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:25 smithi177 bash[68594]: cluster 2024-09-13T14:48:24.504294+0000 mgr.y (mgr.26252) 662 : cluster [DBG] pgmap v1130: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:25 smithi177 bash[68594]: cluster 2024-09-13T14:48:24.504294+0000 mgr.y (mgr.26252) 662 : cluster [DBG] pgmap v1130: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:26.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:26.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:26.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:26.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:26.380 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:26.413 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:26.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:26.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:26.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 44 16 57944 57928 82.2761 72.25 0.0152671 0.0121488 2024-09-13T14:48:26.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:26.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:26.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:26.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:26.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:26.779 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:26.890 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:48:26.890 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:48:26.890 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-13T14:48:26.890 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd out 1 2024-09-13T14:48:26.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:26.939 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:26.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:27.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:27.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:27.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:27.193 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:27.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:27.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:27.305 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:48:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:48:27] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.51.0" 2024-09-13T14:48:27.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:27.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:27.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:27.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:27.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:27.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 45 16 59167 59151 82.1462 76.4375 0.00970895 0.0121685 2024-09-13T14:48:27.564 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:27.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:27.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:27.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:27.710 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:27 smithi099 bash[84576]: cluster 2024-09-13T14:48:26.504688+0000 mgr.y (mgr.26252) 663 : cluster [DBG] pgmap v1131: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:27.710 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:27 smithi099 bash[84576]: cluster 2024-09-13T14:48:26.504688+0000 mgr.y (mgr.26252) 663 : cluster [DBG] pgmap v1131: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:27.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:27 smithi099 bash[87147]: cluster 2024-09-13T14:48:26.504688+0000 mgr.y (mgr.26252) 663 : cluster [DBG] pgmap v1131: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:27.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:27 smithi099 bash[87147]: cluster 2024-09-13T14:48:26.504688+0000 mgr.y (mgr.26252) 663 : cluster [DBG] pgmap v1131: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:27.726 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:27.830 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:27.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:27.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:27.960 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:27.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:27 smithi177 bash[68594]: cluster 2024-09-13T14:48:26.504688+0000 mgr.y (mgr.26252) 663 : cluster [DBG] pgmap v1131: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:27.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:27 smithi177 bash[68594]: cluster 2024-09-13T14:48:26.504688+0000 mgr.y (mgr.26252) 663 : cluster [DBG] pgmap v1131: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:48:27.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.990 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:27.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:28.091 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:28.195 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:28.196 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:28.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.240 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:28.343 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:28.370 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:28.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:28.403 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:28.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:28.501 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:28.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:28.536 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 46 16 60500 60484 82.1714 83.3125 0.00884011 0.0121631 2024-09-13T14:48:28.604 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:28.737 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.774 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:28.878 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.902 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:29.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:29.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:29.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:29.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:29.164 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:29.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:29.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:29.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:29.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:29.398 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:29.423 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:29.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:29.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:29.537 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 47 16 61766 61750 82.1064 79.125 0.0023112 0.0121492 2024-09-13T14:48:29.548 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:29.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:29.678 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:29.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:29.793 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:29 smithi177 bash[68594]: cluster 2024-09-13T14:48:28.505207+0000 mgr.y (mgr.26252) 664 : cluster [DBG] pgmap v1132: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:48:29.793 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:29 smithi177 bash[68594]: cluster 2024-09-13T14:48:28.505207+0000 mgr.y (mgr.26252) 664 : cluster [DBG] pgmap v1132: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:48:29.833 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:29.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:29.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:29.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:29 smithi099 bash[84576]: cluster 2024-09-13T14:48:28.505207+0000 mgr.y (mgr.26252) 664 : cluster [DBG] pgmap v1132: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:48:29.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:29 smithi099 bash[84576]: cluster 2024-09-13T14:48:28.505207+0000 mgr.y (mgr.26252) 664 : cluster [DBG] pgmap v1132: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:48:29.951 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:29 smithi099 bash[87147]: cluster 2024-09-13T14:48:28.505207+0000 mgr.y (mgr.26252) 664 : cluster [DBG] pgmap v1132: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:48:29.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:29 smithi099 bash[87147]: cluster 2024-09-13T14:48:28.505207+0000 mgr.y (mgr.26252) 664 : cluster [DBG] pgmap v1132: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-13T14:48:29.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:29.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:30.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:30.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:30.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:30.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:30.224 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:30.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:30.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:30.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:30.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:30.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:30.477 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:48:30 smithi177 bash[61439]: ts=2024-09-13T14:48:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:30.497 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:30.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:30.537 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 48 15 63053 63038 82.0728 80.5 0.00517499 0.012179 2024-09-13T14:48:30.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:30.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:30.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:30.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:30.771 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:30.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:30.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:30.898 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:30.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:31.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:31.032 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:31.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:31.136 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:31.158 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:31.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:31.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:31.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:31.388 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:31.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:31.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:31.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:31.537 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 49 16 64437 64421 82.1617 86.4375 0.0202245 0.0121652 2024-09-13T14:48:31.542 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:31.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:31.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:31.670 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:31.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:31.794 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:31.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:31.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:31.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:31.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:31.963 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:31 smithi099 bash[84576]: cluster 2024-09-13T14:48:30.505654+0000 mgr.y (mgr.26252) 665 : cluster [DBG] pgmap v1133: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:48:31.963 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:31 smithi099 bash[84576]: cluster 2024-09-13T14:48:30.505654+0000 mgr.y (mgr.26252) 665 : cluster [DBG] pgmap v1133: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:48:31.964 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:31 smithi099 bash[87147]: cluster 2024-09-13T14:48:30.505654+0000 mgr.y (mgr.26252) 665 : cluster [DBG] pgmap v1133: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:48:31.964 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:31 smithi099 bash[87147]: cluster 2024-09-13T14:48:30.505654+0000 mgr.y (mgr.26252) 665 : cluster [DBG] pgmap v1133: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:48:31.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:48:32.031 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:32.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:32.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:32.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:31 smithi177 bash[68594]: cluster 2024-09-13T14:48:30.505654+0000 mgr.y (mgr.26252) 665 : cluster [DBG] pgmap v1133: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:48:32.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:31 smithi177 bash[68594]: cluster 2024-09-13T14:48:30.505654+0000 mgr.y (mgr.26252) 665 : cluster [DBG] pgmap v1133: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:48:32.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:32.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:32.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:32.333 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:32.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:32.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:32.465 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:32.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:32.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:32.537 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 50 16 65546 65530 81.9046 69.3125 0.0102737 0.0122038 2024-09-13T14:48:32.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:32.617 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:32.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:32.720 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:32.745 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:32.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:32.848 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:32.876 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:32.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:32.978 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:33.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:33.111 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:33.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:33.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:33.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:33.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:33.375 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:33.402 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:33.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:33.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:33.540 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 51 16 66627 66611 81.619 67.5625 0.0102311 0.0122467 2024-09-13T14:48:33.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:33.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:33.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:33.677 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:33.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:33.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:33.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:33 smithi099 bash[84576]: cluster 2024-09-13T14:48:32.506312+0000 mgr.y (mgr.26252) 666 : cluster [DBG] pgmap v1134: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:48:33.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:33 smithi099 bash[84576]: cluster 2024-09-13T14:48:32.506312+0000 mgr.y (mgr.26252) 666 : cluster [DBG] pgmap v1134: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:48:33.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:33 smithi099 bash[84576]: audit 2024-09-13T14:48:33.272076+0000 mon.a (mon.0) 1644 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:33.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:33 smithi099 bash[84576]: audit 2024-09-13T14:48:33.272076+0000 mon.a (mon.0) 1644 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:33.791 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:33 smithi099 bash[87147]: cluster 2024-09-13T14:48:32.506312+0000 mgr.y (mgr.26252) 666 : cluster [DBG] pgmap v1134: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:48:33.792 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:33 smithi099 bash[87147]: cluster 2024-09-13T14:48:32.506312+0000 mgr.y (mgr.26252) 666 : cluster [DBG] pgmap v1134: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:48:33.792 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:33 smithi099 bash[87147]: audit 2024-09-13T14:48:33.272076+0000 mon.a (mon.0) 1644 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:33.792 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:33 smithi099 bash[87147]: audit 2024-09-13T14:48:33.272076+0000 mon.a (mon.0) 1644 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:33.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:33.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:33.917 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:33.940 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:33.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:34.043 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:34.053 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:33 smithi177 bash[68594]: cluster 2024-09-13T14:48:32.506312+0000 mgr.y (mgr.26252) 666 : cluster [DBG] pgmap v1134: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:48:34.053 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:33 smithi177 bash[68594]: cluster 2024-09-13T14:48:32.506312+0000 mgr.y (mgr.26252) 666 : cluster [DBG] pgmap v1134: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:48:34.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:33 smithi177 bash[68594]: audit 2024-09-13T14:48:33.272076+0000 mon.a (mon.0) 1644 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:34.054 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:33 smithi177 bash[68594]: audit 2024-09-13T14:48:33.272076+0000 mon.a (mon.0) 1644 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:34.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:34.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:34.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:34.209 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:34.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:34.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:34.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:34.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:34.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:34.461 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:34.488 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:34.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:34.542 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 52 16 67926 67910 81.6068 81.1875 0.0159718 0.0122485 2024-09-13T14:48:34.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:34.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:34.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:34.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:34.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:34.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:34.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:34.896 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:34.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:34.999 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:35.023 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:35.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:35.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:35.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:35.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:35.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:35.283 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:35.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:35.410 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:35.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:35.513 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:35.540 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:35.542 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 53 16 69076 69060 81.4232 71.875 0.00691124 0.0122762 2024-09-13T14:48:35.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:35.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:35.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:35.782 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:35.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:35.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:35 smithi099 bash[84576]: cluster 2024-09-13T14:48:34.508760+0000 mgr.y (mgr.26252) 667 : cluster [DBG] pgmap v1135: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-13T14:48:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:35 smithi099 bash[84576]: cluster 2024-09-13T14:48:34.508760+0000 mgr.y (mgr.26252) 667 : cluster [DBG] pgmap v1135: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-13T14:48:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:35 smithi099 bash[87147]: cluster 2024-09-13T14:48:34.508760+0000 mgr.y (mgr.26252) 667 : cluster [DBG] pgmap v1135: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-13T14:48:35.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:35 smithi099 bash[87147]: cluster 2024-09-13T14:48:34.508760+0000 mgr.y (mgr.26252) 667 : cluster [DBG] pgmap v1135: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-13T14:48:35.920 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:35.944 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:35.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:36.046 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:36.058 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:35 smithi177 bash[68594]: cluster 2024-09-13T14:48:34.508760+0000 mgr.y (mgr.26252) 667 : cluster [DBG] pgmap v1135: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-13T14:48:36.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:35 smithi177 bash[68594]: cluster 2024-09-13T14:48:34.508760+0000 mgr.y (mgr.26252) 667 : cluster [DBG] pgmap v1135: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-09-13T14:48:36.082 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:36.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:36.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:36.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:36.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:36.322 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:36.346 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:36.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:36.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:36.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:36.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:36.542 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 54 16 70117 70101 81.1201 65.0625 0.0163127 0.0123224 2024-09-13T14:48:36.581 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:36.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:36.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:36.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:36.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:36.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:36.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:36.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:36.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:36.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:37.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:37.111 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:37.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:37.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:37.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.257 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:37.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:48:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:48:37] "GET /metrics HTTP/1.1" 200 36752 "" "Prometheus/2.51.0" 2024-09-13T14:48:37.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:37.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:37.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:37.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:37.542 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 55 16 71283 71267 80.9701 72.875 0.0133516 0.0123452 2024-09-13T14:48:37.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:37.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:37.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:37.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:37.883 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:37.905 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:37.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:38.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:37 smithi177 bash[68594]: cluster 2024-09-13T14:48:36.509152+0000 mgr.y (mgr.26252) 668 : cluster [DBG] pgmap v1136: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:48:38.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:37 smithi177 bash[68594]: cluster 2024-09-13T14:48:36.509152+0000 mgr.y (mgr.26252) 668 : cluster [DBG] pgmap v1136: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:48:38.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:38.148 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:38.181 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:38.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:38.296 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:37 smithi099 bash[84576]: cluster 2024-09-13T14:48:36.509152+0000 mgr.y (mgr.26252) 668 : cluster [DBG] pgmap v1136: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:48:38.297 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:37 smithi099 bash[84576]: cluster 2024-09-13T14:48:36.509152+0000 mgr.y (mgr.26252) 668 : cluster [DBG] pgmap v1136: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:48:38.297 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:37 smithi099 bash[87147]: cluster 2024-09-13T14:48:36.509152+0000 mgr.y (mgr.26252) 668 : cluster [DBG] pgmap v1136: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:48:38.297 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:37 smithi099 bash[87147]: cluster 2024-09-13T14:48:36.509152+0000 mgr.y (mgr.26252) 668 : cluster [DBG] pgmap v1136: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:48:38.309 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:38.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:38.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:38.440 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:38.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:38.542 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:38.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 56 16 72267 72251 80.6224 61.5 0.0146672 0.0123978 2024-09-13T14:48:38.565 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:38.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:38.591 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:48:38.591 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:38.630 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:48:38.630 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:38.669 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:48:38.669 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:38.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:38.698 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:38.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:38.723 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:48:38.723 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:38.745 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:48:38.745 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:38.771 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:48:38.771 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:38.803 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:48:38.803 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:38.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.813 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:38 smithi177 bash[68594]: audit 2024-09-13T14:48:38.551784+0000 mon.b (mon.1) 6 : audit [DBG] from='client.? 172.21.15.99:0/339226233' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:38.813 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:38 smithi177 bash[68594]: audit 2024-09-13T14:48:38.551784+0000 mon.b (mon.1) 6 : audit [DBG] from='client.? 172.21.15.99:0/339226233' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:38.829 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:48:38.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.829 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:38.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:38.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:38.966 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:38.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:39.070 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:39.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:38 smithi099 bash[84576]: audit 2024-09-13T14:48:38.551784+0000 mon.b (mon.1) 6 : audit [DBG] from='client.? 172.21.15.99:0/339226233' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:39.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:38 smithi099 bash[84576]: audit 2024-09-13T14:48:38.551784+0000 mon.b (mon.1) 6 : audit [DBG] from='client.? 172.21.15.99:0/339226233' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:38 smithi099 bash[87147]: audit 2024-09-13T14:48:38.551784+0000 mon.b (mon.1) 6 : audit [DBG] from='client.? 172.21.15.99:0/339226233' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:39.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:38 smithi099 bash[87147]: audit 2024-09-13T14:48:38.551784+0000 mon.b (mon.1) 6 : audit [DBG] from='client.? 172.21.15.99:0/339226233' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:39.106 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:39.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:39.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:39.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:39.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:39.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:39.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:39.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:39.494 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:39.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:39.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 57 16 73451 73435 80.506 74 0.0114355 0.012415 2024-09-13T14:48:39.600 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:39.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:39.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:39.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:39.740 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:39 smithi177 bash[68594]: cluster 2024-09-13T14:48:38.509770+0000 mgr.y (mgr.26252) 669 : cluster [DBG] pgmap v1137: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:48:39.740 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:39 smithi177 bash[68594]: cluster 2024-09-13T14:48:38.509770+0000 mgr.y (mgr.26252) 669 : cluster [DBG] pgmap v1137: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:48:39.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:39.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:39.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:39.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:39.929 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:39 smithi099 bash[84576]: cluster 2024-09-13T14:48:38.509770+0000 mgr.y (mgr.26252) 669 : cluster [DBG] pgmap v1137: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:48:39.930 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:39 smithi099 bash[84576]: cluster 2024-09-13T14:48:38.509770+0000 mgr.y (mgr.26252) 669 : cluster [DBG] pgmap v1137: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:48:39.930 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:39 smithi099 bash[87147]: cluster 2024-09-13T14:48:38.509770+0000 mgr.y (mgr.26252) 669 : cluster [DBG] pgmap v1137: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:48:39.930 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:39 smithi099 bash[87147]: cluster 2024-09-13T14:48:38.509770+0000 mgr.y (mgr.26252) 669 : cluster [DBG] pgmap v1137: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:48:39.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:40.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:40.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:40.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:40.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:40.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:40.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:48:40 smithi177 bash[61439]: ts=2024-09-13T14:48:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:40.392 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:40.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:40.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:40.522 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.542 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 58 16 74281 74265 80.0124 51.875 0.00327138 0.0124722 2024-09-13T14:48:40.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:40.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:40.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:40.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:40.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:40.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:40.944 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:41.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:41.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:41.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:41.174 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:41.194 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:41.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:41.297 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:41.321 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:41.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:41.423 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:41.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 59 16 74979 74963 79.3957 43.625 0.0204328 0.0125902 2024-09-13T14:48:41.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:41.592 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:41.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:41.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.728 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:41.834 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:41 smithi177 bash[68594]: cluster 2024-09-13T14:48:40.510650+0000 mgr.y (mgr.26252) 670 : cluster [DBG] pgmap v1138: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:48:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:41 smithi177 bash[68594]: cluster 2024-09-13T14:48:40.510650+0000 mgr.y (mgr.26252) 670 : cluster [DBG] pgmap v1138: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:48:41.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:41.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:41.964 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.985 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:42.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:42.130 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:42.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:42.234 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:42.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:42.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:42.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:41 smithi099 bash[84576]: cluster 2024-09-13T14:48:40.510650+0000 mgr.y (mgr.26252) 670 : cluster [DBG] pgmap v1138: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:48:42.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:41 smithi099 bash[84576]: cluster 2024-09-13T14:48:40.510650+0000 mgr.y (mgr.26252) 670 : cluster [DBG] pgmap v1138: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:48:42.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:41 smithi099 bash[87147]: cluster 2024-09-13T14:48:40.510650+0000 mgr.y (mgr.26252) 670 : cluster [DBG] pgmap v1138: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:48:42.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:41 smithi099 bash[87147]: cluster 2024-09-13T14:48:40.510650+0000 mgr.y (mgr.26252) 670 : cluster [DBG] pgmap v1138: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:48:42.364 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:42.394 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:42.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:42.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:42.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:42.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:42.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T14:48:42.546493+0000 min lat: 0.00109707 max lat: 1.73119 avg lat: 0.0126368 2024-09-13T14:48:42.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:48:42.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 60 16 75898 75882 79.0297 57.4375 0.0026085 0.0126368 2024-09-13T14:48:42.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:42.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:42.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:42.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:42.768 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:42.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:42.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:42.897 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:42.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:43.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:43.036 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:43.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:43.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:43.167 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:43.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:43.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:43.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:43.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:43.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:43.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:43.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:43.541 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:43.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 61 16 76926 76910 78.7873 64.25 0.00739422 0.0126871 2024-09-13T14:48:43.565 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:43.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:43.648 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:43.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:43.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:43.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:43.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:43.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:43 smithi099 bash[84576]: cluster 2024-09-13T14:48:42.511110+0000 mgr.y (mgr.26252) 671 : cluster [DBG] pgmap v1139: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:48:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:43 smithi099 bash[84576]: cluster 2024-09-13T14:48:42.511110+0000 mgr.y (mgr.26252) 671 : cluster [DBG] pgmap v1139: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:48:43.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:43 smithi099 bash[87147]: cluster 2024-09-13T14:48:42.511110+0000 mgr.y (mgr.26252) 671 : cluster [DBG] pgmap v1139: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:48:43.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:43 smithi099 bash[87147]: cluster 2024-09-13T14:48:42.511110+0000 mgr.y (mgr.26252) 671 : cluster [DBG] pgmap v1139: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:48:43.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:43.941 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:43.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:44.044 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:44.053 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:43 smithi177 bash[68594]: cluster 2024-09-13T14:48:42.511110+0000 mgr.y (mgr.26252) 671 : cluster [DBG] pgmap v1139: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:48:44.053 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:43 smithi177 bash[68594]: cluster 2024-09-13T14:48:42.511110+0000 mgr.y (mgr.26252) 671 : cluster [DBG] pgmap v1139: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T14:48:44.065 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:44.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:44.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:44.206 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:44.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:44.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:44.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.340 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:44.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:44.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:44.474 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:44.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:44.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 62 16 77733 77717 78.33 50.4375 0.0182646 0.0127615 2024-09-13T14:48:44.580 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:44.608 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:44.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:44.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:44.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:44.841 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:44.867 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:44.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:44.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:44.997 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:45.065 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:48:45.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:45.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:45.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:45.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:45.277 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:45.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:45.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:45.403 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:45.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:45.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:45.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:45.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:45.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 63 16 78543 78527 77.8902 50.625 0.00393757 0.0128165 2024-09-13T14:48:45.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:45.659 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:45.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:45.726 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T14:48:45.764 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:45.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:45.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:45 smithi177 bash[68594]: cluster 2024-09-13T14:48:44.511748+0000 mgr.y (mgr.26252) 672 : cluster [DBG] pgmap v1140: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:48:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:45 smithi177 bash[68594]: cluster 2024-09-13T14:48:44.511748+0000 mgr.y (mgr.26252) 672 : cluster [DBG] pgmap v1140: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:48:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:45 smithi177 bash[68594]: audit 2024-09-13T14:48:45.207094+0000 mon.a (mon.0) 1645 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:45.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:45 smithi177 bash[68594]: audit 2024-09-13T14:48:45.207094+0000 mon.a (mon.0) 1645 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:45.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:45.922 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:45.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:46.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:46.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:45 smithi099 bash[84576]: cluster 2024-09-13T14:48:44.511748+0000 mgr.y (mgr.26252) 672 : cluster [DBG] pgmap v1140: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:48:46.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:45 smithi099 bash[84576]: cluster 2024-09-13T14:48:44.511748+0000 mgr.y (mgr.26252) 672 : cluster [DBG] pgmap v1140: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:48:46.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:45 smithi099 bash[84576]: audit 2024-09-13T14:48:45.207094+0000 mon.a (mon.0) 1645 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:46.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:45 smithi099 bash[84576]: audit 2024-09-13T14:48:45.207094+0000 mon.a (mon.0) 1645 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:46.037 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:45 smithi099 bash[87147]: cluster 2024-09-13T14:48:44.511748+0000 mgr.y (mgr.26252) 672 : cluster [DBG] pgmap v1140: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:48:46.037 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:45 smithi099 bash[87147]: cluster 2024-09-13T14:48:44.511748+0000 mgr.y (mgr.26252) 672 : cluster [DBG] pgmap v1140: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:48:46.037 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:45 smithi099 bash[87147]: audit 2024-09-13T14:48:45.207094+0000 mon.a (mon.0) 1645 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:46.037 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:45 smithi099 bash[87147]: audit 2024-09-13T14:48:45.207094+0000 mon.a (mon.0) 1645 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:46.049 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:46.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:46.152 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:46.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:46.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:46.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:46.323 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:46.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:46.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:46.458 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:46.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:46.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 64 16 79599 79583 77.7043 66 0.00373348 0.0128641 2024-09-13T14:48:46.562 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:46.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:46.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:46.693 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:46.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:46.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:46.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:46.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:46.948 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:46.958 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:46 smithi177 bash[68594]: cluster 2024-09-13T14:48:45.727220+0000 mon.a (mon.0) 1646 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:46.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:46 smithi177 bash[68594]: cluster 2024-09-13T14:48:45.727220+0000 mon.a (mon.0) 1646 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:46.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:46 smithi177 bash[68594]: cluster 2024-09-13T14:48:45.727251+0000 mon.a (mon.0) 1647 : cluster [INF] Cluster is now healthy 2024-09-13T14:48:46.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:46 smithi177 bash[68594]: cluster 2024-09-13T14:48:45.727251+0000 mon.a (mon.0) 1647 : cluster [INF] Cluster is now healthy 2024-09-13T14:48:46.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:46 smithi177 bash[68594]: audit 2024-09-13T14:48:45.728814+0000 mon.a (mon.0) 1648 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:46.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:46 smithi177 bash[68594]: audit 2024-09-13T14:48:45.728814+0000 mon.a (mon.0) 1648 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:46.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:46 smithi177 bash[68594]: cluster 2024-09-13T14:48:45.734765+0000 mon.a (mon.0) 1649 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2024-09-13T14:48:46.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:46 smithi177 bash[68594]: cluster 2024-09-13T14:48:45.734765+0000 mon.a (mon.0) 1649 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2024-09-13T14:48:46.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:46.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:47.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[84576]: cluster 2024-09-13T14:48:45.727220+0000 mon.a (mon.0) 1646 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:47.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[84576]: cluster 2024-09-13T14:48:45.727220+0000 mon.a (mon.0) 1646 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:47.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[84576]: cluster 2024-09-13T14:48:45.727251+0000 mon.a (mon.0) 1647 : cluster [INF] Cluster is now healthy 2024-09-13T14:48:47.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[84576]: cluster 2024-09-13T14:48:45.727251+0000 mon.a (mon.0) 1647 : cluster [INF] Cluster is now healthy 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[84576]: audit 2024-09-13T14:48:45.728814+0000 mon.a (mon.0) 1648 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[84576]: audit 2024-09-13T14:48:45.728814+0000 mon.a (mon.0) 1648 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[84576]: cluster 2024-09-13T14:48:45.734765+0000 mon.a (mon.0) 1649 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[84576]: cluster 2024-09-13T14:48:45.734765+0000 mon.a (mon.0) 1649 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[87147]: cluster 2024-09-13T14:48:45.727220+0000 mon.a (mon.0) 1646 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[87147]: cluster 2024-09-13T14:48:45.727220+0000 mon.a (mon.0) 1646 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[87147]: cluster 2024-09-13T14:48:45.727251+0000 mon.a (mon.0) 1647 : cluster [INF] Cluster is now healthy 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[87147]: cluster 2024-09-13T14:48:45.727251+0000 mon.a (mon.0) 1647 : cluster [INF] Cluster is now healthy 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[87147]: audit 2024-09-13T14:48:45.728814+0000 mon.a (mon.0) 1648 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[87147]: audit 2024-09-13T14:48:45.728814+0000 mon.a (mon.0) 1648 : audit [INF] from='client.? 172.21.15.99:0/4030504649' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[87147]: cluster 2024-09-13T14:48:45.734765+0000 mon.a (mon.0) 1649 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2024-09-13T14:48:47.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:46 smithi099 bash[87147]: cluster 2024-09-13T14:48:45.734765+0000 mon.a (mon.0) 1649 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2024-09-13T14:48:47.082 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:47.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:47.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:47.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:47.218 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:48:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:48:47] "GET /metrics HTTP/1.1" 200 36752 "" "Prometheus/2.51.0" 2024-09-13T14:48:47.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:47.341 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:47.368 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:47.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:47.473 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:47.515 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:47.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:47.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 65 16 80596 80580 77.4674 62.3125 0.0125058 0.0129036 2024-09-13T14:48:47.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:47.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:47.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:47.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:47.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.785 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:47.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:47.886 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:47.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:47.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:48.014 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:48.023 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:47 smithi177 bash[68594]: cluster 2024-09-13T14:48:46.512134+0000 mgr.y (mgr.26252) 673 : cluster [DBG] pgmap v1142: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:48:48.024 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:47 smithi177 bash[68594]: cluster 2024-09-13T14:48:46.512134+0000 mgr.y (mgr.26252) 673 : cluster [DBG] pgmap v1142: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:48:48.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:48.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:47 smithi099 bash[84576]: cluster 2024-09-13T14:48:46.512134+0000 mgr.y (mgr.26252) 673 : cluster [DBG] pgmap v1142: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:48:48.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:47 smithi099 bash[84576]: cluster 2024-09-13T14:48:46.512134+0000 mgr.y (mgr.26252) 673 : cluster [DBG] pgmap v1142: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:48:48.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:47 smithi099 bash[87147]: cluster 2024-09-13T14:48:46.512134+0000 mgr.y (mgr.26252) 673 : cluster [DBG] pgmap v1142: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:48:48.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:47 smithi099 bash[87147]: cluster 2024-09-13T14:48:46.512134+0000 mgr.y (mgr.26252) 673 : cluster [DBG] pgmap v1142: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:48:48.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:48.173 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:48.278 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:48.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:48.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:48.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:48.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:48.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:48.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 66 16 81738 81722 77.375 71.375 0.0166755 0.0129183 2024-09-13T14:48:48.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:48.576 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:48.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:48.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:48.705 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:48.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:48.810 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:48.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:48.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:48.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:48.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:49.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:48 smithi099 bash[84576]: debug 2024-09-13T14:48:48.969+0000 7f0688bd2640 -1 mon.a@0(leader).osd e974 definitely_dead 0 2024-09-13T14:48:49.098 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:49.126 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:49.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:49.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:49.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:49.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:49.289 INFO:teuthology.orchestra.run.smithi099.stderr:marked out osd.1. 2024-09-13T14:48:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: audit 2024-09-13T14:48:48.277413+0000 mon.a (mon.0) 1650 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: audit 2024-09-13T14:48:48.277413+0000 mon.a (mon.0) 1650 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: audit 2024-09-13T14:48:48.278849+0000 mon.a (mon.0) 1651 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: audit 2024-09-13T14:48:48.278849+0000 mon.a (mon.0) 1651 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: cluster 2024-09-13T14:48:48.512687+0000 mgr.y (mgr.26252) 674 : cluster [DBG] pgmap v1143: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-13T14:48:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: cluster 2024-09-13T14:48:48.512687+0000 mgr.y (mgr.26252) 674 : cluster [DBG] pgmap v1143: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-13T14:48:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: audit 2024-09-13T14:48:48.970790+0000 mon.c (mon.2) 53 : audit [INF] from='client.? 172.21.15.99:0/817820358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: audit 2024-09-13T14:48:48.970790+0000 mon.c (mon.2) 53 : audit [INF] from='client.? 172.21.15.99:0/817820358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: audit 2024-09-13T14:48:48.971233+0000 mon.a (mon.0) 1652 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: audit 2024-09-13T14:48:48.971233+0000 mon.a (mon.0) 1652 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: cluster 2024-09-13T14:48:48.971295+0000 mon.a (mon.0) 1653 : cluster [INF] Client client.admin marked osd.1 out, while it was still marked up 2024-09-13T14:48:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:49 smithi177 bash[68594]: cluster 2024-09-13T14:48:48.971295+0000 mon.a (mon.0) 1653 : cluster [INF] Client client.admin marked osd.1 out, while it was still marked up 2024-09-13T14:48:49.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:49.382 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:49.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:49.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: audit 2024-09-13T14:48:48.277413+0000 mon.a (mon.0) 1650 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: audit 2024-09-13T14:48:48.277413+0000 mon.a (mon.0) 1650 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: audit 2024-09-13T14:48:48.278849+0000 mon.a (mon.0) 1651 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: audit 2024-09-13T14:48:48.278849+0000 mon.a (mon.0) 1651 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: cluster 2024-09-13T14:48:48.512687+0000 mgr.y (mgr.26252) 674 : cluster [DBG] pgmap v1143: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: cluster 2024-09-13T14:48:48.512687+0000 mgr.y (mgr.26252) 674 : cluster [DBG] pgmap v1143: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: audit 2024-09-13T14:48:48.970790+0000 mon.c (mon.2) 53 : audit [INF] from='client.? 172.21.15.99:0/817820358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: audit 2024-09-13T14:48:48.970790+0000 mon.c (mon.2) 53 : audit [INF] from='client.? 172.21.15.99:0/817820358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: audit 2024-09-13T14:48:48.971233+0000 mon.a (mon.0) 1652 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: audit 2024-09-13T14:48:48.971233+0000 mon.a (mon.0) 1652 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.495 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: cluster 2024-09-13T14:48:48.971295+0000 mon.a (mon.0) 1653 : cluster [INF] Client client.admin marked osd.1 out, while it was still marked up 2024-09-13T14:48:49.496 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[84576]: cluster 2024-09-13T14:48:48.971295+0000 mon.a (mon.0) 1653 : cluster [INF] Client client.admin marked osd.1 out, while it was still marked up 2024-09-13T14:48:49.496 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: audit 2024-09-13T14:48:48.277413+0000 mon.a (mon.0) 1650 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:49.496 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: audit 2024-09-13T14:48:48.277413+0000 mon.a (mon.0) 1650 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:48:49.496 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: audit 2024-09-13T14:48:48.278849+0000 mon.a (mon.0) 1651 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:49.496 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: audit 2024-09-13T14:48:48.278849+0000 mon.a (mon.0) 1651 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:49.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: cluster 2024-09-13T14:48:48.512687+0000 mgr.y (mgr.26252) 674 : cluster [DBG] pgmap v1143: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-13T14:48:49.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: cluster 2024-09-13T14:48:48.512687+0000 mgr.y (mgr.26252) 674 : cluster [DBG] pgmap v1143: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-13T14:48:49.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: audit 2024-09-13T14:48:48.970790+0000 mon.c (mon.2) 53 : audit [INF] from='client.? 172.21.15.99:0/817820358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: audit 2024-09-13T14:48:48.970790+0000 mon.c (mon.2) 53 : audit [INF] from='client.? 172.21.15.99:0/817820358' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: audit 2024-09-13T14:48:48.971233+0000 mon.a (mon.0) 1652 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: audit 2024-09-13T14:48:48.971233+0000 mon.a (mon.0) 1652 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T14:48:49.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: cluster 2024-09-13T14:48:48.971295+0000 mon.a (mon.0) 1653 : cluster [INF] Client client.admin marked osd.1 out, while it was still marked up 2024-09-13T14:48:49.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:49 smithi099 bash[87147]: cluster 2024-09-13T14:48:48.971295+0000 mon.a (mon.0) 1653 : cluster [INF] Client client.admin marked osd.1 out, while it was still marked up 2024-09-13T14:48:49.508 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:49.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:49.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 67 16 82523 82507 76.9524 49.0625 0.00310458 0.01294 2024-09-13T14:48:49.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:49.633 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:49.736 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:49.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:49.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:49.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:49.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:49.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:49.991 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:50.022 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:50.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:50.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:50.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:50.302 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:48:50 smithi177 bash[61439]: ts=2024-09-13T14:48:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:48:50.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.393 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:50 smithi099 bash[84576]: audit 2024-09-13T14:48:49.282145+0000 mon.a (mon.0) 1654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-13T14:48:50.393 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:50 smithi099 bash[84576]: audit 2024-09-13T14:48:49.282145+0000 mon.a (mon.0) 1654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-13T14:48:50.393 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:50 smithi099 bash[84576]: cluster 2024-09-13T14:48:49.287369+0000 mon.a (mon.0) 1655 : cluster [DBG] osdmap e975: 8 total, 8 up, 7 in 2024-09-13T14:48:50.393 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:50 smithi099 bash[84576]: cluster 2024-09-13T14:48:49.287369+0000 mon.a (mon.0) 1655 : cluster [DBG] osdmap e975: 8 total, 8 up, 7 in 2024-09-13T14:48:50.393 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:50 smithi099 bash[87147]: audit 2024-09-13T14:48:49.282145+0000 mon.a (mon.0) 1654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-13T14:48:50.393 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:50 smithi099 bash[87147]: audit 2024-09-13T14:48:49.282145+0000 mon.a (mon.0) 1654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-13T14:48:50.393 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:50 smithi099 bash[87147]: cluster 2024-09-13T14:48:49.287369+0000 mon.a (mon.0) 1655 : cluster [DBG] osdmap e975: 8 total, 8 up, 7 in 2024-09-13T14:48:50.393 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:50 smithi099 bash[87147]: cluster 2024-09-13T14:48:49.287369+0000 mon.a (mon.0) 1655 : cluster [DBG] osdmap e975: 8 total, 8 up, 7 in 2024-09-13T14:48:50.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:50.510 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:50.532 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:50.543 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 68 16 82523 82507 75.8208 0 - 0.01294 2024-09-13T14:48:50.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:48:50.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:50 smithi177 bash[68594]: audit 2024-09-13T14:48:49.282145+0000 mon.a (mon.0) 1654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-13T14:48:50.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:50 smithi177 bash[68594]: audit 2024-09-13T14:48:49.282145+0000 mon.a (mon.0) 1654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-13T14:48:50.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:50 smithi177 bash[68594]: cluster 2024-09-13T14:48:49.287369+0000 mon.a (mon.0) 1655 : cluster [DBG] osdmap e975: 8 total, 8 up, 7 in 2024-09-13T14:48:50.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:50 smithi177 bash[68594]: cluster 2024-09-13T14:48:49.287369+0000 mon.a (mon.0) 1655 : cluster [DBG] osdmap e975: 8 total, 8 up, 7 in 2024-09-13T14:48:50.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.678 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:50.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:50.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:50.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:50.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:50.911 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:50.934 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:50.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:51.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:51.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:51.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:51.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:51.192 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:51.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:51.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:51.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:51.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:51.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:51.462 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:51.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:51.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 69 16 82528 82512 74.7266 0.15625 2.28896 0.0130491 2024-09-13T14:48:51.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:51.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:51 smithi099 bash[84576]: cluster 2024-09-13T14:48:50.289710+0000 mon.a (mon.0) 1656 : cluster [DBG] osdmap e976: 8 total, 8 up, 7 in 2024-09-13T14:48:51.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:51 smithi099 bash[84576]: cluster 2024-09-13T14:48:50.289710+0000 mon.a (mon.0) 1656 : cluster [DBG] osdmap e976: 8 total, 8 up, 7 in 2024-09-13T14:48:51.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:51 smithi099 bash[84576]: cluster 2024-09-13T14:48:50.513397+0000 mgr.y (mgr.26252) 675 : cluster [DBG] pgmap v1146: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2024-09-13T14:48:51.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:51 smithi099 bash[84576]: cluster 2024-09-13T14:48:50.513397+0000 mgr.y (mgr.26252) 675 : cluster [DBG] pgmap v1146: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2024-09-13T14:48:51.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:51 smithi099 bash[87147]: cluster 2024-09-13T14:48:50.289710+0000 mon.a (mon.0) 1656 : cluster [DBG] osdmap e976: 8 total, 8 up, 7 in 2024-09-13T14:48:51.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:51 smithi099 bash[87147]: cluster 2024-09-13T14:48:50.289710+0000 mon.a (mon.0) 1656 : cluster [DBG] osdmap e976: 8 total, 8 up, 7 in 2024-09-13T14:48:51.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:51 smithi099 bash[87147]: cluster 2024-09-13T14:48:50.513397+0000 mgr.y (mgr.26252) 675 : cluster [DBG] pgmap v1146: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2024-09-13T14:48:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:51 smithi099 bash[87147]: cluster 2024-09-13T14:48:50.513397+0000 mgr.y (mgr.26252) 675 : cluster [DBG] pgmap v1146: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2024-09-13T14:48:51.588 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:51.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:51 smithi177 bash[68594]: cluster 2024-09-13T14:48:50.289710+0000 mon.a (mon.0) 1656 : cluster [DBG] osdmap e976: 8 total, 8 up, 7 in 2024-09-13T14:48:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:51 smithi177 bash[68594]: cluster 2024-09-13T14:48:50.289710+0000 mon.a (mon.0) 1656 : cluster [DBG] osdmap e976: 8 total, 8 up, 7 in 2024-09-13T14:48:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:51 smithi177 bash[68594]: cluster 2024-09-13T14:48:50.513397+0000 mgr.y (mgr.26252) 675 : cluster [DBG] pgmap v1146: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2024-09-13T14:48:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:51 smithi177 bash[68594]: cluster 2024-09-13T14:48:50.513397+0000 mgr.y (mgr.26252) 675 : cluster [DBG] pgmap v1146: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2024-09-13T14:48:51.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:51.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:51.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:51.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:51.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:51.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:51.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:51.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:52.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:52.114 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:52.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:52.218 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:52.244 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:52.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:52 smithi177 bash[68594]: cluster 2024-09-13T14:48:51.321116+0000 mon.a (mon.0) 1657 : cluster [DBG] osdmap e977: 8 total, 8 up, 7 in 2024-09-13T14:48:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:52 smithi177 bash[68594]: cluster 2024-09-13T14:48:51.321116+0000 mon.a (mon.0) 1657 : cluster [DBG] osdmap e977: 8 total, 8 up, 7 in 2024-09-13T14:48:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:52 smithi177 bash[68594]: audit 2024-09-13T14:48:51.622105+0000 mon.a (mon.0) 1658 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]} : dispatch 2024-09-13T14:48:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:52 smithi177 bash[68594]: audit 2024-09-13T14:48:51.622105+0000 mon.a (mon.0) 1658 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]} : dispatch 2024-09-13T14:48:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:52 smithi177 bash[68594]: audit 2024-09-13T14:48:51.622364+0000 mon.a (mon.0) 1659 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]} : dispatch 2024-09-13T14:48:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:52 smithi177 bash[68594]: audit 2024-09-13T14:48:51.622364+0000 mon.a (mon.0) 1659 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]} : dispatch 2024-09-13T14:48:52.388 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:52.491 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:52.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:52.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:52.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 70 16 83213 83197 74.2706 42.8125 0.00629281 0.0134577 2024-09-13T14:48:52.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[84576]: cluster 2024-09-13T14:48:51.321116+0000 mon.a (mon.0) 1657 : cluster [DBG] osdmap e977: 8 total, 8 up, 7 in 2024-09-13T14:48:52.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[84576]: cluster 2024-09-13T14:48:51.321116+0000 mon.a (mon.0) 1657 : cluster [DBG] osdmap e977: 8 total, 8 up, 7 in 2024-09-13T14:48:52.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[84576]: audit 2024-09-13T14:48:51.622105+0000 mon.a (mon.0) 1658 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]} : dispatch 2024-09-13T14:48:52.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[84576]: audit 2024-09-13T14:48:51.622105+0000 mon.a (mon.0) 1658 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]} : dispatch 2024-09-13T14:48:52.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[84576]: audit 2024-09-13T14:48:51.622364+0000 mon.a (mon.0) 1659 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]} : dispatch 2024-09-13T14:48:52.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[84576]: audit 2024-09-13T14:48:51.622364+0000 mon.a (mon.0) 1659 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]} : dispatch 2024-09-13T14:48:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[87147]: cluster 2024-09-13T14:48:51.321116+0000 mon.a (mon.0) 1657 : cluster [DBG] osdmap e977: 8 total, 8 up, 7 in 2024-09-13T14:48:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[87147]: cluster 2024-09-13T14:48:51.321116+0000 mon.a (mon.0) 1657 : cluster [DBG] osdmap e977: 8 total, 8 up, 7 in 2024-09-13T14:48:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[87147]: audit 2024-09-13T14:48:51.622105+0000 mon.a (mon.0) 1658 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]} : dispatch 2024-09-13T14:48:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[87147]: audit 2024-09-13T14:48:51.622105+0000 mon.a (mon.0) 1658 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]} : dispatch 2024-09-13T14:48:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[87147]: audit 2024-09-13T14:48:51.622364+0000 mon.a (mon.0) 1659 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]} : dispatch 2024-09-13T14:48:52.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:52 smithi099 bash[87147]: audit 2024-09-13T14:48:51.622364+0000 mon.a (mon.0) 1659 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]} : dispatch 2024-09-13T14:48:52.624 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:52.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:52.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:52.758 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:52.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:52.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:52.938 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:52.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:53.042 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:53.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:53.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:53.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:53.302 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:53.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:53.456 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:53.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:53.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 71 16 83982 83966 73.9015 48.0625 0.00734852 0.0135254 2024-09-13T14:48:53.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:53 smithi177 bash[68594]: audit 2024-09-13T14:48:52.314784+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-09-13T14:48:53.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:53 smithi177 bash[68594]: audit 2024-09-13T14:48:52.314784+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-09-13T14:48:53.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:53 smithi177 bash[68594]: audit 2024-09-13T14:48:52.314845+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2024-09-13T14:48:53.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:53 smithi177 bash[68594]: audit 2024-09-13T14:48:52.314845+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2024-09-13T14:48:53.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:53 smithi177 bash[68594]: cluster 2024-09-13T14:48:52.318310+0000 mon.a (mon.0) 1662 : cluster [DBG] osdmap e978: 8 total, 8 up, 7 in 2024-09-13T14:48:53.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:53 smithi177 bash[68594]: cluster 2024-09-13T14:48:52.318310+0000 mon.a (mon.0) 1662 : cluster [DBG] osdmap e978: 8 total, 8 up, 7 in 2024-09-13T14:48:53.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:53 smithi177 bash[68594]: cluster 2024-09-13T14:48:52.513842+0000 mgr.y (mgr.26252) 676 : cluster [DBG] pgmap v1149: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 43 MiB/s wr, 680 op/s; 5148/246141 objects degraded (2.091%); 4890/246141 objects misplaced (1.987%); 496 KiB/s, 7 objects/s recovering 2024-09-13T14:48:53.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:53 smithi177 bash[68594]: cluster 2024-09-13T14:48:52.513842+0000 mgr.y (mgr.26252) 676 : cluster [DBG] pgmap v1149: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 43 MiB/s wr, 680 op/s; 5148/246141 objects degraded (2.091%); 4890/246141 objects misplaced (1.987%); 496 KiB/s, 7 objects/s recovering 2024-09-13T14:48:53.587 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:53.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:53.704 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[84576]: audit 2024-09-13T14:48:52.314784+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-09-13T14:48:53.704 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[84576]: audit 2024-09-13T14:48:52.314784+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-09-13T14:48:53.705 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[84576]: audit 2024-09-13T14:48:52.314845+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2024-09-13T14:48:53.705 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[84576]: audit 2024-09-13T14:48:52.314845+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2024-09-13T14:48:53.705 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[84576]: cluster 2024-09-13T14:48:52.318310+0000 mon.a (mon.0) 1662 : cluster [DBG] osdmap e978: 8 total, 8 up, 7 in 2024-09-13T14:48:53.705 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[84576]: cluster 2024-09-13T14:48:52.318310+0000 mon.a (mon.0) 1662 : cluster [DBG] osdmap e978: 8 total, 8 up, 7 in 2024-09-13T14:48:53.705 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[84576]: cluster 2024-09-13T14:48:52.513842+0000 mgr.y (mgr.26252) 676 : cluster [DBG] pgmap v1149: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 43 MiB/s wr, 680 op/s; 5148/246141 objects degraded (2.091%); 4890/246141 objects misplaced (1.987%); 496 KiB/s, 7 objects/s recovering 2024-09-13T14:48:53.705 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[84576]: cluster 2024-09-13T14:48:52.513842+0000 mgr.y (mgr.26252) 676 : cluster [DBG] pgmap v1149: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 43 MiB/s wr, 680 op/s; 5148/246141 objects degraded (2.091%); 4890/246141 objects misplaced (1.987%); 496 KiB/s, 7 objects/s recovering 2024-09-13T14:48:53.705 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[87147]: audit 2024-09-13T14:48:52.314784+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-09-13T14:48:53.705 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[87147]: audit 2024-09-13T14:48:52.314784+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 5]}]': finished 2024-09-13T14:48:53.705 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[87147]: audit 2024-09-13T14:48:52.314845+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2024-09-13T14:48:53.705 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[87147]: audit 2024-09-13T14:48:52.314845+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2024-09-13T14:48:53.706 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[87147]: cluster 2024-09-13T14:48:52.318310+0000 mon.a (mon.0) 1662 : cluster [DBG] osdmap e978: 8 total, 8 up, 7 in 2024-09-13T14:48:53.706 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[87147]: cluster 2024-09-13T14:48:52.318310+0000 mon.a (mon.0) 1662 : cluster [DBG] osdmap e978: 8 total, 8 up, 7 in 2024-09-13T14:48:53.706 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[87147]: cluster 2024-09-13T14:48:52.513842+0000 mgr.y (mgr.26252) 676 : cluster [DBG] pgmap v1149: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 43 MiB/s wr, 680 op/s; 5148/246141 objects degraded (2.091%); 4890/246141 objects misplaced (1.987%); 496 KiB/s, 7 objects/s recovering 2024-09-13T14:48:53.706 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:53 smithi099 bash[87147]: cluster 2024-09-13T14:48:52.513842+0000 mgr.y (mgr.26252) 676 : cluster [DBG] pgmap v1149: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 43 MiB/s wr, 680 op/s; 5148/246141 objects degraded (2.091%); 4890/246141 objects misplaced (1.987%); 496 KiB/s, 7 objects/s recovering 2024-09-13T14:48:53.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:53.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:53.822 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:53.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:53.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:53.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:53.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:54.085 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:54.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:54.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:54.212 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:54.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.242 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:54.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:54.278 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:54.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:54.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:54 smithi177 bash[68594]: cluster 2024-09-13T14:48:53.320805+0000 mon.a (mon.0) 1663 : cluster [WRN] Health check failed: Degraded data redundancy: 5148/246141 objects degraded (2.091%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:54.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:54 smithi177 bash[68594]: cluster 2024-09-13T14:48:53.320805+0000 mon.a (mon.0) 1663 : cluster [WRN] Health check failed: Degraded data redundancy: 5148/246141 objects degraded (2.091%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:54.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:54 smithi177 bash[68594]: cluster 2024-09-13T14:48:53.342905+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e979: 8 total, 8 up, 7 in 2024-09-13T14:48:54.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:54 smithi177 bash[68594]: cluster 2024-09-13T14:48:53.342905+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e979: 8 total, 8 up, 7 in 2024-09-13T14:48:54.376 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:54.482 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:54.504 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:54.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:54.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 72 16 84812 84796 73.5955 51.875 0.0409322 0.0135821 2024-09-13T14:48:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:54 smithi099 bash[84576]: cluster 2024-09-13T14:48:53.320805+0000 mon.a (mon.0) 1663 : cluster [WRN] Health check failed: Degraded data redundancy: 5148/246141 objects degraded (2.091%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:54.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:54 smithi099 bash[84576]: cluster 2024-09-13T14:48:53.320805+0000 mon.a (mon.0) 1663 : cluster [WRN] Health check failed: Degraded data redundancy: 5148/246141 objects degraded (2.091%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:54 smithi099 bash[84576]: cluster 2024-09-13T14:48:53.342905+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e979: 8 total, 8 up, 7 in 2024-09-13T14:48:54.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:54 smithi099 bash[84576]: cluster 2024-09-13T14:48:53.342905+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e979: 8 total, 8 up, 7 in 2024-09-13T14:48:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:54 smithi099 bash[87147]: cluster 2024-09-13T14:48:53.320805+0000 mon.a (mon.0) 1663 : cluster [WRN] Health check failed: Degraded data redundancy: 5148/246141 objects degraded (2.091%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:54 smithi099 bash[87147]: cluster 2024-09-13T14:48:53.320805+0000 mon.a (mon.0) 1663 : cluster [WRN] Health check failed: Degraded data redundancy: 5148/246141 objects degraded (2.091%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:54 smithi099 bash[87147]: cluster 2024-09-13T14:48:53.342905+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e979: 8 total, 8 up, 7 in 2024-09-13T14:48:54.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:54 smithi099 bash[87147]: cluster 2024-09-13T14:48:53.342905+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e979: 8 total, 8 up, 7 in 2024-09-13T14:48:54.608 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:54.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:54.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:54.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:54.759 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:54.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:54.896 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:54.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:55.001 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:55.030 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:55.134 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:55.169 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:55.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:55.304 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:55.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:55.407 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:55.429 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:55.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:55.531 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:55.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 73 16 85538 85522 73.2089 45.375 0.0119421 0.0136329 2024-09-13T14:48:55.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:55 smithi099 bash[84576]: cluster 2024-09-13T14:48:54.514723+0000 mgr.y (mgr.26252) 677 : cluster [DBG] pgmap v1151: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 565 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 954 KiB/s, 15 objects/s recovering 2024-09-13T14:48:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:55 smithi099 bash[84576]: cluster 2024-09-13T14:48:54.514723+0000 mgr.y (mgr.26252) 677 : cluster [DBG] pgmap v1151: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 565 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 954 KiB/s, 15 objects/s recovering 2024-09-13T14:48:55.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:55 smithi099 bash[87147]: cluster 2024-09-13T14:48:54.514723+0000 mgr.y (mgr.26252) 677 : cluster [DBG] pgmap v1151: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 565 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 954 KiB/s, 15 objects/s recovering 2024-09-13T14:48:55.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:55 smithi099 bash[87147]: cluster 2024-09-13T14:48:54.514723+0000 mgr.y (mgr.26252) 677 : cluster [DBG] pgmap v1151: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 565 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 954 KiB/s, 15 objects/s recovering 2024-09-13T14:48:55.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:55 smithi177 bash[68594]: cluster 2024-09-13T14:48:54.514723+0000 mgr.y (mgr.26252) 677 : cluster [DBG] pgmap v1151: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 565 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 954 KiB/s, 15 objects/s recovering 2024-09-13T14:48:55.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:55 smithi177 bash[68594]: cluster 2024-09-13T14:48:54.514723+0000 mgr.y (mgr.26252) 677 : cluster [DBG] pgmap v1151: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 565 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 954 KiB/s, 15 objects/s recovering 2024-09-13T14:48:55.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:55.701 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:55.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:55.804 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:55.831 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:55.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:55.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:55.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:56.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:56.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:56.190 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:56.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:56.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:56.317 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:56.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:56.362 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:56.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:56.466 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:56.506 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:56.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:56.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 74 16 86230 86214 72.804 43.25 0.0170424 0.0137292 2024-09-13T14:48:56.612 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:56.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:56.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:56.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:56.761 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:48:56.762 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:48:56.762 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-13T14:48:56.762 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:48:56.762 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:48:56.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-13T14:48:56.774 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:56.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:56.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:56.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:57.021 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:57.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:57.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:57.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:57.167 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:57.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:57.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:57.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:57.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:48:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:48:57] "GET /metrics HTTP/1.1" 200 36748 "" "Prometheus/2.51.0" 2024-09-13T14:48:57.390 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:48:57.390 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T14:48:57.390 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T14:48:57.390 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T14:48:57.390 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:48:57.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:57.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:57.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:57.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:57.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 75 16 86937 86921 72.4224 44.1875 0.016102 0.0138017 2024-09-13T14:48:57.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:57.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:57.664 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:57.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:57.796 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:57.823 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:57.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:57 smithi177 bash[68594]: cluster 2024-09-13T14:48:56.515174+0000 mgr.y (mgr.26252) 678 : cluster [DBG] pgmap v1152: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 25 MiB/s wr, 398 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 672 KiB/s, 11 objects/s recovering 2024-09-13T14:48:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:57 smithi177 bash[68594]: cluster 2024-09-13T14:48:56.515174+0000 mgr.y (mgr.26252) 678 : cluster [DBG] pgmap v1152: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 25 MiB/s wr, 398 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 672 KiB/s, 11 objects/s recovering 2024-09-13T14:48:57.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:57.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:57.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:57.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:57 smithi099 bash[84576]: cluster 2024-09-13T14:48:56.515174+0000 mgr.y (mgr.26252) 678 : cluster [DBG] pgmap v1152: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 25 MiB/s wr, 398 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 672 KiB/s, 11 objects/s recovering 2024-09-13T14:48:57.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:57 smithi099 bash[84576]: cluster 2024-09-13T14:48:56.515174+0000 mgr.y (mgr.26252) 678 : cluster [DBG] pgmap v1152: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 25 MiB/s wr, 398 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 672 KiB/s, 11 objects/s recovering 2024-09-13T14:48:57.961 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:57 smithi099 bash[87147]: cluster 2024-09-13T14:48:56.515174+0000 mgr.y (mgr.26252) 678 : cluster [DBG] pgmap v1152: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 25 MiB/s wr, 398 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 672 KiB/s, 11 objects/s recovering 2024-09-13T14:48:57.961 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:57 smithi099 bash[87147]: cluster 2024-09-13T14:48:56.515174+0000 mgr.y (mgr.26252) 678 : cluster [DBG] pgmap v1152: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 25 MiB/s wr, 398 op/s; 15335/250719 objects degraded (6.116%); 10242/250719 objects misplaced (4.085%); 672 KiB/s, 11 objects/s recovering 2024-09-13T14:48:58.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:58.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.082 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:58.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:58.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:58.206 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:58.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:58.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:58.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:58.485 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:58.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:58.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 76 16 87642 87626 72.0492 44.0625 0.0151905 0.0138733 2024-09-13T14:48:58.589 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:58.615 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:58.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:58.757 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:58.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:58 smithi177 bash[68594]: cluster 2024-09-13T14:48:58.572826+0000 mon.a (mon.0) 1665 : cluster [WRN] Health check update: Degraded data redundancy: 15335/258312 objects degraded (5.937%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:48:58.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:58 smithi177 bash[68594]: cluster 2024-09-13T14:48:58.572826+0000 mon.a (mon.0) 1665 : cluster [WRN] Health check update: Degraded data redundancy: 15335/258312 objects degraded (5.937%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:48:58.863 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:58.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:58.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:59.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:59.009 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:58 smithi099 bash[84576]: cluster 2024-09-13T14:48:58.572826+0000 mon.a (mon.0) 1665 : cluster [WRN] Health check update: Degraded data redundancy: 15335/258312 objects degraded (5.937%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:48:59.009 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:58 smithi099 bash[84576]: cluster 2024-09-13T14:48:58.572826+0000 mon.a (mon.0) 1665 : cluster [WRN] Health check update: Degraded data redundancy: 15335/258312 objects degraded (5.937%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:48:59.009 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:58 smithi099 bash[87147]: cluster 2024-09-13T14:48:58.572826+0000 mon.a (mon.0) 1665 : cluster [WRN] Health check update: Degraded data redundancy: 15335/258312 objects degraded (5.937%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:48:59.009 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:58 smithi099 bash[87147]: cluster 2024-09-13T14:48:58.572826+0000 mon.a (mon.0) 1665 : cluster [WRN] Health check update: Degraded data redundancy: 15335/258312 objects degraded (5.937%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:48:59.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:59.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:59.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:59.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:59.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:59.290 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:59.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:59.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:59.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:59.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:59.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:59.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 77 16 88472 88456 71.7872 51.875 0.00796635 0.0139235 2024-09-13T14:48:59.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:59.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:59.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:59.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:59.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:59.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:59.825 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:59.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:59 smithi177 bash[68594]: cluster 2024-09-13T14:48:58.515872+0000 mgr.y (mgr.26252) 679 : cluster [DBG] pgmap v1153: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 15335/258312 objects degraded (5.937%); 9986/258312 objects misplaced (3.866%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:48:59.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:48:59 smithi177 bash[68594]: cluster 2024-09-13T14:48:58.515872+0000 mgr.y (mgr.26252) 679 : cluster [DBG] pgmap v1153: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 15335/258312 objects degraded (5.937%); 9986/258312 objects misplaced (3.866%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:48:59.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:59.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:59.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:59.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:59.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:00.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:59 smithi099 bash[84576]: cluster 2024-09-13T14:48:58.515872+0000 mgr.y (mgr.26252) 679 : cluster [DBG] pgmap v1153: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 15335/258312 objects degraded (5.937%); 9986/258312 objects misplaced (3.866%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:00.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:48:59 smithi099 bash[84576]: cluster 2024-09-13T14:48:58.515872+0000 mgr.y (mgr.26252) 679 : cluster [DBG] pgmap v1153: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 15335/258312 objects degraded (5.937%); 9986/258312 objects misplaced (3.866%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:00.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:59 smithi099 bash[87147]: cluster 2024-09-13T14:48:58.515872+0000 mgr.y (mgr.26252) 679 : cluster [DBG] pgmap v1153: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 15335/258312 objects degraded (5.937%); 9986/258312 objects misplaced (3.866%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:00.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:48:59 smithi099 bash[87147]: cluster 2024-09-13T14:48:58.515872+0000 mgr.y (mgr.26252) 679 : cluster [DBG] pgmap v1153: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 683 op/s; 15335/258312 objects degraded (5.937%); 9986/258312 objects misplaced (3.866%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:00.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:00.113 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:00.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:00.217 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:00.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:00.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:00.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:00.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:00.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:00.476 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:00.484 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:49:00 smithi177 bash[61439]: ts=2024-09-13T14:49:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:00.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:00.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:00.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 78 16 89061 89045 71.3388 36.8125 0.0234945 0.0140122 2024-09-13T14:49:00.610 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:00.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:00.746 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:00.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:00.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:00.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:00.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:00.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:01.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:01.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:01.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:01.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:01.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:01.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:01.278 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:01.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:01.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:01.412 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:01.436 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:01.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:01.541 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:01.544 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 79 15 89841 89826 71.0536 48.8125 0.0229544 0.0140678 2024-09-13T14:49:01.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:01.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:01.598 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:49:01.678 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:01.705 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:01.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:01.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:01.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:01.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:01.873 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:01 smithi099 bash[84576]: cluster 2024-09-13T14:49:00.516510+0000 mgr.y (mgr.26252) 680 : cluster [DBG] pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 15335/261792 objects degraded (5.858%); 9986/261792 objects misplaced (3.814%); 2.4 MiB/s, 39 objects/s recovering 2024-09-13T14:49:01.873 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:01 smithi099 bash[84576]: cluster 2024-09-13T14:49:00.516510+0000 mgr.y (mgr.26252) 680 : cluster [DBG] pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 15335/261792 objects degraded (5.858%); 9986/261792 objects misplaced (3.814%); 2.4 MiB/s, 39 objects/s recovering 2024-09-13T14:49:01.873 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:01 smithi099 bash[87147]: cluster 2024-09-13T14:49:00.516510+0000 mgr.y (mgr.26252) 680 : cluster [DBG] pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 15335/261792 objects degraded (5.858%); 9986/261792 objects misplaced (3.814%); 2.4 MiB/s, 39 objects/s recovering 2024-09-13T14:49:01.873 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:01 smithi099 bash[87147]: cluster 2024-09-13T14:49:00.516510+0000 mgr.y (mgr.26252) 680 : cluster [DBG] pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 15335/261792 objects degraded (5.858%); 9986/261792 objects misplaced (3.814%); 2.4 MiB/s, 39 objects/s recovering 2024-09-13T14:49:01.941 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:01.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:01.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:02.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:02.104 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:02.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:02.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:01 smithi177 bash[68594]: cluster 2024-09-13T14:49:00.516510+0000 mgr.y (mgr.26252) 680 : cluster [DBG] pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 15335/261792 objects degraded (5.858%); 9986/261792 objects misplaced (3.814%); 2.4 MiB/s, 39 objects/s recovering 2024-09-13T14:49:02.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:01 smithi177 bash[68594]: cluster 2024-09-13T14:49:00.516510+0000 mgr.y (mgr.26252) 680 : cluster [DBG] pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 42 MiB/s wr, 676 op/s; 15335/261792 objects degraded (5.858%); 9986/261792 objects misplaced (3.814%); 2.4 MiB/s, 39 objects/s recovering 2024-09-13T14:49:02.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:02.227 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:02.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:02.330 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:02.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:02.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:02.469 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:02.500 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:02.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:02.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T14:49:02.548429+0000 min lat: 0.00109707 max lat: 2.50527 avg lat: 0.014126 2024-09-13T14:49:02.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:49:02.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 80 16 90601 90585 70.7584 47.4375 0.0354418 0.014126 2024-09-13T14:49:02.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:02.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:02.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:02.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:02.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:02.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:02.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:02.901 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:03.005 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:03.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:03.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:03.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:03.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:03.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:03.310 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:03.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:03.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:49:03.415 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:03.482 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:03.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:03.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 81 16 91348 91332 70.4612 46.6875 0.0231921 0.0141842 2024-09-13T14:49:03.585 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:03.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:03.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:03.710 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:03.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[87147]: cluster 2024-09-13T14:49:02.517108+0000 mgr.y (mgr.26252) 681 : cluster [DBG] pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 15335/265275 objects degraded (5.781%); 9860/265275 objects misplaced (3.717%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:49:03.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[87147]: cluster 2024-09-13T14:49:02.517108+0000 mgr.y (mgr.26252) 681 : cluster [DBG] pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 15335/265275 objects degraded (5.781%); 9860/265275 objects misplaced (3.717%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:49:03.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[87147]: audit 2024-09-13T14:49:03.277098+0000 mon.a (mon.0) 1666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:03.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[87147]: audit 2024-09-13T14:49:03.277098+0000 mon.a (mon.0) 1666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:03.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[87147]: audit 2024-09-13T14:49:03.278526+0000 mon.a (mon.0) 1667 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:03.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[87147]: audit 2024-09-13T14:49:03.278526+0000 mon.a (mon.0) 1667 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:03.718 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[84576]: cluster 2024-09-13T14:49:02.517108+0000 mgr.y (mgr.26252) 681 : cluster [DBG] pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 15335/265275 objects degraded (5.781%); 9860/265275 objects misplaced (3.717%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:49:03.718 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[84576]: cluster 2024-09-13T14:49:02.517108+0000 mgr.y (mgr.26252) 681 : cluster [DBG] pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 15335/265275 objects degraded (5.781%); 9860/265275 objects misplaced (3.717%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:49:03.719 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[84576]: audit 2024-09-13T14:49:03.277098+0000 mon.a (mon.0) 1666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:03.719 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[84576]: audit 2024-09-13T14:49:03.277098+0000 mon.a (mon.0) 1666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:03.719 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[84576]: audit 2024-09-13T14:49:03.278526+0000 mon.a (mon.0) 1667 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:03.719 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:03 smithi099 bash[84576]: audit 2024-09-13T14:49:03.278526+0000 mon.a (mon.0) 1667 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:03.728 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:03.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:03.831 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:03.858 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:03.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:03.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:03 smithi177 bash[68594]: cluster 2024-09-13T14:49:02.517108+0000 mgr.y (mgr.26252) 681 : cluster [DBG] pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 15335/265275 objects degraded (5.781%); 9860/265275 objects misplaced (3.717%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:49:03.888 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:03 smithi177 bash[68594]: cluster 2024-09-13T14:49:02.517108+0000 mgr.y (mgr.26252) 681 : cluster [DBG] pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 15335/265275 objects degraded (5.781%); 9860/265275 objects misplaced (3.717%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:49:03.888 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:03 smithi177 bash[68594]: audit 2024-09-13T14:49:03.277098+0000 mon.a (mon.0) 1666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:03.888 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:03 smithi177 bash[68594]: audit 2024-09-13T14:49:03.277098+0000 mon.a (mon.0) 1666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:03.888 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:03 smithi177 bash[68594]: audit 2024-09-13T14:49:03.278526+0000 mon.a (mon.0) 1667 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:03.888 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:03 smithi177 bash[68594]: audit 2024-09-13T14:49:03.278526+0000 mon.a (mon.0) 1667 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:03.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:03.997 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:04.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:04.103 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:04.133 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:04.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:04.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:04.260 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:04.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:04.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:04.396 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:04.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:04.499 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:04.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:04.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 82 16 91889 91873 70.0142 33.8125 0.0101775 0.0142773 2024-09-13T14:49:04.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:04.647 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:04.692 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:04.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:04.795 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:04.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:04.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:04.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:04.946 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:04.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:05.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:05.081 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:05.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:05.206 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:05.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:05.309 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:05.342 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:05.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:05.477 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:05.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:05.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 83 16 92692 92676 69.7753 50.1875 0.0102039 0.0143266 2024-09-13T14:49:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:05 smithi099 bash[84576]: audit 2024-09-13T14:49:04.327754+0000 mon.a (mon.0) 1668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:05 smithi099 bash[84576]: audit 2024-09-13T14:49:04.327754+0000 mon.a (mon.0) 1668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:05 smithi099 bash[84576]: cluster 2024-09-13T14:49:04.517899+0000 mgr.y (mgr.26252) 682 : cluster [DBG] pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 715 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:49:05.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:05 smithi099 bash[84576]: cluster 2024-09-13T14:49:04.517899+0000 mgr.y (mgr.26252) 682 : cluster [DBG] pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 715 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:49:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:05 smithi099 bash[87147]: audit 2024-09-13T14:49:04.327754+0000 mon.a (mon.0) 1668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:05 smithi099 bash[87147]: audit 2024-09-13T14:49:04.327754+0000 mon.a (mon.0) 1668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:05 smithi099 bash[87147]: cluster 2024-09-13T14:49:04.517899+0000 mgr.y (mgr.26252) 682 : cluster [DBG] pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 715 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:49:05.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:05 smithi099 bash[87147]: cluster 2024-09-13T14:49:04.517899+0000 mgr.y (mgr.26252) 682 : cluster [DBG] pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 715 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:49:05.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:05.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:05 smithi177 bash[68594]: audit 2024-09-13T14:49:04.327754+0000 mon.a (mon.0) 1668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:05.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:05 smithi177 bash[68594]: audit 2024-09-13T14:49:04.327754+0000 mon.a (mon.0) 1668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:05.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:05 smithi177 bash[68594]: cluster 2024-09-13T14:49:04.517899+0000 mgr.y (mgr.26252) 682 : cluster [DBG] pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 715 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:49:05.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:05 smithi177 bash[68594]: cluster 2024-09-13T14:49:04.517899+0000 mgr.y (mgr.26252) 682 : cluster [DBG] pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 715 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:49:05.616 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:05.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:05.722 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:05.753 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:05.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:05.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:05.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:05.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:05.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:06.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:06.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:06.121 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:06.144 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:06.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:06.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:06.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:06.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:06.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:06 smithi177 bash[68594]: cluster 2024-09-13T14:49:05.325087+0000 mon.a (mon.0) 1669 : cluster [WRN] Health check update: Degraded data redundancy: 15335/272727 objects degraded (5.623%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:06.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:06 smithi177 bash[68594]: cluster 2024-09-13T14:49:05.325087+0000 mon.a (mon.0) 1669 : cluster [WRN] Health check update: Degraded data redundancy: 15335/272727 objects degraded (5.623%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:06.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:06.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:06.509 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:06.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:06.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 84 16 93477 93461 69.5287 49.0625 0.0385328 0.0143758 2024-09-13T14:49:06.625 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:06 smithi099 bash[84576]: cluster 2024-09-13T14:49:05.325087+0000 mon.a (mon.0) 1669 : cluster [WRN] Health check update: Degraded data redundancy: 15335/272727 objects degraded (5.623%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:06.625 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:06 smithi099 bash[84576]: cluster 2024-09-13T14:49:05.325087+0000 mon.a (mon.0) 1669 : cluster [WRN] Health check update: Degraded data redundancy: 15335/272727 objects degraded (5.623%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:06.625 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:06 smithi099 bash[87147]: cluster 2024-09-13T14:49:05.325087+0000 mon.a (mon.0) 1669 : cluster [WRN] Health check update: Degraded data redundancy: 15335/272727 objects degraded (5.623%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:06.625 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:06 smithi099 bash[87147]: cluster 2024-09-13T14:49:05.325087+0000 mon.a (mon.0) 1669 : cluster [WRN] Health check update: Degraded data redundancy: 15335/272727 objects degraded (5.623%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:06.637 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:06.674 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:06.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:06.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:06.800 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:06.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:06.906 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:06.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:06.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:07.057 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:07.082 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:07.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:07.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:07.197 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:49:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:49:07] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T14:49:07.213 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:07.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:07.316 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:07.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:07.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:07.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:07.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:07.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:07.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 85 16 94258 94242 69.2849 48.8125 0.0232307 0.0144271 2024-09-13T14:49:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:07 smithi099 bash[84576]: cluster 2024-09-13T14:49:06.518399+0000 mgr.y (mgr.26252) 683 : cluster [DBG] pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:49:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:07 smithi099 bash[84576]: cluster 2024-09-13T14:49:06.518399+0000 mgr.y (mgr.26252) 683 : cluster [DBG] pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:49:07.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:07 smithi099 bash[87147]: cluster 2024-09-13T14:49:06.518399+0000 mgr.y (mgr.26252) 683 : cluster [DBG] pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:49:07.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:07 smithi099 bash[87147]: cluster 2024-09-13T14:49:06.518399+0000 mgr.y (mgr.26252) 683 : cluster [DBG] pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:49:07.575 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:07.592 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:07.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:07.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:07 smithi177 bash[68594]: cluster 2024-09-13T14:49:06.518399+0000 mgr.y (mgr.26252) 683 : cluster [DBG] pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:49:07.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:07 smithi177 bash[68594]: cluster 2024-09-13T14:49:06.518399+0000 mgr.y (mgr.26252) 683 : cluster [DBG] pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s; 15335/272727 objects degraded (5.623%); 9775/272727 objects misplaced (3.584%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:49:07.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:07.714 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:07.817 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:07.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:07.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:07.945 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:07.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:07.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:08.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:08.113 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:08.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:08.216 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:08.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:08.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:08.255 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T14:49:08.255 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:49:08.255 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T14:49:08.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:08.372 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:08.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:08.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:08.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:08.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:08.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 86 16 94908 94892 68.9516 40.625 0.0122079 0.0144968 2024-09-13T14:49:08.617 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:08.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:08.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:08.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:08.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:08.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:08.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:08.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:08.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:09.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:09.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:09.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:09.255 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:09.289 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:09.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:09.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:09.422 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:09.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:09.525 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:09.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 87 16 95567 95551 68.6325 41.1875 0.0469528 0.0145622 2024-09-13T14:49:09.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:09.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:09.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:09.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:09.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:09.786 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:09.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:09 smithi099 bash[84576]: cluster 2024-09-13T14:49:08.519122+0000 mgr.y (mgr.26252) 684 : cluster [DBG] pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s; 15335/280143 objects degraded (5.474%); 9498/280143 objects misplaced (3.390%); 3.9 MiB/s, 61 objects/s recovering 2024-09-13T14:49:09.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:09 smithi099 bash[84576]: cluster 2024-09-13T14:49:08.519122+0000 mgr.y (mgr.26252) 684 : cluster [DBG] pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s; 15335/280143 objects degraded (5.474%); 9498/280143 objects misplaced (3.390%); 3.9 MiB/s, 61 objects/s recovering 2024-09-13T14:49:09.795 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:09 smithi099 bash[87147]: cluster 2024-09-13T14:49:08.519122+0000 mgr.y (mgr.26252) 684 : cluster [DBG] pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s; 15335/280143 objects degraded (5.474%); 9498/280143 objects misplaced (3.390%); 3.9 MiB/s, 61 objects/s recovering 2024-09-13T14:49:09.795 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:09 smithi099 bash[87147]: cluster 2024-09-13T14:49:08.519122+0000 mgr.y (mgr.26252) 684 : cluster [DBG] pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s; 15335/280143 objects degraded (5.474%); 9498/280143 objects misplaced (3.390%); 3.9 MiB/s, 61 objects/s recovering 2024-09-13T14:49:09.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:09.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:09.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:09 smithi177 bash[68594]: cluster 2024-09-13T14:49:08.519122+0000 mgr.y (mgr.26252) 684 : cluster [DBG] pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s; 15335/280143 objects degraded (5.474%); 9498/280143 objects misplaced (3.390%); 3.9 MiB/s, 61 objects/s recovering 2024-09-13T14:49:09.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:09 smithi177 bash[68594]: cluster 2024-09-13T14:49:08.519122+0000 mgr.y (mgr.26252) 684 : cluster [DBG] pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s; 15335/280143 objects degraded (5.474%); 9498/280143 objects misplaced (3.390%); 3.9 MiB/s, 61 objects/s recovering 2024-09-13T14:49:09.929 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:09.959 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:09.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:10.066 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:10.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:10.196 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:10.224 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:10.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:10.328 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:10.350 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:10.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:10.454 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:10.464 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:49:10 smithi177 bash[61439]: ts=2024-09-13T14:49:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:10.498 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:10.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:10.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 88 16 96356 96340 68.4129 49.3125 0.0115775 0.0146102 2024-09-13T14:49:10.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:10.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:10.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:10.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:10.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:10.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:10 smithi177 bash[68594]: cluster 2024-09-13T14:49:10.572963+0000 mon.a (mon.0) 1670 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283659 objects degraded (5.406%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:10.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:10 smithi177 bash[68594]: cluster 2024-09-13T14:49:10.572963+0000 mon.a (mon.0) 1670 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283659 objects degraded (5.406%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:10.889 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:10.889 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:10 smithi099 bash[84576]: cluster 2024-09-13T14:49:10.572963+0000 mon.a (mon.0) 1670 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283659 objects degraded (5.406%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:10.890 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:10 smithi099 bash[84576]: cluster 2024-09-13T14:49:10.572963+0000 mon.a (mon.0) 1670 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283659 objects degraded (5.406%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:10.890 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:10 smithi099 bash[87147]: cluster 2024-09-13T14:49:10.572963+0000 mon.a (mon.0) 1670 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283659 objects degraded (5.406%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:10.890 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:10 smithi099 bash[87147]: cluster 2024-09-13T14:49:10.572963+0000 mon.a (mon.0) 1670 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283659 objects degraded (5.406%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:10.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:10.992 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:11.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:11.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:11.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:11.144 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:11.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:11.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:11.285 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:11.394 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:11.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:11.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:11.529 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:11.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 89 16 97086 97070 68.1568 45.625 0.012546 0.0146661 2024-09-13T14:49:11.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:11.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:11.657 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:11.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:11.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:11.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:11.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:11 smithi177 bash[68594]: cluster 2024-09-13T14:49:10.519718+0000 mgr.y (mgr.26252) 685 : cluster [DBG] pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 15335/283659 objects degraded (5.406%); 9498/283659 objects misplaced (3.348%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:11.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:11 smithi177 bash[68594]: cluster 2024-09-13T14:49:10.519718+0000 mgr.y (mgr.26252) 685 : cluster [DBG] pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 15335/283659 objects degraded (5.406%); 9498/283659 objects misplaced (3.348%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:11.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:11.942 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:11 smithi099 bash[84576]: cluster 2024-09-13T14:49:10.519718+0000 mgr.y (mgr.26252) 685 : cluster [DBG] pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 15335/283659 objects degraded (5.406%); 9498/283659 objects misplaced (3.348%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:11.942 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:11 smithi099 bash[84576]: cluster 2024-09-13T14:49:10.519718+0000 mgr.y (mgr.26252) 685 : cluster [DBG] pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 15335/283659 objects degraded (5.406%); 9498/283659 objects misplaced (3.348%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:11.942 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:11 smithi099 bash[87147]: cluster 2024-09-13T14:49:10.519718+0000 mgr.y (mgr.26252) 685 : cluster [DBG] pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 15335/283659 objects degraded (5.406%); 9498/283659 objects misplaced (3.348%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:11.942 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:11 smithi099 bash[87147]: cluster 2024-09-13T14:49:10.519718+0000 mgr.y (mgr.26252) 685 : cluster [DBG] pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 15335/283659 objects degraded (5.406%); 9498/283659 objects misplaced (3.348%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:12.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:12.068 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:12.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:12.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:12.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:12.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:12.278 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:12.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:12.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:12.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:12.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:12.545 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 90 2 97785 97783 67.8946 44.5625 0.0166321 0.0147233 2024-09-13T14:49:12.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:12.592 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:12.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:12.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:12.718 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:12.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:12.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:12.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:12.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:12.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:12.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:12.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:13.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:13.107 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:13.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:13.236 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:13.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:13.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:13.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:13.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:13.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:13.496 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:13.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total time run: 90.0206 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total writes made: 97785 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Write size: 65536 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object size: 65536 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Bandwidth (MB/sec): 67.8907 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Bandwidth: 33.6837 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max bandwidth (MB/sec): 169.5 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min bandwidth (MB/sec): 0 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average IOPS: 1086 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev IOPS: 538.95 2024-09-13T14:49:13.546 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max IOPS: 2712 2024-09-13T14:49:13.547 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min IOPS: 0 2024-09-13T14:49:13.547 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average Latency(s): 0.0147236 2024-09-13T14:49:13.547 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Latency(s): 0.0517609 2024-09-13T14:49:13.547 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max latency(s): 2.50527 2024-09-13T14:49:13.547 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min latency(s): 0.00109707 2024-09-13T14:49:13.560 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:49:13.598 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:13.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:13.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:13.737 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:13.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:13.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:13.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:13 smithi099 bash[84576]: cluster 2024-09-13T14:49:12.520247+0000 mgr.y (mgr.26252) 686 : cluster [DBG] pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 661 op/s; 15335/285597 objects degraded (5.369%); 9498/285597 objects misplaced (3.326%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:13 smithi177 bash[68594]: cluster 2024-09-13T14:49:12.520247+0000 mgr.y (mgr.26252) 686 : cluster [DBG] pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 661 op/s; 15335/285597 objects degraded (5.369%); 9498/285597 objects misplaced (3.326%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:13 smithi177 bash[68594]: cluster 2024-09-13T14:49:12.520247+0000 mgr.y (mgr.26252) 686 : cluster [DBG] pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 661 op/s; 15335/285597 objects degraded (5.369%); 9498/285597 objects misplaced (3.326%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:13.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:13.898 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:13.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:14.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:14.027 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:14.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:14.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:13 smithi099 bash[84576]: cluster 2024-09-13T14:49:12.520247+0000 mgr.y (mgr.26252) 686 : cluster [DBG] pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 661 op/s; 15335/285597 objects degraded (5.369%); 9498/285597 objects misplaced (3.326%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:13 smithi099 bash[87147]: cluster 2024-09-13T14:49:12.520247+0000 mgr.y (mgr.26252) 686 : cluster [DBG] pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 661 op/s; 15335/285597 objects degraded (5.369%); 9498/285597 objects misplaced (3.326%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:14.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:13 smithi099 bash[87147]: cluster 2024-09-13T14:49:12.520247+0000 mgr.y (mgr.26252) 686 : cluster [DBG] pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 661 op/s; 15335/285597 objects degraded (5.369%); 9498/285597 objects misplaced (3.326%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:49:14.131 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:14.162 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:14.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:14.267 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:14.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:14.291 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:49:14.292 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:49:14.292 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-13T14:49:14.292 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:49:14.292 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:49:14.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-13T14:49:14.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:14.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:14.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:14.514 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:14.539 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:14.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:14.643 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:14.665 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:14.768 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:14.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:14.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:14.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:14.923 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:14.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:15.026 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:15.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:15.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:15.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:15.181 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:15.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:15.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:15.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:15.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:15.444 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:15.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:15.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:15.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:15.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:15.714 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:15.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:15.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:15 smithi099 bash[84576]: cluster 2024-09-13T14:49:14.521109+0000 mgr.y (mgr.26252) 687 : cluster [DBG] pgmap v1161: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 782 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:15 smithi099 bash[84576]: cluster 2024-09-13T14:49:14.521109+0000 mgr.y (mgr.26252) 687 : cluster [DBG] pgmap v1161: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 782 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:15.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:15 smithi099 bash[87147]: cluster 2024-09-13T14:49:14.521109+0000 mgr.y (mgr.26252) 687 : cluster [DBG] pgmap v1161: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 782 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:15.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:15 smithi099 bash[87147]: cluster 2024-09-13T14:49:14.521109+0000 mgr.y (mgr.26252) 687 : cluster [DBG] pgmap v1161: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 782 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:15.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:15.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:15.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:15.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:15 smithi177 bash[68594]: cluster 2024-09-13T14:49:14.521109+0000 mgr.y (mgr.26252) 687 : cluster [DBG] pgmap v1161: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 782 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:15.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:15 smithi177 bash[68594]: cluster 2024-09-13T14:49:14.521109+0000 mgr.y (mgr.26252) 687 : cluster [DBG] pgmap v1161: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 782 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:15.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:16.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:16.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:16.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:16.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:16.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:16.479 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:16.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:16.637 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:16.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:16.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:16.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:16.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:16 smithi099 bash[84576]: cluster 2024-09-13T14:49:16.318408+0000 mon.a (mon.0) 1671 : cluster [WRN] Health check update: Degraded data redundancy: 15335/292824 objects degraded (5.237%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:16.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:16 smithi099 bash[84576]: cluster 2024-09-13T14:49:16.318408+0000 mon.a (mon.0) 1671 : cluster [WRN] Health check update: Degraded data redundancy: 15335/292824 objects degraded (5.237%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:16.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:16 smithi099 bash[87147]: cluster 2024-09-13T14:49:16.318408+0000 mon.a (mon.0) 1671 : cluster [WRN] Health check update: Degraded data redundancy: 15335/292824 objects degraded (5.237%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:16.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:16 smithi099 bash[87147]: cluster 2024-09-13T14:49:16.318408+0000 mon.a (mon.0) 1671 : cluster [WRN] Health check update: Degraded data redundancy: 15335/292824 objects degraded (5.237%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:16.878 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:16.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:16 smithi177 bash[68594]: cluster 2024-09-13T14:49:16.318408+0000 mon.a (mon.0) 1671 : cluster [WRN] Health check update: Degraded data redundancy: 15335/292824 objects degraded (5.237%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:16.888 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:16 smithi177 bash[68594]: cluster 2024-09-13T14:49:16.318408+0000 mon.a (mon.0) 1671 : cluster [WRN] Health check update: Degraded data redundancy: 15335/292824 objects degraded (5.237%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:16.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:16.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:17.009 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:17.043 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:17.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:17.148 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:17.178 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:17.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:17.281 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:17.308 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:49:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:49:17] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T14:49:17.412 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:17.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:17.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:17.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:17.570 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:17.678 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:17.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:17.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:17.829 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:17.833 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:49:17.833 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:17.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:17 smithi099 bash[84576]: cluster 2024-09-13T14:49:16.521582+0000 mgr.y (mgr.26252) 688 : cluster [DBG] pgmap v1162: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 575 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:49:17.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:17 smithi099 bash[84576]: cluster 2024-09-13T14:49:16.521582+0000 mgr.y (mgr.26252) 688 : cluster [DBG] pgmap v1162: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 575 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:49:17.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:17 smithi099 bash[84576]: audit 2024-09-13T14:49:16.666322+0000 mon.a (mon.0) 1672 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:17.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:17 smithi099 bash[84576]: audit 2024-09-13T14:49:16.666322+0000 mon.a (mon.0) 1672 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:17.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:17 smithi099 bash[87147]: cluster 2024-09-13T14:49:16.521582+0000 mgr.y (mgr.26252) 688 : cluster [DBG] pgmap v1162: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 575 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:49:17.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:17 smithi099 bash[87147]: cluster 2024-09-13T14:49:16.521582+0000 mgr.y (mgr.26252) 688 : cluster [DBG] pgmap v1162: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 575 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:49:17.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:17 smithi099 bash[87147]: audit 2024-09-13T14:49:16.666322+0000 mon.a (mon.0) 1672 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:17.840 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:17 smithi099 bash[87147]: audit 2024-09-13T14:49:16.666322+0000 mon.a (mon.0) 1672 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:17 smithi177 bash[68594]: cluster 2024-09-13T14:49:16.521582+0000 mgr.y (mgr.26252) 688 : cluster [DBG] pgmap v1162: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 575 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:49:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:17 smithi177 bash[68594]: cluster 2024-09-13T14:49:16.521582+0000 mgr.y (mgr.26252) 688 : cluster [DBG] pgmap v1162: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 575 op/s; 15335/292824 objects degraded (5.237%); 9234/292824 objects misplaced (3.153%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:49:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:17 smithi177 bash[68594]: audit 2024-09-13T14:49:16.666322+0000 mon.a (mon.0) 1672 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:17 smithi177 bash[68594]: audit 2024-09-13T14:49:16.666322+0000 mon.a (mon.0) 1672 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:49:17.878 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:49:17.878 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:17.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:17.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:17.912 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:17.912 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:49:17.939 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:49:17.939 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:17.977 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:49:17.978 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:17.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:18.021 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:49:18.021 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:18.036 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:18.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:18.055 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:49:18.055 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:18.083 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:49:18.083 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:49:18.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:18.162 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:18.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:18.265 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:18.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:18.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:18.400 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:18.435 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:18.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:18.540 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:18.578 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:18.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:18.674 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T14:49:18.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:18.708 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:18.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:18.810 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[84576]: audit 2024-09-13T14:49:17.787392+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.99:0/974214632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[84576]: audit 2024-09-13T14:49:17.787392+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.99:0/974214632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[84576]: audit 2024-09-13T14:49:18.233425+0000 mon.c (mon.2) 54 : audit [INF] from='client.? 172.21.15.99:0/1933766021' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[84576]: audit 2024-09-13T14:49:18.233425+0000 mon.c (mon.2) 54 : audit [INF] from='client.? 172.21.15.99:0/1933766021' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[84576]: audit 2024-09-13T14:49:18.234343+0000 mon.a (mon.0) 1674 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[84576]: audit 2024-09-13T14:49:18.234343+0000 mon.a (mon.0) 1674 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:18.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[84576]: audit 2024-09-13T14:49:18.272981+0000 mon.a (mon.0) 1675 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:18.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[84576]: audit 2024-09-13T14:49:18.272981+0000 mon.a (mon.0) 1675 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[87147]: audit 2024-09-13T14:49:17.787392+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.99:0/974214632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[87147]: audit 2024-09-13T14:49:17.787392+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.99:0/974214632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[87147]: audit 2024-09-13T14:49:18.233425+0000 mon.c (mon.2) 54 : audit [INF] from='client.? 172.21.15.99:0/1933766021' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[87147]: audit 2024-09-13T14:49:18.233425+0000 mon.c (mon.2) 54 : audit [INF] from='client.? 172.21.15.99:0/1933766021' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[87147]: audit 2024-09-13T14:49:18.234343+0000 mon.a (mon.0) 1674 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[87147]: audit 2024-09-13T14:49:18.234343+0000 mon.a (mon.0) 1674 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[87147]: audit 2024-09-13T14:49:18.272981+0000 mon.a (mon.0) 1675 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:18.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:18 smithi099 bash[87147]: audit 2024-09-13T14:49:18.272981+0000 mon.a (mon.0) 1675 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:18.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.842 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:18.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:18.943 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:18.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:18.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:19.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:19.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:18 smithi177 bash[68594]: audit 2024-09-13T14:49:17.787392+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.99:0/974214632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:19.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:18 smithi177 bash[68594]: audit 2024-09-13T14:49:17.787392+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.99:0/974214632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:19.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:18 smithi177 bash[68594]: audit 2024-09-13T14:49:18.233425+0000 mon.c (mon.2) 54 : audit [INF] from='client.? 172.21.15.99:0/1933766021' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:19.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:18 smithi177 bash[68594]: audit 2024-09-13T14:49:18.233425+0000 mon.c (mon.2) 54 : audit [INF] from='client.? 172.21.15.99:0/1933766021' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:19.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:18 smithi177 bash[68594]: audit 2024-09-13T14:49:18.234343+0000 mon.a (mon.0) 1674 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:19.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:18 smithi177 bash[68594]: audit 2024-09-13T14:49:18.234343+0000 mon.a (mon.0) 1674 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:49:19.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:18 smithi177 bash[68594]: audit 2024-09-13T14:49:18.272981+0000 mon.a (mon.0) 1675 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:19.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:18 smithi177 bash[68594]: audit 2024-09-13T14:49:18.272981+0000 mon.a (mon.0) 1675 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:19.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:19.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:19.216 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:19.235 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:19.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:19.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:19.371 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:19.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:19.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:19.503 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:19.608 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:19.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:19.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:19.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:19.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:19.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[84576]: cluster 2024-09-13T14:49:18.522334+0000 mgr.y (mgr.26252) 689 : cluster [DBG] pgmap v1163: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 85 B/s rd, 36 MiB/s wr, 759 op/s; 15335/286803 objects degraded (5.347%); 8937/286803 objects misplaced (3.116%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:49:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[84576]: cluster 2024-09-13T14:49:18.522334+0000 mgr.y (mgr.26252) 689 : cluster [DBG] pgmap v1163: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 85 B/s rd, 36 MiB/s wr, 759 op/s; 15335/286803 objects degraded (5.347%); 8937/286803 objects misplaced (3.116%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:49:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[84576]: cluster 2024-09-13T14:49:18.650460+0000 mon.a (mon.0) 1676 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[84576]: cluster 2024-09-13T14:49:18.650460+0000 mon.a (mon.0) 1676 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[84576]: audit 2024-09-13T14:49:18.666819+0000 mon.a (mon.0) 1677 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[84576]: audit 2024-09-13T14:49:18.666819+0000 mon.a (mon.0) 1677 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[84576]: cluster 2024-09-13T14:49:18.678568+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e980: 8 total, 8 up, 7 in 2024-09-13T14:49:19.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[84576]: cluster 2024-09-13T14:49:18.678568+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e980: 8 total, 8 up, 7 in 2024-09-13T14:49:19.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[87147]: cluster 2024-09-13T14:49:18.522334+0000 mgr.y (mgr.26252) 689 : cluster [DBG] pgmap v1163: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 85 B/s rd, 36 MiB/s wr, 759 op/s; 15335/286803 objects degraded (5.347%); 8937/286803 objects misplaced (3.116%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:49:19.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[87147]: cluster 2024-09-13T14:49:18.522334+0000 mgr.y (mgr.26252) 689 : cluster [DBG] pgmap v1163: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 85 B/s rd, 36 MiB/s wr, 759 op/s; 15335/286803 objects degraded (5.347%); 8937/286803 objects misplaced (3.116%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:49:19.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[87147]: cluster 2024-09-13T14:49:18.650460+0000 mon.a (mon.0) 1676 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[87147]: cluster 2024-09-13T14:49:18.650460+0000 mon.a (mon.0) 1676 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[87147]: audit 2024-09-13T14:49:18.666819+0000 mon.a (mon.0) 1677 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[87147]: audit 2024-09-13T14:49:18.666819+0000 mon.a (mon.0) 1677 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[87147]: cluster 2024-09-13T14:49:18.678568+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e980: 8 total, 8 up, 7 in 2024-09-13T14:49:19.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:19 smithi099 bash[87147]: cluster 2024-09-13T14:49:18.678568+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e980: 8 total, 8 up, 7 in 2024-09-13T14:49:19.875 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:19.909 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:19.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:20.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:20.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:19 smithi177 bash[68594]: cluster 2024-09-13T14:49:18.522334+0000 mgr.y (mgr.26252) 689 : cluster [DBG] pgmap v1163: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 85 B/s rd, 36 MiB/s wr, 759 op/s; 15335/286803 objects degraded (5.347%); 8937/286803 objects misplaced (3.116%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:49:20.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:19 smithi177 bash[68594]: cluster 2024-09-13T14:49:18.522334+0000 mgr.y (mgr.26252) 689 : cluster [DBG] pgmap v1163: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 85 B/s rd, 36 MiB/s wr, 759 op/s; 15335/286803 objects degraded (5.347%); 8937/286803 objects misplaced (3.116%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:49:20.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:19 smithi177 bash[68594]: cluster 2024-09-13T14:49:18.650460+0000 mon.a (mon.0) 1676 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:20.022 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:19 smithi177 bash[68594]: cluster 2024-09-13T14:49:18.650460+0000 mon.a (mon.0) 1676 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:20.022 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:19 smithi177 bash[68594]: audit 2024-09-13T14:49:18.666819+0000 mon.a (mon.0) 1677 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:20.022 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:19 smithi177 bash[68594]: audit 2024-09-13T14:49:18.666819+0000 mon.a (mon.0) 1677 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:49:20.022 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:19 smithi177 bash[68594]: cluster 2024-09-13T14:49:18.678568+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e980: 8 total, 8 up, 7 in 2024-09-13T14:49:20.022 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:19 smithi177 bash[68594]: cluster 2024-09-13T14:49:18.678568+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e980: 8 total, 8 up, 7 in 2024-09-13T14:49:20.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:20.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:20.140 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:20.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:20.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:20.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:20.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:49:20 smithi177 bash[61439]: ts=2024-09-13T14:49:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:20.408 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:20.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:20.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:20.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:20.686 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:20.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:20.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:20.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:20.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:20.956 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:20.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:20.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:21.083 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:21.114 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:21.220 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:21.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:21.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:21.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:21.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:21.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:21.501 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:21.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:21.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:21.627 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:21.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:21.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:21.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:21.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:21 smithi099 bash[84576]: cluster 2024-09-13T14:49:20.522869+0000 mgr.y (mgr.26252) 690 : cluster [DBG] pgmap v1165: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 20 MiB/s wr, 663 op/s; 15335/283311 objects degraded (5.413%); 8937/283311 objects misplaced (3.154%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:21.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:21 smithi099 bash[84576]: cluster 2024-09-13T14:49:20.522869+0000 mgr.y (mgr.26252) 690 : cluster [DBG] pgmap v1165: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 20 MiB/s wr, 663 op/s; 15335/283311 objects degraded (5.413%); 8937/283311 objects misplaced (3.154%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:21.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:21 smithi099 bash[84576]: cluster 2024-09-13T14:49:21.319202+0000 mon.a (mon.0) 1679 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283311 objects degraded (5.413%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:21.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:21 smithi099 bash[84576]: cluster 2024-09-13T14:49:21.319202+0000 mon.a (mon.0) 1679 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283311 objects degraded (5.413%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:21 smithi099 bash[87147]: cluster 2024-09-13T14:49:20.522869+0000 mgr.y (mgr.26252) 690 : cluster [DBG] pgmap v1165: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 20 MiB/s wr, 663 op/s; 15335/283311 objects degraded (5.413%); 8937/283311 objects misplaced (3.154%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:21 smithi099 bash[87147]: cluster 2024-09-13T14:49:20.522869+0000 mgr.y (mgr.26252) 690 : cluster [DBG] pgmap v1165: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 20 MiB/s wr, 663 op/s; 15335/283311 objects degraded (5.413%); 8937/283311 objects misplaced (3.154%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:21 smithi099 bash[87147]: cluster 2024-09-13T14:49:21.319202+0000 mon.a (mon.0) 1679 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283311 objects degraded (5.413%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:21.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:21 smithi099 bash[87147]: cluster 2024-09-13T14:49:21.319202+0000 mon.a (mon.0) 1679 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283311 objects degraded (5.413%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:21.853 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:21.877 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:21.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:21.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:22.012 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:22.012 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:21 smithi177 bash[68594]: cluster 2024-09-13T14:49:20.522869+0000 mgr.y (mgr.26252) 690 : cluster [DBG] pgmap v1165: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 20 MiB/s wr, 663 op/s; 15335/283311 objects degraded (5.413%); 8937/283311 objects misplaced (3.154%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:22.013 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:21 smithi177 bash[68594]: cluster 2024-09-13T14:49:20.522869+0000 mgr.y (mgr.26252) 690 : cluster [DBG] pgmap v1165: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 20 MiB/s wr, 663 op/s; 15335/283311 objects degraded (5.413%); 8937/283311 objects misplaced (3.154%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:22.013 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:21 smithi177 bash[68594]: cluster 2024-09-13T14:49:21.319202+0000 mon.a (mon.0) 1679 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283311 objects degraded (5.413%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:22.013 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:21 smithi177 bash[68594]: cluster 2024-09-13T14:49:21.319202+0000 mon.a (mon.0) 1679 : cluster [WRN] Health check update: Degraded data redundancy: 15335/283311 objects degraded (5.413%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:22.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:22.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:22.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:22.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:22.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:22.282 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:22.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:22.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:22.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:22.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:22.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:22.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:22.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:22.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:22.693 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:22.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:22.828 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:22.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:22.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:22.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:22.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:23.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:23.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:23.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:23.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:23.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:23.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:23.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:23.388 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:23.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:23.497 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:23.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:23.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:23.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:23.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:23.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:23.840 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:23.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:23.929 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[84576]: cluster 2024-09-13T14:49:22.523353+0000 mgr.y (mgr.26252) 691 : cluster [DBG] pgmap v1166: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 672 op/s; 15335/281100 objects degraded (5.455%); 8937/281100 objects misplaced (3.179%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:23.929 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[84576]: cluster 2024-09-13T14:49:22.523353+0000 mgr.y (mgr.26252) 691 : cluster [DBG] pgmap v1166: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 672 op/s; 15335/281100 objects degraded (5.455%); 8937/281100 objects misplaced (3.179%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:23.929 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[84576]: audit 2024-09-13T14:49:23.255916+0000 mon.a (mon.0) 1680 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.929 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[84576]: audit 2024-09-13T14:49:23.255916+0000 mon.a (mon.0) 1680 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.929 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[84576]: audit 2024-09-13T14:49:23.267309+0000 mon.a (mon.0) 1681 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.929 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[84576]: audit 2024-09-13T14:49:23.267309+0000 mon.a (mon.0) 1681 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.930 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[87147]: cluster 2024-09-13T14:49:22.523353+0000 mgr.y (mgr.26252) 691 : cluster [DBG] pgmap v1166: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 672 op/s; 15335/281100 objects degraded (5.455%); 8937/281100 objects misplaced (3.179%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:23.930 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[87147]: cluster 2024-09-13T14:49:22.523353+0000 mgr.y (mgr.26252) 691 : cluster [DBG] pgmap v1166: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 672 op/s; 15335/281100 objects degraded (5.455%); 8937/281100 objects misplaced (3.179%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:23.930 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[87147]: audit 2024-09-13T14:49:23.255916+0000 mon.a (mon.0) 1680 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.930 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[87147]: audit 2024-09-13T14:49:23.255916+0000 mon.a (mon.0) 1680 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.930 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[87147]: audit 2024-09-13T14:49:23.267309+0000 mon.a (mon.0) 1681 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.930 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:23 smithi099 bash[87147]: audit 2024-09-13T14:49:23.267309+0000 mon.a (mon.0) 1681 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:23.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:23 smithi177 bash[68594]: cluster 2024-09-13T14:49:22.523353+0000 mgr.y (mgr.26252) 691 : cluster [DBG] pgmap v1166: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 672 op/s; 15335/281100 objects degraded (5.455%); 8937/281100 objects misplaced (3.179%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:23.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:23 smithi177 bash[68594]: cluster 2024-09-13T14:49:22.523353+0000 mgr.y (mgr.26252) 691 : cluster [DBG] pgmap v1166: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 672 op/s; 15335/281100 objects degraded (5.455%); 8937/281100 objects misplaced (3.179%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:23.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:23 smithi177 bash[68594]: audit 2024-09-13T14:49:23.255916+0000 mon.a (mon.0) 1680 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:23 smithi177 bash[68594]: audit 2024-09-13T14:49:23.255916+0000 mon.a (mon.0) 1680 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:23 smithi177 bash[68594]: audit 2024-09-13T14:49:23.267309+0000 mon.a (mon.0) 1681 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:23 smithi177 bash[68594]: audit 2024-09-13T14:49:23.267309+0000 mon.a (mon.0) 1681 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:23.967 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:23.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:24.075 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:24.104 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:24.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:24.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:24.234 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:24.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:24.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:24.370 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:24.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:24.406 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:24.479 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:24.497 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:24.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:24.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:24.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:24.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:24.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:24.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:24.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:24.859 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:24.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:24.986 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:25.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:25.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:25.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:25.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:25.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:25.268 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:25.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:25.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:25.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:25.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:25.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:25.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:25.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:25.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:25.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:25.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:25.805 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:25.909 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:25.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:49:25.934 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:49:25.940 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:25.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:25.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:25 smithi099 bash[84576]: cluster 2024-09-13T14:49:24.523962+0000 mgr.y (mgr.26252) 692 : cluster [DBG] pgmap v1167: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:25.952 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:25 smithi099 bash[84576]: cluster 2024-09-13T14:49:24.523962+0000 mgr.y (mgr.26252) 692 : cluster [DBG] pgmap v1167: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:25.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:25 smithi099 bash[87147]: cluster 2024-09-13T14:49:24.523962+0000 mgr.y (mgr.26252) 692 : cluster [DBG] pgmap v1167: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:25.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:25 smithi099 bash[87147]: cluster 2024-09-13T14:49:24.523962+0000 mgr.y (mgr.26252) 692 : cluster [DBG] pgmap v1167: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:26.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:25 smithi177 bash[68594]: cluster 2024-09-13T14:49:24.523962+0000 mgr.y (mgr.26252) 692 : cluster [DBG] pgmap v1167: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:25 smithi177 bash[68594]: cluster 2024-09-13T14:49:24.523962+0000 mgr.y (mgr.26252) 692 : cluster [DBG] pgmap v1167: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:26.143 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:26.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:26.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:26.272 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:26.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:26.377 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:26.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:26.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:26.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:26.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:26.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:26.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:26.696 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:26.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:26.800 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:26.823 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:26.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:26.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:26.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:26 smithi177 bash[68594]: cluster 2024-09-13T14:49:26.319940+0000 mon.a (mon.0) 1682 : cluster [WRN] Health check update: Degraded data redundancy: 15335/271881 objects degraded (5.640%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:26.935 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:26 smithi177 bash[68594]: cluster 2024-09-13T14:49:26.319940+0000 mon.a (mon.0) 1682 : cluster [WRN] Health check update: Degraded data redundancy: 15335/271881 objects degraded (5.640%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:26.950 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:26.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:26.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:26 smithi099 bash[84576]: cluster 2024-09-13T14:49:26.319940+0000 mon.a (mon.0) 1682 : cluster [WRN] Health check update: Degraded data redundancy: 15335/271881 objects degraded (5.640%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:26.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:26 smithi099 bash[84576]: cluster 2024-09-13T14:49:26.319940+0000 mon.a (mon.0) 1682 : cluster [WRN] Health check update: Degraded data redundancy: 15335/271881 objects degraded (5.640%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:26.965 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:26 smithi099 bash[87147]: cluster 2024-09-13T14:49:26.319940+0000 mon.a (mon.0) 1682 : cluster [WRN] Health check update: Degraded data redundancy: 15335/271881 objects degraded (5.640%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:26.966 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:26 smithi099 bash[87147]: cluster 2024-09-13T14:49:26.319940+0000 mon.a (mon.0) 1682 : cluster [WRN] Health check update: Degraded data redundancy: 15335/271881 objects degraded (5.640%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:27.052 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:27.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:27.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:27.184 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:27.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:27.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:27.311 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:27.341 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:27.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:27.345 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:49:27] "GET /metrics HTTP/1.1" 200 36772 "" "Prometheus/2.51.0" 2024-09-13T14:49:27.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:27.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:27.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:27.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:27.610 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:27.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:27.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:27.738 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:27.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[84576]: cluster 2024-09-13T14:49:26.524402+0000 mgr.y (mgr.26252) 693 : cluster [DBG] pgmap v1168: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[84576]: cluster 2024-09-13T14:49:26.524402+0000 mgr.y (mgr.26252) 693 : cluster [DBG] pgmap v1168: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[84576]: audit 2024-09-13T14:49:27.609487+0000 mon.a (mon.0) 1683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[84576]: audit 2024-09-13T14:49:27.609487+0000 mon.a (mon.0) 1683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[84576]: audit 2024-09-13T14:49:27.625302+0000 mon.a (mon.0) 1684 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[84576]: audit 2024-09-13T14:49:27.625302+0000 mon.a (mon.0) 1684 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[87147]: cluster 2024-09-13T14:49:26.524402+0000 mgr.y (mgr.26252) 693 : cluster [DBG] pgmap v1168: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[87147]: cluster 2024-09-13T14:49:26.524402+0000 mgr.y (mgr.26252) 693 : cluster [DBG] pgmap v1168: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[87147]: audit 2024-09-13T14:49:27.609487+0000 mon.a (mon.0) 1683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[87147]: audit 2024-09-13T14:49:27.609487+0000 mon.a (mon.0) 1683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[87147]: audit 2024-09-13T14:49:27.625302+0000 mon.a (mon.0) 1684 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:27 smithi099 bash[87147]: audit 2024-09-13T14:49:27.625302+0000 mon.a (mon.0) 1684 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:27 smithi177 bash[68594]: cluster 2024-09-13T14:49:26.524402+0000 mgr.y (mgr.26252) 693 : cluster [DBG] pgmap v1168: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:27 smithi177 bash[68594]: cluster 2024-09-13T14:49:26.524402+0000 mgr.y (mgr.26252) 693 : cluster [DBG] pgmap v1168: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 640 KiB/s wr, 718 op/s; 15335/271881 objects degraded (5.640%); 8671/271881 objects misplaced (3.189%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:49:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:27 smithi177 bash[68594]: audit 2024-09-13T14:49:27.609487+0000 mon.a (mon.0) 1683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:27 smithi177 bash[68594]: audit 2024-09-13T14:49:27.609487+0000 mon.a (mon.0) 1683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:27 smithi177 bash[68594]: audit 2024-09-13T14:49:27.625302+0000 mon.a (mon.0) 1684 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:27 smithi177 bash[68594]: audit 2024-09-13T14:49:27.625302+0000 mon.a (mon.0) 1684 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:27.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:27.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:27.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:28.004 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:28.108 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:28.132 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:28.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:28.235 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:28.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:28.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:28.363 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:28.399 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:28.502 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:28.536 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:28.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:28.643 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:28.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:28.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:28.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:28.807 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:28.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[84576]: audit 2024-09-13T14:49:28.397221+0000 mon.a (mon.0) 1685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:49:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[84576]: audit 2024-09-13T14:49:28.397221+0000 mon.a (mon.0) 1685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:49:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[84576]: audit 2024-09-13T14:49:28.399998+0000 mon.a (mon.0) 1686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:49:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[84576]: audit 2024-09-13T14:49:28.399998+0000 mon.a (mon.0) 1686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:49:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[84576]: audit 2024-09-13T14:49:28.407131+0000 mon.a (mon.0) 1687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[84576]: audit 2024-09-13T14:49:28.407131+0000 mon.a (mon.0) 1687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:28.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[87147]: audit 2024-09-13T14:49:28.397221+0000 mon.a (mon.0) 1685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:49:28.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[87147]: audit 2024-09-13T14:49:28.397221+0000 mon.a (mon.0) 1685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:49:28.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[87147]: audit 2024-09-13T14:49:28.399998+0000 mon.a (mon.0) 1686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:49:28.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[87147]: audit 2024-09-13T14:49:28.399998+0000 mon.a (mon.0) 1686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:49:28.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[87147]: audit 2024-09-13T14:49:28.407131+0000 mon.a (mon.0) 1687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:28.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:28 smithi099 bash[87147]: audit 2024-09-13T14:49:28.407131+0000 mon.a (mon.0) 1687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:28.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:28.923 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:28 smithi177 bash[68594]: audit 2024-09-13T14:49:28.397221+0000 mon.a (mon.0) 1685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:49:28.923 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:28 smithi177 bash[68594]: audit 2024-09-13T14:49:28.397221+0000 mon.a (mon.0) 1685 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:49:28.923 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:28 smithi177 bash[68594]: audit 2024-09-13T14:49:28.399998+0000 mon.a (mon.0) 1686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:49:28.923 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:28 smithi177 bash[68594]: audit 2024-09-13T14:49:28.399998+0000 mon.a (mon.0) 1686 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:49:28.924 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:28 smithi177 bash[68594]: audit 2024-09-13T14:49:28.407131+0000 mon.a (mon.0) 1687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:28.924 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:28 smithi177 bash[68594]: audit 2024-09-13T14:49:28.407131+0000 mon.a (mon.0) 1687 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:28.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:28.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:29.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:29.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:29.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:29.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:29.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:29.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:29.223 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T14:49:29.223 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:49:29.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:29.340 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:29.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:29.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:29.476 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:29.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:29.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:29.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:29.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:29.709 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:29.733 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:29.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:29.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:29.853 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:29 smithi099 bash[84576]: cluster 2024-09-13T14:49:28.524949+0000 mgr.y (mgr.26252) 694 : cluster [DBG] pgmap v1169: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 15335/265401 objects degraded (5.778%); 8408/265401 objects misplaced (3.168%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:29.854 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:29 smithi099 bash[84576]: cluster 2024-09-13T14:49:28.524949+0000 mgr.y (mgr.26252) 694 : cluster [DBG] pgmap v1169: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 15335/265401 objects degraded (5.778%); 8408/265401 objects misplaced (3.168%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:29.854 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:29 smithi099 bash[87147]: cluster 2024-09-13T14:49:28.524949+0000 mgr.y (mgr.26252) 694 : cluster [DBG] pgmap v1169: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 15335/265401 objects degraded (5.778%); 8408/265401 objects misplaced (3.168%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:29.854 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:29 smithi099 bash[87147]: cluster 2024-09-13T14:49:28.524949+0000 mgr.y (mgr.26252) 694 : cluster [DBG] pgmap v1169: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 15335/265401 objects degraded (5.778%); 8408/265401 objects misplaced (3.168%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:29.861 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:29.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:29.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:30.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:30.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:30.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:30.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:29 smithi177 bash[68594]: cluster 2024-09-13T14:49:28.524949+0000 mgr.y (mgr.26252) 694 : cluster [DBG] pgmap v1169: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 15335/265401 objects degraded (5.778%); 8408/265401 objects misplaced (3.168%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:30.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:29 smithi177 bash[68594]: cluster 2024-09-13T14:49:28.524949+0000 mgr.y (mgr.26252) 694 : cluster [DBG] pgmap v1169: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 15335/265401 objects degraded (5.778%); 8408/265401 objects misplaced (3.168%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:49:30.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:30.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:30.235 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:30.266 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:30.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:30.370 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:30.400 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:30.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:30.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:30.537 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:30.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:30.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:49:30 smithi177 bash[61439]: ts=2024-09-13T14:49:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:30.641 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:30.668 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:30.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:30.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:30.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:30.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:30.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:30.952 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:30.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:31.057 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:31.081 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:31.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:31.184 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:31.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:31.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:31.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:31.346 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:31.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:31.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:31.477 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:31.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:31.582 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:31.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:31.718 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:31.736 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:31 smithi099 bash[84576]: cluster 2024-09-13T14:49:30.525716+0000 mgr.y (mgr.26252) 695 : cluster [DBG] pgmap v1170: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 15335/262587 objects degraded (5.840%); 8408/262587 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:31 smithi099 bash[84576]: cluster 2024-09-13T14:49:30.525716+0000 mgr.y (mgr.26252) 695 : cluster [DBG] pgmap v1170: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 15335/262587 objects degraded (5.840%); 8408/262587 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:31 smithi099 bash[84576]: cluster 2024-09-13T14:49:31.320735+0000 mon.a (mon.0) 1688 : cluster [WRN] Health check update: Degraded data redundancy: 15335/262587 objects degraded (5.840%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:31.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:31 smithi099 bash[84576]: cluster 2024-09-13T14:49:31.320735+0000 mon.a (mon.0) 1688 : cluster [WRN] Health check update: Degraded data redundancy: 15335/262587 objects degraded (5.840%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:31.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:31 smithi099 bash[87147]: cluster 2024-09-13T14:49:30.525716+0000 mgr.y (mgr.26252) 695 : cluster [DBG] pgmap v1170: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 15335/262587 objects degraded (5.840%); 8408/262587 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:31.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:31 smithi099 bash[87147]: cluster 2024-09-13T14:49:30.525716+0000 mgr.y (mgr.26252) 695 : cluster [DBG] pgmap v1170: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 15335/262587 objects degraded (5.840%); 8408/262587 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:31.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:31 smithi099 bash[87147]: cluster 2024-09-13T14:49:31.320735+0000 mon.a (mon.0) 1688 : cluster [WRN] Health check update: Degraded data redundancy: 15335/262587 objects degraded (5.840%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:31.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:31 smithi099 bash[87147]: cluster 2024-09-13T14:49:31.320735+0000 mon.a (mon.0) 1688 : cluster [WRN] Health check update: Degraded data redundancy: 15335/262587 objects degraded (5.840%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:31.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:31.847 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:31 smithi177 bash[68594]: cluster 2024-09-13T14:49:30.525716+0000 mgr.y (mgr.26252) 695 : cluster [DBG] pgmap v1170: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 15335/262587 objects degraded (5.840%); 8408/262587 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:31.847 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:31 smithi177 bash[68594]: cluster 2024-09-13T14:49:30.525716+0000 mgr.y (mgr.26252) 695 : cluster [DBG] pgmap v1170: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 15335/262587 objects degraded (5.840%); 8408/262587 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:31.847 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:31 smithi177 bash[68594]: cluster 2024-09-13T14:49:31.320735+0000 mon.a (mon.0) 1688 : cluster [WRN] Health check update: Degraded data redundancy: 15335/262587 objects degraded (5.840%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:31.847 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:31 smithi177 bash[68594]: cluster 2024-09-13T14:49:31.320735+0000 mon.a (mon.0) 1688 : cluster [WRN] Health check update: Degraded data redundancy: 15335/262587 objects degraded (5.840%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:31.867 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:31.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:31.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:31.994 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:31.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:32.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:32.120 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:32.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:32.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:32.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:32.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:32.357 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:32.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:32.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:32.489 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:32.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:32.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:32.620 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:32.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:32.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:32.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:32.770 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:32.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:32.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:32.896 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:33.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:33.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:33.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:33.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:33.266 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:49:33.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:33.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:33.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:33.449 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:33.598 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.655 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:33.760 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:33.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:33.801 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:33.801 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:33.901 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:33.913 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[84576]: cluster 2024-09-13T14:49:32.526356+0000 mgr.y (mgr.26252) 696 : cluster [DBG] pgmap v1171: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 15335/260613 objects degraded (5.884%); 8408/260613 objects misplaced (3.226%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:33.913 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[84576]: cluster 2024-09-13T14:49:32.526356+0000 mgr.y (mgr.26252) 696 : cluster [DBG] pgmap v1171: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 15335/260613 objects degraded (5.884%); 8408/260613 objects misplaced (3.226%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:33.913 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[84576]: audit 2024-09-13T14:49:33.279172+0000 mon.a (mon.0) 1689 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:33.913 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[84576]: audit 2024-09-13T14:49:33.279172+0000 mon.a (mon.0) 1689 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:33.913 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[84576]: audit 2024-09-13T14:49:33.283301+0000 mon.a (mon.0) 1690 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:33.913 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[84576]: audit 2024-09-13T14:49:33.283301+0000 mon.a (mon.0) 1690 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:33.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[87147]: cluster 2024-09-13T14:49:32.526356+0000 mgr.y (mgr.26252) 696 : cluster [DBG] pgmap v1171: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 15335/260613 objects degraded (5.884%); 8408/260613 objects misplaced (3.226%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:33.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[87147]: cluster 2024-09-13T14:49:32.526356+0000 mgr.y (mgr.26252) 696 : cluster [DBG] pgmap v1171: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 15335/260613 objects degraded (5.884%); 8408/260613 objects misplaced (3.226%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:33.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[87147]: audit 2024-09-13T14:49:33.279172+0000 mon.a (mon.0) 1689 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:33.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[87147]: audit 2024-09-13T14:49:33.279172+0000 mon.a (mon.0) 1689 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:33.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[87147]: audit 2024-09-13T14:49:33.283301+0000 mon.a (mon.0) 1690 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:33.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:33 smithi099 bash[87147]: audit 2024-09-13T14:49:33.283301+0000 mon.a (mon.0) 1690 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:33.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:33.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:34.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:34.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:33 smithi177 bash[68594]: cluster 2024-09-13T14:49:32.526356+0000 mgr.y (mgr.26252) 696 : cluster [DBG] pgmap v1171: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 15335/260613 objects degraded (5.884%); 8408/260613 objects misplaced (3.226%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:34.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:33 smithi177 bash[68594]: cluster 2024-09-13T14:49:32.526356+0000 mgr.y (mgr.26252) 696 : cluster [DBG] pgmap v1171: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 15335/260613 objects degraded (5.884%); 8408/260613 objects misplaced (3.226%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:34.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:33 smithi177 bash[68594]: audit 2024-09-13T14:49:33.279172+0000 mon.a (mon.0) 1689 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:34.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:33 smithi177 bash[68594]: audit 2024-09-13T14:49:33.279172+0000 mon.a (mon.0) 1689 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:49:34.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:33 smithi177 bash[68594]: audit 2024-09-13T14:49:33.283301+0000 mon.a (mon.0) 1690 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:34.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:33 smithi177 bash[68594]: audit 2024-09-13T14:49:33.283301+0000 mon.a (mon.0) 1690 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:34.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:34.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:34.153 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:34.187 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:34.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:34.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:34.315 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:34.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:34.418 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:34.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:34.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:34.568 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:34.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:34.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:34.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:34.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:34.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:34.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:34.867 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:34.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:34.970 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:34.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:34.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:35.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:35.119 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:35.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:35.221 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:35.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.251 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:35.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:35.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:35.386 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:35.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:35.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:35.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:35.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:35.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:35.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:35.788 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:35.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:35 smithi099 bash[84576]: cluster 2024-09-13T14:49:34.527101+0000 mgr.y (mgr.26252) 697 : cluster [DBG] pgmap v1172: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 4.2 MiB/s, 66 objects/s recovering 2024-09-13T14:49:35.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:35 smithi099 bash[84576]: cluster 2024-09-13T14:49:34.527101+0000 mgr.y (mgr.26252) 697 : cluster [DBG] pgmap v1172: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 4.2 MiB/s, 66 objects/s recovering 2024-09-13T14:49:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:35 smithi099 bash[87147]: cluster 2024-09-13T14:49:34.527101+0000 mgr.y (mgr.26252) 697 : cluster [DBG] pgmap v1172: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 4.2 MiB/s, 66 objects/s recovering 2024-09-13T14:49:35.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:35 smithi099 bash[87147]: cluster 2024-09-13T14:49:34.527101+0000 mgr.y (mgr.26252) 697 : cluster [DBG] pgmap v1172: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 4.2 MiB/s, 66 objects/s recovering 2024-09-13T14:49:35.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:35.921 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:35.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:36.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:36.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:36.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:36.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:35 smithi177 bash[68594]: cluster 2024-09-13T14:49:34.527101+0000 mgr.y (mgr.26252) 697 : cluster [DBG] pgmap v1172: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 4.2 MiB/s, 66 objects/s recovering 2024-09-13T14:49:36.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:35 smithi177 bash[68594]: cluster 2024-09-13T14:49:34.527101+0000 mgr.y (mgr.26252) 697 : cluster [DBG] pgmap v1172: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 4.2 MiB/s, 66 objects/s recovering 2024-09-13T14:49:36.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:36.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:36.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:36.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:36.312 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:36.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:36.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:36.444 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:36.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:36.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:36.579 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:36.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:36.745 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:36.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:36.847 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:36.871 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:36.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:36.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:36.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:36 smithi177 bash[68594]: cluster 2024-09-13T14:49:36.321587+0000 mon.a (mon.0) 1691 : cluster [WRN] Health check update: Degraded data redundancy: 15387/252348 objects degraded (6.098%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:36.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:36 smithi177 bash[68594]: cluster 2024-09-13T14:49:36.321587+0000 mon.a (mon.0) 1691 : cluster [WRN] Health check update: Degraded data redundancy: 15387/252348 objects degraded (6.098%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:37.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:37.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:37.059 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:36 smithi099 bash[84576]: cluster 2024-09-13T14:49:36.321587+0000 mon.a (mon.0) 1691 : cluster [WRN] Health check update: Degraded data redundancy: 15387/252348 objects degraded (6.098%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:37.060 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:36 smithi099 bash[84576]: cluster 2024-09-13T14:49:36.321587+0000 mon.a (mon.0) 1691 : cluster [WRN] Health check update: Degraded data redundancy: 15387/252348 objects degraded (6.098%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:37.060 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:36 smithi099 bash[87147]: cluster 2024-09-13T14:49:36.321587+0000 mon.a (mon.0) 1691 : cluster [WRN] Health check update: Degraded data redundancy: 15387/252348 objects degraded (6.098%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:37.060 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:36 smithi099 bash[87147]: cluster 2024-09-13T14:49:36.321587+0000 mon.a (mon.0) 1691 : cluster [WRN] Health check update: Degraded data redundancy: 15387/252348 objects degraded (6.098%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:37.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:37.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:37.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:37.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:37.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:49:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:49:37] "GET /metrics HTTP/1.1" 200 36779 "" "Prometheus/2.51.0" 2024-09-13T14:49:37.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:37.410 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:37.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:37.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:37.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:37.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:37.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:37.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:37.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:37.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:37.789 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:37 smithi099 bash[84576]: cluster 2024-09-13T14:49:36.529662+0000 mgr.y (mgr.26252) 698 : cluster [DBG] pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:37.789 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:37 smithi099 bash[84576]: cluster 2024-09-13T14:49:36.529662+0000 mgr.y (mgr.26252) 698 : cluster [DBG] pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:37.790 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:37 smithi099 bash[87147]: cluster 2024-09-13T14:49:36.529662+0000 mgr.y (mgr.26252) 698 : cluster [DBG] pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:37.790 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:37 smithi099 bash[87147]: cluster 2024-09-13T14:49:36.529662+0000 mgr.y (mgr.26252) 698 : cluster [DBG] pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:37.805 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:37.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:37.815 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:37.909 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:37.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:37.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:38.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:38.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:37 smithi177 bash[68594]: cluster 2024-09-13T14:49:36.529662+0000 mgr.y (mgr.26252) 698 : cluster [DBG] pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:38.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:37 smithi177 bash[68594]: cluster 2024-09-13T14:49:36.529662+0000 mgr.y (mgr.26252) 698 : cluster [DBG] pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 15387/252348 objects degraded (6.098%); 8081/252348 objects misplaced (3.202%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:49:38.056 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:38.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:38.161 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:38.188 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:38.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:38.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:38.324 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:38.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:38.433 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:38.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:38.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:38.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:38.590 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:38.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:38.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:38.713 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:38.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:38.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:38.843 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:38.946 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:38.975 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:38.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:39.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:39.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:39.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:39.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:39.239 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:39.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:39.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:39.367 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:39.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:39.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:39.498 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:39.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:39.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:39.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:39.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:39.736 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:39.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:39.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:39.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:39.896 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:39 smithi099 bash[84576]: cluster 2024-09-13T14:49:38.530383+0000 mgr.y (mgr.26252) 699 : cluster [DBG] pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 15527/245442 objects degraded (6.326%); 7658/245442 objects misplaced (3.120%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:49:39.896 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:39 smithi099 bash[84576]: cluster 2024-09-13T14:49:38.530383+0000 mgr.y (mgr.26252) 699 : cluster [DBG] pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 15527/245442 objects degraded (6.326%); 7658/245442 objects misplaced (3.120%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:49:39.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:39 smithi099 bash[87147]: cluster 2024-09-13T14:49:38.530383+0000 mgr.y (mgr.26252) 699 : cluster [DBG] pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 15527/245442 objects degraded (6.326%); 7658/245442 objects misplaced (3.120%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:49:39.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:39 smithi099 bash[87147]: cluster 2024-09-13T14:49:38.530383+0000 mgr.y (mgr.26252) 699 : cluster [DBG] pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 15527/245442 objects degraded (6.326%); 7658/245442 objects misplaced (3.120%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:49:39.899 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:39.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:40.005 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:40.013 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:39 smithi177 bash[68594]: cluster 2024-09-13T14:49:38.530383+0000 mgr.y (mgr.26252) 699 : cluster [DBG] pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 15527/245442 objects degraded (6.326%); 7658/245442 objects misplaced (3.120%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:49:40.013 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:39 smithi177 bash[68594]: cluster 2024-09-13T14:49:38.530383+0000 mgr.y (mgr.26252) 699 : cluster [DBG] pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 15527/245442 objects degraded (6.326%); 7658/245442 objects misplaced (3.120%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:49:40.038 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:40.142 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:40.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:40.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:40.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:40.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:40.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:40.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:49:40 smithi177 bash[61439]: ts=2024-09-13T14:49:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:40.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:40.443 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:40.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:40.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:40.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:40.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:40.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:40.709 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:40.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:40.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:40.834 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:40.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:40.937 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:40.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:41.078 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:41.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:41.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:41.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:41.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:41.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:41.337 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:41.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:41.462 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:41.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:41.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:41.599 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:41.625 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:41.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:41.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:41.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:41.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:41.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:41.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:41.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:42.026 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:42.048 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:49:42.048 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:49:42.048 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:49:42.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:41 smithi099 bash[84576]: cluster 2024-09-13T14:49:40.534399+0000 mgr.y (mgr.26252) 700 : cluster [DBG] pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 15527/242346 objects degraded (6.407%); 7658/242346 objects misplaced (3.160%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:42.062 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:41 smithi099 bash[84576]: cluster 2024-09-13T14:49:40.534399+0000 mgr.y (mgr.26252) 700 : cluster [DBG] pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 15527/242346 objects degraded (6.407%); 7658/242346 objects misplaced (3.160%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:42.062 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:41 smithi099 bash[84576]: cluster 2024-09-13T14:49:41.322416+0000 mon.a (mon.0) 1692 : cluster [WRN] Health check update: Degraded data redundancy: 15527/242346 objects degraded (6.407%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:42.062 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:41 smithi099 bash[84576]: cluster 2024-09-13T14:49:41.322416+0000 mon.a (mon.0) 1692 : cluster [WRN] Health check update: Degraded data redundancy: 15527/242346 objects degraded (6.407%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:42.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:41 smithi099 bash[87147]: cluster 2024-09-13T14:49:40.534399+0000 mgr.y (mgr.26252) 700 : cluster [DBG] pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 15527/242346 objects degraded (6.407%); 7658/242346 objects misplaced (3.160%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:42.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:41 smithi099 bash[87147]: cluster 2024-09-13T14:49:40.534399+0000 mgr.y (mgr.26252) 700 : cluster [DBG] pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 15527/242346 objects degraded (6.407%); 7658/242346 objects misplaced (3.160%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:42.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:41 smithi099 bash[87147]: cluster 2024-09-13T14:49:41.322416+0000 mon.a (mon.0) 1692 : cluster [WRN] Health check update: Degraded data redundancy: 15527/242346 objects degraded (6.407%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:42.063 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:41 smithi099 bash[87147]: cluster 2024-09-13T14:49:41.322416+0000 mon.a (mon.0) 1692 : cluster [WRN] Health check update: Degraded data redundancy: 15527/242346 objects degraded (6.407%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:42.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:41 smithi177 bash[68594]: cluster 2024-09-13T14:49:40.534399+0000 mgr.y (mgr.26252) 700 : cluster [DBG] pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 15527/242346 objects degraded (6.407%); 7658/242346 objects misplaced (3.160%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:42.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:41 smithi177 bash[68594]: cluster 2024-09-13T14:49:40.534399+0000 mgr.y (mgr.26252) 700 : cluster [DBG] pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 15527/242346 objects degraded (6.407%); 7658/242346 objects misplaced (3.160%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:42.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:41 smithi177 bash[68594]: cluster 2024-09-13T14:49:41.322416+0000 mon.a (mon.0) 1692 : cluster [WRN] Health check update: Degraded data redundancy: 15527/242346 objects degraded (6.407%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:42.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:41 smithi177 bash[68594]: cluster 2024-09-13T14:49:41.322416+0000 mon.a (mon.0) 1692 : cluster [WRN] Health check update: Degraded data redundancy: 15527/242346 objects degraded (6.407%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:42.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:42.168 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:42.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:42.271 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:42.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:42.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:42.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:42.451 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:42.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:42.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:42.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:42.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:42.687 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:42.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:42.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:42.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:42.860 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:42.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:42.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:42.993 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:42.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:43.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:43.122 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:43.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:43.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:43.253 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:43.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:43.356 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:43.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:43.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:43.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:43.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:43.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:43.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:43.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:43.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:43.758 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:43.783 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:43.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:43 smithi099 bash[84576]: cluster 2024-09-13T14:49:42.535107+0000 mgr.y (mgr.26252) 701 : cluster [DBG] pgmap v1176: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 15527/240492 objects degraded (6.456%); 7658/240492 objects misplaced (3.184%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:43.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:43 smithi099 bash[84576]: cluster 2024-09-13T14:49:42.535107+0000 mgr.y (mgr.26252) 701 : cluster [DBG] pgmap v1176: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 15527/240492 objects degraded (6.456%); 7658/240492 objects misplaced (3.184%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:43.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:43 smithi099 bash[87147]: cluster 2024-09-13T14:49:42.535107+0000 mgr.y (mgr.26252) 701 : cluster [DBG] pgmap v1176: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 15527/240492 objects degraded (6.456%); 7658/240492 objects misplaced (3.184%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:43.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:43 smithi099 bash[87147]: cluster 2024-09-13T14:49:42.535107+0000 mgr.y (mgr.26252) 701 : cluster [DBG] pgmap v1176: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 15527/240492 objects degraded (6.456%); 7658/240492 objects misplaced (3.184%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:43.885 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:43.894 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:43 smithi177 bash[68594]: cluster 2024-09-13T14:49:42.535107+0000 mgr.y (mgr.26252) 701 : cluster [DBG] pgmap v1176: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 15527/240492 objects degraded (6.456%); 7658/240492 objects misplaced (3.184%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:43.894 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:43 smithi177 bash[68594]: cluster 2024-09-13T14:49:42.535107+0000 mgr.y (mgr.26252) 701 : cluster [DBG] pgmap v1176: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 15527/240492 objects degraded (6.456%); 7658/240492 objects misplaced (3.184%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:43.915 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:44.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.048 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:44.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:44.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:44.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:44.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:44.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:44.320 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:44.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:44.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:44.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:44.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:44.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:44.586 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:44.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:44.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:44.712 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:44.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:44.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:44.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:44.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:44.945 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:44.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:44.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:45.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:45.100 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:45.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:45.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:45.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:45.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:45.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:45.366 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:45.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:45.474 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:45.502 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:45.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:45.609 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:45.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:45.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:45.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:45.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:45.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:45.878 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:45.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:45.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:45.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:45 smithi099 bash[84576]: cluster 2024-09-13T14:49:44.535891+0000 mgr.y (mgr.26252) 702 : cluster [DBG] pgmap v1177: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 788 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:45.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:45 smithi099 bash[84576]: cluster 2024-09-13T14:49:44.535891+0000 mgr.y (mgr.26252) 702 : cluster [DBG] pgmap v1177: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 788 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:45.991 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:45 smithi099 bash[87147]: cluster 2024-09-13T14:49:44.535891+0000 mgr.y (mgr.26252) 702 : cluster [DBG] pgmap v1177: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 788 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:45.992 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:45 smithi099 bash[87147]: cluster 2024-09-13T14:49:44.535891+0000 mgr.y (mgr.26252) 702 : cluster [DBG] pgmap v1177: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 788 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:46.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:46.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:46.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:46.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:45 smithi177 bash[68594]: cluster 2024-09-13T14:49:44.535891+0000 mgr.y (mgr.26252) 702 : cluster [DBG] pgmap v1177: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 788 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:46.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:45 smithi177 bash[68594]: cluster 2024-09-13T14:49:44.535891+0000 mgr.y (mgr.26252) 702 : cluster [DBG] pgmap v1177: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 788 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:46.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:46.164 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:46.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:46.270 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:46.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:46.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:46.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:46.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.425 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:46.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:46.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:46.551 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:46.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:46.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:46.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:46.807 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:46.815 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:46 smithi177 bash[68594]: cluster 2024-09-13T14:49:46.323153+0000 mon.a (mon.0) 1693 : cluster [WRN] Health check update: Degraded data redundancy: 15659/232221 objects degraded (6.743%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:46.815 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:46 smithi177 bash[68594]: cluster 2024-09-13T14:49:46.323153+0000 mon.a (mon.0) 1693 : cluster [WRN] Health check update: Degraded data redundancy: 15659/232221 objects degraded (6.743%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:46.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:46.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:46.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:46.960 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:46.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:47.062 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:47.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:47.140 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:46 smithi099 bash[84576]: cluster 2024-09-13T14:49:46.323153+0000 mon.a (mon.0) 1693 : cluster [WRN] Health check update: Degraded data redundancy: 15659/232221 objects degraded (6.743%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:47.140 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:46 smithi099 bash[84576]: cluster 2024-09-13T14:49:46.323153+0000 mon.a (mon.0) 1693 : cluster [WRN] Health check update: Degraded data redundancy: 15659/232221 objects degraded (6.743%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:47.140 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:46 smithi099 bash[87147]: cluster 2024-09-13T14:49:46.323153+0000 mon.a (mon.0) 1693 : cluster [WRN] Health check update: Degraded data redundancy: 15659/232221 objects degraded (6.743%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:47.140 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:46 smithi099 bash[87147]: cluster 2024-09-13T14:49:46.323153+0000 mon.a (mon.0) 1693 : cluster [WRN] Health check update: Degraded data redundancy: 15659/232221 objects degraded (6.743%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:47.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.236 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:47.342 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:47.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:47.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:47.412 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:49:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:49:47] "GET /metrics HTTP/1.1" 200 36779 "" "Prometheus/2.51.0" 2024-09-13T14:49:47.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.514 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:47.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:47.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:47.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:47.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:48.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:47 smithi099 bash[84576]: cluster 2024-09-13T14:49:46.536403+0000 mgr.y (mgr.26252) 703 : cluster [DBG] pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:48.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:47 smithi099 bash[84576]: cluster 2024-09-13T14:49:46.536403+0000 mgr.y (mgr.26252) 703 : cluster [DBG] pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:48.023 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:47 smithi099 bash[87147]: cluster 2024-09-13T14:49:46.536403+0000 mgr.y (mgr.26252) 703 : cluster [DBG] pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:48.024 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:47 smithi099 bash[87147]: cluster 2024-09-13T14:49:46.536403+0000 mgr.y (mgr.26252) 703 : cluster [DBG] pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:48.049 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:48.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:47 smithi177 bash[68594]: cluster 2024-09-13T14:49:46.536403+0000 mgr.y (mgr.26252) 703 : cluster [DBG] pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:48.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:47 smithi177 bash[68594]: cluster 2024-09-13T14:49:46.536403+0000 mgr.y (mgr.26252) 703 : cluster [DBG] pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 15659/232221 objects degraded (6.743%); 7219/232221 objects misplaced (3.109%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:49:48.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.179 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:48.283 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:48.310 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:48.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:48.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:48.443 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:48.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:48.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.575 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:48.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:48.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:48.851 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:48.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:48.954 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:48.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:48.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:49.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:48 smithi099 bash[84576]: audit 2024-09-13T14:49:48.273713+0000 mon.a (mon.0) 1694 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:49.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:48 smithi099 bash[84576]: audit 2024-09-13T14:49:48.273713+0000 mon.a (mon.0) 1694 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:49.037 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:48 smithi099 bash[87147]: audit 2024-09-13T14:49:48.273713+0000 mon.a (mon.0) 1694 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:49.038 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:48 smithi099 bash[87147]: audit 2024-09-13T14:49:48.273713+0000 mon.a (mon.0) 1694 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:49.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:49.082 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:48 smithi177 bash[68594]: audit 2024-09-13T14:49:48.273713+0000 mon.a (mon.0) 1694 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:49.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:48 smithi177 bash[68594]: audit 2024-09-13T14:49:48.273713+0000 mon.a (mon.0) 1694 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:49.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:49.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:49.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:49.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:49.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:49.352 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:49.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:49.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:49.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:49.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:49.671 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:49.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:49.774 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:49.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:49.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:49.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:49.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:50.037 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:50.057 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:50.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:49 smithi099 bash[84576]: cluster 2024-09-13T14:49:48.537094+0000 mgr.y (mgr.26252) 704 : cluster [DBG] pgmap v1179: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 15785/225117 objects degraded (7.012%); 6832/225117 objects misplaced (3.035%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:49:50.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:49 smithi099 bash[84576]: cluster 2024-09-13T14:49:48.537094+0000 mgr.y (mgr.26252) 704 : cluster [DBG] pgmap v1179: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 15785/225117 objects degraded (7.012%); 6832/225117 objects misplaced (3.035%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:49:50.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:49 smithi099 bash[87147]: cluster 2024-09-13T14:49:48.537094+0000 mgr.y (mgr.26252) 704 : cluster [DBG] pgmap v1179: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 15785/225117 objects degraded (7.012%); 6832/225117 objects misplaced (3.035%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:49:50.070 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:49 smithi099 bash[87147]: cluster 2024-09-13T14:49:48.537094+0000 mgr.y (mgr.26252) 704 : cluster [DBG] pgmap v1179: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 15785/225117 objects degraded (7.012%); 6832/225117 objects misplaced (3.035%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:49:50.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:49 smithi177 bash[68594]: cluster 2024-09-13T14:49:48.537094+0000 mgr.y (mgr.26252) 704 : cluster [DBG] pgmap v1179: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 15785/225117 objects degraded (7.012%); 6832/225117 objects misplaced (3.035%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:49:50.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:49 smithi177 bash[68594]: cluster 2024-09-13T14:49:48.537094+0000 mgr.y (mgr.26252) 704 : cluster [DBG] pgmap v1179: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 15785/225117 objects degraded (7.012%); 6832/225117 objects misplaced (3.035%); 4.4 MiB/s, 70 objects/s recovering 2024-09-13T14:49:50.161 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:50.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:50.292 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:50.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:50.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:50.430 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:50.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:50.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:50.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:50.577 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:50.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:50.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:49:50 smithi177 bash[61439]: ts=2024-09-13T14:49:50.216Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:49:50.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:50.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:50.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:50.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:50.863 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:50.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:50.970 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:50.998 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:51.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:51.103 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:51.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:51.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:51.191 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:49:51.191 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:49:51.192 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2024-09-13T14:49:51.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:49:51.236 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:51.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:51.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:51.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:51.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:51.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:51.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.539 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:51.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:51.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:51.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:51.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:51.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:51.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:51.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:51.926 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:51.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:52.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:52.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:51 smithi177 bash[68594]: cluster 2024-09-13T14:49:50.537725+0000 mgr.y (mgr.26252) 705 : cluster [DBG] pgmap v1180: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 15785/221718 objects degraded (7.119%); 6832/221718 objects misplaced (3.081%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:52.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:51 smithi177 bash[68594]: cluster 2024-09-13T14:49:50.537725+0000 mgr.y (mgr.26252) 705 : cluster [DBG] pgmap v1180: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 15785/221718 objects degraded (7.119%); 6832/221718 objects misplaced (3.081%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:52.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:51 smithi177 bash[68594]: cluster 2024-09-13T14:49:51.329952+0000 mon.a (mon.0) 1695 : cluster [WRN] Health check update: Degraded data redundancy: 15785/221718 objects degraded (7.119%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:52.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:51 smithi177 bash[68594]: cluster 2024-09-13T14:49:51.329952+0000 mon.a (mon.0) 1695 : cluster [WRN] Health check update: Degraded data redundancy: 15785/221718 objects degraded (7.119%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:52.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:52.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:52.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:52.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:52.283 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:52.321 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:52.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:52.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:52.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:52.550 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:52.589 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:52.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:52.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.726 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:52.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:52.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:52.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:52.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:52.922 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:52 smithi099 bash[84576]: cluster 2024-09-13T14:49:50.537725+0000 mgr.y (mgr.26252) 705 : cluster [DBG] pgmap v1180: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 15785/221718 objects degraded (7.119%); 6832/221718 objects misplaced (3.081%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:52.923 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:52 smithi099 bash[84576]: cluster 2024-09-13T14:49:50.537725+0000 mgr.y (mgr.26252) 705 : cluster [DBG] pgmap v1180: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 15785/221718 objects degraded (7.119%); 6832/221718 objects misplaced (3.081%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:52.923 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:52 smithi099 bash[84576]: cluster 2024-09-13T14:49:51.329952+0000 mon.a (mon.0) 1695 : cluster [WRN] Health check update: Degraded data redundancy: 15785/221718 objects degraded (7.119%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:52.923 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:52 smithi099 bash[84576]: cluster 2024-09-13T14:49:51.329952+0000 mon.a (mon.0) 1695 : cluster [WRN] Health check update: Degraded data redundancy: 15785/221718 objects degraded (7.119%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:52.961 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:52.987 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:52.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:53.089 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:53.124 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:53.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:53.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:53.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:53.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:53.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:52 smithi099 bash[87147]: cluster 2024-09-13T14:49:50.537725+0000 mgr.y (mgr.26252) 705 : cluster [DBG] pgmap v1180: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 15785/221718 objects degraded (7.119%); 6832/221718 objects misplaced (3.081%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:53.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:52 smithi099 bash[87147]: cluster 2024-09-13T14:49:50.537725+0000 mgr.y (mgr.26252) 705 : cluster [DBG] pgmap v1180: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 15785/221718 objects degraded (7.119%); 6832/221718 objects misplaced (3.081%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:53.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:52 smithi099 bash[87147]: cluster 2024-09-13T14:49:51.329952+0000 mon.a (mon.0) 1695 : cluster [WRN] Health check update: Degraded data redundancy: 15785/221718 objects degraded (7.119%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:53.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:52 smithi099 bash[87147]: cluster 2024-09-13T14:49:51.329952+0000 mon.a (mon.0) 1695 : cluster [WRN] Health check update: Degraded data redundancy: 15785/221718 objects degraded (7.119%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T14:49:53.369 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:53.394 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:53.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:53.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:53.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:53.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:53.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:53.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:53.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:53.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:53.787 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:53.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:53.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:53.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:54.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:54.050 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:53 smithi177 bash[68594]: cluster 2024-09-13T14:49:52.538333+0000 mgr.y (mgr.26252) 706 : cluster [DBG] pgmap v1181: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 15785/219987 objects degraded (7.175%); 6832/219987 objects misplaced (3.106%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:54.050 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:53 smithi177 bash[68594]: cluster 2024-09-13T14:49:52.538333+0000 mgr.y (mgr.26252) 706 : cluster [DBG] pgmap v1181: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 15785/219987 objects degraded (7.175%); 6832/219987 objects misplaced (3.106%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:54.060 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:54.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:54.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:54.164 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:53 smithi099 bash[84576]: cluster 2024-09-13T14:49:52.538333+0000 mgr.y (mgr.26252) 706 : cluster [DBG] pgmap v1181: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 15785/219987 objects degraded (7.175%); 6832/219987 objects misplaced (3.106%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:54.164 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:53 smithi099 bash[84576]: cluster 2024-09-13T14:49:52.538333+0000 mgr.y (mgr.26252) 706 : cluster [DBG] pgmap v1181: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 15785/219987 objects degraded (7.175%); 6832/219987 objects misplaced (3.106%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:54.165 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:53 smithi099 bash[87147]: cluster 2024-09-13T14:49:52.538333+0000 mgr.y (mgr.26252) 706 : cluster [DBG] pgmap v1181: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 15785/219987 objects degraded (7.175%); 6832/219987 objects misplaced (3.106%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:54.165 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:53 smithi099 bash[87147]: cluster 2024-09-13T14:49:52.538333+0000 mgr.y (mgr.26252) 706 : cluster [DBG] pgmap v1181: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 15785/219987 objects degraded (7.175%); 6832/219987 objects misplaced (3.106%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:49:54.194 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:54.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:54.328 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:54.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:54.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:54.462 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:54.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:54.566 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:54.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:54.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:54.711 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:54.750 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:54.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:54.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:54.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:54.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:55.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:55.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:55.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:55.158 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:55.260 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:55.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:55.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:55.388 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:55.420 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:55.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:55.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:55.545 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:55.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:55.681 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:55.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:55.785 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:55.813 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:55.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:55.926 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:55 smithi099 bash[84576]: cluster 2024-09-13T14:49:54.539176+0000 mgr.y (mgr.26252) 707 : cluster [DBG] pgmap v1182: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:55.927 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:55 smithi099 bash[84576]: cluster 2024-09-13T14:49:54.539176+0000 mgr.y (mgr.26252) 707 : cluster [DBG] pgmap v1182: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:55.927 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:55 smithi099 bash[87147]: cluster 2024-09-13T14:49:54.539176+0000 mgr.y (mgr.26252) 707 : cluster [DBG] pgmap v1182: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:55.927 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:55 smithi099 bash[87147]: cluster 2024-09-13T14:49:54.539176+0000 mgr.y (mgr.26252) 707 : cluster [DBG] pgmap v1182: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:55.947 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:56.001 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:56.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:56.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:56.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:56.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:55 smithi177 bash[68594]: cluster 2024-09-13T14:49:54.539176+0000 mgr.y (mgr.26252) 707 : cluster [DBG] pgmap v1182: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:56.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:55 smithi177 bash[68594]: cluster 2024-09-13T14:49:54.539176+0000 mgr.y (mgr.26252) 707 : cluster [DBG] pgmap v1182: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:49:56.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:56.197 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:56.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:56.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:56.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:56.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:56.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:56.461 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:56.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:56.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:56.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:56.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:56.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:56.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:56.828 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:56.837 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:56 smithi177 bash[68594]: cluster 2024-09-13T14:49:56.331048+0000 mon.a (mon.0) 1696 : cluster [WRN] Health check update: Degraded data redundancy: 15869/211914 objects degraded (7.488%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:56.837 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:56 smithi177 bash[68594]: cluster 2024-09-13T14:49:56.331048+0000 mon.a (mon.0) 1696 : cluster [WRN] Health check update: Degraded data redundancy: 15869/211914 objects degraded (7.488%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:56.851 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:56.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:56.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:56.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:56.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:57.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:57.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:57.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:57.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:57.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:57.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:56 smithi099 bash[84576]: cluster 2024-09-13T14:49:56.331048+0000 mon.a (mon.0) 1696 : cluster [WRN] Health check update: Degraded data redundancy: 15869/211914 objects degraded (7.488%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:57.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:56 smithi099 bash[84576]: cluster 2024-09-13T14:49:56.331048+0000 mon.a (mon.0) 1696 : cluster [WRN] Health check update: Degraded data redundancy: 15869/211914 objects degraded (7.488%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:57.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:56 smithi099 bash[87147]: cluster 2024-09-13T14:49:56.331048+0000 mon.a (mon.0) 1696 : cluster [WRN] Health check update: Degraded data redundancy: 15869/211914 objects degraded (7.488%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:57.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:56 smithi099 bash[87147]: cluster 2024-09-13T14:49:56.331048+0000 mon.a (mon.0) 1696 : cluster [WRN] Health check update: Degraded data redundancy: 15869/211914 objects degraded (7.488%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:57.332 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:57.356 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:57.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:57.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:57.489 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:57.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:57.578 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:49:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:49:57] "GET /metrics HTTP/1.1" 200 36780 "" "Prometheus/2.51.0" 2024-09-13T14:49:57.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:57.615 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:57.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:57.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:57.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:57.863 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:57.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:57.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:57.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:58.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:58.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:58.028 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:57 smithi099 bash[84576]: cluster 2024-09-13T14:49:56.539684+0000 mgr.y (mgr.26252) 708 : cluster [DBG] pgmap v1183: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:58.028 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:57 smithi099 bash[84576]: cluster 2024-09-13T14:49:56.539684+0000 mgr.y (mgr.26252) 708 : cluster [DBG] pgmap v1183: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:58.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:57 smithi177 bash[68594]: cluster 2024-09-13T14:49:56.539684+0000 mgr.y (mgr.26252) 708 : cluster [DBG] pgmap v1183: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:58.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:57 smithi177 bash[68594]: cluster 2024-09-13T14:49:56.539684+0000 mgr.y (mgr.26252) 708 : cluster [DBG] pgmap v1183: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:58.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:58.150 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:58.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:58.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:58 smithi099 bash[87147]: cluster 2024-09-13T14:49:56.539684+0000 mgr.y (mgr.26252) 708 : cluster [DBG] pgmap v1183: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:58.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:58 smithi099 bash[87147]: cluster 2024-09-13T14:49:56.539684+0000 mgr.y (mgr.26252) 708 : cluster [DBG] pgmap v1183: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 15869/211914 objects degraded (7.488%); 6445/211914 objects misplaced (3.041%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:49:58.254 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:58.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:58.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:58.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:58.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:58.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:58.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:58.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:58.647 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:58.672 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:58.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:58.778 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:58.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:58.904 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:58.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:58.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:59.046 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:59.070 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:59.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:59.175 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:59.201 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:59.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:59.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:59.333 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:59.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:59.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:59.467 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:59.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:59.569 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:59.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:59.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:59.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:59.728 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:49:59.728 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:59.729 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:59.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:59.771 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:49:59.771 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:59.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:59.854 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:49:59.855 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:59.864 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:59.892 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:59.892 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:49:59.924 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:49:59.925 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:59.957 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:49:59.957 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:59.968 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:59.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:59.991 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:49:59.991 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:59.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:00.025 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:50:00.025 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:00.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:00.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:59 smithi177 bash[68594]: cluster 2024-09-13T14:49:58.540560+0000 mgr.y (mgr.26252) 709 : cluster [DBG] pgmap v1184: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 15985/205626 objects degraded (7.774%); 6015/205626 objects misplaced (2.925%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:50:00.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:59 smithi177 bash[68594]: cluster 2024-09-13T14:49:58.540560+0000 mgr.y (mgr.26252) 709 : cluster [DBG] pgmap v1184: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 15985/205626 objects degraded (7.774%); 6015/205626 objects misplaced (2.925%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:50:00.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:59 smithi177 bash[68594]: audit 2024-09-13T14:49:59.681303+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.99:0/816816938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:00.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:49:59 smithi177 bash[68594]: audit 2024-09-13T14:49:59.681303+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.99:0/816816938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:00.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:00.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:00.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:59 smithi099 bash[84576]: cluster 2024-09-13T14:49:58.540560+0000 mgr.y (mgr.26252) 709 : cluster [DBG] pgmap v1184: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 15985/205626 objects degraded (7.774%); 6015/205626 objects misplaced (2.925%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:50:00.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:59 smithi099 bash[84576]: cluster 2024-09-13T14:49:58.540560+0000 mgr.y (mgr.26252) 709 : cluster [DBG] pgmap v1184: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 15985/205626 objects degraded (7.774%); 6015/205626 objects misplaced (2.925%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:50:00.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:59 smithi099 bash[84576]: audit 2024-09-13T14:49:59.681303+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.99:0/816816938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:00.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:49:59 smithi099 bash[84576]: audit 2024-09-13T14:49:59.681303+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.99:0/816816938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:00.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:59 smithi099 bash[87147]: cluster 2024-09-13T14:49:58.540560+0000 mgr.y (mgr.26252) 709 : cluster [DBG] pgmap v1184: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 15985/205626 objects degraded (7.774%); 6015/205626 objects misplaced (2.925%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:50:00.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:59 smithi099 bash[87147]: cluster 2024-09-13T14:49:58.540560+0000 mgr.y (mgr.26252) 709 : cluster [DBG] pgmap v1184: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 15985/205626 objects degraded (7.774%); 6015/205626 objects misplaced (2.925%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:50:00.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:59 smithi099 bash[87147]: audit 2024-09-13T14:49:59.681303+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.99:0/816816938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:00.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:49:59 smithi099 bash[87147]: audit 2024-09-13T14:49:59.681303+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.99:0/816816938' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:00.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:00.247 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[61439]: ts=2024-09-13T14:50:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:00.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:00.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:00.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:00.388 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:00.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:00.493 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:00.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:00.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:00.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:00.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:00.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:00.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:00.788 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:00.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:00.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:00.902 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000078+0000 mon.a (mon.0) 1697 : cluster [WRN] Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:00.902 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000078+0000 mon.a (mon.0) 1697 : cluster [WRN] Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:00.902 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000116+0000 mon.a (mon.0) 1698 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:00.902 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000116+0000 mon.a (mon.0) 1698 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:00.902 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000139+0000 mon.a (mon.0) 1699 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:00.902 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000139+0000 mon.a (mon.0) 1699 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:00.902 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000152+0000 mon.a (mon.0) 1700 : cluster [WRN] pg 41.5 is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2024-09-13T14:50:00.902 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000152+0000 mon.a (mon.0) 1700 : cluster [WRN] pg 41.5 is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2024-09-13T14:50:00.902 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000165+0000 mon.a (mon.0) 1701 : cluster [WRN] pg 41.b is stuck undersized for 67s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2024-09-13T14:50:00.903 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000165+0000 mon.a (mon.0) 1701 : cluster [WRN] pg 41.b is stuck undersized for 67s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2024-09-13T14:50:00.903 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000179+0000 mon.a (mon.0) 1702 : cluster [WRN] pg 41.c is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:00.903 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000179+0000 mon.a (mon.0) 1702 : cluster [WRN] pg 41.c is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:00.903 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000190+0000 mon.a (mon.0) 1703 : cluster [WRN] pg 41.d is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:00.903 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000190+0000 mon.a (mon.0) 1703 : cluster [WRN] pg 41.d is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:00.903 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000203+0000 mon.a (mon.0) 1704 : cluster [WRN] pg 41.f is stuck undersized for 67s, current state active+recovering+undersized+remapped, last acting [2,3] 2024-09-13T14:50:00.903 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:00 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.000203+0000 mon.a (mon.0) 1704 : cluster [WRN] pg 41.f is stuck undersized for 67s, current state active+recovering+undersized+remapped, last acting [2,3] 2024-09-13T14:50:00.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:00.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:01.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:01.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:01.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000078+0000 mon.a (mon.0) 1697 : cluster [WRN] Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:01.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000078+0000 mon.a (mon.0) 1697 : cluster [WRN] Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:01.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000116+0000 mon.a (mon.0) 1698 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:01.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000116+0000 mon.a (mon.0) 1698 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:01.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000139+0000 mon.a (mon.0) 1699 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000139+0000 mon.a (mon.0) 1699 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000152+0000 mon.a (mon.0) 1700 : cluster [WRN] pg 41.5 is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000152+0000 mon.a (mon.0) 1700 : cluster [WRN] pg 41.5 is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000165+0000 mon.a (mon.0) 1701 : cluster [WRN] pg 41.b is stuck undersized for 67s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000165+0000 mon.a (mon.0) 1701 : cluster [WRN] pg 41.b is stuck undersized for 67s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000179+0000 mon.a (mon.0) 1702 : cluster [WRN] pg 41.c is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000179+0000 mon.a (mon.0) 1702 : cluster [WRN] pg 41.c is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000190+0000 mon.a (mon.0) 1703 : cluster [WRN] pg 41.d is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000190+0000 mon.a (mon.0) 1703 : cluster [WRN] pg 41.d is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000203+0000 mon.a (mon.0) 1704 : cluster [WRN] pg 41.f is stuck undersized for 67s, current state active+recovering+undersized+remapped, last acting [2,3] 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.000203+0000 mon.a (mon.0) 1704 : cluster [WRN] pg 41.f is stuck undersized for 67s, current state active+recovering+undersized+remapped, last acting [2,3] 2024-09-13T14:50:01.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000078+0000 mon.a (mon.0) 1697 : cluster [WRN] Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000078+0000 mon.a (mon.0) 1697 : cluster [WRN] Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000116+0000 mon.a (mon.0) 1698 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000116+0000 mon.a (mon.0) 1698 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000139+0000 mon.a (mon.0) 1699 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000139+0000 mon.a (mon.0) 1699 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 15985/205626 objects degraded (7.774%), 4 pgs degraded, 5 pgs undersized 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000152+0000 mon.a (mon.0) 1700 : cluster [WRN] pg 41.5 is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000152+0000 mon.a (mon.0) 1700 : cluster [WRN] pg 41.5 is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000165+0000 mon.a (mon.0) 1701 : cluster [WRN] pg 41.b is stuck undersized for 67s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000165+0000 mon.a (mon.0) 1701 : cluster [WRN] pg 41.b is stuck undersized for 67s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000179+0000 mon.a (mon.0) 1702 : cluster [WRN] pg 41.c is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000179+0000 mon.a (mon.0) 1702 : cluster [WRN] pg 41.c is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000190+0000 mon.a (mon.0) 1703 : cluster [WRN] pg 41.d is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000190+0000 mon.a (mon.0) 1703 : cluster [WRN] pg 41.d is stuck undersized for 67s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-09-13T14:50:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000203+0000 mon.a (mon.0) 1704 : cluster [WRN] pg 41.f is stuck undersized for 67s, current state active+recovering+undersized+remapped, last acting [2,3] 2024-09-13T14:50:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:00 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.000203+0000 mon.a (mon.0) 1704 : cluster [WRN] pg 41.f is stuck undersized for 67s, current state active+recovering+undersized+remapped, last acting [2,3] 2024-09-13T14:50:01.152 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:01.172 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:01.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:01.304 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:01.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:01.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:01.445 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:01.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:01.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:01.583 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:01.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:01.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:01.717 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:01.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:01.823 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:01.854 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:01.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:01.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:01.984 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:01.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:02.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:02.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:01 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.541115+0000 mgr.y (mgr.26252) 710 : cluster [DBG] pgmap v1185: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 15985/202695 objects degraded (7.886%); 6015/202695 objects misplaced (2.968%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:02.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:01 smithi177 bash[68594]: cluster 2024-09-13T14:50:00.541115+0000 mgr.y (mgr.26252) 710 : cluster [DBG] pgmap v1185: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 15985/202695 objects degraded (7.886%); 6015/202695 objects misplaced (2.968%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:02.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:01 smithi177 bash[68594]: cluster 2024-09-13T14:50:01.331857+0000 mon.a (mon.0) 1705 : cluster [WRN] Health check update: Degraded data redundancy: 15985/202695 objects degraded (7.886%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:02.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:01 smithi177 bash[68594]: cluster 2024-09-13T14:50:01.331857+0000 mon.a (mon.0) 1705 : cluster [WRN] Health check update: Degraded data redundancy: 15985/202695 objects degraded (7.886%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:02.117 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:02.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:02.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:01 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.541115+0000 mgr.y (mgr.26252) 710 : cluster [DBG] pgmap v1185: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 15985/202695 objects degraded (7.886%); 6015/202695 objects misplaced (2.968%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:02.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:01 smithi099 bash[84576]: cluster 2024-09-13T14:50:00.541115+0000 mgr.y (mgr.26252) 710 : cluster [DBG] pgmap v1185: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 15985/202695 objects degraded (7.886%); 6015/202695 objects misplaced (2.968%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:02.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:01 smithi099 bash[84576]: cluster 2024-09-13T14:50:01.331857+0000 mon.a (mon.0) 1705 : cluster [WRN] Health check update: Degraded data redundancy: 15985/202695 objects degraded (7.886%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:02.203 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:01 smithi099 bash[84576]: cluster 2024-09-13T14:50:01.331857+0000 mon.a (mon.0) 1705 : cluster [WRN] Health check update: Degraded data redundancy: 15985/202695 objects degraded (7.886%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:02.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:01 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.541115+0000 mgr.y (mgr.26252) 710 : cluster [DBG] pgmap v1185: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 15985/202695 objects degraded (7.886%); 6015/202695 objects misplaced (2.968%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:02.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:01 smithi099 bash[87147]: cluster 2024-09-13T14:50:00.541115+0000 mgr.y (mgr.26252) 710 : cluster [DBG] pgmap v1185: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 15985/202695 objects degraded (7.886%); 6015/202695 objects misplaced (2.968%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:02.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:01 smithi099 bash[87147]: cluster 2024-09-13T14:50:01.331857+0000 mon.a (mon.0) 1705 : cluster [WRN] Health check update: Degraded data redundancy: 15985/202695 objects degraded (7.886%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:02.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:01 smithi099 bash[87147]: cluster 2024-09-13T14:50:01.331857+0000 mon.a (mon.0) 1705 : cluster [WRN] Health check update: Degraded data redundancy: 15985/202695 objects degraded (7.886%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:02.221 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:02.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:02.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:02.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:02.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:02.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:02.491 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:02.513 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:02.616 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:02.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:02.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:02.752 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:02.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:02.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:02.921 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:02.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:03.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:03.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:03.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:03.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:03.210 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:03.315 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:03.342 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:03.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:03.449 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:03.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:03.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:03.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:03.590 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:03.693 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:03.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:03.834 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:03.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:03.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:03.960 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:03.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:03 smithi177 bash[68594]: cluster 2024-09-13T14:50:02.541725+0000 mgr.y (mgr.26252) 711 : cluster [DBG] pgmap v1186: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 15985/200973 objects degraded (7.954%); 6015/200973 objects misplaced (2.993%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:03.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:03 smithi177 bash[68594]: cluster 2024-09-13T14:50:02.541725+0000 mgr.y (mgr.26252) 711 : cluster [DBG] pgmap v1186: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 15985/200973 objects degraded (7.954%); 6015/200973 objects misplaced (2.993%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:03.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:03 smithi177 bash[68594]: audit 2024-09-13T14:50:03.273873+0000 mon.a (mon.0) 1706 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:03.968 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:03 smithi177 bash[68594]: audit 2024-09-13T14:50:03.273873+0000 mon.a (mon.0) 1706 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:03.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:04.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:03 smithi099 bash[84576]: cluster 2024-09-13T14:50:02.541725+0000 mgr.y (mgr.26252) 711 : cluster [DBG] pgmap v1186: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 15985/200973 objects degraded (7.954%); 6015/200973 objects misplaced (2.993%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:04.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:03 smithi099 bash[84576]: cluster 2024-09-13T14:50:02.541725+0000 mgr.y (mgr.26252) 711 : cluster [DBG] pgmap v1186: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 15985/200973 objects degraded (7.954%); 6015/200973 objects misplaced (2.993%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:04.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:03 smithi099 bash[84576]: audit 2024-09-13T14:50:03.273873+0000 mon.a (mon.0) 1706 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:04.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:03 smithi099 bash[84576]: audit 2024-09-13T14:50:03.273873+0000 mon.a (mon.0) 1706 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:04.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:03 smithi099 bash[87147]: cluster 2024-09-13T14:50:02.541725+0000 mgr.y (mgr.26252) 711 : cluster [DBG] pgmap v1186: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 15985/200973 objects degraded (7.954%); 6015/200973 objects misplaced (2.993%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:04.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:03 smithi099 bash[87147]: cluster 2024-09-13T14:50:02.541725+0000 mgr.y (mgr.26252) 711 : cluster [DBG] pgmap v1186: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 15985/200973 objects degraded (7.954%); 6015/200973 objects misplaced (2.993%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:50:04.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:03 smithi099 bash[87147]: audit 2024-09-13T14:50:03.273873+0000 mon.a (mon.0) 1706 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:04.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:03 smithi099 bash[87147]: audit 2024-09-13T14:50:03.273873+0000 mon.a (mon.0) 1706 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:04.080 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:04.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:04.212 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:04.240 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:04.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:04.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:04.368 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:04.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:04.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:04.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:04.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:04.608 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:04.632 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:04.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:04.737 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:04.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:04.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:04.876 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:50:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T14:50:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:132544 41.5 41:aadc68a9:::benchmark_data_smithi099_126887_object34756:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e980)", 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:04.862023+0000", 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.016814600999999998, 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01684335, 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 132544 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.862023+0000", 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.862023+0000", 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.862024+0000", 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.44e-06 2024-09-13T14:50:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:50:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.862030+0000", 2024-09-13T14:50:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3360000000000002e-06 2024-09-13T14:50:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:50:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.862045+0000", 2024-09-13T14:50:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.556e-05 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.862054+0000", 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4999999999999999e-06 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.863983+0000", 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019288109999999999 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.864145+0000", 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016253099999999999 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.864306+0000", 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016075900000000001 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.876607+0000", 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012301546999999999 2024-09-13T14:50:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:132548 41.5 41:a1e06abb:::benchmark_data_smithi099_126887_object34760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e980)", 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:04.864882+0000", 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.013955726999999999, 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014056604, 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:50:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:50:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:50:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 132548 2024-09-13T14:50:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:50:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:50:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.864882+0000", 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.864882+0000", 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.864883+0000", 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.567e-06 2024-09-13T14:50:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.864889+0000", 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0410000000000003e-06 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.864903+0000", 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3801000000000001e-05 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:50:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.864911+0000", 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9019999999999999e-06 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.878223+0000", 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013311536000000001 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:132554 41.d e980/976 41:bf4fd24e:::benchmark_data_smithi099_126887_object34766:head v 980'8202, mlcod=0'0)", 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:04.872872+0000", 2024-09-13T14:50:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.005965049, 2024-09-13T14:50:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0061167870000000003, 2024-09-13T14:50:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:50:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:50:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T14:50:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:50:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:50:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.872872+0000", 2024-09-13T14:50:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:50:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.872872+0000", 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.872874+0000", 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1650000000000001e-06 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:50:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.872882+0000", 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9419999999999995e-06 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.872899+0000", 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7927999999999998e-05 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.872908+0000", 2024-09-13T14:50:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5380000000000002e-06 2024-09-13T14:50:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.877768+0000", 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004859515 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.877820+0000", 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2858000000000001e-05 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:50:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:132537 41.d e980/976 41:b01227ac:::benchmark_data_smithi099_126887_object34749:head v 980'8201, mlcod=0'0)", 2024-09-13T14:50:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:04.854039+0000", 2024-09-13T14:50:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.024798293999999999, 2024-09-13T14:50:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.024996675999999999, 2024-09-13T14:50:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:50:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:50:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T14:50:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:50:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:50:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854039+0000", 2024-09-13T14:50:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:50:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:50:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854039+0000", 2024-09-13T14:50:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854040+0000", 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7400000000000003e-07 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854046+0000", 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.319e-06 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:50:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854061+0000", 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5257e-05 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854082+0000", 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0585e-05 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854367+0000", 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00028531599999999998 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854388+0000", 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0812999999999999e-05 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:132534 41.f 41:fa7ef19a:::benchmark_data_smithi099_126887_object34746:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e980)", 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:04.854154+0000", 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.024683017000000002, 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.024933012000000001, 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:50:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 132534 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854154+0000", 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:50:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854154+0000", 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854157+0000", 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1880000000000001e-06 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854160+0000", 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8279999999999996e-06 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854171+0000", 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0491e-05 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.854177+0000", 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2419999999999997e-06 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.855021+0000", 2024-09-13T14:50:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00084333800000000001 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.855161+0000", 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014038200000000001 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.855285+0000", 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012425399999999999 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:50:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.861868+0000", 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006582577 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:04.865898+0000", 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040303600000000002 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 5 2024-09-13T14:50:04.902 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:50:04.903 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:04.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:04.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:05.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:05.056 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:05.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:05.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:05.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:05.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:05.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:05.314 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:05.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:05.418 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:05.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:05.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:05.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:05.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:05.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:05.675 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:05.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:05.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:05.831 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:05.841 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T14:50:05.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:05.948 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[84576]: cluster 2024-09-13T14:50:04.542916+0000 mgr.y (mgr.26252) 712 : cluster [DBG] pgmap v1187: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T14:50:05.948 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[84576]: cluster 2024-09-13T14:50:04.542916+0000 mgr.y (mgr.26252) 712 : cluster [DBG] pgmap v1187: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T14:50:05.948 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[84576]: audit 2024-09-13T14:50:05.190142+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.99:0/2918851204' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:05.948 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[84576]: audit 2024-09-13T14:50:05.190142+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.99:0/2918851204' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:05.948 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[84576]: audit 2024-09-13T14:50:05.195072+0000 mon.a (mon.0) 1707 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:05.949 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[84576]: audit 2024-09-13T14:50:05.195072+0000 mon.a (mon.0) 1707 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:05.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[87147]: cluster 2024-09-13T14:50:04.542916+0000 mgr.y (mgr.26252) 712 : cluster [DBG] pgmap v1187: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T14:50:05.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[87147]: cluster 2024-09-13T14:50:04.542916+0000 mgr.y (mgr.26252) 712 : cluster [DBG] pgmap v1187: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T14:50:05.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[87147]: audit 2024-09-13T14:50:05.190142+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.99:0/2918851204' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:05.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[87147]: audit 2024-09-13T14:50:05.190142+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.99:0/2918851204' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:05.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[87147]: audit 2024-09-13T14:50:05.195072+0000 mon.a (mon.0) 1707 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:05.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:05 smithi099 bash[87147]: audit 2024-09-13T14:50:05.195072+0000 mon.a (mon.0) 1707 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:05.968 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:05.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:06.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:06.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:05 smithi177 bash[68594]: cluster 2024-09-13T14:50:04.542916+0000 mgr.y (mgr.26252) 712 : cluster [DBG] pgmap v1187: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T14:50:06.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:05 smithi177 bash[68594]: cluster 2024-09-13T14:50:04.542916+0000 mgr.y (mgr.26252) 712 : cluster [DBG] pgmap v1187: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 762 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T14:50:06.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:05 smithi177 bash[68594]: audit 2024-09-13T14:50:05.190142+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.99:0/2918851204' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:06.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:05 smithi177 bash[68594]: audit 2024-09-13T14:50:05.190142+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.99:0/2918851204' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:06.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:05 smithi177 bash[68594]: audit 2024-09-13T14:50:05.195072+0000 mon.a (mon.0) 1707 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:06.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:05 smithi177 bash[68594]: audit 2024-09-13T14:50:05.195072+0000 mon.a (mon.0) 1707 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:06.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.106 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:06.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:06.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:06.230 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:06.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:06.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:06.373 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:06.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:06.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:50:06.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:06.507 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:06.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:06.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:06.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.639 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:06.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:06.740 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:06.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:06.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:06.877 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:06.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:06.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:07.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:07.041 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:06 smithi177 bash[68594]: cluster 2024-09-13T14:50:05.831489+0000 mon.a (mon.0) 1708 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:07.041 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:06 smithi177 bash[68594]: cluster 2024-09-13T14:50:05.831489+0000 mon.a (mon.0) 1708 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:07.041 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:06 smithi177 bash[68594]: audit 2024-09-13T14:50:05.837560+0000 mon.a (mon.0) 1709 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:07.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:06 smithi177 bash[68594]: audit 2024-09-13T14:50:05.837560+0000 mon.a (mon.0) 1709 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:07.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:06 smithi177 bash[68594]: cluster 2024-09-13T14:50:05.841040+0000 mon.a (mon.0) 1710 : cluster [DBG] osdmap e981: 8 total, 8 up, 7 in 2024-09-13T14:50:07.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:06 smithi177 bash[68594]: cluster 2024-09-13T14:50:05.841040+0000 mon.a (mon.0) 1710 : cluster [DBG] osdmap e981: 8 total, 8 up, 7 in 2024-09-13T14:50:07.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:06 smithi177 bash[68594]: cluster 2024-09-13T14:50:06.332717+0000 mon.a (mon.0) 1711 : cluster [WRN] Health check update: Degraded data redundancy: 16105/192510 objects degraded (8.366%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:07.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:06 smithi177 bash[68594]: cluster 2024-09-13T14:50:06.332717+0000 mon.a (mon.0) 1711 : cluster [WRN] Health check update: Degraded data redundancy: 16105/192510 objects degraded (8.366%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:07.058 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:07.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:07.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[84576]: cluster 2024-09-13T14:50:05.831489+0000 mon.a (mon.0) 1708 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:07.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[84576]: cluster 2024-09-13T14:50:05.831489+0000 mon.a (mon.0) 1708 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:07.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[84576]: audit 2024-09-13T14:50:05.837560+0000 mon.a (mon.0) 1709 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:07.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[84576]: audit 2024-09-13T14:50:05.837560+0000 mon.a (mon.0) 1709 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:07.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[84576]: cluster 2024-09-13T14:50:05.841040+0000 mon.a (mon.0) 1710 : cluster [DBG] osdmap e981: 8 total, 8 up, 7 in 2024-09-13T14:50:07.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[84576]: cluster 2024-09-13T14:50:05.841040+0000 mon.a (mon.0) 1710 : cluster [DBG] osdmap e981: 8 total, 8 up, 7 in 2024-09-13T14:50:07.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[84576]: cluster 2024-09-13T14:50:06.332717+0000 mon.a (mon.0) 1711 : cluster [WRN] Health check update: Degraded data redundancy: 16105/192510 objects degraded (8.366%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[84576]: cluster 2024-09-13T14:50:06.332717+0000 mon.a (mon.0) 1711 : cluster [WRN] Health check update: Degraded data redundancy: 16105/192510 objects degraded (8.366%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[87147]: cluster 2024-09-13T14:50:05.831489+0000 mon.a (mon.0) 1708 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[87147]: cluster 2024-09-13T14:50:05.831489+0000 mon.a (mon.0) 1708 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[87147]: audit 2024-09-13T14:50:05.837560+0000 mon.a (mon.0) 1709 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[87147]: audit 2024-09-13T14:50:05.837560+0000 mon.a (mon.0) 1709 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[87147]: cluster 2024-09-13T14:50:05.841040+0000 mon.a (mon.0) 1710 : cluster [DBG] osdmap e981: 8 total, 8 up, 7 in 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[87147]: cluster 2024-09-13T14:50:05.841040+0000 mon.a (mon.0) 1710 : cluster [DBG] osdmap e981: 8 total, 8 up, 7 in 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[87147]: cluster 2024-09-13T14:50:06.332717+0000 mon.a (mon.0) 1711 : cluster [WRN] Health check update: Degraded data redundancy: 16105/192510 objects degraded (8.366%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:07.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:06 smithi099 bash[87147]: cluster 2024-09-13T14:50:06.332717+0000 mon.a (mon.0) 1711 : cluster [WRN] Health check update: Degraded data redundancy: 16105/192510 objects degraded (8.366%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:07.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:07.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:07.289 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:07.322 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:07.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:07.426 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:07.439 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:50:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:50:07] "GET /metrics HTTP/1.1" 200 36777 "" "Prometheus/2.51.0" 2024-09-13T14:50:07.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:07.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:07.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:07.588 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:07.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:07.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.731 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:07.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:07.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:07.865 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:07.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:07.968 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:07.996 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:08.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:08.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:08.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:07 smithi177 bash[68594]: cluster 2024-09-13T14:50:06.543564+0000 mgr.y (mgr.26252) 713 : cluster [DBG] pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 4.1 MiB/s, 66 objects/s recovering 2024-09-13T14:50:08.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:07 smithi177 bash[68594]: cluster 2024-09-13T14:50:06.543564+0000 mgr.y (mgr.26252) 713 : cluster [DBG] pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 4.1 MiB/s, 66 objects/s recovering 2024-09-13T14:50:08.131 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:08.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:08.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:08.255 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:08.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:08.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:07 smithi099 bash[84576]: cluster 2024-09-13T14:50:06.543564+0000 mgr.y (mgr.26252) 713 : cluster [DBG] pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 4.1 MiB/s, 66 objects/s recovering 2024-09-13T14:50:08.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:07 smithi099 bash[84576]: cluster 2024-09-13T14:50:06.543564+0000 mgr.y (mgr.26252) 713 : cluster [DBG] pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 4.1 MiB/s, 66 objects/s recovering 2024-09-13T14:50:08.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:07 smithi099 bash[87147]: cluster 2024-09-13T14:50:06.543564+0000 mgr.y (mgr.26252) 713 : cluster [DBG] pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 4.1 MiB/s, 66 objects/s recovering 2024-09-13T14:50:08.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:07 smithi099 bash[87147]: cluster 2024-09-13T14:50:06.543564+0000 mgr.y (mgr.26252) 713 : cluster [DBG] pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 16105/192510 objects degraded (8.366%); 5548/192510 objects misplaced (2.882%); 4.1 MiB/s, 66 objects/s recovering 2024-09-13T14:50:08.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:08.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:08.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:08.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:08.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:08.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:08.641 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:08.672 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:08.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:08.778 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:08.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:08.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:08.908 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:08.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:08.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:09.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:09.049 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:09.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:09.152 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:09.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:09.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:09.279 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:09.298 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:09.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:09.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:09.426 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:09.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:09.529 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:09.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:09.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:09.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:09.702 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:09.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:09.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:09.839 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:09.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:09.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:09.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:10.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:09 smithi099 bash[84576]: cluster 2024-09-13T14:50:08.544342+0000 mgr.y (mgr.26252) 714 : cluster [DBG] pgmap v1190: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 16213/187401 objects degraded (8.652%); 5088/187401 objects misplaced (2.715%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:10.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:09 smithi099 bash[84576]: cluster 2024-09-13T14:50:08.544342+0000 mgr.y (mgr.26252) 714 : cluster [DBG] pgmap v1190: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 16213/187401 objects degraded (8.652%); 5088/187401 objects misplaced (2.715%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:10.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:09 smithi099 bash[87147]: cluster 2024-09-13T14:50:08.544342+0000 mgr.y (mgr.26252) 714 : cluster [DBG] pgmap v1190: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 16213/187401 objects degraded (8.652%); 5088/187401 objects misplaced (2.715%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:10.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:09 smithi099 bash[87147]: cluster 2024-09-13T14:50:08.544342+0000 mgr.y (mgr.26252) 714 : cluster [DBG] pgmap v1190: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 16213/187401 objects degraded (8.652%); 5088/187401 objects misplaced (2.715%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:10.078 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:10.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:10.212 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:10.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:09 smithi177 bash[68594]: cluster 2024-09-13T14:50:08.544342+0000 mgr.y (mgr.26252) 714 : cluster [DBG] pgmap v1190: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 16213/187401 objects degraded (8.652%); 5088/187401 objects misplaced (2.715%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:10.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:09 smithi177 bash[68594]: cluster 2024-09-13T14:50:08.544342+0000 mgr.y (mgr.26252) 714 : cluster [DBG] pgmap v1190: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 16213/187401 objects degraded (8.652%); 5088/187401 objects misplaced (2.715%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:10.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:10.315 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:50:10.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:10.363 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:10.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:10.471 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:10.480 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:50:10 smithi177 bash[61439]: ts=2024-09-13T14:50:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:10.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:10.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:10.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:10.633 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:10.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:10.736 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:10.774 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:10.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:10.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:10.910 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:10.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:11.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:11.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:11.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:11.186 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:11.293 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:11.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:11.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:11.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:11.460 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:11.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:11.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:11.596 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:11.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:11.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:11.724 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:11.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:11.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:11.861 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:11.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:11.967 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:11.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:11.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:12.091 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:12.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:11 smithi177 bash[68594]: cluster 2024-09-13T14:50:10.545485+0000 mgr.y (mgr.26252) 715 : cluster [DBG] pgmap v1191: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 16213/182190 objects degraded (8.899%); 5088/182190 objects misplaced (2.793%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:12.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:11 smithi177 bash[68594]: cluster 2024-09-13T14:50:10.545485+0000 mgr.y (mgr.26252) 715 : cluster [DBG] pgmap v1191: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 16213/182190 objects degraded (8.899%); 5088/182190 objects misplaced (2.793%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:12.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:11 smithi177 bash[68594]: cluster 2024-09-13T14:50:11.333491+0000 mon.a (mon.0) 1712 : cluster [WRN] Health check update: Degraded data redundancy: 16213/182190 objects degraded (8.899%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:12.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:11 smithi177 bash[68594]: cluster 2024-09-13T14:50:11.333491+0000 mon.a (mon.0) 1712 : cluster [WRN] Health check update: Degraded data redundancy: 16213/182190 objects degraded (8.899%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:12.118 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:12.156 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:11 smithi099 bash[84576]: cluster 2024-09-13T14:50:10.545485+0000 mgr.y (mgr.26252) 715 : cluster [DBG] pgmap v1191: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 16213/182190 objects degraded (8.899%); 5088/182190 objects misplaced (2.793%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:12.156 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:11 smithi099 bash[84576]: cluster 2024-09-13T14:50:10.545485+0000 mgr.y (mgr.26252) 715 : cluster [DBG] pgmap v1191: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 16213/182190 objects degraded (8.899%); 5088/182190 objects misplaced (2.793%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:12.156 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:11 smithi099 bash[84576]: cluster 2024-09-13T14:50:11.333491+0000 mon.a (mon.0) 1712 : cluster [WRN] Health check update: Degraded data redundancy: 16213/182190 objects degraded (8.899%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:12.157 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:11 smithi099 bash[84576]: cluster 2024-09-13T14:50:11.333491+0000 mon.a (mon.0) 1712 : cluster [WRN] Health check update: Degraded data redundancy: 16213/182190 objects degraded (8.899%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:12.157 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:11 smithi099 bash[87147]: cluster 2024-09-13T14:50:10.545485+0000 mgr.y (mgr.26252) 715 : cluster [DBG] pgmap v1191: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 16213/182190 objects degraded (8.899%); 5088/182190 objects misplaced (2.793%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:12.157 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:11 smithi099 bash[87147]: cluster 2024-09-13T14:50:10.545485+0000 mgr.y (mgr.26252) 715 : cluster [DBG] pgmap v1191: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 16213/182190 objects degraded (8.899%); 5088/182190 objects misplaced (2.793%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:12.157 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:11 smithi099 bash[87147]: cluster 2024-09-13T14:50:11.333491+0000 mon.a (mon.0) 1712 : cluster [WRN] Health check update: Degraded data redundancy: 16213/182190 objects degraded (8.899%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:12.157 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:11 smithi099 bash[87147]: cluster 2024-09-13T14:50:11.333491+0000 mon.a (mon.0) 1712 : cluster [WRN] Health check update: Degraded data redundancy: 16213/182190 objects degraded (8.899%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:12.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:12.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:12.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:12.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:50:12.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:12.411 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:12.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:12.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:12.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:12.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:12.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:12.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:12.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:12.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:12.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:12.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:12.930 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:12.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:13.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:13.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:13.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:13.129 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:13.177 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:13.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:13.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:13.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:13.330 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:13.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:13.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:13.467 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:13.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:13.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:13.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:13.704 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:13.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:13.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:13.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:13.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:13.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:13.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:13 smithi099 bash[84576]: cluster 2024-09-13T14:50:12.546169+0000 mgr.y (mgr.26252) 716 : cluster [DBG] pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 16213/180330 objects degraded (8.991%); 5088/180330 objects misplaced (2.821%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:13.981 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:13 smithi099 bash[84576]: cluster 2024-09-13T14:50:12.546169+0000 mgr.y (mgr.26252) 716 : cluster [DBG] pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 16213/180330 objects degraded (8.991%); 5088/180330 objects misplaced (2.821%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:13.981 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:13 smithi099 bash[87147]: cluster 2024-09-13T14:50:12.546169+0000 mgr.y (mgr.26252) 716 : cluster [DBG] pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 16213/180330 objects degraded (8.991%); 5088/180330 objects misplaced (2.821%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:13.982 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:13 smithi099 bash[87147]: cluster 2024-09-13T14:50:12.546169+0000 mgr.y (mgr.26252) 716 : cluster [DBG] pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 16213/180330 objects degraded (8.991%); 5088/180330 objects misplaced (2.821%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:14.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:14.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:14.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:14.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:14.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:14.303 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:14.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:13 smithi177 bash[68594]: cluster 2024-09-13T14:50:12.546169+0000 mgr.y (mgr.26252) 716 : cluster [DBG] pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 16213/180330 objects degraded (8.991%); 5088/180330 objects misplaced (2.821%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:13 smithi177 bash[68594]: cluster 2024-09-13T14:50:12.546169+0000 mgr.y (mgr.26252) 716 : cluster [DBG] pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 16213/180330 objects degraded (8.991%); 5088/180330 objects misplaced (2.821%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:14.406 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:14.434 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:14.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:14.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:14.564 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:14.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:14.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:14.693 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:14.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:14.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:14.807 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 14:50:14 smithi177 bash[60145]: logger=cleanup t=2024-09-13T14:50:14.507497687Z level=info msg="Completed cleanup jobs" duration=64.987147ms 2024-09-13T14:50:14.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:14.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:14.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:14.969 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:14.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:15.074 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:15.165 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:15.198 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.220 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:15.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:15.348 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:15.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:15.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:15.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:15.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:15.715 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:15.743 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:15.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:15.848 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:15.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:15.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:15.979 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:15.982 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:15 smithi099 bash[84576]: cluster 2024-09-13T14:50:14.547110+0000 mgr.y (mgr.26252) 717 : cluster [DBG] pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:50:15.982 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:15 smithi099 bash[84576]: cluster 2024-09-13T14:50:14.547110+0000 mgr.y (mgr.26252) 717 : cluster [DBG] pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:50:15.983 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:15 smithi099 bash[87147]: cluster 2024-09-13T14:50:14.547110+0000 mgr.y (mgr.26252) 717 : cluster [DBG] pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:50:15.983 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:15 smithi099 bash[87147]: cluster 2024-09-13T14:50:14.547110+0000 mgr.y (mgr.26252) 717 : cluster [DBG] pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:50:16.002 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:16.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:16.105 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:16.146 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:16.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:16.185 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:16.249 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:16.293 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:16.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:16.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:15 smithi177 bash[68594]: cluster 2024-09-13T14:50:14.547110+0000 mgr.y (mgr.26252) 717 : cluster [DBG] pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:50:16.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:15 smithi177 bash[68594]: cluster 2024-09-13T14:50:14.547110+0000 mgr.y (mgr.26252) 717 : cluster [DBG] pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.5 MiB/s, 72 objects/s recovering 2024-09-13T14:50:16.396 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:16.419 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:16.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:16.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:16.538 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:50:16.543 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:50:16.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:16.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:16.680 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:16.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:16.784 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:16.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:16.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:16.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:16.927 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:16 smithi177 bash[68594]: cluster 2024-09-13T14:50:16.334256+0000 mon.a (mon.0) 1713 : cluster [WRN] Health check update: Degraded data redundancy: 16286/173187 objects degraded (9.404%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:16.927 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:16 smithi177 bash[68594]: cluster 2024-09-13T14:50:16.334256+0000 mon.a (mon.0) 1713 : cluster [WRN] Health check update: Degraded data redundancy: 16286/173187 objects degraded (9.404%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:16.960 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:17.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:17.088 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:17.108 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:16 smithi099 bash[84576]: cluster 2024-09-13T14:50:16.334256+0000 mon.a (mon.0) 1713 : cluster [WRN] Health check update: Degraded data redundancy: 16286/173187 objects degraded (9.404%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:17.108 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:16 smithi099 bash[84576]: cluster 2024-09-13T14:50:16.334256+0000 mon.a (mon.0) 1713 : cluster [WRN] Health check update: Degraded data redundancy: 16286/173187 objects degraded (9.404%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:17.109 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:50:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:50:17] "GET /metrics HTTP/1.1" 200 36777 "" "Prometheus/2.51.0" 2024-09-13T14:50:17.109 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:16 smithi099 bash[87147]: cluster 2024-09-13T14:50:16.334256+0000 mon.a (mon.0) 1713 : cluster [WRN] Health check update: Degraded data redundancy: 16286/173187 objects degraded (9.404%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:17.109 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:16 smithi099 bash[87147]: cluster 2024-09-13T14:50:16.334256+0000 mon.a (mon.0) 1713 : cluster [WRN] Health check update: Degraded data redundancy: 16286/173187 objects degraded (9.404%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:17.191 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:17.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:17.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:17.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:17.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:17.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:17.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:17.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:17.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:17.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:17.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:17.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:17.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:17.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:17.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:17.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:17.992 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:18.002 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:17 smithi177 bash[68594]: audit 2024-09-13T14:50:16.535078+0000 mgr.y (mgr.26252) 718 : audit [DBG] from='client.45230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:18.002 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:17 smithi177 bash[68594]: audit 2024-09-13T14:50:16.535078+0000 mgr.y (mgr.26252) 718 : audit [DBG] from='client.45230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:18.002 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:17 smithi177 bash[68594]: cluster 2024-09-13T14:50:16.547601+0000 mgr.y (mgr.26252) 719 : cluster [DBG] pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:50:18.002 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:17 smithi177 bash[68594]: cluster 2024-09-13T14:50:16.547601+0000 mgr.y (mgr.26252) 719 : cluster [DBG] pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:50:18.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.030 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:18.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:18.131 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:18.158 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:18.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:18.260 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:18.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:18.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:18.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:18 smithi099 bash[84576]: audit 2024-09-13T14:50:16.535078+0000 mgr.y (mgr.26252) 718 : audit [DBG] from='client.45230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:18.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:18 smithi099 bash[84576]: audit 2024-09-13T14:50:16.535078+0000 mgr.y (mgr.26252) 718 : audit [DBG] from='client.45230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:18.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:18 smithi099 bash[84576]: cluster 2024-09-13T14:50:16.547601+0000 mgr.y (mgr.26252) 719 : cluster [DBG] pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:50:18.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:18 smithi099 bash[84576]: cluster 2024-09-13T14:50:16.547601+0000 mgr.y (mgr.26252) 719 : cluster [DBG] pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:50:18.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:17 smithi099 bash[87147]: audit 2024-09-13T14:50:16.535078+0000 mgr.y (mgr.26252) 718 : audit [DBG] from='client.45230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:18.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:17 smithi099 bash[87147]: audit 2024-09-13T14:50:16.535078+0000 mgr.y (mgr.26252) 718 : audit [DBG] from='client.45230 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:50:18.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:17 smithi099 bash[87147]: cluster 2024-09-13T14:50:16.547601+0000 mgr.y (mgr.26252) 719 : cluster [DBG] pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:50:18.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:17 smithi099 bash[87147]: cluster 2024-09-13T14:50:16.547601+0000 mgr.y (mgr.26252) 719 : cluster [DBG] pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 16286/173187 objects degraded (9.404%); 4644/173187 objects misplaced (2.681%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:50:18.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:18.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:18.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:18.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:18.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:18.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:18.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:18.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:18.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:18.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:18.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:18.916 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:18.935 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:18.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:19.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:18 smithi099 bash[84576]: audit 2024-09-13T14:50:18.274200+0000 mon.a (mon.0) 1714 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:19.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:19 smithi099 bash[87147]: audit 2024-09-13T14:50:18.274200+0000 mon.a (mon.0) 1714 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:19.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:19 smithi099 bash[87147]: audit 2024-09-13T14:50:18.274200+0000 mon.a (mon.0) 1714 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:19.038 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:19.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:19.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:18 smithi177 bash[68594]: audit 2024-09-13T14:50:18.274200+0000 mon.a (mon.0) 1714 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:19.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:18 smithi177 bash[68594]: audit 2024-09-13T14:50:18.274200+0000 mon.a (mon.0) 1714 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:19.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:19.181 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:19 smithi099 bash[84576]: audit 2024-09-13T14:50:18.274200+0000 mon.a (mon.0) 1714 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:19.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:19.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:19.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:19.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:19.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:19.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:19.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:19.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:19.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:19.607 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:19.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:19.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:19.735 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:19.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:19.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:19.870 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:19.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:19.977 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:20.002 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:20.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:20.106 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:20.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:19 smithi177 bash[68594]: cluster 2024-09-13T14:50:18.548172+0000 mgr.y (mgr.26252) 720 : cluster [DBG] pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 16387/168351 objects degraded (9.734%); 4213/168351 objects misplaced (2.503%); 5.5 MiB/s, 87 objects/s recovering 2024-09-13T14:50:20.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:19 smithi177 bash[68594]: cluster 2024-09-13T14:50:18.548172+0000 mgr.y (mgr.26252) 720 : cluster [DBG] pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 16387/168351 objects degraded (9.734%); 4213/168351 objects misplaced (2.503%); 5.5 MiB/s, 87 objects/s recovering 2024-09-13T14:50:20.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:20.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:20.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:20.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:20.292 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:20 smithi099 bash[84576]: cluster 2024-09-13T14:50:18.548172+0000 mgr.y (mgr.26252) 720 : cluster [DBG] pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 16387/168351 objects degraded (9.734%); 4213/168351 objects misplaced (2.503%); 5.5 MiB/s, 87 objects/s recovering 2024-09-13T14:50:20.292 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:20 smithi099 bash[84576]: cluster 2024-09-13T14:50:18.548172+0000 mgr.y (mgr.26252) 720 : cluster [DBG] pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 16387/168351 objects degraded (9.734%); 4213/168351 objects misplaced (2.503%); 5.5 MiB/s, 87 objects/s recovering 2024-09-13T14:50:20.292 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:20 smithi099 bash[87147]: cluster 2024-09-13T14:50:18.548172+0000 mgr.y (mgr.26252) 720 : cluster [DBG] pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 16387/168351 objects degraded (9.734%); 4213/168351 objects misplaced (2.503%); 5.5 MiB/s, 87 objects/s recovering 2024-09-13T14:50:20.292 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:20 smithi099 bash[87147]: cluster 2024-09-13T14:50:18.548172+0000 mgr.y (mgr.26252) 720 : cluster [DBG] pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 16387/168351 objects degraded (9.734%); 4213/168351 objects misplaced (2.503%); 5.5 MiB/s, 87 objects/s recovering 2024-09-13T14:50:20.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:20.394 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:20.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:20.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:20.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:20.557 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:20.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:20.613 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:50:20 smithi177 bash[61439]: ts=2024-09-13T14:50:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:20.660 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:20.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:20.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:20.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:20.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:20.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:20.934 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:20.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:20.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:21.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:21.083 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:21.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:21.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:21.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:21.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:21.323 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:21.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:21.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:21.445 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:21.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:21.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:21.580 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:21.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:21.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:21.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:21.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:21.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:21.842 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:21.875 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:21.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:21.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:21.992 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:21 smithi177 bash[68594]: cluster 2024-09-13T14:50:20.549282+0000 mgr.y (mgr.26252) 721 : cluster [DBG] pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16387/163692 objects degraded (10.011%); 4213/163692 objects misplaced (2.574%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:21.992 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:21 smithi177 bash[68594]: cluster 2024-09-13T14:50:20.549282+0000 mgr.y (mgr.26252) 721 : cluster [DBG] pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16387/163692 objects degraded (10.011%); 4213/163692 objects misplaced (2.574%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:21.992 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:21 smithi177 bash[68594]: cluster 2024-09-13T14:50:21.335082+0000 mon.a (mon.0) 1715 : cluster [WRN] Health check update: Degraded data redundancy: 16387/163692 objects degraded (10.011%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:21.992 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:21 smithi177 bash[68594]: cluster 2024-09-13T14:50:21.335082+0000 mon.a (mon.0) 1715 : cluster [WRN] Health check update: Degraded data redundancy: 16387/163692 objects degraded (10.011%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:22.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:22.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:22.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:21 smithi099 bash[84576]: cluster 2024-09-13T14:50:20.549282+0000 mgr.y (mgr.26252) 721 : cluster [DBG] pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16387/163692 objects degraded (10.011%); 4213/163692 objects misplaced (2.574%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:22.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:21 smithi099 bash[84576]: cluster 2024-09-13T14:50:20.549282+0000 mgr.y (mgr.26252) 721 : cluster [DBG] pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16387/163692 objects degraded (10.011%); 4213/163692 objects misplaced (2.574%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:22.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:21 smithi099 bash[84576]: cluster 2024-09-13T14:50:21.335082+0000 mon.a (mon.0) 1715 : cluster [WRN] Health check update: Degraded data redundancy: 16387/163692 objects degraded (10.011%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:22.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:21 smithi099 bash[84576]: cluster 2024-09-13T14:50:21.335082+0000 mon.a (mon.0) 1715 : cluster [WRN] Health check update: Degraded data redundancy: 16387/163692 objects degraded (10.011%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:22.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:21 smithi099 bash[87147]: cluster 2024-09-13T14:50:20.549282+0000 mgr.y (mgr.26252) 721 : cluster [DBG] pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16387/163692 objects degraded (10.011%); 4213/163692 objects misplaced (2.574%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:22.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:21 smithi099 bash[87147]: cluster 2024-09-13T14:50:20.549282+0000 mgr.y (mgr.26252) 721 : cluster [DBG] pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16387/163692 objects degraded (10.011%); 4213/163692 objects misplaced (2.574%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:22.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:21 smithi099 bash[87147]: cluster 2024-09-13T14:50:21.335082+0000 mon.a (mon.0) 1715 : cluster [WRN] Health check update: Degraded data redundancy: 16387/163692 objects degraded (10.011%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:22.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:21 smithi099 bash[87147]: cluster 2024-09-13T14:50:21.335082+0000 mon.a (mon.0) 1715 : cluster [WRN] Health check update: Degraded data redundancy: 16387/163692 objects degraded (10.011%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:22.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:22.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:22.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:22.245 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:22.272 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:22.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:22.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:22.402 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:22.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:22.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:22.540 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:22.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:22.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:22.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:22.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:22.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:22.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:22.908 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:22.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:22.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:23.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:23.058 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:23.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:23.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:22 smithi177 bash[68594]: cluster 2024-09-13T14:50:22.550326+0000 mgr.y (mgr.26252) 722 : cluster [DBG] pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 16387/162189 objects degraded (10.104%); 4213/162189 objects misplaced (2.598%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:23.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:22 smithi177 bash[68594]: cluster 2024-09-13T14:50:22.550326+0000 mgr.y (mgr.26252) 722 : cluster [DBG] pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 16387/162189 objects degraded (10.104%); 4213/162189 objects misplaced (2.598%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:23.166 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:23.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:23.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:23.209 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:22 smithi099 bash[84576]: cluster 2024-09-13T14:50:22.550326+0000 mgr.y (mgr.26252) 722 : cluster [DBG] pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 16387/162189 objects degraded (10.104%); 4213/162189 objects misplaced (2.598%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:23.209 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:22 smithi099 bash[84576]: cluster 2024-09-13T14:50:22.550326+0000 mgr.y (mgr.26252) 722 : cluster [DBG] pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 16387/162189 objects degraded (10.104%); 4213/162189 objects misplaced (2.598%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:23.210 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:22 smithi099 bash[87147]: cluster 2024-09-13T14:50:22.550326+0000 mgr.y (mgr.26252) 722 : cluster [DBG] pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 16387/162189 objects degraded (10.104%); 4213/162189 objects misplaced (2.598%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:23.210 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:22 smithi099 bash[87147]: cluster 2024-09-13T14:50:22.550326+0000 mgr.y (mgr.26252) 722 : cluster [DBG] pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 16387/162189 objects degraded (10.104%); 4213/162189 objects misplaced (2.598%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T14:50:23.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:23.334 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:23.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:23.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:23.457 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:23.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:23.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:23.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:23.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:23.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:23.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:23.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:23.802 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1193,"stamp":"2024-09-13T14:50:14.546363+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3782154213,"num_objects":57729,"num_object_clones":0,"num_object_copies":173187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16286,"num_objects_misplaced":4644,"num_objects_unfound":0,"num_objects_dirty":57729,"num_whiteouts":0,"num_read":345,"num_read_kb":800,"num_write":138081,"num_write_kb":6259900,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7711,"num_bytes_recovered":503977439,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119933,"ondisk_log_size":119933,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11805788,"kb_used_data":10367396,"kb_used_omap":286,"kb_used_meta":1437729,"kb_avail":644410276,"statfs":{"total":671965249536,"available":659876122624,"internally_reserved":0,"allocated":10616213504,"data_stored":10591176158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":293460,"internal_metadata":1472234924},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,6,7,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-422117376,"num_objects":-6441,"num_object_clones":0,"num_object_copies":-19323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":181,"num_objects_misplaced":-904,"num_objects_unfound":0,"num_objects_dirty":-6441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6441,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":723,"num_bytes_recovered":47382528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004132"},"pg_stats":[{"pgid":"41.f","version":"981'8171","reported_seq":14415,"reported_epoch":981,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-13T14:50:12.757119+0000","last_change":"2024-09-13T14:48:51.598650+0000","last_active":"2024-09-13T14:50:12.757119+0000","last_peered":"2024-09-13T14:50:12.757119+0000","last_clean":"2024-09-13T14:48:49.258461+0000","last_became_active":"2024-09-13T14:48:51.535954+0000","last_became_peered":"2024-09-13T14:48:51.535954+0000","last_unstale":"2024-09-13T14:50:12.757119+0000","last_undegraded":"2024-09-13T14:50:12.757119+0000","last_fullsized":"2024-09-13T14:48:51.318646+0000","mapping_epoch":976,"log_start":"0'0","ondisk_log_start":"0'0","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":8171,"log_dups_size":0,"ondisk_log_size":8171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232849408,"num_objects":3553,"num_object_clones":0,"num_object_copies":10659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1832,"num_objects_unfound":0,"num_objects_dirty":3553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8171,"num_write_kb":375168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3089,"num_bytes_recovered":202440704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2,3","objects":1832},{"shards":"2,3","objects":1721}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"41.e","version":"981'8712","reported_seq":8748,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:14.104619+0000","last_change":"2024-09-13T14:47:17.865396+0000","last_active":"2024-09-13T14:50:14.104619+0000","last_peered":"2024-09-13T14:50:14.104619+0000","last_clean":"2024-09-13T14:50:14.104619+0000","last_became_active":"2024-09-13T14:47:17.864558+0000","last_became_peered":"2024-09-13T14:47:17.864558+0000","last_unstale":"2024-09-13T14:50:14.104619+0000","last_undegraded":"2024-09-13T14:50:14.104619+0000","last_fullsized":"2024-09-13T14:50:14.104619+0000","mapping_epoch":953,"log_start":"0'0","ondisk_log_start":"0'0","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":8712,"log_dups_size":0,"ondisk_log_size":8712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:46.006485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230686720,"num_objects":3520,"num_object_clones":0,"num_object_copies":10560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8712,"num_write_kb":391424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"41.d","version":"981'8532","reported_seq":8602,"reported_epoch":981,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:50:13.637776+0000","last_change":"2024-09-13T14:48:51.563400+0000","last_active":"2024-09-13T14:50:13.637776+0000","last_peered":"2024-09-13T14:50:13.637776+0000","last_clean":"2024-09-13T14:48:49.253715+0000","last_became_active":"2024-09-13T14:48:51.556136+0000","last_became_peered":"2024-09-13T14:48:51.556136+0000","last_unstale":"2024-09-13T14:50:13.637776+0000","last_undegraded":"2024-09-13T14:48:51.361665+0000","last_fullsized":"2024-09-13T14:48:51.326027+0000","mapping_epoch":976,"log_start":"970'1500","ondisk_log_start":"970'1500","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232849432,"num_objects":3554,"num_object_clones":0,"num_object_copies":10662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5092,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3554,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8532,"num_write_kb":386689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":2603},{"shards":"0,7","objects":-1538}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"41.c","version":"981'8531","reported_seq":8601,"reported_epoch":981,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:50:13.566094+0000","last_change":"2024-09-13T14:48:51.561428+0000","last_active":"2024-09-13T14:50:13.566094+0000","last_peered":"2024-09-13T14:50:13.566094+0000","last_clean":"2024-09-13T14:48:49.232955+0000","last_became_active":"2024-09-13T14:48:51.552435+0000","last_became_peered":"2024-09-13T14:48:51.552435+0000","last_unstale":"2024-09-13T14:50:13.566094+0000","last_undegraded":"2024-09-13T14:48:51.361493+0000","last_fullsized":"2024-09-13T14:48:51.322450+0000","mapping_epoch":976,"log_start":"970'1500","ondisk_log_start":"970'1500","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232718336,"num_objects":3551,"num_object_clones":0,"num_object_copies":10653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8531,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":2605},{"shards":"0,7","objects":-1544}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"41.b","version":"981'8902","reported_seq":12211,"reported_epoch":981,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:50:13.108800+0000","last_change":"2024-09-13T14:49:30.902213+0000","last_active":"2024-09-13T14:50:13.108800+0000","last_peered":"2024-09-13T14:50:13.108800+0000","last_clean":"2024-09-13T14:48:49.237246+0000","last_became_active":"2024-09-13T14:48:51.508676+0000","last_became_peered":"2024-09-13T14:48:51.508676+0000","last_unstale":"2024-09-13T14:50:13.108800+0000","last_undegraded":"2024-09-13T14:49:30.898579+0000","last_fullsized":"2024-09-13T14:48:51.330096+0000","mapping_epoch":976,"log_start":"972'2200","ondisk_log_start":"972'2200","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":6702,"log_dups_size":0,"ondisk_log_size":6702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246284288,"num_objects":3758,"num_object_clones":0,"num_object_copies":11274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":951,"num_objects_misplaced":2812,"num_objects_unfound":0,"num_objects_dirty":3758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8902,"num_write_kb":405120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1621,"num_bytes_recovered":106233856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":2812},{"shards":"3,7","objects":-5}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"41.a","version":"981'8574","reported_seq":9680,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:11.884146+0000","last_change":"2024-09-13T14:48:02.319705+0000","last_active":"2024-09-13T14:50:11.884146+0000","last_peered":"2024-09-13T14:50:11.884146+0000","last_clean":"2024-09-13T14:50:11.884146+0000","last_became_active":"2024-09-13T14:48:02.316389+0000","last_became_peered":"2024-09-13T14:48:02.316389+0000","last_unstale":"2024-09-13T14:50:11.884146+0000","last_undegraded":"2024-09-13T14:50:11.884146+0000","last_fullsized":"2024-09-13T14:50:11.884146+0000","mapping_epoch":971,"log_start":"968'1400","ondisk_log_start":"968'1400","created":953,"last_epoch_clean":972,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":7174,"log_dups_size":0,"ondisk_log_size":7174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:51.701746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235667456,"num_objects":3596,"num_object_clones":0,"num_object_copies":10788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8574,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":513,"num_bytes_recovered":33619968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"41.9","version":"981'8477","reported_seq":8545,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:13.632918+0000","last_change":"2024-09-13T14:47:17.871387+0000","last_active":"2024-09-13T14:50:13.632918+0000","last_peered":"2024-09-13T14:50:13.632918+0000","last_clean":"2024-09-13T14:50:13.632918+0000","last_became_active":"2024-09-13T14:47:17.871087+0000","last_became_peered":"2024-09-13T14:47:17.871087+0000","last_unstale":"2024-09-13T14:50:13.632918+0000","last_undegraded":"2024-09-13T14:50:13.632918+0000","last_fullsized":"2024-09-13T14:50:13.632918+0000","mapping_epoch":953,"log_start":"970'1500","ondisk_log_start":"970'1500","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:38.938377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234029056,"num_objects":3571,"num_object_clones":0,"num_object_copies":10713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8477,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"41.8","version":"981'8895","reported_seq":9943,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:14.096220+0000","last_change":"2024-09-13T14:47:56.421303+0000","last_active":"2024-09-13T14:50:14.096220+0000","last_peered":"2024-09-13T14:50:14.096220+0000","last_clean":"2024-09-13T14:50:14.096220+0000","last_became_active":"2024-09-13T14:47:50.313636+0000","last_became_peered":"2024-09-13T14:47:50.313636+0000","last_unstale":"2024-09-13T14:50:14.096220+0000","last_undegraded":"2024-09-13T14:50:14.096220+0000","last_fullsized":"2024-09-13T14:50:14.096220+0000","mapping_epoch":961,"log_start":"0'0","ondisk_log_start":"0'0","created":953,"last_epoch_clean":966,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":8895,"log_dups_size":0,"ondisk_log_size":8895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:40.546330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241106944,"num_objects":3679,"num_object_clones":0,"num_object_copies":11037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8895,"num_write_kb":402368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":494,"num_bytes_recovered":32374784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"41.7","version":"981'8626","reported_seq":8662,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:14.011518+0000","last_change":"2024-09-13T14:47:17.866262+0000","last_active":"2024-09-13T14:50:14.011518+0000","last_peered":"2024-09-13T14:50:14.011518+0000","last_clean":"2024-09-13T14:50:14.011518+0000","last_became_active":"2024-09-13T14:47:17.865546+0000","last_became_peered":"2024-09-13T14:47:17.865546+0000","last_unstale":"2024-09-13T14:50:14.011518+0000","last_undegraded":"2024-09-13T14:50:14.011518+0000","last_fullsized":"2024-09-13T14:50:14.011518+0000","mapping_epoch":953,"log_start":"964'1100","ondisk_log_start":"964'1100","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":7526,"log_dups_size":0,"ondisk_log_size":7526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:20:19.446374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228065280,"num_objects":3480,"num_object_clones":0,"num_object_copies":10440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8626,"num_write_kb":387392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"41.6","version":"981'8593","reported_seq":9659,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:11.878437+0000","last_change":"2024-09-13T14:47:57.490906+0000","last_active":"2024-09-13T14:50:11.878437+0000","last_peered":"2024-09-13T14:50:11.878437+0000","last_clean":"2024-09-13T14:50:11.878437+0000","last_became_active":"2024-09-13T14:47:57.489207+0000","last_became_peered":"2024-09-13T14:47:57.489207+0000","last_unstale":"2024-09-13T14:50:11.878437+0000","last_undegraded":"2024-09-13T14:50:11.878437+0000","last_fullsized":"2024-09-13T14:50:11.878437+0000","mapping_epoch":967,"log_start":"968'1400","ondisk_log_start":"968'1400","created":953,"last_epoch_clean":968,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":7193,"log_dups_size":0,"ondisk_log_size":7193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:42:53.102363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239403008,"num_objects":3653,"num_object_clones":0,"num_object_copies":10959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8593,"num_write_kb":391872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":496,"num_bytes_recovered":32505856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"41.5","version":"981'8602","reported_seq":8669,"reported_epoch":981,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:50:12.565302+0000","last_change":"2024-09-13T14:48:51.750293+0000","last_active":"2024-09-13T14:50:12.565302+0000","last_peered":"2024-09-13T14:50:12.565302+0000","last_clean":"2024-09-13T14:48:49.249873+0000","last_became_active":"2024-09-13T14:48:51.739776+0000","last_became_peered":"2024-09-13T14:48:51.739776+0000","last_unstale":"2024-09-13T14:50:12.565302+0000","last_undegraded":"2024-09-13T14:48:51.352166+0000","last_fullsized":"2024-09-13T14:48:51.319659+0000","mapping_epoch":976,"log_start":"0'0","ondisk_log_start":"0'0","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":8602,"log_dups_size":0,"ondisk_log_size":8602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242221056,"num_objects":3696,"num_object_clones":0,"num_object_copies":11088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8602,"num_write_kb":393536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,1,2","objects":2695},{"shards":"0,2","objects":-1452}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"41.4","version":"981'8704","reported_seq":8740,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:13.936761+0000","last_change":"2024-09-13T14:47:17.869661+0000","last_active":"2024-09-13T14:50:13.936761+0000","last_peered":"2024-09-13T14:50:13.936761+0000","last_clean":"2024-09-13T14:50:13.936761+0000","last_became_active":"2024-09-13T14:47:17.868932+0000","last_became_peered":"2024-09-13T14:47:17.868932+0000","last_unstale":"2024-09-13T14:50:13.936761+0000","last_undegraded":"2024-09-13T14:50:13.936761+0000","last_fullsized":"2024-09-13T14:50:13.936761+0000","mapping_epoch":953,"log_start":"966'1200","ondisk_log_start":"966'1200","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":7504,"log_dups_size":0,"ondisk_log_size":7504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:39:56.429223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234094592,"num_objects":3572,"num_object_clones":0,"num_object_copies":10716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8704,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"41.3","version":"981'8465","reported_seq":9482,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:13.070921+0000","last_change":"2024-09-13T14:47:52.333231+0000","last_active":"2024-09-13T14:50:13.070921+0000","last_peered":"2024-09-13T14:50:13.070921+0000","last_clean":"2024-09-13T14:50:13.070921+0000","last_became_active":"2024-09-13T14:47:52.329428+0000","last_became_peered":"2024-09-13T14:47:52.329428+0000","last_unstale":"2024-09-13T14:50:13.070921+0000","last_undegraded":"2024-09-13T14:50:13.070921+0000","last_fullsized":"2024-09-13T14:50:13.070921+0000","mapping_epoch":963,"log_start":"972'1700","ondisk_log_start":"972'1700","created":953,"last_epoch_clean":964,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":6765,"log_dups_size":0,"ondisk_log_size":6765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:23.513545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236126208,"num_objects":3603,"num_object_clones":0,"num_object_copies":10809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8465,"num_write_kb":386176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":470,"num_bytes_recovered":30801920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"41.2","version":"981'8776","reported_seq":9833,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:13.908596+0000","last_change":"2024-09-13T14:48:00.977950+0000","last_active":"2024-09-13T14:50:13.908596+0000","last_peered":"2024-09-13T14:50:13.908596+0000","last_clean":"2024-09-13T14:50:13.908596+0000","last_became_active":"2024-09-13T14:47:59.506115+0000","last_became_peered":"2024-09-13T14:47:59.506115+0000","last_unstale":"2024-09-13T14:50:13.908596+0000","last_undegraded":"2024-09-13T14:50:13.908596+0000","last_fullsized":"2024-09-13T14:50:13.908596+0000","mapping_epoch":969,"log_start":"966'1200","ondisk_log_start":"966'1200","created":953,"last_epoch_clean":970,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":7576,"log_dups_size":0,"ondisk_log_size":7576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:01:25.035332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236322816,"num_objects":3606,"num_object_clones":0,"num_object_copies":10818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8776,"num_write_kb":396224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":499,"num_bytes_recovered":32702464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"41.1","version":"981'8811","reported_seq":8879,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:13.024915+0000","last_change":"2024-09-13T14:47:17.873466+0000","last_active":"2024-09-13T14:50:13.024915+0000","last_peered":"2024-09-13T14:50:13.024915+0000","last_clean":"2024-09-13T14:50:13.024915+0000","last_became_active":"2024-09-13T14:47:17.873085+0000","last_became_peered":"2024-09-13T14:47:17.873085+0000","last_unstale":"2024-09-13T14:50:13.024915+0000","last_undegraded":"2024-09-13T14:50:13.024915+0000","last_fullsized":"2024-09-13T14:50:13.024915+0000","mapping_epoch":953,"log_start":"972'2100","ondisk_log_start":"972'2100","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:11.804439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241369088,"num_objects":3683,"num_object_clones":0,"num_object_copies":11049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8811,"num_write_kb":399808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"41.0","version":"981'8498","reported_seq":9570,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:11.888161+0000","last_change":"2024-09-13T14:47:55.473007+0000","last_active":"2024-09-13T14:50:11.888161+0000","last_peered":"2024-09-13T14:50:11.888161+0000","last_clean":"2024-09-13T14:50:11.888161+0000","last_became_active":"2024-09-13T14:47:55.470399+0000","last_became_peered":"2024-09-13T14:47:55.470399+0000","last_unstale":"2024-09-13T14:50:11.888161+0000","last_undegraded":"2024-09-13T14:50:11.888161+0000","last_fullsized":"2024-09-13T14:50:11.888161+0000","mapping_epoch":965,"log_start":"967'1300","ondisk_log_start":"967'1300","created":953,"last_epoch_clean":966,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":7198,"log_dups_size":0,"ondisk_log_size":7198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:50:25.545728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237764608,"num_objects":3628,"num_object_clones":0,"num_object_copies":10884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8498,"num_write_kb":388032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":499,"num_bytes_recovered":32702464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":1350,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.859608+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T14:50:05.859608+0000","last_peered":"2024-09-13T14:50:05.859608+0000","last_clean":"2024-09-13T14:50:05.859608+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T14:50:05.859608+0000","last_undegraded":"2024-09-13T14:50:05.859608+0000","last_fullsized":"2024-09-13T14:50:05.859608+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":64,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.844130+0000","last_change":"2024-09-13T14:48:50.321123+0000","last_active":"2024-09-13T14:50:05.844130+0000","last_peered":"2024-09-13T14:50:05.844130+0000","last_clean":"2024-09-13T14:50:05.844130+0000","last_became_active":"2024-09-13T14:48:50.319957+0000","last_became_peered":"2024-09-13T14:48:50.319957+0000","last_unstale":"2024-09-13T14:50:05.844130+0000","last_undegraded":"2024-09-13T14:50:05.844130+0000","last_fullsized":"2024-09-13T14:50:05.844130+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:41.504593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1201,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.866674+0000","last_change":"2024-09-13T14:39:34.403170+0000","last_active":"2024-09-13T14:50:05.866674+0000","last_peered":"2024-09-13T14:50:05.866674+0000","last_clean":"2024-09-13T14:50:05.866674+0000","last_became_active":"2024-09-13T14:39:34.389001+0000","last_became_peered":"2024-09-13T14:39:34.389001+0000","last_unstale":"2024-09-13T14:50:05.866674+0000","last_undegraded":"2024-09-13T14:50:05.866674+0000","last_fullsized":"2024-09-13T14:50:05.866674+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:38.542452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.676145+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T14:49:18.676145+0000","last_peered":"2024-09-13T14:49:18.676145+0000","last_clean":"2024-09-13T14:49:18.676145+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T14:49:18.676145+0000","last_undegraded":"2024-09-13T14:49:18.676145+0000","last_fullsized":"2024-09-13T14:49:18.676145+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":966,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675878+0000","last_change":"2024-09-13T14:47:46.337827+0000","last_active":"2024-09-13T14:49:18.675878+0000","last_peered":"2024-09-13T14:49:18.675878+0000","last_clean":"2024-09-13T14:49:18.675878+0000","last_became_active":"2024-09-13T14:47:46.321140+0000","last_became_peered":"2024-09-13T14:47:46.321140+0000","last_unstale":"2024-09-13T14:49:18.675878+0000","last_undegraded":"2024-09-13T14:49:18.675878+0000","last_fullsized":"2024-09-13T14:49:18.675878+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:15.997382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":56,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.842478+0000","last_change":"2024-09-13T14:47:46.333043+0000","last_active":"2024-09-13T14:50:05.842478+0000","last_peered":"2024-09-13T14:50:05.842478+0000","last_clean":"2024-09-13T14:50:05.842478+0000","last_became_active":"2024-09-13T14:47:46.332292+0000","last_became_peered":"2024-09-13T14:47:46.332292+0000","last_unstale":"2024-09-13T14:50:05.842478+0000","last_undegraded":"2024-09-13T14:50:05.842478+0000","last_fullsized":"2024-09-13T14:50:05.842478+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:38.942334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":997,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675690+0000","last_change":"2024-09-13T14:48:50.325866+0000","last_active":"2024-09-13T14:49:18.675690+0000","last_peered":"2024-09-13T14:49:18.675690+0000","last_clean":"2024-09-13T14:49:18.675690+0000","last_became_active":"2024-09-13T14:48:50.306654+0000","last_became_peered":"2024-09-13T14:48:50.306654+0000","last_unstale":"2024-09-13T14:49:18.675690+0000","last_undegraded":"2024-09-13T14:49:18.675690+0000","last_fullsized":"2024-09-13T14:49:18.675690+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:31:52.323885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":63,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.842221+0000","last_change":"2024-09-13T14:48:50.319669+0000","last_active":"2024-09-13T14:50:05.842221+0000","last_peered":"2024-09-13T14:50:05.842221+0000","last_clean":"2024-09-13T14:50:05.842221+0000","last_became_active":"2024-09-13T14:48:50.318195+0000","last_became_peered":"2024-09-13T14:48:50.318195+0000","last_unstale":"2024-09-13T14:50:05.842221+0000","last_undegraded":"2024-09-13T14:50:05.842221+0000","last_fullsized":"2024-09-13T14:50:05.842221+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:47.189218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1358,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.875553+0000","last_change":"2024-09-13T14:47:46.330059+0000","last_active":"2024-09-13T14:50:05.875553+0000","last_peered":"2024-09-13T14:50:05.875553+0000","last_clean":"2024-09-13T14:50:05.875553+0000","last_became_active":"2024-09-13T14:47:46.329344+0000","last_became_peered":"2024-09-13T14:47:46.329344+0000","last_unstale":"2024-09-13T14:50:05.875553+0000","last_undegraded":"2024-09-13T14:50:05.875553+0000","last_fullsized":"2024-09-13T14:50:05.875553+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:58:43.470176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1255,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.845121+0000","last_change":"2024-09-13T14:48:50.313031+0000","last_active":"2024-09-13T14:50:05.845121+0000","last_peered":"2024-09-13T14:50:05.845121+0000","last_clean":"2024-09-13T14:50:05.845121+0000","last_became_active":"2024-09-13T14:48:50.312337+0000","last_became_peered":"2024-09-13T14:48:50.312337+0000","last_unstale":"2024-09-13T14:50:05.845121+0000","last_undegraded":"2024-09-13T14:50:05.845121+0000","last_fullsized":"2024-09-13T14:50:05.845121+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:17:02.018596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":63,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.842426+0000","last_change":"2024-09-13T14:48:50.302422+0000","last_active":"2024-09-13T14:50:05.842426+0000","last_peered":"2024-09-13T14:50:05.842426+0000","last_clean":"2024-09-13T14:50:05.842426+0000","last_became_active":"2024-09-13T14:48:50.301099+0000","last_became_peered":"2024-09-13T14:48:50.301099+0000","last_unstale":"2024-09-13T14:50:05.842426+0000","last_undegraded":"2024-09-13T14:50:05.842426+0000","last_fullsized":"2024-09-13T14:50:05.842426+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:06:19.459783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1241,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.866546+0000","last_change":"2024-09-13T14:48:50.318471+0000","last_active":"2024-09-13T14:50:05.866546+0000","last_peered":"2024-09-13T14:50:05.866546+0000","last_clean":"2024-09-13T14:50:05.866546+0000","last_became_active":"2024-09-13T14:48:50.318104+0000","last_became_peered":"2024-09-13T14:48:50.318104+0000","last_unstale":"2024-09-13T14:50:05.866546+0000","last_undegraded":"2024-09-13T14:50:05.866546+0000","last_fullsized":"2024-09-13T14:50:05.866546+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:40.089045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1278,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.863181+0000","last_change":"2024-09-13T14:48:50.320505+0000","last_active":"2024-09-13T14:50:05.863181+0000","last_peered":"2024-09-13T14:50:05.863181+0000","last_clean":"2024-09-13T14:50:05.863181+0000","last_became_active":"2024-09-13T14:48:50.318302+0000","last_became_peered":"2024-09-13T14:48:50.318302+0000","last_unstale":"2024-09-13T14:50:05.863181+0000","last_undegraded":"2024-09-13T14:50:05.863181+0000","last_fullsized":"2024-09-13T14:50:05.863181+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:44:02.320626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1347,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.858970+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T14:50:05.858970+0000","last_peered":"2024-09-13T14:50:05.858970+0000","last_clean":"2024-09-13T14:50:05.858970+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T14:50:05.858970+0000","last_undegraded":"2024-09-13T14:50:05.858970+0000","last_fullsized":"2024-09-13T14:50:05.858970+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1234,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.852607+0000","last_change":"2024-09-13T14:48:50.319178+0000","last_active":"2024-09-13T14:50:05.852607+0000","last_peered":"2024-09-13T14:50:05.852607+0000","last_clean":"2024-09-13T14:50:05.852607+0000","last_became_active":"2024-09-13T14:48:50.318773+0000","last_became_peered":"2024-09-13T14:48:50.318773+0000","last_unstale":"2024-09-13T14:50:05.852607+0000","last_undegraded":"2024-09-13T14:50:05.852607+0000","last_fullsized":"2024-09-13T14:50:05.852607+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:01.123295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":948,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.679731+0000","last_change":"2024-09-13T14:48:50.310483+0000","last_active":"2024-09-13T14:49:18.679731+0000","last_peered":"2024-09-13T14:49:18.679731+0000","last_clean":"2024-09-13T14:49:18.679731+0000","last_became_active":"2024-09-13T14:48:50.310017+0000","last_became_peered":"2024-09-13T14:48:50.310017+0000","last_unstale":"2024-09-13T14:49:18.679731+0000","last_undegraded":"2024-09-13T14:49:18.679731+0000","last_fullsized":"2024-09-13T14:49:18.679731+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:05.652854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1214,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.852520+0000","last_change":"2024-09-13T14:48:50.324418+0000","last_active":"2024-09-13T14:50:05.852520+0000","last_peered":"2024-09-13T14:50:05.852520+0000","last_clean":"2024-09-13T14:50:05.852520+0000","last_became_active":"2024-09-13T14:48:50.324073+0000","last_became_peered":"2024-09-13T14:48:50.324073+0000","last_unstale":"2024-09-13T14:50:05.852520+0000","last_undegraded":"2024-09-13T14:50:05.852520+0000","last_fullsized":"2024-09-13T14:50:05.852520+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:47.192606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1338,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.859699+0000","last_change":"2024-09-13T14:39:34.318266+0000","last_active":"2024-09-13T14:50:05.859699+0000","last_peered":"2024-09-13T14:50:05.859699+0000","last_clean":"2024-09-13T14:50:05.859699+0000","last_became_active":"2024-09-13T14:39:34.315973+0000","last_became_peered":"2024-09-13T14:39:34.315973+0000","last_unstale":"2024-09-13T14:50:05.859699+0000","last_undegraded":"2024-09-13T14:50:05.859699+0000","last_fullsized":"2024-09-13T14:50:05.859699+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:15.270282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1186,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.863280+0000","last_change":"2024-09-13T14:39:34.338378+0000","last_active":"2024-09-13T14:50:05.863280+0000","last_peered":"2024-09-13T14:50:05.863280+0000","last_clean":"2024-09-13T14:50:05.863280+0000","last_became_active":"2024-09-13T14:39:34.338005+0000","last_became_peered":"2024-09-13T14:39:34.338005+0000","last_unstale":"2024-09-13T14:50:05.863280+0000","last_undegraded":"2024-09-13T14:50:05.863280+0000","last_fullsized":"2024-09-13T14:50:05.863280+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:29.120379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":945,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.676075+0000","last_change":"2024-09-13T14:48:50.318723+0000","last_active":"2024-09-13T14:49:18.676075+0000","last_peered":"2024-09-13T14:49:18.676075+0000","last_clean":"2024-09-13T14:49:18.676075+0000","last_became_active":"2024-09-13T14:48:50.318132+0000","last_became_peered":"2024-09-13T14:48:50.318132+0000","last_unstale":"2024-09-13T14:49:18.676075+0000","last_undegraded":"2024-09-13T14:49:18.676075+0000","last_fullsized":"2024-09-13T14:49:18.676075+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:25.352103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":992,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.677535+0000","last_change":"2024-09-13T14:47:46.354886+0000","last_active":"2024-09-13T14:49:18.677535+0000","last_peered":"2024-09-13T14:49:18.677535+0000","last_clean":"2024-09-13T14:49:18.677535+0000","last_became_active":"2024-09-13T14:47:46.329947+0000","last_became_peered":"2024-09-13T14:47:46.329947+0000","last_unstale":"2024-09-13T14:49:18.677535+0000","last_undegraded":"2024-09-13T14:49:18.677535+0000","last_fullsized":"2024-09-13T14:49:18.677535+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:22.443920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":676,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.679453+0000","last_change":"2024-09-13T14:48:50.312407+0000","last_active":"2024-09-13T14:49:18.679453+0000","last_peered":"2024-09-13T14:49:18.679453+0000","last_clean":"2024-09-13T14:49:18.679453+0000","last_became_active":"2024-09-13T14:48:50.308224+0000","last_became_peered":"2024-09-13T14:48:50.308224+0000","last_unstale":"2024-09-13T14:49:18.679453+0000","last_undegraded":"2024-09-13T14:49:18.679453+0000","last_fullsized":"2024-09-13T14:49:18.679453+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:45.050288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":67,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.841836+0000","last_change":"2024-09-13T14:47:46.344536+0000","last_active":"2024-09-13T14:50:05.841836+0000","last_peered":"2024-09-13T14:50:05.841836+0000","last_clean":"2024-09-13T14:50:05.841836+0000","last_became_active":"2024-09-13T14:47:46.321131+0000","last_became_peered":"2024-09-13T14:47:46.321131+0000","last_unstale":"2024-09-13T14:50:05.841836+0000","last_undegraded":"2024-09-13T14:50:05.841836+0000","last_fullsized":"2024-09-13T14:50:05.841836+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:31.114472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":64,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.842303+0000","last_change":"2024-09-13T14:48:50.310695+0000","last_active":"2024-09-13T14:50:05.842303+0000","last_peered":"2024-09-13T14:50:05.842303+0000","last_clean":"2024-09-13T14:50:05.842303+0000","last_became_active":"2024-09-13T14:48:50.309622+0000","last_became_peered":"2024-09-13T14:48:50.309622+0000","last_unstale":"2024-09-13T14:50:05.842303+0000","last_undegraded":"2024-09-13T14:50:05.842303+0000","last_fullsized":"2024-09-13T14:50:05.842303+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:56.001951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.842957+0000","last_change":"2024-09-13T14:47:46.318785+0000","last_active":"2024-09-13T14:50:05.842957+0000","last_peered":"2024-09-13T14:50:05.842957+0000","last_clean":"2024-09-13T14:50:05.842957+0000","last_became_active":"2024-09-13T14:47:46.318438+0000","last_became_peered":"2024-09-13T14:47:46.318438+0000","last_unstale":"2024-09-13T14:50:05.842957+0000","last_undegraded":"2024-09-13T14:50:05.842957+0000","last_fullsized":"2024-09-13T14:50:05.842957+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:44.745378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":951,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.679794+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T14:49:18.679794+0000","last_peered":"2024-09-13T14:49:18.679794+0000","last_clean":"2024-09-13T14:49:18.679794+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T14:49:18.679794+0000","last_undegraded":"2024-09-13T14:49:18.679794+0000","last_fullsized":"2024-09-13T14:49:18.679794+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1388,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.859199+0000","last_change":"2024-09-13T14:48:50.315638+0000","last_active":"2024-09-13T14:50:05.859199+0000","last_peered":"2024-09-13T14:50:05.859199+0000","last_clean":"2024-09-13T14:50:05.859199+0000","last_became_active":"2024-09-13T14:48:50.307974+0000","last_became_peered":"2024-09-13T14:48:50.307974+0000","last_unstale":"2024-09-13T14:50:05.859199+0000","last_undegraded":"2024-09-13T14:50:05.859199+0000","last_fullsized":"2024-09-13T14:50:05.859199+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:59.437169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1184,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.852805+0000","last_change":"2024-09-13T14:39:34.349945+0000","last_active":"2024-09-13T14:50:05.852805+0000","last_peered":"2024-09-13T14:50:05.852805+0000","last_clean":"2024-09-13T14:50:05.852805+0000","last_became_active":"2024-09-13T14:39:34.346730+0000","last_became_peered":"2024-09-13T14:39:34.346730+0000","last_unstale":"2024-09-13T14:50:05.852805+0000","last_undegraded":"2024-09-13T14:50:05.852805+0000","last_fullsized":"2024-09-13T14:50:05.852805+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:48.854603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1273,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.679613+0000","last_change":"2024-09-13T14:48:50.316513+0000","last_active":"2024-09-13T14:49:18.679613+0000","last_peered":"2024-09-13T14:49:18.679613+0000","last_clean":"2024-09-13T14:49:18.679613+0000","last_became_active":"2024-09-13T14:48:50.315976+0000","last_became_peered":"2024-09-13T14:48:50.315976+0000","last_unstale":"2024-09-13T14:49:18.679613+0000","last_undegraded":"2024-09-13T14:49:18.679613+0000","last_fullsized":"2024-09-13T14:49:18.679613+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:44.529556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1187,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.867067+0000","last_change":"2024-09-13T14:39:34.401432+0000","last_active":"2024-09-13T14:50:05.867067+0000","last_peered":"2024-09-13T14:50:05.867067+0000","last_clean":"2024-09-13T14:50:05.867067+0000","last_became_active":"2024-09-13T14:39:34.395458+0000","last_became_peered":"2024-09-13T14:39:34.395458+0000","last_unstale":"2024-09-13T14:50:05.867067+0000","last_undegraded":"2024-09-13T14:50:05.867067+0000","last_fullsized":"2024-09-13T14:50:05.867067+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:17.007388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1328,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.875333+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T14:50:05.875333+0000","last_peered":"2024-09-13T14:50:05.875333+0000","last_clean":"2024-09-13T14:50:05.875333+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T14:50:05.875333+0000","last_undegraded":"2024-09-13T14:50:05.875333+0000","last_fullsized":"2024-09-13T14:50:05.875333+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":987,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.677762+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T14:49:18.677762+0000","last_peered":"2024-09-13T14:49:18.677762+0000","last_clean":"2024-09-13T14:49:18.677762+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T14:49:18.677762+0000","last_undegraded":"2024-09-13T14:49:18.677762+0000","last_fullsized":"2024-09-13T14:49:18.677762+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":681,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.845067+0000","last_change":"2024-09-13T14:48:50.307105+0000","last_active":"2024-09-13T14:50:05.845067+0000","last_peered":"2024-09-13T14:50:05.845067+0000","last_clean":"2024-09-13T14:50:05.845067+0000","last_became_active":"2024-09-13T14:48:50.306324+0000","last_became_peered":"2024-09-13T14:48:50.306324+0000","last_unstale":"2024-09-13T14:50:05.845067+0000","last_undegraded":"2024-09-13T14:50:05.845067+0000","last_fullsized":"2024-09-13T14:50:05.845067+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:22.934075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1263,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675350+0000","last_change":"2024-09-13T14:48:50.307874+0000","last_active":"2024-09-13T14:49:18.675350+0000","last_peered":"2024-09-13T14:49:18.675350+0000","last_clean":"2024-09-13T14:49:18.675350+0000","last_became_active":"2024-09-13T14:48:50.305966+0000","last_became_peered":"2024-09-13T14:48:50.305966+0000","last_unstale":"2024-09-13T14:49:18.675350+0000","last_undegraded":"2024-09-13T14:49:18.675350+0000","last_fullsized":"2024-09-13T14:49:18.675350+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:06.157706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":621,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.862626+0000","last_change":"2024-09-13T14:48:50.306120+0000","last_active":"2024-09-13T14:50:05.862626+0000","last_peered":"2024-09-13T14:50:05.862626+0000","last_clean":"2024-09-13T14:50:05.862626+0000","last_became_active":"2024-09-13T14:48:50.305574+0000","last_became_peered":"2024-09-13T14:48:50.305574+0000","last_unstale":"2024-09-13T14:50:05.862626+0000","last_undegraded":"2024-09-13T14:50:05.862626+0000","last_fullsized":"2024-09-13T14:50:05.862626+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:15:16.993023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":952,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675916+0000","last_change":"2024-09-13T14:39:34.321388+0000","last_active":"2024-09-13T14:49:18.675916+0000","last_peered":"2024-09-13T14:49:18.675916+0000","last_clean":"2024-09-13T14:49:18.675916+0000","last_became_active":"2024-09-13T14:39:34.321025+0000","last_became_peered":"2024-09-13T14:39:34.321025+0000","last_unstale":"2024-09-13T14:49:18.675916+0000","last_undegraded":"2024-09-13T14:49:18.675916+0000","last_fullsized":"2024-09-13T14:49:18.675916+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:52.593825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":760,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.845167+0000","last_change":"2024-09-13T14:48:50.312676+0000","last_active":"2024-09-13T14:50:05.845167+0000","last_peered":"2024-09-13T14:50:05.845167+0000","last_clean":"2024-09-13T14:50:05.845167+0000","last_became_active":"2024-09-13T14:48:50.311904+0000","last_became_peered":"2024-09-13T14:48:50.311904+0000","last_unstale":"2024-09-13T14:50:05.845167+0000","last_undegraded":"2024-09-13T14:50:05.845167+0000","last_fullsized":"2024-09-13T14:50:05.845167+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:55.991145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":57,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.842314+0000","last_change":"2024-09-13T14:47:46.324072+0000","last_active":"2024-09-13T14:50:05.842314+0000","last_peered":"2024-09-13T14:50:05.842314+0000","last_clean":"2024-09-13T14:50:05.842314+0000","last_became_active":"2024-09-13T14:47:46.322613+0000","last_became_peered":"2024-09-13T14:47:46.322613+0000","last_unstale":"2024-09-13T14:50:05.842314+0000","last_undegraded":"2024-09-13T14:50:05.842314+0000","last_fullsized":"2024-09-13T14:50:05.842314+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:45.314706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":963,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675233+0000","last_change":"2024-09-13T14:48:50.330611+0000","last_active":"2024-09-13T14:49:18.675233+0000","last_peered":"2024-09-13T14:49:18.675233+0000","last_clean":"2024-09-13T14:49:18.675233+0000","last_became_active":"2024-09-13T14:48:50.310242+0000","last_became_peered":"2024-09-13T14:48:50.310242+0000","last_unstale":"2024-09-13T14:49:18.675233+0000","last_undegraded":"2024-09-13T14:49:18.675233+0000","last_fullsized":"2024-09-13T14:49:18.675233+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:28.097810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":598,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.676047+0000","last_change":"2024-09-13T14:48:50.309417+0000","last_active":"2024-09-13T14:49:18.676047+0000","last_peered":"2024-09-13T14:49:18.676047+0000","last_clean":"2024-09-13T14:49:18.676047+0000","last_became_active":"2024-09-13T14:48:50.308664+0000","last_became_peered":"2024-09-13T14:48:50.308664+0000","last_unstale":"2024-09-13T14:49:18.676047+0000","last_undegraded":"2024-09-13T14:49:18.676047+0000","last_fullsized":"2024-09-13T14:49:18.676047+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:15.509105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":681,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.844867+0000","last_change":"2024-09-13T14:48:50.304113+0000","last_active":"2024-09-13T14:50:05.844867+0000","last_peered":"2024-09-13T14:50:05.844867+0000","last_clean":"2024-09-13T14:50:05.844867+0000","last_became_active":"2024-09-13T14:48:50.303641+0000","last_became_peered":"2024-09-13T14:48:50.303641+0000","last_unstale":"2024-09-13T14:50:05.844867+0000","last_undegraded":"2024-09-13T14:50:05.844867+0000","last_fullsized":"2024-09-13T14:50:05.844867+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:11.186019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1237,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.860949+0000","last_change":"2024-09-13T14:39:34.322533+0000","last_active":"2024-09-13T14:50:05.860949+0000","last_peered":"2024-09-13T14:50:05.860949+0000","last_clean":"2024-09-13T14:50:05.860949+0000","last_became_active":"2024-09-13T14:39:34.322070+0000","last_became_peered":"2024-09-13T14:39:34.322070+0000","last_unstale":"2024-09-13T14:50:05.860949+0000","last_undegraded":"2024-09-13T14:50:05.860949+0000","last_fullsized":"2024-09-13T14:50:05.860949+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:43.746420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1357,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.858855+0000","last_change":"2024-09-13T14:48:50.305592+0000","last_active":"2024-09-13T14:50:05.858855+0000","last_peered":"2024-09-13T14:50:05.858855+0000","last_clean":"2024-09-13T14:50:05.858855+0000","last_became_active":"2024-09-13T14:48:50.305223+0000","last_became_peered":"2024-09-13T14:48:50.305223+0000","last_unstale":"2024-09-13T14:50:05.858855+0000","last_undegraded":"2024-09-13T14:50:05.858855+0000","last_fullsized":"2024-09-13T14:50:05.858855+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:40:12.253906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1364,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.862367+0000","last_change":"2024-09-13T14:47:46.329033+0000","last_active":"2024-09-13T14:50:05.862367+0000","last_peered":"2024-09-13T14:50:05.862367+0000","last_clean":"2024-09-13T14:50:05.862367+0000","last_became_active":"2024-09-13T14:47:46.328203+0000","last_became_peered":"2024-09-13T14:47:46.328203+0000","last_unstale":"2024-09-13T14:50:05.862367+0000","last_undegraded":"2024-09-13T14:50:05.862367+0000","last_fullsized":"2024-09-13T14:50:05.862367+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:23.440178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":977,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.677221+0000","last_change":"2024-09-13T14:48:50.317044+0000","last_active":"2024-09-13T14:49:18.677221+0000","last_peered":"2024-09-13T14:49:18.677221+0000","last_clean":"2024-09-13T14:49:18.677221+0000","last_became_active":"2024-09-13T14:48:50.316179+0000","last_became_peered":"2024-09-13T14:48:50.316179+0000","last_unstale":"2024-09-13T14:49:18.677221+0000","last_undegraded":"2024-09-13T14:49:18.677221+0000","last_fullsized":"2024-09-13T14:49:18.677221+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:06:00.880130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":958,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.680981+0000","last_change":"2024-09-13T14:47:46.320830+0000","last_active":"2024-09-13T14:49:18.680981+0000","last_peered":"2024-09-13T14:49:18.680981+0000","last_clean":"2024-09-13T14:49:18.680981+0000","last_became_active":"2024-09-13T14:47:46.320443+0000","last_became_peered":"2024-09-13T14:47:46.320443+0000","last_unstale":"2024-09-13T14:49:18.680981+0000","last_undegraded":"2024-09-13T14:49:18.680981+0000","last_fullsized":"2024-09-13T14:49:18.680981+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:46.279580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1186,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.866390+0000","last_change":"2024-09-13T14:39:34.356129+0000","last_active":"2024-09-13T14:50:05.866390+0000","last_peered":"2024-09-13T14:50:05.866390+0000","last_clean":"2024-09-13T14:50:05.866390+0000","last_became_active":"2024-09-13T14:39:34.355288+0000","last_became_peered":"2024-09-13T14:39:34.355288+0000","last_unstale":"2024-09-13T14:50:05.866390+0000","last_undegraded":"2024-09-13T14:50:05.866390+0000","last_fullsized":"2024-09-13T14:50:05.866390+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.725003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":948,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675479+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T14:49:18.675479+0000","last_peered":"2024-09-13T14:49:18.675479+0000","last_clean":"2024-09-13T14:49:18.675479+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T14:49:18.675479+0000","last_undegraded":"2024-09-13T14:49:18.675479+0000","last_fullsized":"2024-09-13T14:49:18.675479+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1055,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.677246+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T14:49:18.677246+0000","last_peered":"2024-09-13T14:49:18.677246+0000","last_clean":"2024-09-13T14:49:18.677246+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T14:49:18.677246+0000","last_undegraded":"2024-09-13T14:49:18.677246+0000","last_fullsized":"2024-09-13T14:49:18.677246+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1279,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.867161+0000","last_change":"2024-09-13T14:48:50.323890+0000","last_active":"2024-09-13T14:50:05.867161+0000","last_peered":"2024-09-13T14:50:05.867161+0000","last_clean":"2024-09-13T14:50:05.867161+0000","last_became_active":"2024-09-13T14:48:50.323324+0000","last_became_peered":"2024-09-13T14:48:50.323324+0000","last_unstale":"2024-09-13T14:50:05.867161+0000","last_undegraded":"2024-09-13T14:50:05.867161+0000","last_fullsized":"2024-09-13T14:50:05.867161+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:23:46.125319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1217,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.852711+0000","last_change":"2024-09-13T14:48:50.325837+0000","last_active":"2024-09-13T14:50:05.852711+0000","last_peered":"2024-09-13T14:50:05.852711+0000","last_clean":"2024-09-13T14:50:05.852711+0000","last_became_active":"2024-09-13T14:48:50.325420+0000","last_became_peered":"2024-09-13T14:48:50.325420+0000","last_unstale":"2024-09-13T14:50:05.852711+0000","last_undegraded":"2024-09-13T14:50:05.852711+0000","last_fullsized":"2024-09-13T14:50:05.852711+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:34.804111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":64,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.842388+0000","last_change":"2024-09-13T14:48:50.319803+0000","last_active":"2024-09-13T14:50:05.842388+0000","last_peered":"2024-09-13T14:50:05.842388+0000","last_clean":"2024-09-13T14:50:05.842388+0000","last_became_active":"2024-09-13T14:48:50.318366+0000","last_became_peered":"2024-09-13T14:48:50.318366+0000","last_unstale":"2024-09-13T14:50:05.842388+0000","last_undegraded":"2024-09-13T14:50:05.842388+0000","last_fullsized":"2024-09-13T14:50:05.842388+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:00.668596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":57,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.842870+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T14:50:05.842870+0000","last_peered":"2024-09-13T14:50:05.842870+0000","last_clean":"2024-09-13T14:50:05.842870+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T14:50:05.842870+0000","last_undegraded":"2024-09-13T14:50:05.842870+0000","last_fullsized":"2024-09-13T14:50:05.842870+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":64,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.842073+0000","last_change":"2024-09-13T14:48:50.315439+0000","last_active":"2024-09-13T14:50:05.842073+0000","last_peered":"2024-09-13T14:50:05.842073+0000","last_clean":"2024-09-13T14:50:05.842073+0000","last_became_active":"2024-09-13T14:48:50.314478+0000","last_became_peered":"2024-09-13T14:48:50.314478+0000","last_unstale":"2024-09-13T14:50:05.842073+0000","last_undegraded":"2024-09-13T14:50:05.842073+0000","last_fullsized":"2024-09-13T14:50:05.842073+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:22.490918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":986,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.679535+0000","last_change":"2024-09-13T14:48:50.308531+0000","last_active":"2024-09-13T14:49:18.679535+0000","last_peered":"2024-09-13T14:49:18.679535+0000","last_clean":"2024-09-13T14:49:18.679535+0000","last_became_active":"2024-09-13T14:48:50.307100+0000","last_became_peered":"2024-09-13T14:48:50.307100+0000","last_unstale":"2024-09-13T14:49:18.679535+0000","last_undegraded":"2024-09-13T14:49:18.679535+0000","last_fullsized":"2024-09-13T14:49:18.679535+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:59.904615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":57,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.841701+0000","last_change":"2024-09-13T14:47:46.325496+0000","last_active":"2024-09-13T14:50:05.841701+0000","last_peered":"2024-09-13T14:50:05.841701+0000","last_clean":"2024-09-13T14:50:05.841701+0000","last_became_active":"2024-09-13T14:47:46.325099+0000","last_became_peered":"2024-09-13T14:47:46.325099+0000","last_unstale":"2024-09-13T14:50:05.841701+0000","last_undegraded":"2024-09-13T14:50:05.841701+0000","last_fullsized":"2024-09-13T14:50:05.841701+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:48.803608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1221,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.876545+0000","last_change":"2024-09-13T14:47:46.349682+0000","last_active":"2024-09-13T14:50:05.876545+0000","last_peered":"2024-09-13T14:50:05.876545+0000","last_clean":"2024-09-13T14:50:05.876545+0000","last_became_active":"2024-09-13T14:47:46.341656+0000","last_became_peered":"2024-09-13T14:47:46.341656+0000","last_unstale":"2024-09-13T14:50:05.876545+0000","last_undegraded":"2024-09-13T14:50:05.876545+0000","last_fullsized":"2024-09-13T14:50:05.876545+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:02.727169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1371,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.875476+0000","last_change":"2024-09-13T14:48:50.317959+0000","last_active":"2024-09-13T14:50:05.875476+0000","last_peered":"2024-09-13T14:50:05.875476+0000","last_clean":"2024-09-13T14:50:05.875476+0000","last_became_active":"2024-09-13T14:48:50.317578+0000","last_became_peered":"2024-09-13T14:48:50.317578+0000","last_unstale":"2024-09-13T14:50:05.875476+0000","last_undegraded":"2024-09-13T14:50:05.875476+0000","last_fullsized":"2024-09-13T14:50:05.875476+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:17.441358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":935,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675615+0000","last_change":"2024-09-13T14:39:34.393673+0000","last_active":"2024-09-13T14:49:18.675615+0000","last_peered":"2024-09-13T14:49:18.675615+0000","last_clean":"2024-09-13T14:49:18.675615+0000","last_became_active":"2024-09-13T14:39:34.393186+0000","last_became_peered":"2024-09-13T14:39:34.393186+0000","last_unstale":"2024-09-13T14:49:18.675615+0000","last_undegraded":"2024-09-13T14:49:18.675615+0000","last_fullsized":"2024-09-13T14:49:18.675615+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:23.130084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":948,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.679374+0000","last_change":"2024-09-13T14:48:50.313376+0000","last_active":"2024-09-13T14:49:18.679374+0000","last_peered":"2024-09-13T14:49:18.679374+0000","last_clean":"2024-09-13T14:49:18.679374+0000","last_became_active":"2024-09-13T14:48:50.311937+0000","last_became_peered":"2024-09-13T14:48:50.311937+0000","last_unstale":"2024-09-13T14:49:18.679374+0000","last_undegraded":"2024-09-13T14:49:18.679374+0000","last_fullsized":"2024-09-13T14:49:18.679374+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:27.168900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":923,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.677685+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:49:18.677685+0000","last_peered":"2024-09-13T14:49:18.677685+0000","last_clean":"2024-09-13T14:49:18.677685+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:49:18.677685+0000","last_undegraded":"2024-09-13T14:49:18.677685+0000","last_fullsized":"2024-09-13T14:49:18.677685+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":64,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.843037+0000","last_change":"2024-09-13T14:48:50.317851+0000","last_active":"2024-09-13T14:50:05.843037+0000","last_peered":"2024-09-13T14:50:05.843037+0000","last_clean":"2024-09-13T14:50:05.843037+0000","last_became_active":"2024-09-13T14:48:50.317295+0000","last_became_peered":"2024-09-13T14:48:50.317295+0000","last_unstale":"2024-09-13T14:50:05.843037+0000","last_undegraded":"2024-09-13T14:50:05.843037+0000","last_fullsized":"2024-09-13T14:50:05.843037+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:04.019051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1000,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.676276+0000","last_change":"2024-09-13T14:48:50.327871+0000","last_active":"2024-09-13T14:49:18.676276+0000","last_peered":"2024-09-13T14:49:18.676276+0000","last_clean":"2024-09-13T14:49:18.676276+0000","last_became_active":"2024-09-13T14:48:50.316167+0000","last_became_peered":"2024-09-13T14:48:50.316167+0000","last_unstale":"2024-09-13T14:49:18.676276+0000","last_undegraded":"2024-09-13T14:49:18.676276+0000","last_fullsized":"2024-09-13T14:49:18.676276+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:52.315823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":917,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.677545+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:49:18.677545+0000","last_peered":"2024-09-13T14:49:18.677545+0000","last_clean":"2024-09-13T14:49:18.677545+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:49:18.677545+0000","last_undegraded":"2024-09-13T14:49:18.677545+0000","last_fullsized":"2024-09-13T14:49:18.677545+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":985,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675827+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T14:49:18.675827+0000","last_peered":"2024-09-13T14:49:18.675827+0000","last_clean":"2024-09-13T14:49:18.675827+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T14:49:18.675827+0000","last_undegraded":"2024-09-13T14:49:18.675827+0000","last_fullsized":"2024-09-13T14:49:18.675827+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":951,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.680995+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T14:49:18.680995+0000","last_peered":"2024-09-13T14:49:18.680995+0000","last_clean":"2024-09-13T14:49:18.680995+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T14:49:18.680995+0000","last_undegraded":"2024-09-13T14:49:18.680995+0000","last_fullsized":"2024-09-13T14:49:18.680995+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1216,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.866922+0000","last_change":"2024-09-13T14:48:50.322285+0000","last_active":"2024-09-13T14:50:05.866922+0000","last_peered":"2024-09-13T14:50:05.866922+0000","last_clean":"2024-09-13T14:50:05.866922+0000","last_became_active":"2024-09-13T14:48:50.321935+0000","last_became_peered":"2024-09-13T14:48:50.321935+0000","last_unstale":"2024-09-13T14:50:05.866922+0000","last_undegraded":"2024-09-13T14:50:05.866922+0000","last_fullsized":"2024-09-13T14:50:05.866922+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:47.484432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":958,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675341+0000","last_change":"2024-09-13T14:47:46.322428+0000","last_active":"2024-09-13T14:49:18.675341+0000","last_peered":"2024-09-13T14:49:18.675341+0000","last_clean":"2024-09-13T14:49:18.675341+0000","last_became_active":"2024-09-13T14:47:46.321955+0000","last_became_peered":"2024-09-13T14:47:46.321955+0000","last_unstale":"2024-09-13T14:49:18.675341+0000","last_undegraded":"2024-09-13T14:49:18.675341+0000","last_fullsized":"2024-09-13T14:49:18.675341+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:47.614065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1338,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.859012+0000","last_change":"2024-09-13T14:39:34.362694+0000","last_active":"2024-09-13T14:50:05.859012+0000","last_peered":"2024-09-13T14:50:05.859012+0000","last_clean":"2024-09-13T14:50:05.859012+0000","last_became_active":"2024-09-13T14:39:34.362435+0000","last_became_peered":"2024-09-13T14:39:34.362435+0000","last_unstale":"2024-09-13T14:50:05.859012+0000","last_undegraded":"2024-09-13T14:50:05.859012+0000","last_fullsized":"2024-09-13T14:50:05.859012+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:33.350328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1339,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.862922+0000","last_change":"2024-09-13T14:39:34.360941+0000","last_active":"2024-09-13T14:50:05.862922+0000","last_peered":"2024-09-13T14:50:05.862922+0000","last_clean":"2024-09-13T14:50:05.862922+0000","last_became_active":"2024-09-13T14:39:34.360383+0000","last_became_peered":"2024-09-13T14:39:34.360383+0000","last_unstale":"2024-09-13T14:50:05.862922+0000","last_undegraded":"2024-09-13T14:50:05.862922+0000","last_fullsized":"2024-09-13T14:50:05.862922+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:51.344296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":934,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675241+0000","last_change":"2024-09-13T14:39:34.380634+0000","last_active":"2024-09-13T14:49:18.675241+0000","last_peered":"2024-09-13T14:49:18.675241+0000","last_clean":"2024-09-13T14:49:18.675241+0000","last_became_active":"2024-09-13T14:39:34.379785+0000","last_became_peered":"2024-09-13T14:39:34.379785+0000","last_unstale":"2024-09-13T14:49:18.675241+0000","last_undegraded":"2024-09-13T14:49:18.675241+0000","last_fullsized":"2024-09-13T14:49:18.675241+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:02.436193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":991,"reported_epoch":980,"state":"active+clean","last_fresh":"2024-09-13T14:49:18.675689+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T14:49:18.675689+0000","last_peered":"2024-09-13T14:49:18.675689+0000","last_clean":"2024-09-13T14:49:18.675689+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T14:49:18.675689+0000","last_undegraded":"2024-09-13T14:49:18.675689+0000","last_fullsized":"2024-09-13T14:49:18.675689+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1216,"reported_epoch":981,"state":"active+clean","last_fresh":"2024-09-13T14:50:05.868204+0000","last_change":"2024-09-13T14:48:50.322415+0000","last_active":"2024-09-13T14:50:05.868204+0000","last_peered":"2024-09-13T14:50:05.868204+0000","last_clean":"2024-09-13T14:50:05.868204+0000","last_became_active":"2024-09-13T14:48:50.321926+0000","last_became_peered":"2024-09-13T14:48:50.321926+0000","last_unstale":"2024-09-13T14:50:05.868204+0000","last_undegraded":"2024-09-13T14:50:05.868204+0000","last_fullsized":"2024-09-13T14:50:05.868204+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:35:43.108789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":41,"num_pg":16,"stat_sum":{"num_bytes":3781558296,"num_objects":57703,"num_object_clones":0,"num_object_copies":173109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16286,"num_objects_misplaced":4644,"num_objects_unfound":0,"num_objects_dirty":57703,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137869,"num_write_kb":6258241,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7681,"num_bytes_recovered":503382016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12186103808,"data_stored":12186091592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119769,"ondisk_log_size":119769,"up":48,"acting":43,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397870,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1697832,"kb_used_data":1387936,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92047320,"statfs":{"total":95995035648,"available":94256455680,"internally_reserved":0,"allocated":1421246464,"data_stored":1417654759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594105,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1760732,"kb_used_data":1455060,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91984420,"statfs":{"total":95995035648,"available":94192046080,"internally_reserved":0,"allocated":1489981440,"data_stored":1486401872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790339,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1457616,"kb_used_data":1160264,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92287536,"statfs":{"total":95995035648,"available":94502436864,"internally_reserved":0,"allocated":1188110336,"data_stored":1184538330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980831867,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1666012,"kb_used_data":1525588,"kb_used_omap":76,"kb_used_meta":140339,"kb_avail":92079140,"statfs":{"total":95995035648,"available":94289039360,"internally_reserved":0,"allocated":1562202112,"data_stored":1558622069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":143707968},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911106,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1571080,"kb_used_data":1470848,"kb_used_omap":77,"kb_used_meta":100082,"kb_avail":92174072,"statfs":{"total":95995035648,"available":94386249728,"internally_reserved":0,"allocated":1506148352,"data_stored":1502583997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":102484789},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023049,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1843280,"kb_used_data":1698568,"kb_used_omap":71,"kb_used_meta":144504,"kb_avail":91901872,"statfs":{"total":95995035648,"available":94107516928,"internally_reserved":0,"allocated":1739333632,"data_stored":1735761631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":147972768},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2757369004179,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195371,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1809236,"kb_used_data":1669132,"kb_used_omap":56,"kb_used_meta":139975,"kb_avail":91935916,"statfs":{"total":95995035648,"available":94142377984,"internally_reserved":0,"allocated":1709191168,"data_stored":1705613500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":143334543},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1696403456,"data_stored":1696399384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1680343040,"data_stored":1680343040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1726353408,"data_stored":1726349336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1489174528,"data_stored":1489174528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1549336576,"data_stored":1549336576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1163984896,"data_stored":1163984896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1469841408,"data_stored":1469841408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1410666496,"data_stored":1410662424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:50:23.807 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:50:23.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_0 pg_num 26 2024-09-13T14:50:23.833 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:23.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.863 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:23.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:23.964 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:24.004 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:24.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:24.107 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:24.133 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:24.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:24.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:24.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:24.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:24.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:24.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:24.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:24.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:24.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:24.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:24.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:24.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:24.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:24.781 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:24.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:24.884 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:24.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:24.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:25.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:25.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:25.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:25.201 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:25.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:25.309 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.336 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:25.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:25.437 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:25.474 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:25.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:25.577 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:25.607 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:25.713 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:25.723 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:25 smithi099 bash[84576]: cluster 2024-09-13T14:50:24.552523+0000 mgr.y (mgr.26252) 723 : cluster [DBG] pgmap v1198: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:50:25.723 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:25 smithi099 bash[84576]: cluster 2024-09-13T14:50:24.552523+0000 mgr.y (mgr.26252) 723 : cluster [DBG] pgmap v1198: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:50:25.723 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:25 smithi099 bash[87147]: cluster 2024-09-13T14:50:24.552523+0000 mgr.y (mgr.26252) 723 : cluster [DBG] pgmap v1198: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:50:25.724 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:25 smithi099 bash[87147]: cluster 2024-09-13T14:50:24.552523+0000 mgr.y (mgr.26252) 723 : cluster [DBG] pgmap v1198: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:50:25.733 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:25.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:25.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:25.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:25 smithi177 bash[68594]: cluster 2024-09-13T14:50:24.552523+0000 mgr.y (mgr.26252) 723 : cluster [DBG] pgmap v1198: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:50:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:25 smithi177 bash[68594]: cluster 2024-09-13T14:50:24.552523+0000 mgr.y (mgr.26252) 723 : cluster [DBG] pgmap v1198: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:50:25.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:25.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:26.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:26.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:26.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:26.143 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:26.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:26.282 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:26.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:26.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:26.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:26.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:26.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:26.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:26.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:26.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:26.691 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:26.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:26.796 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:26.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:26 smithi099 bash[84576]: cluster 2024-09-13T14:50:26.337926+0000 mon.a (mon.0) 1716 : cluster [WRN] Health check update: Degraded data redundancy: 16468/154809 objects degraded (10.638%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:26.806 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:26 smithi099 bash[84576]: cluster 2024-09-13T14:50:26.337926+0000 mon.a (mon.0) 1716 : cluster [WRN] Health check update: Degraded data redundancy: 16468/154809 objects degraded (10.638%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:26.806 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:26 smithi099 bash[87147]: cluster 2024-09-13T14:50:26.337926+0000 mon.a (mon.0) 1716 : cluster [WRN] Health check update: Degraded data redundancy: 16468/154809 objects degraded (10.638%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:26.806 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:26 smithi099 bash[87147]: cluster 2024-09-13T14:50:26.337926+0000 mon.a (mon.0) 1716 : cluster [WRN] Health check update: Degraded data redundancy: 16468/154809 objects degraded (10.638%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:26.828 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:26.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:26.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:26 smithi177 bash[68594]: cluster 2024-09-13T14:50:26.337926+0000 mon.a (mon.0) 1716 : cluster [WRN] Health check update: Degraded data redundancy: 16468/154809 objects degraded (10.638%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:26.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:26 smithi177 bash[68594]: cluster 2024-09-13T14:50:26.337926+0000 mon.a (mon.0) 1716 : cluster [WRN] Health check update: Degraded data redundancy: 16468/154809 objects degraded (10.638%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:26.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:26.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:26.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:27.077 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:50:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:50:27] "GET /metrics HTTP/1.1" 200 36779 "" "Prometheus/2.51.0" 2024-09-13T14:50:27.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:27.107 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:27.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:27.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:27.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:27.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:27.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:27.379 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:27.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:27.502 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:27.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:27.605 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:27.637 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:27.742 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:27.786 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:27.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:27.893 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:27.910 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:27.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:27.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:27 smithi099 bash[87147]: cluster 2024-09-13T14:50:26.553282+0000 mgr.y (mgr.26252) 724 : cluster [DBG] pgmap v1199: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:50:27.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:27 smithi099 bash[87147]: cluster 2024-09-13T14:50:26.553282+0000 mgr.y (mgr.26252) 724 : cluster [DBG] pgmap v1199: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:50:27.914 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:27 smithi099 bash[84576]: cluster 2024-09-13T14:50:26.553282+0000 mgr.y (mgr.26252) 724 : cluster [DBG] pgmap v1199: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:50:27.914 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:27 smithi099 bash[84576]: cluster 2024-09-13T14:50:26.553282+0000 mgr.y (mgr.26252) 724 : cluster [DBG] pgmap v1199: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:50:28.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:28.053 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:28.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:27 smithi177 bash[68594]: cluster 2024-09-13T14:50:26.553282+0000 mgr.y (mgr.26252) 724 : cluster [DBG] pgmap v1199: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:50:28.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:27 smithi177 bash[68594]: cluster 2024-09-13T14:50:26.553282+0000 mgr.y (mgr.26252) 724 : cluster [DBG] pgmap v1199: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 16468/154809 objects degraded (10.638%); 3760/154809 objects misplaced (2.429%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:50:28.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:28.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:28.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:28.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:28.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:28.435 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:28.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:28.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:28.559 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:28.577 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:28.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:28.680 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:28.724 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:28.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:28 smithi099 bash[84576]: audit 2024-09-13T14:50:28.521694+0000 mon.a (mon.0) 1717 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:50:28.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:28 smithi099 bash[84576]: audit 2024-09-13T14:50:28.521694+0000 mon.a (mon.0) 1717 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:50:28.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:28 smithi099 bash[87147]: audit 2024-09-13T14:50:28.521694+0000 mon.a (mon.0) 1717 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:50:28.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:28 smithi099 bash[87147]: audit 2024-09-13T14:50:28.521694+0000 mon.a (mon.0) 1717 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:50:28.830 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:28.871 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:28.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:28.979 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:29.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:29.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:28 smithi177 bash[68594]: audit 2024-09-13T14:50:28.521694+0000 mon.a (mon.0) 1717 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:50:29.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:28 smithi177 bash[68594]: audit 2024-09-13T14:50:28.521694+0000 mon.a (mon.0) 1717 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:50:29.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:29.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:29.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:29.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:29.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:29.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:29.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:29.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:29.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:29.540 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:29.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:29.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:29.670 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:29.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:29.774 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:29.782 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:29 smithi099 bash[84576]: cluster 2024-09-13T14:50:28.553837+0000 mgr.y (mgr.26252) 725 : cluster [DBG] pgmap v1200: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 16521/150459 objects degraded (10.980%); 3312/150459 objects misplaced (2.201%); 5.7 MiB/s, 91 objects/s recovering 2024-09-13T14:50:29.783 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:29 smithi099 bash[84576]: cluster 2024-09-13T14:50:28.553837+0000 mgr.y (mgr.26252) 725 : cluster [DBG] pgmap v1200: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 16521/150459 objects degraded (10.980%); 3312/150459 objects misplaced (2.201%); 5.7 MiB/s, 91 objects/s recovering 2024-09-13T14:50:29.783 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:29 smithi099 bash[87147]: cluster 2024-09-13T14:50:28.553837+0000 mgr.y (mgr.26252) 725 : cluster [DBG] pgmap v1200: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 16521/150459 objects degraded (10.980%); 3312/150459 objects misplaced (2.201%); 5.7 MiB/s, 91 objects/s recovering 2024-09-13T14:50:29.783 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:29 smithi099 bash[87147]: cluster 2024-09-13T14:50:28.553837+0000 mgr.y (mgr.26252) 725 : cluster [DBG] pgmap v1200: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 16521/150459 objects degraded (10.980%); 3312/150459 objects misplaced (2.201%); 5.7 MiB/s, 91 objects/s recovering 2024-09-13T14:50:29.794 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:29.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:29 smithi177 bash[68594]: cluster 2024-09-13T14:50:28.553837+0000 mgr.y (mgr.26252) 725 : cluster [DBG] pgmap v1200: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 16521/150459 objects degraded (10.980%); 3312/150459 objects misplaced (2.201%); 5.7 MiB/s, 91 objects/s recovering 2024-09-13T14:50:29.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:29 smithi177 bash[68594]: cluster 2024-09-13T14:50:28.553837+0000 mgr.y (mgr.26252) 725 : cluster [DBG] pgmap v1200: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 16521/150459 objects degraded (10.980%); 3312/150459 objects misplaced (2.201%); 5.7 MiB/s, 91 objects/s recovering 2024-09-13T14:50:29.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:29.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:30.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:30.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:30.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:30.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:30.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:30.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:30.278 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:30.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:30.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:30.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:30.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:30.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:30.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:30.546 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:50:30 smithi177 bash[61439]: ts=2024-09-13T14:50:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:30.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:30.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:30.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:30.696 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:30.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:30.801 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:30.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:30.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:30.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:30.992 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:30.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:31.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:31.121 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:31.226 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:31.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:31.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:31.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:31.409 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:31.514 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:31.543 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:31.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:31.658 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:31 smithi177 bash[68594]: cluster 2024-09-13T14:50:30.554571+0000 mgr.y (mgr.26252) 726 : cluster [DBG] pgmap v1201: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 16521/145881 objects degraded (11.325%); 3312/145881 objects misplaced (2.270%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:31.658 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:31 smithi177 bash[68594]: cluster 2024-09-13T14:50:30.554571+0000 mgr.y (mgr.26252) 726 : cluster [DBG] pgmap v1201: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 16521/145881 objects degraded (11.325%); 3312/145881 objects misplaced (2.270%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:31.658 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:31 smithi177 bash[68594]: cluster 2024-09-13T14:50:31.339785+0000 mon.a (mon.0) 1718 : cluster [WRN] Health check update: Degraded data redundancy: 16521/145881 objects degraded (11.325%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:31.658 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:31 smithi177 bash[68594]: cluster 2024-09-13T14:50:31.339785+0000 mon.a (mon.0) 1718 : cluster [WRN] Health check update: Degraded data redundancy: 16521/145881 objects degraded (11.325%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:31.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:31.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:31.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:31.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:31.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:31.916 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:31.925 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:31 smithi099 bash[84576]: cluster 2024-09-13T14:50:30.554571+0000 mgr.y (mgr.26252) 726 : cluster [DBG] pgmap v1201: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 16521/145881 objects degraded (11.325%); 3312/145881 objects misplaced (2.270%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:31.925 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:31 smithi099 bash[84576]: cluster 2024-09-13T14:50:30.554571+0000 mgr.y (mgr.26252) 726 : cluster [DBG] pgmap v1201: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 16521/145881 objects degraded (11.325%); 3312/145881 objects misplaced (2.270%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:31.925 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:31 smithi099 bash[84576]: cluster 2024-09-13T14:50:31.339785+0000 mon.a (mon.0) 1718 : cluster [WRN] Health check update: Degraded data redundancy: 16521/145881 objects degraded (11.325%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:31.925 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:31 smithi099 bash[84576]: cluster 2024-09-13T14:50:31.339785+0000 mon.a (mon.0) 1718 : cluster [WRN] Health check update: Degraded data redundancy: 16521/145881 objects degraded (11.325%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:31.925 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:31 smithi099 bash[87147]: cluster 2024-09-13T14:50:30.554571+0000 mgr.y (mgr.26252) 726 : cluster [DBG] pgmap v1201: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 16521/145881 objects degraded (11.325%); 3312/145881 objects misplaced (2.270%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:31.926 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:31 smithi099 bash[87147]: cluster 2024-09-13T14:50:30.554571+0000 mgr.y (mgr.26252) 726 : cluster [DBG] pgmap v1201: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 16521/145881 objects degraded (11.325%); 3312/145881 objects misplaced (2.270%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:31.926 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:31 smithi099 bash[87147]: cluster 2024-09-13T14:50:31.339785+0000 mon.a (mon.0) 1718 : cluster [WRN] Health check update: Degraded data redundancy: 16521/145881 objects degraded (11.325%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:31.926 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:31 smithi099 bash[87147]: cluster 2024-09-13T14:50:31.339785+0000 mon.a (mon.0) 1718 : cluster [WRN] Health check update: Degraded data redundancy: 16521/145881 objects degraded (11.325%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:31.948 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:31.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:32.053 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:32.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:32.189 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:32.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:32.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:32.323 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:32.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:32.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:32.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:32.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:32.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:32.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:32.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:32.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:32.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:32.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:32.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:32.783 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:50:32.783 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:32.801 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:50:32.801 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:32.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:32.869 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:50:32.869 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:32.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:32.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:32.905 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:50:32.906 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:32.944 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:50:32.944 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:32.970 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:32.970 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:50:32.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:32.995 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:50:32.996 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:33.017 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:33.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:33.028 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:50:33.028 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:33.065 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:33.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:33.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:33.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:33.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:33.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:33.396 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:33.415 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:33.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:33.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:33.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:33.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:33.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:33.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:33.684 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[84576]: cluster 2024-09-13T14:50:32.555138+0000 mgr.y (mgr.26252) 727 : cluster [DBG] pgmap v1202: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 16521/144240 objects degraded (11.454%); 3312/144240 objects misplaced (2.296%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:33.684 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[84576]: cluster 2024-09-13T14:50:32.555138+0000 mgr.y (mgr.26252) 727 : cluster [DBG] pgmap v1202: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 16521/144240 objects degraded (11.454%); 3312/144240 objects misplaced (2.296%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:33.684 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[84576]: audit 2024-09-13T14:50:32.765754+0000 mon.a (mon.0) 1719 : audit [DBG] from='client.? 172.21.15.99:0/2322655562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:33.684 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[84576]: audit 2024-09-13T14:50:32.765754+0000 mon.a (mon.0) 1719 : audit [DBG] from='client.? 172.21.15.99:0/2322655562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:33.684 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[84576]: audit 2024-09-13T14:50:33.274410+0000 mon.a (mon.0) 1720 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:33.684 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[84576]: audit 2024-09-13T14:50:33.274410+0000 mon.a (mon.0) 1720 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:33.684 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[87147]: cluster 2024-09-13T14:50:32.555138+0000 mgr.y (mgr.26252) 727 : cluster [DBG] pgmap v1202: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 16521/144240 objects degraded (11.454%); 3312/144240 objects misplaced (2.296%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:33.685 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[87147]: cluster 2024-09-13T14:50:32.555138+0000 mgr.y (mgr.26252) 727 : cluster [DBG] pgmap v1202: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 16521/144240 objects degraded (11.454%); 3312/144240 objects misplaced (2.296%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:33.685 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[87147]: audit 2024-09-13T14:50:32.765754+0000 mon.a (mon.0) 1719 : audit [DBG] from='client.? 172.21.15.99:0/2322655562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:33.685 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[87147]: audit 2024-09-13T14:50:32.765754+0000 mon.a (mon.0) 1719 : audit [DBG] from='client.? 172.21.15.99:0/2322655562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:33.685 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[87147]: audit 2024-09-13T14:50:33.274410+0000 mon.a (mon.0) 1720 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:33.685 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:33 smithi099 bash[87147]: audit 2024-09-13T14:50:33.274410+0000 mon.a (mon.0) 1720 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:33.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:33.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:33.804 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:33.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:33.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:33.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:33 smithi177 bash[68594]: cluster 2024-09-13T14:50:32.555138+0000 mgr.y (mgr.26252) 727 : cluster [DBG] pgmap v1202: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 16521/144240 objects degraded (11.454%); 3312/144240 objects misplaced (2.296%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:33.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:33 smithi177 bash[68594]: cluster 2024-09-13T14:50:32.555138+0000 mgr.y (mgr.26252) 727 : cluster [DBG] pgmap v1202: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 16521/144240 objects degraded (11.454%); 3312/144240 objects misplaced (2.296%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:50:33.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:33 smithi177 bash[68594]: audit 2024-09-13T14:50:32.765754+0000 mon.a (mon.0) 1719 : audit [DBG] from='client.? 172.21.15.99:0/2322655562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:33.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:33 smithi177 bash[68594]: audit 2024-09-13T14:50:32.765754+0000 mon.a (mon.0) 1719 : audit [DBG] from='client.? 172.21.15.99:0/2322655562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:33.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:33 smithi177 bash[68594]: audit 2024-09-13T14:50:33.274410+0000 mon.a (mon.0) 1720 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:33.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:33 smithi177 bash[68594]: audit 2024-09-13T14:50:33.274410+0000 mon.a (mon.0) 1720 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:33.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:34.027 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:34.059 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:34.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:34.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:34.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:34.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:34.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:34.327 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:34.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:34.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:34.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:34.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:34.575 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:34.607 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:34.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:34.713 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:34.745 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:34.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:34.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:34.877 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:34.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:35.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:35.123 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:35.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:35.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:35.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:35.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:35.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:35.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:35.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:35.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:35.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:35.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:35.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:35.631 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[84576]: cluster 2024-09-13T14:50:34.555789+0000 mgr.y (mgr.26252) 728 : cluster [DBG] pgmap v1203: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:50:35.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:35.670 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:35 smithi177 bash[68594]: cluster 2024-09-13T14:50:34.555789+0000 mgr.y (mgr.26252) 728 : cluster [DBG] pgmap v1203: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:50:35.670 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:35 smithi177 bash[68594]: cluster 2024-09-13T14:50:34.555789+0000 mgr.y (mgr.26252) 728 : cluster [DBG] pgmap v1203: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:50:35.671 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:35 smithi177 bash[68594]: audit 2024-09-13T14:50:34.963276+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:35.671 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:35 smithi177 bash[68594]: audit 2024-09-13T14:50:34.963276+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:35.671 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:35 smithi177 bash[68594]: audit 2024-09-13T14:50:34.973050+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:35.671 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:35 smithi177 bash[68594]: audit 2024-09-13T14:50:34.973050+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.703 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:35.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:35.804 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:35.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:35.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:35.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:35.963 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:36.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:36.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[84576]: cluster 2024-09-13T14:50:34.555789+0000 mgr.y (mgr.26252) 728 : cluster [DBG] pgmap v1203: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:50:36.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[84576]: audit 2024-09-13T14:50:34.963276+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:36.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[84576]: audit 2024-09-13T14:50:34.963276+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:36.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[84576]: audit 2024-09-13T14:50:34.973050+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:36.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[84576]: audit 2024-09-13T14:50:34.973050+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:36.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[87147]: cluster 2024-09-13T14:50:34.555789+0000 mgr.y (mgr.26252) 728 : cluster [DBG] pgmap v1203: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:50:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[87147]: cluster 2024-09-13T14:50:34.555789+0000 mgr.y (mgr.26252) 728 : cluster [DBG] pgmap v1203: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:50:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[87147]: audit 2024-09-13T14:50:34.963276+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[87147]: audit 2024-09-13T14:50:34.963276+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[87147]: audit 2024-09-13T14:50:34.973050+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:36.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:35 smithi099 bash[87147]: audit 2024-09-13T14:50:34.973050+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:36.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:36.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:36.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:36.238 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:36.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:36.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:36.371 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:36.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:36.476 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:36.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:36.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:36.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:36.652 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:36.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:36.760 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:36.769 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:36 smithi177 bash[68594]: cluster 2024-09-13T14:50:36.340406+0000 mon.a (mon.0) 1723 : cluster [WRN] Health check update: Degraded data redundancy: 16633/137004 objects degraded (12.141%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:36.769 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:36 smithi177 bash[68594]: cluster 2024-09-13T14:50:36.340406+0000 mon.a (mon.0) 1723 : cluster [WRN] Health check update: Degraded data redundancy: 16633/137004 objects degraded (12.141%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:36.785 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:36.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:36.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:36.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:36.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:37.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:37.031 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:36 smithi099 bash[84576]: cluster 2024-09-13T14:50:36.340406+0000 mon.a (mon.0) 1723 : cluster [WRN] Health check update: Degraded data redundancy: 16633/137004 objects degraded (12.141%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:37.031 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:36 smithi099 bash[84576]: cluster 2024-09-13T14:50:36.340406+0000 mon.a (mon.0) 1723 : cluster [WRN] Health check update: Degraded data redundancy: 16633/137004 objects degraded (12.141%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:37.031 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:36 smithi099 bash[87147]: cluster 2024-09-13T14:50:36.340406+0000 mon.a (mon.0) 1723 : cluster [WRN] Health check update: Degraded data redundancy: 16633/137004 objects degraded (12.141%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:37.031 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:36 smithi099 bash[87147]: cluster 2024-09-13T14:50:36.340406+0000 mon.a (mon.0) 1723 : cluster [WRN] Health check update: Degraded data redundancy: 16633/137004 objects degraded (12.141%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:37.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:37.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:37.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:37.169 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:37.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:37.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:37.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:50:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:50:37] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-13T14:50:37.397 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:37.434 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:37.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:37.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:37.571 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:37.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:37.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:37.705 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:37.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:37.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:37.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:37.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:37 smithi177 bash[68594]: cluster 2024-09-13T14:50:36.556330+0000 mgr.y (mgr.26252) 729 : cluster [DBG] pgmap v1204: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:37 smithi177 bash[68594]: cluster 2024-09-13T14:50:36.556330+0000 mgr.y (mgr.26252) 729 : cluster [DBG] pgmap v1204: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:37.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:37.952 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:37 smithi099 bash[84576]: cluster 2024-09-13T14:50:36.556330+0000 mgr.y (mgr.26252) 729 : cluster [DBG] pgmap v1204: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:37.953 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:37 smithi099 bash[84576]: cluster 2024-09-13T14:50:36.556330+0000 mgr.y (mgr.26252) 729 : cluster [DBG] pgmap v1204: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:37.956 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:37 smithi099 bash[87147]: cluster 2024-09-13T14:50:36.556330+0000 mgr.y (mgr.26252) 729 : cluster [DBG] pgmap v1204: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:37.956 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:37 smithi099 bash[87147]: cluster 2024-09-13T14:50:36.556330+0000 mgr.y (mgr.26252) 729 : cluster [DBG] pgmap v1204: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 16633/137004 objects degraded (12.141%); 2758/137004 objects misplaced (2.013%); 4.4 MiB/s, 69 objects/s recovering 2024-09-13T14:50:37.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:37.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:38.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:38.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:38.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:38.205 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:38.235 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:38.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:38.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:38.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:38.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:38.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:38.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:38.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:38.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:38.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:38.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:38.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:38.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:38.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:38.863 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:38.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:38.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:38.999 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:39.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:39.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:39.133 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:39.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:39.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:39.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:39.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:39.373 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:39.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:39.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:39.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:39.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:39.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:39.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:39.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:39.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:39.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:39 smithi099 bash[84576]: cluster 2024-09-13T14:50:38.557375+0000 mgr.y (mgr.26252) 730 : cluster [DBG] pgmap v1205: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 16671/132525 objects degraded (12.580%); 2205/132525 objects misplaced (1.664%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:50:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:39 smithi099 bash[84576]: cluster 2024-09-13T14:50:38.557375+0000 mgr.y (mgr.26252) 730 : cluster [DBG] pgmap v1205: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 16671/132525 objects degraded (12.580%); 2205/132525 objects misplaced (1.664%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:50:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:39 smithi099 bash[87147]: cluster 2024-09-13T14:50:38.557375+0000 mgr.y (mgr.26252) 730 : cluster [DBG] pgmap v1205: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 16671/132525 objects degraded (12.580%); 2205/132525 objects misplaced (1.664%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:50:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:39 smithi099 bash[87147]: cluster 2024-09-13T14:50:38.557375+0000 mgr.y (mgr.26252) 730 : cluster [DBG] pgmap v1205: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 16671/132525 objects degraded (12.580%); 2205/132525 objects misplaced (1.664%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:50:39.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:39.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:39.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:40.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:40.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:40.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:40.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:39 smithi177 bash[68594]: cluster 2024-09-13T14:50:38.557375+0000 mgr.y (mgr.26252) 730 : cluster [DBG] pgmap v1205: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 16671/132525 objects degraded (12.580%); 2205/132525 objects misplaced (1.664%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:50:40.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:39 smithi177 bash[68594]: cluster 2024-09-13T14:50:38.557375+0000 mgr.y (mgr.26252) 730 : cluster [DBG] pgmap v1205: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 16671/132525 objects degraded (12.580%); 2205/132525 objects misplaced (1.664%); 7.0 MiB/s, 112 objects/s recovering 2024-09-13T14:50:40.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:40.178 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:40.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:40.283 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:40.310 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:40.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:40.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:40.429 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:50:40 smithi177 bash[61439]: ts=2024-09-13T14:50:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:40.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:40.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:40.575 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:40.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:40.679 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:40.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:40.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:40.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:40.862 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:40.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:41.003 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:41.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:41.109 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:41.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:41.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:41.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:41.296 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:41.401 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:41.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:41.508 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:50:41.531 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:41.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:41.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:41.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:41.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:41.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:41.736 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[84576]: cluster 2024-09-13T14:50:40.558577+0000 mgr.y (mgr.26252) 731 : cluster [DBG] pgmap v1206: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16671/128106 objects degraded (13.013%); 2205/128106 objects misplaced (1.721%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:41.736 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[84576]: cluster 2024-09-13T14:50:40.558577+0000 mgr.y (mgr.26252) 731 : cluster [DBG] pgmap v1206: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16671/128106 objects degraded (13.013%); 2205/128106 objects misplaced (1.721%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:41.736 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[84576]: audit 2024-09-13T14:50:40.976593+0000 mon.a (mon.0) 1724 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.736 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[84576]: audit 2024-09-13T14:50:40.976593+0000 mon.a (mon.0) 1724 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.736 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[84576]: audit 2024-09-13T14:50:40.988058+0000 mon.a (mon.0) 1725 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[84576]: audit 2024-09-13T14:50:40.988058+0000 mon.a (mon.0) 1725 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[84576]: cluster 2024-09-13T14:50:41.341206+0000 mon.a (mon.0) 1726 : cluster [WRN] Health check update: Degraded data redundancy: 16671/128106 objects degraded (13.013%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:41.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[84576]: cluster 2024-09-13T14:50:41.341206+0000 mon.a (mon.0) 1726 : cluster [WRN] Health check update: Degraded data redundancy: 16671/128106 objects degraded (13.013%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:41.738 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[87147]: cluster 2024-09-13T14:50:40.558577+0000 mgr.y (mgr.26252) 731 : cluster [DBG] pgmap v1206: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16671/128106 objects degraded (13.013%); 2205/128106 objects misplaced (1.721%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:41.738 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[87147]: cluster 2024-09-13T14:50:40.558577+0000 mgr.y (mgr.26252) 731 : cluster [DBG] pgmap v1206: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16671/128106 objects degraded (13.013%); 2205/128106 objects misplaced (1.721%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:41.738 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[87147]: audit 2024-09-13T14:50:40.976593+0000 mon.a (mon.0) 1724 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.738 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[87147]: audit 2024-09-13T14:50:40.976593+0000 mon.a (mon.0) 1724 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.738 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[87147]: audit 2024-09-13T14:50:40.988058+0000 mon.a (mon.0) 1725 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.738 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[87147]: audit 2024-09-13T14:50:40.988058+0000 mon.a (mon.0) 1725 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.738 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[87147]: cluster 2024-09-13T14:50:41.341206+0000 mon.a (mon.0) 1726 : cluster [WRN] Health check update: Degraded data redundancy: 16671/128106 objects degraded (13.013%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:41.739 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:41 smithi099 bash[87147]: cluster 2024-09-13T14:50:41.341206+0000 mon.a (mon.0) 1726 : cluster [WRN] Health check update: Degraded data redundancy: 16671/128106 objects degraded (13.013%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:41.796 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:41.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:41.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:41 smithi177 bash[68594]: cluster 2024-09-13T14:50:40.558577+0000 mgr.y (mgr.26252) 731 : cluster [DBG] pgmap v1206: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16671/128106 objects degraded (13.013%); 2205/128106 objects misplaced (1.721%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:41 smithi177 bash[68594]: cluster 2024-09-13T14:50:40.558577+0000 mgr.y (mgr.26252) 731 : cluster [DBG] pgmap v1206: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 16671/128106 objects degraded (13.013%); 2205/128106 objects misplaced (1.721%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:41 smithi177 bash[68594]: audit 2024-09-13T14:50:40.976593+0000 mon.a (mon.0) 1724 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:41 smithi177 bash[68594]: audit 2024-09-13T14:50:40.976593+0000 mon.a (mon.0) 1724 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:41 smithi177 bash[68594]: audit 2024-09-13T14:50:40.988058+0000 mon.a (mon.0) 1725 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:41 smithi177 bash[68594]: audit 2024-09-13T14:50:40.988058+0000 mon.a (mon.0) 1725 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:41 smithi177 bash[68594]: cluster 2024-09-13T14:50:41.341206+0000 mon.a (mon.0) 1726 : cluster [WRN] Health check update: Degraded data redundancy: 16671/128106 objects degraded (13.013%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:41 smithi177 bash[68594]: cluster 2024-09-13T14:50:41.341206+0000 mon.a (mon.0) 1726 : cluster [WRN] Health check update: Degraded data redundancy: 16671/128106 objects degraded (13.013%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:41.951 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:41.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:41.973 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T14:50:41.973 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:50:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:42.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:42.104 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:42.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:42.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:42.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:42.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:42.372 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:42.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:42.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:42.502 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:42.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:42.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:42.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:42.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:42.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:42.759 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:42.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:42.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[84576]: audit 2024-09-13T14:50:41.972775+0000 mon.a (mon.0) 1727 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:42.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[84576]: audit 2024-09-13T14:50:41.972775+0000 mon.a (mon.0) 1727 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:42.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[84576]: audit 2024-09-13T14:50:41.973938+0000 mon.a (mon.0) 1728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:42.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[84576]: audit 2024-09-13T14:50:41.973938+0000 mon.a (mon.0) 1728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:42.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[84576]: audit 2024-09-13T14:50:41.984391+0000 mon.a (mon.0) 1729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[84576]: audit 2024-09-13T14:50:41.984391+0000 mon.a (mon.0) 1729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[84576]: audit 2024-09-13T14:50:42.271753+0000 mon.c (mon.2) 56 : audit [INF] from='client.? 172.21.15.99:0/1579332587' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[84576]: audit 2024-09-13T14:50:42.271753+0000 mon.c (mon.2) 56 : audit [INF] from='client.? 172.21.15.99:0/1579332587' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[84576]: audit 2024-09-13T14:50:42.272924+0000 mon.a (mon.0) 1730 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[84576]: audit 2024-09-13T14:50:42.272924+0000 mon.a (mon.0) 1730 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[87147]: audit 2024-09-13T14:50:41.972775+0000 mon.a (mon.0) 1727 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[87147]: audit 2024-09-13T14:50:41.972775+0000 mon.a (mon.0) 1727 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[87147]: audit 2024-09-13T14:50:41.973938+0000 mon.a (mon.0) 1728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[87147]: audit 2024-09-13T14:50:41.973938+0000 mon.a (mon.0) 1728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[87147]: audit 2024-09-13T14:50:41.984391+0000 mon.a (mon.0) 1729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:42.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[87147]: audit 2024-09-13T14:50:41.984391+0000 mon.a (mon.0) 1729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:42.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[87147]: audit 2024-09-13T14:50:42.271753+0000 mon.c (mon.2) 56 : audit [INF] from='client.? 172.21.15.99:0/1579332587' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[87147]: audit 2024-09-13T14:50:42.271753+0000 mon.c (mon.2) 56 : audit [INF] from='client.? 172.21.15.99:0/1579332587' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[87147]: audit 2024-09-13T14:50:42.272924+0000 mon.a (mon.0) 1730 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:42 smithi099 bash[87147]: audit 2024-09-13T14:50:42.272924+0000 mon.a (mon.0) 1730 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:42.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:42 smithi177 bash[68594]: audit 2024-09-13T14:50:41.972775+0000 mon.a (mon.0) 1727 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:42.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:42 smithi177 bash[68594]: audit 2024-09-13T14:50:41.972775+0000 mon.a (mon.0) 1727 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:50:42.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:42 smithi177 bash[68594]: audit 2024-09-13T14:50:41.973938+0000 mon.a (mon.0) 1728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:42.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:42 smithi177 bash[68594]: audit 2024-09-13T14:50:41.973938+0000 mon.a (mon.0) 1728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:50:42.872 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:42 smithi177 bash[68594]: audit 2024-09-13T14:50:41.984391+0000 mon.a (mon.0) 1729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:42.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:42 smithi177 bash[68594]: audit 2024-09-13T14:50:41.984391+0000 mon.a (mon.0) 1729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:50:42.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:42 smithi177 bash[68594]: audit 2024-09-13T14:50:42.271753+0000 mon.c (mon.2) 56 : audit [INF] from='client.? 172.21.15.99:0/1579332587' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:42 smithi177 bash[68594]: audit 2024-09-13T14:50:42.271753+0000 mon.c (mon.2) 56 : audit [INF] from='client.? 172.21.15.99:0/1579332587' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:42 smithi177 bash[68594]: audit 2024-09-13T14:50:42.272924+0000 mon.a (mon.0) 1730 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.873 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:42 smithi177 bash[68594]: audit 2024-09-13T14:50:42.272924+0000 mon.a (mon.0) 1730 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:42.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:42.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:42.991 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:42.993 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T14:50:43.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:43.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:43.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:43.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:43.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:43.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:43.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:43.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:43.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:43.406 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:43.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:43.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:43.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:43.661 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:43.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:43.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:43.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:43.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:43 smithi177 bash[68594]: cluster 2024-09-13T14:50:42.559158+0000 mgr.y (mgr.26252) 732 : cluster [DBG] pgmap v1207: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 16671/126588 objects degraded (13.169%); 2205/126588 objects misplaced (1.742%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:43 smithi177 bash[68594]: cluster 2024-09-13T14:50:42.559158+0000 mgr.y (mgr.26252) 732 : cluster [DBG] pgmap v1207: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 16671/126588 objects degraded (13.169%); 2205/126588 objects misplaced (1.742%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:43 smithi177 bash[68594]: cluster 2024-09-13T14:50:42.981584+0000 mon.a (mon.0) 1731 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:43 smithi177 bash[68594]: cluster 2024-09-13T14:50:42.981584+0000 mon.a (mon.0) 1731 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:43 smithi177 bash[68594]: audit 2024-09-13T14:50:42.987544+0000 mon.a (mon.0) 1732 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:43 smithi177 bash[68594]: audit 2024-09-13T14:50:42.987544+0000 mon.a (mon.0) 1732 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:43 smithi177 bash[68594]: cluster 2024-09-13T14:50:42.994876+0000 mon.a (mon.0) 1733 : cluster [DBG] osdmap e982: 8 total, 8 up, 7 in 2024-09-13T14:50:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:43 smithi177 bash[68594]: cluster 2024-09-13T14:50:42.994876+0000 mon.a (mon.0) 1733 : cluster [DBG] osdmap e982: 8 total, 8 up, 7 in 2024-09-13T14:50:43.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:43.937 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:43.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:44.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:44.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:44.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:44.157 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[84576]: cluster 2024-09-13T14:50:42.559158+0000 mgr.y (mgr.26252) 732 : cluster [DBG] pgmap v1207: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 16671/126588 objects degraded (13.169%); 2205/126588 objects misplaced (1.742%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:44.158 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[84576]: cluster 2024-09-13T14:50:42.559158+0000 mgr.y (mgr.26252) 732 : cluster [DBG] pgmap v1207: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 16671/126588 objects degraded (13.169%); 2205/126588 objects misplaced (1.742%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:44.158 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[84576]: cluster 2024-09-13T14:50:42.981584+0000 mon.a (mon.0) 1731 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:44.158 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[84576]: cluster 2024-09-13T14:50:42.981584+0000 mon.a (mon.0) 1731 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:44.158 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[84576]: audit 2024-09-13T14:50:42.987544+0000 mon.a (mon.0) 1732 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:44.158 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[84576]: audit 2024-09-13T14:50:42.987544+0000 mon.a (mon.0) 1732 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:44.158 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[84576]: cluster 2024-09-13T14:50:42.994876+0000 mon.a (mon.0) 1733 : cluster [DBG] osdmap e982: 8 total, 8 up, 7 in 2024-09-13T14:50:44.158 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[84576]: cluster 2024-09-13T14:50:42.994876+0000 mon.a (mon.0) 1733 : cluster [DBG] osdmap e982: 8 total, 8 up, 7 in 2024-09-13T14:50:44.158 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[87147]: cluster 2024-09-13T14:50:42.559158+0000 mgr.y (mgr.26252) 732 : cluster [DBG] pgmap v1207: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 16671/126588 objects degraded (13.169%); 2205/126588 objects misplaced (1.742%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:44.158 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[87147]: cluster 2024-09-13T14:50:42.559158+0000 mgr.y (mgr.26252) 732 : cluster [DBG] pgmap v1207: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 16671/126588 objects degraded (13.169%); 2205/126588 objects misplaced (1.742%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:50:44.159 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[87147]: cluster 2024-09-13T14:50:42.981584+0000 mon.a (mon.0) 1731 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:44.159 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[87147]: cluster 2024-09-13T14:50:42.981584+0000 mon.a (mon.0) 1731 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:44.159 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[87147]: audit 2024-09-13T14:50:42.987544+0000 mon.a (mon.0) 1732 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:44.159 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[87147]: audit 2024-09-13T14:50:42.987544+0000 mon.a (mon.0) 1732 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:44.159 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[87147]: cluster 2024-09-13T14:50:42.994876+0000 mon.a (mon.0) 1733 : cluster [DBG] osdmap e982: 8 total, 8 up, 7 in 2024-09-13T14:50:44.159 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:43 smithi099 bash[87147]: cluster 2024-09-13T14:50:42.994876+0000 mon.a (mon.0) 1733 : cluster [DBG] osdmap e982: 8 total, 8 up, 7 in 2024-09-13T14:50:44.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:44.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:44.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:44.348 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:44.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:44.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:44.485 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:44.591 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:44.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:44.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:44.736 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:44.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:44.839 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:44.864 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:44.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:44.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:44.987 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:44.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:45.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:45 smithi099 bash[84576]: cluster 2024-09-13T14:50:43.997520+0000 mon.a (mon.0) 1734 : cluster [DBG] osdmap e983: 8 total, 8 up, 7 in 2024-09-13T14:50:45.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:45 smithi099 bash[84576]: cluster 2024-09-13T14:50:43.997520+0000 mon.a (mon.0) 1734 : cluster [DBG] osdmap e983: 8 total, 8 up, 7 in 2024-09-13T14:50:45.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:45 smithi099 bash[84576]: cluster 2024-09-13T14:50:44.559806+0000 mgr.y (mgr.26252) 733 : cluster [DBG] pgmap v1210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 7.6 MiB/s, 121 objects/s recovering 2024-09-13T14:50:45.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:45 smithi099 bash[84576]: cluster 2024-09-13T14:50:44.559806+0000 mgr.y (mgr.26252) 733 : cluster [DBG] pgmap v1210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 7.6 MiB/s, 121 objects/s recovering 2024-09-13T14:50:45.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:45 smithi099 bash[87147]: cluster 2024-09-13T14:50:43.997520+0000 mon.a (mon.0) 1734 : cluster [DBG] osdmap e983: 8 total, 8 up, 7 in 2024-09-13T14:50:45.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:45 smithi099 bash[87147]: cluster 2024-09-13T14:50:43.997520+0000 mon.a (mon.0) 1734 : cluster [DBG] osdmap e983: 8 total, 8 up, 7 in 2024-09-13T14:50:45.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:45 smithi099 bash[87147]: cluster 2024-09-13T14:50:44.559806+0000 mgr.y (mgr.26252) 733 : cluster [DBG] pgmap v1210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 7.6 MiB/s, 121 objects/s recovering 2024-09-13T14:50:45.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:45 smithi099 bash[87147]: cluster 2024-09-13T14:50:44.559806+0000 mgr.y (mgr.26252) 733 : cluster [DBG] pgmap v1210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 7.6 MiB/s, 121 objects/s recovering 2024-09-13T14:50:45.091 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:45.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:45 smithi177 bash[68594]: cluster 2024-09-13T14:50:43.997520+0000 mon.a (mon.0) 1734 : cluster [DBG] osdmap e983: 8 total, 8 up, 7 in 2024-09-13T14:50:45.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:45 smithi177 bash[68594]: cluster 2024-09-13T14:50:43.997520+0000 mon.a (mon.0) 1734 : cluster [DBG] osdmap e983: 8 total, 8 up, 7 in 2024-09-13T14:50:45.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:45 smithi177 bash[68594]: cluster 2024-09-13T14:50:44.559806+0000 mgr.y (mgr.26252) 733 : cluster [DBG] pgmap v1210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 7.6 MiB/s, 121 objects/s recovering 2024-09-13T14:50:45.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:45 smithi177 bash[68594]: cluster 2024-09-13T14:50:44.559806+0000 mgr.y (mgr.26252) 733 : cluster [DBG] pgmap v1210: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 7.6 MiB/s, 121 objects/s recovering 2024-09-13T14:50:45.113 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:45.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:45.216 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:45.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:45.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:45.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:45.381 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:45.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:45.485 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:45.502 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:45.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:45.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:45.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:45.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:45.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:45.762 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:45.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:45.864 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:45.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:45.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:46.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:46.029 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:46.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:46.136 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:46.144 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:46 smithi177 bash[68594]: cluster 2024-09-13T14:50:45.011747+0000 mon.a (mon.0) 1735 : cluster [DBG] osdmap e984: 8 total, 8 up, 7 in 2024-09-13T14:50:46.144 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:46 smithi177 bash[68594]: cluster 2024-09-13T14:50:45.011747+0000 mon.a (mon.0) 1735 : cluster [DBG] osdmap e984: 8 total, 8 up, 7 in 2024-09-13T14:50:46.162 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:46.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:46.266 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:46.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:46.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:46.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:46.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:46.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:46.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:46.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:46.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:46 smithi099 bash[84576]: cluster 2024-09-13T14:50:45.011747+0000 mon.a (mon.0) 1735 : cluster [DBG] osdmap e984: 8 total, 8 up, 7 in 2024-09-13T14:50:46.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:46 smithi099 bash[84576]: cluster 2024-09-13T14:50:45.011747+0000 mon.a (mon.0) 1735 : cluster [DBG] osdmap e984: 8 total, 8 up, 7 in 2024-09-13T14:50:46.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:46 smithi099 bash[87147]: cluster 2024-09-13T14:50:45.011747+0000 mon.a (mon.0) 1735 : cluster [DBG] osdmap e984: 8 total, 8 up, 7 in 2024-09-13T14:50:46.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:46 smithi099 bash[87147]: cluster 2024-09-13T14:50:45.011747+0000 mon.a (mon.0) 1735 : cluster [DBG] osdmap e984: 8 total, 8 up, 7 in 2024-09-13T14:50:46.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:46.685 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:46.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:46.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:50:46.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:46.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:46.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:46.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:46.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:46.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:47.018 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:47 smithi099 bash[87147]: cluster 2024-09-13T14:50:46.342106+0000 mon.a (mon.0) 1736 : cluster [WRN] Health check update: Degraded data redundancy: 16722/119175 objects degraded (14.031%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:47.019 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:47 smithi099 bash[87147]: cluster 2024-09-13T14:50:46.342106+0000 mon.a (mon.0) 1736 : cluster [WRN] Health check update: Degraded data redundancy: 16722/119175 objects degraded (14.031%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:47.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:47.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:47.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:47.130 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:50:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:50:47] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-13T14:50:47.131 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:47 smithi099 bash[84576]: cluster 2024-09-13T14:50:46.342106+0000 mon.a (mon.0) 1736 : cluster [WRN] Health check update: Degraded data redundancy: 16722/119175 objects degraded (14.031%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:47.131 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:47 smithi099 bash[84576]: cluster 2024-09-13T14:50:46.342106+0000 mon.a (mon.0) 1736 : cluster [WRN] Health check update: Degraded data redundancy: 16722/119175 objects degraded (14.031%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:47.131 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:47 smithi099 bash[84576]: cluster 2024-09-13T14:50:46.560313+0000 mgr.y (mgr.26252) 734 : cluster [DBG] pgmap v1212: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:50:47.131 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:47 smithi099 bash[84576]: cluster 2024-09-13T14:50:46.560313+0000 mgr.y (mgr.26252) 734 : cluster [DBG] pgmap v1212: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:50:47.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:47 smithi099 bash[87147]: cluster 2024-09-13T14:50:46.560313+0000 mgr.y (mgr.26252) 734 : cluster [DBG] pgmap v1212: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:50:47.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:47 smithi099 bash[87147]: cluster 2024-09-13T14:50:46.560313+0000 mgr.y (mgr.26252) 734 : cluster [DBG] pgmap v1212: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:50:47.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:47.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:47.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:47.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:47 smithi177 bash[68594]: cluster 2024-09-13T14:50:46.342106+0000 mon.a (mon.0) 1736 : cluster [WRN] Health check update: Degraded data redundancy: 16722/119175 objects degraded (14.031%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:47.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:47 smithi177 bash[68594]: cluster 2024-09-13T14:50:46.342106+0000 mon.a (mon.0) 1736 : cluster [WRN] Health check update: Degraded data redundancy: 16722/119175 objects degraded (14.031%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:47.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:47 smithi177 bash[68594]: cluster 2024-09-13T14:50:46.560313+0000 mgr.y (mgr.26252) 734 : cluster [DBG] pgmap v1212: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:50:47.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:47 smithi177 bash[68594]: cluster 2024-09-13T14:50:46.560313+0000 mgr.y (mgr.26252) 734 : cluster [DBG] pgmap v1212: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 16722/119175 objects degraded (14.031%); 1700/119175 objects misplaced (1.426%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:50:47.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:47.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:47.463 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:47.503 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:47.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:47.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:47.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:47.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:47.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:47.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:47.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:47.873 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:47.903 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:47.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:48.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:48.029 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:48.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:48.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:48.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:48.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:48.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:48.311 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:48.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:48.418 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:48.444 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:48.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:48.547 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:48.557 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:48 smithi099 bash[84576]: audit 2024-09-13T14:50:48.277093+0000 mon.a (mon.0) 1737 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:48.557 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:48 smithi099 bash[84576]: audit 2024-09-13T14:50:48.277093+0000 mon.a (mon.0) 1737 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:48.558 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:48 smithi099 bash[87147]: audit 2024-09-13T14:50:48.277093+0000 mon.a (mon.0) 1737 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:48.558 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:48 smithi099 bash[87147]: audit 2024-09-13T14:50:48.277093+0000 mon.a (mon.0) 1737 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:48.590 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:48.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:48.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:48 smithi177 bash[68594]: audit 2024-09-13T14:50:48.277093+0000 mon.a (mon.0) 1737 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:48.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:48 smithi177 bash[68594]: audit 2024-09-13T14:50:48.277093+0000 mon.a (mon.0) 1737 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:48.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:50:48.693 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:48.729 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:48.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:48.834 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:48.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:48.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:48.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:49.009 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:49.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:49.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:49.149 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:49.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:49.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:49.277 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:49.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:49.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:49.401 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:49.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:49.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:49.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:49.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:49.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:49 smithi099 bash[84576]: cluster 2024-09-13T14:50:48.562371+0000 mgr.y (mgr.26252) 735 : cluster [DBG] pgmap v1213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 16738/114633 objects degraded (14.601%); 1412/114633 objects misplaced (1.232%); 7.6 MiB/s, 120 objects/s recovering 2024-09-13T14:50:49.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:49 smithi099 bash[84576]: cluster 2024-09-13T14:50:48.562371+0000 mgr.y (mgr.26252) 735 : cluster [DBG] pgmap v1213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 16738/114633 objects degraded (14.601%); 1412/114633 objects misplaced (1.232%); 7.6 MiB/s, 120 objects/s recovering 2024-09-13T14:50:49.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:49 smithi099 bash[87147]: cluster 2024-09-13T14:50:48.562371+0000 mgr.y (mgr.26252) 735 : cluster [DBG] pgmap v1213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 16738/114633 objects degraded (14.601%); 1412/114633 objects misplaced (1.232%); 7.6 MiB/s, 120 objects/s recovering 2024-09-13T14:50:49.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:49 smithi099 bash[87147]: cluster 2024-09-13T14:50:48.562371+0000 mgr.y (mgr.26252) 735 : cluster [DBG] pgmap v1213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 16738/114633 objects degraded (14.601%); 1412/114633 objects misplaced (1.232%); 7.6 MiB/s, 120 objects/s recovering 2024-09-13T14:50:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:49 smithi177 bash[68594]: cluster 2024-09-13T14:50:48.562371+0000 mgr.y (mgr.26252) 735 : cluster [DBG] pgmap v1213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 16738/114633 objects degraded (14.601%); 1412/114633 objects misplaced (1.232%); 7.6 MiB/s, 120 objects/s recovering 2024-09-13T14:50:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:49 smithi177 bash[68594]: cluster 2024-09-13T14:50:48.562371+0000 mgr.y (mgr.26252) 735 : cluster [DBG] pgmap v1213: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 16738/114633 objects degraded (14.601%); 1412/114633 objects misplaced (1.232%); 7.6 MiB/s, 120 objects/s recovering 2024-09-13T14:50:49.642 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:49.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:49.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:49.775 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:49.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:49.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:49.902 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:49.926 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:49.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:50.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:50.061 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:50.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:50.197 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:50.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:50.304 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:50.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:50.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:50.347 INFO:teuthology.orchestra.run.smithi099.stderr:set pool 41 pg_num to 26 2024-09-13T14:50:50.446 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:50.480 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:50.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:50.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:50.599 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:50:50 smithi177 bash[61439]: ts=2024-09-13T14:50:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:50:50.599 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:50 smithi177 bash[68594]: audit 2024-09-13T14:50:49.573781+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.99:0/3643711439' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.599 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:50 smithi177 bash[68594]: audit 2024-09-13T14:50:49.573781+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.99:0/3643711439' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.599 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:50 smithi177 bash[68594]: audit 2024-09-13T14:50:49.576605+0000 mon.a (mon.0) 1738 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.600 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:50 smithi177 bash[68594]: audit 2024-09-13T14:50:49.576605+0000 mon.a (mon.0) 1738 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.623 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:50.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:50.692 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:50 smithi099 bash[84576]: audit 2024-09-13T14:50:49.573781+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.99:0/3643711439' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.692 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:50 smithi099 bash[84576]: audit 2024-09-13T14:50:49.573781+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.99:0/3643711439' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.692 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:50 smithi099 bash[84576]: audit 2024-09-13T14:50:49.576605+0000 mon.a (mon.0) 1738 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.693 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:50 smithi099 bash[84576]: audit 2024-09-13T14:50:49.576605+0000 mon.a (mon.0) 1738 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.693 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:50 smithi099 bash[87147]: audit 2024-09-13T14:50:49.573781+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.99:0/3643711439' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.693 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:50 smithi099 bash[87147]: audit 2024-09-13T14:50:49.573781+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.99:0/3643711439' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.693 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:50 smithi099 bash[87147]: audit 2024-09-13T14:50:49.576605+0000 mon.a (mon.0) 1738 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.693 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:50 smithi099 bash[87147]: audit 2024-09-13T14:50:49.576605+0000 mon.a (mon.0) 1738 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:50:50.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:50.752 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:50.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:50.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:50.907 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:50.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:51.015 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:51.036 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:51.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:51.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:51.170 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:51.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:51.276 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:51.304 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:51.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:51.407 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:51.498 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:51.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:51.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[84576]: audit 2024-09-13T14:50:50.345621+0000 mon.a (mon.0) 1739 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:50:51.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[84576]: audit 2024-09-13T14:50:50.345621+0000 mon.a (mon.0) 1739 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[84576]: cluster 2024-09-13T14:50:50.358181+0000 mon.a (mon.0) 1740 : cluster [DBG] osdmap e985: 8 total, 8 up, 7 in 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[84576]: cluster 2024-09-13T14:50:50.358181+0000 mon.a (mon.0) 1740 : cluster [DBG] osdmap e985: 8 total, 8 up, 7 in 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[84576]: cluster 2024-09-13T14:50:50.562853+0000 mgr.y (mgr.26252) 736 : cluster [DBG] pgmap v1215: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 16738/110028 objects degraded (15.212%); 1412/110028 objects misplaced (1.283%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[84576]: cluster 2024-09-13T14:50:50.562853+0000 mgr.y (mgr.26252) 736 : cluster [DBG] pgmap v1215: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 16738/110028 objects degraded (15.212%); 1412/110028 objects misplaced (1.283%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[84576]: audit 2024-09-13T14:50:50.563577+0000 mon.a (mon.0) 1741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[84576]: audit 2024-09-13T14:50:50.563577+0000 mon.a (mon.0) 1741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[84576]: cluster 2024-09-13T14:50:51.342922+0000 mon.a (mon.0) 1742 : cluster [WRN] Health check update: Degraded data redundancy: 16738/114633 objects degraded (14.601%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[84576]: cluster 2024-09-13T14:50:51.342922+0000 mon.a (mon.0) 1742 : cluster [WRN] Health check update: Degraded data redundancy: 16738/114633 objects degraded (14.601%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[87147]: audit 2024-09-13T14:50:50.345621+0000 mon.a (mon.0) 1739 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[87147]: audit 2024-09-13T14:50:50.345621+0000 mon.a (mon.0) 1739 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:50:51.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[87147]: cluster 2024-09-13T14:50:50.358181+0000 mon.a (mon.0) 1740 : cluster [DBG] osdmap e985: 8 total, 8 up, 7 in 2024-09-13T14:50:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[87147]: cluster 2024-09-13T14:50:50.358181+0000 mon.a (mon.0) 1740 : cluster [DBG] osdmap e985: 8 total, 8 up, 7 in 2024-09-13T14:50:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[87147]: cluster 2024-09-13T14:50:50.562853+0000 mgr.y (mgr.26252) 736 : cluster [DBG] pgmap v1215: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 16738/110028 objects degraded (15.212%); 1412/110028 objects misplaced (1.283%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:50:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[87147]: cluster 2024-09-13T14:50:50.562853+0000 mgr.y (mgr.26252) 736 : cluster [DBG] pgmap v1215: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 16738/110028 objects degraded (15.212%); 1412/110028 objects misplaced (1.283%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:50:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[87147]: audit 2024-09-13T14:50:50.563577+0000 mon.a (mon.0) 1741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:50:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[87147]: audit 2024-09-13T14:50:50.563577+0000 mon.a (mon.0) 1741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:50:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[87147]: cluster 2024-09-13T14:50:51.342922+0000 mon.a (mon.0) 1742 : cluster [WRN] Health check update: Degraded data redundancy: 16738/114633 objects degraded (14.601%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:51.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:51 smithi099 bash[87147]: cluster 2024-09-13T14:50:51.342922+0000 mon.a (mon.0) 1742 : cluster [WRN] Health check update: Degraded data redundancy: 16738/114633 objects degraded (14.601%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:51.602 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:51 smithi177 bash[68594]: audit 2024-09-13T14:50:50.345621+0000 mon.a (mon.0) 1739 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:50:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:51 smithi177 bash[68594]: audit 2024-09-13T14:50:50.345621+0000 mon.a (mon.0) 1739 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:50:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:51 smithi177 bash[68594]: cluster 2024-09-13T14:50:50.358181+0000 mon.a (mon.0) 1740 : cluster [DBG] osdmap e985: 8 total, 8 up, 7 in 2024-09-13T14:50:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:51 smithi177 bash[68594]: cluster 2024-09-13T14:50:50.358181+0000 mon.a (mon.0) 1740 : cluster [DBG] osdmap e985: 8 total, 8 up, 7 in 2024-09-13T14:50:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:51 smithi177 bash[68594]: cluster 2024-09-13T14:50:50.562853+0000 mgr.y (mgr.26252) 736 : cluster [DBG] pgmap v1215: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 16738/110028 objects degraded (15.212%); 1412/110028 objects misplaced (1.283%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:50:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:51 smithi177 bash[68594]: cluster 2024-09-13T14:50:50.562853+0000 mgr.y (mgr.26252) 736 : cluster [DBG] pgmap v1215: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 16738/110028 objects degraded (15.212%); 1412/110028 objects misplaced (1.283%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:50:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:51 smithi177 bash[68594]: audit 2024-09-13T14:50:50.563577+0000 mon.a (mon.0) 1741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:50:51.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:51 smithi177 bash[68594]: audit 2024-09-13T14:50:50.563577+0000 mon.a (mon.0) 1741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:50:51.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:51 smithi177 bash[68594]: cluster 2024-09-13T14:50:51.342922+0000 mon.a (mon.0) 1742 : cluster [WRN] Health check update: Degraded data redundancy: 16738/114633 objects degraded (14.601%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:51.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:51 smithi177 bash[68594]: cluster 2024-09-13T14:50:51.342922+0000 mon.a (mon.0) 1742 : cluster [WRN] Health check update: Degraded data redundancy: 16738/114633 objects degraded (14.601%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:51.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:51.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:51.769 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:51.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:51.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:51.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:51.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:52.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:52.052 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:52.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:52.154 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:52.177 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:52.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:52.280 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:52.308 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:52.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:52.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:52.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:52.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:52.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:52.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:52 smithi177 bash[68594]: audit 2024-09-13T14:50:51.370513+0000 mon.a (mon.0) 1743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:50:52.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:52 smithi177 bash[68594]: audit 2024-09-13T14:50:51.370513+0000 mon.a (mon.0) 1743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:50:52.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:52 smithi177 bash[68594]: cluster 2024-09-13T14:50:51.378876+0000 mon.a (mon.0) 1744 : cluster [DBG] osdmap e986: 8 total, 8 up, 7 in 2024-09-13T14:50:52.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:52 smithi177 bash[68594]: cluster 2024-09-13T14:50:51.378876+0000 mon.a (mon.0) 1744 : cluster [DBG] osdmap e986: 8 total, 8 up, 7 in 2024-09-13T14:50:52.564 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:52.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:52.670 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:52.684 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:52 smithi099 bash[87147]: audit 2024-09-13T14:50:51.370513+0000 mon.a (mon.0) 1743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:50:52.684 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:52 smithi099 bash[87147]: audit 2024-09-13T14:50:51.370513+0000 mon.a (mon.0) 1743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:50:52.684 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:52 smithi099 bash[87147]: cluster 2024-09-13T14:50:51.378876+0000 mon.a (mon.0) 1744 : cluster [DBG] osdmap e986: 8 total, 8 up, 7 in 2024-09-13T14:50:52.684 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:52 smithi099 bash[87147]: cluster 2024-09-13T14:50:51.378876+0000 mon.a (mon.0) 1744 : cluster [DBG] osdmap e986: 8 total, 8 up, 7 in 2024-09-13T14:50:52.685 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:52 smithi099 bash[84576]: audit 2024-09-13T14:50:51.370513+0000 mon.a (mon.0) 1743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:50:52.685 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:52 smithi099 bash[84576]: audit 2024-09-13T14:50:51.370513+0000 mon.a (mon.0) 1743 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:50:52.685 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:52 smithi099 bash[84576]: cluster 2024-09-13T14:50:51.378876+0000 mon.a (mon.0) 1744 : cluster [DBG] osdmap e986: 8 total, 8 up, 7 in 2024-09-13T14:50:52.685 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:52 smithi099 bash[84576]: cluster 2024-09-13T14:50:51.378876+0000 mon.a (mon.0) 1744 : cluster [DBG] osdmap e986: 8 total, 8 up, 7 in 2024-09-13T14:50:52.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:52.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:52.799 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:52.827 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:52.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:52.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:52.963 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:52.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:53.066 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:53.094 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:53.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:53.096 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:53.174 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:53.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:53.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:53.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:53.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:53.341 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:53.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:53.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:53.455 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.391776+0000 mon.a (mon.0) 1745 : cluster [DBG] osdmap e987: 8 total, 8 up, 7 in 2024-09-13T14:50:53.455 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.391776+0000 mon.a (mon.0) 1745 : cluster [DBG] osdmap e987: 8 total, 8 up, 7 in 2024-09-13T14:50:53.455 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.563314+0000 mgr.y (mgr.26252) 737 : cluster [DBG] pgmap v1218: 99 pgs: 2 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16738/108126 objects degraded (15.480%); 1412/108126 objects misplaced (1.306%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:50:53.455 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.563314+0000 mgr.y (mgr.26252) 737 : cluster [DBG] pgmap v1218: 99 pgs: 2 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16738/108126 objects degraded (15.480%); 1412/108126 objects misplaced (1.306%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:50:53.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.648527+0000 osd.3 (osd.3) 1 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:50:53.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.648527+0000 osd.3 (osd.3) 1 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:50:53.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.947642+0000 osd.5 (osd.5) 3 : cluster [DBG] 41.8 scrub starts 2024-09-13T14:50:53.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.947642+0000 osd.5 (osd.5) 3 : cluster [DBG] 41.8 scrub starts 2024-09-13T14:50:53.456 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.980119+0000 osd.3 (osd.3) 2 : cluster [DBG] 41.3 scrub ok 2024-09-13T14:50:53.457 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.980119+0000 osd.3 (osd.3) 2 : cluster [DBG] 41.3 scrub ok 2024-09-13T14:50:53.457 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.106205+0000 osd.5 (osd.5) 4 : cluster [DBG] 41.8 scrub ok 2024-09-13T14:50:53.457 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.106205+0000 osd.5 (osd.5) 4 : cluster [DBG] 41.8 scrub ok 2024-09-13T14:50:53.457 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.391776+0000 mon.a (mon.0) 1745 : cluster [DBG] osdmap e987: 8 total, 8 up, 7 in 2024-09-13T14:50:53.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.391776+0000 mon.a (mon.0) 1745 : cluster [DBG] osdmap e987: 8 total, 8 up, 7 in 2024-09-13T14:50:53.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.563314+0000 mgr.y (mgr.26252) 737 : cluster [DBG] pgmap v1218: 99 pgs: 2 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16738/108126 objects degraded (15.480%); 1412/108126 objects misplaced (1.306%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:50:53.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.563314+0000 mgr.y (mgr.26252) 737 : cluster [DBG] pgmap v1218: 99 pgs: 2 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16738/108126 objects degraded (15.480%); 1412/108126 objects misplaced (1.306%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:50:53.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.648527+0000 osd.3 (osd.3) 1 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:50:53.459 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.648527+0000 osd.3 (osd.3) 1 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:50:53.459 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.947642+0000 osd.5 (osd.5) 3 : cluster [DBG] 41.8 scrub starts 2024-09-13T14:50:53.459 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.947642+0000 osd.5 (osd.5) 3 : cluster [DBG] 41.8 scrub starts 2024-09-13T14:50:53.459 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.980119+0000 osd.3 (osd.3) 2 : cluster [DBG] 41.3 scrub ok 2024-09-13T14:50:53.459 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.980119+0000 osd.3 (osd.3) 2 : cluster [DBG] 41.3 scrub ok 2024-09-13T14:50:53.459 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.106205+0000 osd.5 (osd.5) 4 : cluster [DBG] 41.8 scrub ok 2024-09-13T14:50:53.460 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:53 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.106205+0000 osd.5 (osd.5) 4 : cluster [DBG] 41.8 scrub ok 2024-09-13T14:50:53.467 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:53.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:53.570 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:53.593 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:53.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.391776+0000 mon.a (mon.0) 1745 : cluster [DBG] osdmap e987: 8 total, 8 up, 7 in 2024-09-13T14:50:53.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.391776+0000 mon.a (mon.0) 1745 : cluster [DBG] osdmap e987: 8 total, 8 up, 7 in 2024-09-13T14:50:53.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.563314+0000 mgr.y (mgr.26252) 737 : cluster [DBG] pgmap v1218: 99 pgs: 2 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16738/108126 objects degraded (15.480%); 1412/108126 objects misplaced (1.306%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:50:53.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.563314+0000 mgr.y (mgr.26252) 737 : cluster [DBG] pgmap v1218: 99 pgs: 2 peering, 10 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16738/108126 objects degraded (15.480%); 1412/108126 objects misplaced (1.306%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:50:53.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.648527+0000 osd.3 (osd.3) 1 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:50:53.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.648527+0000 osd.3 (osd.3) 1 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:50:53.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.947642+0000 osd.5 (osd.5) 3 : cluster [DBG] 41.8 scrub starts 2024-09-13T14:50:53.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.947642+0000 osd.5 (osd.5) 3 : cluster [DBG] 41.8 scrub starts 2024-09-13T14:50:53.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.980119+0000 osd.3 (osd.3) 2 : cluster [DBG] 41.3 scrub ok 2024-09-13T14:50:53.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.980119+0000 osd.3 (osd.3) 2 : cluster [DBG] 41.3 scrub ok 2024-09-13T14:50:53.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.106205+0000 osd.5 (osd.5) 4 : cluster [DBG] 41.8 scrub ok 2024-09-13T14:50:53.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:53 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.106205+0000 osd.5 (osd.5) 4 : cluster [DBG] 41.8 scrub ok 2024-09-13T14:50:53.696 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:53.714 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:53.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:53.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:53.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:53.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:53.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:53.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:53.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:54.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:54.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:54.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:54.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:54.239 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:54.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:54.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:54.367 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:54.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:54.508 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:54.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:54.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.820144+0000 osd.4 (osd.4) 1 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:50:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.820144+0000 osd.4 (osd.4) 1 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:50:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.940704+0000 osd.0 (osd.0) 1 : cluster [DBG] 41.9 scrub starts 2024-09-13T14:50:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.940704+0000 osd.0 (osd.0) 1 : cluster [DBG] 41.9 scrub starts 2024-09-13T14:50:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.990370+0000 osd.4 (osd.4) 2 : cluster [DBG] 41.0 scrub ok 2024-09-13T14:50:54.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:52.990370+0000 osd.4 (osd.4) 2 : cluster [DBG] 41.0 scrub ok 2024-09-13T14:50:54.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.126959+0000 osd.0 (osd.0) 2 : cluster [DBG] 41.9 scrub ok 2024-09-13T14:50:54.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.126959+0000 osd.0 (osd.0) 2 : cluster [DBG] 41.9 scrub ok 2024-09-13T14:50:54.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.408215+0000 mon.a (mon.0) 1746 : cluster [DBG] osdmap e988: 8 total, 8 up, 7 in 2024-09-13T14:50:54.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.408215+0000 mon.a (mon.0) 1746 : cluster [DBG] osdmap e988: 8 total, 8 up, 7 in 2024-09-13T14:50:54.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.602379+0000 osd.3 (osd.3) 3 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:50:54.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.602379+0000 osd.3 (osd.3) 3 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:50:54.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.944574+0000 osd.3 (osd.3) 4 : cluster [DBG] 41.1 scrub ok 2024-09-13T14:50:54.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:54 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.944574+0000 osd.3 (osd.3) 4 : cluster [DBG] 41.1 scrub ok 2024-09-13T14:50:54.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:54.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:54.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:54.765 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:54.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:54.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.820144+0000 osd.4 (osd.4) 1 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:50:54.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.820144+0000 osd.4 (osd.4) 1 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:50:54.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.940704+0000 osd.0 (osd.0) 1 : cluster [DBG] 41.9 scrub starts 2024-09-13T14:50:54.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.940704+0000 osd.0 (osd.0) 1 : cluster [DBG] 41.9 scrub starts 2024-09-13T14:50:54.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.990370+0000 osd.4 (osd.4) 2 : cluster [DBG] 41.0 scrub ok 2024-09-13T14:50:54.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:52.990370+0000 osd.4 (osd.4) 2 : cluster [DBG] 41.0 scrub ok 2024-09-13T14:50:54.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.126959+0000 osd.0 (osd.0) 2 : cluster [DBG] 41.9 scrub ok 2024-09-13T14:50:54.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.126959+0000 osd.0 (osd.0) 2 : cluster [DBG] 41.9 scrub ok 2024-09-13T14:50:54.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.408215+0000 mon.a (mon.0) 1746 : cluster [DBG] osdmap e988: 8 total, 8 up, 7 in 2024-09-13T14:50:54.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.408215+0000 mon.a (mon.0) 1746 : cluster [DBG] osdmap e988: 8 total, 8 up, 7 in 2024-09-13T14:50:54.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.602379+0000 osd.3 (osd.3) 3 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:50:54.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.602379+0000 osd.3 (osd.3) 3 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:50:54.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.944574+0000 osd.3 (osd.3) 4 : cluster [DBG] 41.1 scrub ok 2024-09-13T14:50:54.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.944574+0000 osd.3 (osd.3) 4 : cluster [DBG] 41.1 scrub ok 2024-09-13T14:50:54.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.820144+0000 osd.4 (osd.4) 1 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:50:54.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.820144+0000 osd.4 (osd.4) 1 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:50:54.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.940704+0000 osd.0 (osd.0) 1 : cluster [DBG] 41.9 scrub starts 2024-09-13T14:50:54.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.940704+0000 osd.0 (osd.0) 1 : cluster [DBG] 41.9 scrub starts 2024-09-13T14:50:54.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.990370+0000 osd.4 (osd.4) 2 : cluster [DBG] 41.0 scrub ok 2024-09-13T14:50:54.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:52.990370+0000 osd.4 (osd.4) 2 : cluster [DBG] 41.0 scrub ok 2024-09-13T14:50:54.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.126959+0000 osd.0 (osd.0) 2 : cluster [DBG] 41.9 scrub ok 2024-09-13T14:50:54.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.126959+0000 osd.0 (osd.0) 2 : cluster [DBG] 41.9 scrub ok 2024-09-13T14:50:54.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.408215+0000 mon.a (mon.0) 1746 : cluster [DBG] osdmap e988: 8 total, 8 up, 7 in 2024-09-13T14:50:54.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.408215+0000 mon.a (mon.0) 1746 : cluster [DBG] osdmap e988: 8 total, 8 up, 7 in 2024-09-13T14:50:54.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.602379+0000 osd.3 (osd.3) 3 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:50:54.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.602379+0000 osd.3 (osd.3) 3 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:50:54.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.944574+0000 osd.3 (osd.3) 4 : cluster [DBG] 41.1 scrub ok 2024-09-13T14:50:54.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:54 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.944574+0000 osd.3 (osd.3) 4 : cluster [DBG] 41.1 scrub ok 2024-09-13T14:50:54.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:54.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:54.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:54.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:55.022 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:55.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:55.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:55.146 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:55.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:55.206 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:55.249 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:55.270 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:55.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:55.372 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:55.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:55.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:55.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:55.526 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:55.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:55.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.796888+0000 osd.4 (osd.4) 3 : cluster [DBG] 41.6 scrub starts 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.796888+0000 osd.4 (osd.4) 3 : cluster [DBG] 41.6 scrub starts 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.934261+0000 osd.0 (osd.0) 3 : cluster [DBG] 41.19 scrub starts 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:53.934261+0000 osd.0 (osd.0) 3 : cluster [DBG] 41.19 scrub starts 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.004634+0000 osd.4 (osd.4) 4 : cluster [DBG] 41.6 scrub ok 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.004634+0000 osd.4 (osd.4) 4 : cluster [DBG] 41.6 scrub ok 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.287614+0000 osd.0 (osd.0) 4 : cluster [DBG] 41.19 scrub ok 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.287614+0000 osd.0 (osd.0) 4 : cluster [DBG] 41.19 scrub ok 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.564055+0000 mgr.y (mgr.26252) 738 : cluster [DBG] pgmap v1220: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.564055+0000 mgr.y (mgr.26252) 738 : cluster [DBG] pgmap v1220: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.894743+0000 osd.5 (osd.5) 5 : cluster [DBG] 41.18 scrub starts 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.894743+0000 osd.5 (osd.5) 5 : cluster [DBG] 41.18 scrub starts 2024-09-13T14:50:55.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:55.152061+0000 osd.5 (osd.5) 6 : cluster [DBG] 41.18 scrub ok 2024-09-13T14:50:55.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:55 smithi177 bash[68594]: cluster 2024-09-13T14:50:55.152061+0000 osd.5 (osd.5) 6 : cluster [DBG] 41.18 scrub ok 2024-09-13T14:50:55.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:55.672 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:55.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.796888+0000 osd.4 (osd.4) 3 : cluster [DBG] 41.6 scrub starts 2024-09-13T14:50:55.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.796888+0000 osd.4 (osd.4) 3 : cluster [DBG] 41.6 scrub starts 2024-09-13T14:50:55.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.934261+0000 osd.0 (osd.0) 3 : cluster [DBG] 41.19 scrub starts 2024-09-13T14:50:55.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:53.934261+0000 osd.0 (osd.0) 3 : cluster [DBG] 41.19 scrub starts 2024-09-13T14:50:55.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.004634+0000 osd.4 (osd.4) 4 : cluster [DBG] 41.6 scrub ok 2024-09-13T14:50:55.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.004634+0000 osd.4 (osd.4) 4 : cluster [DBG] 41.6 scrub ok 2024-09-13T14:50:55.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.287614+0000 osd.0 (osd.0) 4 : cluster [DBG] 41.19 scrub ok 2024-09-13T14:50:55.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.287614+0000 osd.0 (osd.0) 4 : cluster [DBG] 41.19 scrub ok 2024-09-13T14:50:55.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.564055+0000 mgr.y (mgr.26252) 738 : cluster [DBG] pgmap v1220: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:55.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.564055+0000 mgr.y (mgr.26252) 738 : cluster [DBG] pgmap v1220: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:55.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.894743+0000 osd.5 (osd.5) 5 : cluster [DBG] 41.18 scrub starts 2024-09-13T14:50:55.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.894743+0000 osd.5 (osd.5) 5 : cluster [DBG] 41.18 scrub starts 2024-09-13T14:50:55.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:55.152061+0000 osd.5 (osd.5) 6 : cluster [DBG] 41.18 scrub ok 2024-09-13T14:50:55.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[84576]: cluster 2024-09-13T14:50:55.152061+0000 osd.5 (osd.5) 6 : cluster [DBG] 41.18 scrub ok 2024-09-13T14:50:55.676 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.796888+0000 osd.4 (osd.4) 3 : cluster [DBG] 41.6 scrub starts 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.796888+0000 osd.4 (osd.4) 3 : cluster [DBG] 41.6 scrub starts 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.934261+0000 osd.0 (osd.0) 3 : cluster [DBG] 41.19 scrub starts 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:53.934261+0000 osd.0 (osd.0) 3 : cluster [DBG] 41.19 scrub starts 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.004634+0000 osd.4 (osd.4) 4 : cluster [DBG] 41.6 scrub ok 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.004634+0000 osd.4 (osd.4) 4 : cluster [DBG] 41.6 scrub ok 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.287614+0000 osd.0 (osd.0) 4 : cluster [DBG] 41.19 scrub ok 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.287614+0000 osd.0 (osd.0) 4 : cluster [DBG] 41.19 scrub ok 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.564055+0000 mgr.y (mgr.26252) 738 : cluster [DBG] pgmap v1220: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.564055+0000 mgr.y (mgr.26252) 738 : cluster [DBG] pgmap v1220: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.894743+0000 osd.5 (osd.5) 5 : cluster [DBG] 41.18 scrub starts 2024-09-13T14:50:55.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.894743+0000 osd.5 (osd.5) 5 : cluster [DBG] 41.18 scrub starts 2024-09-13T14:50:55.716 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:55.152061+0000 osd.5 (osd.5) 6 : cluster [DBG] 41.18 scrub ok 2024-09-13T14:50:55.716 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:55 smithi099 bash[87147]: cluster 2024-09-13T14:50:55.152061+0000 osd.5 (osd.5) 6 : cluster [DBG] 41.18 scrub ok 2024-09-13T14:50:55.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:55.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:55.842 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:55.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:55.948 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:55.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:55.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:56.082 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:56.107 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:56.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:56.211 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:56.241 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:56.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:56.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:56.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:56.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:56.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:56.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:56 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.554000+0000 osd.3 (osd.3) 5 : cluster [DBG] 41.13 scrub starts 2024-09-13T14:50:56.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:56 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.554000+0000 osd.3 (osd.3) 5 : cluster [DBG] 41.13 scrub starts 2024-09-13T14:50:56.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:56 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.838737+0000 osd.4 (osd.4) 5 : cluster [DBG] 41.10 scrub starts 2024-09-13T14:50:56.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:56 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.838737+0000 osd.4 (osd.4) 5 : cluster [DBG] 41.10 scrub starts 2024-09-13T14:50:56.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:56 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.894410+0000 osd.3 (osd.3) 6 : cluster [DBG] 41.13 scrub ok 2024-09-13T14:50:56.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:56 smithi177 bash[68594]: cluster 2024-09-13T14:50:54.894410+0000 osd.3 (osd.3) 6 : cluster [DBG] 41.13 scrub ok 2024-09-13T14:50:56.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:56 smithi177 bash[68594]: cluster 2024-09-13T14:50:55.030425+0000 osd.4 (osd.4) 6 : cluster [DBG] 41.10 scrub ok 2024-09-13T14:50:56.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:56 smithi177 bash[68594]: cluster 2024-09-13T14:50:55.030425+0000 osd.4 (osd.4) 6 : cluster [DBG] 41.10 scrub ok 2024-09-13T14:50:56.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:56 smithi177 bash[68594]: cluster 2024-09-13T14:50:56.343750+0000 mon.a (mon.0) 1747 : cluster [WRN] Health check update: Degraded data redundancy: 14241/101676 objects degraded (14.006%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:56.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:56 smithi177 bash[68594]: cluster 2024-09-13T14:50:56.343750+0000 mon.a (mon.0) 1747 : cluster [WRN] Health check update: Degraded data redundancy: 14241/101676 objects degraded (14.006%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:56.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:56.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:56.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:56.648 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:56.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:56.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:56.773 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:56.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.554000+0000 osd.3 (osd.3) 5 : cluster [DBG] 41.13 scrub starts 2024-09-13T14:50:56.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.554000+0000 osd.3 (osd.3) 5 : cluster [DBG] 41.13 scrub starts 2024-09-13T14:50:56.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.838737+0000 osd.4 (osd.4) 5 : cluster [DBG] 41.10 scrub starts 2024-09-13T14:50:56.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.838737+0000 osd.4 (osd.4) 5 : cluster [DBG] 41.10 scrub starts 2024-09-13T14:50:56.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.894410+0000 osd.3 (osd.3) 6 : cluster [DBG] 41.13 scrub ok 2024-09-13T14:50:56.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[84576]: cluster 2024-09-13T14:50:54.894410+0000 osd.3 (osd.3) 6 : cluster [DBG] 41.13 scrub ok 2024-09-13T14:50:56.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[84576]: cluster 2024-09-13T14:50:55.030425+0000 osd.4 (osd.4) 6 : cluster [DBG] 41.10 scrub ok 2024-09-13T14:50:56.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[84576]: cluster 2024-09-13T14:50:55.030425+0000 osd.4 (osd.4) 6 : cluster [DBG] 41.10 scrub ok 2024-09-13T14:50:56.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[84576]: cluster 2024-09-13T14:50:56.343750+0000 mon.a (mon.0) 1747 : cluster [WRN] Health check update: Degraded data redundancy: 14241/101676 objects degraded (14.006%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:56.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[84576]: cluster 2024-09-13T14:50:56.343750+0000 mon.a (mon.0) 1747 : cluster [WRN] Health check update: Degraded data redundancy: 14241/101676 objects degraded (14.006%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:56.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.554000+0000 osd.3 (osd.3) 5 : cluster [DBG] 41.13 scrub starts 2024-09-13T14:50:56.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.554000+0000 osd.3 (osd.3) 5 : cluster [DBG] 41.13 scrub starts 2024-09-13T14:50:56.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.838737+0000 osd.4 (osd.4) 5 : cluster [DBG] 41.10 scrub starts 2024-09-13T14:50:56.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.838737+0000 osd.4 (osd.4) 5 : cluster [DBG] 41.10 scrub starts 2024-09-13T14:50:56.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.894410+0000 osd.3 (osd.3) 6 : cluster [DBG] 41.13 scrub ok 2024-09-13T14:50:56.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[87147]: cluster 2024-09-13T14:50:54.894410+0000 osd.3 (osd.3) 6 : cluster [DBG] 41.13 scrub ok 2024-09-13T14:50:56.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[87147]: cluster 2024-09-13T14:50:55.030425+0000 osd.4 (osd.4) 6 : cluster [DBG] 41.10 scrub ok 2024-09-13T14:50:56.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[87147]: cluster 2024-09-13T14:50:55.030425+0000 osd.4 (osd.4) 6 : cluster [DBG] 41.10 scrub ok 2024-09-13T14:50:56.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[87147]: cluster 2024-09-13T14:50:56.343750+0000 mon.a (mon.0) 1747 : cluster [WRN] Health check update: Degraded data redundancy: 14241/101676 objects degraded (14.006%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:56.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:56 smithi099 bash[87147]: cluster 2024-09-13T14:50:56.343750+0000 mon.a (mon.0) 1747 : cluster [WRN] Health check update: Degraded data redundancy: 14241/101676 objects degraded (14.006%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:50:56.875 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:56.904 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:56.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:57.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:57.032 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:57.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:57.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:57.161 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:57.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:57.264 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:57.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:57.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:50:57] "GET /metrics HTTP/1.1" 200 36769 "" "Prometheus/2.51.0" 2024-09-13T14:50:57.388 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:57.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:57.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:57.519 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:57.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:55.573834+0000 osd.3 (osd.3) 7 : cluster [DBG] 41.11 scrub starts 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:55.573834+0000 osd.3 (osd.3) 7 : cluster [DBG] 41.11 scrub starts 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:55.843976+0000 osd.3 (osd.3) 8 : cluster [DBG] 41.11 scrub ok 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:55.843976+0000 osd.3 (osd.3) 8 : cluster [DBG] 41.11 scrub ok 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:55.875804+0000 osd.4 (osd.4) 7 : cluster [DBG] 41.16 scrub starts 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:55.875804+0000 osd.4 (osd.4) 7 : cluster [DBG] 41.16 scrub starts 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:56.179688+0000 osd.4 (osd.4) 8 : cluster [DBG] 41.16 scrub ok 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:56.179688+0000 osd.4 (osd.4) 8 : cluster [DBG] 41.16 scrub ok 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:56.564497+0000 mgr.y (mgr.26252) 739 : cluster [DBG] pgmap v1221: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:56.564497+0000 mgr.y (mgr.26252) 739 : cluster [DBG] pgmap v1221: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:56.732508+0000 osd.6 (osd.6) 3 : cluster [DBG] 41.7 scrub starts 2024-09-13T14:50:57.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:57 smithi177 bash[68594]: cluster 2024-09-13T14:50:56.732508+0000 osd.6 (osd.6) 3 : cluster [DBG] 41.7 scrub starts 2024-09-13T14:50:57.545 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:57.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:57.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:57.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:57.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:57.699 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:50:57.699 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:50:57.700 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2024-09-13T14:50:57.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:55.573834+0000 osd.3 (osd.3) 7 : cluster [DBG] 41.11 scrub starts 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:55.573834+0000 osd.3 (osd.3) 7 : cluster [DBG] 41.11 scrub starts 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:55.843976+0000 osd.3 (osd.3) 8 : cluster [DBG] 41.11 scrub ok 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:55.843976+0000 osd.3 (osd.3) 8 : cluster [DBG] 41.11 scrub ok 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:55.875804+0000 osd.4 (osd.4) 7 : cluster [DBG] 41.16 scrub starts 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:55.875804+0000 osd.4 (osd.4) 7 : cluster [DBG] 41.16 scrub starts 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:56.179688+0000 osd.4 (osd.4) 8 : cluster [DBG] 41.16 scrub ok 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:56.179688+0000 osd.4 (osd.4) 8 : cluster [DBG] 41.16 scrub ok 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:56.564497+0000 mgr.y (mgr.26252) 739 : cluster [DBG] pgmap v1221: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:56.564497+0000 mgr.y (mgr.26252) 739 : cluster [DBG] pgmap v1221: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:57.731 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:56.732508+0000 osd.6 (osd.6) 3 : cluster [DBG] 41.7 scrub starts 2024-09-13T14:50:57.732 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[84576]: cluster 2024-09-13T14:50:56.732508+0000 osd.6 (osd.6) 3 : cluster [DBG] 41.7 scrub starts 2024-09-13T14:50:57.778 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:57.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:57.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:57.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:57.926 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:58.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:58.038 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:55.573834+0000 osd.3 (osd.3) 7 : cluster [DBG] 41.11 scrub starts 2024-09-13T14:50:58.038 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:55.573834+0000 osd.3 (osd.3) 7 : cluster [DBG] 41.11 scrub starts 2024-09-13T14:50:58.038 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:55.843976+0000 osd.3 (osd.3) 8 : cluster [DBG] 41.11 scrub ok 2024-09-13T14:50:58.038 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:55.843976+0000 osd.3 (osd.3) 8 : cluster [DBG] 41.11 scrub ok 2024-09-13T14:50:58.038 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:55.875804+0000 osd.4 (osd.4) 7 : cluster [DBG] 41.16 scrub starts 2024-09-13T14:50:58.038 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:55.875804+0000 osd.4 (osd.4) 7 : cluster [DBG] 41.16 scrub starts 2024-09-13T14:50:58.038 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:56.179688+0000 osd.4 (osd.4) 8 : cluster [DBG] 41.16 scrub ok 2024-09-13T14:50:58.038 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:56.179688+0000 osd.4 (osd.4) 8 : cluster [DBG] 41.16 scrub ok 2024-09-13T14:50:58.039 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:56.564497+0000 mgr.y (mgr.26252) 739 : cluster [DBG] pgmap v1221: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:58.039 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:56.564497+0000 mgr.y (mgr.26252) 739 : cluster [DBG] pgmap v1221: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 peering, 2 unknown, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 14241/101676 objects degraded (14.006%); 1218/101676 objects misplaced (1.198%) 2024-09-13T14:50:58.039 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:56.732508+0000 osd.6 (osd.6) 3 : cluster [DBG] 41.7 scrub starts 2024-09-13T14:50:58.039 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:57 smithi099 bash[87147]: cluster 2024-09-13T14:50:56.732508+0000 osd.6 (osd.6) 3 : cluster [DBG] 41.7 scrub starts 2024-09-13T14:50:58.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:58.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:58.154 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:58.179 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:58.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:58.281 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.306 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:58.412 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:58.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:58.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:58.542 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:58 smithi177 bash[68594]: cluster 2024-09-13T14:50:57.011404+0000 osd.6 (osd.6) 4 : cluster [DBG] 41.7 scrub ok 2024-09-13T14:50:58.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:58 smithi177 bash[68594]: cluster 2024-09-13T14:50:57.011404+0000 osd.6 (osd.6) 4 : cluster [DBG] 41.7 scrub ok 2024-09-13T14:50:58.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:58 smithi177 bash[68594]: cluster 2024-09-13T14:50:57.722774+0000 osd.6 (osd.6) 5 : cluster [DBG] 41.4 scrub starts 2024-09-13T14:50:58.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:58 smithi177 bash[68594]: cluster 2024-09-13T14:50:57.722774+0000 osd.6 (osd.6) 5 : cluster [DBG] 41.4 scrub starts 2024-09-13T14:50:58.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:58.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:58.697 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:58.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:58.801 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:58.814 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:58 smithi099 bash[84576]: cluster 2024-09-13T14:50:57.011404+0000 osd.6 (osd.6) 4 : cluster [DBG] 41.7 scrub ok 2024-09-13T14:50:58.814 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:58 smithi099 bash[84576]: cluster 2024-09-13T14:50:57.011404+0000 osd.6 (osd.6) 4 : cluster [DBG] 41.7 scrub ok 2024-09-13T14:50:58.814 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:58 smithi099 bash[84576]: cluster 2024-09-13T14:50:57.722774+0000 osd.6 (osd.6) 5 : cluster [DBG] 41.4 scrub starts 2024-09-13T14:50:58.814 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:58 smithi099 bash[84576]: cluster 2024-09-13T14:50:57.722774+0000 osd.6 (osd.6) 5 : cluster [DBG] 41.4 scrub starts 2024-09-13T14:50:58.814 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:58 smithi099 bash[87147]: cluster 2024-09-13T14:50:57.011404+0000 osd.6 (osd.6) 4 : cluster [DBG] 41.7 scrub ok 2024-09-13T14:50:58.814 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:58 smithi099 bash[87147]: cluster 2024-09-13T14:50:57.011404+0000 osd.6 (osd.6) 4 : cluster [DBG] 41.7 scrub ok 2024-09-13T14:50:58.814 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:58 smithi099 bash[87147]: cluster 2024-09-13T14:50:57.722774+0000 osd.6 (osd.6) 5 : cluster [DBG] 41.4 scrub starts 2024-09-13T14:50:58.814 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:58 smithi099 bash[87147]: cluster 2024-09-13T14:50:57.722774+0000 osd.6 (osd.6) 5 : cluster [DBG] 41.4 scrub starts 2024-09-13T14:50:58.830 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:58.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:58.935 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:59.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:59.099 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:59.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:59.202 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:59.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:59.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:59.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:59.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:59.462 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:59.497 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:59.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:59.622 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:59 smithi099 bash[84576]: cluster 2024-09-13T14:50:58.526821+0000 osd.6 (osd.6) 6 : cluster [DBG] 41.4 scrub ok 2024-09-13T14:50:59.622 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:59 smithi099 bash[84576]: cluster 2024-09-13T14:50:58.526821+0000 osd.6 (osd.6) 6 : cluster [DBG] 41.4 scrub ok 2024-09-13T14:50:59.623 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:59 smithi099 bash[84576]: cluster 2024-09-13T14:50:58.565194+0000 mgr.y (mgr.26252) 740 : cluster [DBG] pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 16492/96804 objects degraded (17.036%); 1048/96804 objects misplaced (1.083%) 2024-09-13T14:50:59.623 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:50:59 smithi099 bash[84576]: cluster 2024-09-13T14:50:58.565194+0000 mgr.y (mgr.26252) 740 : cluster [DBG] pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 16492/96804 objects degraded (17.036%); 1048/96804 objects misplaced (1.083%) 2024-09-13T14:50:59.624 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:59 smithi099 bash[87147]: cluster 2024-09-13T14:50:58.526821+0000 osd.6 (osd.6) 6 : cluster [DBG] 41.4 scrub ok 2024-09-13T14:50:59.624 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:59 smithi099 bash[87147]: cluster 2024-09-13T14:50:58.526821+0000 osd.6 (osd.6) 6 : cluster [DBG] 41.4 scrub ok 2024-09-13T14:50:59.624 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:59 smithi099 bash[87147]: cluster 2024-09-13T14:50:58.565194+0000 mgr.y (mgr.26252) 740 : cluster [DBG] pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 16492/96804 objects degraded (17.036%); 1048/96804 objects misplaced (1.083%) 2024-09-13T14:50:59.625 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:50:59 smithi099 bash[87147]: cluster 2024-09-13T14:50:58.565194+0000 mgr.y (mgr.26252) 740 : cluster [DBG] pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 16492/96804 objects degraded (17.036%); 1048/96804 objects misplaced (1.083%) 2024-09-13T14:50:59.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:59.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:59.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:59.739 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:59 smithi177 bash[68594]: cluster 2024-09-13T14:50:58.526821+0000 osd.6 (osd.6) 6 : cluster [DBG] 41.4 scrub ok 2024-09-13T14:50:59.739 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:59 smithi177 bash[68594]: cluster 2024-09-13T14:50:58.526821+0000 osd.6 (osd.6) 6 : cluster [DBG] 41.4 scrub ok 2024-09-13T14:50:59.739 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:59 smithi177 bash[68594]: cluster 2024-09-13T14:50:58.565194+0000 mgr.y (mgr.26252) 740 : cluster [DBG] pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 16492/96804 objects degraded (17.036%); 1048/96804 objects misplaced (1.083%) 2024-09-13T14:50:59.739 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:50:59 smithi177 bash[68594]: cluster 2024-09-13T14:50:58.565194+0000 mgr.y (mgr.26252) 740 : cluster [DBG] pgmap v1222: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 2.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 16492/96804 objects degraded (17.036%); 1048/96804 objects misplaced (1.083%) 2024-09-13T14:50:59.755 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:59.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:59.858 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.878 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:59.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:59.979 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:00.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:00.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:00.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:00.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:00.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:00.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:00.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:00.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:00.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:00.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:00.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:00.509 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:00.526 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:00.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:51:00 smithi177 bash[61439]: ts=2024-09-13T14:51:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:00.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:00.678 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:00.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:00.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:00.817 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:00.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:00.920 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:00.946 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:00.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:01.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:01.073 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:01.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:01.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:01.201 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:01.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:01.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:01.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:01.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:01.465 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:01.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:01.568 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:01.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:01.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:01.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:01.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:01.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:01.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[84576]: cluster 2024-09-13T14:51:00.565797+0000 mgr.y (mgr.26252) 741 : cluster [DBG] pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16492/91335 objects degraded (18.057%); 1048/91335 objects misplaced (1.147%) 2024-09-13T14:51:01.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[84576]: cluster 2024-09-13T14:51:00.565797+0000 mgr.y (mgr.26252) 741 : cluster [DBG] pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16492/91335 objects degraded (18.057%); 1048/91335 objects misplaced (1.147%) 2024-09-13T14:51:01.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[84576]: cluster 2024-09-13T14:51:00.686448+0000 osd.6 (osd.6) 7 : cluster [DBG] 41.17 scrub starts 2024-09-13T14:51:01.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[84576]: cluster 2024-09-13T14:51:00.686448+0000 osd.6 (osd.6) 7 : cluster [DBG] 41.17 scrub starts 2024-09-13T14:51:01.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[84576]: cluster 2024-09-13T14:51:01.344564+0000 mon.a (mon.0) 1748 : cluster [WRN] Health check update: Degraded data redundancy: 16492/91335 objects degraded (18.057%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:01.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[84576]: cluster 2024-09-13T14:51:01.344564+0000 mon.a (mon.0) 1748 : cluster [WRN] Health check update: Degraded data redundancy: 16492/91335 objects degraded (18.057%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:01.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[87147]: cluster 2024-09-13T14:51:00.565797+0000 mgr.y (mgr.26252) 741 : cluster [DBG] pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16492/91335 objects degraded (18.057%); 1048/91335 objects misplaced (1.147%) 2024-09-13T14:51:01.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[87147]: cluster 2024-09-13T14:51:00.565797+0000 mgr.y (mgr.26252) 741 : cluster [DBG] pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16492/91335 objects degraded (18.057%); 1048/91335 objects misplaced (1.147%) 2024-09-13T14:51:01.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[87147]: cluster 2024-09-13T14:51:00.686448+0000 osd.6 (osd.6) 7 : cluster [DBG] 41.17 scrub starts 2024-09-13T14:51:01.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[87147]: cluster 2024-09-13T14:51:00.686448+0000 osd.6 (osd.6) 7 : cluster [DBG] 41.17 scrub starts 2024-09-13T14:51:01.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[87147]: cluster 2024-09-13T14:51:01.344564+0000 mon.a (mon.0) 1748 : cluster [WRN] Health check update: Degraded data redundancy: 16492/91335 objects degraded (18.057%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:01.830 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:01 smithi099 bash[87147]: cluster 2024-09-13T14:51:01.344564+0000 mon.a (mon.0) 1748 : cluster [WRN] Health check update: Degraded data redundancy: 16492/91335 objects degraded (18.057%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:01.853 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:01.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:01 smithi177 bash[68594]: cluster 2024-09-13T14:51:00.565797+0000 mgr.y (mgr.26252) 741 : cluster [DBG] pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16492/91335 objects degraded (18.057%); 1048/91335 objects misplaced (1.147%) 2024-09-13T14:51:01.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:01 smithi177 bash[68594]: cluster 2024-09-13T14:51:00.565797+0000 mgr.y (mgr.26252) 741 : cluster [DBG] pgmap v1223: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 16492/91335 objects degraded (18.057%); 1048/91335 objects misplaced (1.147%) 2024-09-13T14:51:01.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:01 smithi177 bash[68594]: cluster 2024-09-13T14:51:00.686448+0000 osd.6 (osd.6) 7 : cluster [DBG] 41.17 scrub starts 2024-09-13T14:51:01.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:01 smithi177 bash[68594]: cluster 2024-09-13T14:51:00.686448+0000 osd.6 (osd.6) 7 : cluster [DBG] 41.17 scrub starts 2024-09-13T14:51:01.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:01 smithi177 bash[68594]: cluster 2024-09-13T14:51:01.344564+0000 mon.a (mon.0) 1748 : cluster [WRN] Health check update: Degraded data redundancy: 16492/91335 objects degraded (18.057%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:01.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:01 smithi177 bash[68594]: cluster 2024-09-13T14:51:01.344564+0000 mon.a (mon.0) 1748 : cluster [WRN] Health check update: Degraded data redundancy: 16492/91335 objects degraded (18.057%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:01.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:01.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:01.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:02.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:02.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:02.157 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:02.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:02.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:02.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:02.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:02.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:02.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:02.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:02.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:02.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:02.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:02.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:02.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:02 smithi099 bash[84576]: cluster 2024-09-13T14:51:00.886898+0000 osd.6 (osd.6) 8 : cluster [DBG] 41.17 scrub ok 2024-09-13T14:51:02.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:02 smithi099 bash[84576]: cluster 2024-09-13T14:51:00.886898+0000 osd.6 (osd.6) 8 : cluster [DBG] 41.17 scrub ok 2024-09-13T14:51:02.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:02 smithi099 bash[84576]: cluster 2024-09-13T14:51:01.672976+0000 osd.6 (osd.6) 9 : cluster [DBG] 41.14 scrub starts 2024-09-13T14:51:02.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:02 smithi099 bash[84576]: cluster 2024-09-13T14:51:01.672976+0000 osd.6 (osd.6) 9 : cluster [DBG] 41.14 scrub starts 2024-09-13T14:51:02.791 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:02 smithi099 bash[87147]: cluster 2024-09-13T14:51:00.886898+0000 osd.6 (osd.6) 8 : cluster [DBG] 41.17 scrub ok 2024-09-13T14:51:02.791 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:02 smithi099 bash[87147]: cluster 2024-09-13T14:51:00.886898+0000 osd.6 (osd.6) 8 : cluster [DBG] 41.17 scrub ok 2024-09-13T14:51:02.791 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:02 smithi099 bash[87147]: cluster 2024-09-13T14:51:01.672976+0000 osd.6 (osd.6) 9 : cluster [DBG] 41.14 scrub starts 2024-09-13T14:51:02.791 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:02 smithi099 bash[87147]: cluster 2024-09-13T14:51:01.672976+0000 osd.6 (osd.6) 9 : cluster [DBG] 41.14 scrub starts 2024-09-13T14:51:02.808 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:02.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:02.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:02.924 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:02 smithi177 bash[68594]: cluster 2024-09-13T14:51:00.886898+0000 osd.6 (osd.6) 8 : cluster [DBG] 41.17 scrub ok 2024-09-13T14:51:02.924 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:02 smithi177 bash[68594]: cluster 2024-09-13T14:51:00.886898+0000 osd.6 (osd.6) 8 : cluster [DBG] 41.17 scrub ok 2024-09-13T14:51:02.924 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:02 smithi177 bash[68594]: cluster 2024-09-13T14:51:01.672976+0000 osd.6 (osd.6) 9 : cluster [DBG] 41.14 scrub starts 2024-09-13T14:51:02.924 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:02 smithi177 bash[68594]: cluster 2024-09-13T14:51:01.672976+0000 osd.6 (osd.6) 9 : cluster [DBG] 41.14 scrub starts 2024-09-13T14:51:02.937 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:02.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:03.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:03.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.066 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:03.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:03.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:03.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:03.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:03.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:03.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:03.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:03.422 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:03.456 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:03.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:03.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:03.591 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:03.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:03.717 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:03.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[84576]: cluster 2024-09-13T14:51:02.157862+0000 osd.6 (osd.6) 10 : cluster [DBG] 41.14 scrub ok 2024-09-13T14:51:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[84576]: cluster 2024-09-13T14:51:02.157862+0000 osd.6 (osd.6) 10 : cluster [DBG] 41.14 scrub ok 2024-09-13T14:51:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[84576]: cluster 2024-09-13T14:51:02.566383+0000 mgr.y (mgr.26252) 742 : cluster [DBG] pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 16492/89142 objects degraded (18.501%); 1048/89142 objects misplaced (1.176%) 2024-09-13T14:51:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[84576]: cluster 2024-09-13T14:51:02.566383+0000 mgr.y (mgr.26252) 742 : cluster [DBG] pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 16492/89142 objects degraded (18.501%); 1048/89142 objects misplaced (1.176%) 2024-09-13T14:51:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[84576]: audit 2024-09-13T14:51:03.277435+0000 mon.a (mon.0) 1749 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[84576]: audit 2024-09-13T14:51:03.277435+0000 mon.a (mon.0) 1749 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[87147]: cluster 2024-09-13T14:51:02.157862+0000 osd.6 (osd.6) 10 : cluster [DBG] 41.14 scrub ok 2024-09-13T14:51:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[87147]: cluster 2024-09-13T14:51:02.157862+0000 osd.6 (osd.6) 10 : cluster [DBG] 41.14 scrub ok 2024-09-13T14:51:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[87147]: cluster 2024-09-13T14:51:02.566383+0000 mgr.y (mgr.26252) 742 : cluster [DBG] pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 16492/89142 objects degraded (18.501%); 1048/89142 objects misplaced (1.176%) 2024-09-13T14:51:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[87147]: cluster 2024-09-13T14:51:02.566383+0000 mgr.y (mgr.26252) 742 : cluster [DBG] pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 16492/89142 objects degraded (18.501%); 1048/89142 objects misplaced (1.176%) 2024-09-13T14:51:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[87147]: audit 2024-09-13T14:51:03.277435+0000 mon.a (mon.0) 1749 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:03 smithi099 bash[87147]: audit 2024-09-13T14:51:03.277435+0000 mon.a (mon.0) 1749 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:03.824 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:03.851 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:03.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:03.957 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:03.966 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:03 smithi177 bash[68594]: cluster 2024-09-13T14:51:02.157862+0000 osd.6 (osd.6) 10 : cluster [DBG] 41.14 scrub ok 2024-09-13T14:51:03.966 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:03 smithi177 bash[68594]: cluster 2024-09-13T14:51:02.157862+0000 osd.6 (osd.6) 10 : cluster [DBG] 41.14 scrub ok 2024-09-13T14:51:03.966 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:03 smithi177 bash[68594]: cluster 2024-09-13T14:51:02.566383+0000 mgr.y (mgr.26252) 742 : cluster [DBG] pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 16492/89142 objects degraded (18.501%); 1048/89142 objects misplaced (1.176%) 2024-09-13T14:51:03.966 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:03 smithi177 bash[68594]: cluster 2024-09-13T14:51:02.566383+0000 mgr.y (mgr.26252) 742 : cluster [DBG] pgmap v1224: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 16492/89142 objects degraded (18.501%); 1048/89142 objects misplaced (1.176%) 2024-09-13T14:51:03.966 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:03 smithi177 bash[68594]: audit 2024-09-13T14:51:03.277435+0000 mon.a (mon.0) 1749 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:03.966 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:03 smithi177 bash[68594]: audit 2024-09-13T14:51:03.277435+0000 mon.a (mon.0) 1749 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:03.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:03.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:04.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:04.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:04.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:04.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:04.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:04.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:04.339 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:04.359 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:04.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:04.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:04.488 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:04.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:04.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:04.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:04.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:04.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:04.738 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:04.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:04.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:04.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:04.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:04.974 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:04.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:05.102 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:05.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:05.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:05.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:05.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:05.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:05.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:05.385 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:05.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:05.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:05.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:05.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:05.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:05.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:05.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:05.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:05 smithi177 bash[68594]: cluster 2024-09-13T14:51:04.567050+0000 mgr.y (mgr.26252) 743 : cluster [DBG] pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:05 smithi177 bash[68594]: cluster 2024-09-13T14:51:04.567050+0000 mgr.y (mgr.26252) 743 : cluster [DBG] pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:05.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:05.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:05.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:06.017 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:06.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:05 smithi099 bash[84576]: cluster 2024-09-13T14:51:04.567050+0000 mgr.y (mgr.26252) 743 : cluster [DBG] pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:06.036 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:05 smithi099 bash[84576]: cluster 2024-09-13T14:51:04.567050+0000 mgr.y (mgr.26252) 743 : cluster [DBG] pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:06.036 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:05 smithi099 bash[87147]: cluster 2024-09-13T14:51:04.567050+0000 mgr.y (mgr.26252) 743 : cluster [DBG] pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:06.037 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:05 smithi099 bash[87147]: cluster 2024-09-13T14:51:04.567050+0000 mgr.y (mgr.26252) 743 : cluster [DBG] pgmap v1225: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:06.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:06.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:06.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:06.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:06.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:06.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:06.291 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:06.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:06.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:06.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:06.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:06.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:06.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:06.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:06.653 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:06.686 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:06.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:06.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:06.817 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:06.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:06.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:06 smithi177 bash[68594]: cluster 2024-09-13T14:51:06.345318+0000 mon.a (mon.0) 1750 : cluster [WRN] Health check update: Degraded data redundancy: 16365/78147 objects degraded (20.941%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:06.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:06 smithi177 bash[68594]: cluster 2024-09-13T14:51:06.345318+0000 mon.a (mon.0) 1750 : cluster [WRN] Health check update: Degraded data redundancy: 16365/78147 objects degraded (20.941%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:06.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:06.950 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:07.029 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:06 smithi099 bash[84576]: cluster 2024-09-13T14:51:06.345318+0000 mon.a (mon.0) 1750 : cluster [WRN] Health check update: Degraded data redundancy: 16365/78147 objects degraded (20.941%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:07.029 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:06 smithi099 bash[84576]: cluster 2024-09-13T14:51:06.345318+0000 mon.a (mon.0) 1750 : cluster [WRN] Health check update: Degraded data redundancy: 16365/78147 objects degraded (20.941%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:07.029 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:06 smithi099 bash[87147]: cluster 2024-09-13T14:51:06.345318+0000 mon.a (mon.0) 1750 : cluster [WRN] Health check update: Degraded data redundancy: 16365/78147 objects degraded (20.941%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:07.029 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:06 smithi099 bash[87147]: cluster 2024-09-13T14:51:06.345318+0000 mon.a (mon.0) 1750 : cluster [WRN] Health check update: Degraded data redundancy: 16365/78147 objects degraded (20.941%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:07.058 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:07.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:07.179 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:07.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:07.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:07.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:07.343 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:07.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:07.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:07.477 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:07.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:07.512 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:51:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:51:07] "GET /metrics HTTP/1.1" 200 36751 "" "Prometheus/2.51.0" 2024-09-13T14:51:07.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:07.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:07.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:07.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:07.742 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:07.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:07 smithi099 bash[84576]: cluster 2024-09-13T14:51:06.567557+0000 mgr.y (mgr.26252) 744 : cluster [DBG] pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:07.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:07 smithi099 bash[84576]: cluster 2024-09-13T14:51:06.567557+0000 mgr.y (mgr.26252) 744 : cluster [DBG] pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:07.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:07 smithi099 bash[87147]: cluster 2024-09-13T14:51:06.567557+0000 mgr.y (mgr.26252) 744 : cluster [DBG] pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:07 smithi099 bash[87147]: cluster 2024-09-13T14:51:06.567557+0000 mgr.y (mgr.26252) 744 : cluster [DBG] pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:07.847 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:07.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:07.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:07.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:08.011 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:08.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:08.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:07 smithi177 bash[68594]: cluster 2024-09-13T14:51:06.567557+0000 mgr.y (mgr.26252) 744 : cluster [DBG] pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:08.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:07 smithi177 bash[68594]: cluster 2024-09-13T14:51:06.567557+0000 mgr.y (mgr.26252) 744 : cluster [DBG] pgmap v1226: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 16365/78147 objects degraded (20.941%); 739/78147 objects misplaced (0.946%) 2024-09-13T14:51:08.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:08.133 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:08.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:08.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:08.243 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:08.257 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:08.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:08.363 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:08.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:08.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:08.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:08.520 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:08.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:08.626 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:08.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:08.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:08.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:08.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:08.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:08.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:08.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:09.021 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:09.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:09.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:09.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:09.172 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:09.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:09.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:09.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:09.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:09.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:09.439 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:09.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:09.542 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:09.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:09.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:09.699 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:09.801 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:09 smithi099 bash[84576]: cluster 2024-09-13T14:51:08.568260+0000 mgr.y (mgr.26252) 745 : cluster [DBG] pgmap v1227: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 16164/70323 objects degraded (22.985%); 471/70323 objects misplaced (0.670%) 2024-09-13T14:51:09.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:09 smithi099 bash[84576]: cluster 2024-09-13T14:51:08.568260+0000 mgr.y (mgr.26252) 745 : cluster [DBG] pgmap v1227: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 16164/70323 objects degraded (22.985%); 471/70323 objects misplaced (0.670%) 2024-09-13T14:51:09.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:09 smithi099 bash[87147]: cluster 2024-09-13T14:51:08.568260+0000 mgr.y (mgr.26252) 745 : cluster [DBG] pgmap v1227: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 16164/70323 objects degraded (22.985%); 471/70323 objects misplaced (0.670%) 2024-09-13T14:51:09.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:09 smithi099 bash[87147]: cluster 2024-09-13T14:51:08.568260+0000 mgr.y (mgr.26252) 745 : cluster [DBG] pgmap v1227: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 16164/70323 objects degraded (22.985%); 471/70323 objects misplaced (0.670%) 2024-09-13T14:51:09.826 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:09.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:09.932 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:09.959 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:09.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:10.063 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:10.072 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:09 smithi177 bash[68594]: cluster 2024-09-13T14:51:08.568260+0000 mgr.y (mgr.26252) 745 : cluster [DBG] pgmap v1227: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 16164/70323 objects degraded (22.985%); 471/70323 objects misplaced (0.670%) 2024-09-13T14:51:10.072 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:09 smithi177 bash[68594]: cluster 2024-09-13T14:51:08.568260+0000 mgr.y (mgr.26252) 745 : cluster [DBG] pgmap v1227: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 16164/70323 objects degraded (22.985%); 471/70323 objects misplaced (0.670%) 2024-09-13T14:51:10.092 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:10.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:10.195 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:10.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:10.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:10.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:10.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:10.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:51:10 smithi177 bash[61439]: ts=2024-09-13T14:51:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:10.449 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:10.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:10.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:10.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:10.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:10.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:10.713 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:10.739 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:10.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:10.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:10.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:10.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:11.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:11.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:11.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:11.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:11.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:11.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:11.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:11.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:11.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:11.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:11.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:11.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:11.531 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:11.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:11.633 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:11.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:11.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:11.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:11.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:11.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:11 smithi099 bash[84576]: cluster 2024-09-13T14:51:10.568903+0000 mgr.y (mgr.26252) 746 : cluster [DBG] pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 16164/66783 objects degraded (24.204%); 471/66783 objects misplaced (0.705%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:51:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:11 smithi099 bash[84576]: cluster 2024-09-13T14:51:10.568903+0000 mgr.y (mgr.26252) 746 : cluster [DBG] pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 16164/66783 objects degraded (24.204%); 471/66783 objects misplaced (0.705%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:51:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:11 smithi099 bash[84576]: cluster 2024-09-13T14:51:11.348764+0000 mon.a (mon.0) 1751 : cluster [WRN] Health check update: Degraded data redundancy: 16164/66783 objects degraded (24.204%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:11 smithi099 bash[84576]: cluster 2024-09-13T14:51:11.348764+0000 mon.a (mon.0) 1751 : cluster [WRN] Health check update: Degraded data redundancy: 16164/66783 objects degraded (24.204%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:11.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:11 smithi099 bash[87147]: cluster 2024-09-13T14:51:10.568903+0000 mgr.y (mgr.26252) 746 : cluster [DBG] pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 16164/66783 objects degraded (24.204%); 471/66783 objects misplaced (0.705%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:51:11.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:11 smithi099 bash[87147]: cluster 2024-09-13T14:51:10.568903+0000 mgr.y (mgr.26252) 746 : cluster [DBG] pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 16164/66783 objects degraded (24.204%); 471/66783 objects misplaced (0.705%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:51:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:11 smithi099 bash[87147]: cluster 2024-09-13T14:51:11.348764+0000 mon.a (mon.0) 1751 : cluster [WRN] Health check update: Degraded data redundancy: 16164/66783 objects degraded (24.204%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:11 smithi099 bash[87147]: cluster 2024-09-13T14:51:11.348764+0000 mon.a (mon.0) 1751 : cluster [WRN] Health check update: Degraded data redundancy: 16164/66783 objects degraded (24.204%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:11.917 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:11.926 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:11 smithi177 bash[68594]: cluster 2024-09-13T14:51:10.568903+0000 mgr.y (mgr.26252) 746 : cluster [DBG] pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 16164/66783 objects degraded (24.204%); 471/66783 objects misplaced (0.705%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:51:11.926 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:11 smithi177 bash[68594]: cluster 2024-09-13T14:51:10.568903+0000 mgr.y (mgr.26252) 746 : cluster [DBG] pgmap v1228: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 16164/66783 objects degraded (24.204%); 471/66783 objects misplaced (0.705%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T14:51:11.926 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:11 smithi177 bash[68594]: cluster 2024-09-13T14:51:11.348764+0000 mon.a (mon.0) 1751 : cluster [WRN] Health check update: Degraded data redundancy: 16164/66783 objects degraded (24.204%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:11.927 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:11 smithi177 bash[68594]: cluster 2024-09-13T14:51:11.348764+0000 mon.a (mon.0) 1751 : cluster [WRN] Health check update: Degraded data redundancy: 16164/66783 objects degraded (24.204%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:11.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:11.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:12.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:12.064 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:12.170 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:12.202 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:12.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:12.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:12.335 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:12.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:12.439 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:12.467 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:12.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:12.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:12.599 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:12.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:12.705 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:12.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:12.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:12.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:12.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:12.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:12.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:13.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:13.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:13.111 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:13.138 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:13.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:13.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:13.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:13.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:13.363 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:13.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:13.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:13.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:13.521 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:13.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:13.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:13.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:13.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:13.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:13.775 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:13 smithi177 bash[68594]: cluster 2024-09-13T14:51:12.569524+0000 mgr.y (mgr.26252) 747 : cluster [DBG] pgmap v1229: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 16164/62379 objects degraded (25.913%); 471/62379 objects misplaced (0.755%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:51:13.775 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:13 smithi177 bash[68594]: cluster 2024-09-13T14:51:12.569524+0000 mgr.y (mgr.26252) 747 : cluster [DBG] pgmap v1229: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 16164/62379 objects degraded (25.913%); 471/62379 objects misplaced (0.755%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:51:13.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:13.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:13.897 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:13.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:13.907 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:13 smithi099 bash[84576]: cluster 2024-09-13T14:51:12.569524+0000 mgr.y (mgr.26252) 747 : cluster [DBG] pgmap v1229: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 16164/62379 objects degraded (25.913%); 471/62379 objects misplaced (0.755%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:51:13.907 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:13 smithi099 bash[84576]: cluster 2024-09-13T14:51:12.569524+0000 mgr.y (mgr.26252) 747 : cluster [DBG] pgmap v1229: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 16164/62379 objects degraded (25.913%); 471/62379 objects misplaced (0.755%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:51:13.907 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:13 smithi099 bash[87147]: cluster 2024-09-13T14:51:12.569524+0000 mgr.y (mgr.26252) 747 : cluster [DBG] pgmap v1229: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 16164/62379 objects degraded (25.913%); 471/62379 objects misplaced (0.755%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:51:13.907 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:13 smithi099 bash[87147]: cluster 2024-09-13T14:51:12.569524+0000 mgr.y (mgr.26252) 747 : cluster [DBG] pgmap v1229: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 16164/62379 objects degraded (25.913%); 471/62379 objects misplaced (0.755%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:51:14.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:14.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:14.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:14.131 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:14.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:14.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:14.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:14.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:14.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:14.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:14.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:14.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:14.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:14.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:14.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:14.642 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:14.642 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:51:14.661 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:14.688 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:51:14.690 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:14.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:14.715 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:51:14.716 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:51:14.747 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:14.777 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:51:14.778 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:14.793 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:14.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:14 smithi099 bash[84576]: audit 2024-09-13T14:51:14.613258+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.99:0/715825645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:14.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:14 smithi099 bash[84576]: audit 2024-09-13T14:51:14.613258+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.99:0/715825645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:14.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:14 smithi099 bash[87147]: audit 2024-09-13T14:51:14.613258+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.99:0/715825645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:14.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:14 smithi099 bash[87147]: audit 2024-09-13T14:51:14.613258+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.99:0/715825645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:51:14.811 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:14.830 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:14.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:14.839 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:51:14.840 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:14.871 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:51:14.872 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:14.934 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:14.959 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:14.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:15.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:15.069 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:51:15.069 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T14:51:15.069 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T14:51:15.069 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T14:51:15.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.070 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7901 41.6 41:61447ba6:::benchmark_data_smithi099_126887_object7900:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:51:15.070 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314278+0000", 2024-09-13T14:51:15.070 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 208.75586285099999, 2024-09-13T14:51:15.070 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.087476222, 2024-09-13T14:51:15.070 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.070 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.070 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7901 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314278+0000", 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.071 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314278+0000", 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314282+0000", 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9299999999999996e-06 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314349+0000", 2024-09-13T14:51:15.072 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6754999999999994e-05 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314398+0000", 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9196000000000001e-05 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314424+0000", 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5945e-05 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.073 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314609+0000", 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018503399999999999 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314624+0000", 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5310000000000001e-05 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.323631+0000", 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0090065409999999999 2024-09-13T14:51:15.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.323656+0000", 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5174e-05 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.391048+0000", 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.067391907 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.391274+0000", 2024-09-13T14:51:15.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022600300000000001 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.391435+0000", 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000161385 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.394112+0000", 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026761419999999998 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.394773+0000", 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00066148599999999998 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.400932+0000", 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0061586280000000002 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.401186+0000", 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000254045 2024-09-13T14:51:15.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.401755+0000", 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00056874099999999999 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7929 41.6 41:694a6c3a:::benchmark_data_smithi099_126887_object7928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314510+0000", 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 208.75563142600001, 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0880818320000001, 2024-09-13T14:51:15.078 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7929 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314510+0000", 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.079 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314510+0000", 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314512+0000", 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0499999999999999e-06 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.080 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314652+0000", 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000140603 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314679+0000", 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7042999999999998e-05 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314699+0000", 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9208999999999999e-05 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.081 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314850+0000", 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015126699999999999 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314867+0000", 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.721e-05 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.323779+0000", 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0089114480000000006 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.082 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.323804+0000", 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5023000000000001e-05 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.392562+0000", 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0687581370000001 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.392702+0000", 2024-09-13T14:51:15.083 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000139794 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,7", 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.392804+0000", 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102939 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.394737+0000", 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019330129999999999 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.084 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.394808+0000", 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0246999999999993e-05 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.401985+0000", 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0071771070000000003 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402085+0000", 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010058800000000001 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.085 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402592+0000", 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00050615399999999998 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13535 41.f 41:f954659f:::benchmark_data_smithi099_126887_object13534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e962)", 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.103662+0000", 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 203.966479612, 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45471350100000002, 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.086 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13535 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103662+0000", 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.087 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103662+0000", 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103663+0000", 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4449999999999999e-06 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103685+0000", 2024-09-13T14:51:15.088 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1624000000000002e-05 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103696+0000", 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.155e-05 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103706+0000", 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0254999999999999e-05 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103766+0000", 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9277000000000002e-05 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103890+0000", 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000124015 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104039+0000", 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014949200000000001 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.090 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.105610+0000", 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015709719999999999 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.105689+0000", 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9066999999999994e-05 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558212+0000", 2024-09-13T14:51:15.091 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.452522333 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558311+0000", 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9794e-05 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558375+0000", 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3676999999999994e-05 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.092 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:13532 41.1 e962/953 41:8d998828:::benchmark_data_smithi099_126887_object13531:head v 962'859, mlcod=962'856)", 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.104667+0000", 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 203.96547390500001, 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45439524399999998, 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.093 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104667+0000", 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104667+0000", 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104668+0000", 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0100000000000004e-07 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.094 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104691+0000", 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3311000000000001e-05 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104703+0000", 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.163e-05 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104711+0000", 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8040000000000002e-06 2024-09-13T14:51:15.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104800+0000", 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8757000000000004e-05 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104819+0000", 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8950999999999999e-05 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559008+0000", 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45418946300000002 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559062+0000", 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4527000000000002e-05 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.097 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:13529 41.3 e962/959 41:cb165151:::benchmark_data_smithi099_126887_object13528:head v 962'809, mlcod=962'807)", 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.104731+0000", 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 203.96541022299999, 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45468538200000003, 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104731+0000", 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.098 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104731+0000", 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104731+0000", 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3099999999999999e-07 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104759+0000", 2024-09-13T14:51:15.099 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7469000000000001e-05 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104767+0000", 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0209999999999999e-06 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104772+0000", 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1819999999999997e-06 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.100 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104818+0000", 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5584000000000003e-05 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104838+0000", 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0211e-05 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106901+0000", 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002062885 2024-09-13T14:51:15.101 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559416+0000", 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45251569899999999 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82469 41.5 41:a947eb55:::benchmark_data_smithi099_126887_object82468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e974)", 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.247637+0000", 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.822503708, 2024-09-13T14:51:15.102 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5025099439999998, 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82469 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.247637+0000", 2024-09-13T14:51:15.103 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.247637+0000", 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.247639+0000", 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.066e-06 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.104 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.247658+0000", 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9239999999999999e-05 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.247664+0000", 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2589999999999997e-06 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.247668+0000", 2024-09-13T14:51:15.105 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8569999999999997e-06 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.249957+0000", 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022893259999999999 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.250092+0000", 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000134614 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.106 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.250225+0000", 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000133535 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258472+0000", 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0082466220000000003 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258484+0000", 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1837e-05 2024-09-13T14:51:15.107 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311164+0000", 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.052680103999999998 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311175+0000", 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0618000000000001e-05 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.108 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.295820+0000", 2024-09-13T14:51:15.109 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98464500799999999 2024-09-13T14:51:15.109 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.109 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.109 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.109 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.295837+0000", 2024-09-13T14:51:15.109 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7099999999999999e-05 2024-09-13T14:51:15.109 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.110 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.110 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.110 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.750053+0000", 2024-09-13T14:51:15.110 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4542164639999999 2024-09-13T14:51:15.110 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.110 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.110 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.111 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.750147+0000", 2024-09-13T14:51:15.111 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4294000000000002e-05 2024-09-13T14:51:15.111 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.111 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.111 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.111 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.111 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.111 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82480 41.f 41:f6f03880:::benchmark_data_smithi099_126887_object82479:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e974)", 2024-09-13T14:51:15.112 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.258405+0000", 2024-09-13T14:51:15.112 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.81173613999999, 2024-09-13T14:51:15.112 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2867408020000002, 2024-09-13T14:51:15.112 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.112 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.112 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:51:15.112 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.112 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.112 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82480 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258405+0000", 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258405+0000", 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.113 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258407+0000", 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6640000000000001e-06 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258428+0000", 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0928999999999999e-05 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.114 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258438+0000", 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0302999999999999e-05 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258445+0000", 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5410000000000003e-06 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258524+0000", 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8362999999999995e-05 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.115 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258641+0000", 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117225 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258771+0000", 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013036699999999999 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261940+0000", 2024-09-13T14:51:15.116 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003168176 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.268005+0000", 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006065239 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.299391+0000", 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.031386598000000002 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.117 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.299408+0000", 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6858999999999999e-05 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296908+0000", 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99749927199999999 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296922+0000", 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4209999999999999e-05 2024-09-13T14:51:15.118 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.544979+0000", 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.248056922 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.545146+0000", 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016713399999999999 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.119 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82482 41.5 41:a6f203dd:::benchmark_data_smithi099_126887_object82481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e974)", 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.261475+0000", 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.80866652700001, 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4917984309999999, 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82482 2024-09-13T14:51:15.120 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261475+0000", 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261475+0000", 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.121 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261476+0000", 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.725e-06 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261505+0000", 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8634999999999999e-05 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261524+0000", 2024-09-13T14:51:15.122 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8964999999999999e-05 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261533+0000", 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9409999999999996e-06 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.262003+0000", 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00046970999999999997 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.123 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.262135+0000", 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013255600000000001 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.262267+0000", 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132213 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.267460+0000", 2024-09-13T14:51:15.124 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051922039999999997 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.268046+0000", 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00058689500000000004 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311545+0000", 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.043498439 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.125 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311551+0000", 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7429999999999999e-06 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.295950+0000", 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98439968099999997 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.295962+0000", 2024-09-13T14:51:15.126 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1256e-05 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.753147+0000", 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.457185768 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.753273+0000", 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012569999999999999 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.127 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82486 41.5 41:aff9bb28:::benchmark_data_smithi099_126887_object82485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e974)", 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.261571+0000", 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.808569745, 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4918477370000001, 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.128 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82486 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261571+0000", 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261571+0000", 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.129 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261572+0000", 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4099999999999998e-07 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261594+0000", 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2107e-05 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.130 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261603+0000", 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5920000000000001e-06 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261609+0000", 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3210000000000001e-06 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.264739+0000", 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003129953 2024-09-13T14:51:15.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.264828+0000", 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8527000000000001e-05 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.265088+0000", 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026058599999999999 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.132 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.133 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.268107+0000", 2024-09-13T14:51:15.133 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030193490000000002 2024-09-13T14:51:15.133 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.270709+0000", 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026011889999999998 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311568+0000", 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.040859070999999997 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.134 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311572+0000", 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5340000000000001e-06 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296005+0000", 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98443301599999999 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296016+0000", 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1090999999999999e-05 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.135 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.753325+0000", 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.45730885 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.753419+0000", 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.391e-05 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.136 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82497 41.5 41:a1b85fb5:::benchmark_data_smithi099_126887_object82496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e974)", 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.273260+0000", 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.79688124899999, 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.48033204, 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82497 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.137 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.273260+0000", 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.273260+0000", 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.273261+0000", 2024-09-13T14:51:15.138 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.409e-06 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.273280+0000", 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8853000000000001e-05 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.273295+0000", 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4746000000000001e-05 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.139 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.273302+0000", 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3379999999999999e-06 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.273354+0000", 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2232999999999999e-05 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.273472+0000", 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117887 2024-09-13T14:51:15.140 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.273580+0000", 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000107421 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.278914+0000", 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0053343430000000001 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.141 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.278943+0000", 2024-09-13T14:51:15.142 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8815000000000001e-05 2024-09-13T14:51:15.142 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.142 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.142 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.142 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311589+0000", 2024-09-13T14:51:15.142 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032645754999999999 2024-09-13T14:51:15.142 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.142 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.142 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.142 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311593+0000", 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.172e-06 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296056+0000", 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98446309600000004 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296067+0000", 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0643e-05 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.143 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.753456+0000", 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.457389568 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.753592+0000", 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000135761 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.144 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82505 41.f 41:f3c96f24:::benchmark_data_smithi099_126887_object82504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e974)", 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.281677+0000", 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.78846405300001, 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2661155000000002, 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82505 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.145 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.281677+0000", 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.281677+0000", 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.281679+0000", 2024-09-13T14:51:15.146 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1390000000000002e-06 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.281718+0000", 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9195e-05 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.281734+0000", 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5403000000000001e-05 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.281742+0000", 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1999999999999994e-06 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.281804+0000", 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2447999999999996e-05 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.281966+0000", 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016191600000000001 2024-09-13T14:51:15.148 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.282406+0000", 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00043957300000000001 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.285867+0000", 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034606060000000002 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.289705+0000", 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0038385540000000001 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.299451+0000", 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097456720000000004 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.299461+0000", 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0227999999999999e-05 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.150 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296967+0000", 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99750580300000002 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296978+0000", 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1635e-05 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.547675+0000", 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2506967900000001 2024-09-13T14:51:15.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.547793+0000", 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011733799999999999 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82511 41.f 41:fe708b07:::benchmark_data_smithi099_126887_object82510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e974)", 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.309971+0000", 2024-09-13T14:51:15.152 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.760170265, 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2540167179999999, 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82511 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.309971+0000", 2024-09-13T14:51:15.153 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.309971+0000", 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.309973+0000", 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8220000000000001e-06 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.154 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310002+0000", 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8938e-05 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310013+0000", 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1751e-05 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310022+0000", 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4619999999999996e-06 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.155 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311048+0000", 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001025668 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311055+0000", 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9789999999999996e-06 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.297075+0000", 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98602071199999997 2024-09-13T14:51:15.156 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.297087+0000", 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1286e-05 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.547857+0000", 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.250770607 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.548000+0000", 2024-09-13T14:51:15.157 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000142881 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.548166+0000", 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016571199999999999 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.553932+0000", 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057664150000000004 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.158 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.557951+0000", 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040191920000000004 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.561552+0000", 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003600925 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.561613+0000", 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0386000000000001e-05 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.159 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.563988+0000", 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023749819999999999 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82517 41.5 41:a07071fa:::benchmark_data_smithi099_126887_object82516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e975)", 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.311462+0000", 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.75867871400001, 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4704043690000002, 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.160 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82517 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311462+0000", 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.161 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311462+0000", 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311464+0000", 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3370000000000001e-06 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.162 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311487+0000", 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3346999999999999e-05 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311498+0000", 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0679999999999999e-05 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311505+0000", 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0330000000000003e-06 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.163 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311605+0000", 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100361 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311609+0000", 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.461e-06 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296097+0000", 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98448849800000005 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.164 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296105+0000", 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3270000000000001e-06 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.753647+0000", 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4575415540000001 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.753805+0000", 2024-09-13T14:51:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015768800000000001 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.754030+0000", 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022548699999999999 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.770273+0000", 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016243210000000001 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.166 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.778789+0000", 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085151740000000004 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.778916+0000", 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012748600000000001 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.778975+0000", 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9307e-05 2024-09-13T14:51:15.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.781867+0000", 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028914190000000001 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82520 41.5 41:ae1ac5a9:::benchmark_data_smithi099_126887_object82519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e975)", 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.318652+0000", 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.751488954, 2024-09-13T14:51:15.168 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4657732459999999, 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82520 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.318652+0000", 2024-09-13T14:51:15.169 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.318652+0000", 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.318654+0000", 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4789999999999999e-06 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.170 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.318673+0000", 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9218000000000001e-05 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.318688+0000", 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4657e-05 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.318697+0000", 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4709999999999996e-06 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.171 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.320598+0000", 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001900669 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.320609+0000", 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1514e-05 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296133+0000", 2024-09-13T14:51:15.172 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97552361399999998 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296141+0000", 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7489999999999998e-06 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.756378+0000", 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.460237137 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.173 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.756549+0000", 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017155800000000001 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.756727+0000", 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017742800000000001 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.770280+0000", 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013553004 2024-09-13T14:51:15.174 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.778860+0000", 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085798820000000005 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.782022+0000", 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031626699999999998 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.782077+0000", 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4929000000000002e-05 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.784425+0000", 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023482669999999998 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.176 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82521 41.5 41:a2e4d2cb:::benchmark_data_smithi099_126887_object82520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e975)", 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.320620+0000", 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.74952147799999, 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.475588949, 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82521 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.177 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.178 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.320620+0000", 2024-09-13T14:51:15.178 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.178 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.178 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.178 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.178 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.320620+0000", 2024-09-13T14:51:15.178 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.178 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.178 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.178 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.320621+0000", 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4279999999999999e-06 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.320645+0000", 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3541999999999999e-05 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.320660+0000", 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5557999999999999e-05 2024-09-13T14:51:15.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.320670+0000", 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8400000000000007e-06 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.320731+0000", 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1353999999999996e-05 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.320739+0000", 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0879999999999995e-06 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296169+0000", 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97542986499999995 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296177+0000", 2024-09-13T14:51:15.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9689999999999994e-06 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.759529+0000", 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.463351547 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.759711+0000", 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018203699999999999 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.759910+0000", 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019950600000000001 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.770285+0000", 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010375018 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.781964+0000", 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011678818000000001 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.793606+0000", 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011642151999999999 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.793669+0000", 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2674000000000003e-05 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.796209+0000", 2024-09-13T14:51:15.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025395529999999999 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82523 41.5 41:a3090b6f:::benchmark_data_smithi099_126887_object82522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e975)", 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.324307+0000", 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 145.745833801, 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.476410065, 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.185 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82523 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.324307+0000", 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.324307+0000", 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.324309+0000", 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4470000000000001e-06 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.324330+0000", 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1202000000000001e-05 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.324345+0000", 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4790000000000001e-05 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.324351+0000", 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4640000000000003e-06 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.324414+0000", 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2755000000000005e-05 2024-09-13T14:51:15.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.324427+0000", 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3053e-05 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296209+0000", 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97178183900000004 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.296217+0000", 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.072e-06 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.761907+0000", 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4656904829999999 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.762047+0000", 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013998400000000001 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.762221+0000", 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017395 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.775852+0000", 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013630256 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.784508+0000", 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0086558580000000006 2024-09-13T14:51:15.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.796266+0000", 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011758696000000001 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.796307+0000", 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1162000000000003e-05 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.192 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.800717+0000", 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044100540000000001 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137611 41.f 41:f3e29f70:::benchmark_data_smithi099_126887_object39823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.521997+0000", 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 62.548143682999999, 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.46795233000000003, 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.193 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137611 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.521997+0000", 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.521997+0000", 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.521999+0000", 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.609e-06 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522002+0000", 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3380000000000001e-06 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522011+0000", 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3259999999999996e-06 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522016+0000", 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2970000000000003e-06 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.526224+0000", 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0042076049999999997 2024-09-13T14:51:15.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.526371+0000", 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014755700000000001 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.526519+0000", 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014766599999999999 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.197 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.531915+0000", 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0053958069999999999 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.549794+0000", 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017879692999999999 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.985725+0000", 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.43593075399999998 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.198 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.986090+0000", 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00036506800000000002 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.989950+0000", 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0038596099999999999 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.199 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137613 41.f 41:f4b91036:::benchmark_data_smithi099_126887_object39825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.522025+0000", 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 62.548116282999999, 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.49653631599999998, 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137613 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522025+0000", 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522025+0000", 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522025+0000", 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.27e-07 2024-09-13T14:51:15.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522027+0000", 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.717e-06 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522032+0000", 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.916e-06 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522036+0000", 2024-09-13T14:51:15.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9249999999999997e-06 2024-09-13T14:51:15.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.531941+0000", 2024-09-13T14:51:15.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0099056219999999993 2024-09-13T14:51:15.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.532041+0000", 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9648000000000005e-05 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.532127+0000", 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5906e-05 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.536271+0000", 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004143847 2024-09-13T14:51:15.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.549818+0000", 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013547201 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.017054+0000", 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.46723604499999999 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.017107+0000", 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3396999999999997e-05 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.018561+0000", 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014537650000000001 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137628 41.f 41:f2dde755:::benchmark_data_smithi099_126887_object39840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:51:15.206 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.543546+0000", 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 62.526594906, 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.475842716, 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137628 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543546+0000", 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543546+0000", 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543547+0000", 2024-09-13T14:51:15.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.237e-06 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543553+0000", 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0250000000000002e-06 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543564+0000", 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1265e-05 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543570+0000", 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1750000000000002e-06 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.546441+0000", 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028709289999999999 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.546593+0000", 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000151669 2024-09-13T14:51:15.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.546748+0000", 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015586800000000001 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.550002+0000", 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032534450000000002 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.561684+0000", 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011681881999999999 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.018608+0000", 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45692447000000003 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.018659+0000", 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0742000000000003e-05 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.019389+0000", 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000730009 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137635 41.f 41:fa83ca08:::benchmark_data_smithi099_126887_object39847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.554069+0000", 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 62.516072463, 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.46605477099999998, 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:51:15.213 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137635 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.554069+0000", 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.554069+0000", 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.554070+0000", 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6020000000000001e-06 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.554075+0000", 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8579999999999999e-06 2024-09-13T14:51:15.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.554086+0000", 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1052e-05 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.554092+0000", 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4099999999999999e-06 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.554148+0000", 2024-09-13T14:51:15.216 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6060999999999997e-05 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.554259+0000", 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110955 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.554361+0000", 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010254799999999999 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.217 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.561647+0000", 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0072861009999999997 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.562091+0000", 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00044377000000000002 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.019426+0000", 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45733513799999997 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.019472+0000", 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5979999999999997e-05 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.020123+0000", 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00065129599999999999 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:51:15.219 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:51:15.220 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:14 smithi177 bash[68594]: audit 2024-09-13T14:51:14.613258+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.99:0/715825645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:15.220 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:14 smithi177 bash[68594]: audit 2024-09-13T14:51:14.613258+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.99:0/715825645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.221 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:15.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:15.322 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:15.350 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:15.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:15.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:15.484 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:15.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:15.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:15.612 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:15.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:15.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:15.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:15.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:15 smithi177 bash[68594]: cluster 2024-09-13T14:51:14.570336+0000 mgr.y (mgr.26252) 748 : cluster [DBG] pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 6.9 MiB/s, 111 objects/s recovering 2024-09-13T14:51:15.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:15 smithi177 bash[68594]: cluster 2024-09-13T14:51:14.570336+0000 mgr.y (mgr.26252) 748 : cluster [DBG] pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 6.9 MiB/s, 111 objects/s recovering 2024-09-13T14:51:15.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:15.885 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:15.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:15.978 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:15 smithi099 bash[84576]: cluster 2024-09-13T14:51:14.570336+0000 mgr.y (mgr.26252) 748 : cluster [DBG] pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 6.9 MiB/s, 111 objects/s recovering 2024-09-13T14:51:15.978 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:15 smithi099 bash[84576]: cluster 2024-09-13T14:51:14.570336+0000 mgr.y (mgr.26252) 748 : cluster [DBG] pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 6.9 MiB/s, 111 objects/s recovering 2024-09-13T14:51:15.979 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:15 smithi099 bash[87147]: cluster 2024-09-13T14:51:14.570336+0000 mgr.y (mgr.26252) 748 : cluster [DBG] pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 6.9 MiB/s, 111 objects/s recovering 2024-09-13T14:51:15.979 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:15 smithi099 bash[87147]: cluster 2024-09-13T14:51:14.570336+0000 mgr.y (mgr.26252) 748 : cluster [DBG] pgmap v1230: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 6.9 MiB/s, 111 objects/s recovering 2024-09-13T14:51:15.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:16.012 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:16.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:16.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:16.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:16.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:16.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:16.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:16.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:16.373 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:16.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:16.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:16.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:16.540 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:16.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:16.644 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:16.671 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:16.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:16.690 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T14:51:16.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:16.799 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:16.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:16.852 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[84576]: audit 2024-09-13T14:51:15.801225+0000 mon.b (mon.1) 10 : audit [INF] from='client.? 172.21.15.99:0/126461784' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.852 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[84576]: audit 2024-09-13T14:51:15.801225+0000 mon.b (mon.1) 10 : audit [INF] from='client.? 172.21.15.99:0/126461784' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.852 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[84576]: audit 2024-09-13T14:51:15.804003+0000 mon.a (mon.0) 1752 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.852 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[84576]: audit 2024-09-13T14:51:15.804003+0000 mon.a (mon.0) 1752 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.853 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[84576]: cluster 2024-09-13T14:51:16.349390+0000 mon.a (mon.0) 1753 : cluster [WRN] Health check update: Degraded data redundancy: 16027/54888 objects degraded (29.199%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:16.853 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[84576]: cluster 2024-09-13T14:51:16.349390+0000 mon.a (mon.0) 1753 : cluster [WRN] Health check update: Degraded data redundancy: 16027/54888 objects degraded (29.199%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:16.853 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[87147]: audit 2024-09-13T14:51:15.801225+0000 mon.b (mon.1) 10 : audit [INF] from='client.? 172.21.15.99:0/126461784' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.853 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[87147]: audit 2024-09-13T14:51:15.801225+0000 mon.b (mon.1) 10 : audit [INF] from='client.? 172.21.15.99:0/126461784' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.853 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[87147]: audit 2024-09-13T14:51:15.804003+0000 mon.a (mon.0) 1752 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.853 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[87147]: audit 2024-09-13T14:51:15.804003+0000 mon.a (mon.0) 1752 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.853 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[87147]: cluster 2024-09-13T14:51:16.349390+0000 mon.a (mon.0) 1753 : cluster [WRN] Health check update: Degraded data redundancy: 16027/54888 objects degraded (29.199%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:16.853 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:16 smithi099 bash[87147]: cluster 2024-09-13T14:51:16.349390+0000 mon.a (mon.0) 1753 : cluster [WRN] Health check update: Degraded data redundancy: 16027/54888 objects degraded (29.199%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:16.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:16.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:16 smithi177 bash[68594]: audit 2024-09-13T14:51:15.801225+0000 mon.b (mon.1) 10 : audit [INF] from='client.? 172.21.15.99:0/126461784' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:16 smithi177 bash[68594]: audit 2024-09-13T14:51:15.801225+0000 mon.b (mon.1) 10 : audit [INF] from='client.? 172.21.15.99:0/126461784' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:16 smithi177 bash[68594]: audit 2024-09-13T14:51:15.804003+0000 mon.a (mon.0) 1752 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:16 smithi177 bash[68594]: audit 2024-09-13T14:51:15.804003+0000 mon.a (mon.0) 1752 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:16.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:16 smithi177 bash[68594]: cluster 2024-09-13T14:51:16.349390+0000 mon.a (mon.0) 1753 : cluster [WRN] Health check update: Degraded data redundancy: 16027/54888 objects degraded (29.199%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:16.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:16 smithi177 bash[68594]: cluster 2024-09-13T14:51:16.349390+0000 mon.a (mon.0) 1753 : cluster [WRN] Health check update: Degraded data redundancy: 16027/54888 objects degraded (29.199%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:16.929 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:17.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:17.063 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:17.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:17.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:17.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:17.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:17.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:17.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:51:17] "GET /metrics HTTP/1.1" 200 36751 "" "Prometheus/2.51.0" 2024-09-13T14:51:17.334 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:17.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:17.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:17.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:17.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:17.587 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:17.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:17.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:17.726 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:17.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:17.829 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:17.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[84576]: cluster 2024-09-13T14:51:16.570944+0000 mgr.y (mgr.26252) 749 : cluster [DBG] pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 4.7 MiB/s, 74 objects/s recovering 2024-09-13T14:51:17.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[84576]: cluster 2024-09-13T14:51:16.570944+0000 mgr.y (mgr.26252) 749 : cluster [DBG] pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 4.7 MiB/s, 74 objects/s recovering 2024-09-13T14:51:17.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[84576]: cluster 2024-09-13T14:51:16.676593+0000 mon.a (mon.0) 1754 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:17.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[84576]: cluster 2024-09-13T14:51:16.676593+0000 mon.a (mon.0) 1754 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:17.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[84576]: audit 2024-09-13T14:51:16.687033+0000 mon.a (mon.0) 1755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:17.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[84576]: audit 2024-09-13T14:51:16.687033+0000 mon.a (mon.0) 1755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:17.837 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[84576]: cluster 2024-09-13T14:51:16.690774+0000 mon.a (mon.0) 1756 : cluster [DBG] osdmap e989: 8 total, 8 up, 7 in 2024-09-13T14:51:17.838 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[84576]: cluster 2024-09-13T14:51:16.690774+0000 mon.a (mon.0) 1756 : cluster [DBG] osdmap e989: 8 total, 8 up, 7 in 2024-09-13T14:51:17.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[87147]: cluster 2024-09-13T14:51:16.570944+0000 mgr.y (mgr.26252) 749 : cluster [DBG] pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 4.7 MiB/s, 74 objects/s recovering 2024-09-13T14:51:17.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[87147]: cluster 2024-09-13T14:51:16.570944+0000 mgr.y (mgr.26252) 749 : cluster [DBG] pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 4.7 MiB/s, 74 objects/s recovering 2024-09-13T14:51:17.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[87147]: cluster 2024-09-13T14:51:16.676593+0000 mon.a (mon.0) 1754 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:17.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[87147]: cluster 2024-09-13T14:51:16.676593+0000 mon.a (mon.0) 1754 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:17.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[87147]: audit 2024-09-13T14:51:16.687033+0000 mon.a (mon.0) 1755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:17.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[87147]: audit 2024-09-13T14:51:16.687033+0000 mon.a (mon.0) 1755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:17.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[87147]: cluster 2024-09-13T14:51:16.690774+0000 mon.a (mon.0) 1756 : cluster [DBG] osdmap e989: 8 total, 8 up, 7 in 2024-09-13T14:51:17.838 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:17 smithi099 bash[87147]: cluster 2024-09-13T14:51:16.690774+0000 mon.a (mon.0) 1756 : cluster [DBG] osdmap e989: 8 total, 8 up, 7 in 2024-09-13T14:51:17.852 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:17.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:17 smithi177 bash[68594]: cluster 2024-09-13T14:51:16.570944+0000 mgr.y (mgr.26252) 749 : cluster [DBG] pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 4.7 MiB/s, 74 objects/s recovering 2024-09-13T14:51:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:17 smithi177 bash[68594]: cluster 2024-09-13T14:51:16.570944+0000 mgr.y (mgr.26252) 749 : cluster [DBG] pgmap v1231: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 16027/54888 objects degraded (29.199%); 180/54888 objects misplaced (0.328%); 4.7 MiB/s, 74 objects/s recovering 2024-09-13T14:51:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:17 smithi177 bash[68594]: cluster 2024-09-13T14:51:16.676593+0000 mon.a (mon.0) 1754 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:17 smithi177 bash[68594]: cluster 2024-09-13T14:51:16.676593+0000 mon.a (mon.0) 1754 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:17 smithi177 bash[68594]: audit 2024-09-13T14:51:16.687033+0000 mon.a (mon.0) 1755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:17 smithi177 bash[68594]: audit 2024-09-13T14:51:16.687033+0000 mon.a (mon.0) 1755 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:17 smithi177 bash[68594]: cluster 2024-09-13T14:51:16.690774+0000 mon.a (mon.0) 1756 : cluster [DBG] osdmap e989: 8 total, 8 up, 7 in 2024-09-13T14:51:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:17 smithi177 bash[68594]: cluster 2024-09-13T14:51:16.690774+0000 mon.a (mon.0) 1756 : cluster [DBG] osdmap e989: 8 total, 8 up, 7 in 2024-09-13T14:51:17.957 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:17.986 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:17.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:18.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:18.113 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:18.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:18.216 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:18.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:18.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:18.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:18.371 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:18.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:18.506 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:18.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:18.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:18.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:18.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:18.783 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:18.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:18.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:18.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:18.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:19.036 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:19.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:19.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:19.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:19.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:19.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:19.279 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:19.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:19.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:19 smithi177 bash[68594]: audit 2024-09-13T14:51:18.281169+0000 mon.a (mon.0) 1757 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:19.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:19 smithi177 bash[68594]: audit 2024-09-13T14:51:18.281169+0000 mon.a (mon.0) 1757 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:19.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:19 smithi177 bash[68594]: audit 2024-09-13T14:51:18.282270+0000 mon.a (mon.0) 1758 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:19.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:19 smithi177 bash[68594]: audit 2024-09-13T14:51:18.282270+0000 mon.a (mon.0) 1758 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:19.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:19 smithi177 bash[68594]: cluster 2024-09-13T14:51:18.571542+0000 mgr.y (mgr.26252) 750 : cluster [DBG] pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 969 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 15785/46548 objects degraded (33.911%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:19.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:19 smithi177 bash[68594]: cluster 2024-09-13T14:51:18.571542+0000 mgr.y (mgr.26252) 750 : cluster [DBG] pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 969 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 15785/46548 objects degraded (33.911%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:19.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:19.438 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:19.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:19.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:19.555 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[84576]: audit 2024-09-13T14:51:18.281169+0000 mon.a (mon.0) 1757 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:19.555 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[84576]: audit 2024-09-13T14:51:18.281169+0000 mon.a (mon.0) 1757 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:19.555 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[84576]: audit 2024-09-13T14:51:18.282270+0000 mon.a (mon.0) 1758 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:19.555 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[84576]: audit 2024-09-13T14:51:18.282270+0000 mon.a (mon.0) 1758 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:19.555 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[84576]: cluster 2024-09-13T14:51:18.571542+0000 mgr.y (mgr.26252) 750 : cluster [DBG] pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 969 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 15785/46548 objects degraded (33.911%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:19.555 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[84576]: cluster 2024-09-13T14:51:18.571542+0000 mgr.y (mgr.26252) 750 : cluster [DBG] pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 969 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 15785/46548 objects degraded (33.911%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:19.556 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[87147]: audit 2024-09-13T14:51:18.281169+0000 mon.a (mon.0) 1757 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:19.556 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[87147]: audit 2024-09-13T14:51:18.281169+0000 mon.a (mon.0) 1757 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:19.556 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[87147]: audit 2024-09-13T14:51:18.282270+0000 mon.a (mon.0) 1758 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:19.556 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[87147]: audit 2024-09-13T14:51:18.282270+0000 mon.a (mon.0) 1758 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:19.556 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[87147]: cluster 2024-09-13T14:51:18.571542+0000 mgr.y (mgr.26252) 750 : cluster [DBG] pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 969 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 15785/46548 objects degraded (33.911%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:19.556 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:19 smithi099 bash[87147]: cluster 2024-09-13T14:51:18.571542+0000 mgr.y (mgr.26252) 750 : cluster [DBG] pgmap v1233: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 969 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 15785/46548 objects degraded (33.911%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:19.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:19.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:19.687 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:19.721 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:19.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:19.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:19.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:19.979 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:20.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:20.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:20.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:20.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:20.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:20.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:20.284 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:20.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:20.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:20.413 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:20.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:20.519 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:20.527 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:51:20 smithi177 bash[61439]: ts=2024-09-13T14:51:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:20.540 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:20.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:20.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:20.671 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:20.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:20.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:20.794 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:20.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:20.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:20.924 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:20.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:21.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:21.058 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:21.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:21.161 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:21.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:21.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:21.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:21.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:21.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:51:21.412 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:21.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:21.546 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:21.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:21.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:21.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:21.699 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:21.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:21.801 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:21.810 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:21 smithi099 bash[84576]: cluster 2024-09-13T14:51:20.572298+0000 mgr.y (mgr.26252) 751 : cluster [DBG] pgmap v1234: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 887 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 15785/42627 objects degraded (37.031%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:21.810 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:21 smithi099 bash[84576]: cluster 2024-09-13T14:51:20.572298+0000 mgr.y (mgr.26252) 751 : cluster [DBG] pgmap v1234: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 887 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 15785/42627 objects degraded (37.031%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:21.810 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:21 smithi099 bash[84576]: cluster 2024-09-13T14:51:21.350212+0000 mon.a (mon.0) 1759 : cluster [WRN] Health check update: Degraded data redundancy: 15785/42627 objects degraded (37.031%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:21.810 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:21 smithi099 bash[84576]: cluster 2024-09-13T14:51:21.350212+0000 mon.a (mon.0) 1759 : cluster [WRN] Health check update: Degraded data redundancy: 15785/42627 objects degraded (37.031%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:21.811 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:21 smithi099 bash[87147]: cluster 2024-09-13T14:51:20.572298+0000 mgr.y (mgr.26252) 751 : cluster [DBG] pgmap v1234: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 887 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 15785/42627 objects degraded (37.031%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:21.811 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:21 smithi099 bash[87147]: cluster 2024-09-13T14:51:20.572298+0000 mgr.y (mgr.26252) 751 : cluster [DBG] pgmap v1234: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 887 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 15785/42627 objects degraded (37.031%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:21.811 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:21 smithi099 bash[87147]: cluster 2024-09-13T14:51:21.350212+0000 mon.a (mon.0) 1759 : cluster [WRN] Health check update: Degraded data redundancy: 15785/42627 objects degraded (37.031%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:21.811 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:21 smithi099 bash[87147]: cluster 2024-09-13T14:51:21.350212+0000 mon.a (mon.0) 1759 : cluster [WRN] Health check update: Degraded data redundancy: 15785/42627 objects degraded (37.031%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:21.835 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:21.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:21 smithi177 bash[68594]: cluster 2024-09-13T14:51:20.572298+0000 mgr.y (mgr.26252) 751 : cluster [DBG] pgmap v1234: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 887 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 15785/42627 objects degraded (37.031%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:21.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:21 smithi177 bash[68594]: cluster 2024-09-13T14:51:20.572298+0000 mgr.y (mgr.26252) 751 : cluster [DBG] pgmap v1234: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 887 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 15785/42627 objects degraded (37.031%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:21.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:21 smithi177 bash[68594]: cluster 2024-09-13T14:51:21.350212+0000 mon.a (mon.0) 1759 : cluster [WRN] Health check update: Degraded data redundancy: 15785/42627 objects degraded (37.031%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:21.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:21 smithi177 bash[68594]: cluster 2024-09-13T14:51:21.350212+0000 mon.a (mon.0) 1759 : cluster [WRN] Health check update: Degraded data redundancy: 15785/42627 objects degraded (37.031%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:21.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:21.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:21.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:22.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:22.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:22.190 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:22.215 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:22.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:22.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:22.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:22.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:22.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:22.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:22.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:22.588 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:22.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:22.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:22.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:22.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:22.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:22.856 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:22.878 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:22.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:23.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:23.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:23.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:23.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:23.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:23.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:23.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:51:23.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:23.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:23.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:23.399 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:23.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:51:23.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:23.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:23.529 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:23.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:23.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:23.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:23.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:23.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:23.772 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:23 smithi099 bash[87147]: cluster 2024-09-13T14:51:22.572942+0000 mgr.y (mgr.26252) 752 : cluster [DBG] pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 787 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 15785/37842 objects degraded (41.713%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:23.772 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:23 smithi099 bash[87147]: cluster 2024-09-13T14:51:22.572942+0000 mgr.y (mgr.26252) 752 : cluster [DBG] pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 787 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 15785/37842 objects degraded (41.713%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:23.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:23 smithi099 bash[84576]: cluster 2024-09-13T14:51:22.572942+0000 mgr.y (mgr.26252) 752 : cluster [DBG] pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 787 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 15785/37842 objects degraded (41.713%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:23.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:23 smithi099 bash[84576]: cluster 2024-09-13T14:51:22.572942+0000 mgr.y (mgr.26252) 752 : cluster [DBG] pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 787 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 15785/37842 objects degraded (41.713%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:23.789 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:23.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:23 smithi177 bash[68594]: cluster 2024-09-13T14:51:22.572942+0000 mgr.y (mgr.26252) 752 : cluster [DBG] pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 787 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 15785/37842 objects degraded (41.713%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:23.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:23 smithi177 bash[68594]: cluster 2024-09-13T14:51:22.572942+0000 mgr.y (mgr.26252) 752 : cluster [DBG] pgmap v1235: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 787 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 15785/37842 objects degraded (41.713%); 4.8 MiB/s, 84 objects/s recovering 2024-09-13T14:51:23.894 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:23.926 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:23.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:24.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:24.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:24.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:24.157 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:24.187 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:24.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:24.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:24.310 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:24.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:24.413 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:24.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:24.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:24.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:24.578 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:24.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:24.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:24.708 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:24.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:51:24.733 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:51:24.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:24.847 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:24.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:24.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:24.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:25.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:25.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:25.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:25.203 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:25.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:25.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:25.329 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:25.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:25.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:25.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:25.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:25.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:25.582 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:25.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:25.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:25.721 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:25.766 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:25.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:25.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:25.889 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:25.890 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:25 smithi177 bash[68594]: cluster 2024-09-13T14:51:24.573718+0000 mgr.y (mgr.26252) 753 : cluster [DBG] pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:25.891 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:25 smithi177 bash[68594]: cluster 2024-09-13T14:51:24.573718+0000 mgr.y (mgr.26252) 753 : cluster [DBG] pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:25.891 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:25 smithi177 bash[68594]: audit 2024-09-13T14:51:24.713508+0000 mgr.y (mgr.26252) 754 : audit [DBG] from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:51:25.891 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:25 smithi177 bash[68594]: audit 2024-09-13T14:51:24.713508+0000 mgr.y (mgr.26252) 754 : audit [DBG] from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:51:25.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:25.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:26.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:26.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:26.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:26.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:26.168 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:25 smithi099 bash[84576]: cluster 2024-09-13T14:51:24.573718+0000 mgr.y (mgr.26252) 753 : cluster [DBG] pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:26.168 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:25 smithi099 bash[84576]: cluster 2024-09-13T14:51:24.573718+0000 mgr.y (mgr.26252) 753 : cluster [DBG] pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:26.168 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:25 smithi099 bash[84576]: audit 2024-09-13T14:51:24.713508+0000 mgr.y (mgr.26252) 754 : audit [DBG] from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:51:26.168 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:25 smithi099 bash[84576]: audit 2024-09-13T14:51:24.713508+0000 mgr.y (mgr.26252) 754 : audit [DBG] from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:51:26.169 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:25 smithi099 bash[87147]: cluster 2024-09-13T14:51:24.573718+0000 mgr.y (mgr.26252) 753 : cluster [DBG] pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:26.169 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:25 smithi099 bash[87147]: cluster 2024-09-13T14:51:24.573718+0000 mgr.y (mgr.26252) 753 : cluster [DBG] pgmap v1236: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:26.169 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:25 smithi099 bash[87147]: audit 2024-09-13T14:51:24.713508+0000 mgr.y (mgr.26252) 754 : audit [DBG] from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:51:26.169 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:25 smithi099 bash[87147]: audit 2024-09-13T14:51:24.713508+0000 mgr.y (mgr.26252) 754 : audit [DBG] from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:51:26.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:26.298 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:26.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:26.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:26.551 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:26.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:26.653 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:26.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:26.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:26.761 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1236,"stamp":"2024-09-13T14:51:24.573122+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":651040741,"num_objects":9952,"num_object_clones":0,"num_object_copies":29856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15434,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9952,"num_whiteouts":0,"num_read":345,"num_read_kb":800,"num_write":104167,"num_write_kb":3125884,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11248,"num_bytes_recovered":707007967,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":213535,"ondisk_log_size":213535,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3665692,"kb_used_data":2026276,"kb_used_omap":286,"kb_used_meta":1639009,"kb_avail":652550372,"statfs":{"total":671965249536,"available":668211580928,"internally_reserved":0,"allocated":2074906624,"data_stored":2049456446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":293460,"internal_metadata":1678345644},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,9,2,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-546832384,"num_objects":-8344,"num_object_clones":0,"num_object_copies":-25032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-593,"num_objects_misplaced":-180,"num_objects_unfound":0,"num_objects_dirty":-8344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8344,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":773,"num_bytes_recovered":22020096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003408"},"pg_stats":[{"pgid":"41.19","version":"989'10664","reported_seq":10776,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:22.286448+0000","last_change":"2024-09-13T14:50:54.287821+0000","last_active":"2024-09-13T14:51:22.286448+0000","last_peered":"2024-09-13T14:51:22.286448+0000","last_clean":"2024-09-13T14:51:22.286448+0000","last_became_active":"2024-09-13T14:50:52.465138+0000","last_became_peered":"2024-09-13T14:50:52.465138+0000","last_unstale":"2024-09-13T14:51:22.286448+0000","last_undegraded":"2024-09-13T14:51:22.286448+0000","last_fullsized":"2024-09-13T14:51:22.286448+0000","mapping_epoch":986,"log_start":"972'2900","ondisk_log_start":"972'2900","created":986,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":5,"last_scrub":"988'9928","last_scrub_stamp":"2024-09-13T14:50:54.287621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:54.287621+0000","objects_scrubbed":1059,"log_size":7764,"log_dups_size":0,"ondisk_log_size":7764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:52.069616+0000","scrub_duration":351,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20905984,"num_objects":319,"num_object_clones":0,"num_object_copies":957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":740,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"41.18","version":"989'11114","reported_seq":12230,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:24.111090+0000","last_change":"2024-09-13T14:50:55.152124+0000","last_active":"2024-09-13T14:51:24.111090+0000","last_peered":"2024-09-13T14:51:24.111090+0000","last_clean":"2024-09-13T14:51:24.111090+0000","last_became_active":"2024-09-13T14:50:52.463593+0000","last_became_peered":"2024-09-13T14:50:52.463593+0000","last_unstale":"2024-09-13T14:51:24.111090+0000","last_undegraded":"2024-09-13T14:51:24.111090+0000","last_fullsized":"2024-09-13T14:51:24.111090+0000","mapping_epoch":986,"log_start":"956'300","ondisk_log_start":"956'300","created":986,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":5,"last_scrub":"988'10403","last_scrub_stamp":"2024-09-13T14:50:55.152065+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:55.152065+0000","objects_scrubbed":1031,"log_size":10814,"log_dups_size":0,"ondisk_log_size":10814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:20:07.557647+0000","scrub_duration":0.25726823900000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20774912,"num_objects":317,"num_object_clones":0,"num_object_copies":951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":714,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"41.17","version":"989'10794","reported_seq":10890,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:23.960553+0000","last_change":"2024-09-13T14:51:00.886983+0000","last_active":"2024-09-13T14:51:23.960553+0000","last_peered":"2024-09-13T14:51:23.960553+0000","last_clean":"2024-09-13T14:51:23.960553+0000","last_became_active":"2024-09-13T14:50:52.457937+0000","last_became_peered":"2024-09-13T14:50:52.457937+0000","last_unstale":"2024-09-13T14:51:23.960553+0000","last_undegraded":"2024-09-13T14:51:23.960553+0000","last_fullsized":"2024-09-13T14:51:23.960553+0000","mapping_epoch":986,"log_start":"972'2500","ondisk_log_start":"972'2500","created":986,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":5,"last_scrub":"988'10221","last_scrub_stamp":"2024-09-13T14:51:00.886904+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:51:00.886904+0000","objects_scrubbed":814,"log_size":8294,"log_dups_size":0,"ondisk_log_size":8294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:28:26.977295+0000","scrub_duration":0.20036731499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15728640,"num_objects":240,"num_object_clones":0,"num_object_copies":720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":574,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"41.16","version":"989'10838","reported_seq":11948,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:21.821965+0000","last_change":"2024-09-13T14:50:56.180613+0000","last_active":"2024-09-13T14:51:21.821965+0000","last_peered":"2024-09-13T14:51:21.821965+0000","last_clean":"2024-09-13T14:51:21.821965+0000","last_became_active":"2024-09-13T14:50:52.466586+0000","last_became_peered":"2024-09-13T14:50:52.466586+0000","last_unstale":"2024-09-13T14:51:21.821965+0000","last_undegraded":"2024-09-13T14:51:21.821965+0000","last_fullsized":"2024-09-13T14:51:21.821965+0000","mapping_epoch":986,"log_start":"972'2900","ondisk_log_start":"972'2900","created":986,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":5,"last_scrub":"988'10132","last_scrub_stamp":"2024-09-13T14:50:56.179695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:56.179695+0000","objects_scrubbed":1045,"log_size":7938,"log_dups_size":0,"ondisk_log_size":7938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:20:44.681852+0000","scrub_duration":303,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22020096,"num_objects":336,"num_object_clones":0,"num_object_copies":1008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":709,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"41.15","version":"989'10894","reported_seq":12332,"reported_epoch":989,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-13T14:51:22.914595+0000","last_change":"2024-09-13T14:50:52.587106+0000","last_active":"2024-09-13T14:51:22.914595+0000","last_peered":"2024-09-13T14:51:22.914595+0000","last_clean":"2024-09-13T14:48:49.249873+0000","last_became_active":"2024-09-13T14:50:52.539545+0000","last_became_peered":"2024-09-13T14:50:52.539545+0000","last_unstale":"2024-09-13T14:51:22.914595+0000","last_undegraded":"2024-09-13T14:50:52.537824+0000","last_fullsized":"2024-09-13T14:51:22.914595+0000","mapping_epoch":986,"log_start":"956'300","ondisk_log_start":"956'300","created":986,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":10594,"log_dups_size":0,"ondisk_log_size":10594,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18808832,"num_objects":287,"num_object_clones":0,"num_object_copies":861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1973,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5862,"num_write_kb":196768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":675,"num_bytes_recovered":44105728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,2,6","objects":287}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"41.14","version":"989'10898","reported_seq":10997,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:23.996332+0000","last_change":"2024-09-13T14:51:02.157921+0000","last_active":"2024-09-13T14:51:23.996332+0000","last_peered":"2024-09-13T14:51:23.996332+0000","last_clean":"2024-09-13T14:51:23.996332+0000","last_became_active":"2024-09-13T14:50:52.460459+0000","last_became_peered":"2024-09-13T14:50:52.460459+0000","last_unstale":"2024-09-13T14:51:23.996332+0000","last_undegraded":"2024-09-13T14:51:23.996332+0000","last_fullsized":"2024-09-13T14:51:23.996332+0000","mapping_epoch":986,"log_start":"972'2500","ondisk_log_start":"972'2500","created":986,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":5,"last_scrub":"988'10338","last_scrub_stamp":"2024-09-13T14:51:02.157867+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:51:02.157867+0000","objects_scrubbed":855,"log_size":8398,"log_dups_size":0,"ondisk_log_size":8398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:16.414690+0000","scrub_duration":0.48482796099999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18808832,"num_objects":287,"num_object_clones":0,"num_object_copies":861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":568,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"41.13","version":"989'10689","reported_seq":11749,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:23.363426+0000","last_change":"2024-09-13T14:50:54.894492+0000","last_active":"2024-09-13T14:51:23.363426+0000","last_peered":"2024-09-13T14:51:23.363426+0000","last_clean":"2024-09-13T14:51:23.363426+0000","last_became_active":"2024-09-13T14:50:52.438714+0000","last_became_peered":"2024-09-13T14:50:52.438714+0000","last_unstale":"2024-09-13T14:51:23.363426+0000","last_undegraded":"2024-09-13T14:51:23.363426+0000","last_fullsized":"2024-09-13T14:51:23.363426+0000","mapping_epoch":986,"log_start":"973'3200","ondisk_log_start":"973'3200","created":986,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":5,"last_scrub":"988'9968","last_scrub_stamp":"2024-09-13T14:50:54.894414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:54.894414+0000","objects_scrubbed":1023,"log_size":7489,"log_dups_size":0,"ondisk_log_size":7489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:33:52.430611+0000","scrub_duration":339,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19595264,"num_objects":299,"num_object_clones":0,"num_object_copies":897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":724,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"41.12","version":"989'11059","reported_seq":12134,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:23.946429+0000","last_change":"2024-09-13T14:50:52.464064+0000","last_active":"2024-09-13T14:51:23.946429+0000","last_peered":"2024-09-13T14:51:23.946429+0000","last_clean":"2024-09-13T14:51:23.946429+0000","last_became_active":"2024-09-13T14:50:52.442212+0000","last_became_peered":"2024-09-13T14:50:52.442212+0000","last_unstale":"2024-09-13T14:51:23.946429+0000","last_undegraded":"2024-09-13T14:51:23.946429+0000","last_fullsized":"2024-09-13T14:51:23.946429+0000","mapping_epoch":986,"log_start":"972'2700","ondisk_log_start":"972'2700","created":986,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":8359,"log_dups_size":0,"ondisk_log_size":8359,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16121856,"num_objects":246,"num_object_clones":0,"num_object_copies":738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5945,"num_write_kb":198112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":250,"num_bytes_recovered":16351232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"41.11","version":"989'11100","reported_seq":11211,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:23.309472+0000","last_change":"2024-09-13T14:50:55.844308+0000","last_active":"2024-09-13T14:51:23.309472+0000","last_peered":"2024-09-13T14:51:23.309472+0000","last_clean":"2024-09-13T14:51:23.309472+0000","last_became_active":"2024-09-13T14:50:52.457229+0000","last_became_peered":"2024-09-13T14:50:52.457229+0000","last_unstale":"2024-09-13T14:51:23.309472+0000","last_undegraded":"2024-09-13T14:51:23.309472+0000","last_fullsized":"2024-09-13T14:51:23.309472+0000","mapping_epoch":986,"log_start":"973'3600","ondisk_log_start":"973'3600","created":986,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":5,"last_scrub":"988'10357","last_scrub_stamp":"2024-09-13T14:50:55.843983+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:55.843983+0000","objects_scrubbed":1054,"log_size":7500,"log_dups_size":0,"ondisk_log_size":7500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:21.746514+0000","scrub_duration":271,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20185088,"num_objects":308,"num_object_clones":0,"num_object_copies":924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":746,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"41.10","version":"989'10703","reported_seq":11818,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:21.846367+0000","last_change":"2024-09-13T14:50:55.030534+0000","last_active":"2024-09-13T14:51:21.846367+0000","last_peered":"2024-09-13T14:51:21.846367+0000","last_clean":"2024-09-13T14:51:21.846367+0000","last_became_active":"2024-09-13T14:50:52.441899+0000","last_became_peered":"2024-09-13T14:50:52.441899+0000","last_unstale":"2024-09-13T14:51:21.846367+0000","last_undegraded":"2024-09-13T14:51:21.846367+0000","last_fullsized":"2024-09-13T14:51:21.846367+0000","mapping_epoch":986,"log_start":"972'2800","ondisk_log_start":"972'2800","created":986,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":5,"last_scrub":"988'10054","last_scrub_stamp":"2024-09-13T14:50:55.030434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:55.030434+0000","objects_scrubbed":1017,"log_size":7903,"log_dups_size":0,"ondisk_log_size":7903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:52.244286+0000","scrub_duration":191,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23855104,"num_objects":364,"num_object_clones":0,"num_object_copies":1092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":653,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"41.f","version":"989'11113","reported_seq":21050,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:22.933063+0000","last_change":"2024-09-13T14:50:45.040692+0000","last_active":"2024-09-13T14:51:22.933063+0000","last_peered":"2024-09-13T14:51:22.933063+0000","last_clean":"2024-09-13T14:51:22.933063+0000","last_became_active":"2024-09-13T14:50:45.020513+0000","last_became_peered":"2024-09-13T14:50:45.020513+0000","last_unstale":"2024-09-13T14:51:22.933063+0000","last_undegraded":"2024-09-13T14:51:22.933063+0000","last_fullsized":"2024-09-13T14:51:22.933063+0000","mapping_epoch":983,"log_start":"972'2697","ondisk_log_start":"972'2697","created":953,"last_epoch_clean":984,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":8416,"log_dups_size":0,"ondisk_log_size":8416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:52:27.048502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40042496,"num_objects":611,"num_object_clones":0,"num_object_copies":1833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11113,"num_write_kb":375168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4921,"num_bytes_recovered":322502656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"41.e","version":"989'11695","reported_seq":11739,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:24.136821+0000","last_change":"2024-09-13T14:47:17.865396+0000","last_active":"2024-09-13T14:51:24.136821+0000","last_peered":"2024-09-13T14:51:24.136821+0000","last_clean":"2024-09-13T14:51:24.136821+0000","last_became_active":"2024-09-13T14:47:17.864558+0000","last_became_peered":"2024-09-13T14:47:17.864558+0000","last_unstale":"2024-09-13T14:51:24.136821+0000","last_undegraded":"2024-09-13T14:51:24.136821+0000","last_fullsized":"2024-09-13T14:51:24.136821+0000","mapping_epoch":953,"log_start":"972'2248","ondisk_log_start":"972'2248","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":9447,"log_dups_size":0,"ondisk_log_size":9447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:46.006485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35192832,"num_objects":537,"num_object_clones":0,"num_object_copies":1611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11695,"num_write_kb":391424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"41.d","version":"989'11419","reported_seq":11506,"reported_epoch":989,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:51:22.348995+0000","last_change":"2024-09-13T14:48:51.563400+0000","last_active":"2024-09-13T14:51:22.348995+0000","last_peered":"2024-09-13T14:51:22.348995+0000","last_clean":"2024-09-13T14:48:49.253715+0000","last_became_active":"2024-09-13T14:48:51.556136+0000","last_became_peered":"2024-09-13T14:48:51.556136+0000","last_unstale":"2024-09-13T14:51:22.348995+0000","last_undegraded":"2024-09-13T14:48:51.361665+0000","last_fullsized":"2024-09-13T14:48:51.326027+0000","mapping_epoch":976,"log_start":"974'5023","ondisk_log_start":"974'5023","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":6396,"log_dups_size":0,"ondisk_log_size":6396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43647000,"num_objects":667,"num_object_clones":0,"num_object_copies":2001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5092,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":667,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11419,"num_write_kb":386689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,7","objects":-4425}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"41.c","version":"989'11430","reported_seq":11517,"reported_epoch":989,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:51:22.329281+0000","last_change":"2024-09-13T14:48:51.561428+0000","last_active":"2024-09-13T14:51:22.329281+0000","last_peered":"2024-09-13T14:51:22.329281+0000","last_clean":"2024-09-13T14:48:49.232955+0000","last_became_active":"2024-09-13T14:48:51.552435+0000","last_became_peered":"2024-09-13T14:48:51.552435+0000","last_unstale":"2024-09-13T14:51:22.329281+0000","last_undegraded":"2024-09-13T14:48:51.361493+0000","last_fullsized":"2024-09-13T14:48:51.322450+0000","mapping_epoch":976,"log_start":"974'5012","ondisk_log_start":"974'5012","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":6418,"log_dups_size":0,"ondisk_log_size":6418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42729472,"num_objects":652,"num_object_clones":0,"num_object_copies":1956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11430,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,7","objects":-4443}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"41.b","version":"989'12026","reported_seq":20211,"reported_epoch":989,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:51:23.370154+0000","last_change":"2024-09-13T14:49:30.902213+0000","last_active":"2024-09-13T14:51:23.370154+0000","last_peered":"2024-09-13T14:51:23.370154+0000","last_clean":"2024-09-13T14:48:49.237246+0000","last_became_active":"2024-09-13T14:48:51.508676+0000","last_became_peered":"2024-09-13T14:48:51.508676+0000","last_unstale":"2024-09-13T14:51:23.370154+0000","last_undegraded":"2024-09-13T14:49:30.898579+0000","last_fullsized":"2024-09-13T14:48:51.330096+0000","mapping_epoch":976,"log_start":"979'5773","ondisk_log_start":"979'5773","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":6253,"log_dups_size":0,"ondisk_log_size":6253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41549824,"num_objects":634,"num_object_clones":0,"num_object_copies":1902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1334,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12026,"num_write_kb":405120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4050,"num_bytes_recovered":236781568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"3,7","objects":-700}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"41.a","version":"989'11528","reported_seq":12652,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:21.852203+0000","last_change":"2024-09-13T14:48:02.319705+0000","last_active":"2024-09-13T14:51:21.852203+0000","last_peered":"2024-09-13T14:51:21.852203+0000","last_clean":"2024-09-13T14:51:21.852203+0000","last_became_active":"2024-09-13T14:48:02.316389+0000","last_became_peered":"2024-09-13T14:48:02.316389+0000","last_unstale":"2024-09-13T14:51:21.852203+0000","last_undegraded":"2024-09-13T14:51:21.852203+0000","last_fullsized":"2024-09-13T14:51:21.852203+0000","mapping_epoch":971,"log_start":"974'5071","ondisk_log_start":"974'5071","created":953,"last_epoch_clean":972,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":6457,"log_dups_size":0,"ondisk_log_size":6457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:50.283954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42074112,"num_objects":642,"num_object_clones":0,"num_object_copies":1926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11528,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":513,"num_bytes_recovered":33619968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"41.9","version":"989'10618","reported_seq":10729,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:22.375983+0000","last_change":"2024-09-13T14:50:53.127094+0000","last_active":"2024-09-13T14:51:22.375983+0000","last_peered":"2024-09-13T14:51:22.375983+0000","last_clean":"2024-09-13T14:51:22.375983+0000","last_became_active":"2024-09-13T14:50:52.397349+0000","last_became_peered":"2024-09-13T14:50:52.397349+0000","last_unstale":"2024-09-13T14:51:22.375983+0000","last_undegraded":"2024-09-13T14:51:22.375983+0000","last_fullsized":"2024-09-13T14:51:22.375983+0000","mapping_epoch":953,"log_start":"972'2900","ondisk_log_start":"972'2900","created":953,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":0,"last_scrub":"987'9911","last_scrub_stamp":"2024-09-13T14:50:53.126965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:53.126965+0000","objects_scrubbed":1051,"log_size":7718,"log_dups_size":0,"ondisk_log_size":7718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:24:17.296129+0000","scrub_duration":187,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22478848,"num_objects":343,"num_object_clones":0,"num_object_copies":1029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":708,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"41.8","version":"989'11188","reported_seq":12309,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:24.045824+0000","last_change":"2024-09-13T14:50:53.106271+0000","last_active":"2024-09-13T14:51:24.045824+0000","last_peered":"2024-09-13T14:51:24.045824+0000","last_clean":"2024-09-13T14:51:24.045824+0000","last_became_active":"2024-09-13T14:50:52.397426+0000","last_became_peered":"2024-09-13T14:50:52.397426+0000","last_unstale":"2024-09-13T14:51:24.045824+0000","last_undegraded":"2024-09-13T14:51:24.045824+0000","last_fullsized":"2024-09-13T14:51:24.045824+0000","mapping_epoch":961,"log_start":"956'300","ondisk_log_start":"956'300","created":953,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":0,"last_scrub":"987'10366","last_scrub_stamp":"2024-09-13T14:50:53.106210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:53.106210+0000","objects_scrubbed":1131,"log_size":10888,"log_dups_size":0,"ondisk_log_size":10888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:35:35.481352+0000","scrub_duration":0.15850714499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20250624,"num_objects":309,"num_object_clones":0,"num_object_copies":927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":822,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"41.7","version":"989'10860","reported_seq":10964,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:24.000554+0000","last_change":"2024-09-13T14:50:57.011480+0000","last_active":"2024-09-13T14:51:24.000554+0000","last_peered":"2024-09-13T14:51:24.000554+0000","last_clean":"2024-09-13T14:51:24.000554+0000","last_became_active":"2024-09-13T14:50:52.398122+0000","last_became_peered":"2024-09-13T14:50:52.398122+0000","last_unstale":"2024-09-13T14:51:24.000554+0000","last_undegraded":"2024-09-13T14:51:24.000554+0000","last_fullsized":"2024-09-13T14:51:24.000554+0000","mapping_epoch":953,"log_start":"972'2500","ondisk_log_start":"972'2500","created":953,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":0,"last_scrub":"988'10145","last_scrub_stamp":"2024-09-13T14:50:57.011409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:57.011409+0000","objects_scrubbed":975,"log_size":8360,"log_dups_size":0,"ondisk_log_size":8360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:32.526543+0000","scrub_duration":0.27882780699999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16842752,"num_objects":257,"num_object_clones":0,"num_object_copies":771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":718,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"41.6","version":"989'10807","reported_seq":11916,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:21.830160+0000","last_change":"2024-09-13T14:50:54.004728+0000","last_active":"2024-09-13T14:51:21.830160+0000","last_peered":"2024-09-13T14:51:21.830160+0000","last_clean":"2024-09-13T14:51:21.830160+0000","last_became_active":"2024-09-13T14:50:52.420975+0000","last_became_peered":"2024-09-13T14:50:52.420975+0000","last_unstale":"2024-09-13T14:51:21.830160+0000","last_undegraded":"2024-09-13T14:51:21.830160+0000","last_fullsized":"2024-09-13T14:51:21.830160+0000","mapping_epoch":967,"log_start":"972'2900","ondisk_log_start":"972'2900","created":953,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":0,"last_scrub":"988'10089","last_scrub_stamp":"2024-09-13T14:50:54.004639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:54.004639+0000","objects_scrubbed":1044,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:44:56.276708+0000","scrub_duration":207,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21168128,"num_objects":323,"num_object_clones":0,"num_object_copies":969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":721,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"41.5","version":"989'10809","reported_seq":12014,"reported_epoch":989,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-13T14:51:22.889338+0000","last_change":"2024-09-13T14:50:53.473029+0000","last_active":"2024-09-13T14:51:22.889338+0000","last_peered":"2024-09-13T14:51:22.889338+0000","last_clean":"2024-09-13T14:48:49.249873+0000","last_became_active":"2024-09-13T14:50:53.446120+0000","last_became_peered":"2024-09-13T14:50:53.446120+0000","last_unstale":"2024-09-13T14:51:22.889338+0000","last_undegraded":"2024-09-13T14:50:53.416686+0000","last_fullsized":"2024-09-13T14:51:22.889338+0000","mapping_epoch":987,"log_start":"956'300","ondisk_log_start":"956'300","created":953,"last_epoch_clean":954,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":10509,"log_dups_size":0,"ondisk_log_size":10509,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24379392,"num_objects":372,"num_object_clones":0,"num_object_copies":1116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1940,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5777,"num_write_kb":196768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":560,"num_bytes_recovered":36700160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":["2","0","1"],"object_location_counts":[{"shards":"0,2,6","objects":372}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"41.4","version":"989'10897","reported_seq":11000,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:24.061913+0000","last_change":"2024-09-13T14:50:58.526889+0000","last_active":"2024-09-13T14:51:24.061913+0000","last_peered":"2024-09-13T14:51:24.061913+0000","last_clean":"2024-09-13T14:51:24.061913+0000","last_became_active":"2024-09-13T14:50:52.426004+0000","last_became_peered":"2024-09-13T14:50:52.426004+0000","last_unstale":"2024-09-13T14:51:24.061913+0000","last_undegraded":"2024-09-13T14:51:24.061913+0000","last_fullsized":"2024-09-13T14:51:24.061913+0000","mapping_epoch":953,"log_start":"972'2500","ondisk_log_start":"972'2500","created":953,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":0,"last_scrub":"988'10210","last_scrub_stamp":"2024-09-13T14:50:58.526827+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:58.526827+0000","objects_scrubbed":974,"log_size":8397,"log_dups_size":0,"ondisk_log_size":8397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:47.481012+0000","scrub_duration":0.803985544,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18546688,"num_objects":283,"num_object_clones":0,"num_object_copies":849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":691,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"41.3","version":"989'10688","reported_seq":11749,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:23.296882+0000","last_change":"2024-09-13T14:50:52.980238+0000","last_active":"2024-09-13T14:51:23.296882+0000","last_peered":"2024-09-13T14:51:23.296882+0000","last_clean":"2024-09-13T14:51:23.296882+0000","last_became_active":"2024-09-13T14:50:52.416903+0000","last_became_peered":"2024-09-13T14:50:52.416903+0000","last_unstale":"2024-09-13T14:51:23.296882+0000","last_undegraded":"2024-09-13T14:51:23.296882+0000","last_fullsized":"2024-09-13T14:51:23.296882+0000","mapping_epoch":963,"log_start":"973'3200","ondisk_log_start":"973'3200","created":953,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":0,"last_scrub":"987'9935","last_scrub_stamp":"2024-09-13T14:50:52.980128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:52.980128+0000","objects_scrubbed":1070,"log_size":7488,"log_dups_size":0,"ondisk_log_size":7488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:15:45.621385+0000","scrub_duration":331,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20512768,"num_objects":313,"num_object_clones":0,"num_object_copies":939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":757,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"41.2","version":"989'10964","reported_seq":12040,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:23.946536+0000","last_change":"2024-09-13T14:50:52.440777+0000","last_active":"2024-09-13T14:51:23.946536+0000","last_peered":"2024-09-13T14:51:23.946536+0000","last_clean":"2024-09-13T14:51:23.946536+0000","last_became_active":"2024-09-13T14:50:52.422093+0000","last_became_peered":"2024-09-13T14:50:52.422093+0000","last_unstale":"2024-09-13T14:51:23.946536+0000","last_undegraded":"2024-09-13T14:51:23.946536+0000","last_fullsized":"2024-09-13T14:51:23.946536+0000","mapping_epoch":969,"log_start":"972'2700","ondisk_log_start":"972'2700","created":953,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:47:16.845531+0000","objects_scrubbed":0,"log_size":8264,"log_dups_size":0,"ondisk_log_size":8264,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22347776,"num_objects":341,"num_object_clones":0,"num_object_copies":1023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5850,"num_write_kb":198112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":249,"num_bytes_recovered":16351232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"41.1","version":"989'11045","reported_seq":11157,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:23.297030+0000","last_change":"2024-09-13T14:50:53.944666+0000","last_active":"2024-09-13T14:51:23.297030+0000","last_peered":"2024-09-13T14:51:23.297030+0000","last_clean":"2024-09-13T14:51:23.297030+0000","last_became_active":"2024-09-13T14:50:52.428558+0000","last_became_peered":"2024-09-13T14:50:52.428558+0000","last_unstale":"2024-09-13T14:51:23.297030+0000","last_undegraded":"2024-09-13T14:51:23.297030+0000","last_fullsized":"2024-09-13T14:51:23.297030+0000","mapping_epoch":953,"log_start":"973'3600","ondisk_log_start":"973'3600","created":953,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":0,"last_scrub":"988'10307","last_scrub_stamp":"2024-09-13T14:50:53.944580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:53.944580+0000","objects_scrubbed":1065,"log_size":7445,"log_dups_size":0,"ondisk_log_size":7445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:16:27.064591+0000","scrub_duration":339,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21299200,"num_objects":325,"num_object_clones":0,"num_object_copies":975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":740,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"41.0","version":"989'10755","reported_seq":11870,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:21.798458+0000","last_change":"2024-09-13T14:50:52.990595+0000","last_active":"2024-09-13T14:51:21.798458+0000","last_peered":"2024-09-13T14:51:21.798458+0000","last_clean":"2024-09-13T14:51:21.798458+0000","last_became_active":"2024-09-13T14:50:52.394526+0000","last_became_peered":"2024-09-13T14:50:52.394526+0000","last_unstale":"2024-09-13T14:51:21.798458+0000","last_undegraded":"2024-09-13T14:51:21.798458+0000","last_fullsized":"2024-09-13T14:51:21.798458+0000","mapping_epoch":965,"log_start":"972'2800","ondisk_log_start":"972'2800","created":953,"last_epoch_clean":987,"parent":"0.0","parent_split_bits":0,"last_scrub":"987'10032","last_scrub_stamp":"2024-09-13T14:50:52.990378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:47:16.845531+0000","last_clean_scrub_stamp":"2024-09-13T14:50:52.990378+0000","objects_scrubbed":1039,"log_size":7955,"log_dups_size":0,"ondisk_log_size":7955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:36:25.251132+0000","scrub_duration":171,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20578304,"num_objects":314,"num_object_clones":0,"num_object_copies":942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":725,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":1367,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.699800+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T14:51:16.699800+0000","last_peered":"2024-09-13T14:51:16.699800+0000","last_clean":"2024-09-13T14:51:16.699800+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T14:51:16.699800+0000","last_undegraded":"2024-09-13T14:51:16.699800+0000","last_fullsized":"2024-09-13T14:51:16.699800+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":80,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.687921+0000","last_change":"2024-09-13T14:48:50.321123+0000","last_active":"2024-09-13T14:51:16.687921+0000","last_peered":"2024-09-13T14:51:16.687921+0000","last_clean":"2024-09-13T14:51:16.687921+0000","last_became_active":"2024-09-13T14:48:50.319957+0000","last_became_peered":"2024-09-13T14:48:50.319957+0000","last_unstale":"2024-09-13T14:51:16.687921+0000","last_undegraded":"2024-09-13T14:51:16.687921+0000","last_fullsized":"2024-09-13T14:51:16.687921+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:47:41.504593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1217,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.704646+0000","last_change":"2024-09-13T14:39:34.403170+0000","last_active":"2024-09-13T14:51:16.704646+0000","last_peered":"2024-09-13T14:51:16.704646+0000","last_clean":"2024-09-13T14:51:16.704646+0000","last_became_active":"2024-09-13T14:39:34.389001+0000","last_became_peered":"2024-09-13T14:39:34.389001+0000","last_unstale":"2024-09-13T14:51:16.704646+0000","last_undegraded":"2024-09-13T14:51:16.704646+0000","last_fullsized":"2024-09-13T14:51:16.704646+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:38.542452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":20,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396301+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T14:50:53.396301+0000","last_peered":"2024-09-13T14:50:53.396301+0000","last_clean":"2024-09-13T14:50:53.396301+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T14:50:53.396301+0000","last_undegraded":"2024-09-13T14:50:53.396301+0000","last_fullsized":"2024-09-13T14:50:53.396301+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":974,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396658+0000","last_change":"2024-09-13T14:47:46.337827+0000","last_active":"2024-09-13T14:50:53.396658+0000","last_peered":"2024-09-13T14:50:53.396658+0000","last_clean":"2024-09-13T14:50:53.396658+0000","last_became_active":"2024-09-13T14:47:46.321140+0000","last_became_peered":"2024-09-13T14:47:46.321140+0000","last_unstale":"2024-09-13T14:50:53.396658+0000","last_undegraded":"2024-09-13T14:50:53.396658+0000","last_fullsized":"2024-09-13T14:50:53.396658+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:15.997382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":73,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.689004+0000","last_change":"2024-09-13T14:47:46.333043+0000","last_active":"2024-09-13T14:51:16.689004+0000","last_peered":"2024-09-13T14:51:16.689004+0000","last_clean":"2024-09-13T14:51:16.689004+0000","last_became_active":"2024-09-13T14:47:46.332292+0000","last_became_peered":"2024-09-13T14:47:46.332292+0000","last_unstale":"2024-09-13T14:51:16.689004+0000","last_undegraded":"2024-09-13T14:51:16.689004+0000","last_fullsized":"2024-09-13T14:51:16.689004+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:38.942334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1005,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396724+0000","last_change":"2024-09-13T14:48:50.325866+0000","last_active":"2024-09-13T14:50:53.396724+0000","last_peered":"2024-09-13T14:50:53.396724+0000","last_clean":"2024-09-13T14:50:53.396724+0000","last_became_active":"2024-09-13T14:48:50.306654+0000","last_became_peered":"2024-09-13T14:48:50.306654+0000","last_unstale":"2024-09-13T14:50:53.396724+0000","last_undegraded":"2024-09-13T14:50:53.396724+0000","last_fullsized":"2024-09-13T14:50:53.396724+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:31:52.323885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":80,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.688280+0000","last_change":"2024-09-13T14:48:50.319669+0000","last_active":"2024-09-13T14:51:16.688280+0000","last_peered":"2024-09-13T14:51:16.688280+0000","last_clean":"2024-09-13T14:51:16.688280+0000","last_became_active":"2024-09-13T14:48:50.318195+0000","last_became_peered":"2024-09-13T14:48:50.318195+0000","last_unstale":"2024-09-13T14:51:16.688280+0000","last_undegraded":"2024-09-13T14:51:16.688280+0000","last_fullsized":"2024-09-13T14:51:16.688280+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:47.189218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1375,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.706867+0000","last_change":"2024-09-13T14:47:46.330059+0000","last_active":"2024-09-13T14:51:16.706867+0000","last_peered":"2024-09-13T14:51:16.706867+0000","last_clean":"2024-09-13T14:51:16.706867+0000","last_became_active":"2024-09-13T14:47:46.329344+0000","last_became_peered":"2024-09-13T14:47:46.329344+0000","last_unstale":"2024-09-13T14:51:16.706867+0000","last_undegraded":"2024-09-13T14:51:16.706867+0000","last_fullsized":"2024-09-13T14:51:16.706867+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:58:43.470176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1271,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.725605+0000","last_change":"2024-09-13T14:48:50.313031+0000","last_active":"2024-09-13T14:51:16.725605+0000","last_peered":"2024-09-13T14:51:16.725605+0000","last_clean":"2024-09-13T14:51:16.725605+0000","last_became_active":"2024-09-13T14:48:50.312337+0000","last_became_peered":"2024-09-13T14:48:50.312337+0000","last_unstale":"2024-09-13T14:51:16.725605+0000","last_undegraded":"2024-09-13T14:51:16.725605+0000","last_fullsized":"2024-09-13T14:51:16.725605+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:17:02.018596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":80,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.688872+0000","last_change":"2024-09-13T14:48:50.302422+0000","last_active":"2024-09-13T14:51:16.688872+0000","last_peered":"2024-09-13T14:51:16.688872+0000","last_clean":"2024-09-13T14:51:16.688872+0000","last_became_active":"2024-09-13T14:48:50.301099+0000","last_became_peered":"2024-09-13T14:48:50.301099+0000","last_unstale":"2024-09-13T14:51:16.688872+0000","last_undegraded":"2024-09-13T14:51:16.688872+0000","last_fullsized":"2024-09-13T14:51:16.688872+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:06:19.459783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1257,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.701713+0000","last_change":"2024-09-13T14:48:50.318471+0000","last_active":"2024-09-13T14:51:16.701713+0000","last_peered":"2024-09-13T14:51:16.701713+0000","last_clean":"2024-09-13T14:51:16.701713+0000","last_became_active":"2024-09-13T14:48:50.318104+0000","last_became_peered":"2024-09-13T14:48:50.318104+0000","last_unstale":"2024-09-13T14:51:16.701713+0000","last_undegraded":"2024-09-13T14:51:16.701713+0000","last_fullsized":"2024-09-13T14:51:16.701713+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:40.089045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1295,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.702106+0000","last_change":"2024-09-13T14:48:50.320505+0000","last_active":"2024-09-13T14:51:16.702106+0000","last_peered":"2024-09-13T14:51:16.702106+0000","last_clean":"2024-09-13T14:51:16.702106+0000","last_became_active":"2024-09-13T14:48:50.318302+0000","last_became_peered":"2024-09-13T14:48:50.318302+0000","last_unstale":"2024-09-13T14:51:16.702106+0000","last_undegraded":"2024-09-13T14:51:16.702106+0000","last_fullsized":"2024-09-13T14:51:16.702106+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:44:02.320626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1363,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.700894+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T14:51:16.700894+0000","last_peered":"2024-09-13T14:51:16.700894+0000","last_clean":"2024-09-13T14:51:16.700894+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T14:51:16.700894+0000","last_undegraded":"2024-09-13T14:51:16.700894+0000","last_fullsized":"2024-09-13T14:51:16.700894+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1250,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.701463+0000","last_change":"2024-09-13T14:48:50.319178+0000","last_active":"2024-09-13T14:51:16.701463+0000","last_peered":"2024-09-13T14:51:16.701463+0000","last_clean":"2024-09-13T14:51:16.701463+0000","last_became_active":"2024-09-13T14:48:50.318773+0000","last_became_peered":"2024-09-13T14:48:50.318773+0000","last_unstale":"2024-09-13T14:51:16.701463+0000","last_undegraded":"2024-09-13T14:51:16.701463+0000","last_fullsized":"2024-09-13T14:51:16.701463+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:01.123295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":956,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.414343+0000","last_change":"2024-09-13T14:48:50.310483+0000","last_active":"2024-09-13T14:50:53.414343+0000","last_peered":"2024-09-13T14:50:53.414343+0000","last_clean":"2024-09-13T14:50:53.414343+0000","last_became_active":"2024-09-13T14:48:50.310017+0000","last_became_peered":"2024-09-13T14:48:50.310017+0000","last_unstale":"2024-09-13T14:50:53.414343+0000","last_undegraded":"2024-09-13T14:50:53.414343+0000","last_fullsized":"2024-09-13T14:50:53.414343+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:05.652854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1231,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.701372+0000","last_change":"2024-09-13T14:48:50.324418+0000","last_active":"2024-09-13T14:51:16.701372+0000","last_peered":"2024-09-13T14:51:16.701372+0000","last_clean":"2024-09-13T14:51:16.701372+0000","last_became_active":"2024-09-13T14:48:50.324073+0000","last_became_peered":"2024-09-13T14:48:50.324073+0000","last_unstale":"2024-09-13T14:51:16.701372+0000","last_undegraded":"2024-09-13T14:51:16.701372+0000","last_fullsized":"2024-09-13T14:51:16.701372+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:47.192606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1355,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.699881+0000","last_change":"2024-09-13T14:39:34.318266+0000","last_active":"2024-09-13T14:51:16.699881+0000","last_peered":"2024-09-13T14:51:16.699881+0000","last_clean":"2024-09-13T14:51:16.699881+0000","last_became_active":"2024-09-13T14:39:34.315973+0000","last_became_peered":"2024-09-13T14:39:34.315973+0000","last_unstale":"2024-09-13T14:51:16.699881+0000","last_undegraded":"2024-09-13T14:51:16.699881+0000","last_fullsized":"2024-09-13T14:51:16.699881+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:15.270282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1203,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.702151+0000","last_change":"2024-09-13T14:39:34.338378+0000","last_active":"2024-09-13T14:51:16.702151+0000","last_peered":"2024-09-13T14:51:16.702151+0000","last_clean":"2024-09-13T14:51:16.702151+0000","last_became_active":"2024-09-13T14:39:34.338005+0000","last_became_peered":"2024-09-13T14:39:34.338005+0000","last_unstale":"2024-09-13T14:51:16.702151+0000","last_undegraded":"2024-09-13T14:51:16.702151+0000","last_fullsized":"2024-09-13T14:51:16.702151+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:29.120379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":953,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396721+0000","last_change":"2024-09-13T14:48:50.318723+0000","last_active":"2024-09-13T14:50:53.396721+0000","last_peered":"2024-09-13T14:50:53.396721+0000","last_clean":"2024-09-13T14:50:53.396721+0000","last_became_active":"2024-09-13T14:48:50.318132+0000","last_became_peered":"2024-09-13T14:48:50.318132+0000","last_unstale":"2024-09-13T14:50:53.396721+0000","last_undegraded":"2024-09-13T14:50:53.396721+0000","last_fullsized":"2024-09-13T14:50:53.396721+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:25.352103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1000,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.433074+0000","last_change":"2024-09-13T14:47:46.354886+0000","last_active":"2024-09-13T14:50:53.433074+0000","last_peered":"2024-09-13T14:50:53.433074+0000","last_clean":"2024-09-13T14:50:53.433074+0000","last_became_active":"2024-09-13T14:47:46.329947+0000","last_became_peered":"2024-09-13T14:47:46.329947+0000","last_unstale":"2024-09-13T14:50:53.433074+0000","last_undegraded":"2024-09-13T14:50:53.433074+0000","last_fullsized":"2024-09-13T14:50:53.433074+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:22.443920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":684,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.414137+0000","last_change":"2024-09-13T14:48:50.312407+0000","last_active":"2024-09-13T14:50:53.414137+0000","last_peered":"2024-09-13T14:50:53.414137+0000","last_clean":"2024-09-13T14:50:53.414137+0000","last_became_active":"2024-09-13T14:48:50.308224+0000","last_became_peered":"2024-09-13T14:48:50.308224+0000","last_unstale":"2024-09-13T14:50:53.414137+0000","last_undegraded":"2024-09-13T14:50:53.414137+0000","last_fullsized":"2024-09-13T14:50:53.414137+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:59:45.050288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":83,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.688396+0000","last_change":"2024-09-13T14:47:46.344536+0000","last_active":"2024-09-13T14:51:16.688396+0000","last_peered":"2024-09-13T14:51:16.688396+0000","last_clean":"2024-09-13T14:51:16.688396+0000","last_became_active":"2024-09-13T14:47:46.321131+0000","last_became_peered":"2024-09-13T14:47:46.321131+0000","last_unstale":"2024-09-13T14:51:16.688396+0000","last_undegraded":"2024-09-13T14:51:16.688396+0000","last_fullsized":"2024-09-13T14:51:16.688396+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:31.114472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":80,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.688290+0000","last_change":"2024-09-13T14:48:50.310695+0000","last_active":"2024-09-13T14:51:16.688290+0000","last_peered":"2024-09-13T14:51:16.688290+0000","last_clean":"2024-09-13T14:51:16.688290+0000","last_became_active":"2024-09-13T14:48:50.309622+0000","last_became_peered":"2024-09-13T14:48:50.309622+0000","last_unstale":"2024-09-13T14:51:16.688290+0000","last_undegraded":"2024-09-13T14:51:16.688290+0000","last_fullsized":"2024-09-13T14:51:16.688290+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:56.001951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":73,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.687904+0000","last_change":"2024-09-13T14:47:46.318785+0000","last_active":"2024-09-13T14:51:16.687904+0000","last_peered":"2024-09-13T14:51:16.687904+0000","last_clean":"2024-09-13T14:51:16.687904+0000","last_became_active":"2024-09-13T14:47:46.318438+0000","last_became_peered":"2024-09-13T14:47:46.318438+0000","last_unstale":"2024-09-13T14:51:16.687904+0000","last_undegraded":"2024-09-13T14:51:16.687904+0000","last_fullsized":"2024-09-13T14:51:16.687904+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:44.745378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":959,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.414523+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T14:50:53.414523+0000","last_peered":"2024-09-13T14:50:53.414523+0000","last_clean":"2024-09-13T14:50:53.414523+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T14:50:53.414523+0000","last_undegraded":"2024-09-13T14:50:53.414523+0000","last_fullsized":"2024-09-13T14:50:53.414523+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1405,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.701062+0000","last_change":"2024-09-13T14:48:50.315638+0000","last_active":"2024-09-13T14:51:16.701062+0000","last_peered":"2024-09-13T14:51:16.701062+0000","last_clean":"2024-09-13T14:51:16.701062+0000","last_became_active":"2024-09-13T14:48:50.307974+0000","last_became_peered":"2024-09-13T14:48:50.307974+0000","last_unstale":"2024-09-13T14:51:16.701062+0000","last_undegraded":"2024-09-13T14:51:16.701062+0000","last_fullsized":"2024-09-13T14:51:16.701062+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:59.437169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1200,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.701548+0000","last_change":"2024-09-13T14:39:34.349945+0000","last_active":"2024-09-13T14:51:16.701548+0000","last_peered":"2024-09-13T14:51:16.701548+0000","last_clean":"2024-09-13T14:51:16.701548+0000","last_became_active":"2024-09-13T14:39:34.346730+0000","last_became_peered":"2024-09-13T14:39:34.346730+0000","last_unstale":"2024-09-13T14:51:16.701548+0000","last_undegraded":"2024-09-13T14:51:16.701548+0000","last_fullsized":"2024-09-13T14:51:16.701548+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:48.854603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1281,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.414025+0000","last_change":"2024-09-13T14:48:50.316513+0000","last_active":"2024-09-13T14:50:53.414025+0000","last_peered":"2024-09-13T14:50:53.414025+0000","last_clean":"2024-09-13T14:50:53.414025+0000","last_became_active":"2024-09-13T14:48:50.315976+0000","last_became_peered":"2024-09-13T14:48:50.315976+0000","last_unstale":"2024-09-13T14:50:53.414025+0000","last_undegraded":"2024-09-13T14:50:53.414025+0000","last_fullsized":"2024-09-13T14:50:53.414025+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:44.529556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1204,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.702360+0000","last_change":"2024-09-13T14:39:34.401432+0000","last_active":"2024-09-13T14:51:16.702360+0000","last_peered":"2024-09-13T14:51:16.702360+0000","last_clean":"2024-09-13T14:51:16.702360+0000","last_became_active":"2024-09-13T14:39:34.395458+0000","last_became_peered":"2024-09-13T14:39:34.395458+0000","last_unstale":"2024-09-13T14:51:16.702360+0000","last_undegraded":"2024-09-13T14:51:16.702360+0000","last_fullsized":"2024-09-13T14:51:16.702360+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:17.007388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1345,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.706770+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T14:51:16.706770+0000","last_peered":"2024-09-13T14:51:16.706770+0000","last_clean":"2024-09-13T14:51:16.706770+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T14:51:16.706770+0000","last_undegraded":"2024-09-13T14:51:16.706770+0000","last_fullsized":"2024-09-13T14:51:16.706770+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":995,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.433181+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T14:50:53.433181+0000","last_peered":"2024-09-13T14:50:53.433181+0000","last_clean":"2024-09-13T14:50:53.433181+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T14:50:53.433181+0000","last_undegraded":"2024-09-13T14:50:53.433181+0000","last_fullsized":"2024-09-13T14:50:53.433181+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":698,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.725513+0000","last_change":"2024-09-13T14:48:50.307105+0000","last_active":"2024-09-13T14:51:16.725513+0000","last_peered":"2024-09-13T14:51:16.725513+0000","last_clean":"2024-09-13T14:51:16.725513+0000","last_became_active":"2024-09-13T14:48:50.306324+0000","last_became_peered":"2024-09-13T14:48:50.306324+0000","last_unstale":"2024-09-13T14:51:16.725513+0000","last_undegraded":"2024-09-13T14:51:16.725513+0000","last_fullsized":"2024-09-13T14:51:16.725513+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:22.934075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1271,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396401+0000","last_change":"2024-09-13T14:48:50.307874+0000","last_active":"2024-09-13T14:50:53.396401+0000","last_peered":"2024-09-13T14:50:53.396401+0000","last_clean":"2024-09-13T14:50:53.396401+0000","last_became_active":"2024-09-13T14:48:50.305966+0000","last_became_peered":"2024-09-13T14:48:50.305966+0000","last_unstale":"2024-09-13T14:50:53.396401+0000","last_undegraded":"2024-09-13T14:50:53.396401+0000","last_fullsized":"2024-09-13T14:50:53.396401+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:06.157706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":638,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.702214+0000","last_change":"2024-09-13T14:48:50.306120+0000","last_active":"2024-09-13T14:51:16.702214+0000","last_peered":"2024-09-13T14:51:16.702214+0000","last_clean":"2024-09-13T14:51:16.702214+0000","last_became_active":"2024-09-13T14:48:50.305574+0000","last_became_peered":"2024-09-13T14:48:50.305574+0000","last_unstale":"2024-09-13T14:51:16.702214+0000","last_undegraded":"2024-09-13T14:51:16.702214+0000","last_fullsized":"2024-09-13T14:51:16.702214+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:15:16.993023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":960,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396697+0000","last_change":"2024-09-13T14:39:34.321388+0000","last_active":"2024-09-13T14:50:53.396697+0000","last_peered":"2024-09-13T14:50:53.396697+0000","last_clean":"2024-09-13T14:50:53.396697+0000","last_became_active":"2024-09-13T14:39:34.321025+0000","last_became_peered":"2024-09-13T14:39:34.321025+0000","last_unstale":"2024-09-13T14:50:53.396697+0000","last_undegraded":"2024-09-13T14:50:53.396697+0000","last_fullsized":"2024-09-13T14:50:53.396697+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:52.593825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":777,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.725683+0000","last_change":"2024-09-13T14:48:50.312676+0000","last_active":"2024-09-13T14:51:16.725683+0000","last_peered":"2024-09-13T14:51:16.725683+0000","last_clean":"2024-09-13T14:51:16.725683+0000","last_became_active":"2024-09-13T14:48:50.311904+0000","last_became_peered":"2024-09-13T14:48:50.311904+0000","last_unstale":"2024-09-13T14:51:16.725683+0000","last_undegraded":"2024-09-13T14:51:16.725683+0000","last_fullsized":"2024-09-13T14:51:16.725683+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:55.991145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.688381+0000","last_change":"2024-09-13T14:47:46.324072+0000","last_active":"2024-09-13T14:51:16.688381+0000","last_peered":"2024-09-13T14:51:16.688381+0000","last_clean":"2024-09-13T14:51:16.688381+0000","last_became_active":"2024-09-13T14:47:46.322613+0000","last_became_peered":"2024-09-13T14:47:46.322613+0000","last_unstale":"2024-09-13T14:51:16.688381+0000","last_undegraded":"2024-09-13T14:51:16.688381+0000","last_fullsized":"2024-09-13T14:51:16.688381+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:45.314706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":971,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396310+0000","last_change":"2024-09-13T14:48:50.330611+0000","last_active":"2024-09-13T14:50:53.396310+0000","last_peered":"2024-09-13T14:50:53.396310+0000","last_clean":"2024-09-13T14:50:53.396310+0000","last_became_active":"2024-09-13T14:48:50.310242+0000","last_became_peered":"2024-09-13T14:48:50.310242+0000","last_unstale":"2024-09-13T14:50:53.396310+0000","last_undegraded":"2024-09-13T14:50:53.396310+0000","last_fullsized":"2024-09-13T14:50:53.396310+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:28.097810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":606,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396784+0000","last_change":"2024-09-13T14:48:50.309417+0000","last_active":"2024-09-13T14:50:53.396784+0000","last_peered":"2024-09-13T14:50:53.396784+0000","last_clean":"2024-09-13T14:50:53.396784+0000","last_became_active":"2024-09-13T14:48:50.308664+0000","last_became_peered":"2024-09-13T14:48:50.308664+0000","last_unstale":"2024-09-13T14:50:53.396784+0000","last_undegraded":"2024-09-13T14:50:53.396784+0000","last_fullsized":"2024-09-13T14:50:53.396784+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:15.509105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":698,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.722798+0000","last_change":"2024-09-13T14:48:50.304113+0000","last_active":"2024-09-13T14:51:16.722798+0000","last_peered":"2024-09-13T14:51:16.722798+0000","last_clean":"2024-09-13T14:51:16.722798+0000","last_became_active":"2024-09-13T14:48:50.303641+0000","last_became_peered":"2024-09-13T14:48:50.303641+0000","last_unstale":"2024-09-13T14:51:16.722798+0000","last_undegraded":"2024-09-13T14:51:16.722798+0000","last_fullsized":"2024-09-13T14:51:16.722798+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:53:11.186019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1253,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.723127+0000","last_change":"2024-09-13T14:39:34.322533+0000","last_active":"2024-09-13T14:51:16.723127+0000","last_peered":"2024-09-13T14:51:16.723127+0000","last_clean":"2024-09-13T14:51:16.723127+0000","last_became_active":"2024-09-13T14:39:34.322070+0000","last_became_peered":"2024-09-13T14:39:34.322070+0000","last_unstale":"2024-09-13T14:51:16.723127+0000","last_undegraded":"2024-09-13T14:51:16.723127+0000","last_fullsized":"2024-09-13T14:51:16.723127+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:43.746420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1374,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.700853+0000","last_change":"2024-09-13T14:48:50.305592+0000","last_active":"2024-09-13T14:51:16.700853+0000","last_peered":"2024-09-13T14:51:16.700853+0000","last_clean":"2024-09-13T14:51:16.700853+0000","last_became_active":"2024-09-13T14:48:50.305223+0000","last_became_peered":"2024-09-13T14:48:50.305223+0000","last_unstale":"2024-09-13T14:51:16.700853+0000","last_undegraded":"2024-09-13T14:51:16.700853+0000","last_fullsized":"2024-09-13T14:51:16.700853+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:40:12.253906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1381,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.697940+0000","last_change":"2024-09-13T14:47:46.329033+0000","last_active":"2024-09-13T14:51:16.697940+0000","last_peered":"2024-09-13T14:51:16.697940+0000","last_clean":"2024-09-13T14:51:16.697940+0000","last_became_active":"2024-09-13T14:47:46.328203+0000","last_became_peered":"2024-09-13T14:47:46.328203+0000","last_unstale":"2024-09-13T14:51:16.697940+0000","last_undegraded":"2024-09-13T14:51:16.697940+0000","last_fullsized":"2024-09-13T14:51:16.697940+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:23.440178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":985,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.432275+0000","last_change":"2024-09-13T14:48:50.317044+0000","last_active":"2024-09-13T14:50:53.432275+0000","last_peered":"2024-09-13T14:50:53.432275+0000","last_clean":"2024-09-13T14:50:53.432275+0000","last_became_active":"2024-09-13T14:48:50.316179+0000","last_became_peered":"2024-09-13T14:48:50.316179+0000","last_unstale":"2024-09-13T14:50:53.432275+0000","last_undegraded":"2024-09-13T14:50:53.432275+0000","last_fullsized":"2024-09-13T14:50:53.432275+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:06:00.880130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":966,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.432099+0000","last_change":"2024-09-13T14:47:46.320830+0000","last_active":"2024-09-13T14:50:53.432099+0000","last_peered":"2024-09-13T14:50:53.432099+0000","last_clean":"2024-09-13T14:50:53.432099+0000","last_became_active":"2024-09-13T14:47:46.320443+0000","last_became_peered":"2024-09-13T14:47:46.320443+0000","last_unstale":"2024-09-13T14:50:53.432099+0000","last_undegraded":"2024-09-13T14:50:53.432099+0000","last_fullsized":"2024-09-13T14:50:53.432099+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:46.279580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1202,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.701697+0000","last_change":"2024-09-13T14:39:34.356129+0000","last_active":"2024-09-13T14:51:16.701697+0000","last_peered":"2024-09-13T14:51:16.701697+0000","last_clean":"2024-09-13T14:51:16.701697+0000","last_became_active":"2024-09-13T14:39:34.355288+0000","last_became_peered":"2024-09-13T14:39:34.355288+0000","last_unstale":"2024-09-13T14:51:16.701697+0000","last_undegraded":"2024-09-13T14:51:16.701697+0000","last_fullsized":"2024-09-13T14:51:16.701697+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.725003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":956,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396470+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T14:50:53.396470+0000","last_peered":"2024-09-13T14:50:53.396470+0000","last_clean":"2024-09-13T14:50:53.396470+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T14:50:53.396470+0000","last_undegraded":"2024-09-13T14:50:53.396470+0000","last_fullsized":"2024-09-13T14:50:53.396470+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1063,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.432427+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T14:50:53.432427+0000","last_peered":"2024-09-13T14:50:53.432427+0000","last_clean":"2024-09-13T14:50:53.432427+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T14:50:53.432427+0000","last_undegraded":"2024-09-13T14:50:53.432427+0000","last_fullsized":"2024-09-13T14:50:53.432427+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1296,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.702447+0000","last_change":"2024-09-13T14:48:50.323890+0000","last_active":"2024-09-13T14:51:16.702447+0000","last_peered":"2024-09-13T14:51:16.702447+0000","last_clean":"2024-09-13T14:51:16.702447+0000","last_became_active":"2024-09-13T14:48:50.323324+0000","last_became_peered":"2024-09-13T14:48:50.323324+0000","last_unstale":"2024-09-13T14:51:16.702447+0000","last_undegraded":"2024-09-13T14:51:16.702447+0000","last_fullsized":"2024-09-13T14:51:16.702447+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:23:46.125319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1234,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.701621+0000","last_change":"2024-09-13T14:48:50.325837+0000","last_active":"2024-09-13T14:51:16.701621+0000","last_peered":"2024-09-13T14:51:16.701621+0000","last_clean":"2024-09-13T14:51:16.701621+0000","last_became_active":"2024-09-13T14:48:50.325420+0000","last_became_peered":"2024-09-13T14:48:50.325420+0000","last_unstale":"2024-09-13T14:51:16.701621+0000","last_undegraded":"2024-09-13T14:51:16.701621+0000","last_fullsized":"2024-09-13T14:51:16.701621+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:34.804111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":80,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.688875+0000","last_change":"2024-09-13T14:48:50.319803+0000","last_active":"2024-09-13T14:51:16.688875+0000","last_peered":"2024-09-13T14:51:16.688875+0000","last_clean":"2024-09-13T14:51:16.688875+0000","last_became_active":"2024-09-13T14:48:50.318366+0000","last_became_peered":"2024-09-13T14:48:50.318366+0000","last_unstale":"2024-09-13T14:51:16.688875+0000","last_undegraded":"2024-09-13T14:51:16.688875+0000","last_fullsized":"2024-09-13T14:51:16.688875+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:00.668596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":73,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.687812+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T14:51:16.687812+0000","last_peered":"2024-09-13T14:51:16.687812+0000","last_clean":"2024-09-13T14:51:16.687812+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T14:51:16.687812+0000","last_undegraded":"2024-09-13T14:51:16.687812+0000","last_fullsized":"2024-09-13T14:51:16.687812+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":80,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.688614+0000","last_change":"2024-09-13T14:48:50.315439+0000","last_active":"2024-09-13T14:51:16.688614+0000","last_peered":"2024-09-13T14:51:16.688614+0000","last_clean":"2024-09-13T14:51:16.688614+0000","last_became_active":"2024-09-13T14:48:50.314478+0000","last_became_peered":"2024-09-13T14:48:50.314478+0000","last_unstale":"2024-09-13T14:51:16.688614+0000","last_undegraded":"2024-09-13T14:51:16.688614+0000","last_fullsized":"2024-09-13T14:51:16.688614+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:22.490918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":994,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.414436+0000","last_change":"2024-09-13T14:48:50.308531+0000","last_active":"2024-09-13T14:50:53.414436+0000","last_peered":"2024-09-13T14:50:53.414436+0000","last_clean":"2024-09-13T14:50:53.414436+0000","last_became_active":"2024-09-13T14:48:50.307100+0000","last_became_peered":"2024-09-13T14:48:50.307100+0000","last_unstale":"2024-09-13T14:50:53.414436+0000","last_undegraded":"2024-09-13T14:50:53.414436+0000","last_fullsized":"2024-09-13T14:50:53.414436+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:51:59.904615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":73,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.688294+0000","last_change":"2024-09-13T14:47:46.325496+0000","last_active":"2024-09-13T14:51:16.688294+0000","last_peered":"2024-09-13T14:51:16.688294+0000","last_clean":"2024-09-13T14:51:16.688294+0000","last_became_active":"2024-09-13T14:47:46.325099+0000","last_became_peered":"2024-09-13T14:47:46.325099+0000","last_unstale":"2024-09-13T14:51:16.688294+0000","last_undegraded":"2024-09-13T14:51:16.688294+0000","last_fullsized":"2024-09-13T14:51:16.688294+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:48.803608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1238,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.709410+0000","last_change":"2024-09-13T14:47:46.349682+0000","last_active":"2024-09-13T14:51:16.709410+0000","last_peered":"2024-09-13T14:51:16.709410+0000","last_clean":"2024-09-13T14:51:16.709410+0000","last_became_active":"2024-09-13T14:47:46.341656+0000","last_became_peered":"2024-09-13T14:47:46.341656+0000","last_unstale":"2024-09-13T14:51:16.709410+0000","last_undegraded":"2024-09-13T14:51:16.709410+0000","last_fullsized":"2024-09-13T14:51:16.709410+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:02.727169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1387,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.706906+0000","last_change":"2024-09-13T14:48:50.317959+0000","last_active":"2024-09-13T14:51:16.706906+0000","last_peered":"2024-09-13T14:51:16.706906+0000","last_clean":"2024-09-13T14:51:16.706906+0000","last_became_active":"2024-09-13T14:48:50.317578+0000","last_became_peered":"2024-09-13T14:48:50.317578+0000","last_unstale":"2024-09-13T14:51:16.706906+0000","last_undegraded":"2024-09-13T14:51:16.706906+0000","last_fullsized":"2024-09-13T14:51:16.706906+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:17.441358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":943,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396542+0000","last_change":"2024-09-13T14:39:34.393673+0000","last_active":"2024-09-13T14:50:53.396542+0000","last_peered":"2024-09-13T14:50:53.396542+0000","last_clean":"2024-09-13T14:50:53.396542+0000","last_became_active":"2024-09-13T14:39:34.393186+0000","last_became_peered":"2024-09-13T14:39:34.393186+0000","last_unstale":"2024-09-13T14:50:53.396542+0000","last_undegraded":"2024-09-13T14:50:53.396542+0000","last_fullsized":"2024-09-13T14:50:53.396542+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:23.130084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":956,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.414230+0000","last_change":"2024-09-13T14:48:50.313376+0000","last_active":"2024-09-13T14:50:53.414230+0000","last_peered":"2024-09-13T14:50:53.414230+0000","last_clean":"2024-09-13T14:50:53.414230+0000","last_became_active":"2024-09-13T14:48:50.311937+0000","last_became_peered":"2024-09-13T14:48:50.311937+0000","last_unstale":"2024-09-13T14:50:53.414230+0000","last_undegraded":"2024-09-13T14:50:53.414230+0000","last_fullsized":"2024-09-13T14:50:53.414230+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:27.168900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":931,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.433334+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:50:53.433334+0000","last_peered":"2024-09-13T14:50:53.433334+0000","last_clean":"2024-09-13T14:50:53.433334+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:50:53.433334+0000","last_undegraded":"2024-09-13T14:50:53.433334+0000","last_fullsized":"2024-09-13T14:50:53.433334+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":80,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.687985+0000","last_change":"2024-09-13T14:48:50.317851+0000","last_active":"2024-09-13T14:51:16.687985+0000","last_peered":"2024-09-13T14:51:16.687985+0000","last_clean":"2024-09-13T14:51:16.687985+0000","last_became_active":"2024-09-13T14:48:50.317295+0000","last_became_peered":"2024-09-13T14:48:50.317295+0000","last_unstale":"2024-09-13T14:51:16.687985+0000","last_undegraded":"2024-09-13T14:51:16.687985+0000","last_fullsized":"2024-09-13T14:51:16.687985+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:04.019051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1008,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.433704+0000","last_change":"2024-09-13T14:48:50.327871+0000","last_active":"2024-09-13T14:50:53.433704+0000","last_peered":"2024-09-13T14:50:53.433704+0000","last_clean":"2024-09-13T14:50:53.433704+0000","last_became_active":"2024-09-13T14:48:50.316167+0000","last_became_peered":"2024-09-13T14:48:50.316167+0000","last_unstale":"2024-09-13T14:50:53.433704+0000","last_undegraded":"2024-09-13T14:50:53.433704+0000","last_fullsized":"2024-09-13T14:50:53.433704+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:03:52.315823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":925,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396484+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:50:53.396484+0000","last_peered":"2024-09-13T14:50:53.396484+0000","last_clean":"2024-09-13T14:50:53.396484+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:50:53.396484+0000","last_undegraded":"2024-09-13T14:50:53.396484+0000","last_fullsized":"2024-09-13T14:50:53.396484+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":993,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.433380+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T14:50:53.433380+0000","last_peered":"2024-09-13T14:50:53.433380+0000","last_clean":"2024-09-13T14:50:53.433380+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T14:50:53.433380+0000","last_undegraded":"2024-09-13T14:50:53.433380+0000","last_fullsized":"2024-09-13T14:50:53.433380+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":959,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.414461+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T14:50:53.414461+0000","last_peered":"2024-09-13T14:50:53.414461+0000","last_clean":"2024-09-13T14:50:53.414461+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T14:50:53.414461+0000","last_undegraded":"2024-09-13T14:50:53.414461+0000","last_fullsized":"2024-09-13T14:50:53.414461+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1232,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.702191+0000","last_change":"2024-09-13T14:48:50.322285+0000","last_active":"2024-09-13T14:51:16.702191+0000","last_peered":"2024-09-13T14:51:16.702191+0000","last_clean":"2024-09-13T14:51:16.702191+0000","last_became_active":"2024-09-13T14:48:50.321935+0000","last_became_peered":"2024-09-13T14:48:50.321935+0000","last_unstale":"2024-09-13T14:51:16.702191+0000","last_undegraded":"2024-09-13T14:51:16.702191+0000","last_fullsized":"2024-09-13T14:51:16.702191+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:47.484432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":966,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.432633+0000","last_change":"2024-09-13T14:47:46.322428+0000","last_active":"2024-09-13T14:50:53.432633+0000","last_peered":"2024-09-13T14:50:53.432633+0000","last_clean":"2024-09-13T14:50:53.432633+0000","last_became_active":"2024-09-13T14:47:46.321955+0000","last_became_peered":"2024-09-13T14:47:46.321955+0000","last_unstale":"2024-09-13T14:50:53.432633+0000","last_undegraded":"2024-09-13T14:50:53.432633+0000","last_fullsized":"2024-09-13T14:50:53.432633+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:47.614065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1355,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.700959+0000","last_change":"2024-09-13T14:39:34.362694+0000","last_active":"2024-09-13T14:51:16.700959+0000","last_peered":"2024-09-13T14:51:16.700959+0000","last_clean":"2024-09-13T14:51:16.700959+0000","last_became_active":"2024-09-13T14:39:34.362435+0000","last_became_peered":"2024-09-13T14:39:34.362435+0000","last_unstale":"2024-09-13T14:51:16.700959+0000","last_undegraded":"2024-09-13T14:51:16.700959+0000","last_fullsized":"2024-09-13T14:51:16.700959+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:33.350328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1355,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.698451+0000","last_change":"2024-09-13T14:39:34.360941+0000","last_active":"2024-09-13T14:51:16.698451+0000","last_peered":"2024-09-13T14:51:16.698451+0000","last_clean":"2024-09-13T14:51:16.698451+0000","last_became_active":"2024-09-13T14:39:34.360383+0000","last_became_peered":"2024-09-13T14:39:34.360383+0000","last_unstale":"2024-09-13T14:51:16.698451+0000","last_undegraded":"2024-09-13T14:51:16.698451+0000","last_fullsized":"2024-09-13T14:51:16.698451+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:51.344296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":942,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.396596+0000","last_change":"2024-09-13T14:39:34.380634+0000","last_active":"2024-09-13T14:50:53.396596+0000","last_peered":"2024-09-13T14:50:53.396596+0000","last_clean":"2024-09-13T14:50:53.396596+0000","last_became_active":"2024-09-13T14:39:34.379785+0000","last_became_peered":"2024-09-13T14:39:34.379785+0000","last_unstale":"2024-09-13T14:50:53.396596+0000","last_undegraded":"2024-09-13T14:50:53.396596+0000","last_fullsized":"2024-09-13T14:50:53.396596+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:02.436193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":999,"reported_epoch":988,"state":"active+clean","last_fresh":"2024-09-13T14:50:53.432833+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T14:50:53.432833+0000","last_peered":"2024-09-13T14:50:53.432833+0000","last_clean":"2024-09-13T14:50:53.432833+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T14:50:53.432833+0000","last_undegraded":"2024-09-13T14:50:53.432833+0000","last_fullsized":"2024-09-13T14:50:53.432833+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1233,"reported_epoch":989,"state":"active+clean","last_fresh":"2024-09-13T14:51:16.703602+0000","last_change":"2024-09-13T14:48:50.322415+0000","last_active":"2024-09-13T14:51:16.703602+0000","last_peered":"2024-09-13T14:51:16.703602+0000","last_clean":"2024-09-13T14:51:16.703602+0000","last_became_active":"2024-09-13T14:48:50.321926+0000","last_became_peered":"2024-09-13T14:48:50.321926+0000","last_unstale":"2024-09-13T14:51:16.703602+0000","last_undegraded":"2024-09-13T14:51:16.703602+0000","last_fullsized":"2024-09-13T14:51:16.703602+0000","mapping_epoch":975,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":976,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:35:43.108789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":41,"num_pg":26,"stat_sum":{"num_bytes":650444824,"num_objects":9926,"num_object_clones":0,"num_object_copies":29778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15434,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9926,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":103955,"num_write_kb":3124225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11218,"num_bytes_recovered":706412544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3290443776,"data_stored":3290431560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":213371,"ondisk_log_size":213371,"up":78,"acting":75,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397884,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":530856,"kb_used_data":233504,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93214296,"statfs":{"total":95995035648,"available":95451439104,"internally_reserved":0,"allocated":239108096,"data_stored":235457415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594119,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":581084,"kb_used_data":275412,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93164068,"statfs":{"total":95995035648,"available":95400005632,"internally_reserved":0,"allocated":282021888,"data_stored":278383344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790353,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":500752,"kb_used_data":195080,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93244400,"statfs":{"total":95995035648,"available":95482265600,"internally_reserved":0,"allocated":199761920,"data_stored":196130938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980831881,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":498460,"kb_used_data":321940,"kb_used_omap":76,"kb_used_meta":176435,"kb_avail":93246692,"statfs":{"total":95995035648,"available":95484612608,"internally_reserved":0,"allocated":329666560,"data_stored":326027541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":180670272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911120,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":448968,"kb_used_data":300160,"kb_used_omap":77,"kb_used_meta":148658,"kb_avail":93296184,"statfs":{"total":95995035648,"available":95535292416,"internally_reserved":0,"allocated":307363840,"data_stored":303740509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":152226613},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023063,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":547472,"kb_used_data":344584,"kb_used_omap":71,"kb_used_meta":202808,"kb_avail":93197680,"statfs":{"total":95995035648,"available":95434424320,"internally_reserved":0,"allocated":352854016,"data_stored":349223039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":207676064},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2757369004193,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195385,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":558100,"kb_used_data":355596,"kb_used_omap":56,"kb_used_meta":202439,"kb_avail":93187052,"statfs":{"total":95995035648,"available":95423541248,"internally_reserved":0,"allocated":364130304,"data_stored":360493660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":207297679},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":343150592,"data_stored":343146520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1357840384,"data_stored":1357840384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":331681792,"data_stored":331677720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":278134784,"data_stored":278134784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":311099392,"data_stored":311099392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":183107584,"data_stored":183107584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":260964352,"data_stored":260964352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":41,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":224464896,"data_stored":224460824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:51:26.766 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:51:26.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_0 pg_num 36 2024-09-13T14:51:26.799 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:26.808 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:26 smithi177 bash[68594]: cluster 2024-09-13T14:51:26.351012+0000 mon.a (mon.0) 1760 : cluster [WRN] Health check update: Degraded data redundancy: 15434/29856 objects degraded (51.695%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:26.808 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:26 smithi177 bash[68594]: cluster 2024-09-13T14:51:26.351012+0000 mon.a (mon.0) 1760 : cluster [WRN] Health check update: Degraded data redundancy: 15434/29856 objects degraded (51.695%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:26.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:26.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:26.932 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:26.946 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:26 smithi099 bash[84576]: cluster 2024-09-13T14:51:26.351012+0000 mon.a (mon.0) 1760 : cluster [WRN] Health check update: Degraded data redundancy: 15434/29856 objects degraded (51.695%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:26.946 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:26 smithi099 bash[84576]: cluster 2024-09-13T14:51:26.351012+0000 mon.a (mon.0) 1760 : cluster [WRN] Health check update: Degraded data redundancy: 15434/29856 objects degraded (51.695%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:26.946 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:26 smithi099 bash[87147]: cluster 2024-09-13T14:51:26.351012+0000 mon.a (mon.0) 1760 : cluster [WRN] Health check update: Degraded data redundancy: 15434/29856 objects degraded (51.695%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:26.946 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:26 smithi099 bash[87147]: cluster 2024-09-13T14:51:26.351012+0000 mon.a (mon.0) 1760 : cluster [WRN] Health check update: Degraded data redundancy: 15434/29856 objects degraded (51.695%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:26.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:26.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:27.082 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:27.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:27.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:27.178 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:27.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:27.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:27.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:27.283 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:51:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:51:27] "GET /metrics HTTP/1.1" 200 36770 "" "Prometheus/2.51.0" 2024-09-13T14:51:27.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:27.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:27.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:27.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:27.496 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:27.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:27.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:27.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:27.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:27.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:27.741 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:27 smithi177 bash[68594]: cluster 2024-09-13T14:51:26.574231+0000 mgr.y (mgr.26252) 755 : cluster [DBG] pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:27.741 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:27 smithi177 bash[68594]: cluster 2024-09-13T14:51:26.574231+0000 mgr.y (mgr.26252) 755 : cluster [DBG] pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:27.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:27.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:27.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:27.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:27.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:28.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:28.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:27 smithi099 bash[84576]: cluster 2024-09-13T14:51:26.574231+0000 mgr.y (mgr.26252) 755 : cluster [DBG] pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:28.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:27 smithi099 bash[84576]: cluster 2024-09-13T14:51:26.574231+0000 mgr.y (mgr.26252) 755 : cluster [DBG] pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:28.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:27 smithi099 bash[87147]: cluster 2024-09-13T14:51:26.574231+0000 mgr.y (mgr.26252) 755 : cluster [DBG] pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:28.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:27 smithi099 bash[87147]: cluster 2024-09-13T14:51:26.574231+0000 mgr.y (mgr.26252) 755 : cluster [DBG] pgmap v1237: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 621 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 834 op/s; 15434/29856 objects degraded (51.695%); 2.1 MiB/s, 77 objects/s recovering 2024-09-13T14:51:28.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.136 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:28.240 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:28.261 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:28.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:28.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:28.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:28.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:28.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:28.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:28.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:28.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:28.771 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:28.796 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:28.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:28.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.934 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:29.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:29.063 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:29.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:29.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:29.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:29.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:29.270 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:29.270 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:29.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:29.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:29.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:29.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:29.440 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:29.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:29.543 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:29.568 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:29.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:30.029 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:29 smithi099 bash[84576]: cluster 2024-09-13T14:51:28.575196+0000 mgr.y (mgr.26252) 756 : cluster [DBG] pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 422 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 969 op/s; 15209/20304 objects degraded (74.906%); 1.8 MiB/s, 83 objects/s recovering 2024-09-13T14:51:30.030 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:29 smithi099 bash[84576]: cluster 2024-09-13T14:51:28.575196+0000 mgr.y (mgr.26252) 756 : cluster [DBG] pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 422 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 969 op/s; 15209/20304 objects degraded (74.906%); 1.8 MiB/s, 83 objects/s recovering 2024-09-13T14:51:30.030 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:29 smithi099 bash[87147]: cluster 2024-09-13T14:51:28.575196+0000 mgr.y (mgr.26252) 756 : cluster [DBG] pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 422 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 969 op/s; 15209/20304 objects degraded (74.906%); 1.8 MiB/s, 83 objects/s recovering 2024-09-13T14:51:30.030 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:29 smithi099 bash[87147]: cluster 2024-09-13T14:51:28.575196+0000 mgr.y (mgr.26252) 756 : cluster [DBG] pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 422 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 969 op/s; 15209/20304 objects degraded (74.906%); 1.8 MiB/s, 83 objects/s recovering 2024-09-13T14:51:30.061 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:30.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:30.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:29 smithi177 bash[68594]: cluster 2024-09-13T14:51:28.575196+0000 mgr.y (mgr.26252) 756 : cluster [DBG] pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 422 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 969 op/s; 15209/20304 objects degraded (74.906%); 1.8 MiB/s, 83 objects/s recovering 2024-09-13T14:51:30.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:29 smithi177 bash[68594]: cluster 2024-09-13T14:51:28.575196+0000 mgr.y (mgr.26252) 756 : cluster [DBG] pgmap v1238: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 422 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 969 op/s; 15209/20304 objects degraded (74.906%); 1.8 MiB/s, 83 objects/s recovering 2024-09-13T14:51:30.166 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:30.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:30.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:30.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.323 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:30.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.468 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:30.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:30.580 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:51:30 smithi177 bash[61439]: ts=2024-09-13T14:51:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:30.598 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:30.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:30.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:30.850 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:30.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:30.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:30.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:30.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:31.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:31.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:31.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:31.205 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:31.233 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:31.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:31.308 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:31.336 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:31.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:31.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:31.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:31.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:31.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:31.598 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:31.626 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:31.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:31.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:31.748 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:31.850 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:31.868 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:31.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:31.948 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:31 smithi099 bash[84576]: cluster 2024-09-13T14:51:30.575931+0000 mgr.y (mgr.26252) 757 : cluster [DBG] pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 332 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 15209/15969 objects degraded (95.241%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:31.948 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:31 smithi099 bash[84576]: cluster 2024-09-13T14:51:30.575931+0000 mgr.y (mgr.26252) 757 : cluster [DBG] pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 332 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 15209/15969 objects degraded (95.241%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:31.948 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:31 smithi099 bash[84576]: cluster 2024-09-13T14:51:31.351846+0000 mon.a (mon.0) 1761 : cluster [WRN] Health check update: Degraded data redundancy: 15209/15969 objects degraded (95.241%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:31.948 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:31 smithi099 bash[84576]: cluster 2024-09-13T14:51:31.351846+0000 mon.a (mon.0) 1761 : cluster [WRN] Health check update: Degraded data redundancy: 15209/15969 objects degraded (95.241%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:31.948 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:31 smithi099 bash[87147]: cluster 2024-09-13T14:51:30.575931+0000 mgr.y (mgr.26252) 757 : cluster [DBG] pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 332 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 15209/15969 objects degraded (95.241%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:31.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:31 smithi099 bash[87147]: cluster 2024-09-13T14:51:30.575931+0000 mgr.y (mgr.26252) 757 : cluster [DBG] pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 332 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 15209/15969 objects degraded (95.241%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:31.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:31 smithi099 bash[87147]: cluster 2024-09-13T14:51:31.351846+0000 mon.a (mon.0) 1761 : cluster [WRN] Health check update: Degraded data redundancy: 15209/15969 objects degraded (95.241%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:31.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:31 smithi099 bash[87147]: cluster 2024-09-13T14:51:31.351846+0000 mon.a (mon.0) 1761 : cluster [WRN] Health check update: Degraded data redundancy: 15209/15969 objects degraded (95.241%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:31.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:31.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:31 smithi177 bash[68594]: cluster 2024-09-13T14:51:30.575931+0000 mgr.y (mgr.26252) 757 : cluster [DBG] pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 332 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 15209/15969 objects degraded (95.241%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:31.985 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:31 smithi177 bash[68594]: cluster 2024-09-13T14:51:30.575931+0000 mgr.y (mgr.26252) 757 : cluster [DBG] pgmap v1239: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 332 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 15209/15969 objects degraded (95.241%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:31.985 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:31 smithi177 bash[68594]: cluster 2024-09-13T14:51:31.351846+0000 mon.a (mon.0) 1761 : cluster [WRN] Health check update: Degraded data redundancy: 15209/15969 objects degraded (95.241%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:31.985 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:31 smithi177 bash[68594]: cluster 2024-09-13T14:51:31.351846+0000 mon.a (mon.0) 1761 : cluster [WRN] Health check update: Degraded data redundancy: 15209/15969 objects degraded (95.241%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:32.002 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:32.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:32.133 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:32.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:32.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:32.257 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:32.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:32.394 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:32.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:32.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:32.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:32.628 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:32.662 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:32.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:32.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:32.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:32.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:32.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:32.924 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:33.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:33.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:33.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:33.166 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:33.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:33.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:33.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:33.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:33.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:33.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:33.462 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:33.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:33.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:33.596 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:33.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:33.688 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:33 smithi099 bash[84576]: cluster 2024-09-13T14:51:32.576567+0000 mgr.y (mgr.26252) 758 : cluster [DBG] pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 238 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 865 op/s; 15209/11460 objects degraded (132.714%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:33.688 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:33 smithi099 bash[84576]: cluster 2024-09-13T14:51:32.576567+0000 mgr.y (mgr.26252) 758 : cluster [DBG] pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 238 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 865 op/s; 15209/11460 objects degraded (132.714%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:33.688 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:33 smithi099 bash[84576]: audit 2024-09-13T14:51:33.278045+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:33.688 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:33 smithi099 bash[84576]: audit 2024-09-13T14:51:33.278045+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:33.689 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:33 smithi099 bash[87147]: cluster 2024-09-13T14:51:32.576567+0000 mgr.y (mgr.26252) 758 : cluster [DBG] pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 238 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 865 op/s; 15209/11460 objects degraded (132.714%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:33.689 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:33 smithi099 bash[87147]: cluster 2024-09-13T14:51:32.576567+0000 mgr.y (mgr.26252) 758 : cluster [DBG] pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 238 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 865 op/s; 15209/11460 objects degraded (132.714%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:33.689 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:33 smithi099 bash[87147]: audit 2024-09-13T14:51:33.278045+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:33.689 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:33 smithi099 bash[87147]: audit 2024-09-13T14:51:33.278045+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:33.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:33.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.733 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:33.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:33.834 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:33.858 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:33.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:33.961 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:34.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:34.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:33 smithi177 bash[68594]: cluster 2024-09-13T14:51:32.576567+0000 mgr.y (mgr.26252) 758 : cluster [DBG] pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 238 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 865 op/s; 15209/11460 objects degraded (132.714%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:33 smithi177 bash[68594]: cluster 2024-09-13T14:51:32.576567+0000 mgr.y (mgr.26252) 758 : cluster [DBG] pgmap v1240: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 238 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 865 op/s; 15209/11460 objects degraded (132.714%); 0 B/s, 47 objects/s recovering 2024-09-13T14:51:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:33 smithi177 bash[68594]: audit 2024-09-13T14:51:33.278045+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:33 smithi177 bash[68594]: audit 2024-09-13T14:51:33.278045+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:34.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:34.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:34.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:34.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:34.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:34.366 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:34.389 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:34.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:34.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:34.520 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:34.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:34.552 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Removed 97785 objects 2024-09-13T14:51:34.552 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Clean up completed and total clean up time :140.992 2024-09-13T14:51:34.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:34.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:34.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:34.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:34.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:34.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:34.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:34.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:34.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:35.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:35.028 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:35.131 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:35.157 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:35.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:35.260 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:35.287 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:35.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:35.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:35.415 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:35.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:35.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:35.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:35.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:35.677 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:35.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:35.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:35.791 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:35 smithi177 bash[68594]: cluster 2024-09-13T14:51:34.577154+0000 mgr.y (mgr.26252) 759 : cluster [DBG] pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:35.791 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:35 smithi177 bash[68594]: cluster 2024-09-13T14:51:34.577154+0000 mgr.y (mgr.26252) 759 : cluster [DBG] pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:35.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:35.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:35.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:35.937 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:36.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:36.067 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:36.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:36.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:35 smithi099 bash[84576]: cluster 2024-09-13T14:51:34.577154+0000 mgr.y (mgr.26252) 759 : cluster [DBG] pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:36.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:35 smithi099 bash[84576]: cluster 2024-09-13T14:51:34.577154+0000 mgr.y (mgr.26252) 759 : cluster [DBG] pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:36.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:35 smithi099 bash[87147]: cluster 2024-09-13T14:51:34.577154+0000 mgr.y (mgr.26252) 759 : cluster [DBG] pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:36.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:35 smithi099 bash[87147]: cluster 2024-09-13T14:51:34.577154+0000 mgr.y (mgr.26252) 759 : cluster [DBG] pgmap v1241: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:36.170 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:36.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:36.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:36.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:36.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:36.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:36.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:36.550 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:36.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:36.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:36.700 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:36.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:36.803 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:36.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:36 smithi099 bash[84576]: cluster 2024-09-13T14:51:36.352619+0000 mon.a (mon.0) 1763 : cluster [WRN] Health check update: Degraded data redundancy: 14984/4104 objects degraded (365.107%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:36.813 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:36 smithi099 bash[84576]: cluster 2024-09-13T14:51:36.352619+0000 mon.a (mon.0) 1763 : cluster [WRN] Health check update: Degraded data redundancy: 14984/4104 objects degraded (365.107%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:36.813 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:36 smithi099 bash[87147]: cluster 2024-09-13T14:51:36.352619+0000 mon.a (mon.0) 1763 : cluster [WRN] Health check update: Degraded data redundancy: 14984/4104 objects degraded (365.107%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:36.813 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:36 smithi099 bash[87147]: cluster 2024-09-13T14:51:36.352619+0000 mon.a (mon.0) 1763 : cluster [WRN] Health check update: Degraded data redundancy: 14984/4104 objects degraded (365.107%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:36.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:36.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:36.929 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:36.949 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:36.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:37.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:37.070 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:51:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:51:37] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-13T14:51:37.073 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:37.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:37.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:36 smithi177 bash[68594]: cluster 2024-09-13T14:51:36.352619+0000 mon.a (mon.0) 1763 : cluster [WRN] Health check update: Degraded data redundancy: 14984/4104 objects degraded (365.107%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:37.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:36 smithi177 bash[68594]: cluster 2024-09-13T14:51:36.352619+0000 mon.a (mon.0) 1763 : cluster [WRN] Health check update: Degraded data redundancy: 14984/4104 objects degraded (365.107%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:37.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:37.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:37.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:37.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:37.329 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:37.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:37.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:37.458 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:37.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:37.564 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:37.587 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:37.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:37.690 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.698 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:37 smithi099 bash[84576]: cluster 2024-09-13T14:51:36.577574+0000 mgr.y (mgr.26252) 760 : cluster [DBG] pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 37 objects/s recovering 2024-09-13T14:51:37.698 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:37 smithi099 bash[84576]: cluster 2024-09-13T14:51:36.577574+0000 mgr.y (mgr.26252) 760 : cluster [DBG] pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 37 objects/s recovering 2024-09-13T14:51:37.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:37 smithi099 bash[87147]: cluster 2024-09-13T14:51:36.577574+0000 mgr.y (mgr.26252) 760 : cluster [DBG] pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 37 objects/s recovering 2024-09-13T14:51:37.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:37 smithi099 bash[87147]: cluster 2024-09-13T14:51:36.577574+0000 mgr.y (mgr.26252) 760 : cluster [DBG] pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 37 objects/s recovering 2024-09-13T14:51:37.717 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:37.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:37.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:37.983 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:37.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:38.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:38.097 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:37 smithi177 bash[68594]: cluster 2024-09-13T14:51:36.577574+0000 mgr.y (mgr.26252) 760 : cluster [DBG] pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 37 objects/s recovering 2024-09-13T14:51:38.098 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:37 smithi177 bash[68594]: cluster 2024-09-13T14:51:36.577574+0000 mgr.y (mgr.26252) 760 : cluster [DBG] pgmap v1242: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 84 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 14984/4104 objects degraded (365.107%); 0 B/s, 37 objects/s recovering 2024-09-13T14:51:38.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:38.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:38.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:38.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:38.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:38.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:38.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:38.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:38.473 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:38.499 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:38.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:38.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:38.626 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:38.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:38.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:38.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:38.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:38.860 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:38.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:38.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:38.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:39.002 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:39.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:39.105 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:39.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:39.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:39.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:39.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:39.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:39.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:39.382 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:39.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:39.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:39.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:39.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:39.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:39.644 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:39.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:39.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:39.755 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:39 smithi177 bash[68594]: cluster 2024-09-13T14:51:38.578402+0000 mgr.y (mgr.26252) 761 : cluster [DBG] pgmap v1243: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 13 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 12127/669 objects degraded (1812.706%); 0 B/s, 113 objects/s recovering 2024-09-13T14:51:39.755 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:39 smithi177 bash[68594]: cluster 2024-09-13T14:51:38.578402+0000 mgr.y (mgr.26252) 761 : cluster [DBG] pgmap v1243: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 13 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 12127/669 objects degraded (1812.706%); 0 B/s, 113 objects/s recovering 2024-09-13T14:51:39.755 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:39 smithi177 bash[68594]: cluster 2024-09-13T14:51:38.682969+0000 mon.a (mon.0) 1764 : cluster [DBG] osdmap e990: 8 total, 8 up, 7 in 2024-09-13T14:51:39.755 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:39 smithi177 bash[68594]: cluster 2024-09-13T14:51:38.682969+0000 mon.a (mon.0) 1764 : cluster [DBG] osdmap e990: 8 total, 8 up, 7 in 2024-09-13T14:51:39.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:39.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:39.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:39.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:39.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:39.996 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:40.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:40.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:40.128 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:40.152 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:40.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:40.186 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:39 smithi099 bash[84576]: cluster 2024-09-13T14:51:38.578402+0000 mgr.y (mgr.26252) 761 : cluster [DBG] pgmap v1243: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 13 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 12127/669 objects degraded (1812.706%); 0 B/s, 113 objects/s recovering 2024-09-13T14:51:40.186 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:39 smithi099 bash[84576]: cluster 2024-09-13T14:51:38.578402+0000 mgr.y (mgr.26252) 761 : cluster [DBG] pgmap v1243: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 13 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 12127/669 objects degraded (1812.706%); 0 B/s, 113 objects/s recovering 2024-09-13T14:51:40.186 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:39 smithi099 bash[84576]: cluster 2024-09-13T14:51:38.682969+0000 mon.a (mon.0) 1764 : cluster [DBG] osdmap e990: 8 total, 8 up, 7 in 2024-09-13T14:51:40.186 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:39 smithi099 bash[84576]: cluster 2024-09-13T14:51:38.682969+0000 mon.a (mon.0) 1764 : cluster [DBG] osdmap e990: 8 total, 8 up, 7 in 2024-09-13T14:51:40.187 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:39 smithi099 bash[87147]: cluster 2024-09-13T14:51:38.578402+0000 mgr.y (mgr.26252) 761 : cluster [DBG] pgmap v1243: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 13 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 12127/669 objects degraded (1812.706%); 0 B/s, 113 objects/s recovering 2024-09-13T14:51:40.187 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:39 smithi099 bash[87147]: cluster 2024-09-13T14:51:38.578402+0000 mgr.y (mgr.26252) 761 : cluster [DBG] pgmap v1243: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 13 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 12127/669 objects degraded (1812.706%); 0 B/s, 113 objects/s recovering 2024-09-13T14:51:40.187 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:39 smithi099 bash[87147]: cluster 2024-09-13T14:51:38.682969+0000 mon.a (mon.0) 1764 : cluster [DBG] osdmap e990: 8 total, 8 up, 7 in 2024-09-13T14:51:40.187 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:39 smithi099 bash[87147]: cluster 2024-09-13T14:51:38.682969+0000 mon.a (mon.0) 1764 : cluster [DBG] osdmap e990: 8 total, 8 up, 7 in 2024-09-13T14:51:40.255 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:40.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:40.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:40.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:51:40 smithi177 bash[61439]: ts=2024-09-13T14:51:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:40.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:40.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:40.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:40.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:40.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:40.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:40.641 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:40.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:40.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:40.796 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:40.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:40.907 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:40 smithi099 bash[87147]: cluster 2024-09-13T14:51:39.691784+0000 mon.a (mon.0) 1765 : cluster [DBG] osdmap e991: 8 total, 8 up, 7 in 2024-09-13T14:51:40.907 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:40 smithi099 bash[87147]: cluster 2024-09-13T14:51:39.691784+0000 mon.a (mon.0) 1765 : cluster [DBG] osdmap e991: 8 total, 8 up, 7 in 2024-09-13T14:51:40.907 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:40 smithi099 bash[84576]: cluster 2024-09-13T14:51:39.691784+0000 mon.a (mon.0) 1765 : cluster [DBG] osdmap e991: 8 total, 8 up, 7 in 2024-09-13T14:51:40.907 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:40 smithi099 bash[84576]: cluster 2024-09-13T14:51:39.691784+0000 mon.a (mon.0) 1765 : cluster [DBG] osdmap e991: 8 total, 8 up, 7 in 2024-09-13T14:51:40.922 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:40.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:41.026 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:41.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:40 smithi177 bash[68594]: cluster 2024-09-13T14:51:39.691784+0000 mon.a (mon.0) 1765 : cluster [DBG] osdmap e991: 8 total, 8 up, 7 in 2024-09-13T14:51:41.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:40 smithi177 bash[68594]: cluster 2024-09-13T14:51:39.691784+0000 mon.a (mon.0) 1765 : cluster [DBG] osdmap e991: 8 total, 8 up, 7 in 2024-09-13T14:51:41.052 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:41.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:41.156 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:41.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:41.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:41.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:41.310 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:41.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:41.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:41.441 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:41.544 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:41.570 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:41.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:41.699 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:41.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:41.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:41.814 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:41 smithi177 bash[68594]: cluster 2024-09-13T14:51:40.578902+0000 mgr.y (mgr.26252) 762 : cluster [DBG] pgmap v1246: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 12127/288 objects degraded (4210.764%); 0 B/s, 141 objects/s recovering 2024-09-13T14:51:41.814 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:41 smithi177 bash[68594]: cluster 2024-09-13T14:51:40.578902+0000 mgr.y (mgr.26252) 762 : cluster [DBG] pgmap v1246: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 12127/288 objects degraded (4210.764%); 0 B/s, 141 objects/s recovering 2024-09-13T14:51:41.814 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:41 smithi177 bash[68594]: cluster 2024-09-13T14:51:41.353300+0000 mon.a (mon.0) 1766 : cluster [WRN] Health check update: Degraded data redundancy: 12127/288 objects degraded (4210.764%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:41.815 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:41 smithi177 bash[68594]: cluster 2024-09-13T14:51:41.353300+0000 mon.a (mon.0) 1766 : cluster [WRN] Health check update: Degraded data redundancy: 12127/288 objects degraded (4210.764%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:41.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:41.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:41.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:41.946 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:41 smithi099 bash[87147]: cluster 2024-09-13T14:51:40.578902+0000 mgr.y (mgr.26252) 762 : cluster [DBG] pgmap v1246: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 12127/288 objects degraded (4210.764%); 0 B/s, 141 objects/s recovering 2024-09-13T14:51:41.946 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:41 smithi099 bash[87147]: cluster 2024-09-13T14:51:40.578902+0000 mgr.y (mgr.26252) 762 : cluster [DBG] pgmap v1246: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 12127/288 objects degraded (4210.764%); 0 B/s, 141 objects/s recovering 2024-09-13T14:51:41.946 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:41 smithi099 bash[87147]: cluster 2024-09-13T14:51:41.353300+0000 mon.a (mon.0) 1766 : cluster [WRN] Health check update: Degraded data redundancy: 12127/288 objects degraded (4210.764%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:41.946 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:41 smithi099 bash[87147]: cluster 2024-09-13T14:51:41.353300+0000 mon.a (mon.0) 1766 : cluster [WRN] Health check update: Degraded data redundancy: 12127/288 objects degraded (4210.764%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:41.946 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:41 smithi099 bash[84576]: cluster 2024-09-13T14:51:40.578902+0000 mgr.y (mgr.26252) 762 : cluster [DBG] pgmap v1246: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 12127/288 objects degraded (4210.764%); 0 B/s, 141 objects/s recovering 2024-09-13T14:51:41.946 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:41 smithi099 bash[84576]: cluster 2024-09-13T14:51:40.578902+0000 mgr.y (mgr.26252) 762 : cluster [DBG] pgmap v1246: 99 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 12127/288 objects degraded (4210.764%); 0 B/s, 141 objects/s recovering 2024-09-13T14:51:41.947 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:41 smithi099 bash[84576]: cluster 2024-09-13T14:51:41.353300+0000 mon.a (mon.0) 1766 : cluster [WRN] Health check update: Degraded data redundancy: 12127/288 objects degraded (4210.764%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:41.947 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:41 smithi099 bash[84576]: cluster 2024-09-13T14:51:41.353300+0000 mon.a (mon.0) 1766 : cluster [WRN] Health check update: Degraded data redundancy: 12127/288 objects degraded (4210.764%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:41.962 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:41.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:42.065 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:42.091 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:42.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:42.194 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:42.221 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:42.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:42.325 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:42.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:42.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:42.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:42.475 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:42.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:42.577 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:42.600 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:42.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:42.703 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:42.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:42.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:42.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:42.847 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:42.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:42.950 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:42.973 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:42 smithi099 bash[84576]: audit 2024-09-13T14:51:42.107182+0000 mon.a (mon.0) 1767 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:42.973 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:42 smithi099 bash[84576]: audit 2024-09-13T14:51:42.107182+0000 mon.a (mon.0) 1767 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:42.973 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:42 smithi099 bash[87147]: audit 2024-09-13T14:51:42.107182+0000 mon.a (mon.0) 1767 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:42.973 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:42 smithi099 bash[87147]: audit 2024-09-13T14:51:42.107182+0000 mon.a (mon.0) 1767 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:42.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:42.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:43.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:43.099 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:43.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:43.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:42 smithi177 bash[68594]: audit 2024-09-13T14:51:42.107182+0000 mon.a (mon.0) 1767 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:43.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:42 smithi177 bash[68594]: audit 2024-09-13T14:51:42.107182+0000 mon.a (mon.0) 1767 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:51:43.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:43.224 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:43.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:43.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:43.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:43.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:43.449 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:43.472 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:43.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:43.575 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:43.602 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:43.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:43.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:43.725 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:43.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:43.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:43.857 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:43.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:43 smithi177 bash[68594]: cluster 2024-09-13T14:51:42.579568+0000 mgr.y (mgr.26252) 763 : cluster [DBG] pgmap v1247: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 172 objects/s recovering 2024-09-13T14:51:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:43 smithi177 bash[68594]: cluster 2024-09-13T14:51:42.579568+0000 mgr.y (mgr.26252) 763 : cluster [DBG] pgmap v1247: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 172 objects/s recovering 2024-09-13T14:51:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:43 smithi177 bash[68594]: audit 2024-09-13T14:51:42.800554+0000 mon.a (mon.0) 1768 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:43 smithi177 bash[68594]: audit 2024-09-13T14:51:42.800554+0000 mon.a (mon.0) 1768 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:43 smithi177 bash[68594]: audit 2024-09-13T14:51:42.805641+0000 mon.a (mon.0) 1769 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:43 smithi177 bash[68594]: audit 2024-09-13T14:51:42.805641+0000 mon.a (mon.0) 1769 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:43.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:43.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:44.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[84576]: cluster 2024-09-13T14:51:42.579568+0000 mgr.y (mgr.26252) 763 : cluster [DBG] pgmap v1247: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 172 objects/s recovering 2024-09-13T14:51:44.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[84576]: cluster 2024-09-13T14:51:42.579568+0000 mgr.y (mgr.26252) 763 : cluster [DBG] pgmap v1247: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 172 objects/s recovering 2024-09-13T14:51:44.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[84576]: audit 2024-09-13T14:51:42.800554+0000 mon.a (mon.0) 1768 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:44.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[84576]: audit 2024-09-13T14:51:42.800554+0000 mon.a (mon.0) 1768 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:44.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[84576]: audit 2024-09-13T14:51:42.805641+0000 mon.a (mon.0) 1769 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:44.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[84576]: audit 2024-09-13T14:51:42.805641+0000 mon.a (mon.0) 1769 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:44.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[87147]: cluster 2024-09-13T14:51:42.579568+0000 mgr.y (mgr.26252) 763 : cluster [DBG] pgmap v1247: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 172 objects/s recovering 2024-09-13T14:51:44.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[87147]: cluster 2024-09-13T14:51:42.579568+0000 mgr.y (mgr.26252) 763 : cluster [DBG] pgmap v1247: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 94 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 172 objects/s recovering 2024-09-13T14:51:44.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[87147]: audit 2024-09-13T14:51:42.800554+0000 mon.a (mon.0) 1768 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:44.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[87147]: audit 2024-09-13T14:51:42.800554+0000 mon.a (mon.0) 1768 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:44.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[87147]: audit 2024-09-13T14:51:42.805641+0000 mon.a (mon.0) 1769 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:44.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:43 smithi099 bash[87147]: audit 2024-09-13T14:51:42.805641+0000 mon.a (mon.0) 1769 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:44.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:44.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:44.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:44.203 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:44.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:44.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:44.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:44.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:44.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:44.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:44.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:44.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:44.588 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:44.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:44.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:44.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:44.745 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:44.849 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:44.877 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:44.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:44.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:45.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:45.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:45.134 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:45.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:45.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:45.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:45.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:45.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:45.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:45.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:45.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:45.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:45.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:45.646 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:45.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:45.749 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:45.773 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:45.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:45.875 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:45.898 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:45.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:46.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:46.009 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:45 smithi099 bash[84576]: cluster 2024-09-13T14:51:44.580277+0000 mgr.y (mgr.26252) 764 : cluster [DBG] pgmap v1248: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 324 objects/s recovering 2024-09-13T14:51:46.009 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:45 smithi099 bash[84576]: cluster 2024-09-13T14:51:44.580277+0000 mgr.y (mgr.26252) 764 : cluster [DBG] pgmap v1248: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 324 objects/s recovering 2024-09-13T14:51:46.009 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:45 smithi099 bash[87147]: cluster 2024-09-13T14:51:44.580277+0000 mgr.y (mgr.26252) 764 : cluster [DBG] pgmap v1248: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 324 objects/s recovering 2024-09-13T14:51:46.010 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:45 smithi099 bash[87147]: cluster 2024-09-13T14:51:44.580277+0000 mgr.y (mgr.26252) 764 : cluster [DBG] pgmap v1248: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 324 objects/s recovering 2024-09-13T14:51:46.020 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:46.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:46.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:45 smithi177 bash[68594]: cluster 2024-09-13T14:51:44.580277+0000 mgr.y (mgr.26252) 764 : cluster [DBG] pgmap v1248: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 324 objects/s recovering 2024-09-13T14:51:46.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:45 smithi177 bash[68594]: cluster 2024-09-13T14:51:44.580277+0000 mgr.y (mgr.26252) 764 : cluster [DBG] pgmap v1248: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 324 objects/s recovering 2024-09-13T14:51:46.123 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:46.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:46.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:46.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:46.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:46.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:46.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:46.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:46.508 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:46.535 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:46.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:46.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:46.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:46.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:46.769 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:46.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:46.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:46.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:46.917 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:47.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:47.042 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:47.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:47.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:46 smithi099 bash[84576]: cluster 2024-09-13T14:51:46.354131+0000 mon.a (mon.0) 1770 : cluster [WRN] Health check update: Degraded data redundancy: 11651/288 objects degraded (4045.486%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:47.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:46 smithi099 bash[84576]: cluster 2024-09-13T14:51:46.354131+0000 mon.a (mon.0) 1770 : cluster [WRN] Health check update: Degraded data redundancy: 11651/288 objects degraded (4045.486%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:47.071 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:51:47] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-13T14:51:47.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:46 smithi099 bash[87147]: cluster 2024-09-13T14:51:46.354131+0000 mon.a (mon.0) 1770 : cluster [WRN] Health check update: Degraded data redundancy: 11651/288 objects degraded (4045.486%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:47.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:46 smithi099 bash[87147]: cluster 2024-09-13T14:51:46.354131+0000 mon.a (mon.0) 1770 : cluster [WRN] Health check update: Degraded data redundancy: 11651/288 objects degraded (4045.486%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:47.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:46 smithi177 bash[68594]: cluster 2024-09-13T14:51:46.354131+0000 mon.a (mon.0) 1770 : cluster [WRN] Health check update: Degraded data redundancy: 11651/288 objects degraded (4045.486%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:47.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:46 smithi177 bash[68594]: cluster 2024-09-13T14:51:46.354131+0000 mon.a (mon.0) 1770 : cluster [WRN] Health check update: Degraded data redundancy: 11651/288 objects degraded (4045.486%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:47.146 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:47.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:47.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:47.275 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:47.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:47.399 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:47.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:47.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:47.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:47.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:47.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:47.646 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:47.673 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:47.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:47.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:47.796 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:47.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:47.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:47.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[84576]: cluster 2024-09-13T14:51:46.580698+0000 mgr.y (mgr.26252) 765 : cluster [DBG] pgmap v1249: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 210 objects/s recovering 2024-09-13T14:51:47.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[84576]: cluster 2024-09-13T14:51:46.580698+0000 mgr.y (mgr.26252) 765 : cluster [DBG] pgmap v1249: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 210 objects/s recovering 2024-09-13T14:51:47.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[84576]: cluster 2024-09-13T14:51:46.746393+0000 osd.2 (osd.2) 1 : cluster [DBG] 41.15 scrub starts 2024-09-13T14:51:47.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[84576]: cluster 2024-09-13T14:51:46.746393+0000 osd.2 (osd.2) 1 : cluster [DBG] 41.15 scrub starts 2024-09-13T14:51:47.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[84576]: cluster 2024-09-13T14:51:46.756664+0000 osd.2 (osd.2) 2 : cluster [DBG] 41.15 scrub ok 2024-09-13T14:51:47.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[84576]: cluster 2024-09-13T14:51:46.756664+0000 osd.2 (osd.2) 2 : cluster [DBG] 41.15 scrub ok 2024-09-13T14:51:47.912 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[87147]: cluster 2024-09-13T14:51:46.580698+0000 mgr.y (mgr.26252) 765 : cluster [DBG] pgmap v1249: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 210 objects/s recovering 2024-09-13T14:51:47.912 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[87147]: cluster 2024-09-13T14:51:46.580698+0000 mgr.y (mgr.26252) 765 : cluster [DBG] pgmap v1249: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 210 objects/s recovering 2024-09-13T14:51:47.912 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[87147]: cluster 2024-09-13T14:51:46.746393+0000 osd.2 (osd.2) 1 : cluster [DBG] 41.15 scrub starts 2024-09-13T14:51:47.912 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[87147]: cluster 2024-09-13T14:51:46.746393+0000 osd.2 (osd.2) 1 : cluster [DBG] 41.15 scrub starts 2024-09-13T14:51:47.912 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[87147]: cluster 2024-09-13T14:51:46.756664+0000 osd.2 (osd.2) 2 : cluster [DBG] 41.15 scrub ok 2024-09-13T14:51:47.912 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:47 smithi099 bash[87147]: cluster 2024-09-13T14:51:46.756664+0000 osd.2 (osd.2) 2 : cluster [DBG] 41.15 scrub ok 2024-09-13T14:51:47.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:47.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:48.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:48.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:47 smithi177 bash[68594]: cluster 2024-09-13T14:51:46.580698+0000 mgr.y (mgr.26252) 765 : cluster [DBG] pgmap v1249: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 210 objects/s recovering 2024-09-13T14:51:48.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:47 smithi177 bash[68594]: cluster 2024-09-13T14:51:46.580698+0000 mgr.y (mgr.26252) 765 : cluster [DBG] pgmap v1249: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 95 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 11651/288 objects degraded (4045.486%); 0 B/s, 210 objects/s recovering 2024-09-13T14:51:48.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:47 smithi177 bash[68594]: cluster 2024-09-13T14:51:46.746393+0000 osd.2 (osd.2) 1 : cluster [DBG] 41.15 scrub starts 2024-09-13T14:51:48.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:47 smithi177 bash[68594]: cluster 2024-09-13T14:51:46.746393+0000 osd.2 (osd.2) 1 : cluster [DBG] 41.15 scrub starts 2024-09-13T14:51:48.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:47 smithi177 bash[68594]: cluster 2024-09-13T14:51:46.756664+0000 osd.2 (osd.2) 2 : cluster [DBG] 41.15 scrub ok 2024-09-13T14:51:48.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:47 smithi177 bash[68594]: cluster 2024-09-13T14:51:46.756664+0000 osd.2 (osd.2) 2 : cluster [DBG] 41.15 scrub ok 2024-09-13T14:51:48.051 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:48.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:48.154 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:48.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:48.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:48.304 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:48.407 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:48.429 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:48.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:48.531 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:48.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:48.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:48.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:48.686 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:48.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:48.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:48.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:48.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:48.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:48.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:49.038 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:49.061 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:49.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:49.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:49.184 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[84576]: audit 2024-09-13T14:51:48.180341+0000 mon.a (mon.0) 1771 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[84576]: audit 2024-09-13T14:51:48.180341+0000 mon.a (mon.0) 1771 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[84576]: audit 2024-09-13T14:51:48.189006+0000 mon.a (mon.0) 1772 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[84576]: audit 2024-09-13T14:51:48.189006+0000 mon.a (mon.0) 1772 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[84576]: audit 2024-09-13T14:51:48.278848+0000 mon.a (mon.0) 1773 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:49.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[84576]: audit 2024-09-13T14:51:48.278848+0000 mon.a (mon.0) 1773 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:49.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[84576]: cluster 2024-09-13T14:51:48.581710+0000 mgr.y (mgr.26252) 766 : cluster [DBG] pgmap v1250: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 41 objects/s recovering 2024-09-13T14:51:49.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[84576]: cluster 2024-09-13T14:51:48.581710+0000 mgr.y (mgr.26252) 766 : cluster [DBG] pgmap v1250: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 41 objects/s recovering 2024-09-13T14:51:49.185 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[87147]: audit 2024-09-13T14:51:48.180341+0000 mon.a (mon.0) 1771 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.185 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[87147]: audit 2024-09-13T14:51:48.180341+0000 mon.a (mon.0) 1771 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.185 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[87147]: audit 2024-09-13T14:51:48.189006+0000 mon.a (mon.0) 1772 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[87147]: audit 2024-09-13T14:51:48.189006+0000 mon.a (mon.0) 1772 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[87147]: audit 2024-09-13T14:51:48.278848+0000 mon.a (mon.0) 1773 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:49.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[87147]: audit 2024-09-13T14:51:48.278848+0000 mon.a (mon.0) 1773 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:49.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[87147]: cluster 2024-09-13T14:51:48.581710+0000 mgr.y (mgr.26252) 766 : cluster [DBG] pgmap v1250: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 41 objects/s recovering 2024-09-13T14:51:49.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:49 smithi099 bash[87147]: cluster 2024-09-13T14:51:48.581710+0000 mgr.y (mgr.26252) 766 : cluster [DBG] pgmap v1250: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 41 objects/s recovering 2024-09-13T14:51:49.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:49.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:49.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:49.321 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:49.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:49.424 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.437 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:49 smithi177 bash[68594]: audit 2024-09-13T14:51:48.180341+0000 mon.a (mon.0) 1771 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.437 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:49 smithi177 bash[68594]: audit 2024-09-13T14:51:48.180341+0000 mon.a (mon.0) 1771 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.437 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:49 smithi177 bash[68594]: audit 2024-09-13T14:51:48.189006+0000 mon.a (mon.0) 1772 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.437 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:49 smithi177 bash[68594]: audit 2024-09-13T14:51:48.189006+0000 mon.a (mon.0) 1772 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:51:49.437 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:49 smithi177 bash[68594]: audit 2024-09-13T14:51:48.278848+0000 mon.a (mon.0) 1773 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:49.437 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:49 smithi177 bash[68594]: audit 2024-09-13T14:51:48.278848+0000 mon.a (mon.0) 1773 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:49.437 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:49 smithi177 bash[68594]: cluster 2024-09-13T14:51:48.581710+0000 mgr.y (mgr.26252) 766 : cluster [DBG] pgmap v1250: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 41 objects/s recovering 2024-09-13T14:51:49.438 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:49 smithi177 bash[68594]: cluster 2024-09-13T14:51:48.581710+0000 mgr.y (mgr.26252) 766 : cluster [DBG] pgmap v1250: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 41 objects/s recovering 2024-09-13T14:51:49.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:49.557 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.584 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:49.687 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:49.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:49.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:49.819 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:49.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:49.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:50.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:50.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:50.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:50.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:50.235 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:50.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:50.339 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:50.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:51:50 smithi177 bash[61439]: ts=2024-09-13T14:51:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:51:50.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:50.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:50.470 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:50.496 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:50.599 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:50.625 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:50.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:50.728 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:50.749 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:50.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:50.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:50.878 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:50.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:50.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:51.003 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:51.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:51.106 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:51.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:51.230 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:51.251 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:51.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:51.354 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:51.375 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:51.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:51.478 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:51.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:51.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:51.511 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:51:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T14:51:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2024-09-13T14:51:51.512 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:51:51.601 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:51.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:51.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:51.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:51.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:51.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:51.859 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:51.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:51 smithi177 bash[68594]: cluster 2024-09-13T14:51:50.583083+0000 mgr.y (mgr.26252) 767 : cluster [DBG] pgmap v1251: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 38 objects/s recovering 2024-09-13T14:51:51.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:51 smithi177 bash[68594]: cluster 2024-09-13T14:51:50.583083+0000 mgr.y (mgr.26252) 767 : cluster [DBG] pgmap v1251: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 38 objects/s recovering 2024-09-13T14:51:51.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:51 smithi177 bash[68594]: cluster 2024-09-13T14:51:51.355229+0000 mon.a (mon.0) 1774 : cluster [WRN] Health check update: Degraded data redundancy: 9334/288 objects degraded (3240.972%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:51.870 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:51 smithi177 bash[68594]: cluster 2024-09-13T14:51:51.355229+0000 mon.a (mon.0) 1774 : cluster [WRN] Health check update: Degraded data redundancy: 9334/288 objects degraded (3240.972%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:51.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:51.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:51.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:52.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:52.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:51 smithi099 bash[84576]: cluster 2024-09-13T14:51:50.583083+0000 mgr.y (mgr.26252) 767 : cluster [DBG] pgmap v1251: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 38 objects/s recovering 2024-09-13T14:51:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:51 smithi099 bash[84576]: cluster 2024-09-13T14:51:50.583083+0000 mgr.y (mgr.26252) 767 : cluster [DBG] pgmap v1251: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 38 objects/s recovering 2024-09-13T14:51:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:51 smithi099 bash[84576]: cluster 2024-09-13T14:51:51.355229+0000 mon.a (mon.0) 1774 : cluster [WRN] Health check update: Degraded data redundancy: 9334/288 objects degraded (3240.972%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:51 smithi099 bash[84576]: cluster 2024-09-13T14:51:51.355229+0000 mon.a (mon.0) 1774 : cluster [WRN] Health check update: Degraded data redundancy: 9334/288 objects degraded (3240.972%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:52.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:51 smithi099 bash[87147]: cluster 2024-09-13T14:51:50.583083+0000 mgr.y (mgr.26252) 767 : cluster [DBG] pgmap v1251: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 38 objects/s recovering 2024-09-13T14:51:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:51 smithi099 bash[87147]: cluster 2024-09-13T14:51:50.583083+0000 mgr.y (mgr.26252) 767 : cluster [DBG] pgmap v1251: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 9334/288 objects degraded (3240.972%); 0 B/s, 38 objects/s recovering 2024-09-13T14:51:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:51 smithi099 bash[87147]: cluster 2024-09-13T14:51:51.355229+0000 mon.a (mon.0) 1774 : cluster [WRN] Health check update: Degraded data redundancy: 9334/288 objects degraded (3240.972%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:52.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:51 smithi099 bash[87147]: cluster 2024-09-13T14:51:51.355229+0000 mon.a (mon.0) 1774 : cluster [WRN] Health check update: Degraded data redundancy: 9334/288 objects degraded (3240.972%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:52.114 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:52.140 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:52.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:52.243 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:52.266 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:52.369 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:52.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:52.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:52.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:52.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:52.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:52.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:52.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:52.760 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:52.787 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:52.889 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:52.909 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:53.012 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:53.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:53.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:53.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:53.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:53.269 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:53.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:53.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:53.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:53.423 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:53.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:53.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:53.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:53.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:53.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:53.682 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:53.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:53.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:53.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:53.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:53.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:53 smithi177 bash[68594]: cluster 2024-09-13T14:51:52.584169+0000 mgr.y (mgr.26252) 768 : cluster [DBG] pgmap v1252: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:53.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:53 smithi177 bash[68594]: cluster 2024-09-13T14:51:52.584169+0000 mgr.y (mgr.26252) 768 : cluster [DBG] pgmap v1252: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:53.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:53.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:54.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:53 smithi099 bash[84576]: cluster 2024-09-13T14:51:52.584169+0000 mgr.y (mgr.26252) 768 : cluster [DBG] pgmap v1252: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:54.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:53 smithi099 bash[84576]: cluster 2024-09-13T14:51:52.584169+0000 mgr.y (mgr.26252) 768 : cluster [DBG] pgmap v1252: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:54.016 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:53 smithi099 bash[87147]: cluster 2024-09-13T14:51:52.584169+0000 mgr.y (mgr.26252) 768 : cluster [DBG] pgmap v1252: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:54.016 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:53 smithi099 bash[87147]: cluster 2024-09-13T14:51:52.584169+0000 mgr.y (mgr.26252) 768 : cluster [DBG] pgmap v1252: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 66 objects/s recovering 2024-09-13T14:51:54.043 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:54.067 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:54.170 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:54.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:54.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:54.321 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:54.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:54.424 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:54.448 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:54.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:54.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:54.571 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:54.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:54.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:54.699 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:54.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:54.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:54.828 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:54.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:54.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:54.956 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:55.059 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:55.084 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:55.188 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:55.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:55.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:55.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:55.344 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:55.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:55.447 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:55.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:55.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:55.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:55.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:55.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:55.706 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:55.716 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:55 smithi177 bash[68594]: cluster 2024-09-13T14:51:54.584911+0000 mgr.y (mgr.26252) 769 : cluster [DBG] pgmap v1253: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 95 objects/s recovering 2024-09-13T14:51:55.717 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:55 smithi177 bash[68594]: cluster 2024-09-13T14:51:54.584911+0000 mgr.y (mgr.26252) 769 : cluster [DBG] pgmap v1253: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 95 objects/s recovering 2024-09-13T14:51:55.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:55.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:55.833 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:55.860 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:55.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:55.940 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:55 smithi099 bash[84576]: cluster 2024-09-13T14:51:54.584911+0000 mgr.y (mgr.26252) 769 : cluster [DBG] pgmap v1253: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 95 objects/s recovering 2024-09-13T14:51:55.940 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:55 smithi099 bash[84576]: cluster 2024-09-13T14:51:54.584911+0000 mgr.y (mgr.26252) 769 : cluster [DBG] pgmap v1253: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 95 objects/s recovering 2024-09-13T14:51:55.940 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:55 smithi099 bash[87147]: cluster 2024-09-13T14:51:54.584911+0000 mgr.y (mgr.26252) 769 : cluster [DBG] pgmap v1253: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 95 objects/s recovering 2024-09-13T14:51:55.941 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:55 smithi099 bash[87147]: cluster 2024-09-13T14:51:54.584911+0000 mgr.y (mgr.26252) 769 : cluster [DBG] pgmap v1253: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%); 0 B/s, 95 objects/s recovering 2024-09-13T14:51:55.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:55.988 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:55.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:56.090 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:56.116 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:56.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:56.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:56.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:56.352 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:56.378 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:56.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:56.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:56.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:56.609 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:56.633 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:56.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:56.736 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:56.763 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:56.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:56.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:56 smithi099 bash[84576]: cluster 2024-09-13T14:51:56.356077+0000 mon.a (mon.0) 1775 : cluster [WRN] Health check update: Degraded data redundancy: 8954/288 objects degraded (3109.028%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:56.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:56 smithi099 bash[84576]: cluster 2024-09-13T14:51:56.356077+0000 mon.a (mon.0) 1775 : cluster [WRN] Health check update: Degraded data redundancy: 8954/288 objects degraded (3109.028%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:56.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:56 smithi099 bash[87147]: cluster 2024-09-13T14:51:56.356077+0000 mon.a (mon.0) 1775 : cluster [WRN] Health check update: Degraded data redundancy: 8954/288 objects degraded (3109.028%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:56.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:56 smithi099 bash[87147]: cluster 2024-09-13T14:51:56.356077+0000 mon.a (mon.0) 1775 : cluster [WRN] Health check update: Degraded data redundancy: 8954/288 objects degraded (3109.028%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:56.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:56.878 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:56 smithi177 bash[68594]: cluster 2024-09-13T14:51:56.356077+0000 mon.a (mon.0) 1775 : cluster [WRN] Health check update: Degraded data redundancy: 8954/288 objects degraded (3109.028%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:56.878 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:56 smithi177 bash[68594]: cluster 2024-09-13T14:51:56.356077+0000 mon.a (mon.0) 1775 : cluster [WRN] Health check update: Degraded data redundancy: 8954/288 objects degraded (3109.028%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:51:56.891 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:56.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:56.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:57.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:57.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:57.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:57.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:57.272 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:57.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:51:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:51:57] "GET /metrics HTTP/1.1" 200 36757 "" "Prometheus/2.51.0" 2024-09-13T14:51:57.377 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:57.502 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:57.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:57.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.652 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:57.755 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:57.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:57.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:57.885 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:57.912 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:57.913 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:57 smithi177 bash[68594]: cluster 2024-09-13T14:51:56.585330+0000 mgr.y (mgr.26252) 770 : cluster [DBG] pgmap v1254: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%) 2024-09-13T14:51:57.913 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:57 smithi177 bash[68594]: cluster 2024-09-13T14:51:56.585330+0000 mgr.y (mgr.26252) 770 : cluster [DBG] pgmap v1254: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%) 2024-09-13T14:51:57.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:58.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:58.026 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:57 smithi099 bash[84576]: cluster 2024-09-13T14:51:56.585330+0000 mgr.y (mgr.26252) 770 : cluster [DBG] pgmap v1254: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%) 2024-09-13T14:51:58.026 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:57 smithi099 bash[84576]: cluster 2024-09-13T14:51:56.585330+0000 mgr.y (mgr.26252) 770 : cluster [DBG] pgmap v1254: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%) 2024-09-13T14:51:58.026 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:57 smithi099 bash[87147]: cluster 2024-09-13T14:51:56.585330+0000 mgr.y (mgr.26252) 770 : cluster [DBG] pgmap v1254: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%) 2024-09-13T14:51:58.026 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:57 smithi099 bash[87147]: cluster 2024-09-13T14:51:56.585330+0000 mgr.y (mgr.26252) 770 : cluster [DBG] pgmap v1254: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 96 active+clean; 4.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8954/288 objects degraded (3109.028%) 2024-09-13T14:51:58.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:58.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:58.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:58.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:58.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:58.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:58.289 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:58.392 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:58.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:58.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:58.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:58.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:58.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:58.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:58.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:58.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:58.786 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:58.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:58.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:58.860 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:51:58.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:58.925 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:58 smithi177 bash[68594]: cluster 2024-09-13T14:51:57.746254+0000 osd.2 (osd.2) 3 : cluster [DBG] 41.5 scrub starts 2024-09-13T14:51:58.925 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:58 smithi177 bash[68594]: cluster 2024-09-13T14:51:57.746254+0000 osd.2 (osd.2) 3 : cluster [DBG] 41.5 scrub starts 2024-09-13T14:51:58.925 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:58 smithi177 bash[68594]: cluster 2024-09-13T14:51:57.751308+0000 osd.2 (osd.2) 4 : cluster [DBG] 41.5 scrub ok 2024-09-13T14:51:58.925 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:58 smithi177 bash[68594]: cluster 2024-09-13T14:51:57.751308+0000 osd.2 (osd.2) 4 : cluster [DBG] 41.5 scrub ok 2024-09-13T14:51:58.941 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:59.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:59.055 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:58 smithi099 bash[84576]: cluster 2024-09-13T14:51:57.746254+0000 osd.2 (osd.2) 3 : cluster [DBG] 41.5 scrub starts 2024-09-13T14:51:59.055 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:58 smithi099 bash[84576]: cluster 2024-09-13T14:51:57.746254+0000 osd.2 (osd.2) 3 : cluster [DBG] 41.5 scrub starts 2024-09-13T14:51:59.055 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:58 smithi099 bash[84576]: cluster 2024-09-13T14:51:57.751308+0000 osd.2 (osd.2) 4 : cluster [DBG] 41.5 scrub ok 2024-09-13T14:51:59.056 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:58 smithi099 bash[84576]: cluster 2024-09-13T14:51:57.751308+0000 osd.2 (osd.2) 4 : cluster [DBG] 41.5 scrub ok 2024-09-13T14:51:59.056 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:58 smithi099 bash[87147]: cluster 2024-09-13T14:51:57.746254+0000 osd.2 (osd.2) 3 : cluster [DBG] 41.5 scrub starts 2024-09-13T14:51:59.056 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:58 smithi099 bash[87147]: cluster 2024-09-13T14:51:57.746254+0000 osd.2 (osd.2) 3 : cluster [DBG] 41.5 scrub starts 2024-09-13T14:51:59.056 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:58 smithi099 bash[87147]: cluster 2024-09-13T14:51:57.751308+0000 osd.2 (osd.2) 4 : cluster [DBG] 41.5 scrub ok 2024-09-13T14:51:59.056 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:58 smithi099 bash[87147]: cluster 2024-09-13T14:51:57.751308+0000 osd.2 (osd.2) 4 : cluster [DBG] 41.5 scrub ok 2024-09-13T14:51:59.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:59.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:59.172 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:59.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:59.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:59.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:59.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:59.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:59.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:59.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:59.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:59.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:59.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:59.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:59.710 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:59.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:59.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:59.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:59.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:59 smithi177 bash[68594]: cluster 2024-09-13T14:51:58.586480+0000 mgr.y (mgr.26252) 771 : cluster [DBG] pgmap v1255: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:51:59.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:51:59 smithi177 bash[68594]: cluster 2024-09-13T14:51:58.586480+0000 mgr.y (mgr.26252) 771 : cluster [DBG] pgmap v1255: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:51:59.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:59.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:59.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:00.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:59 smithi099 bash[84576]: cluster 2024-09-13T14:51:58.586480+0000 mgr.y (mgr.26252) 771 : cluster [DBG] pgmap v1255: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:52:00.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:51:59 smithi099 bash[84576]: cluster 2024-09-13T14:51:58.586480+0000 mgr.y (mgr.26252) 771 : cluster [DBG] pgmap v1255: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:52:00.039 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:59 smithi099 bash[87147]: cluster 2024-09-13T14:51:58.586480+0000 mgr.y (mgr.26252) 771 : cluster [DBG] pgmap v1255: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:52:00.039 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:51:59 smithi099 bash[87147]: cluster 2024-09-13T14:51:58.586480+0000 mgr.y (mgr.26252) 771 : cluster [DBG] pgmap v1255: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:52:00.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:00.096 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:00.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:00.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:00.220 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:00.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:00.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:00.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:00.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:00.447 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:00.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:00.608 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:52:00 smithi177 bash[61439]: ts=2024-09-13T14:52:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:00.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:00.710 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:00.753 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:00.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:00.856 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:00.888 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:00.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:00.991 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:01.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:01.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:01.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:01.149 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:01.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:01.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:01.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:01.375 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:01.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:01.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:01.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:01.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:01.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: audit 2024-09-13T14:52:00.288072+0000 mon.a (mon.0) 1776 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: audit 2024-09-13T14:52:00.288072+0000 mon.a (mon.0) 1776 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: audit 2024-09-13T14:52:00.294287+0000 mon.a (mon.0) 1777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: audit 2024-09-13T14:52:00.294287+0000 mon.a (mon.0) 1777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: cluster 2024-09-13T14:52:00.587064+0000 mgr.y (mgr.26252) 772 : cluster [DBG] pgmap v1256: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:52:01.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: cluster 2024-09-13T14:52:00.587064+0000 mgr.y (mgr.26252) 772 : cluster [DBG] pgmap v1256: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:52:01.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: audit 2024-09-13T14:52:01.009030+0000 mon.a (mon.0) 1778 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:52:01.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: audit 2024-09-13T14:52:01.009030+0000 mon.a (mon.0) 1778 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:52:01.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: audit 2024-09-13T14:52:01.009816+0000 mon.a (mon.0) 1779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:52:01.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: audit 2024-09-13T14:52:01.009816+0000 mon.a (mon.0) 1779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:52:01.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: audit 2024-09-13T14:52:01.016259+0000 mon.a (mon.0) 1780 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:01 smithi177 bash[68594]: audit 2024-09-13T14:52:01.016259+0000 mon.a (mon.0) 1780 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:01.537 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: audit 2024-09-13T14:52:00.288072+0000 mon.a (mon.0) 1776 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.537 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: audit 2024-09-13T14:52:00.288072+0000 mon.a (mon.0) 1776 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.538 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: audit 2024-09-13T14:52:00.294287+0000 mon.a (mon.0) 1777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.538 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: audit 2024-09-13T14:52:00.294287+0000 mon.a (mon.0) 1777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.538 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: cluster 2024-09-13T14:52:00.587064+0000 mgr.y (mgr.26252) 772 : cluster [DBG] pgmap v1256: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:52:01.538 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: cluster 2024-09-13T14:52:00.587064+0000 mgr.y (mgr.26252) 772 : cluster [DBG] pgmap v1256: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:52:01.538 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: audit 2024-09-13T14:52:01.009030+0000 mon.a (mon.0) 1778 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:52:01.538 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: audit 2024-09-13T14:52:01.009030+0000 mon.a (mon.0) 1778 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: audit 2024-09-13T14:52:01.009816+0000 mon.a (mon.0) 1779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: audit 2024-09-13T14:52:01.009816+0000 mon.a (mon.0) 1779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: audit 2024-09-13T14:52:01.016259+0000 mon.a (mon.0) 1780 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[84576]: audit 2024-09-13T14:52:01.016259+0000 mon.a (mon.0) 1780 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: audit 2024-09-13T14:52:00.288072+0000 mon.a (mon.0) 1776 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: audit 2024-09-13T14:52:00.288072+0000 mon.a (mon.0) 1776 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: audit 2024-09-13T14:52:00.294287+0000 mon.a (mon.0) 1777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: audit 2024-09-13T14:52:00.294287+0000 mon.a (mon.0) 1777 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: cluster 2024-09-13T14:52:00.587064+0000 mgr.y (mgr.26252) 772 : cluster [DBG] pgmap v1256: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: cluster 2024-09-13T14:52:00.587064+0000 mgr.y (mgr.26252) 772 : cluster [DBG] pgmap v1256: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8551/210 objects degraded (4071.905%) 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: audit 2024-09-13T14:52:01.009030+0000 mon.a (mon.0) 1778 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:52:01.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: audit 2024-09-13T14:52:01.009030+0000 mon.a (mon.0) 1778 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:52:01.540 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: audit 2024-09-13T14:52:01.009816+0000 mon.a (mon.0) 1779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:52:01.540 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: audit 2024-09-13T14:52:01.009816+0000 mon.a (mon.0) 1779 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:52:01.540 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: audit 2024-09-13T14:52:01.016259+0000 mon.a (mon.0) 1780 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.540 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:01 smithi099 bash[87147]: audit 2024-09-13T14:52:01.016259+0000 mon.a (mon.0) 1780 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:01.637 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:01.668 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:01.755 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:52:01.756 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:01.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:01.773 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:52:01.773 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:01.791 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:52:01.791 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:01.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:01.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:01.813 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:52:01.813 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:01.832 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:52:01.833 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:01.850 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:52:01.851 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:01.871 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:52:01.871 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:01.890 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:52:01.890 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:01.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:01.929 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:01.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:02.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:02.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:02.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:02.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:02.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:02.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:02.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:02.303 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T14:52:02.304 INFO:teuthology.orchestra.run.smithi099.stderr:set pool 41 pg_num to 36 2024-09-13T14:52:02.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:02.421 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:02.431 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[87147]: cluster 2024-09-13T14:52:01.356786+0000 mon.a (mon.0) 1781 : cluster [WRN] Health check update: Degraded data redundancy: 8551/210 objects degraded (4071.905%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:02.432 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[87147]: cluster 2024-09-13T14:52:01.356786+0000 mon.a (mon.0) 1781 : cluster [WRN] Health check update: Degraded data redundancy: 8551/210 objects degraded (4071.905%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:02.432 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[87147]: audit 2024-09-13T14:52:01.586542+0000 mon.a (mon.0) 1782 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:02.432 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[87147]: audit 2024-09-13T14:52:01.586542+0000 mon.a (mon.0) 1782 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:02.432 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[87147]: audit 2024-09-13T14:52:01.740641+0000 mon.a (mon.0) 1783 : audit [DBG] from='client.? 172.21.15.99:0/3153022160' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:02.432 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[87147]: audit 2024-09-13T14:52:01.740641+0000 mon.a (mon.0) 1783 : audit [DBG] from='client.? 172.21.15.99:0/3153022160' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:02.433 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[87147]: audit 2024-09-13T14:52:01.905274+0000 mon.a (mon.0) 1784 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T14:52:02.433 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[87147]: audit 2024-09-13T14:52:01.905274+0000 mon.a (mon.0) 1784 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T14:52:02.434 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[84576]: cluster 2024-09-13T14:52:01.356786+0000 mon.a (mon.0) 1781 : cluster [WRN] Health check update: Degraded data redundancy: 8551/210 objects degraded (4071.905%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:02.434 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[84576]: cluster 2024-09-13T14:52:01.356786+0000 mon.a (mon.0) 1781 : cluster [WRN] Health check update: Degraded data redundancy: 8551/210 objects degraded (4071.905%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:02.434 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[84576]: audit 2024-09-13T14:52:01.586542+0000 mon.a (mon.0) 1782 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:02.434 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[84576]: audit 2024-09-13T14:52:01.586542+0000 mon.a (mon.0) 1782 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:02.434 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[84576]: audit 2024-09-13T14:52:01.740641+0000 mon.a (mon.0) 1783 : audit [DBG] from='client.? 172.21.15.99:0/3153022160' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:02.435 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[84576]: audit 2024-09-13T14:52:01.740641+0000 mon.a (mon.0) 1783 : audit [DBG] from='client.? 172.21.15.99:0/3153022160' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:02.435 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[84576]: audit 2024-09-13T14:52:01.905274+0000 mon.a (mon.0) 1784 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T14:52:02.435 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:02 smithi099 bash[84576]: audit 2024-09-13T14:52:01.905274+0000 mon.a (mon.0) 1784 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T14:52:02.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:02.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:02.546 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:02.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:02 smithi177 bash[68594]: cluster 2024-09-13T14:52:01.356786+0000 mon.a (mon.0) 1781 : cluster [WRN] Health check update: Degraded data redundancy: 8551/210 objects degraded (4071.905%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:02.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:02 smithi177 bash[68594]: cluster 2024-09-13T14:52:01.356786+0000 mon.a (mon.0) 1781 : cluster [WRN] Health check update: Degraded data redundancy: 8551/210 objects degraded (4071.905%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:02.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:02 smithi177 bash[68594]: audit 2024-09-13T14:52:01.586542+0000 mon.a (mon.0) 1782 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:02.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:02 smithi177 bash[68594]: audit 2024-09-13T14:52:01.586542+0000 mon.a (mon.0) 1782 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:02.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:02 smithi177 bash[68594]: audit 2024-09-13T14:52:01.740641+0000 mon.a (mon.0) 1783 : audit [DBG] from='client.? 172.21.15.99:0/3153022160' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:02.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:02 smithi177 bash[68594]: audit 2024-09-13T14:52:01.740641+0000 mon.a (mon.0) 1783 : audit [DBG] from='client.? 172.21.15.99:0/3153022160' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:02.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:02 smithi177 bash[68594]: audit 2024-09-13T14:52:01.905274+0000 mon.a (mon.0) 1784 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T14:52:02.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:02 smithi177 bash[68594]: audit 2024-09-13T14:52:01.905274+0000 mon.a (mon.0) 1784 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T14:52:02.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:02.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:02.675 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:02.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:02.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:02.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:02.833 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:02.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:02.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:02.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:02.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:03.068 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:03.096 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:03.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:03.199 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:03.225 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:03.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:03.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: cluster 2024-09-13T14:52:02.295968+0000 mon.a (mon.0) 1785 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: cluster 2024-09-13T14:52:02.295968+0000 mon.a (mon.0) 1785 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: audit 2024-09-13T14:52:02.307001+0000 mon.a (mon.0) 1786 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: audit 2024-09-13T14:52:02.307001+0000 mon.a (mon.0) 1786 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: audit 2024-09-13T14:52:02.307178+0000 mon.a (mon.0) 1787 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: audit 2024-09-13T14:52:02.307178+0000 mon.a (mon.0) 1787 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: cluster 2024-09-13T14:52:02.316840+0000 mon.a (mon.0) 1788 : cluster [DBG] osdmap e992: 8 total, 8 up, 7 in 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: cluster 2024-09-13T14:52:02.316840+0000 mon.a (mon.0) 1788 : cluster [DBG] osdmap e992: 8 total, 8 up, 7 in 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: cluster 2024-09-13T14:52:02.587901+0000 mgr.y (mgr.26252) 773 : cluster [DBG] pgmap v1258: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/210 objects degraded (3814.762%); 0 B/s, 12 objects/s recovering 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: cluster 2024-09-13T14:52:02.587901+0000 mgr.y (mgr.26252) 773 : cluster [DBG] pgmap v1258: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/210 objects degraded (3814.762%); 0 B/s, 12 objects/s recovering 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: audit 2024-09-13T14:52:02.588889+0000 mon.a (mon.0) 1789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"} : dispatch 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: audit 2024-09-13T14:52:02.588889+0000 mon.a (mon.0) 1789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"} : dispatch 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: audit 2024-09-13T14:52:03.283645+0000 mon.a (mon.0) 1790 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: audit 2024-09-13T14:52:03.283645+0000 mon.a (mon.0) 1790 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: audit 2024-09-13T14:52:03.284852+0000 mon.a (mon.0) 1791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:03 smithi177 bash[68594]: audit 2024-09-13T14:52:03.284852+0000 mon.a (mon.0) 1791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:03.365 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:03.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:03.471 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:03.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: cluster 2024-09-13T14:52:02.295968+0000 mon.a (mon.0) 1785 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: cluster 2024-09-13T14:52:02.295968+0000 mon.a (mon.0) 1785 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: audit 2024-09-13T14:52:02.307001+0000 mon.a (mon.0) 1786 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: audit 2024-09-13T14:52:02.307001+0000 mon.a (mon.0) 1786 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: audit 2024-09-13T14:52:02.307178+0000 mon.a (mon.0) 1787 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2024-09-13T14:52:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: audit 2024-09-13T14:52:02.307178+0000 mon.a (mon.0) 1787 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2024-09-13T14:52:03.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: cluster 2024-09-13T14:52:02.316840+0000 mon.a (mon.0) 1788 : cluster [DBG] osdmap e992: 8 total, 8 up, 7 in 2024-09-13T14:52:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: cluster 2024-09-13T14:52:02.316840+0000 mon.a (mon.0) 1788 : cluster [DBG] osdmap e992: 8 total, 8 up, 7 in 2024-09-13T14:52:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: cluster 2024-09-13T14:52:02.587901+0000 mgr.y (mgr.26252) 773 : cluster [DBG] pgmap v1258: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/210 objects degraded (3814.762%); 0 B/s, 12 objects/s recovering 2024-09-13T14:52:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: cluster 2024-09-13T14:52:02.587901+0000 mgr.y (mgr.26252) 773 : cluster [DBG] pgmap v1258: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/210 objects degraded (3814.762%); 0 B/s, 12 objects/s recovering 2024-09-13T14:52:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: audit 2024-09-13T14:52:02.588889+0000 mon.a (mon.0) 1789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"} : dispatch 2024-09-13T14:52:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: audit 2024-09-13T14:52:02.588889+0000 mon.a (mon.0) 1789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"} : dispatch 2024-09-13T14:52:03.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: audit 2024-09-13T14:52:03.283645+0000 mon.a (mon.0) 1790 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:03.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: audit 2024-09-13T14:52:03.283645+0000 mon.a (mon.0) 1790 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:03.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: audit 2024-09-13T14:52:03.284852+0000 mon.a (mon.0) 1791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:03.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[84576]: audit 2024-09-13T14:52:03.284852+0000 mon.a (mon.0) 1791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:03.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: cluster 2024-09-13T14:52:02.295968+0000 mon.a (mon.0) 1785 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:03.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: cluster 2024-09-13T14:52:02.295968+0000 mon.a (mon.0) 1785 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:03.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: audit 2024-09-13T14:52:02.307001+0000 mon.a (mon.0) 1786 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:03.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: audit 2024-09-13T14:52:02.307001+0000 mon.a (mon.0) 1786 : audit [INF] from='client.? 172.21.15.99:0/3647337462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:03.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: audit 2024-09-13T14:52:02.307178+0000 mon.a (mon.0) 1787 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2024-09-13T14:52:03.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: audit 2024-09-13T14:52:02.307178+0000 mon.a (mon.0) 1787 : audit [INF] from='client.? 172.21.15.99:0/159952614' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2024-09-13T14:52:03.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: cluster 2024-09-13T14:52:02.316840+0000 mon.a (mon.0) 1788 : cluster [DBG] osdmap e992: 8 total, 8 up, 7 in 2024-09-13T14:52:03.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: cluster 2024-09-13T14:52:02.316840+0000 mon.a (mon.0) 1788 : cluster [DBG] osdmap e992: 8 total, 8 up, 7 in 2024-09-13T14:52:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: cluster 2024-09-13T14:52:02.587901+0000 mgr.y (mgr.26252) 773 : cluster [DBG] pgmap v1258: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/210 objects degraded (3814.762%); 0 B/s, 12 objects/s recovering 2024-09-13T14:52:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: cluster 2024-09-13T14:52:02.587901+0000 mgr.y (mgr.26252) 773 : cluster [DBG] pgmap v1258: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/210 objects degraded (3814.762%); 0 B/s, 12 objects/s recovering 2024-09-13T14:52:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: audit 2024-09-13T14:52:02.588889+0000 mon.a (mon.0) 1789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"} : dispatch 2024-09-13T14:52:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: audit 2024-09-13T14:52:02.588889+0000 mon.a (mon.0) 1789 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"} : dispatch 2024-09-13T14:52:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: audit 2024-09-13T14:52:03.283645+0000 mon.a (mon.0) 1790 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: audit 2024-09-13T14:52:03.283645+0000 mon.a (mon.0) 1790 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: audit 2024-09-13T14:52:03.284852+0000 mon.a (mon.0) 1791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:03 smithi099 bash[87147]: audit 2024-09-13T14:52:03.284852+0000 mon.a (mon.0) 1791 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:03.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:03.686 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:03.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:03.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:03.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:03.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:03.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:03.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:04.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:04.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:04.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:04.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:04.206 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:04.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:04.309 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:04.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:04.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:04.457 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:04.481 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:04.583 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:04.594 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:04 smithi177 bash[68594]: audit 2024-09-13T14:52:03.314788+0000 mon.a (mon.0) 1792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2024-09-13T14:52:04.594 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:04 smithi177 bash[68594]: audit 2024-09-13T14:52:03.314788+0000 mon.a (mon.0) 1792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2024-09-13T14:52:04.594 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:04 smithi177 bash[68594]: cluster 2024-09-13T14:52:03.319623+0000 mon.a (mon.0) 1793 : cluster [DBG] osdmap e993: 8 total, 8 up, 7 in 2024-09-13T14:52:04.594 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:04 smithi177 bash[68594]: cluster 2024-09-13T14:52:03.319623+0000 mon.a (mon.0) 1793 : cluster [DBG] osdmap e993: 8 total, 8 up, 7 in 2024-09-13T14:52:04.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:04.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:04.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:04.722 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:04 smithi099 bash[87147]: audit 2024-09-13T14:52:03.314788+0000 mon.a (mon.0) 1792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2024-09-13T14:52:04.723 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:04 smithi099 bash[87147]: audit 2024-09-13T14:52:03.314788+0000 mon.a (mon.0) 1792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2024-09-13T14:52:04.723 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:04 smithi099 bash[87147]: cluster 2024-09-13T14:52:03.319623+0000 mon.a (mon.0) 1793 : cluster [DBG] osdmap e993: 8 total, 8 up, 7 in 2024-09-13T14:52:04.723 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:04 smithi099 bash[87147]: cluster 2024-09-13T14:52:03.319623+0000 mon.a (mon.0) 1793 : cluster [DBG] osdmap e993: 8 total, 8 up, 7 in 2024-09-13T14:52:04.723 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:04 smithi099 bash[84576]: audit 2024-09-13T14:52:03.314788+0000 mon.a (mon.0) 1792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2024-09-13T14:52:04.723 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:04 smithi099 bash[84576]: audit 2024-09-13T14:52:03.314788+0000 mon.a (mon.0) 1792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2024-09-13T14:52:04.723 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:04 smithi099 bash[84576]: cluster 2024-09-13T14:52:03.319623+0000 mon.a (mon.0) 1793 : cluster [DBG] osdmap e993: 8 total, 8 up, 7 in 2024-09-13T14:52:04.723 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:04 smithi099 bash[84576]: cluster 2024-09-13T14:52:03.319623+0000 mon.a (mon.0) 1793 : cluster [DBG] osdmap e993: 8 total, 8 up, 7 in 2024-09-13T14:52:04.738 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:04.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:04.841 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:04.868 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:04.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:04.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:04.998 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:05.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:05.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:05.128 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:05.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:05.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:05.258 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:05.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:05.361 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:05.372 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:05 smithi177 bash[68594]: cluster 2024-09-13T14:52:04.339460+0000 mon.a (mon.0) 1794 : cluster [DBG] osdmap e994: 8 total, 8 up, 7 in 2024-09-13T14:52:05.373 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:05 smithi177 bash[68594]: cluster 2024-09-13T14:52:04.339460+0000 mon.a (mon.0) 1794 : cluster [DBG] osdmap e994: 8 total, 8 up, 7 in 2024-09-13T14:52:05.373 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:05 smithi177 bash[68594]: cluster 2024-09-13T14:52:04.588679+0000 mgr.y (mgr.26252) 774 : cluster [DBG] pgmap v1261: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%); 0 B/s, 69 objects/s recovering 2024-09-13T14:52:05.373 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:05 smithi177 bash[68594]: cluster 2024-09-13T14:52:04.588679+0000 mgr.y (mgr.26252) 774 : cluster [DBG] pgmap v1261: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%); 0 B/s, 69 objects/s recovering 2024-09-13T14:52:05.373 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:05 smithi177 bash[68594]: cluster 2024-09-13T14:52:04.663458+0000 osd.3 (osd.3) 9 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:05.373 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:05 smithi177 bash[68594]: cluster 2024-09-13T14:52:04.663458+0000 osd.3 (osd.3) 9 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:05.389 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:05.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:05.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:05.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:05.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:05.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:05.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:05.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:05.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:05.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:05.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:05.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:05.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:05.908 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[84576]: cluster 2024-09-13T14:52:04.339460+0000 mon.a (mon.0) 1794 : cluster [DBG] osdmap e994: 8 total, 8 up, 7 in 2024-09-13T14:52:05.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[84576]: cluster 2024-09-13T14:52:04.339460+0000 mon.a (mon.0) 1794 : cluster [DBG] osdmap e994: 8 total, 8 up, 7 in 2024-09-13T14:52:05.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[84576]: cluster 2024-09-13T14:52:04.588679+0000 mgr.y (mgr.26252) 774 : cluster [DBG] pgmap v1261: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%); 0 B/s, 69 objects/s recovering 2024-09-13T14:52:05.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[84576]: cluster 2024-09-13T14:52:04.588679+0000 mgr.y (mgr.26252) 774 : cluster [DBG] pgmap v1261: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%); 0 B/s, 69 objects/s recovering 2024-09-13T14:52:05.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[84576]: cluster 2024-09-13T14:52:04.663458+0000 osd.3 (osd.3) 9 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:05.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[84576]: cluster 2024-09-13T14:52:04.663458+0000 osd.3 (osd.3) 9 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:05.909 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[87147]: cluster 2024-09-13T14:52:04.339460+0000 mon.a (mon.0) 1794 : cluster [DBG] osdmap e994: 8 total, 8 up, 7 in 2024-09-13T14:52:05.909 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[87147]: cluster 2024-09-13T14:52:04.339460+0000 mon.a (mon.0) 1794 : cluster [DBG] osdmap e994: 8 total, 8 up, 7 in 2024-09-13T14:52:05.909 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[87147]: cluster 2024-09-13T14:52:04.588679+0000 mgr.y (mgr.26252) 774 : cluster [DBG] pgmap v1261: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%); 0 B/s, 69 objects/s recovering 2024-09-13T14:52:05.909 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[87147]: cluster 2024-09-13T14:52:04.588679+0000 mgr.y (mgr.26252) 774 : cluster [DBG] pgmap v1261: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%); 0 B/s, 69 objects/s recovering 2024-09-13T14:52:05.909 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[87147]: cluster 2024-09-13T14:52:04.663458+0000 osd.3 (osd.3) 9 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:05.910 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:05 smithi099 bash[87147]: cluster 2024-09-13T14:52:04.663458+0000 osd.3 (osd.3) 9 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:06.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:06.033 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:06.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:06.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:06.271 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:06.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.403 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[84576]: cluster 2024-09-13T14:52:04.705721+0000 osd.2 (osd.2) 5 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:06.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[84576]: cluster 2024-09-13T14:52:04.705721+0000 osd.2 (osd.2) 5 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:06.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[84576]: cluster 2024-09-13T14:52:04.770052+0000 osd.4 (osd.4) 9 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:06.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[84576]: cluster 2024-09-13T14:52:04.770052+0000 osd.4 (osd.4) 9 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:06.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[84576]: cluster 2024-09-13T14:52:05.340314+0000 mon.a (mon.0) 1795 : cluster [DBG] osdmap e995: 8 total, 8 up, 7 in 2024-09-13T14:52:06.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[84576]: cluster 2024-09-13T14:52:05.340314+0000 mon.a (mon.0) 1795 : cluster [DBG] osdmap e995: 8 total, 8 up, 7 in 2024-09-13T14:52:06.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[84576]: cluster 2024-09-13T14:52:05.613973+0000 osd.3 (osd.3) 10 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:06.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[84576]: cluster 2024-09-13T14:52:05.613973+0000 osd.3 (osd.3) 10 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:06.404 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[87147]: cluster 2024-09-13T14:52:04.705721+0000 osd.2 (osd.2) 5 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:06.404 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[87147]: cluster 2024-09-13T14:52:04.705721+0000 osd.2 (osd.2) 5 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:06.404 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[87147]: cluster 2024-09-13T14:52:04.770052+0000 osd.4 (osd.4) 9 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:06.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[87147]: cluster 2024-09-13T14:52:04.770052+0000 osd.4 (osd.4) 9 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:06.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[87147]: cluster 2024-09-13T14:52:05.340314+0000 mon.a (mon.0) 1795 : cluster [DBG] osdmap e995: 8 total, 8 up, 7 in 2024-09-13T14:52:06.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[87147]: cluster 2024-09-13T14:52:05.340314+0000 mon.a (mon.0) 1795 : cluster [DBG] osdmap e995: 8 total, 8 up, 7 in 2024-09-13T14:52:06.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[87147]: cluster 2024-09-13T14:52:05.613973+0000 osd.3 (osd.3) 10 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:06.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:06 smithi099 bash[87147]: cluster 2024-09-13T14:52:05.613973+0000 osd.3 (osd.3) 10 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:06.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:06.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:06.541 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:06.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:06.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:06 smithi177 bash[68594]: cluster 2024-09-13T14:52:04.705721+0000 osd.2 (osd.2) 5 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:06.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:06 smithi177 bash[68594]: cluster 2024-09-13T14:52:04.705721+0000 osd.2 (osd.2) 5 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:06.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:06 smithi177 bash[68594]: cluster 2024-09-13T14:52:04.770052+0000 osd.4 (osd.4) 9 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:06 smithi177 bash[68594]: cluster 2024-09-13T14:52:04.770052+0000 osd.4 (osd.4) 9 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:06 smithi177 bash[68594]: cluster 2024-09-13T14:52:05.340314+0000 mon.a (mon.0) 1795 : cluster [DBG] osdmap e995: 8 total, 8 up, 7 in 2024-09-13T14:52:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:06 smithi177 bash[68594]: cluster 2024-09-13T14:52:05.340314+0000 mon.a (mon.0) 1795 : cluster [DBG] osdmap e995: 8 total, 8 up, 7 in 2024-09-13T14:52:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:06 smithi177 bash[68594]: cluster 2024-09-13T14:52:05.613973+0000 osd.3 (osd.3) 10 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:06.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:06 smithi177 bash[68594]: cluster 2024-09-13T14:52:05.613973+0000 osd.3 (osd.3) 10 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:06.643 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.667 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:06.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:06.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:06.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:06.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.920 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:06.991 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-13T14:52:06.991 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-13T14:52:07.024 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:07.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:07.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:07.157 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:07.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:07.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:07.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:07.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:07.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:52:07] "GET /metrics HTTP/1.1" 200 36751 "" "Prometheus/2.51.0" 2024-09-13T14:52:07.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:07.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:07.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:07.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:07.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:07.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:07.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:05.666206+0000 osd.2 (osd.2) 6 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:05.666206+0000 osd.2 (osd.2) 6 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:05.747595+0000 osd.4 (osd.4) 10 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:05.747595+0000 osd.4 (osd.4) 10 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:06.349014+0000 mon.a (mon.0) 1796 : cluster [DBG] osdmap e996: 8 total, 8 up, 7 in 2024-09-13T14:52:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:06.349014+0000 mon.a (mon.0) 1796 : cluster [DBG] osdmap e996: 8 total, 8 up, 7 in 2024-09-13T14:52:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:06.358828+0000 mon.a (mon.0) 1797 : cluster [WRN] Health check update: Degraded data redundancy: 8011/144 objects degraded (5563.194%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:06.358828+0000 mon.a (mon.0) 1797 : cluster [WRN] Health check update: Degraded data redundancy: 8011/144 objects degraded (5563.194%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:06.589187+0000 mgr.y (mgr.26252) 775 : cluster [DBG] pgmap v1264: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%) 2024-09-13T14:52:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:06.589187+0000 mgr.y (mgr.26252) 775 : cluster [DBG] pgmap v1264: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%) 2024-09-13T14:52:07.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:06.620885+0000 osd.3 (osd.3) 11 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:07.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:07 smithi177 bash[68594]: cluster 2024-09-13T14:52:06.620885+0000 osd.3 (osd.3) 11 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:07.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:07.699 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:07.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:07.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:07.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:05.666206+0000 osd.2 (osd.2) 6 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:07.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:05.666206+0000 osd.2 (osd.2) 6 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:07.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:05.747595+0000 osd.4 (osd.4) 10 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:07.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:05.747595+0000 osd.4 (osd.4) 10 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:07.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:06.349014+0000 mon.a (mon.0) 1796 : cluster [DBG] osdmap e996: 8 total, 8 up, 7 in 2024-09-13T14:52:07.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:06.349014+0000 mon.a (mon.0) 1796 : cluster [DBG] osdmap e996: 8 total, 8 up, 7 in 2024-09-13T14:52:07.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:06.358828+0000 mon.a (mon.0) 1797 : cluster [WRN] Health check update: Degraded data redundancy: 8011/144 objects degraded (5563.194%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:06.358828+0000 mon.a (mon.0) 1797 : cluster [WRN] Health check update: Degraded data redundancy: 8011/144 objects degraded (5563.194%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:06.589187+0000 mgr.y (mgr.26252) 775 : cluster [DBG] pgmap v1264: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%) 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:06.589187+0000 mgr.y (mgr.26252) 775 : cluster [DBG] pgmap v1264: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%) 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:06.620885+0000 osd.3 (osd.3) 11 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[87147]: cluster 2024-09-13T14:52:06.620885+0000 osd.3 (osd.3) 11 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:05.666206+0000 osd.2 (osd.2) 6 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:05.666206+0000 osd.2 (osd.2) 6 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:05.747595+0000 osd.4 (osd.4) 10 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:05.747595+0000 osd.4 (osd.4) 10 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:06.349014+0000 mon.a (mon.0) 1796 : cluster [DBG] osdmap e996: 8 total, 8 up, 7 in 2024-09-13T14:52:07.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:06.349014+0000 mon.a (mon.0) 1796 : cluster [DBG] osdmap e996: 8 total, 8 up, 7 in 2024-09-13T14:52:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:06.358828+0000 mon.a (mon.0) 1797 : cluster [WRN] Health check update: Degraded data redundancy: 8011/144 objects degraded (5563.194%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:06.358828+0000 mon.a (mon.0) 1797 : cluster [WRN] Health check update: Degraded data redundancy: 8011/144 objects degraded (5563.194%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:52:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:06.589187+0000 mgr.y (mgr.26252) 775 : cluster [DBG] pgmap v1264: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%) 2024-09-13T14:52:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:06.589187+0000 mgr.y (mgr.26252) 775 : cluster [DBG] pgmap v1264: 109 pgs: 5 peering, 10 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 8011/144 objects degraded (5563.194%) 2024-09-13T14:52:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:06.620885+0000 osd.3 (osd.3) 11 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:07.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:07 smithi099 bash[84576]: cluster 2024-09-13T14:52:06.620885+0000 osd.3 (osd.3) 11 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:07.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:07.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:07.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:07.962 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:07.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:08.065 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:08.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:08.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:08.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:08.217 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:08.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:08.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:08.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:08.449 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:08.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:08 smithi177 bash[68594]: cluster 2024-09-13T14:52:06.729634+0000 osd.4 (osd.4) 11 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:08.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:08 smithi177 bash[68594]: cluster 2024-09-13T14:52:06.729634+0000 osd.4 (osd.4) 11 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:08.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:08 smithi177 bash[68594]: cluster 2024-09-13T14:52:07.618101+0000 osd.3 (osd.3) 12 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:08.461 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:08 smithi177 bash[68594]: cluster 2024-09-13T14:52:07.618101+0000 osd.3 (osd.3) 12 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:08.462 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:08.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:08.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:08.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:08.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:08 smithi099 bash[84576]: cluster 2024-09-13T14:52:06.729634+0000 osd.4 (osd.4) 11 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:08.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:08 smithi099 bash[84576]: cluster 2024-09-13T14:52:06.729634+0000 osd.4 (osd.4) 11 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:08.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:08 smithi099 bash[84576]: cluster 2024-09-13T14:52:07.618101+0000 osd.3 (osd.3) 12 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:08.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:08 smithi099 bash[84576]: cluster 2024-09-13T14:52:07.618101+0000 osd.3 (osd.3) 12 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:08.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:08 smithi099 bash[87147]: cluster 2024-09-13T14:52:06.729634+0000 osd.4 (osd.4) 11 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:08.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:08 smithi099 bash[87147]: cluster 2024-09-13T14:52:06.729634+0000 osd.4 (osd.4) 11 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:08.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:08 smithi099 bash[87147]: cluster 2024-09-13T14:52:07.618101+0000 osd.3 (osd.3) 12 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:08.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:08 smithi099 bash[87147]: cluster 2024-09-13T14:52:07.618101+0000 osd.3 (osd.3) 12 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:08.601 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:08.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:08.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:08.732 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:08.835 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:08.864 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:08.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:52:08.968 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:08.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:52:08.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:08.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:09.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:09.119 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:09.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:09.222 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:09.250 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:09.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:09.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:09.378 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:09.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:09.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:09 smithi177 bash[68594]: cluster 2024-09-13T14:52:07.727456+0000 osd.4 (osd.4) 12 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:09.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:09 smithi177 bash[68594]: cluster 2024-09-13T14:52:07.727456+0000 osd.4 (osd.4) 12 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:09.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:09 smithi177 bash[68594]: cluster 2024-09-13T14:52:08.373611+0000 mon.a (mon.0) 1798 : cluster [DBG] osdmap e997: 8 total, 8 up, 7 in 2024-09-13T14:52:09.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:09 smithi177 bash[68594]: cluster 2024-09-13T14:52:08.373611+0000 mon.a (mon.0) 1798 : cluster [DBG] osdmap e997: 8 total, 8 up, 7 in 2024-09-13T14:52:09.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:09 smithi177 bash[68594]: cluster 2024-09-13T14:52:08.589648+0000 mgr.y (mgr.26252) 776 : cluster [DBG] pgmap v1266: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 peering, 5 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 94 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 6438/144 objects degraded (4470.833%); 0 B/s, 329 objects/s recovering 2024-09-13T14:52:09.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:09 smithi177 bash[68594]: cluster 2024-09-13T14:52:08.589648+0000 mgr.y (mgr.26252) 776 : cluster [DBG] pgmap v1266: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 peering, 5 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 94 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 6438/144 objects degraded (4470.833%); 0 B/s, 329 objects/s recovering 2024-09-13T14:52:09.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:09 smithi177 bash[68594]: cluster 2024-09-13T14:52:08.652940+0000 osd.3 (osd.3) 13 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:09.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:09 smithi177 bash[68594]: cluster 2024-09-13T14:52:08.652940+0000 osd.3 (osd.3) 13 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:09.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:09 smithi177 bash[68594]: cluster 2024-09-13T14:52:09.373632+0000 mon.a (mon.0) 1799 : cluster [DBG] osdmap e998: 8 total, 8 up, 7 in 2024-09-13T14:52:09.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:09 smithi177 bash[68594]: cluster 2024-09-13T14:52:09.373632+0000 mon.a (mon.0) 1799 : cluster [DBG] osdmap e998: 8 total, 8 up, 7 in 2024-09-13T14:52:09.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:09.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:09.608 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:09.618 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[87147]: cluster 2024-09-13T14:52:07.727456+0000 osd.4 (osd.4) 12 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:09.619 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[87147]: cluster 2024-09-13T14:52:07.727456+0000 osd.4 (osd.4) 12 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:09.619 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[87147]: cluster 2024-09-13T14:52:08.373611+0000 mon.a (mon.0) 1798 : cluster [DBG] osdmap e997: 8 total, 8 up, 7 in 2024-09-13T14:52:09.619 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[87147]: cluster 2024-09-13T14:52:08.373611+0000 mon.a (mon.0) 1798 : cluster [DBG] osdmap e997: 8 total, 8 up, 7 in 2024-09-13T14:52:09.619 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[87147]: cluster 2024-09-13T14:52:08.589648+0000 mgr.y (mgr.26252) 776 : cluster [DBG] pgmap v1266: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 peering, 5 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 94 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 6438/144 objects degraded (4470.833%); 0 B/s, 329 objects/s recovering 2024-09-13T14:52:09.619 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[87147]: cluster 2024-09-13T14:52:08.589648+0000 mgr.y (mgr.26252) 776 : cluster [DBG] pgmap v1266: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 peering, 5 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 94 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 6438/144 objects degraded (4470.833%); 0 B/s, 329 objects/s recovering 2024-09-13T14:52:09.619 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[87147]: cluster 2024-09-13T14:52:08.652940+0000 osd.3 (osd.3) 13 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:09.619 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[87147]: cluster 2024-09-13T14:52:08.652940+0000 osd.3 (osd.3) 13 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:09.619 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[87147]: cluster 2024-09-13T14:52:09.373632+0000 mon.a (mon.0) 1799 : cluster [DBG] osdmap e998: 8 total, 8 up, 7 in 2024-09-13T14:52:09.619 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[87147]: cluster 2024-09-13T14:52:09.373632+0000 mon.a (mon.0) 1799 : cluster [DBG] osdmap e998: 8 total, 8 up, 7 in 2024-09-13T14:52:09.619 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[84576]: cluster 2024-09-13T14:52:07.727456+0000 osd.4 (osd.4) 12 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:09.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[84576]: cluster 2024-09-13T14:52:07.727456+0000 osd.4 (osd.4) 12 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:09.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[84576]: cluster 2024-09-13T14:52:08.373611+0000 mon.a (mon.0) 1798 : cluster [DBG] osdmap e997: 8 total, 8 up, 7 in 2024-09-13T14:52:09.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[84576]: cluster 2024-09-13T14:52:08.373611+0000 mon.a (mon.0) 1798 : cluster [DBG] osdmap e997: 8 total, 8 up, 7 in 2024-09-13T14:52:09.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[84576]: cluster 2024-09-13T14:52:08.589648+0000 mgr.y (mgr.26252) 776 : cluster [DBG] pgmap v1266: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 peering, 5 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 94 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 6438/144 objects degraded (4470.833%); 0 B/s, 329 objects/s recovering 2024-09-13T14:52:09.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[84576]: cluster 2024-09-13T14:52:08.589648+0000 mgr.y (mgr.26252) 776 : cluster [DBG] pgmap v1266: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 peering, 5 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 94 active+clean; 1.9 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 6438/144 objects degraded (4470.833%); 0 B/s, 329 objects/s recovering 2024-09-13T14:52:09.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[84576]: cluster 2024-09-13T14:52:08.652940+0000 osd.3 (osd.3) 13 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:09.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[84576]: cluster 2024-09-13T14:52:08.652940+0000 osd.3 (osd.3) 13 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:09.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[84576]: cluster 2024-09-13T14:52:09.373632+0000 mon.a (mon.0) 1799 : cluster [DBG] osdmap e998: 8 total, 8 up, 7 in 2024-09-13T14:52:09.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:09 smithi099 bash[84576]: cluster 2024-09-13T14:52:09.373632+0000 mon.a (mon.0) 1799 : cluster [DBG] osdmap e998: 8 total, 8 up, 7 in 2024-09-13T14:52:09.632 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:09.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:09.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:09.764 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:09.866 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:09.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:09.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:10.027 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:10.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:10.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:10.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:10.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:10.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:10.284 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:10.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:10.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:52:10 smithi177 bash[61439]: ts=2024-09-13T14:52:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:10.388 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:10.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:10.516 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:10 smithi177 bash[68594]: cluster 2024-09-13T14:52:09.621407+0000 osd.3 (osd.3) 14 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:10 smithi177 bash[68594]: cluster 2024-09-13T14:52:09.621407+0000 osd.3 (osd.3) 14 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:10 smithi177 bash[68594]: cluster 2024-09-13T14:52:09.651548+0000 osd.0 (osd.0) 5 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:10 smithi177 bash[68594]: cluster 2024-09-13T14:52:09.651548+0000 osd.0 (osd.0) 5 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:10.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:10.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:10.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:10 smithi099 bash[87147]: cluster 2024-09-13T14:52:09.621407+0000 osd.3 (osd.3) 14 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:10.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:10 smithi099 bash[87147]: cluster 2024-09-13T14:52:09.621407+0000 osd.3 (osd.3) 14 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:10.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:10 smithi099 bash[87147]: cluster 2024-09-13T14:52:09.651548+0000 osd.0 (osd.0) 5 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:10.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:10 smithi099 bash[87147]: cluster 2024-09-13T14:52:09.651548+0000 osd.0 (osd.0) 5 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:10.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:10 smithi099 bash[84576]: cluster 2024-09-13T14:52:09.621407+0000 osd.3 (osd.3) 14 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:10.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:10 smithi099 bash[84576]: cluster 2024-09-13T14:52:09.621407+0000 osd.3 (osd.3) 14 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:10.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:10 smithi099 bash[84576]: cluster 2024-09-13T14:52:09.651548+0000 osd.0 (osd.0) 5 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:10.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:10 smithi099 bash[84576]: cluster 2024-09-13T14:52:09.651548+0000 osd.0 (osd.0) 5 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:10.648 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:10.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:10.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:10.779 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:10.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:10.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:10.937 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:10.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:11.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:11.068 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:11.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:11.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:11.197 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:11.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:11.300 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:11.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:11.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:11.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:11.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:11.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:11.524 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:11.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:11.570 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:11 smithi177 bash[68594]: cluster 2024-09-13T14:52:10.590363+0000 mgr.y (mgr.26252) 777 : cluster [DBG] pgmap v1268: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+scrubbing, 103 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 55 MiB/s wr, 1.75k op/s; 6438/210 objects degraded (3065.714%); 1.5 MiB/s, 290 objects/s recovering 2024-09-13T14:52:11.570 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:11 smithi177 bash[68594]: cluster 2024-09-13T14:52:10.590363+0000 mgr.y (mgr.26252) 777 : cluster [DBG] pgmap v1268: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+scrubbing, 103 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 55 MiB/s wr, 1.75k op/s; 6438/210 objects degraded (3065.714%); 1.5 MiB/s, 290 objects/s recovering 2024-09-13T14:52:11.570 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:11 smithi177 bash[68594]: cluster 2024-09-13T14:52:10.641942+0000 osd.3 (osd.3) 15 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:11.570 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:11 smithi177 bash[68594]: cluster 2024-09-13T14:52:10.641942+0000 osd.3 (osd.3) 15 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:11.570 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:11 smithi177 bash[68594]: cluster 2024-09-13T14:52:11.359616+0000 mon.a (mon.0) 1800 : cluster [WRN] Health check update: Degraded data redundancy: 6438/144 objects degraded (4470.833%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:11.570 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:11 smithi177 bash[68594]: cluster 2024-09-13T14:52:11.359616+0000 mon.a (mon.0) 1800 : cluster [WRN] Health check update: Degraded data redundancy: 6438/144 objects degraded (4470.833%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[87147]: cluster 2024-09-13T14:52:10.590363+0000 mgr.y (mgr.26252) 777 : cluster [DBG] pgmap v1268: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+scrubbing, 103 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 55 MiB/s wr, 1.75k op/s; 6438/210 objects degraded (3065.714%); 1.5 MiB/s, 290 objects/s recovering 2024-09-13T14:52:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[87147]: cluster 2024-09-13T14:52:10.590363+0000 mgr.y (mgr.26252) 777 : cluster [DBG] pgmap v1268: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+scrubbing, 103 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 55 MiB/s wr, 1.75k op/s; 6438/210 objects degraded (3065.714%); 1.5 MiB/s, 290 objects/s recovering 2024-09-13T14:52:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[87147]: cluster 2024-09-13T14:52:10.641942+0000 osd.3 (osd.3) 15 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[87147]: cluster 2024-09-13T14:52:10.641942+0000 osd.3 (osd.3) 15 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[87147]: cluster 2024-09-13T14:52:11.359616+0000 mon.a (mon.0) 1800 : cluster [WRN] Health check update: Degraded data redundancy: 6438/144 objects degraded (4470.833%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:11.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[87147]: cluster 2024-09-13T14:52:11.359616+0000 mon.a (mon.0) 1800 : cluster [WRN] Health check update: Degraded data redundancy: 6438/144 objects degraded (4470.833%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[84576]: cluster 2024-09-13T14:52:10.590363+0000 mgr.y (mgr.26252) 777 : cluster [DBG] pgmap v1268: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+scrubbing, 103 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 55 MiB/s wr, 1.75k op/s; 6438/210 objects degraded (3065.714%); 1.5 MiB/s, 290 objects/s recovering 2024-09-13T14:52:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[84576]: cluster 2024-09-13T14:52:10.590363+0000 mgr.y (mgr.26252) 777 : cluster [DBG] pgmap v1268: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+scrubbing, 103 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 55 MiB/s wr, 1.75k op/s; 6438/210 objects degraded (3065.714%); 1.5 MiB/s, 290 objects/s recovering 2024-09-13T14:52:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[84576]: cluster 2024-09-13T14:52:10.641942+0000 osd.3 (osd.3) 15 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[84576]: cluster 2024-09-13T14:52:10.641942+0000 osd.3 (osd.3) 15 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[84576]: cluster 2024-09-13T14:52:11.359616+0000 mon.a (mon.0) 1800 : cluster [WRN] Health check update: Degraded data redundancy: 6438/144 objects degraded (4470.833%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:11.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:11 smithi099 bash[84576]: cluster 2024-09-13T14:52:11.359616+0000 mon.a (mon.0) 1800 : cluster [WRN] Health check update: Degraded data redundancy: 6438/144 objects degraded (4470.833%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:11.587 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:11.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:11.689 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:11.715 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:11.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:11.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:11.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:11.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:11.946 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:11.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:11.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:11.991 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:52:11.991 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:52:11.991 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-09-13T14:52:11.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd in 1 2024-09-13T14:52:12.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:12.100 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:12.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.223 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:12.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.347 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:12.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:12.476 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:12.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:12.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.587 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[87147]: cluster 2024-09-13T14:52:10.675480+0000 osd.2 (osd.2) 7 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:12.587 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[87147]: cluster 2024-09-13T14:52:10.675480+0000 osd.2 (osd.2) 7 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:12.587 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[87147]: cluster 2024-09-13T14:52:10.764504+0000 osd.4 (osd.4) 13 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:12.587 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[87147]: cluster 2024-09-13T14:52:10.764504+0000 osd.4 (osd.4) 13 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:12.587 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[87147]: cluster 2024-09-13T14:52:11.652395+0000 osd.3 (osd.3) 16 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:12.587 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[87147]: cluster 2024-09-13T14:52:11.652395+0000 osd.3 (osd.3) 16 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:12.587 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[84576]: cluster 2024-09-13T14:52:10.675480+0000 osd.2 (osd.2) 7 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:12.588 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[84576]: cluster 2024-09-13T14:52:10.675480+0000 osd.2 (osd.2) 7 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:12.588 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[84576]: cluster 2024-09-13T14:52:10.764504+0000 osd.4 (osd.4) 13 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:12.588 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[84576]: cluster 2024-09-13T14:52:10.764504+0000 osd.4 (osd.4) 13 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:12.588 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[84576]: cluster 2024-09-13T14:52:11.652395+0000 osd.3 (osd.3) 16 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:12.588 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:12 smithi099 bash[84576]: cluster 2024-09-13T14:52:11.652395+0000 osd.3 (osd.3) 16 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:12.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:12.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.726 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:12.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:12.840 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:12 smithi177 bash[68594]: cluster 2024-09-13T14:52:10.675480+0000 osd.2 (osd.2) 7 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:12.840 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:12 smithi177 bash[68594]: cluster 2024-09-13T14:52:10.675480+0000 osd.2 (osd.2) 7 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:12.840 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:12 smithi177 bash[68594]: cluster 2024-09-13T14:52:10.764504+0000 osd.4 (osd.4) 13 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:12.840 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:12 smithi177 bash[68594]: cluster 2024-09-13T14:52:10.764504+0000 osd.4 (osd.4) 13 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:12.840 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:12 smithi177 bash[68594]: cluster 2024-09-13T14:52:11.652395+0000 osd.3 (osd.3) 16 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:12.841 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:12 smithi177 bash[68594]: cluster 2024-09-13T14:52:11.652395+0000 osd.3 (osd.3) 16 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:12.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:12.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:12.960 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:12.988 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:12.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:13.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:13.117 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:13.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:13.220 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:13.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:13.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:13.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:13.377 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:13.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:13.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:13.504 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:13.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:13.569 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:13.570 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:13.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:13.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:13 smithi177 bash[68594]: cluster 2024-09-13T14:52:11.683728+0000 osd.2 (osd.2) 8 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:13.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:13 smithi177 bash[68594]: cluster 2024-09-13T14:52:11.683728+0000 osd.2 (osd.2) 8 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:13 smithi177 bash[68594]: cluster 2024-09-13T14:52:11.778604+0000 osd.4 (osd.4) 14 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:13 smithi177 bash[68594]: cluster 2024-09-13T14:52:11.778604+0000 osd.4 (osd.4) 14 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:13 smithi177 bash[68594]: cluster 2024-09-13T14:52:12.591188+0000 mgr.y (mgr.26252) 778 : cluster [DBG] pgmap v1269: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+scrubbing, 1 active+recovering, 102 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 46 MiB/s wr, 1.47k op/s; 3904/210 objects degraded (1859.048%); 1.2 MiB/s, 422 objects/s recovering 2024-09-13T14:52:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:13 smithi177 bash[68594]: cluster 2024-09-13T14:52:12.591188+0000 mgr.y (mgr.26252) 778 : cluster [DBG] pgmap v1269: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+scrubbing, 1 active+recovering, 102 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 46 MiB/s wr, 1.47k op/s; 3904/210 objects degraded (1859.048%); 1.2 MiB/s, 422 objects/s recovering 2024-09-13T14:52:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:13 smithi177 bash[68594]: cluster 2024-09-13T14:52:12.697550+0000 osd.3 (osd.3) 17 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:13 smithi177 bash[68594]: cluster 2024-09-13T14:52:12.697550+0000 osd.3 (osd.3) 17 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:13.635 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:13.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:13.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:13.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:13.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:13.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:13.896 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:13.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:13.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[84576]: cluster 2024-09-13T14:52:11.683728+0000 osd.2 (osd.2) 8 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:13.918 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[84576]: cluster 2024-09-13T14:52:11.683728+0000 osd.2 (osd.2) 8 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:13.918 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[84576]: cluster 2024-09-13T14:52:11.778604+0000 osd.4 (osd.4) 14 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:13.918 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[84576]: cluster 2024-09-13T14:52:11.778604+0000 osd.4 (osd.4) 14 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:13.918 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[84576]: cluster 2024-09-13T14:52:12.591188+0000 mgr.y (mgr.26252) 778 : cluster [DBG] pgmap v1269: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+scrubbing, 1 active+recovering, 102 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 46 MiB/s wr, 1.47k op/s; 3904/210 objects degraded (1859.048%); 1.2 MiB/s, 422 objects/s recovering 2024-09-13T14:52:13.918 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[84576]: cluster 2024-09-13T14:52:12.591188+0000 mgr.y (mgr.26252) 778 : cluster [DBG] pgmap v1269: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+scrubbing, 1 active+recovering, 102 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 46 MiB/s wr, 1.47k op/s; 3904/210 objects degraded (1859.048%); 1.2 MiB/s, 422 objects/s recovering 2024-09-13T14:52:13.918 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[84576]: cluster 2024-09-13T14:52:12.697550+0000 osd.3 (osd.3) 17 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:13.918 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[84576]: cluster 2024-09-13T14:52:12.697550+0000 osd.3 (osd.3) 17 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:13.918 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[87147]: cluster 2024-09-13T14:52:11.683728+0000 osd.2 (osd.2) 8 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:13.918 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[87147]: cluster 2024-09-13T14:52:11.683728+0000 osd.2 (osd.2) 8 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:13.918 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[87147]: cluster 2024-09-13T14:52:11.778604+0000 osd.4 (osd.4) 14 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:13.919 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[87147]: cluster 2024-09-13T14:52:11.778604+0000 osd.4 (osd.4) 14 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:13.919 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[87147]: cluster 2024-09-13T14:52:12.591188+0000 mgr.y (mgr.26252) 778 : cluster [DBG] pgmap v1269: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+scrubbing, 1 active+recovering, 102 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 46 MiB/s wr, 1.47k op/s; 3904/210 objects degraded (1859.048%); 1.2 MiB/s, 422 objects/s recovering 2024-09-13T14:52:13.919 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[87147]: cluster 2024-09-13T14:52:12.591188+0000 mgr.y (mgr.26252) 778 : cluster [DBG] pgmap v1269: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+scrubbing, 1 active+recovering, 102 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 46 MiB/s wr, 1.47k op/s; 3904/210 objects degraded (1859.048%); 1.2 MiB/s, 422 objects/s recovering 2024-09-13T14:52:13.919 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[87147]: cluster 2024-09-13T14:52:12.697550+0000 osd.3 (osd.3) 17 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:13.919 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:13 smithi099 bash[87147]: cluster 2024-09-13T14:52:12.697550+0000 osd.3 (osd.3) 17 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:14.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:14.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:14.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:14.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:14.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:14.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:14.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:14.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:14.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:14.398 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:14.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:14.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:14.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:14.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:14.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:14.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:14.663 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:14 smithi099 bash[84576]: cluster 2024-09-13T14:52:12.778569+0000 osd.4 (osd.4) 15 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:14.663 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:14 smithi099 bash[84576]: cluster 2024-09-13T14:52:12.778569+0000 osd.4 (osd.4) 15 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:14.663 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:14 smithi099 bash[84576]: cluster 2024-09-13T14:52:13.655419+0000 osd.3 (osd.3) 18 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:14.663 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:14 smithi099 bash[84576]: cluster 2024-09-13T14:52:13.655419+0000 osd.3 (osd.3) 18 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:14.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:14.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:14.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:14.794 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:14 smithi177 bash[68594]: cluster 2024-09-13T14:52:12.778569+0000 osd.4 (osd.4) 15 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:14.795 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:14 smithi177 bash[68594]: cluster 2024-09-13T14:52:12.778569+0000 osd.4 (osd.4) 15 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:14.795 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:14 smithi177 bash[68594]: cluster 2024-09-13T14:52:13.655419+0000 osd.3 (osd.3) 18 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:14.795 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:14 smithi177 bash[68594]: cluster 2024-09-13T14:52:13.655419+0000 osd.3 (osd.3) 18 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:14.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:14.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:14.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:15.043 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:15.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:14 smithi099 bash[87147]: cluster 2024-09-13T14:52:12.778569+0000 osd.4 (osd.4) 15 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:15.055 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:14 smithi099 bash[87147]: cluster 2024-09-13T14:52:12.778569+0000 osd.4 (osd.4) 15 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:15.055 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:14 smithi099 bash[87147]: cluster 2024-09-13T14:52:13.655419+0000 osd.3 (osd.3) 18 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:15.055 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:14 smithi099 bash[87147]: cluster 2024-09-13T14:52:13.655419+0000 osd.3 (osd.3) 18 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:15.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:15.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:15.175 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:15.202 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:15.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:15.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:15.332 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:15.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:15.436 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:15.458 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:15.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:15.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:15.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:15.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:15 smithi177 bash[68594]: cluster 2024-09-13T14:52:13.805403+0000 osd.4 (osd.4) 16 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:15.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:15 smithi177 bash[68594]: cluster 2024-09-13T14:52:13.805403+0000 osd.4 (osd.4) 16 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:15.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:15 smithi177 bash[68594]: cluster 2024-09-13T14:52:14.592123+0000 mgr.y (mgr.26252) 779 : cluster [DBG] pgmap v1270: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 36 MiB/s wr, 1.15k op/s; 3904/210 objects degraded (1859.048%); 998 KiB/s, 330 objects/s recovering 2024-09-13T14:52:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:15 smithi177 bash[68594]: cluster 2024-09-13T14:52:14.592123+0000 mgr.y (mgr.26252) 779 : cluster [DBG] pgmap v1270: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 36 MiB/s wr, 1.15k op/s; 3904/210 objects degraded (1859.048%); 998 KiB/s, 330 objects/s recovering 2024-09-13T14:52:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:15 smithi177 bash[68594]: cluster 2024-09-13T14:52:14.662178+0000 osd.3 (osd.3) 19 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:15 smithi177 bash[68594]: cluster 2024-09-13T14:52:14.662178+0000 osd.3 (osd.3) 19 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:15.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:15.698 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[84576]: cluster 2024-09-13T14:52:13.805403+0000 osd.4 (osd.4) 16 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[84576]: cluster 2024-09-13T14:52:13.805403+0000 osd.4 (osd.4) 16 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[84576]: cluster 2024-09-13T14:52:14.592123+0000 mgr.y (mgr.26252) 779 : cluster [DBG] pgmap v1270: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 36 MiB/s wr, 1.15k op/s; 3904/210 objects degraded (1859.048%); 998 KiB/s, 330 objects/s recovering 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[84576]: cluster 2024-09-13T14:52:14.592123+0000 mgr.y (mgr.26252) 779 : cluster [DBG] pgmap v1270: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 36 MiB/s wr, 1.15k op/s; 3904/210 objects degraded (1859.048%); 998 KiB/s, 330 objects/s recovering 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[84576]: cluster 2024-09-13T14:52:14.662178+0000 osd.3 (osd.3) 19 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[84576]: cluster 2024-09-13T14:52:14.662178+0000 osd.3 (osd.3) 19 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[87147]: cluster 2024-09-13T14:52:13.805403+0000 osd.4 (osd.4) 16 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[87147]: cluster 2024-09-13T14:52:13.805403+0000 osd.4 (osd.4) 16 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[87147]: cluster 2024-09-13T14:52:14.592123+0000 mgr.y (mgr.26252) 779 : cluster [DBG] pgmap v1270: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 36 MiB/s wr, 1.15k op/s; 3904/210 objects degraded (1859.048%); 998 KiB/s, 330 objects/s recovering 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[87147]: cluster 2024-09-13T14:52:14.592123+0000 mgr.y (mgr.26252) 779 : cluster [DBG] pgmap v1270: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 36 MiB/s wr, 1.15k op/s; 3904/210 objects degraded (1859.048%); 998 KiB/s, 330 objects/s recovering 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[87147]: cluster 2024-09-13T14:52:14.662178+0000 osd.3 (osd.3) 19 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:15.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:15 smithi099 bash[87147]: cluster 2024-09-13T14:52:14.662178+0000 osd.3 (osd.3) 19 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:15.713 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:15.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:15.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:15.844 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:15.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:15.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:15.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:16.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:16.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:16.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:16.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:16.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:16.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:16.368 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:16.470 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:16.482 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:16 smithi177 bash[68594]: cluster 2024-09-13T14:52:15.571829+0000 osd.0 (osd.0) 6 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:16.482 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:16 smithi177 bash[68594]: cluster 2024-09-13T14:52:15.571829+0000 osd.0 (osd.0) 6 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:16.482 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:16 smithi177 bash[68594]: cluster 2024-09-13T14:52:15.661736+0000 osd.3 (osd.3) 20 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:16.483 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:16 smithi177 bash[68594]: cluster 2024-09-13T14:52:15.661736+0000 osd.3 (osd.3) 20 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:16.483 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:16 smithi177 bash[68594]: cluster 2024-09-13T14:52:16.360471+0000 mon.a (mon.0) 1801 : cluster [WRN] Health check update: Degraded data redundancy: 3904/210 objects degraded (1859.048%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:16.483 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:16 smithi177 bash[68594]: cluster 2024-09-13T14:52:16.360471+0000 mon.a (mon.0) 1801 : cluster [WRN] Health check update: Degraded data redundancy: 3904/210 objects degraded (1859.048%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:16.497 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:16.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:16.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:16.626 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:16.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:16.728 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:16.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[84576]: cluster 2024-09-13T14:52:15.571829+0000 osd.0 (osd.0) 6 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:16.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[84576]: cluster 2024-09-13T14:52:15.571829+0000 osd.0 (osd.0) 6 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:16.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[84576]: cluster 2024-09-13T14:52:15.661736+0000 osd.3 (osd.3) 20 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:16.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[84576]: cluster 2024-09-13T14:52:15.661736+0000 osd.3 (osd.3) 20 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:16.741 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[84576]: cluster 2024-09-13T14:52:16.360471+0000 mon.a (mon.0) 1801 : cluster [WRN] Health check update: Degraded data redundancy: 3904/210 objects degraded (1859.048%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:16.741 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[84576]: cluster 2024-09-13T14:52:16.360471+0000 mon.a (mon.0) 1801 : cluster [WRN] Health check update: Degraded data redundancy: 3904/210 objects degraded (1859.048%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:16.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[87147]: cluster 2024-09-13T14:52:15.571829+0000 osd.0 (osd.0) 6 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:16.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[87147]: cluster 2024-09-13T14:52:15.571829+0000 osd.0 (osd.0) 6 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:16.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[87147]: cluster 2024-09-13T14:52:15.661736+0000 osd.3 (osd.3) 20 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:16.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[87147]: cluster 2024-09-13T14:52:15.661736+0000 osd.3 (osd.3) 20 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:16.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[87147]: cluster 2024-09-13T14:52:16.360471+0000 mon.a (mon.0) 1801 : cluster [WRN] Health check update: Degraded data redundancy: 3904/210 objects degraded (1859.048%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:16.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:16 smithi099 bash[87147]: cluster 2024-09-13T14:52:16.360471+0000 mon.a (mon.0) 1801 : cluster [WRN] Health check update: Degraded data redundancy: 3904/210 objects degraded (1859.048%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:52:16.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:16.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:16.858 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:16.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:16.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:17.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:17.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:17.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:17.140 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:17.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:17.243 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:17.252 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:52:17] "GET /metrics HTTP/1.1" 200 36751 "" "Prometheus/2.51.0" 2024-09-13T14:52:17.267 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:17.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:17.370 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:17.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:17.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:17.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:17.518 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:17.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[84576]: cluster 2024-09-13T14:52:15.701902+0000 osd.2 (osd.2) 9 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:17.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[84576]: cluster 2024-09-13T14:52:15.701902+0000 osd.2 (osd.2) 9 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:17.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[84576]: cluster 2024-09-13T14:52:15.818840+0000 osd.4 (osd.4) 17 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:17.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[84576]: cluster 2024-09-13T14:52:15.818840+0000 osd.4 (osd.4) 17 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:17.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[84576]: cluster 2024-09-13T14:52:16.592838+0000 mgr.y (mgr.26252) 780 : cluster [DBG] pgmap v1271: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 35 MiB/s wr, 1.12k op/s; 3904/210 objects degraded (1859.048%); 971 KiB/s, 150 objects/s recovering 2024-09-13T14:52:17.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[84576]: cluster 2024-09-13T14:52:16.592838+0000 mgr.y (mgr.26252) 780 : cluster [DBG] pgmap v1271: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 35 MiB/s wr, 1.12k op/s; 3904/210 objects degraded (1859.048%); 971 KiB/s, 150 objects/s recovering 2024-09-13T14:52:17.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[84576]: cluster 2024-09-13T14:52:16.621361+0000 osd.3 (osd.3) 21 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:17.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[84576]: cluster 2024-09-13T14:52:16.621361+0000 osd.3 (osd.3) 21 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:17.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[87147]: cluster 2024-09-13T14:52:15.701902+0000 osd.2 (osd.2) 9 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:17.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[87147]: cluster 2024-09-13T14:52:15.701902+0000 osd.2 (osd.2) 9 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:17.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[87147]: cluster 2024-09-13T14:52:15.818840+0000 osd.4 (osd.4) 17 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:17.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[87147]: cluster 2024-09-13T14:52:15.818840+0000 osd.4 (osd.4) 17 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:17.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[87147]: cluster 2024-09-13T14:52:16.592838+0000 mgr.y (mgr.26252) 780 : cluster [DBG] pgmap v1271: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 35 MiB/s wr, 1.12k op/s; 3904/210 objects degraded (1859.048%); 971 KiB/s, 150 objects/s recovering 2024-09-13T14:52:17.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[87147]: cluster 2024-09-13T14:52:16.592838+0000 mgr.y (mgr.26252) 780 : cluster [DBG] pgmap v1271: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 35 MiB/s wr, 1.12k op/s; 3904/210 objects degraded (1859.048%); 971 KiB/s, 150 objects/s recovering 2024-09-13T14:52:17.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[87147]: cluster 2024-09-13T14:52:16.621361+0000 osd.3 (osd.3) 21 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:17.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:17 smithi099 bash[87147]: cluster 2024-09-13T14:52:16.621361+0000 osd.3 (osd.3) 21 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:17.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:17.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:17.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:17.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:17.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:17 smithi177 bash[68594]: cluster 2024-09-13T14:52:15.701902+0000 osd.2 (osd.2) 9 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:17.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:17 smithi177 bash[68594]: cluster 2024-09-13T14:52:15.701902+0000 osd.2 (osd.2) 9 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:17.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:17 smithi177 bash[68594]: cluster 2024-09-13T14:52:15.818840+0000 osd.4 (osd.4) 17 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:17.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:17 smithi177 bash[68594]: cluster 2024-09-13T14:52:15.818840+0000 osd.4 (osd.4) 17 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:17.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:17 smithi177 bash[68594]: cluster 2024-09-13T14:52:16.592838+0000 mgr.y (mgr.26252) 780 : cluster [DBG] pgmap v1271: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 35 MiB/s wr, 1.12k op/s; 3904/210 objects degraded (1859.048%); 971 KiB/s, 150 objects/s recovering 2024-09-13T14:52:17.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:17 smithi177 bash[68594]: cluster 2024-09-13T14:52:16.592838+0000 mgr.y (mgr.26252) 780 : cluster [DBG] pgmap v1271: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+scrubbing, 1 active+recovering, 96 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 35 MiB/s wr, 1.12k op/s; 3904/210 objects degraded (1859.048%); 971 KiB/s, 150 objects/s recovering 2024-09-13T14:52:17.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:17 smithi177 bash[68594]: cluster 2024-09-13T14:52:16.621361+0000 osd.3 (osd.3) 21 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:17.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:17 smithi177 bash[68594]: cluster 2024-09-13T14:52:16.621361+0000 osd.3 (osd.3) 21 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:17.777 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:17.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:17.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:17.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:18.009 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:18.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:18.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:18.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:18.168 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:18.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:18.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:18.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:18.402 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:18.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:18.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:18.532 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:18.552 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:18.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:18.658 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:18.667 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[84576]: cluster 2024-09-13T14:52:16.720910+0000 osd.2 (osd.2) 10 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:18.667 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[84576]: cluster 2024-09-13T14:52:16.720910+0000 osd.2 (osd.2) 10 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:18.667 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[84576]: cluster 2024-09-13T14:52:16.807283+0000 osd.4 (osd.4) 18 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:18.667 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[84576]: cluster 2024-09-13T14:52:16.807283+0000 osd.4 (osd.4) 18 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[84576]: cluster 2024-09-13T14:52:17.570705+0000 osd.0 (osd.0) 7 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[84576]: cluster 2024-09-13T14:52:17.570705+0000 osd.0 (osd.0) 7 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[84576]: cluster 2024-09-13T14:52:17.604829+0000 osd.3 (osd.3) 22 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[84576]: cluster 2024-09-13T14:52:17.604829+0000 osd.3 (osd.3) 22 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[84576]: audit 2024-09-13T14:52:18.279446+0000 mon.a (mon.0) 1802 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[84576]: audit 2024-09-13T14:52:18.279446+0000 mon.a (mon.0) 1802 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[87147]: cluster 2024-09-13T14:52:16.720910+0000 osd.2 (osd.2) 10 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[87147]: cluster 2024-09-13T14:52:16.720910+0000 osd.2 (osd.2) 10 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[87147]: cluster 2024-09-13T14:52:16.807283+0000 osd.4 (osd.4) 18 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[87147]: cluster 2024-09-13T14:52:16.807283+0000 osd.4 (osd.4) 18 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:18.668 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[87147]: cluster 2024-09-13T14:52:17.570705+0000 osd.0 (osd.0) 7 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:18.669 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[87147]: cluster 2024-09-13T14:52:17.570705+0000 osd.0 (osd.0) 7 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:18.669 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[87147]: cluster 2024-09-13T14:52:17.604829+0000 osd.3 (osd.3) 22 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:18.669 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[87147]: cluster 2024-09-13T14:52:17.604829+0000 osd.3 (osd.3) 22 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:18.669 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[87147]: audit 2024-09-13T14:52:18.279446+0000 mon.a (mon.0) 1802 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:18.669 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:18 smithi099 bash[87147]: audit 2024-09-13T14:52:18.279446+0000 mon.a (mon.0) 1802 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:18.679 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:18.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:18.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:18.804 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:18.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:18 smithi177 bash[68594]: cluster 2024-09-13T14:52:16.720910+0000 osd.2 (osd.2) 10 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:18 smithi177 bash[68594]: cluster 2024-09-13T14:52:16.720910+0000 osd.2 (osd.2) 10 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:18 smithi177 bash[68594]: cluster 2024-09-13T14:52:16.807283+0000 osd.4 (osd.4) 18 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:18 smithi177 bash[68594]: cluster 2024-09-13T14:52:16.807283+0000 osd.4 (osd.4) 18 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:18 smithi177 bash[68594]: cluster 2024-09-13T14:52:17.570705+0000 osd.0 (osd.0) 7 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:18.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:18 smithi177 bash[68594]: cluster 2024-09-13T14:52:17.570705+0000 osd.0 (osd.0) 7 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:18.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:18 smithi177 bash[68594]: cluster 2024-09-13T14:52:17.604829+0000 osd.3 (osd.3) 22 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:18.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:18 smithi177 bash[68594]: cluster 2024-09-13T14:52:17.604829+0000 osd.3 (osd.3) 22 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:18.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:18 smithi177 bash[68594]: audit 2024-09-13T14:52:18.279446+0000 mon.a (mon.0) 1802 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:18.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:18 smithi177 bash[68594]: audit 2024-09-13T14:52:18.279446+0000 mon.a (mon.0) 1802 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:18.906 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:18.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:18.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:19.038 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:19.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:19.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:19.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:19.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:19.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:19.299 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:19.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:19.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:19.448 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:19.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:19.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:19.574 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:19.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:19 smithi177 bash[68594]: cluster 2024-09-13T14:52:18.571887+0000 osd.3 (osd.3) 23 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:19 smithi177 bash[68594]: cluster 2024-09-13T14:52:18.571887+0000 osd.3 (osd.3) 23 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:19 smithi177 bash[68594]: cluster 2024-09-13T14:52:18.593482+0000 mgr.y (mgr.26252) 781 : cluster [DBG] pgmap v1272: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 29 MiB/s wr, 918 op/s; 3904/210 objects degraded (1859.048%); 798 KiB/s, 281 objects/s recovering 2024-09-13T14:52:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:19 smithi177 bash[68594]: cluster 2024-09-13T14:52:18.593482+0000 mgr.y (mgr.26252) 781 : cluster [DBG] pgmap v1272: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 29 MiB/s wr, 918 op/s; 3904/210 objects degraded (1859.048%); 798 KiB/s, 281 objects/s recovering 2024-09-13T14:52:19.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:19.701 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:19.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:19.755 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:19.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:19 smithi099 bash[84576]: cluster 2024-09-13T14:52:18.571887+0000 osd.3 (osd.3) 23 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:19 smithi099 bash[84576]: cluster 2024-09-13T14:52:18.571887+0000 osd.3 (osd.3) 23 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:19 smithi099 bash[84576]: cluster 2024-09-13T14:52:18.593482+0000 mgr.y (mgr.26252) 781 : cluster [DBG] pgmap v1272: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 29 MiB/s wr, 918 op/s; 3904/210 objects degraded (1859.048%); 798 KiB/s, 281 objects/s recovering 2024-09-13T14:52:19.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:19 smithi099 bash[84576]: cluster 2024-09-13T14:52:18.593482+0000 mgr.y (mgr.26252) 781 : cluster [DBG] pgmap v1272: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 29 MiB/s wr, 918 op/s; 3904/210 objects degraded (1859.048%); 798 KiB/s, 281 objects/s recovering 2024-09-13T14:52:19.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:19 smithi099 bash[87147]: cluster 2024-09-13T14:52:18.571887+0000 osd.3 (osd.3) 23 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:19.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:19 smithi099 bash[87147]: cluster 2024-09-13T14:52:18.571887+0000 osd.3 (osd.3) 23 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:19.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:19 smithi099 bash[87147]: cluster 2024-09-13T14:52:18.593482+0000 mgr.y (mgr.26252) 781 : cluster [DBG] pgmap v1272: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 29 MiB/s wr, 918 op/s; 3904/210 objects degraded (1859.048%); 798 KiB/s, 281 objects/s recovering 2024-09-13T14:52:19.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:19 smithi099 bash[87147]: cluster 2024-09-13T14:52:18.593482+0000 mgr.y (mgr.26252) 781 : cluster [DBG] pgmap v1272: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 29 MiB/s wr, 918 op/s; 3904/210 objects degraded (1859.048%); 798 KiB/s, 281 objects/s recovering 2024-09-13T14:52:19.832 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:19.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:19.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:19.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:20.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:20.092 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:20.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:20.195 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:20.224 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:20.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:20.327 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:20.338 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:52:20 smithi177 bash[61439]: ts=2024-09-13T14:52:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:20.355 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:20.459 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:20.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:20.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:20.593 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:20.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:20 smithi177 bash[68594]: cluster 2024-09-13T14:52:18.783633+0000 osd.4 (osd.4) 19 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:20.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:20 smithi177 bash[68594]: cluster 2024-09-13T14:52:18.783633+0000 osd.4 (osd.4) 19 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:20.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:20 smithi177 bash[68594]: cluster 2024-09-13T14:52:19.595879+0000 osd.3 (osd.3) 24 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:20.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:20 smithi177 bash[68594]: cluster 2024-09-13T14:52:19.595879+0000 osd.3 (osd.3) 24 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:20.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:20.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:20.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:20.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:20.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:20 smithi099 bash[84576]: cluster 2024-09-13T14:52:18.783633+0000 osd.4 (osd.4) 19 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:20 smithi099 bash[84576]: cluster 2024-09-13T14:52:18.783633+0000 osd.4 (osd.4) 19 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:20 smithi099 bash[84576]: cluster 2024-09-13T14:52:19.595879+0000 osd.3 (osd.3) 24 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:20 smithi099 bash[84576]: cluster 2024-09-13T14:52:19.595879+0000 osd.3 (osd.3) 24 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:20 smithi099 bash[87147]: cluster 2024-09-13T14:52:18.783633+0000 osd.4 (osd.4) 19 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:20 smithi099 bash[87147]: cluster 2024-09-13T14:52:18.783633+0000 osd.4 (osd.4) 19 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:20 smithi099 bash[87147]: cluster 2024-09-13T14:52:19.595879+0000 osd.3 (osd.3) 24 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:20 smithi099 bash[87147]: cluster 2024-09-13T14:52:19.595879+0000 osd.3 (osd.3) 24 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:20.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:20.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:20.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:20.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:21.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:21.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:21.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:21.140 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:21.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:21.243 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:21.269 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:21.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:21.371 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:21.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:21.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:21.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:21.510 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[87147]: cluster 2024-09-13T14:52:19.825286+0000 osd.4 (osd.4) 20 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:21.510 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[87147]: cluster 2024-09-13T14:52:19.825286+0000 osd.4 (osd.4) 20 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:21.510 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[87147]: cluster 2024-09-13T14:52:20.563677+0000 osd.3 (osd.3) 25 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:21.510 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[87147]: cluster 2024-09-13T14:52:20.563677+0000 osd.3 (osd.3) 25 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[87147]: cluster 2024-09-13T14:52:20.578810+0000 osd.0 (osd.0) 8 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[87147]: cluster 2024-09-13T14:52:20.578810+0000 osd.0 (osd.0) 8 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[87147]: cluster 2024-09-13T14:52:20.594772+0000 mgr.y (mgr.26252) 782 : cluster [DBG] pgmap v1273: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 3904/210 objects degraded (1859.048%); 0 B/s, 239 objects/s recovering 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[87147]: cluster 2024-09-13T14:52:20.594772+0000 mgr.y (mgr.26252) 782 : cluster [DBG] pgmap v1273: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 3904/210 objects degraded (1859.048%); 0 B/s, 239 objects/s recovering 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[84576]: cluster 2024-09-13T14:52:19.825286+0000 osd.4 (osd.4) 20 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[84576]: cluster 2024-09-13T14:52:19.825286+0000 osd.4 (osd.4) 20 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[84576]: cluster 2024-09-13T14:52:20.563677+0000 osd.3 (osd.3) 25 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[84576]: cluster 2024-09-13T14:52:20.563677+0000 osd.3 (osd.3) 25 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[84576]: cluster 2024-09-13T14:52:20.578810+0000 osd.0 (osd.0) 8 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[84576]: cluster 2024-09-13T14:52:20.578810+0000 osd.0 (osd.0) 8 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:21.511 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[84576]: cluster 2024-09-13T14:52:20.594772+0000 mgr.y (mgr.26252) 782 : cluster [DBG] pgmap v1273: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 3904/210 objects degraded (1859.048%); 0 B/s, 239 objects/s recovering 2024-09-13T14:52:21.512 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:21 smithi099 bash[84576]: cluster 2024-09-13T14:52:20.594772+0000 mgr.y (mgr.26252) 782 : cluster [DBG] pgmap v1273: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 3904/210 objects degraded (1859.048%); 0 B/s, 239 objects/s recovering 2024-09-13T14:52:21.524 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:21.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:21.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:21.649 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:21.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:21.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:21.763 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:21 smithi177 bash[68594]: cluster 2024-09-13T14:52:19.825286+0000 osd.4 (osd.4) 20 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:21.763 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:21 smithi177 bash[68594]: cluster 2024-09-13T14:52:19.825286+0000 osd.4 (osd.4) 20 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:21.763 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:21 smithi177 bash[68594]: cluster 2024-09-13T14:52:20.563677+0000 osd.3 (osd.3) 25 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:21.763 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:21 smithi177 bash[68594]: cluster 2024-09-13T14:52:20.563677+0000 osd.3 (osd.3) 25 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:21.763 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:21 smithi177 bash[68594]: cluster 2024-09-13T14:52:20.578810+0000 osd.0 (osd.0) 8 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:21.763 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:21 smithi177 bash[68594]: cluster 2024-09-13T14:52:20.578810+0000 osd.0 (osd.0) 8 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:21.764 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:21 smithi177 bash[68594]: cluster 2024-09-13T14:52:20.594772+0000 mgr.y (mgr.26252) 782 : cluster [DBG] pgmap v1273: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 3904/210 objects degraded (1859.048%); 0 B/s, 239 objects/s recovering 2024-09-13T14:52:21.764 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:21 smithi177 bash[68594]: cluster 2024-09-13T14:52:20.594772+0000 mgr.y (mgr.26252) 782 : cluster [DBG] pgmap v1273: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 3904/210 objects degraded (1859.048%); 0 B/s, 239 objects/s recovering 2024-09-13T14:52:21.780 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:21.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:21.883 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:21.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:22.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:22.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:22.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:22.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:22.161 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:22.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:22.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:22.292 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:22.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:22.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:22.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:22.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:22.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:22.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:22 smithi177 bash[68594]: cluster 2024-09-13T14:52:21.525526+0000 osd.3 (osd.3) 26 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:22.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:22 smithi177 bash[68594]: cluster 2024-09-13T14:52:21.525526+0000 osd.3 (osd.3) 26 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:22.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:22.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:22.658 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:22.684 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:22.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:22.705 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:22 smithi099 bash[84576]: cluster 2024-09-13T14:52:21.525526+0000 osd.3 (osd.3) 26 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:22.705 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:22 smithi099 bash[84576]: cluster 2024-09-13T14:52:21.525526+0000 osd.3 (osd.3) 26 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:22.705 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:22 smithi099 bash[87147]: cluster 2024-09-13T14:52:21.525526+0000 osd.3 (osd.3) 26 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:22.705 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:22 smithi099 bash[87147]: cluster 2024-09-13T14:52:21.525526+0000 osd.3 (osd.3) 26 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:22.787 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:22.816 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:22.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:22.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:22.948 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:22.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:23.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:23.079 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:23.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:23.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:23.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:23.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:23.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:23.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:23.446 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:23.473 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:23.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:23.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:23.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:23.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:23.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:23.721 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:23 smithi177 bash[68594]: cluster 2024-09-13T14:52:21.688657+0000 osd.2 (osd.2) 11 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:23.721 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:23 smithi177 bash[68594]: cluster 2024-09-13T14:52:21.688657+0000 osd.2 (osd.2) 11 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:23.721 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:23 smithi177 bash[68594]: cluster 2024-09-13T14:52:21.757971+0000 osd.4 (osd.4) 21 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:23.721 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:23 smithi177 bash[68594]: cluster 2024-09-13T14:52:21.757971+0000 osd.4 (osd.4) 21 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:23.722 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:23 smithi177 bash[68594]: cluster 2024-09-13T14:52:22.515436+0000 osd.3 (osd.3) 27 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:23.722 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:23 smithi177 bash[68594]: cluster 2024-09-13T14:52:22.515436+0000 osd.3 (osd.3) 27 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:23.722 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:23 smithi177 bash[68594]: cluster 2024-09-13T14:52:22.595821+0000 mgr.y (mgr.26252) 783 : cluster [DBG] pgmap v1274: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 293 objects/s recovering 2024-09-13T14:52:23.722 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:23 smithi177 bash[68594]: cluster 2024-09-13T14:52:22.595821+0000 mgr.y (mgr.26252) 783 : cluster [DBG] pgmap v1274: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 293 objects/s recovering 2024-09-13T14:52:23.722 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:23 smithi177 bash[68594]: cluster 2024-09-13T14:52:22.636355+0000 osd.0 (osd.0) 9 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:23.722 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:23 smithi177 bash[68594]: cluster 2024-09-13T14:52:22.636355+0000 osd.0 (osd.0) 9 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:23.736 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:23.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[84576]: cluster 2024-09-13T14:52:21.688657+0000 osd.2 (osd.2) 11 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[84576]: cluster 2024-09-13T14:52:21.688657+0000 osd.2 (osd.2) 11 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[84576]: cluster 2024-09-13T14:52:21.757971+0000 osd.4 (osd.4) 21 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[84576]: cluster 2024-09-13T14:52:21.757971+0000 osd.4 (osd.4) 21 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:23.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[84576]: cluster 2024-09-13T14:52:22.515436+0000 osd.3 (osd.3) 27 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[84576]: cluster 2024-09-13T14:52:22.515436+0000 osd.3 (osd.3) 27 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[84576]: cluster 2024-09-13T14:52:22.595821+0000 mgr.y (mgr.26252) 783 : cluster [DBG] pgmap v1274: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 293 objects/s recovering 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[84576]: cluster 2024-09-13T14:52:22.595821+0000 mgr.y (mgr.26252) 783 : cluster [DBG] pgmap v1274: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 293 objects/s recovering 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[84576]: cluster 2024-09-13T14:52:22.636355+0000 osd.0 (osd.0) 9 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[84576]: cluster 2024-09-13T14:52:22.636355+0000 osd.0 (osd.0) 9 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[87147]: cluster 2024-09-13T14:52:21.688657+0000 osd.2 (osd.2) 11 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[87147]: cluster 2024-09-13T14:52:21.688657+0000 osd.2 (osd.2) 11 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[87147]: cluster 2024-09-13T14:52:21.757971+0000 osd.4 (osd.4) 21 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[87147]: cluster 2024-09-13T14:52:21.757971+0000 osd.4 (osd.4) 21 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[87147]: cluster 2024-09-13T14:52:22.515436+0000 osd.3 (osd.3) 27 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:23.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[87147]: cluster 2024-09-13T14:52:22.515436+0000 osd.3 (osd.3) 27 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:23.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[87147]: cluster 2024-09-13T14:52:22.595821+0000 mgr.y (mgr.26252) 783 : cluster [DBG] pgmap v1274: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 293 objects/s recovering 2024-09-13T14:52:23.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[87147]: cluster 2024-09-13T14:52:22.595821+0000 mgr.y (mgr.26252) 783 : cluster [DBG] pgmap v1274: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 293 objects/s recovering 2024-09-13T14:52:23.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[87147]: cluster 2024-09-13T14:52:22.636355+0000 osd.0 (osd.0) 9 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:23.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:23 smithi099 bash[87147]: cluster 2024-09-13T14:52:22.636355+0000 osd.0 (osd.0) 9 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:23.839 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:23.862 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:23.964 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:23.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:23.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:24.096 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:24.124 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:24.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:24.227 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:24.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:24.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:24.386 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:24.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:24.511 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:24.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:24.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:24 smithi177 bash[68594]: cluster 2024-09-13T14:52:22.679424+0000 osd.2 (osd.2) 12 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:24.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:24 smithi177 bash[68594]: cluster 2024-09-13T14:52:22.679424+0000 osd.2 (osd.2) 12 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:24.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:24 smithi177 bash[68594]: cluster 2024-09-13T14:52:22.746733+0000 osd.4 (osd.4) 22 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:24 smithi177 bash[68594]: cluster 2024-09-13T14:52:22.746733+0000 osd.4 (osd.4) 22 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:24 smithi177 bash[68594]: cluster 2024-09-13T14:52:23.443920+0000 mon.a (mon.0) 1803 : cluster [WRN] Health check update: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:24 smithi177 bash[68594]: cluster 2024-09-13T14:52:23.443920+0000 mon.a (mon.0) 1803 : cluster [WRN] Health check update: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:24 smithi177 bash[68594]: cluster 2024-09-13T14:52:23.545227+0000 osd.3 (osd.3) 28 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:24 smithi177 bash[68594]: cluster 2024-09-13T14:52:23.545227+0000 osd.3 (osd.3) 28 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:24.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:24.644 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:24.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:24.747 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:24.770 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:24.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:24.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:24.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:25.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:25.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:25.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:25.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:25.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:25.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:25.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[87147]: cluster 2024-09-13T14:52:22.679424+0000 osd.2 (osd.2) 12 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:25.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[87147]: cluster 2024-09-13T14:52:22.679424+0000 osd.2 (osd.2) 12 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:25.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[87147]: cluster 2024-09-13T14:52:22.746733+0000 osd.4 (osd.4) 22 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:25.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[87147]: cluster 2024-09-13T14:52:22.746733+0000 osd.4 (osd.4) 22 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:25.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[87147]: cluster 2024-09-13T14:52:23.443920+0000 mon.a (mon.0) 1803 : cluster [WRN] Health check update: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:25.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[87147]: cluster 2024-09-13T14:52:23.443920+0000 mon.a (mon.0) 1803 : cluster [WRN] Health check update: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:25.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[87147]: cluster 2024-09-13T14:52:23.545227+0000 osd.3 (osd.3) 28 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:25.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[87147]: cluster 2024-09-13T14:52:23.545227+0000 osd.3 (osd.3) 28 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:25.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[84576]: cluster 2024-09-13T14:52:22.679424+0000 osd.2 (osd.2) 12 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:25.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[84576]: cluster 2024-09-13T14:52:22.679424+0000 osd.2 (osd.2) 12 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:25.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[84576]: cluster 2024-09-13T14:52:22.746733+0000 osd.4 (osd.4) 22 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:25.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[84576]: cluster 2024-09-13T14:52:22.746733+0000 osd.4 (osd.4) 22 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:25.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[84576]: cluster 2024-09-13T14:52:23.443920+0000 mon.a (mon.0) 1803 : cluster [WRN] Health check update: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:25.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[84576]: cluster 2024-09-13T14:52:23.443920+0000 mon.a (mon.0) 1803 : cluster [WRN] Health check update: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:52:25.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[84576]: cluster 2024-09-13T14:52:23.545227+0000 osd.3 (osd.3) 28 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:25.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:24 smithi099 bash[84576]: cluster 2024-09-13T14:52:23.545227+0000 osd.3 (osd.3) 28 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:25.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:25.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:25.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:25.419 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:25.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:25.522 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:25.530 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:25 smithi099 bash[87147]: cluster 2024-09-13T14:52:24.526364+0000 osd.3 (osd.3) 29 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:25.531 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:25 smithi099 bash[87147]: cluster 2024-09-13T14:52:24.526364+0000 osd.3 (osd.3) 29 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:25.531 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:25 smithi099 bash[87147]: cluster 2024-09-13T14:52:24.597265+0000 mgr.y (mgr.26252) 784 : cluster [DBG] pgmap v1275: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:25.531 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:25 smithi099 bash[87147]: cluster 2024-09-13T14:52:24.597265+0000 mgr.y (mgr.26252) 784 : cluster [DBG] pgmap v1275: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:25.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:25 smithi099 bash[84576]: cluster 2024-09-13T14:52:24.526364+0000 osd.3 (osd.3) 29 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:25.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:25 smithi099 bash[84576]: cluster 2024-09-13T14:52:24.526364+0000 osd.3 (osd.3) 29 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:25.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:25 smithi099 bash[84576]: cluster 2024-09-13T14:52:24.597265+0000 mgr.y (mgr.26252) 784 : cluster [DBG] pgmap v1275: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:25.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:25 smithi099 bash[84576]: cluster 2024-09-13T14:52:24.597265+0000 mgr.y (mgr.26252) 784 : cluster [DBG] pgmap v1275: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:25.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:25.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:25.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:25.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:25.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:25.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:25.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:25 smithi177 bash[68594]: cluster 2024-09-13T14:52:24.526364+0000 osd.3 (osd.3) 29 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:25.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:25 smithi177 bash[68594]: cluster 2024-09-13T14:52:24.526364+0000 osd.3 (osd.3) 29 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:25.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:25 smithi177 bash[68594]: cluster 2024-09-13T14:52:24.597265+0000 mgr.y (mgr.26252) 784 : cluster [DBG] pgmap v1275: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:25.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:25 smithi177 bash[68594]: cluster 2024-09-13T14:52:24.597265+0000 mgr.y (mgr.26252) 784 : cluster [DBG] pgmap v1275: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:25.804 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:25.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:25.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:25.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:25.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:26.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:26.065 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:26.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:26.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:26.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:26.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:26.294 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:26.316 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:26.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:26.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:26.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:26.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:26.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:26.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:26.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:26.679 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:26.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:26.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:26.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:26.816 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:26 smithi177 bash[68594]: cluster 2024-09-13T14:52:24.747467+0000 osd.4 (osd.4) 23 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:26.816 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:26 smithi177 bash[68594]: cluster 2024-09-13T14:52:24.747467+0000 osd.4 (osd.4) 23 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:26.816 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:26 smithi177 bash[68594]: cluster 2024-09-13T14:52:25.485788+0000 osd.3 (osd.3) 30 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:26.816 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:26 smithi177 bash[68594]: cluster 2024-09-13T14:52:25.485788+0000 osd.3 (osd.3) 30 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:26.832 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:26.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:26.889 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:26 smithi099 bash[84576]: cluster 2024-09-13T14:52:24.747467+0000 osd.4 (osd.4) 23 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:26.889 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:26 smithi099 bash[84576]: cluster 2024-09-13T14:52:24.747467+0000 osd.4 (osd.4) 23 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:26.889 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:26 smithi099 bash[84576]: cluster 2024-09-13T14:52:25.485788+0000 osd.3 (osd.3) 30 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:26.889 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:26 smithi099 bash[84576]: cluster 2024-09-13T14:52:25.485788+0000 osd.3 (osd.3) 30 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:26.889 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:26 smithi099 bash[87147]: cluster 2024-09-13T14:52:24.747467+0000 osd.4 (osd.4) 23 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:26.889 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:26 smithi099 bash[87147]: cluster 2024-09-13T14:52:24.747467+0000 osd.4 (osd.4) 23 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:26.889 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:26 smithi099 bash[87147]: cluster 2024-09-13T14:52:25.485788+0000 osd.3 (osd.3) 30 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:26.889 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:26 smithi099 bash[87147]: cluster 2024-09-13T14:52:25.485788+0000 osd.3 (osd.3) 30 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:26.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:26.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:26.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:27.060 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:27.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:27.217 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:27.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:27.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:27.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:52:27] "GET /metrics HTTP/1.1" 200 36765 "" "Prometheus/2.51.0" 2024-09-13T14:52:27.348 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:27.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:27.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:27.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:27.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:27.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:27.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:27.719 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[84576]: cluster 2024-09-13T14:52:25.753122+0000 osd.4 (osd.4) 24 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:27.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[87147]: cluster 2024-09-13T14:52:25.753122+0000 osd.4 (osd.4) 24 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:27.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[87147]: cluster 2024-09-13T14:52:25.753122+0000 osd.4 (osd.4) 24 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:27.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[87147]: cluster 2024-09-13T14:52:26.481551+0000 osd.3 (osd.3) 31 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:27.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[87147]: cluster 2024-09-13T14:52:26.481551+0000 osd.3 (osd.3) 31 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:27.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[87147]: cluster 2024-09-13T14:52:26.521512+0000 osd.0 (osd.0) 10 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:27.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[87147]: cluster 2024-09-13T14:52:26.521512+0000 osd.0 (osd.0) 10 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:27.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[87147]: cluster 2024-09-13T14:52:26.597797+0000 mgr.y (mgr.26252) 785 : cluster [DBG] pgmap v1276: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:27.720 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[87147]: cluster 2024-09-13T14:52:26.597797+0000 mgr.y (mgr.26252) 785 : cluster [DBG] pgmap v1276: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:27.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:27 smithi177 bash[68594]: cluster 2024-09-13T14:52:25.753122+0000 osd.4 (osd.4) 24 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:27.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:27 smithi177 bash[68594]: cluster 2024-09-13T14:52:25.753122+0000 osd.4 (osd.4) 24 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:27.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:27 smithi177 bash[68594]: cluster 2024-09-13T14:52:26.481551+0000 osd.3 (osd.3) 31 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:27.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:27 smithi177 bash[68594]: cluster 2024-09-13T14:52:26.481551+0000 osd.3 (osd.3) 31 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:27 smithi177 bash[68594]: cluster 2024-09-13T14:52:26.521512+0000 osd.0 (osd.0) 10 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:27 smithi177 bash[68594]: cluster 2024-09-13T14:52:26.521512+0000 osd.0 (osd.0) 10 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:27 smithi177 bash[68594]: cluster 2024-09-13T14:52:26.597797+0000 mgr.y (mgr.26252) 785 : cluster [DBG] pgmap v1276: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:27 smithi177 bash[68594]: cluster 2024-09-13T14:52:26.597797+0000 mgr.y (mgr.26252) 785 : cluster [DBG] pgmap v1276: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:27.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:27.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:27.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:27.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:28.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:28.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:28.022 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[84576]: cluster 2024-09-13T14:52:25.753122+0000 osd.4 (osd.4) 24 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:28.022 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[84576]: cluster 2024-09-13T14:52:26.481551+0000 osd.3 (osd.3) 31 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:28.022 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[84576]: cluster 2024-09-13T14:52:26.481551+0000 osd.3 (osd.3) 31 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:28.022 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[84576]: cluster 2024-09-13T14:52:26.521512+0000 osd.0 (osd.0) 10 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:28.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[84576]: cluster 2024-09-13T14:52:26.521512+0000 osd.0 (osd.0) 10 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:28.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[84576]: cluster 2024-09-13T14:52:26.597797+0000 mgr.y (mgr.26252) 785 : cluster [DBG] pgmap v1276: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:28.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:27 smithi099 bash[84576]: cluster 2024-09-13T14:52:26.597797+0000 mgr.y (mgr.26252) 785 : cluster [DBG] pgmap v1276: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+clean+scrubbing, 1 active+recovering, 94 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 200 objects/s recovering 2024-09-13T14:52:28.103 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:28.130 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:28.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:28.234 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:28.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:28.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:28.364 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:28.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:28.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:28.494 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:28.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:28 smithi177 bash[68594]: cluster 2024-09-13T14:52:26.790963+0000 osd.2 (osd.2) 13 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:28.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:28 smithi177 bash[68594]: cluster 2024-09-13T14:52:26.790963+0000 osd.2 (osd.2) 13 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:28.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:28 smithi177 bash[68594]: cluster 2024-09-13T14:52:27.461311+0000 osd.3 (osd.3) 32 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:28.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:28 smithi177 bash[68594]: cluster 2024-09-13T14:52:27.461311+0000 osd.3 (osd.3) 32 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:28.523 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:28.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:28.626 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:28.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:28.756 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:28.783 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:28.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:28 smithi099 bash[84576]: cluster 2024-09-13T14:52:26.790963+0000 osd.2 (osd.2) 13 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:28.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:28 smithi099 bash[84576]: cluster 2024-09-13T14:52:26.790963+0000 osd.2 (osd.2) 13 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:28.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:28 smithi099 bash[84576]: cluster 2024-09-13T14:52:27.461311+0000 osd.3 (osd.3) 32 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:28.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:28 smithi099 bash[84576]: cluster 2024-09-13T14:52:27.461311+0000 osd.3 (osd.3) 32 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:28.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:28 smithi099 bash[87147]: cluster 2024-09-13T14:52:26.790963+0000 osd.2 (osd.2) 13 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:28.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:28 smithi099 bash[87147]: cluster 2024-09-13T14:52:26.790963+0000 osd.2 (osd.2) 13 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:28.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:28 smithi099 bash[87147]: cluster 2024-09-13T14:52:27.461311+0000 osd.3 (osd.3) 32 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:28.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:28 smithi099 bash[87147]: cluster 2024-09-13T14:52:27.461311+0000 osd.3 (osd.3) 32 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:28.886 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:28.908 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:28.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:29.011 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:29.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:29.136 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:29.163 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:29.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:29.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:29.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:29.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:29.398 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:29.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:29.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:29.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:29.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:29.661 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[84576]: cluster 2024-09-13T14:52:27.712232+0000 osd.4 (osd.4) 25 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:29.661 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[84576]: cluster 2024-09-13T14:52:27.712232+0000 osd.4 (osd.4) 25 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:29.662 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[84576]: cluster 2024-09-13T14:52:27.800541+0000 osd.2 (osd.2) 14 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:29.662 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[84576]: cluster 2024-09-13T14:52:27.800541+0000 osd.2 (osd.2) 14 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:29.662 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[84576]: cluster 2024-09-13T14:52:28.464440+0000 osd.3 (osd.3) 33 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:29.662 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[84576]: cluster 2024-09-13T14:52:28.464440+0000 osd.3 (osd.3) 33 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:29.662 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[84576]: cluster 2024-09-13T14:52:28.544781+0000 osd.0 (osd.0) 11 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:29.662 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[84576]: cluster 2024-09-13T14:52:28.544781+0000 osd.0 (osd.0) 11 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:29.662 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[84576]: cluster 2024-09-13T14:52:28.598373+0000 mgr.y (mgr.26252) 786 : cluster [DBG] pgmap v1277: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 295 objects/s recovering 2024-09-13T14:52:29.662 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[84576]: cluster 2024-09-13T14:52:28.598373+0000 mgr.y (mgr.26252) 786 : cluster [DBG] pgmap v1277: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 295 objects/s recovering 2024-09-13T14:52:29.662 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[87147]: cluster 2024-09-13T14:52:27.712232+0000 osd.4 (osd.4) 25 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:29.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[87147]: cluster 2024-09-13T14:52:27.712232+0000 osd.4 (osd.4) 25 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:29.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[87147]: cluster 2024-09-13T14:52:27.800541+0000 osd.2 (osd.2) 14 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:29.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[87147]: cluster 2024-09-13T14:52:27.800541+0000 osd.2 (osd.2) 14 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:29.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[87147]: cluster 2024-09-13T14:52:28.464440+0000 osd.3 (osd.3) 33 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:29.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[87147]: cluster 2024-09-13T14:52:28.464440+0000 osd.3 (osd.3) 33 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:29.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[87147]: cluster 2024-09-13T14:52:28.544781+0000 osd.0 (osd.0) 11 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:29.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[87147]: cluster 2024-09-13T14:52:28.544781+0000 osd.0 (osd.0) 11 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:29.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[87147]: cluster 2024-09-13T14:52:28.598373+0000 mgr.y (mgr.26252) 786 : cluster [DBG] pgmap v1277: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 295 objects/s recovering 2024-09-13T14:52:29.663 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:29 smithi099 bash[87147]: cluster 2024-09-13T14:52:28.598373+0000 mgr.y (mgr.26252) 786 : cluster [DBG] pgmap v1277: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 295 objects/s recovering 2024-09-13T14:52:29.673 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:29.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:29.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:29 smithi177 bash[68594]: cluster 2024-09-13T14:52:27.712232+0000 osd.4 (osd.4) 25 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:29.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:29 smithi177 bash[68594]: cluster 2024-09-13T14:52:27.712232+0000 osd.4 (osd.4) 25 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:29.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:29 smithi177 bash[68594]: cluster 2024-09-13T14:52:27.800541+0000 osd.2 (osd.2) 14 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:29.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:29 smithi177 bash[68594]: cluster 2024-09-13T14:52:27.800541+0000 osd.2 (osd.2) 14 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:29.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:29 smithi177 bash[68594]: cluster 2024-09-13T14:52:28.464440+0000 osd.3 (osd.3) 33 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:29.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:29 smithi177 bash[68594]: cluster 2024-09-13T14:52:28.464440+0000 osd.3 (osd.3) 33 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:29.789 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:29 smithi177 bash[68594]: cluster 2024-09-13T14:52:28.544781+0000 osd.0 (osd.0) 11 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:29.789 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:29 smithi177 bash[68594]: cluster 2024-09-13T14:52:28.544781+0000 osd.0 (osd.0) 11 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:29.789 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:29 smithi177 bash[68594]: cluster 2024-09-13T14:52:28.598373+0000 mgr.y (mgr.26252) 786 : cluster [DBG] pgmap v1277: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 295 objects/s recovering 2024-09-13T14:52:29.789 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:29 smithi177 bash[68594]: cluster 2024-09-13T14:52:28.598373+0000 mgr.y (mgr.26252) 786 : cluster [DBG] pgmap v1277: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 295 objects/s recovering 2024-09-13T14:52:29.804 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:29.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:29.908 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:29.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:29.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:30.037 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:30.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:30.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:30.170 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:30.194 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:30.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:30.297 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:30.324 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:30.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:30.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:30.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:30.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:30.478 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:52:30 smithi177 bash[61439]: ts=2024-09-13T14:52:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:30.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:30.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[84576]: cluster 2024-09-13T14:52:28.688810+0000 osd.4 (osd.4) 26 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[84576]: cluster 2024-09-13T14:52:28.688810+0000 osd.4 (osd.4) 26 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[84576]: cluster 2024-09-13T14:52:29.416460+0000 osd.3 (osd.3) 34 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[84576]: cluster 2024-09-13T14:52:29.416460+0000 osd.3 (osd.3) 34 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[84576]: cluster 2024-09-13T14:52:29.584419+0000 osd.0 (osd.0) 12 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[84576]: cluster 2024-09-13T14:52:29.584419+0000 osd.0 (osd.0) 12 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[87147]: cluster 2024-09-13T14:52:28.688810+0000 osd.4 (osd.4) 26 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[87147]: cluster 2024-09-13T14:52:28.688810+0000 osd.4 (osd.4) 26 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[87147]: cluster 2024-09-13T14:52:29.416460+0000 osd.3 (osd.3) 34 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[87147]: cluster 2024-09-13T14:52:29.416460+0000 osd.3 (osd.3) 34 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[87147]: cluster 2024-09-13T14:52:29.584419+0000 osd.0 (osd.0) 12 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:30.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:30 smithi099 bash[87147]: cluster 2024-09-13T14:52:29.584419+0000 osd.0 (osd.0) 12 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:30.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:30.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:30.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:30.701 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:30 smithi177 bash[68594]: cluster 2024-09-13T14:52:28.688810+0000 osd.4 (osd.4) 26 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:30.701 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:30 smithi177 bash[68594]: cluster 2024-09-13T14:52:28.688810+0000 osd.4 (osd.4) 26 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:30.701 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:30 smithi177 bash[68594]: cluster 2024-09-13T14:52:29.416460+0000 osd.3 (osd.3) 34 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:30.701 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:30 smithi177 bash[68594]: cluster 2024-09-13T14:52:29.416460+0000 osd.3 (osd.3) 34 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:30.702 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:30 smithi177 bash[68594]: cluster 2024-09-13T14:52:29.584419+0000 osd.0 (osd.0) 12 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:30.702 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:30 smithi177 bash[68594]: cluster 2024-09-13T14:52:29.584419+0000 osd.0 (osd.0) 12 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:30.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:30.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:30.819 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:30.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:30.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:30.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:30.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:30.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:31.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:31.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:31.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:31.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:31.236 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:31.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:31.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:31.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:31.460 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T14:52:32.487 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:52:32.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:32.489 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:31 smithi177 bash[68594]: cluster 2024-09-13T14:52:30.443025+0000 osd.3 (osd.3) 35 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:32.489 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:31 smithi177 bash[68594]: cluster 2024-09-13T14:52:30.443025+0000 osd.3 (osd.3) 35 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:32.489 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:31 smithi177 bash[68594]: cluster 2024-09-13T14:52:30.599722+0000 mgr.y (mgr.26252) 787 : cluster [DBG] pgmap v1278: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 164 objects/s recovering 2024-09-13T14:52:32.489 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:31 smithi177 bash[68594]: cluster 2024-09-13T14:52:30.599722+0000 mgr.y (mgr.26252) 787 : cluster [DBG] pgmap v1278: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 164 objects/s recovering 2024-09-13T14:52:32.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:31 smithi099 bash[84576]: cluster 2024-09-13T14:52:30.443025+0000 osd.3 (osd.3) 35 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:32.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:31 smithi099 bash[84576]: cluster 2024-09-13T14:52:30.443025+0000 osd.3 (osd.3) 35 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:32.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:31 smithi099 bash[84576]: cluster 2024-09-13T14:52:30.599722+0000 mgr.y (mgr.26252) 787 : cluster [DBG] pgmap v1278: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 164 objects/s recovering 2024-09-13T14:52:32.489 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:31 smithi099 bash[84576]: cluster 2024-09-13T14:52:30.599722+0000 mgr.y (mgr.26252) 787 : cluster [DBG] pgmap v1278: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 164 objects/s recovering 2024-09-13T14:52:32.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:31 smithi099 bash[87147]: cluster 2024-09-13T14:52:30.443025+0000 osd.3 (osd.3) 35 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:32.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:31 smithi099 bash[87147]: cluster 2024-09-13T14:52:30.443025+0000 osd.3 (osd.3) 35 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:32.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:31 smithi099 bash[87147]: cluster 2024-09-13T14:52:30.599722+0000 mgr.y (mgr.26252) 787 : cluster [DBG] pgmap v1278: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 164 objects/s recovering 2024-09-13T14:52:32.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:31 smithi099 bash[87147]: cluster 2024-09-13T14:52:30.599722+0000 mgr.y (mgr.26252) 787 : cluster [DBG] pgmap v1278: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 14 active+clean+scrubbing, 1 active+recovering, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 1346/210 objects degraded (640.952%); 0 B/s, 164 objects/s recovering 2024-09-13T14:52:32.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:32.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:32.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:32.654 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:32.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:32.756 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:32.780 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:32.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:32.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:32 smithi177 bash[68594]: cluster 2024-09-13T14:52:30.642298+0000 osd.4 (osd.4) 27 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:32.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:32 smithi177 bash[68594]: cluster 2024-09-13T14:52:30.642298+0000 osd.4 (osd.4) 27 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:32.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:32 smithi177 bash[68594]: cluster 2024-09-13T14:52:31.397155+0000 osd.3 (osd.3) 36 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:32.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:32 smithi177 bash[68594]: cluster 2024-09-13T14:52:31.397155+0000 osd.3 (osd.3) 36 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:32.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:32 smithi177 bash[68594]: cluster 2024-09-13T14:52:31.617487+0000 osd.0 (osd.0) 13 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:32.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:32 smithi177 bash[68594]: cluster 2024-09-13T14:52:31.617487+0000 osd.0 (osd.0) 13 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:32.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:32.907 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:32.916 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[84576]: cluster 2024-09-13T14:52:30.642298+0000 osd.4 (osd.4) 27 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:32.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[84576]: cluster 2024-09-13T14:52:30.642298+0000 osd.4 (osd.4) 27 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:32.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[84576]: cluster 2024-09-13T14:52:31.397155+0000 osd.3 (osd.3) 36 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:32.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[84576]: cluster 2024-09-13T14:52:31.397155+0000 osd.3 (osd.3) 36 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:32.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[84576]: cluster 2024-09-13T14:52:31.617487+0000 osd.0 (osd.0) 13 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:32.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[84576]: cluster 2024-09-13T14:52:31.617487+0000 osd.0 (osd.0) 13 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:32.917 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[87147]: cluster 2024-09-13T14:52:30.642298+0000 osd.4 (osd.4) 27 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:32.917 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[87147]: cluster 2024-09-13T14:52:30.642298+0000 osd.4 (osd.4) 27 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:32.917 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[87147]: cluster 2024-09-13T14:52:31.397155+0000 osd.3 (osd.3) 36 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:32.917 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[87147]: cluster 2024-09-13T14:52:31.397155+0000 osd.3 (osd.3) 36 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:32.918 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[87147]: cluster 2024-09-13T14:52:31.617487+0000 osd.0 (osd.0) 13 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:32.918 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:32 smithi099 bash[87147]: cluster 2024-09-13T14:52:31.617487+0000 osd.0 (osd.0) 13 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:33.010 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:33.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:33.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:33.140 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:33.169 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:33.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:33.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:33.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:33.431 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:33.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:33.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:33.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:33.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:33.689 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:33.776 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[84576]: cluster 2024-09-13T14:52:31.659443+0000 osd.4 (osd.4) 28 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:33.776 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[84576]: cluster 2024-09-13T14:52:31.659443+0000 osd.4 (osd.4) 28 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:33.776 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[84576]: cluster 2024-09-13T14:52:32.364633+0000 osd.3 (osd.3) 37 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:33.776 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[84576]: cluster 2024-09-13T14:52:32.364633+0000 osd.3 (osd.3) 37 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[84576]: cluster 2024-09-13T14:52:32.600747+0000 mgr.y (mgr.26252) 788 : cluster [DBG] pgmap v1279: 109 pgs: 1 active+recovering+undersized+remapped, 15 active+clean+scrubbing, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 291 objects/s recovering 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[84576]: cluster 2024-09-13T14:52:32.600747+0000 mgr.y (mgr.26252) 788 : cluster [DBG] pgmap v1279: 109 pgs: 1 active+recovering+undersized+remapped, 15 active+clean+scrubbing, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 291 objects/s recovering 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[84576]: audit 2024-09-13T14:52:32.637120+0000 mon.a (mon.0) 1804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[84576]: audit 2024-09-13T14:52:32.637120+0000 mon.a (mon.0) 1804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[84576]: audit 2024-09-13T14:52:33.279974+0000 mon.a (mon.0) 1805 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[84576]: audit 2024-09-13T14:52:33.279974+0000 mon.a (mon.0) 1805 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[87147]: cluster 2024-09-13T14:52:31.659443+0000 osd.4 (osd.4) 28 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[87147]: cluster 2024-09-13T14:52:31.659443+0000 osd.4 (osd.4) 28 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[87147]: cluster 2024-09-13T14:52:32.364633+0000 osd.3 (osd.3) 37 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[87147]: cluster 2024-09-13T14:52:32.364633+0000 osd.3 (osd.3) 37 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:33.777 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[87147]: cluster 2024-09-13T14:52:32.600747+0000 mgr.y (mgr.26252) 788 : cluster [DBG] pgmap v1279: 109 pgs: 1 active+recovering+undersized+remapped, 15 active+clean+scrubbing, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 291 objects/s recovering 2024-09-13T14:52:33.778 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[87147]: cluster 2024-09-13T14:52:32.600747+0000 mgr.y (mgr.26252) 788 : cluster [DBG] pgmap v1279: 109 pgs: 1 active+recovering+undersized+remapped, 15 active+clean+scrubbing, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 291 objects/s recovering 2024-09-13T14:52:33.778 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[87147]: audit 2024-09-13T14:52:32.637120+0000 mon.a (mon.0) 1804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:52:33.778 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[87147]: audit 2024-09-13T14:52:32.637120+0000 mon.a (mon.0) 1804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:52:33.778 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[87147]: audit 2024-09-13T14:52:33.279974+0000 mon.a (mon.0) 1805 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:33.778 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:33 smithi099 bash[87147]: audit 2024-09-13T14:52:33.279974+0000 mon.a (mon.0) 1805 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:33.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:33.812 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:33.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:33.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:33 smithi177 bash[68594]: cluster 2024-09-13T14:52:31.659443+0000 osd.4 (osd.4) 28 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:33 smithi177 bash[68594]: cluster 2024-09-13T14:52:31.659443+0000 osd.4 (osd.4) 28 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:33 smithi177 bash[68594]: cluster 2024-09-13T14:52:32.364633+0000 osd.3 (osd.3) 37 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:33 smithi177 bash[68594]: cluster 2024-09-13T14:52:32.364633+0000 osd.3 (osd.3) 37 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:33 smithi177 bash[68594]: cluster 2024-09-13T14:52:32.600747+0000 mgr.y (mgr.26252) 788 : cluster [DBG] pgmap v1279: 109 pgs: 1 active+recovering+undersized+remapped, 15 active+clean+scrubbing, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 291 objects/s recovering 2024-09-13T14:52:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:33 smithi177 bash[68594]: cluster 2024-09-13T14:52:32.600747+0000 mgr.y (mgr.26252) 788 : cluster [DBG] pgmap v1279: 109 pgs: 1 active+recovering+undersized+remapped, 15 active+clean+scrubbing, 93 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 291 objects/s recovering 2024-09-13T14:52:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:33 smithi177 bash[68594]: audit 2024-09-13T14:52:32.637120+0000 mon.a (mon.0) 1804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:52:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:33 smithi177 bash[68594]: audit 2024-09-13T14:52:32.637120+0000 mon.a (mon.0) 1804 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:52:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:33 smithi177 bash[68594]: audit 2024-09-13T14:52:33.279974+0000 mon.a (mon.0) 1805 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:33 smithi177 bash[68594]: audit 2024-09-13T14:52:33.279974+0000 mon.a (mon.0) 1805 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:33.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:33.941 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:33.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:34.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:34.070 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:34.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:34.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:34.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:34.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:34.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:34.328 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:34.430 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:34.452 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:34.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:34.555 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:34.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:34.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:34.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:34.695 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: cluster 2024-09-13T14:52:32.679751+0000 osd.2 (osd.2) 15 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: cluster 2024-09-13T14:52:32.679751+0000 osd.2 (osd.2) 15 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.406836+0000 osd.3 (osd.3) 38 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.406836+0000 osd.3 (osd.3) 38 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.493508+0000 mon.a (mon.0) 1806 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded) 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.493508+0000 mon.a (mon.0) 1806 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded) 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: audit 2024-09-13T14:52:33.504172+0000 mon.a (mon.0) 1807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: audit 2024-09-13T14:52:33.504172+0000 mon.a (mon.0) 1807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.510770+0000 mon.a (mon.0) 1808 : cluster [DBG] osdmap e999: 8 total, 8 up, 7 in 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.510770+0000 mon.a (mon.0) 1808 : cluster [DBG] osdmap e999: 8 total, 8 up, 7 in 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.703668+0000 osd.0 (osd.0) 14 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:34.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.703668+0000 osd.0 (osd.0) 14 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:34.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: cluster 2024-09-13T14:52:32.679751+0000 osd.2 (osd.2) 15 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:34.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: cluster 2024-09-13T14:52:32.679751+0000 osd.2 (osd.2) 15 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:34.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.406836+0000 osd.3 (osd.3) 38 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:34.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.406836+0000 osd.3 (osd.3) 38 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:34.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.493508+0000 mon.a (mon.0) 1806 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded) 2024-09-13T14:52:34.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.493508+0000 mon.a (mon.0) 1806 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded) 2024-09-13T14:52:34.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: audit 2024-09-13T14:52:33.504172+0000 mon.a (mon.0) 1807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:52:34.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: audit 2024-09-13T14:52:33.504172+0000 mon.a (mon.0) 1807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:52:34.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.510770+0000 mon.a (mon.0) 1808 : cluster [DBG] osdmap e999: 8 total, 8 up, 7 in 2024-09-13T14:52:34.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.510770+0000 mon.a (mon.0) 1808 : cluster [DBG] osdmap e999: 8 total, 8 up, 7 in 2024-09-13T14:52:34.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.703668+0000 osd.0 (osd.0) 14 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:34.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:34 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.703668+0000 osd.0 (osd.0) 14 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:34.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:34.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:34.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:34.838 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:34.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:34.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: cluster 2024-09-13T14:52:32.679751+0000 osd.2 (osd.2) 15 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: cluster 2024-09-13T14:52:32.679751+0000 osd.2 (osd.2) 15 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.406836+0000 osd.3 (osd.3) 38 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.406836+0000 osd.3 (osd.3) 38 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.493508+0000 mon.a (mon.0) 1806 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded) 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.493508+0000 mon.a (mon.0) 1806 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1346/210 objects degraded (640.952%), 1 pg degraded) 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: audit 2024-09-13T14:52:33.504172+0000 mon.a (mon.0) 1807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: audit 2024-09-13T14:52:33.504172+0000 mon.a (mon.0) 1807 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.510770+0000 mon.a (mon.0) 1808 : cluster [DBG] osdmap e999: 8 total, 8 up, 7 in 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.510770+0000 mon.a (mon.0) 1808 : cluster [DBG] osdmap e999: 8 total, 8 up, 7 in 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.703668+0000 osd.0 (osd.0) 14 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:34 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.703668+0000 osd.0 (osd.0) 14 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:34.941 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:34.966 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:35.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:35.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:35.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:35.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:35.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:35.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:35.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:35.356 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:35.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:35.459 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:35.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:35.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.652495+0000 osd.4 (osd.4) 29 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.652495+0000 osd.4 (osd.4) 29 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.715147+0000 osd.2 (osd.2) 16 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:33.715147+0000 osd.2 (osd.2) 16 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:35.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.423809+0000 osd.3 (osd.3) 39 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.423809+0000 osd.3 (osd.3) 39 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.514648+0000 mon.a (mon.0) 1809 : cluster [DBG] osdmap e1000: 8 total, 8 up, 7 in 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.514648+0000 mon.a (mon.0) 1809 : cluster [DBG] osdmap e1000: 8 total, 8 up, 7 in 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.601309+0000 mgr.y (mgr.26252) 789 : cluster [DBG] pgmap v1282: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 333 objects/s recovering 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.601309+0000 mgr.y (mgr.26252) 789 : cluster [DBG] pgmap v1282: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 333 objects/s recovering 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.745864+0000 osd.0 (osd.0) 15 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.745864+0000 osd.0 (osd.0) 15 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.652495+0000 osd.4 (osd.4) 29 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.652495+0000 osd.4 (osd.4) 29 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.715147+0000 osd.2 (osd.2) 16 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:35.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:33.715147+0000 osd.2 (osd.2) 16 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.423809+0000 osd.3 (osd.3) 39 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.423809+0000 osd.3 (osd.3) 39 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.514648+0000 mon.a (mon.0) 1809 : cluster [DBG] osdmap e1000: 8 total, 8 up, 7 in 2024-09-13T14:52:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.514648+0000 mon.a (mon.0) 1809 : cluster [DBG] osdmap e1000: 8 total, 8 up, 7 in 2024-09-13T14:52:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.601309+0000 mgr.y (mgr.26252) 789 : cluster [DBG] pgmap v1282: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 333 objects/s recovering 2024-09-13T14:52:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.601309+0000 mgr.y (mgr.26252) 789 : cluster [DBG] pgmap v1282: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 333 objects/s recovering 2024-09-13T14:52:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.745864+0000 osd.0 (osd.0) 15 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:35 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.745864+0000 osd.0 (osd.0) 15 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:35.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:35.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:35.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:35.724 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:35.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:35.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:35.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.652495+0000 osd.4 (osd.4) 29 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.652495+0000 osd.4 (osd.4) 29 : cluster [DBG] 41.20 scrub starts 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.715147+0000 osd.2 (osd.2) 16 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:33.715147+0000 osd.2 (osd.2) 16 : cluster [DBG] 41.1f scrub starts 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.423809+0000 osd.3 (osd.3) 39 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.423809+0000 osd.3 (osd.3) 39 : cluster [DBG] 41.23 scrub starts 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.514648+0000 mon.a (mon.0) 1809 : cluster [DBG] osdmap e1000: 8 total, 8 up, 7 in 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.514648+0000 mon.a (mon.0) 1809 : cluster [DBG] osdmap e1000: 8 total, 8 up, 7 in 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.601309+0000 mgr.y (mgr.26252) 789 : cluster [DBG] pgmap v1282: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 333 objects/s recovering 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.601309+0000 mgr.y (mgr.26252) 789 : cluster [DBG] pgmap v1282: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 333 objects/s recovering 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.745864+0000 osd.0 (osd.0) 15 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:35 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.745864+0000 osd.0 (osd.0) 15 : cluster [DBG] 41.c scrub starts 2024-09-13T14:52:35.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:35.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:35.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:36.009 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:36.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:36.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:36.140 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:36.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:36.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:36.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:36.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:36.371 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:36.398 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:36.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:36.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:36.512 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.604791+0000 osd.4 (osd.4) 30 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:36.512 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.604791+0000 osd.4 (osd.4) 30 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:36.512 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.686652+0000 osd.2 (osd.2) 17 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:36.512 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:34.686652+0000 osd.2 (osd.2) 17 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.415932+0000 osd.3 (osd.3) 40 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.415932+0000 osd.3 (osd.3) 40 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.505705+0000 mon.a (mon.0) 1810 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.505705+0000 mon.a (mon.0) 1810 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.517217+0000 mon.a (mon.0) 1811 : cluster [DBG] osdmap e1001: 8 total, 8 up, 7 in 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.517217+0000 mon.a (mon.0) 1811 : cluster [DBG] osdmap e1001: 8 total, 8 up, 7 in 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.519076+0000 osd.5 (osd.5) 7 : cluster [DBG] 41.10 starting backfill to osd.0 from (0'0,0'0] MAX to 989'11067 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.519076+0000 osd.5 (osd.5) 7 : cluster [DBG] 41.10 starting backfill to osd.0 from (0'0,0'0] MAX to 989'11067 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.701357+0000 osd.0 (osd.0) 16 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.701357+0000 osd.0 (osd.0) 16 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:36.375560+0000 mon.a (mon.0) 1812 : cluster [DBG] osdmap e1002: 8 total, 8 up, 7 in 2024-09-13T14:52:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:36 smithi177 bash[68594]: cluster 2024-09-13T14:52:36.375560+0000 mon.a (mon.0) 1812 : cluster [DBG] osdmap e1002: 8 total, 8 up, 7 in 2024-09-13T14:52:36.528 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:36.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:36.631 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:36.642 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.604791+0000 osd.4 (osd.4) 30 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:36.642 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.604791+0000 osd.4 (osd.4) 30 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.686652+0000 osd.2 (osd.2) 17 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:34.686652+0000 osd.2 (osd.2) 17 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.415932+0000 osd.3 (osd.3) 40 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.415932+0000 osd.3 (osd.3) 40 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.505705+0000 mon.a (mon.0) 1810 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.505705+0000 mon.a (mon.0) 1810 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.517217+0000 mon.a (mon.0) 1811 : cluster [DBG] osdmap e1001: 8 total, 8 up, 7 in 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.517217+0000 mon.a (mon.0) 1811 : cluster [DBG] osdmap e1001: 8 total, 8 up, 7 in 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.519076+0000 osd.5 (osd.5) 7 : cluster [DBG] 41.10 starting backfill to osd.0 from (0'0,0'0] MAX to 989'11067 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.519076+0000 osd.5 (osd.5) 7 : cluster [DBG] 41.10 starting backfill to osd.0 from (0'0,0'0] MAX to 989'11067 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.701357+0000 osd.0 (osd.0) 16 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.701357+0000 osd.0 (osd.0) 16 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:36.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:36.375560+0000 mon.a (mon.0) 1812 : cluster [DBG] osdmap e1002: 8 total, 8 up, 7 in 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[84576]: cluster 2024-09-13T14:52:36.375560+0000 mon.a (mon.0) 1812 : cluster [DBG] osdmap e1002: 8 total, 8 up, 7 in 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.604791+0000 osd.4 (osd.4) 30 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.604791+0000 osd.4 (osd.4) 30 : cluster [DBG] 41.1a scrub starts 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.686652+0000 osd.2 (osd.2) 17 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:34.686652+0000 osd.2 (osd.2) 17 : cluster [DBG] 41.f scrub starts 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.415932+0000 osd.3 (osd.3) 40 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.415932+0000 osd.3 (osd.3) 40 : cluster [DBG] 41.1b scrub starts 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.505705+0000 mon.a (mon.0) 1810 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.505705+0000 mon.a (mon.0) 1810 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.517217+0000 mon.a (mon.0) 1811 : cluster [DBG] osdmap e1001: 8 total, 8 up, 7 in 2024-09-13T14:52:36.644 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.517217+0000 mon.a (mon.0) 1811 : cluster [DBG] osdmap e1001: 8 total, 8 up, 7 in 2024-09-13T14:52:36.645 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.519076+0000 osd.5 (osd.5) 7 : cluster [DBG] 41.10 starting backfill to osd.0 from (0'0,0'0] MAX to 989'11067 2024-09-13T14:52:36.645 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.519076+0000 osd.5 (osd.5) 7 : cluster [DBG] 41.10 starting backfill to osd.0 from (0'0,0'0] MAX to 989'11067 2024-09-13T14:52:36.645 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.701357+0000 osd.0 (osd.0) 16 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:36.645 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.701357+0000 osd.0 (osd.0) 16 : cluster [DBG] 41.d scrub starts 2024-09-13T14:52:36.645 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:36.375560+0000 mon.a (mon.0) 1812 : cluster [DBG] osdmap e1002: 8 total, 8 up, 7 in 2024-09-13T14:52:36.645 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:36 smithi099 bash[87147]: cluster 2024-09-13T14:52:36.375560+0000 mon.a (mon.0) 1812 : cluster [DBG] osdmap e1002: 8 total, 8 up, 7 in 2024-09-13T14:52:36.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:36.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:36.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:36.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:36.913 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:52:36.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:36.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:37.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:37.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:37.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:37.148 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:37.202 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:52:37] "GET /metrics HTTP/1.1" 200 36751 "" "Prometheus/2.51.0" 2024-09-13T14:52:37.271 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.296 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:37.399 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:37.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:37.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:37.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:37.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:37.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:37.658 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.684 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:37.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.587191+0000 osd.4 (osd.4) 31 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:37.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[84576]: cluster 2024-09-13T14:52:35.587191+0000 osd.4 (osd.4) 31 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:37.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[84576]: cluster 2024-09-13T14:52:36.414775+0000 osd.3 (osd.3) 41 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:37.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[84576]: cluster 2024-09-13T14:52:36.414775+0000 osd.3 (osd.3) 41 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:37.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[84576]: cluster 2024-09-13T14:52:36.602069+0000 mgr.y (mgr.26252) 790 : cluster [DBG] pgmap v1285: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:37.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[84576]: cluster 2024-09-13T14:52:36.602069+0000 mgr.y (mgr.26252) 790 : cluster [DBG] pgmap v1285: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:37.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[84576]: cluster 2024-09-13T14:52:36.691962+0000 osd.0 (osd.0) 17 : cluster [DBG] 41.1c scrub starts 2024-09-13T14:52:37.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[84576]: cluster 2024-09-13T14:52:36.691962+0000 osd.0 (osd.0) 17 : cluster [DBG] 41.1c scrub starts 2024-09-13T14:52:37.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[84576]: cluster 2024-09-13T14:52:37.378806+0000 mon.a (mon.0) 1813 : cluster [DBG] osdmap e1003: 8 total, 8 up, 7 in 2024-09-13T14:52:37.769 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[84576]: cluster 2024-09-13T14:52:37.378806+0000 mon.a (mon.0) 1813 : cluster [DBG] osdmap e1003: 8 total, 8 up, 7 in 2024-09-13T14:52:37.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.587191+0000 osd.4 (osd.4) 31 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:37.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[87147]: cluster 2024-09-13T14:52:35.587191+0000 osd.4 (osd.4) 31 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:37.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[87147]: cluster 2024-09-13T14:52:36.414775+0000 osd.3 (osd.3) 41 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:37.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[87147]: cluster 2024-09-13T14:52:36.414775+0000 osd.3 (osd.3) 41 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:37.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[87147]: cluster 2024-09-13T14:52:36.602069+0000 mgr.y (mgr.26252) 790 : cluster [DBG] pgmap v1285: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:37.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[87147]: cluster 2024-09-13T14:52:36.602069+0000 mgr.y (mgr.26252) 790 : cluster [DBG] pgmap v1285: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:37.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[87147]: cluster 2024-09-13T14:52:36.691962+0000 osd.0 (osd.0) 17 : cluster [DBG] 41.1c scrub starts 2024-09-13T14:52:37.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[87147]: cluster 2024-09-13T14:52:36.691962+0000 osd.0 (osd.0) 17 : cluster [DBG] 41.1c scrub starts 2024-09-13T14:52:37.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[87147]: cluster 2024-09-13T14:52:37.378806+0000 mon.a (mon.0) 1813 : cluster [DBG] osdmap e1003: 8 total, 8 up, 7 in 2024-09-13T14:52:37.770 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:37 smithi099 bash[87147]: cluster 2024-09-13T14:52:37.378806+0000 mon.a (mon.0) 1813 : cluster [DBG] osdmap e1003: 8 total, 8 up, 7 in 2024-09-13T14:52:37.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:37.799 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:37 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.587191+0000 osd.4 (osd.4) 31 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:37.799 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:37 smithi177 bash[68594]: cluster 2024-09-13T14:52:35.587191+0000 osd.4 (osd.4) 31 : cluster [DBG] 41.a scrub starts 2024-09-13T14:52:37.799 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:37 smithi177 bash[68594]: cluster 2024-09-13T14:52:36.414775+0000 osd.3 (osd.3) 41 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:37.799 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:37 smithi177 bash[68594]: cluster 2024-09-13T14:52:36.414775+0000 osd.3 (osd.3) 41 : cluster [DBG] 41.1 scrub starts 2024-09-13T14:52:37.799 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:37 smithi177 bash[68594]: cluster 2024-09-13T14:52:36.602069+0000 mgr.y (mgr.26252) 790 : cluster [DBG] pgmap v1285: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:37.799 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:37 smithi177 bash[68594]: cluster 2024-09-13T14:52:36.602069+0000 mgr.y (mgr.26252) 790 : cluster [DBG] pgmap v1285: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 9 active+clean+scrubbing, 98 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:37.800 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:37 smithi177 bash[68594]: cluster 2024-09-13T14:52:36.691962+0000 osd.0 (osd.0) 17 : cluster [DBG] 41.1c scrub starts 2024-09-13T14:52:37.800 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:37 smithi177 bash[68594]: cluster 2024-09-13T14:52:36.691962+0000 osd.0 (osd.0) 17 : cluster [DBG] 41.1c scrub starts 2024-09-13T14:52:37.800 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:37 smithi177 bash[68594]: cluster 2024-09-13T14:52:37.378806+0000 mon.a (mon.0) 1813 : cluster [DBG] osdmap e1003: 8 total, 8 up, 7 in 2024-09-13T14:52:37.800 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:37 smithi177 bash[68594]: cluster 2024-09-13T14:52:37.378806+0000 mon.a (mon.0) 1813 : cluster [DBG] osdmap e1003: 8 total, 8 up, 7 in 2024-09-13T14:52:37.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:37.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:37.946 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:37.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:38.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:38.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:38.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:38.206 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:38.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:38.309 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:38.335 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:38.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:38.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:38.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:38.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:38.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:38 smithi099 bash[84576]: cluster 2024-09-13T14:52:36.593216+0000 osd.4 (osd.4) 32 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:38.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:38 smithi099 bash[84576]: cluster 2024-09-13T14:52:36.593216+0000 osd.4 (osd.4) 32 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:38.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:38 smithi099 bash[84576]: cluster 2024-09-13T14:52:37.444881+0000 osd.3 (osd.3) 42 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:38.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:38 smithi099 bash[84576]: cluster 2024-09-13T14:52:37.444881+0000 osd.3 (osd.3) 42 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:38.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:38 smithi099 bash[87147]: cluster 2024-09-13T14:52:36.593216+0000 osd.4 (osd.4) 32 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:38.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:38 smithi099 bash[87147]: cluster 2024-09-13T14:52:36.593216+0000 osd.4 (osd.4) 32 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:38.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:38 smithi099 bash[87147]: cluster 2024-09-13T14:52:37.444881+0000 osd.3 (osd.3) 42 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:38.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:38 smithi099 bash[87147]: cluster 2024-09-13T14:52:37.444881+0000 osd.3 (osd.3) 42 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:38.587 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:38.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:38.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:38 smithi177 bash[68594]: cluster 2024-09-13T14:52:36.593216+0000 osd.4 (osd.4) 32 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:38.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:38 smithi177 bash[68594]: cluster 2024-09-13T14:52:36.593216+0000 osd.4 (osd.4) 32 : cluster [DBG] 41.0 scrub starts 2024-09-13T14:52:38.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:38 smithi177 bash[68594]: cluster 2024-09-13T14:52:37.444881+0000 osd.3 (osd.3) 42 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:38.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:38 smithi177 bash[68594]: cluster 2024-09-13T14:52:37.444881+0000 osd.3 (osd.3) 42 : cluster [DBG] 41.21 scrub starts 2024-09-13T14:52:38.690 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:38.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:38.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:38.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:38.846 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:38.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:38.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:38.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:38.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:39.080 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:39.107 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:39.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:39.210 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:39.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:39.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:39.340 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:39.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:39.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:39.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:39.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:39.536 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_0' removed 2024-09-13T14:52:39.601 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:39.632 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:39.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:39.736 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:39.747 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[87147]: cluster 2024-09-13T14:52:38.488802+0000 osd.3 (osd.3) 43 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:39.748 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[87147]: cluster 2024-09-13T14:52:38.488802+0000 osd.3 (osd.3) 43 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:39.748 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[87147]: cluster 2024-09-13T14:52:38.602605+0000 mgr.y (mgr.26252) 791 : cluster [DBG] pgmap v1287: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 102 objects/s recovering 2024-09-13T14:52:39.748 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[87147]: cluster 2024-09-13T14:52:38.602605+0000 mgr.y (mgr.26252) 791 : cluster [DBG] pgmap v1287: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 102 objects/s recovering 2024-09-13T14:52:39.748 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[87147]: audit 2024-09-13T14:52:38.688130+0000 mon.c (mon.2) 57 : audit [INF] from='client.? 172.21.15.99:0/1825713892' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.748 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[87147]: audit 2024-09-13T14:52:38.688130+0000 mon.c (mon.2) 57 : audit [INF] from='client.? 172.21.15.99:0/1825713892' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.748 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[87147]: audit 2024-09-13T14:52:38.688739+0000 mon.a (mon.0) 1814 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.748 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[87147]: audit 2024-09-13T14:52:38.688739+0000 mon.a (mon.0) 1814 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.748 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[84576]: cluster 2024-09-13T14:52:38.488802+0000 osd.3 (osd.3) 43 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:39.748 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[84576]: cluster 2024-09-13T14:52:38.488802+0000 osd.3 (osd.3) 43 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:39.749 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[84576]: cluster 2024-09-13T14:52:38.602605+0000 mgr.y (mgr.26252) 791 : cluster [DBG] pgmap v1287: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 102 objects/s recovering 2024-09-13T14:52:39.749 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[84576]: cluster 2024-09-13T14:52:38.602605+0000 mgr.y (mgr.26252) 791 : cluster [DBG] pgmap v1287: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 102 objects/s recovering 2024-09-13T14:52:39.749 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[84576]: audit 2024-09-13T14:52:38.688130+0000 mon.c (mon.2) 57 : audit [INF] from='client.? 172.21.15.99:0/1825713892' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.749 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[84576]: audit 2024-09-13T14:52:38.688130+0000 mon.c (mon.2) 57 : audit [INF] from='client.? 172.21.15.99:0/1825713892' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.749 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[84576]: audit 2024-09-13T14:52:38.688739+0000 mon.a (mon.0) 1814 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.749 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:39 smithi099 bash[84576]: audit 2024-09-13T14:52:38.688739+0000 mon.a (mon.0) 1814 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.762 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:39.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:39 smithi177 bash[68594]: cluster 2024-09-13T14:52:38.488802+0000 osd.3 (osd.3) 43 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:39 smithi177 bash[68594]: cluster 2024-09-13T14:52:38.488802+0000 osd.3 (osd.3) 43 : cluster [DBG] 41.b scrub starts 2024-09-13T14:52:39.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:39 smithi177 bash[68594]: cluster 2024-09-13T14:52:38.602605+0000 mgr.y (mgr.26252) 791 : cluster [DBG] pgmap v1287: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 102 objects/s recovering 2024-09-13T14:52:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:39 smithi177 bash[68594]: cluster 2024-09-13T14:52:38.602605+0000 mgr.y (mgr.26252) 791 : cluster [DBG] pgmap v1287: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 3.3 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s, 102 objects/s recovering 2024-09-13T14:52:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:39 smithi177 bash[68594]: audit 2024-09-13T14:52:38.688130+0000 mon.c (mon.2) 57 : audit [INF] from='client.? 172.21.15.99:0/1825713892' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:39 smithi177 bash[68594]: audit 2024-09-13T14:52:38.688130+0000 mon.c (mon.2) 57 : audit [INF] from='client.? 172.21.15.99:0/1825713892' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:39 smithi177 bash[68594]: audit 2024-09-13T14:52:38.688739+0000 mon.a (mon.0) 1814 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:39 smithi177 bash[68594]: audit 2024-09-13T14:52:38.688739+0000 mon.a (mon.0) 1814 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:52:39.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:39.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:39.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:39.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:40.017 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:40.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:40.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:40.148 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:40.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:40.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:40.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:40.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:40.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:52:40 smithi177 bash[61439]: ts=2024-09-13T14:52:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:40.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:40.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:40.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:40.532 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:40.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:40.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:40.662 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:40.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:40.765 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:40 smithi177 bash[68594]: cluster 2024-09-13T14:52:39.493188+0000 osd.3 (osd.3) 44 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:40 smithi177 bash[68594]: cluster 2024-09-13T14:52:39.493188+0000 osd.3 (osd.3) 44 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:40 smithi177 bash[68594]: audit 2024-09-13T14:52:39.530021+0000 mon.a (mon.0) 1815 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:52:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:40 smithi177 bash[68594]: audit 2024-09-13T14:52:39.530021+0000 mon.a (mon.0) 1815 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:52:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:40 smithi177 bash[68594]: cluster 2024-09-13T14:52:39.535224+0000 mon.a (mon.0) 1816 : cluster [DBG] osdmap e1004: 8 total, 8 up, 7 in 2024-09-13T14:52:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:40 smithi177 bash[68594]: cluster 2024-09-13T14:52:39.535224+0000 mon.a (mon.0) 1816 : cluster [DBG] osdmap e1004: 8 total, 8 up, 7 in 2024-09-13T14:52:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:40 smithi177 bash[68594]: cluster 2024-09-13T14:52:39.668950+0000 osd.0 (osd.0) 18 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:40 smithi177 bash[68594]: cluster 2024-09-13T14:52:39.668950+0000 osd.0 (osd.0) 18 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:40.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:40.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:40.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[84576]: cluster 2024-09-13T14:52:39.493188+0000 osd.3 (osd.3) 44 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:40.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[84576]: cluster 2024-09-13T14:52:39.493188+0000 osd.3 (osd.3) 44 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:40.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[84576]: audit 2024-09-13T14:52:39.530021+0000 mon.a (mon.0) 1815 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:52:40.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[84576]: audit 2024-09-13T14:52:39.530021+0000 mon.a (mon.0) 1815 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[84576]: cluster 2024-09-13T14:52:39.535224+0000 mon.a (mon.0) 1816 : cluster [DBG] osdmap e1004: 8 total, 8 up, 7 in 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[84576]: cluster 2024-09-13T14:52:39.535224+0000 mon.a (mon.0) 1816 : cluster [DBG] osdmap e1004: 8 total, 8 up, 7 in 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[84576]: cluster 2024-09-13T14:52:39.668950+0000 osd.0 (osd.0) 18 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[84576]: cluster 2024-09-13T14:52:39.668950+0000 osd.0 (osd.0) 18 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[87147]: cluster 2024-09-13T14:52:39.493188+0000 osd.3 (osd.3) 44 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[87147]: cluster 2024-09-13T14:52:39.493188+0000 osd.3 (osd.3) 44 : cluster [DBG] 41.3 scrub starts 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[87147]: audit 2024-09-13T14:52:39.530021+0000 mon.a (mon.0) 1815 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[87147]: audit 2024-09-13T14:52:39.530021+0000 mon.a (mon.0) 1815 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[87147]: cluster 2024-09-13T14:52:39.535224+0000 mon.a (mon.0) 1816 : cluster [DBG] osdmap e1004: 8 total, 8 up, 7 in 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[87147]: cluster 2024-09-13T14:52:39.535224+0000 mon.a (mon.0) 1816 : cluster [DBG] osdmap e1004: 8 total, 8 up, 7 in 2024-09-13T14:52:40.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[87147]: cluster 2024-09-13T14:52:39.668950+0000 osd.0 (osd.0) 18 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:40.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:40 smithi099 bash[87147]: cluster 2024-09-13T14:52:39.668950+0000 osd.0 (osd.0) 18 : cluster [DBG] 41.1d scrub starts 2024-09-13T14:52:40.896 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:40.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:40.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:41.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:41.049 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:52:41.050 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:52:41.050 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-09-13T14:52:41.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-09-13T14:52:41.055 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:41.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:41.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:41.181 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:41.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:41.284 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:41.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:41.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:41.408 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:41.429 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:41.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:41.532 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:41.557 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:41.558 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:41 smithi099 bash[84576]: cluster 2024-09-13T14:52:40.603140+0000 mgr.y (mgr.26252) 792 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:41.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:41 smithi099 bash[84576]: cluster 2024-09-13T14:52:40.603140+0000 mgr.y (mgr.26252) 792 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:41.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:41 smithi099 bash[87147]: cluster 2024-09-13T14:52:40.603140+0000 mgr.y (mgr.26252) 792 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:41.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:41 smithi099 bash[87147]: cluster 2024-09-13T14:52:40.603140+0000 mgr.y (mgr.26252) 792 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:41.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:41.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:41.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:41.791 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:41.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:41.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:41.845 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:52:41.845 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:41 smithi177 bash[68594]: cluster 2024-09-13T14:52:40.603140+0000 mgr.y (mgr.26252) 792 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:41.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:41 smithi177 bash[68594]: cluster 2024-09-13T14:52:40.603140+0000 mgr.y (mgr.26252) 792 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:41.864 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:52:41.865 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:41.882 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:52:41.882 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:41.903 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:52:41.903 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:41.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:41.925 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:52:41.925 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:41.938 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:41.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:41.957 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:52:41.957 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:41.977 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:52:41.977 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:41.994 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:52:41.994 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:42.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:42.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:42.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:42.172 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:42.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:42.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:42.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:42.437 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:42.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:42.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:42.539 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: debug 2024-09-13T14:52:42.224+0000 7f0688bd2640 -1 mon.a@0(leader).osd e1004 definitely_dead 0 2024-09-13T14:52:42.550 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T14:52:42.566 INFO:teuthology.orchestra.run.smithi099.stderr:marked in osd.1. 2024-09-13T14:52:42.566 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:42.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:42.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:42.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: cluster 2024-09-13T14:52:41.531921+0000 mon.a (mon.0) 1817 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:52:42.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: cluster 2024-09-13T14:52:41.531921+0000 mon.a (mon.0) 1817 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:52:42.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: audit 2024-09-13T14:52:41.829525+0000 mon.a (mon.0) 1818 : audit [DBG] from='client.? 172.21.15.99:0/813992319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:42.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: audit 2024-09-13T14:52:41.829525+0000 mon.a (mon.0) 1818 : audit [DBG] from='client.? 172.21.15.99:0/813992319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:42.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: audit 2024-09-13T14:52:42.228430+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.99:0/1354270138' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: audit 2024-09-13T14:52:42.228430+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.99:0/1354270138' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: audit 2024-09-13T14:52:42.229169+0000 mon.a (mon.0) 1819 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: audit 2024-09-13T14:52:42.229169+0000 mon.a (mon.0) 1819 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: audit 2024-09-13T14:52:42.233752+0000 mon.a (mon.0) 1820 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[84576]: audit 2024-09-13T14:52:42.233752+0000 mon.a (mon.0) 1820 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[87147]: cluster 2024-09-13T14:52:41.531921+0000 mon.a (mon.0) 1817 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[87147]: cluster 2024-09-13T14:52:41.531921+0000 mon.a (mon.0) 1817 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[87147]: audit 2024-09-13T14:52:41.829525+0000 mon.a (mon.0) 1818 : audit [DBG] from='client.? 172.21.15.99:0/813992319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[87147]: audit 2024-09-13T14:52:41.829525+0000 mon.a (mon.0) 1818 : audit [DBG] from='client.? 172.21.15.99:0/813992319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[87147]: audit 2024-09-13T14:52:42.228430+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.99:0/1354270138' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[87147]: audit 2024-09-13T14:52:42.228430+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.99:0/1354270138' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[87147]: audit 2024-09-13T14:52:42.229169+0000 mon.a (mon.0) 1819 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[87147]: audit 2024-09-13T14:52:42.229169+0000 mon.a (mon.0) 1819 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.702 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[87147]: audit 2024-09-13T14:52:42.233752+0000 mon.a (mon.0) 1820 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:42.703 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:42 smithi099 bash[87147]: audit 2024-09-13T14:52:42.233752+0000 mon.a (mon.0) 1820 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:42.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.714 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:42.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:42.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:42.839 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:42.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:42.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:42 smithi177 bash[68594]: cluster 2024-09-13T14:52:41.531921+0000 mon.a (mon.0) 1817 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:52:42.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:42 smithi177 bash[68594]: cluster 2024-09-13T14:52:41.531921+0000 mon.a (mon.0) 1817 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:52:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:42 smithi177 bash[68594]: audit 2024-09-13T14:52:41.829525+0000 mon.a (mon.0) 1818 : audit [DBG] from='client.? 172.21.15.99:0/813992319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:42 smithi177 bash[68594]: audit 2024-09-13T14:52:41.829525+0000 mon.a (mon.0) 1818 : audit [DBG] from='client.? 172.21.15.99:0/813992319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:42 smithi177 bash[68594]: audit 2024-09-13T14:52:42.228430+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.99:0/1354270138' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:42 smithi177 bash[68594]: audit 2024-09-13T14:52:42.228430+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.99:0/1354270138' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:42 smithi177 bash[68594]: audit 2024-09-13T14:52:42.229169+0000 mon.a (mon.0) 1819 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:42 smithi177 bash[68594]: audit 2024-09-13T14:52:42.229169+0000 mon.a (mon.0) 1819 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T14:52:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:42 smithi177 bash[68594]: audit 2024-09-13T14:52:42.233752+0000 mon.a (mon.0) 1820 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:42 smithi177 bash[68594]: audit 2024-09-13T14:52:42.233752+0000 mon.a (mon.0) 1820 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:42.942 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:42.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:43.070 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:43.096 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:43.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:43.199 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.225 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:43.328 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:43.355 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:43.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:43.458 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:43.589 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:43.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:43 smithi177 bash[68594]: cluster 2024-09-13T14:52:42.541848+0000 mon.a (mon.0) 1821 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:43.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:43 smithi177 bash[68594]: cluster 2024-09-13T14:52:42.541848+0000 mon.a (mon.0) 1821 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:43.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:43 smithi177 bash[68594]: audit 2024-09-13T14:52:42.553253+0000 mon.a (mon.0) 1822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-13T14:52:43.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:43 smithi177 bash[68594]: audit 2024-09-13T14:52:42.553253+0000 mon.a (mon.0) 1822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-13T14:52:43.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:43 smithi177 bash[68594]: audit 2024-09-13T14:52:42.553410+0000 mon.a (mon.0) 1823 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:43.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:43 smithi177 bash[68594]: audit 2024-09-13T14:52:42.553410+0000 mon.a (mon.0) 1823 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:43.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:43 smithi177 bash[68594]: cluster 2024-09-13T14:52:42.561720+0000 mon.a (mon.0) 1824 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2024-09-13T14:52:43.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:43 smithi177 bash[68594]: cluster 2024-09-13T14:52:42.561720+0000 mon.a (mon.0) 1824 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2024-09-13T14:52:43.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:43 smithi177 bash[68594]: cluster 2024-09-13T14:52:42.603509+0000 mgr.y (mgr.26252) 793 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:43.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:43 smithi177 bash[68594]: cluster 2024-09-13T14:52:42.603509+0000 mgr.y (mgr.26252) 793 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:43.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:43.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:43.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:43.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:43.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:43.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.872 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:43.896 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[87147]: cluster 2024-09-13T14:52:42.541848+0000 mon.a (mon.0) 1821 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:43.896 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[87147]: cluster 2024-09-13T14:52:42.541848+0000 mon.a (mon.0) 1821 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:43.896 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[87147]: audit 2024-09-13T14:52:42.553253+0000 mon.a (mon.0) 1822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-13T14:52:43.896 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[87147]: audit 2024-09-13T14:52:42.553253+0000 mon.a (mon.0) 1822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-13T14:52:43.896 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[87147]: audit 2024-09-13T14:52:42.553410+0000 mon.a (mon.0) 1823 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:43.896 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[87147]: audit 2024-09-13T14:52:42.553410+0000 mon.a (mon.0) 1823 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:43.896 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[87147]: cluster 2024-09-13T14:52:42.561720+0000 mon.a (mon.0) 1824 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[87147]: cluster 2024-09-13T14:52:42.561720+0000 mon.a (mon.0) 1824 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[87147]: cluster 2024-09-13T14:52:42.603509+0000 mgr.y (mgr.26252) 793 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[87147]: cluster 2024-09-13T14:52:42.603509+0000 mgr.y (mgr.26252) 793 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[84576]: cluster 2024-09-13T14:52:42.541848+0000 mon.a (mon.0) 1821 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[84576]: cluster 2024-09-13T14:52:42.541848+0000 mon.a (mon.0) 1821 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[84576]: audit 2024-09-13T14:52:42.553253+0000 mon.a (mon.0) 1822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[84576]: audit 2024-09-13T14:52:42.553253+0000 mon.a (mon.0) 1822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[84576]: audit 2024-09-13T14:52:42.553410+0000 mon.a (mon.0) 1823 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[84576]: audit 2024-09-13T14:52:42.553410+0000 mon.a (mon.0) 1823 : audit [INF] from='client.? 172.21.15.99:0/912796377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[84576]: cluster 2024-09-13T14:52:42.561720+0000 mon.a (mon.0) 1824 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2024-09-13T14:52:43.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[84576]: cluster 2024-09-13T14:52:42.561720+0000 mon.a (mon.0) 1824 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2024-09-13T14:52:43.898 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[84576]: cluster 2024-09-13T14:52:42.603509+0000 mgr.y (mgr.26252) 793 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:43.898 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:43 smithi099 bash[84576]: cluster 2024-09-13T14:52:42.603509+0000 mgr.y (mgr.26252) 793 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2024-09-13T14:52:43.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:44.004 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:44.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:44.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:44.133 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:44.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:44.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:44.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:44.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:44.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:44.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:44.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:44.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:44.628 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:44.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:44 smithi099 bash[84576]: cluster 2024-09-13T14:52:43.565201+0000 mon.a (mon.0) 1825 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2024-09-13T14:52:44.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:44 smithi099 bash[84576]: cluster 2024-09-13T14:52:43.565201+0000 mon.a (mon.0) 1825 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2024-09-13T14:52:44.640 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:44 smithi099 bash[87147]: cluster 2024-09-13T14:52:43.565201+0000 mon.a (mon.0) 1825 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2024-09-13T14:52:44.640 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:44 smithi099 bash[87147]: cluster 2024-09-13T14:52:43.565201+0000 mon.a (mon.0) 1825 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2024-09-13T14:52:44.657 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:44.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:44.682 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-09-13T14:52:44.760 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:44.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:44.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:44 smithi177 bash[68594]: cluster 2024-09-13T14:52:43.565201+0000 mon.a (mon.0) 1825 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2024-09-13T14:52:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:44 smithi177 bash[68594]: cluster 2024-09-13T14:52:43.565201+0000 mon.a (mon.0) 1825 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2024-09-13T14:52:44.886 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:44.912 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:44.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:45.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:45.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:45.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:45.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:52:45.146 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.159 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:45.183 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:45.286 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:45.409 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:45.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:45.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:45.538 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:45.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:45.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:45.670 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:45.681 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:45 smithi177 bash[68594]: cluster 2024-09-13T14:52:44.604388+0000 mgr.y (mgr.26252) 794 : cluster [DBG] pgmap v1293: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:45.681 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:45 smithi177 bash[68594]: cluster 2024-09-13T14:52:44.604388+0000 mgr.y (mgr.26252) 794 : cluster [DBG] pgmap v1293: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:45.697 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:45.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:45.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:45 smithi099 bash[84576]: cluster 2024-09-13T14:52:44.604388+0000 mgr.y (mgr.26252) 794 : cluster [DBG] pgmap v1293: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:45.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:45 smithi099 bash[84576]: cluster 2024-09-13T14:52:44.604388+0000 mgr.y (mgr.26252) 794 : cluster [DBG] pgmap v1293: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:45.812 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:45 smithi099 bash[87147]: cluster 2024-09-13T14:52:44.604388+0000 mgr.y (mgr.26252) 794 : cluster [DBG] pgmap v1293: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:45.812 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:45 smithi099 bash[87147]: cluster 2024-09-13T14:52:44.604388+0000 mgr.y (mgr.26252) 794 : cluster [DBG] pgmap v1293: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:45.827 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:45.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:45.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:45.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:46.060 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:46.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:46.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:46.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:46.217 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:46.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:46.347 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:46.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:46.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:46.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:46.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:46.580 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:46.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:46.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:46.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:52:46.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:46.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:46.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:46.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:46.859 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:46.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:46.982 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:46.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:47.085 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.112 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:52:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:52:47] "GET /metrics HTTP/1.1" 200 36751 "" "Prometheus/2.51.0" 2024-09-13T14:52:47.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:47.203 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:47.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:47.236 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:47.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:47.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:47.366 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:47.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:47.469 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.496 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:47.598 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:47.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:47.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:47.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:47.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:47.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:47.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:47 smithi177 bash[68594]: cluster 2024-09-13T14:52:46.605012+0000 mgr.y (mgr.26252) 795 : cluster [DBG] pgmap v1294: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:47.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:47 smithi177 bash[68594]: cluster 2024-09-13T14:52:46.605012+0000 mgr.y (mgr.26252) 795 : cluster [DBG] pgmap v1294: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:48.012 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:48.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:48.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:47 smithi099 bash[84576]: cluster 2024-09-13T14:52:46.605012+0000 mgr.y (mgr.26252) 795 : cluster [DBG] pgmap v1294: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:48.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:47 smithi099 bash[84576]: cluster 2024-09-13T14:52:46.605012+0000 mgr.y (mgr.26252) 795 : cluster [DBG] pgmap v1294: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:48.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:47 smithi099 bash[87147]: cluster 2024-09-13T14:52:46.605012+0000 mgr.y (mgr.26252) 795 : cluster [DBG] pgmap v1294: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:48.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:47 smithi099 bash[87147]: cluster 2024-09-13T14:52:46.605012+0000 mgr.y (mgr.26252) 795 : cluster [DBG] pgmap v1294: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:48.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:48.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:48.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:48.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:48.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:48.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:48.404 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:48.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:48.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:48.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:48.637 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:48.663 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:48.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:48.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:48.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:48.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:48.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:48.923 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:48.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:49.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:49.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:49.156 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:49.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:49.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:49.233 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:49.286 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:49.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:49.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[84576]: audit 2024-09-13T14:52:48.283745+0000 mon.a (mon.0) 1826 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:49.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[84576]: audit 2024-09-13T14:52:48.283745+0000 mon.a (mon.0) 1826 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:49.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[84576]: audit 2024-09-13T14:52:48.284825+0000 mon.a (mon.0) 1827 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:49.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[84576]: audit 2024-09-13T14:52:48.284825+0000 mon.a (mon.0) 1827 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:49.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[84576]: cluster 2024-09-13T14:52:48.605573+0000 mgr.y (mgr.26252) 796 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:49.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[84576]: cluster 2024-09-13T14:52:48.605573+0000 mgr.y (mgr.26252) 796 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:49.308 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[87147]: audit 2024-09-13T14:52:48.283745+0000 mon.a (mon.0) 1826 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:49.308 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[87147]: audit 2024-09-13T14:52:48.283745+0000 mon.a (mon.0) 1826 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:49.309 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[87147]: audit 2024-09-13T14:52:48.284825+0000 mon.a (mon.0) 1827 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:49.309 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[87147]: audit 2024-09-13T14:52:48.284825+0000 mon.a (mon.0) 1827 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:49.309 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[87147]: cluster 2024-09-13T14:52:48.605573+0000 mgr.y (mgr.26252) 796 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:49.309 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:49 smithi099 bash[87147]: cluster 2024-09-13T14:52:48.605573+0000 mgr.y (mgr.26252) 796 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:49.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:49 smithi177 bash[68594]: audit 2024-09-13T14:52:48.283745+0000 mon.a (mon.0) 1826 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:49 smithi177 bash[68594]: audit 2024-09-13T14:52:48.283745+0000 mon.a (mon.0) 1826 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:49 smithi177 bash[68594]: audit 2024-09-13T14:52:48.284825+0000 mon.a (mon.0) 1827 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:49 smithi177 bash[68594]: audit 2024-09-13T14:52:48.284825+0000 mon.a (mon.0) 1827 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:49 smithi177 bash[68594]: cluster 2024-09-13T14:52:48.605573+0000 mgr.y (mgr.26252) 796 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:49 smithi177 bash[68594]: cluster 2024-09-13T14:52:48.605573+0000 mgr.y (mgr.26252) 796 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:49.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:49.438 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:49.541 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:49.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:49.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:49.670 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:49.684 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:52:49.684 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:52:49.684 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-13T14:52:49.684 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:52:49.684 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:52:49.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-13T14:52:49.699 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:49.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:49.802 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:49.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:49.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:49.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:49.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:50.056 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:50.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:50.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:50.185 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.212 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:50.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:50.326 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:52:50 smithi177 bash[61439]: ts=2024-09-13T14:52:50.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:52:50.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:50.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:50.471 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:50.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:50.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.602 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:50.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:50.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:50.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:50.841 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:50.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:50.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:50.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:51.103 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:51.124 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:51.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:51.227 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:51.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:51.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:51.258 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:51.356 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:51.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:51.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:51.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:51.513 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:51.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:51.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.645 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:51.748 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:51.760 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:51 smithi099 bash[84576]: cluster 2024-09-13T14:52:50.606255+0000 mgr.y (mgr.26252) 797 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:51.760 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:51 smithi099 bash[84576]: cluster 2024-09-13T14:52:50.606255+0000 mgr.y (mgr.26252) 797 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:51.760 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:51 smithi099 bash[87147]: cluster 2024-09-13T14:52:50.606255+0000 mgr.y (mgr.26252) 797 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:51.760 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:51 smithi099 bash[87147]: cluster 2024-09-13T14:52:50.606255+0000 mgr.y (mgr.26252) 797 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:51.777 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:51.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:51.880 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.891 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:51 smithi177 bash[68594]: cluster 2024-09-13T14:52:50.606255+0000 mgr.y (mgr.26252) 797 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:51.891 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:51 smithi177 bash[68594]: cluster 2024-09-13T14:52:50.606255+0000 mgr.y (mgr.26252) 797 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T14:52:51.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:52.011 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:52.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:52.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:52.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:52.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:52.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:52.273 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:52.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:52.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:52.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:52.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:52.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:52.546 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:52.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:52.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:52.677 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:52.702 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:52.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:52.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:52.834 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:52.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:52 smithi177 bash[68594]: audit 2024-09-13T14:52:52.401825+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:52 smithi177 bash[68594]: audit 2024-09-13T14:52:52.401825+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:52 smithi177 bash[68594]: audit 2024-09-13T14:52:52.402304+0000 mon.a (mon.0) 1829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]} : dispatch 2024-09-13T14:52:52.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:52 smithi177 bash[68594]: audit 2024-09-13T14:52:52.402304+0000 mon.a (mon.0) 1829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]} : dispatch 2024-09-13T14:52:52.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:52 smithi177 bash[68594]: audit 2024-09-13T14:52:52.402690+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:52 smithi177 bash[68594]: audit 2024-09-13T14:52:52.402690+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:52.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[84576]: audit 2024-09-13T14:52:52.401825+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[84576]: audit 2024-09-13T14:52:52.401825+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[84576]: audit 2024-09-13T14:52:52.402304+0000 mon.a (mon.0) 1829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]} : dispatch 2024-09-13T14:52:52.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[84576]: audit 2024-09-13T14:52:52.402304+0000 mon.a (mon.0) 1829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]} : dispatch 2024-09-13T14:52:52.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[84576]: audit 2024-09-13T14:52:52.402690+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[84576]: audit 2024-09-13T14:52:52.402690+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.950 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[87147]: audit 2024-09-13T14:52:52.401825+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.951 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[87147]: audit 2024-09-13T14:52:52.401825+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.951 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[87147]: audit 2024-09-13T14:52:52.402304+0000 mon.a (mon.0) 1829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]} : dispatch 2024-09-13T14:52:52.951 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[87147]: audit 2024-09-13T14:52:52.402304+0000 mon.a (mon.0) 1829 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]} : dispatch 2024-09-13T14:52:52.951 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[87147]: audit 2024-09-13T14:52:52.402690+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.951 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:52 smithi099 bash[87147]: audit 2024-09-13T14:52:52.402690+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]} : dispatch 2024-09-13T14:52:52.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.965 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:52.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:53.066 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:53.092 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:53.195 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:53.223 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:53.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:53.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:53.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:53.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:53.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:53.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:53.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:53.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:53.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:53.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:53.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:53.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:53.844 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:53.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:53 smithi177 bash[68594]: cluster 2024-09-13T14:52:52.606994+0000 mgr.y (mgr.26252) 798 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1 B/s, 0 objects/s recovering 2024-09-13T14:52:53.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:53 smithi177 bash[68594]: cluster 2024-09-13T14:52:52.606994+0000 mgr.y (mgr.26252) 798 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1 B/s, 0 objects/s recovering 2024-09-13T14:52:53.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:53 smithi177 bash[68594]: audit 2024-09-13T14:52:52.672062+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-13T14:52:53.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:53 smithi177 bash[68594]: audit 2024-09-13T14:52:52.672062+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-13T14:52:53.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:53 smithi177 bash[68594]: audit 2024-09-13T14:52:52.672253+0000 mon.a (mon.0) 1832 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2024-09-13T14:52:53.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:53 smithi177 bash[68594]: audit 2024-09-13T14:52:52.672253+0000 mon.a (mon.0) 1832 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2024-09-13T14:52:53.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:53 smithi177 bash[68594]: audit 2024-09-13T14:52:52.672420+0000 mon.a (mon.0) 1833 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-13T14:52:53.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:53 smithi177 bash[68594]: audit 2024-09-13T14:52:52.672420+0000 mon.a (mon.0) 1833 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-13T14:52:53.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:53 smithi177 bash[68594]: cluster 2024-09-13T14:52:52.678769+0000 mon.a (mon.0) 1834 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2024-09-13T14:52:53.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:53 smithi177 bash[68594]: cluster 2024-09-13T14:52:52.678769+0000 mon.a (mon.0) 1834 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2024-09-13T14:52:53.872 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:53.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:54.004 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:54.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[87147]: cluster 2024-09-13T14:52:52.606994+0000 mgr.y (mgr.26252) 798 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1 B/s, 0 objects/s recovering 2024-09-13T14:52:54.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[87147]: cluster 2024-09-13T14:52:52.606994+0000 mgr.y (mgr.26252) 798 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1 B/s, 0 objects/s recovering 2024-09-13T14:52:54.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[87147]: audit 2024-09-13T14:52:52.672062+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[87147]: audit 2024-09-13T14:52:52.672062+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[87147]: audit 2024-09-13T14:52:52.672253+0000 mon.a (mon.0) 1832 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[87147]: audit 2024-09-13T14:52:52.672253+0000 mon.a (mon.0) 1832 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[87147]: audit 2024-09-13T14:52:52.672420+0000 mon.a (mon.0) 1833 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[87147]: audit 2024-09-13T14:52:52.672420+0000 mon.a (mon.0) 1833 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[87147]: cluster 2024-09-13T14:52:52.678769+0000 mon.a (mon.0) 1834 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[87147]: cluster 2024-09-13T14:52:52.678769+0000 mon.a (mon.0) 1834 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[84576]: cluster 2024-09-13T14:52:52.606994+0000 mgr.y (mgr.26252) 798 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1 B/s, 0 objects/s recovering 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[84576]: cluster 2024-09-13T14:52:52.606994+0000 mgr.y (mgr.26252) 798 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1 B/s, 0 objects/s recovering 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[84576]: audit 2024-09-13T14:52:52.672062+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-13T14:52:54.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[84576]: audit 2024-09-13T14:52:52.672062+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-13T14:52:54.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[84576]: audit 2024-09-13T14:52:52.672253+0000 mon.a (mon.0) 1832 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2024-09-13T14:52:54.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[84576]: audit 2024-09-13T14:52:52.672253+0000 mon.a (mon.0) 1832 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2024-09-13T14:52:54.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[84576]: audit 2024-09-13T14:52:52.672420+0000 mon.a (mon.0) 1833 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-13T14:52:54.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[84576]: audit 2024-09-13T14:52:52.672420+0000 mon.a (mon.0) 1833 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-13T14:52:54.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[84576]: cluster 2024-09-13T14:52:52.678769+0000 mon.a (mon.0) 1834 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2024-09-13T14:52:54.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:53 smithi099 bash[84576]: cluster 2024-09-13T14:52:52.678769+0000 mon.a (mon.0) 1834 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2024-09-13T14:52:54.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:54.133 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:54.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:54.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:54.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7894 41.3 41:cff57933:::benchmark_data_smithi099_126887_object7893:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e957)", 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:45.659072+0000", 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 308.61043313599998, 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7221457610000002, 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.271 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7894 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.659072+0000", 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.659072+0000", 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.659073+0000", 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2130000000000001e-06 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.659100+0000", 2024-09-13T14:52:54.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7120000000000001e-05 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.659111+0000", 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0764000000000001e-05 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.659119+0000", 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3920000000000003e-06 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.659219+0000", 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9965999999999998e-05 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.659327+0000", 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010735599999999999 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:52:54.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.659422+0000", 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5303000000000006e-05 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.661706+0000", 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022840099999999999 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.689672+0000", 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.027965875000000001 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.689685+0000", 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2798e-05 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.323440+0000", 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.63375570199999998 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.323465+0000", 2024-09-13T14:52:54.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5096000000000002e-05 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.380984+0000", 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.057518505 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.381218+0000", 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000233661 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7907 41.3 41:c20fc544:::benchmark_data_smithi099_126887_object7906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e958)", 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:45.690191+0000", 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 308.57931360700002, 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7000133310000001, 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7907 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.279 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.690191+0000", 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.690191+0000", 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.690193+0000", 2024-09-13T14:52:54.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.367e-06 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.690218+0000", 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.525e-05 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.690228+0000", 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6239999999999997e-06 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.690235+0000", 2024-09-13T14:52:54.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0870000000000001e-06 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.691494+0000", 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012594450000000001 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:45.691504+0000", 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7580000000000004e-06 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.323509+0000", 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.63200473000000001 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.323522+0000", 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3057000000000001e-05 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.382041+0000", 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0585198039999999 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.382373+0000", 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00033162399999999998 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.382663+0000", 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000290134 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.385581+0000", 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029176290000000001 2024-09-13T14:52:54.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.385709+0000", 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127923 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.389452+0000", 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037433929999999998 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.389563+0000", 2024-09-13T14:52:54.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111088 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.390205+0000", 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00064141800000000004 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13529 41.3 41:cb165151:::benchmark_data_smithi099_126887_object13528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e962)", 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.103514+0000", 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 303.16599131999999, 2024-09-13T14:52:54.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2293936169999999, 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13529 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103514+0000", 2024-09-13T14:52:54.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103514+0000", 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103515+0000", 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4300000000000001e-06 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103534+0000", 2024-09-13T14:52:54.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9012e-05 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103546+0000", 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2046e-05 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103554+0000", 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0269999999999995e-06 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103759+0000", 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020447099999999999 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103961+0000", 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000202747 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104134+0000", 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017305200000000001 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.105364+0000", 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012296 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106011+0000", 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00064737799999999999 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.311283+0000", 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.20527171899999999 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.311298+0000", 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4482e-05 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.332677+0000", 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0213794549999999 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.332907+0000", 2024-09-13T14:52:54.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00023019800000000001 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13558 41.3 41:c780be55:::benchmark_data_smithi099_126887_object13557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.563718+0000", 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.70578717299998, 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.78020811899999998, 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.293 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13558 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563718+0000", 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563718+0000", 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563720+0000", 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9910000000000001e-06 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563783+0000", 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3008000000000004e-05 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563803+0000", 2024-09-13T14:52:54.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9850000000000001e-05 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563813+0000", 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0003000000000001e-05 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563913+0000", 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100708 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563926+0000", 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2198000000000001e-05 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.333539+0000", 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.76961326200000002 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.333668+0000", 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012934399999999999 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.333822+0000", 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015411600000000001 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.336862+0000", 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030395639999999998 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.337006+0000", 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014409199999999999 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.343823+0000", 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0068171009999999999 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.343881+0000", 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8443000000000003e-05 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.343926+0000", 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4438999999999999e-05 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.299 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13574 41.3 41:c6bd48c1:::benchmark_data_smithi099_126887_object13573:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.583970+0000", 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.68553517399999, 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.76008950200000003, 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.300 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13574 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.583970+0000", 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.583970+0000", 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.583971+0000", 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3769999999999999e-06 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.584038+0000", 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6919000000000001e-05 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.584056+0000", 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7657999999999999e-05 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.584065+0000", 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3700000000000001e-06 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.584137+0000", 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1748999999999994e-05 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.584147+0000", 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0397999999999999e-05 2024-09-13T14:52:54.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.334945+0000", 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.75079751299999997 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.335099+0000", 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000154604 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.335253+0000", 2024-09-13T14:52:54.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000153843 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.336905+0000", 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001652033 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.337011+0000", 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000105512 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.343969+0000", 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069578829999999998 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.344022+0000", 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3226000000000002e-05 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.344059+0000", 2024-09-13T14:52:54.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7416999999999998e-05 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13587 41.3 41:c1e87c62:::benchmark_data_smithi099_126887_object13586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.594832+0000", 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.67467249200001, 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.75299288099999995, 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.307 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13587 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.594832+0000", 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.594832+0000", 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.594833+0000", 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9999999999999995e-07 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.594873+0000", 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9978999999999998e-05 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.594884+0000", 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0919e-05 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.594892+0000", 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8350000000000004e-06 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.594945+0000", 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2794000000000003e-05 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.594953+0000", 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7400000000000004e-06 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.335913+0000", 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.74096056399999999 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.336046+0000", 2024-09-13T14:52:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013300000000000001 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.336185+0000", 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013887200000000001 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.337945+0000", 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017595480000000001 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.337982+0000", 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7224000000000001e-05 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.347696+0000", 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097140430000000003 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.347772+0000", 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6178000000000001e-05 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.347825+0000", 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3185e-05 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13592 41.3 41:c49eae01:::benchmark_data_smithi099_126887_object13591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.599032+0000", 2024-09-13T14:52:54.314 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.67047319400001, 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.74898030800000004, 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13592 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599032+0000", 2024-09-13T14:52:54.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599032+0000", 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599033+0000", 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.311e-06 2024-09-13T14:52:54.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599074+0000", 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0500000000000002e-05 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599086+0000", 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2928000000000001e-05 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599094+0000", 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.498e-06 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599154+0000", 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0340000000000002e-05 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599161+0000", 2024-09-13T14:52:54.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8589999999999999e-06 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.337039+0000", 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73787769400000003 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.337160+0000", 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120665 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.337295+0000", 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000135752 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.339019+0000", 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001723732 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.339838+0000", 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00081905699999999999 2024-09-13T14:52:54.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.347897+0000", 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0080589489999999993 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.347968+0000", 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1401000000000003e-05 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.348012+0000", 2024-09-13T14:52:54.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3621999999999997e-05 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13593 41.3 41:c27ae073:::benchmark_data_smithi099_126887_object13592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.599837+0000", 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.669668242, 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.74831583099999999, 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.322 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13593 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599837+0000", 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599837+0000", 2024-09-13T14:52:54.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599838+0000", 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.116e-06 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599877+0000", 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8998000000000002e-05 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599892+0000", 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5180999999999999e-05 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599901+0000", 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3940000000000002e-06 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599978+0000", 2024-09-13T14:52:54.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6178000000000001e-05 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.599989+0000", 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1460999999999999e-05 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.338014+0000", 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73802474900000004 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.338130+0000", 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116122 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.338266+0000", 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000135914 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.340606+0000", 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023398329999999999 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.340646+0000", 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0692000000000003e-05 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.348061+0000", 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0074149869999999996 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.348115+0000", 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3312000000000001e-05 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.348153+0000", 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7894e-05 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13615 41.3 41:cf652504:::benchmark_data_smithi099_126887_object13614:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.613365+0000", 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.65613991700002, 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73495050100000003, 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.329 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13615 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.613365+0000", 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.613365+0000", 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.613366+0000", 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.319e-06 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.613417+0000", 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0249e-05 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.613432+0000", 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5826999999999999e-05 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.613440+0000", 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6490000000000008e-06 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.613511+0000", 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0690000000000003e-05 2024-09-13T14:52:54.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.613533+0000", 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2711999999999999e-05 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.339074+0000", 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.725540988 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.339158+0000", 2024-09-13T14:52:54.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3534000000000004e-05 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.339285+0000", 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012678 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.340658+0000", 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013736740000000001 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.340696+0000", 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7064000000000003e-05 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.348216+0000", 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075205289999999998 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.348274+0000", 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7915999999999999e-05 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.348316+0000", 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1569999999999997e-05 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13656 41.3 41:c1b60e23:::benchmark_data_smithi099_126887_object13655:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.639998+0000", 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.62950687599999, 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71301172099999999, 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.336 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13656 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.639998+0000", 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.639998+0000", 2024-09-13T14:52:54.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.639999+0000", 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1909999999999999e-06 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.640028+0000", 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9159e-05 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.640040+0000", 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.181e-05 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.640065+0000", 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4309000000000001e-05 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.640128+0000", 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3324999999999994e-05 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.640144+0000", 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5741999999999999e-05 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.339878+0000", 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.69973443999999996 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.339972+0000", 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3636999999999998e-05 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.340095+0000", 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123328 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.342010+0000", 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019146479999999999 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.343778+0000", 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017686799999999999 2024-09-13T14:52:54.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.352906+0000", 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0091280619999999993 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.352964+0000", 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7627999999999998e-05 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.353010+0000", 2024-09-13T14:52:54.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5762e-05 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13674 41.3 41:c69d5465:::benchmark_data_smithi099_126887_object13673:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.650644+0000", 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.61886116199997, 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70257348600000002, 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.343 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13674 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.650644+0000", 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.650644+0000", 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.650645+0000", 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2279999999999999e-06 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.650680+0000", 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4662000000000001e-05 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.650690+0000", 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8490000000000001e-06 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.650697+0000", 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.34e-06 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.651071+0000", 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00037377899999999997 2024-09-13T14:52:54.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.651082+0000", 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.093e-05 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.340726+0000", 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.68964462699999995 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.340842+0000", 2024-09-13T14:52:54.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116146 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.340931+0000", 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8729999999999999e-05 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.342908+0000", 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001977215 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.344133+0000", 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001224478 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.353059+0000", 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0089257290000000003 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.353179+0000", 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120757 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.349 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.353217+0000", 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8016000000000003e-05 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13676 41.3 41:c5bcde6b:::benchmark_data_smithi099_126887_object13675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.651197+0000", 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.61830826800002, 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70234033200000001, 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.350 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13676 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.651197+0000", 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.651197+0000", 2024-09-13T14:52:54.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.651198+0000", 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0100000000000003e-07 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.651219+0000", 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1002999999999999e-05 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.651228+0000", 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6539999999999994e-06 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.651235+0000", 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8220000000000001e-06 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.651291+0000", 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5874000000000001e-05 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.651300+0000", 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3959999999999995e-06 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.341389+0000", 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.69008904100000001 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.354 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.341480+0000", 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0995999999999994e-05 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.341561+0000", 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0272000000000007e-05 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.343737+0000", 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002176117 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.355 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.345414+0000", 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016768449999999999 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.353424+0000", 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0080101840000000001 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.353491+0000", 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7609999999999998e-05 2024-09-13T14:52:54.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.353537+0000", 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5617e-05 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13720 41.3 41:c176b61c:::benchmark_data_smithi099_126887_object13719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.676164+0000", 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.59334108199999, 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.68072931599999997, 2024-09-13T14:52:54.357 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13720 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.676164+0000", 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.676164+0000", 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.676165+0000", 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4419999999999999e-06 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.676194+0000", 2024-09-13T14:52:54.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9130000000000001e-05 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.676210+0000", 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5790999999999998e-05 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.676219+0000", 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7630000000000006e-06 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.360 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.676292+0000", 2024-09-13T14:52:54.361 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2667000000000001e-05 2024-09-13T14:52:54.361 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.361 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.361 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.361 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.676304+0000", 2024-09-13T14:52:54.361 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2101e-05 2024-09-13T14:52:54.361 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.361 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.361 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.361 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.342041+0000", 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.665737303 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.342130+0000", 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9327999999999996e-05 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.342270+0000", 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000139402 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.362 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.343790+0000", 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015199440000000001 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.345438+0000", 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016481390000000001 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.356778+0000", 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011339973 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.363 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.356848+0000", 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0437000000000003e-05 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.356893+0000", 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4895999999999998e-05 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.364 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13735 41.3 41:cd3c474d:::benchmark_data_smithi099_126887_object13734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e963)", 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.687708+0000", 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 302.581796951, 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.66939357200000005, 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13735 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.365 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.687708+0000", 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.687708+0000", 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.687710+0000", 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.137e-06 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.687735+0000", 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5228999999999998e-05 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.687754+0000", 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.876e-05 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.687763+0000", 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0769999999999997e-06 2024-09-13T14:52:54.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.687845+0000", 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1787999999999997e-05 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.687868+0000", 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3472000000000002e-05 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.342964+0000", 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.65509514800000002 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.343098+0000", 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013407100000000001 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.343266+0000", 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016869399999999999 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.344069+0000", 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00080263800000000005 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.345457+0000", 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013875859999999999 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.357002+0000", 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011545771 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.357058+0000", 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5148000000000001e-05 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:52.357102+0000", 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4053e-05 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.371 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82477 41.b 41:d0d69cbf:::benchmark_data_smithi099_126887_object82476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e974)", 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.254589+0000", 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 245.01491609799999, 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2626653430000001, 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82477 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.372 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.254589+0000", 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.254589+0000", 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.254591+0000", 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2819999999999999e-06 2024-09-13T14:52:54.373 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.254625+0000", 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4097999999999998e-05 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.254652+0000", 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7129000000000001e-05 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.254669+0000", 2024-09-13T14:52:54.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6534000000000001e-05 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.254756+0000", 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7542999999999999e-05 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.254912+0000", 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015584299999999999 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.255058+0000", 2024-09-13T14:52:54.375 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000145702 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.257687+0000", 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026291660000000001 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261404+0000", 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037167939999999998 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.376 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311772+0000", 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.050368128999999998 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311784+0000", 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.201e-05 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.308050+0000", 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99626562299999999 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.308066+0000", 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6107e-05 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.517142+0000", 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2090757940000001 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.517254+0000", 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112589 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.378 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82479 41.b 41:d8ae91e9:::benchmark_data_smithi099_126887_object82478:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e974)", 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.255125+0000", 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 245.014379547, 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2655392220000001, 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82479 2024-09-13T14:52:54.379 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.255125+0000", 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.255125+0000", 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.380 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.255126+0000", 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.92e-07 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.255168+0000", 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1714000000000001e-05 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.255181+0000", 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3162e-05 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.381 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.255189+0000", 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9610000000000005e-06 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.257728+0000", 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025386279999999998 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.257928+0000", 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000200383 2024-09-13T14:52:54.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.258100+0000", 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017151499999999999 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.261563+0000", 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034635529999999999 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.267820+0000", 2024-09-13T14:52:54.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062572610000000001 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311807+0000", 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.043986988999999997 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311812+0000", 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6120000000000001e-06 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.308102+0000", 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99629034900000002 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.308112+0000", 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9860000000000006e-06 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.520527+0000", 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2124148880000001 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.520665+0000", 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000137429 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82510 41.b 41:d404e4fc:::benchmark_data_smithi099_126887_object82509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e974)", 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.291238+0000", 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 244.97826664799999, 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2396514230000002, 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.386 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82510 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.291238+0000", 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.387 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.291238+0000", 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.291240+0000", 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.736e-06 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.291280+0000", 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0141000000000001e-05 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.388 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.291291+0000", 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1112e-05 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.291298+0000", 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6390000000000003e-06 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311827+0000", 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020529353 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.311831+0000", 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5080000000000001e-06 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.308140+0000", 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99630886399999996 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.308147+0000", 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4819999999999997e-06 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.520692+0000", 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.212544839 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.520852+0000", 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016001700000000001 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2024-09-13T14:52:54.391 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.521034+0000", 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018164599999999999 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.524164+0000", 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031300080000000001 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.525544+0000", 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00137991 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.527486+0000", 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019429009999999999 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.527564+0000", 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7627999999999996e-05 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.530890+0000", 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003325639 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.393 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137582 41.b 41:d9931b21:::benchmark_data_smithi099_126887_object39794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.480696+0000", 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 161.78880885000001, 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.525429339, 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137582 2024-09-13T14:52:54.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.480696+0000", 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.480696+0000", 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.480698+0000", 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0949999999999998e-06 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.480705+0000", 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4169999999999997e-06 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.480722+0000", 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7227e-05 2024-09-13T14:52:54.396 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.480730+0000", 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6200000000000005e-06 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.508663+0000", 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.027932322999999998 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for rw locks", 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.508768+0000", 2024-09-13T14:52:54.397 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010504400000000001 2024-09-13T14:52:54.398 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.398 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.398 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.398 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.526018+0000", 2024-09-13T14:52:54.398 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017250481000000002 2024-09-13T14:52:54.398 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.398 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.398 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.399 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.526235+0000", 2024-09-13T14:52:54.400 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000216419 2024-09-13T14:52:54.400 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.400 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.400 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2024-09-13T14:52:54.400 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.526408+0000", 2024-09-13T14:52:54.400 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000173361 2024-09-13T14:52:54.400 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.400 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.400 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.985347+0000", 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45893924200000003 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.991664+0000", 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006317128 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.003547+0000", 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011882925000000001 2024-09-13T14:52:54.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.003660+0000", 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112129 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.006125+0000", 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002465928 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137584 41.b 41:deaf9ba1:::benchmark_data_smithi099_126887_object39796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.482377+0000", 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 161.78712820300001, 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.52109949099999997, 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137584 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.403 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.482377+0000", 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.482377+0000", 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.482378+0000", 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3519999999999999e-06 2024-09-13T14:52:54.404 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.482383+0000", 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5240000000000002e-06 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.482398+0000", 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5235000000000001e-05 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.482405+0000", 2024-09-13T14:52:54.405 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.939e-06 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.509051+0000", 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.026646436999999999 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.509167+0000", 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115501 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.406 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.509295+0000", 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012847100000000001 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.516456+0000", 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0071612439999999998 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.516471+0000", 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4218e-05 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.000670+0000", 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.48419937600000001 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.000733+0000", 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2699000000000004e-05 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.003476+0000", 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027434949999999999 2024-09-13T14:52:54.408 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137609 41.b 41:d77896f1:::benchmark_data_smithi099_126887_object39821:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.520268+0000", 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 161.74923703100001, 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.48916475199999998, 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:52:54.409 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137609 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.520268+0000", 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.520268+0000", 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:52:54.410 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.520270+0000", 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5859999999999999e-06 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.520275+0000", 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5890000000000002e-06 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.520291+0000", 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5658000000000002e-05 2024-09-13T14:52:54.411 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.520299+0000", 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3010000000000007e-06 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.985526+0000", 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.46522709299999998 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.985817+0000", 2024-09-13T14:52:54.412 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000291188 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.986143+0000", 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00032548700000000001 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.006182+0000", 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020039653000000001 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:52:54.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.006198+0000", 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.541e-05 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.006243+0000", 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4687000000000001e-05 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.006324+0000", 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1186999999999994e-05 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:52:54.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:52:54.415 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.009433+0000", 2024-09-13T14:52:54.415 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003108913 2024-09-13T14:52:54.415 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.415 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.415 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.415 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:52:54.415 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:52:54.415 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:52:54.415 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:54.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:54.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:54.541 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:54.566 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:54.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:54.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:54.691 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:54.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:54.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:54.819 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:54.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:54.907 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:54 smithi099 bash[84576]: cluster 2024-09-13T14:52:53.685193+0000 mon.a (mon.0) 1835 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2024-09-13T14:52:54.907 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:54 smithi099 bash[84576]: cluster 2024-09-13T14:52:53.685193+0000 mon.a (mon.0) 1835 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2024-09-13T14:52:54.907 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:54 smithi099 bash[84576]: audit 2024-09-13T14:52:54.372959+0000 mon.a (mon.0) 1836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:54.907 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:54 smithi099 bash[84576]: audit 2024-09-13T14:52:54.372959+0000 mon.a (mon.0) 1836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:54.908 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:54 smithi099 bash[87147]: cluster 2024-09-13T14:52:53.685193+0000 mon.a (mon.0) 1835 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2024-09-13T14:52:54.908 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:54 smithi099 bash[87147]: cluster 2024-09-13T14:52:53.685193+0000 mon.a (mon.0) 1835 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2024-09-13T14:52:54.908 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:54 smithi099 bash[87147]: audit 2024-09-13T14:52:54.372959+0000 mon.a (mon.0) 1836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:54.908 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:54 smithi099 bash[87147]: audit 2024-09-13T14:52:54.372959+0000 mon.a (mon.0) 1836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:54.922 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:54.933 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:54 smithi177 bash[68594]: cluster 2024-09-13T14:52:53.685193+0000 mon.a (mon.0) 1835 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2024-09-13T14:52:54.933 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:54 smithi177 bash[68594]: cluster 2024-09-13T14:52:53.685193+0000 mon.a (mon.0) 1835 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2024-09-13T14:52:54.933 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:54 smithi177 bash[68594]: audit 2024-09-13T14:52:54.372959+0000 mon.a (mon.0) 1836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:54.933 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:54 smithi177 bash[68594]: audit 2024-09-13T14:52:54.372959+0000 mon.a (mon.0) 1836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:52:54.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:54.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:55.053 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:55.082 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:55.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:55.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:55.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:55.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:55.317 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:55.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:55.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:55.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:55.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:55.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:55.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:55.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:55.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:55.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:55.715 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:55 smithi177 bash[68594]: cluster 2024-09-13T14:52:54.607760+0000 mgr.y (mgr.26252) 799 : cluster [DBG] pgmap v1300: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:52:55.715 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:55 smithi177 bash[68594]: cluster 2024-09-13T14:52:54.607760+0000 mgr.y (mgr.26252) 799 : cluster [DBG] pgmap v1300: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:52:55.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:55.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:55.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:55.863 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:55.967 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:55.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:55.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:56.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:55 smithi099 bash[84576]: cluster 2024-09-13T14:52:54.607760+0000 mgr.y (mgr.26252) 799 : cluster [DBG] pgmap v1300: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:52:56.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:55 smithi099 bash[84576]: cluster 2024-09-13T14:52:54.607760+0000 mgr.y (mgr.26252) 799 : cluster [DBG] pgmap v1300: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:52:56.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:55 smithi099 bash[87147]: cluster 2024-09-13T14:52:54.607760+0000 mgr.y (mgr.26252) 799 : cluster [DBG] pgmap v1300: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:52:56.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:55 smithi099 bash[87147]: cluster 2024-09-13T14:52:54.607760+0000 mgr.y (mgr.26252) 799 : cluster [DBG] pgmap v1300: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:52:56.098 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:56.126 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:56.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:56.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:56.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:56.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:56.386 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:56.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:56.489 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:56.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:56.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:56.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:56.644 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:56.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:56.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:56.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:56.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:56.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:56.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:56.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:57.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:57.030 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:57.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:57.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:57.160 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:57.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:57.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:57.275 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:52:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:52:57] "GET /metrics HTTP/1.1" 200 34340 "" "Prometheus/2.51.0" 2024-09-13T14:52:57.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:57.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:57.422 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:57.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:57.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:57.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:57.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:57.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:57.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:57.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:57.786 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:57.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:57.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:57 smithi177 bash[68594]: cluster 2024-09-13T14:52:56.608153+0000 mgr.y (mgr.26252) 800 : cluster [DBG] pgmap v1301: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:57.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:57 smithi177 bash[68594]: cluster 2024-09-13T14:52:56.608153+0000 mgr.y (mgr.26252) 800 : cluster [DBG] pgmap v1301: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:57.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:57.923 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:57 smithi099 bash[84576]: cluster 2024-09-13T14:52:56.608153+0000 mgr.y (mgr.26252) 800 : cluster [DBG] pgmap v1301: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:57.924 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:57 smithi099 bash[84576]: cluster 2024-09-13T14:52:56.608153+0000 mgr.y (mgr.26252) 800 : cluster [DBG] pgmap v1301: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:57.924 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:57 smithi099 bash[87147]: cluster 2024-09-13T14:52:56.608153+0000 mgr.y (mgr.26252) 800 : cluster [DBG] pgmap v1301: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:57.924 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:57 smithi099 bash[87147]: cluster 2024-09-13T14:52:56.608153+0000 mgr.y (mgr.26252) 800 : cluster [DBG] pgmap v1301: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:57.937 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:57.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:58.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:58.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:58.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:58.169 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:58.195 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:58.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:58.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:52:58.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:58.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:58.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:58.427 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:58.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:58.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:58.550 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:58.577 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:58.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:58.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:58.708 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:58.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:58.811 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:58.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:58.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:58.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:58.968 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:59.071 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:59.096 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:59.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:59.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:59.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:59.332 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:59.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:59.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:59.463 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:59.487 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:59.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:59.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:59.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:59.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:59.725 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:59.737 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:59 smithi099 bash[87147]: cluster 2024-09-13T14:52:58.608716+0000 mgr.y (mgr.26252) 801 : cluster [DBG] pgmap v1302: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:59.737 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:52:59 smithi099 bash[87147]: cluster 2024-09-13T14:52:58.608716+0000 mgr.y (mgr.26252) 801 : cluster [DBG] pgmap v1302: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:59.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:59 smithi099 bash[84576]: cluster 2024-09-13T14:52:58.608716+0000 mgr.y (mgr.26252) 801 : cluster [DBG] pgmap v1302: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:59.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:52:59 smithi099 bash[84576]: cluster 2024-09-13T14:52:58.608716+0000 mgr.y (mgr.26252) 801 : cluster [DBG] pgmap v1302: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:59.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:59.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:59.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:59 smithi177 bash[68594]: cluster 2024-09-13T14:52:58.608716+0000 mgr.y (mgr.26252) 801 : cluster [DBG] pgmap v1302: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:59.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:52:59 smithi177 bash[68594]: cluster 2024-09-13T14:52:58.608716+0000 mgr.y (mgr.26252) 801 : cluster [DBG] pgmap v1302: 73 pgs: 2 activating, 71 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:52:59.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:59.986 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:00.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:00.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:00.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:00.139 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:00.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:00.242 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:00.266 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:00.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:00.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:00.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:00.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:00.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:00.521 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:00.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:00.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:53:00 smithi177 bash[61439]: ts=2024-09-13T14:53:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:00.624 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:00.652 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:00.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:00.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:00.783 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:00.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:00.886 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:00.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:01.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:01.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:01.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:01.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:01.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:01.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:01.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:01.294 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:01.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:01.397 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:01.419 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:01.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:01.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:01.545 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:01.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:01.667 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:01.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:01.701 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_1' created 2024-09-13T14:53:01.769 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:01.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:01.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:01.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:01.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:01.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:02.021 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:02.032 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[87147]: cluster 2024-09-13T14:53:00.609239+0000 mgr.y (mgr.26252) 802 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:02.032 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[87147]: cluster 2024-09-13T14:53:00.609239+0000 mgr.y (mgr.26252) 802 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:02.032 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[87147]: audit 2024-09-13T14:53:01.091696+0000 mon.a (mon.0) 1837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:53:02.032 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[87147]: audit 2024-09-13T14:53:01.091696+0000 mon.a (mon.0) 1837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:53:02.032 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[87147]: audit 2024-09-13T14:53:01.468265+0000 mon.a (mon.0) 1838 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:53:02.032 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[87147]: audit 2024-09-13T14:53:01.468265+0000 mon.a (mon.0) 1838 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:53:02.032 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[87147]: audit 2024-09-13T14:53:01.688740+0000 mon.a (mon.0) 1839 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[87147]: audit 2024-09-13T14:53:01.688740+0000 mon.a (mon.0) 1839 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[87147]: audit 2024-09-13T14:53:01.693419+0000 mon.a (mon.0) 1840 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[87147]: audit 2024-09-13T14:53:01.693419+0000 mon.a (mon.0) 1840 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[84576]: cluster 2024-09-13T14:53:00.609239+0000 mgr.y (mgr.26252) 802 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[84576]: cluster 2024-09-13T14:53:00.609239+0000 mgr.y (mgr.26252) 802 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[84576]: audit 2024-09-13T14:53:01.091696+0000 mon.a (mon.0) 1837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[84576]: audit 2024-09-13T14:53:01.091696+0000 mon.a (mon.0) 1837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[84576]: audit 2024-09-13T14:53:01.468265+0000 mon.a (mon.0) 1838 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[84576]: audit 2024-09-13T14:53:01.468265+0000 mon.a (mon.0) 1838 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[84576]: audit 2024-09-13T14:53:01.688740+0000 mon.a (mon.0) 1839 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[84576]: audit 2024-09-13T14:53:01.688740+0000 mon.a (mon.0) 1839 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.034 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[84576]: audit 2024-09-13T14:53:01.693419+0000 mon.a (mon.0) 1840 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.034 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:01 smithi099 bash[84576]: audit 2024-09-13T14:53:01.693419+0000 mon.a (mon.0) 1840 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.047 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:02.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:01 smithi177 bash[68594]: cluster 2024-09-13T14:53:00.609239+0000 mgr.y (mgr.26252) 802 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:02.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:01 smithi177 bash[68594]: cluster 2024-09-13T14:53:00.609239+0000 mgr.y (mgr.26252) 802 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:02.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:01 smithi177 bash[68594]: audit 2024-09-13T14:53:01.091696+0000 mon.a (mon.0) 1837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:53:02.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:01 smithi177 bash[68594]: audit 2024-09-13T14:53:01.091696+0000 mon.a (mon.0) 1837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:53:02.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:01 smithi177 bash[68594]: audit 2024-09-13T14:53:01.468265+0000 mon.a (mon.0) 1838 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:53:02.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:01 smithi177 bash[68594]: audit 2024-09-13T14:53:01.468265+0000 mon.a (mon.0) 1838 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-13T14:53:02.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:01 smithi177 bash[68594]: audit 2024-09-13T14:53:01.688740+0000 mon.a (mon.0) 1839 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:01 smithi177 bash[68594]: audit 2024-09-13T14:53:01.688740+0000 mon.a (mon.0) 1839 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:01 smithi177 bash[68594]: audit 2024-09-13T14:53:01.693419+0000 mon.a (mon.0) 1840 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:01 smithi177 bash[68594]: audit 2024-09-13T14:53:01.693419+0000 mon.a (mon.0) 1840 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:02.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:02.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:02.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:02.279 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:02.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:02.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:02.406 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:02.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:02.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:02.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:02.746 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:02.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:02.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:02.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:02.973 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:53:02.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:02.987 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:02 smithi177 bash[68594]: audit 2024-09-13T14:53:01.704685+0000 mon.a (mon.0) 1841 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:53:02.987 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:02 smithi177 bash[68594]: audit 2024-09-13T14:53:01.704685+0000 mon.a (mon.0) 1841 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:53:02.987 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:02 smithi177 bash[68594]: cluster 2024-09-13T14:53:01.709350+0000 mon.a (mon.0) 1842 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2024-09-13T14:53:02.987 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:02 smithi177 bash[68594]: cluster 2024-09-13T14:53:01.709350+0000 mon.a (mon.0) 1842 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2024-09-13T14:53:03.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:03.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:03.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:02 smithi099 bash[84576]: audit 2024-09-13T14:53:01.704685+0000 mon.a (mon.0) 1841 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:53:03.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:02 smithi099 bash[84576]: audit 2024-09-13T14:53:01.704685+0000 mon.a (mon.0) 1841 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:53:03.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:02 smithi099 bash[84576]: cluster 2024-09-13T14:53:01.709350+0000 mon.a (mon.0) 1842 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2024-09-13T14:53:03.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:02 smithi099 bash[84576]: cluster 2024-09-13T14:53:01.709350+0000 mon.a (mon.0) 1842 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2024-09-13T14:53:03.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:02 smithi099 bash[87147]: audit 2024-09-13T14:53:01.704685+0000 mon.a (mon.0) 1841 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:53:03.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:02 smithi099 bash[87147]: audit 2024-09-13T14:53:01.704685+0000 mon.a (mon.0) 1841 : audit [INF] from='client.? 172.21.15.99:0/3542823447' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:53:03.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:02 smithi099 bash[87147]: cluster 2024-09-13T14:53:01.709350+0000 mon.a (mon.0) 1842 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2024-09-13T14:53:03.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:02 smithi099 bash[87147]: cluster 2024-09-13T14:53:01.709350+0000 mon.a (mon.0) 1842 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2024-09-13T14:53:03.103 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:03.129 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:03.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:03.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:03.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:03.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:03.361 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:03.389 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:03.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:03.494 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:03.518 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:03.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:03.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:03.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:03.685 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:53:03.685 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:03.702 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:53:03.703 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:03.720 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:53:03.720 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: cluster 2024-09-13T14:53:02.610131+0000 mgr.y (mgr.26252) 803 : cluster [DBG] pgmap v1305: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: cluster 2024-09-13T14:53:02.610131+0000 mgr.y (mgr.26252) 803 : cluster [DBG] pgmap v1305: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: cluster 2024-09-13T14:53:02.706301+0000 mon.a (mon.0) 1843 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: cluster 2024-09-13T14:53:02.706301+0000 mon.a (mon.0) 1843 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: cluster 2024-09-13T14:53:02.714838+0000 mon.a (mon.0) 1844 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: cluster 2024-09-13T14:53:02.714838+0000 mon.a (mon.0) 1844 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: audit 2024-09-13T14:53:03.284772+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: audit 2024-09-13T14:53:03.284772+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: audit 2024-09-13T14:53:03.285871+0000 mon.a (mon.0) 1846 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: audit 2024-09-13T14:53:03.285871+0000 mon.a (mon.0) 1846 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: audit 2024-09-13T14:53:03.670433+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.99:0/1457849675' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:03 smithi177 bash[68594]: audit 2024-09-13T14:53:03.670433+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.99:0/1457849675' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.742 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:53:03.742 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:03.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:03.763 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:53:03.763 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:03.764 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:03.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:03.781 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:53:03.781 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:03.800 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:53:03.801 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:03.820 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:53:03.820 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: cluster 2024-09-13T14:53:02.610131+0000 mgr.y (mgr.26252) 803 : cluster [DBG] pgmap v1305: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: cluster 2024-09-13T14:53:02.610131+0000 mgr.y (mgr.26252) 803 : cluster [DBG] pgmap v1305: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: cluster 2024-09-13T14:53:02.706301+0000 mon.a (mon.0) 1843 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:53:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: cluster 2024-09-13T14:53:02.706301+0000 mon.a (mon.0) 1843 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:53:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: cluster 2024-09-13T14:53:02.714838+0000 mon.a (mon.0) 1844 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2024-09-13T14:53:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: cluster 2024-09-13T14:53:02.714838+0000 mon.a (mon.0) 1844 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2024-09-13T14:53:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: audit 2024-09-13T14:53:03.284772+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: audit 2024-09-13T14:53:03.284772+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: audit 2024-09-13T14:53:03.285871+0000 mon.a (mon.0) 1846 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: audit 2024-09-13T14:53:03.285871+0000 mon.a (mon.0) 1846 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: audit 2024-09-13T14:53:03.670433+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.99:0/1457849675' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[84576]: audit 2024-09-13T14:53:03.670433+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.99:0/1457849675' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: cluster 2024-09-13T14:53:02.610131+0000 mgr.y (mgr.26252) 803 : cluster [DBG] pgmap v1305: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: cluster 2024-09-13T14:53:02.610131+0000 mgr.y (mgr.26252) 803 : cluster [DBG] pgmap v1305: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: cluster 2024-09-13T14:53:02.706301+0000 mon.a (mon.0) 1843 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: cluster 2024-09-13T14:53:02.706301+0000 mon.a (mon.0) 1843 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: cluster 2024-09-13T14:53:02.714838+0000 mon.a (mon.0) 1844 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: cluster 2024-09-13T14:53:02.714838+0000 mon.a (mon.0) 1844 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: audit 2024-09-13T14:53:03.284772+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: audit 2024-09-13T14:53:03.284772+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: audit 2024-09-13T14:53:03.285871+0000 mon.a (mon.0) 1846 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: audit 2024-09-13T14:53:03.285871+0000 mon.a (mon.0) 1846 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: audit 2024-09-13T14:53:03.670433+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.99:0/1457849675' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:03 smithi099 bash[87147]: audit 2024-09-13T14:53:03.670433+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.99:0/1457849675' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:03.866 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:03.887 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:03.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:03.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:04.119 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:04.185 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-09-13T14:53:04.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:04.268 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:04.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:04.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:04.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:04.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:04.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:04.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:04.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:04.620 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:04.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:04.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:04.748 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:04.751 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T14:53:04.765 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[84576]: cluster 2024-09-13T14:53:03.728906+0000 mon.a (mon.0) 1847 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2024-09-13T14:53:04.765 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[84576]: cluster 2024-09-13T14:53:03.728906+0000 mon.a (mon.0) 1847 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2024-09-13T14:53:04.766 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[84576]: audit 2024-09-13T14:53:04.516706+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.99:0/3423412557' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:04.766 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[84576]: audit 2024-09-13T14:53:04.516706+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.99:0/3423412557' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:04.766 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[84576]: audit 2024-09-13T14:53:04.517070+0000 mon.a (mon.0) 1848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:04.766 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[84576]: audit 2024-09-13T14:53:04.517070+0000 mon.a (mon.0) 1848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:04.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[87147]: cluster 2024-09-13T14:53:03.728906+0000 mon.a (mon.0) 1847 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2024-09-13T14:53:04.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[87147]: cluster 2024-09-13T14:53:03.728906+0000 mon.a (mon.0) 1847 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2024-09-13T14:53:04.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[87147]: audit 2024-09-13T14:53:04.516706+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.99:0/3423412557' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:04.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[87147]: audit 2024-09-13T14:53:04.516706+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.99:0/3423412557' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:04.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[87147]: audit 2024-09-13T14:53:04.517070+0000 mon.a (mon.0) 1848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:04.766 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:04 smithi099 bash[87147]: audit 2024-09-13T14:53:04.517070+0000 mon.a (mon.0) 1848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:04.769 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:04.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:04.872 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:04.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:05.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:05.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:05.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:04 smithi177 bash[68594]: cluster 2024-09-13T14:53:03.728906+0000 mon.a (mon.0) 1847 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2024-09-13T14:53:05.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:04 smithi177 bash[68594]: cluster 2024-09-13T14:53:03.728906+0000 mon.a (mon.0) 1847 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2024-09-13T14:53:05.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:04 smithi177 bash[68594]: audit 2024-09-13T14:53:04.516706+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.99:0/3423412557' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:05.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:04 smithi177 bash[68594]: audit 2024-09-13T14:53:04.516706+0000 mon.c (mon.2) 60 : audit [INF] from='client.? 172.21.15.99:0/3423412557' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:05.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:04 smithi177 bash[68594]: audit 2024-09-13T14:53:04.517070+0000 mon.a (mon.0) 1848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:05.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:04 smithi177 bash[68594]: audit 2024-09-13T14:53:04.517070+0000 mon.a (mon.0) 1848 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:05.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:05.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:05.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:05.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:05.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:05.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:05.386 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:05.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:05.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:05.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:05.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:05.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:05.647 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:05.674 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:05.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:05.777 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:05.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:05.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:05.929 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:05.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:06.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:06.036 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[87147]: cluster 2024-09-13T14:53:04.611156+0000 mgr.y (mgr.26252) 804 : cluster [DBG] pgmap v1308: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:06.036 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[87147]: cluster 2024-09-13T14:53:04.611156+0000 mgr.y (mgr.26252) 804 : cluster [DBG] pgmap v1308: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:06.036 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[87147]: cluster 2024-09-13T14:53:04.736154+0000 mon.a (mon.0) 1849 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:06.036 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[87147]: cluster 2024-09-13T14:53:04.736154+0000 mon.a (mon.0) 1849 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:06.036 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[87147]: audit 2024-09-13T14:53:04.747581+0000 mon.a (mon.0) 1850 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:06.036 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[87147]: audit 2024-09-13T14:53:04.747581+0000 mon.a (mon.0) 1850 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:06.037 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[87147]: cluster 2024-09-13T14:53:04.755202+0000 mon.a (mon.0) 1851 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2024-09-13T14:53:06.037 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[87147]: cluster 2024-09-13T14:53:04.755202+0000 mon.a (mon.0) 1851 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2024-09-13T14:53:06.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[84576]: cluster 2024-09-13T14:53:04.611156+0000 mgr.y (mgr.26252) 804 : cluster [DBG] pgmap v1308: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:06.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[84576]: cluster 2024-09-13T14:53:04.611156+0000 mgr.y (mgr.26252) 804 : cluster [DBG] pgmap v1308: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:06.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[84576]: cluster 2024-09-13T14:53:04.736154+0000 mon.a (mon.0) 1849 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:06.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[84576]: cluster 2024-09-13T14:53:04.736154+0000 mon.a (mon.0) 1849 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:06.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[84576]: audit 2024-09-13T14:53:04.747581+0000 mon.a (mon.0) 1850 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:06.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[84576]: audit 2024-09-13T14:53:04.747581+0000 mon.a (mon.0) 1850 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:06.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[84576]: cluster 2024-09-13T14:53:04.755202+0000 mon.a (mon.0) 1851 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2024-09-13T14:53:06.037 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:05 smithi099 bash[84576]: cluster 2024-09-13T14:53:04.755202+0000 mon.a (mon.0) 1851 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2024-09-13T14:53:06.043 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:05 smithi177 bash[68594]: cluster 2024-09-13T14:53:04.611156+0000 mgr.y (mgr.26252) 804 : cluster [DBG] pgmap v1308: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:06.043 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:05 smithi177 bash[68594]: cluster 2024-09-13T14:53:04.611156+0000 mgr.y (mgr.26252) 804 : cluster [DBG] pgmap v1308: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:06.043 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:05 smithi177 bash[68594]: cluster 2024-09-13T14:53:04.736154+0000 mon.a (mon.0) 1849 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:06.043 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:05 smithi177 bash[68594]: cluster 2024-09-13T14:53:04.736154+0000 mon.a (mon.0) 1849 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:06.043 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:05 smithi177 bash[68594]: audit 2024-09-13T14:53:04.747581+0000 mon.a (mon.0) 1850 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:06.043 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:05 smithi177 bash[68594]: audit 2024-09-13T14:53:04.747581+0000 mon.a (mon.0) 1850 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:06.044 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:05 smithi177 bash[68594]: cluster 2024-09-13T14:53:04.755202+0000 mon.a (mon.0) 1851 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2024-09-13T14:53:06.044 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:05 smithi177 bash[68594]: cluster 2024-09-13T14:53:04.755202+0000 mon.a (mon.0) 1851 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2024-09-13T14:53:06.058 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:06.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:06.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:06.188 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:06.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:06.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:06.314 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:06.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:06.418 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:06.444 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:06.457 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:06.547 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:06.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:06.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:06.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:06.698 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:06.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:06.800 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:06.820 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:06.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:06.924 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:06.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:06.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:07.054 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:07.078 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:07.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:07.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:07.192 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:53:07] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-13T14:53:07.205 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:07.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:07.308 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:07.327 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:07.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:07.354 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T14:53:07.354 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:53:07.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:53:07.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:07.449 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:07.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:07.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:07.571 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:07.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:07.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:07.697 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:07.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:07.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:07.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[84576]: cluster 2024-09-13T14:53:06.611804+0000 mgr.y (mgr.26252) 805 : cluster [DBG] pgmap v1310: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:07.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[84576]: cluster 2024-09-13T14:53:06.611804+0000 mgr.y (mgr.26252) 805 : cluster [DBG] pgmap v1310: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:07.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[84576]: audit 2024-09-13T14:53:07.253958+0000 mon.a (mon.0) 1852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[84576]: audit 2024-09-13T14:53:07.253958+0000 mon.a (mon.0) 1852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.813 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[84576]: audit 2024-09-13T14:53:07.260312+0000 mon.a (mon.0) 1853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.813 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[84576]: audit 2024-09-13T14:53:07.260312+0000 mon.a (mon.0) 1853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.813 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[87147]: cluster 2024-09-13T14:53:06.611804+0000 mgr.y (mgr.26252) 805 : cluster [DBG] pgmap v1310: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:07.813 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[87147]: cluster 2024-09-13T14:53:06.611804+0000 mgr.y (mgr.26252) 805 : cluster [DBG] pgmap v1310: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:07.813 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[87147]: audit 2024-09-13T14:53:07.253958+0000 mon.a (mon.0) 1852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.813 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[87147]: audit 2024-09-13T14:53:07.253958+0000 mon.a (mon.0) 1852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.813 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[87147]: audit 2024-09-13T14:53:07.260312+0000 mon.a (mon.0) 1853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.813 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:07 smithi099 bash[87147]: audit 2024-09-13T14:53:07.260312+0000 mon.a (mon.0) 1853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.828 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:07.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:07.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:07.944 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:07 smithi177 bash[68594]: cluster 2024-09-13T14:53:06.611804+0000 mgr.y (mgr.26252) 805 : cluster [DBG] pgmap v1310: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:07.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:07 smithi177 bash[68594]: cluster 2024-09-13T14:53:06.611804+0000 mgr.y (mgr.26252) 805 : cluster [DBG] pgmap v1310: 89 pgs: 1 creating+activating, 2 creating+peering, 86 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:07.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:07 smithi177 bash[68594]: audit 2024-09-13T14:53:07.253958+0000 mon.a (mon.0) 1852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:07 smithi177 bash[68594]: audit 2024-09-13T14:53:07.253958+0000 mon.a (mon.0) 1852 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:07 smithi177 bash[68594]: audit 2024-09-13T14:53:07.260312+0000 mon.a (mon.0) 1853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:07 smithi177 bash[68594]: audit 2024-09-13T14:53:07.260312+0000 mon.a (mon.0) 1853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:07.958 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:07.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:08.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:53:08.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:08.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:08.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:08.189 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:08.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:08.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:08.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:53:08.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:08.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:08.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:08.449 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:08.467 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:08.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:08.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:08.594 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:08.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:08.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:08.720 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:08.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:08.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:08.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:08.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:08.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:08.969 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:08.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:09.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:09.091 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:09.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:09.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:09.213 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:09.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:09.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:09.343 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:09.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:09.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:09.473 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:09.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:09.567 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:09.576 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:09.596 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:09.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:09.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:09.720 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:09.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:09.823 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:09.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:09.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:09.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:09.980 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:09.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[87147]: cluster 2024-09-13T14:53:08.612440+0000 mgr.y (mgr.26252) 806 : cluster [DBG] pgmap v1311: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:09.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[87147]: cluster 2024-09-13T14:53:08.612440+0000 mgr.y (mgr.26252) 806 : cluster [DBG] pgmap v1311: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:09.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[87147]: audit 2024-09-13T14:53:09.198179+0000 mon.a (mon.0) 1854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[87147]: audit 2024-09-13T14:53:09.198179+0000 mon.a (mon.0) 1854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[87147]: audit 2024-09-13T14:53:09.204565+0000 mon.a (mon.0) 1855 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[87147]: audit 2024-09-13T14:53:09.204565+0000 mon.a (mon.0) 1855 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[87147]: audit 2024-09-13T14:53:09.385983+0000 mon.a (mon.0) 1856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[87147]: audit 2024-09-13T14:53:09.385983+0000 mon.a (mon.0) 1856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.986 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[84576]: cluster 2024-09-13T14:53:08.612440+0000 mgr.y (mgr.26252) 806 : cluster [DBG] pgmap v1311: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:09.986 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[84576]: cluster 2024-09-13T14:53:08.612440+0000 mgr.y (mgr.26252) 806 : cluster [DBG] pgmap v1311: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:09.986 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[84576]: audit 2024-09-13T14:53:09.198179+0000 mon.a (mon.0) 1854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.986 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[84576]: audit 2024-09-13T14:53:09.198179+0000 mon.a (mon.0) 1854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.986 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[84576]: audit 2024-09-13T14:53:09.204565+0000 mon.a (mon.0) 1855 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.986 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[84576]: audit 2024-09-13T14:53:09.204565+0000 mon.a (mon.0) 1855 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.986 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[84576]: audit 2024-09-13T14:53:09.385983+0000 mon.a (mon.0) 1856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:09.986 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:09 smithi099 bash[84576]: audit 2024-09-13T14:53:09.385983+0000 mon.a (mon.0) 1856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:10.083 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:10.108 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:09 smithi177 bash[68594]: cluster 2024-09-13T14:53:08.612440+0000 mgr.y (mgr.26252) 806 : cluster [DBG] pgmap v1311: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:09 smithi177 bash[68594]: cluster 2024-09-13T14:53:08.612440+0000 mgr.y (mgr.26252) 806 : cluster [DBG] pgmap v1311: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:09 smithi177 bash[68594]: audit 2024-09-13T14:53:09.198179+0000 mon.a (mon.0) 1854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:09 smithi177 bash[68594]: audit 2024-09-13T14:53:09.198179+0000 mon.a (mon.0) 1854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:09 smithi177 bash[68594]: audit 2024-09-13T14:53:09.204565+0000 mon.a (mon.0) 1855 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:09 smithi177 bash[68594]: audit 2024-09-13T14:53:09.204565+0000 mon.a (mon.0) 1855 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:09 smithi177 bash[68594]: audit 2024-09-13T14:53:09.385983+0000 mon.a (mon.0) 1856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:09 smithi177 bash[68594]: audit 2024-09-13T14:53:09.385983+0000 mon.a (mon.0) 1856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:10.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:10.239 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:10.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:10.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:10.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:10.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:10.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:10.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:10.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:10.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:53:10 smithi177 bash[61439]: ts=2024-09-13T14:53:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:10.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:10.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:10.724 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:10.752 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:10.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:10.855 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:10.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:10.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:10.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:10.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[84576]: audit 2024-09-13T14:53:09.788536+0000 mon.a (mon.0) 1857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:10.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[84576]: audit 2024-09-13T14:53:09.788536+0000 mon.a (mon.0) 1857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[84576]: audit 2024-09-13T14:53:09.789568+0000 mon.a (mon.0) 1858 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[84576]: audit 2024-09-13T14:53:09.789568+0000 mon.a (mon.0) 1858 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[84576]: audit 2024-09-13T14:53:09.796390+0000 mon.a (mon.0) 1859 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[84576]: audit 2024-09-13T14:53:09.796390+0000 mon.a (mon.0) 1859 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[87147]: audit 2024-09-13T14:53:09.788536+0000 mon.a (mon.0) 1857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[87147]: audit 2024-09-13T14:53:09.788536+0000 mon.a (mon.0) 1857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[87147]: audit 2024-09-13T14:53:09.789568+0000 mon.a (mon.0) 1858 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[87147]: audit 2024-09-13T14:53:09.789568+0000 mon.a (mon.0) 1858 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[87147]: audit 2024-09-13T14:53:09.796390+0000 mon.a (mon.0) 1859 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:10 smithi099 bash[87147]: audit 2024-09-13T14:53:09.796390+0000 mon.a (mon.0) 1859 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:11.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:11.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:11.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:10 smithi177 bash[68594]: audit 2024-09-13T14:53:09.788536+0000 mon.a (mon.0) 1857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:11.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:10 smithi177 bash[68594]: audit 2024-09-13T14:53:09.788536+0000 mon.a (mon.0) 1857 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:53:11.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:10 smithi177 bash[68594]: audit 2024-09-13T14:53:09.789568+0000 mon.a (mon.0) 1858 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:11.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:10 smithi177 bash[68594]: audit 2024-09-13T14:53:09.789568+0000 mon.a (mon.0) 1858 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:53:11.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:10 smithi177 bash[68594]: audit 2024-09-13T14:53:09.796390+0000 mon.a (mon.0) 1859 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:11.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:10 smithi177 bash[68594]: audit 2024-09-13T14:53:09.796390+0000 mon.a (mon.0) 1859 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:11.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:11.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:11.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:11.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:11.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:11.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:11.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:11.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:11.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:11.508 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:11.536 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:11.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:11.592 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:11.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:11.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:11.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:11.768 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:11.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:11.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:11.928 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:11.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:12.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:12.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:11 smithi177 bash[68594]: cluster 2024-09-13T14:53:10.613113+0000 mgr.y (mgr.26252) 807 : cluster [DBG] pgmap v1312: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:12.043 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:11 smithi177 bash[68594]: cluster 2024-09-13T14:53:10.613113+0000 mgr.y (mgr.26252) 807 : cluster [DBG] pgmap v1312: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:12.060 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:12.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:12.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:11 smithi099 bash[84576]: cluster 2024-09-13T14:53:10.613113+0000 mgr.y (mgr.26252) 807 : cluster [DBG] pgmap v1312: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:12.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:11 smithi099 bash[84576]: cluster 2024-09-13T14:53:10.613113+0000 mgr.y (mgr.26252) 807 : cluster [DBG] pgmap v1312: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:12.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:11 smithi099 bash[87147]: cluster 2024-09-13T14:53:10.613113+0000 mgr.y (mgr.26252) 807 : cluster [DBG] pgmap v1312: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:12.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:11 smithi099 bash[87147]: cluster 2024-09-13T14:53:10.613113+0000 mgr.y (mgr.26252) 807 : cluster [DBG] pgmap v1312: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:12.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:12.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:12.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:12.294 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:12.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:12.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:12.408 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:53:12.409 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "" 2024-09-13T14:53:12.409 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:53:12.418 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:12.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:12.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:12.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:12.577 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:12.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:12.614 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:12.680 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:12.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:12.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:12.810 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:12.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:12.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:12.943 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:12.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:12.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:13.028 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:53:13.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:13.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:13.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:13.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:13.227 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:13.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:13.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:13.356 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:13.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:13.460 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:13.488 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:13.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:13.591 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:13.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:13.721 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:13.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:13.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:13.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:13 smithi177 bash[68594]: cluster 2024-09-13T14:53:12.613790+0000 mgr.y (mgr.26252) 808 : cluster [DBG] pgmap v1313: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:13 smithi177 bash[68594]: cluster 2024-09-13T14:53:12.613790+0000 mgr.y (mgr.26252) 808 : cluster [DBG] pgmap v1313: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:13.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:13.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:13 smithi099 bash[84576]: cluster 2024-09-13T14:53:12.613790+0000 mgr.y (mgr.26252) 808 : cluster [DBG] pgmap v1313: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:13.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:13 smithi099 bash[84576]: cluster 2024-09-13T14:53:12.613790+0000 mgr.y (mgr.26252) 808 : cluster [DBG] pgmap v1313: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:13.951 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:13 smithi099 bash[87147]: cluster 2024-09-13T14:53:12.613790+0000 mgr.y (mgr.26252) 808 : cluster [DBG] pgmap v1313: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:13.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:13 smithi099 bash[87147]: cluster 2024-09-13T14:53:12.613790+0000 mgr.y (mgr.26252) 808 : cluster [DBG] pgmap v1313: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:13.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:14.006 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:14.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:14.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:14.136 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:14.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:14.240 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:14.268 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:14.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:14.370 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:14.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:14.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:14.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:14.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:14.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:14.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:14.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:14.759 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:14.787 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:14.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:14.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:14.913 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:14.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:15.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:15.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:15.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:15.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:15.175 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:15.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:15.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:15.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:15.409 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:15.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:15.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:15.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:15.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:15.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:15.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:15.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:15.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:15.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:15.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:15.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:15.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:15.941 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:15 smithi099 bash[84576]: cluster 2024-09-13T14:53:14.614476+0000 mgr.y (mgr.26252) 809 : cluster [DBG] pgmap v1314: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:15.941 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:15 smithi099 bash[84576]: cluster 2024-09-13T14:53:14.614476+0000 mgr.y (mgr.26252) 809 : cluster [DBG] pgmap v1314: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:15.941 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:15 smithi099 bash[87147]: cluster 2024-09-13T14:53:14.614476+0000 mgr.y (mgr.26252) 809 : cluster [DBG] pgmap v1314: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:15.941 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:15 smithi099 bash[87147]: cluster 2024-09-13T14:53:14.614476+0000 mgr.y (mgr.26252) 809 : cluster [DBG] pgmap v1314: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:15.956 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:15.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:16.059 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:16.070 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:15 smithi177 bash[68594]: cluster 2024-09-13T14:53:14.614476+0000 mgr.y (mgr.26252) 809 : cluster [DBG] pgmap v1314: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:16.070 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:15 smithi177 bash[68594]: cluster 2024-09-13T14:53:14.614476+0000 mgr.y (mgr.26252) 809 : cluster [DBG] pgmap v1314: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:16.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:16.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:16.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:16.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:16.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:16.318 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:16.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:16.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:16.449 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:16.478 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:16.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:16.582 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:16.605 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:16.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:16.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:16.735 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:16.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:16.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:16.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:16.967 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:16.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:16.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:17.101 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:17.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:17.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:17.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:17.252 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:17.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:17.267 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T14:53:17.267 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:53:17.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:53:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:53:17] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-13T14:53:17.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:17.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:17.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:17.479 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:17.500 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:17.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:17.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:17.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:17.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:17.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:17.760 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:17.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:17.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:17.887 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:17.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:18.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:18.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:17 smithi099 bash[84576]: cluster 2024-09-13T14:53:16.615154+0000 mgr.y (mgr.26252) 810 : cluster [DBG] pgmap v1315: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:17 smithi099 bash[84576]: cluster 2024-09-13T14:53:16.615154+0000 mgr.y (mgr.26252) 810 : cluster [DBG] pgmap v1315: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:18.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:17 smithi099 bash[87147]: cluster 2024-09-13T14:53:16.615154+0000 mgr.y (mgr.26252) 810 : cluster [DBG] pgmap v1315: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:18.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:17 smithi099 bash[87147]: cluster 2024-09-13T14:53:16.615154+0000 mgr.y (mgr.26252) 810 : cluster [DBG] pgmap v1315: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:18.095 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:17 smithi177 bash[68594]: cluster 2024-09-13T14:53:16.615154+0000 mgr.y (mgr.26252) 810 : cluster [DBG] pgmap v1315: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:18.095 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:17 smithi177 bash[68594]: cluster 2024-09-13T14:53:16.615154+0000 mgr.y (mgr.26252) 810 : cluster [DBG] pgmap v1315: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:18.096 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:53:18.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:18.150 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:18.253 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:18.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:18.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:18.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:18.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:18.518 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:18.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:18.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:18.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:18.672 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:18.775 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:18.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:18.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:18.861 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:18 smithi099 bash[84576]: audit 2024-09-13T14:53:18.281069+0000 mon.a (mon.0) 1860 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:18.861 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:18 smithi099 bash[84576]: audit 2024-09-13T14:53:18.281069+0000 mon.a (mon.0) 1860 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:18.861 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:18 smithi099 bash[87147]: audit 2024-09-13T14:53:18.281069+0000 mon.a (mon.0) 1860 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:18.861 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:18 smithi099 bash[87147]: audit 2024-09-13T14:53:18.281069+0000 mon.a (mon.0) 1860 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:18.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:18.932 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:18.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:19.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:19.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:18 smithi177 bash[68594]: audit 2024-09-13T14:53:18.281069+0000 mon.a (mon.0) 1860 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:19.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:18 smithi177 bash[68594]: audit 2024-09-13T14:53:18.281069+0000 mon.a (mon.0) 1860 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:19.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:19.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:19.165 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:19.192 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:19.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:19.295 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:19.323 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:19.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:19.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:19.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:19.556 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:19.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:19.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:19.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:19.713 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:19.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:19.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:19.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:19.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:19.947 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:19.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:19.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:20.078 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:20.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:19 smithi177 bash[68594]: cluster 2024-09-13T14:53:18.615905+0000 mgr.y (mgr.26252) 811 : cluster [DBG] pgmap v1316: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:20.090 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:19 smithi177 bash[68594]: cluster 2024-09-13T14:53:18.615905+0000 mgr.y (mgr.26252) 811 : cluster [DBG] pgmap v1316: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:20.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:20.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:20.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:20.238 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:20.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:19 smithi099 bash[84576]: cluster 2024-09-13T14:53:18.615905+0000 mgr.y (mgr.26252) 811 : cluster [DBG] pgmap v1316: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:20.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:19 smithi099 bash[84576]: cluster 2024-09-13T14:53:18.615905+0000 mgr.y (mgr.26252) 811 : cluster [DBG] pgmap v1316: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:20.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:19 smithi099 bash[87147]: cluster 2024-09-13T14:53:18.615905+0000 mgr.y (mgr.26252) 811 : cluster [DBG] pgmap v1316: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:20.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:19 smithi099 bash[87147]: cluster 2024-09-13T14:53:18.615905+0000 mgr.y (mgr.26252) 811 : cluster [DBG] pgmap v1316: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:20.341 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:20.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:53:20 smithi177 bash[61439]: ts=2024-09-13T14:53:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:20.368 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:20.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:20.471 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:20.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:20.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:20.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:20.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:20.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:20.732 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:20.760 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:20.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:20.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:20.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:20.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:21.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:21.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:21.115 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:21.138 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:21.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:21.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:21.400 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:21.502 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:53:21.502 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:53:21.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:21.527 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:21.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:21.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.659 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:21.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.788 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:21.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.903 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:21 smithi177 bash[68594]: cluster 2024-09-13T14:53:20.616937+0000 mgr.y (mgr.26252) 812 : cluster [DBG] pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:21.903 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:21 smithi177 bash[68594]: cluster 2024-09-13T14:53:20.616937+0000 mgr.y (mgr.26252) 812 : cluster [DBG] pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:21.919 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:22.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:22.048 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:22.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:22.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:22.166 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:21 smithi099 bash[84576]: cluster 2024-09-13T14:53:20.616937+0000 mgr.y (mgr.26252) 812 : cluster [DBG] pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:22.166 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:21 smithi099 bash[84576]: cluster 2024-09-13T14:53:20.616937+0000 mgr.y (mgr.26252) 812 : cluster [DBG] pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:22.167 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:21 smithi099 bash[87147]: cluster 2024-09-13T14:53:20.616937+0000 mgr.y (mgr.26252) 812 : cluster [DBG] pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:22.167 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:21 smithi099 bash[87147]: cluster 2024-09-13T14:53:20.616937+0000 mgr.y (mgr.26252) 812 : cluster [DBG] pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:22.179 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:22.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:22.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:22.310 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:22.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:22.439 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:22.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:22.542 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:22.565 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:22.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:22.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:22.687 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:22.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:22.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:22.812 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:22.864 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:53:22.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:22.939 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:22.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:23.042 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:23.067 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:23.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:23.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:23.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:23.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:23.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:23.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:23.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:23.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:23.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:23.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:23.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:23.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:23.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:23.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:23.720 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:23.823 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:23.842 INFO:teuthology.orchestra.run.smithi099.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-09-13T14:53:23.849 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:23.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:23.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:23.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:23.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:24.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:24.086 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:23 smithi177 bash[68594]: cluster 2024-09-13T14:53:22.617480+0000 mgr.y (mgr.26252) 813 : cluster [DBG] pgmap v1318: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:24.086 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:23 smithi177 bash[68594]: cluster 2024-09-13T14:53:22.617480+0000 mgr.y (mgr.26252) 813 : cluster [DBG] pgmap v1318: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:24.087 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:23 smithi177 bash[68594]: audit 2024-09-13T14:53:22.870213+0000 mon.c (mon.2) 61 : audit [INF] from='client.? 172.21.15.99:0/837791672' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.087 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:23 smithi177 bash[68594]: audit 2024-09-13T14:53:22.870213+0000 mon.c (mon.2) 61 : audit [INF] from='client.? 172.21.15.99:0/837791672' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.087 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:23 smithi177 bash[68594]: audit 2024-09-13T14:53:22.870534+0000 mon.a (mon.0) 1861 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.087 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:23 smithi177 bash[68594]: audit 2024-09-13T14:53:22.870534+0000 mon.a (mon.0) 1861 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:24.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:24.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:24.215 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[84576]: cluster 2024-09-13T14:53:22.617480+0000 mgr.y (mgr.26252) 813 : cluster [DBG] pgmap v1318: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:24.215 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[84576]: cluster 2024-09-13T14:53:22.617480+0000 mgr.y (mgr.26252) 813 : cluster [DBG] pgmap v1318: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:24.215 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[84576]: audit 2024-09-13T14:53:22.870213+0000 mon.c (mon.2) 61 : audit [INF] from='client.? 172.21.15.99:0/837791672' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.215 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[84576]: audit 2024-09-13T14:53:22.870213+0000 mon.c (mon.2) 61 : audit [INF] from='client.? 172.21.15.99:0/837791672' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.216 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[84576]: audit 2024-09-13T14:53:22.870534+0000 mon.a (mon.0) 1861 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.216 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[84576]: audit 2024-09-13T14:53:22.870534+0000 mon.a (mon.0) 1861 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.216 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[87147]: cluster 2024-09-13T14:53:22.617480+0000 mgr.y (mgr.26252) 813 : cluster [DBG] pgmap v1318: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:24.216 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[87147]: cluster 2024-09-13T14:53:22.617480+0000 mgr.y (mgr.26252) 813 : cluster [DBG] pgmap v1318: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:24.216 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[87147]: audit 2024-09-13T14:53:22.870213+0000 mon.c (mon.2) 61 : audit [INF] from='client.? 172.21.15.99:0/837791672' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.216 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[87147]: audit 2024-09-13T14:53:22.870213+0000 mon.c (mon.2) 61 : audit [INF] from='client.? 172.21.15.99:0/837791672' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.216 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[87147]: audit 2024-09-13T14:53:22.870534+0000 mon.a (mon.0) 1861 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.216 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:23 smithi099 bash[87147]: audit 2024-09-13T14:53:22.870534+0000 mon.a (mon.0) 1861 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:53:24.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:24.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:24.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:24.362 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:24.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:24.465 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:24.492 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:24.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:24.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:24.617 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:24.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:24.721 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:24.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:24.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:24.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:24.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:24 smithi177 bash[68594]: audit 2024-09-13T14:53:23.838324+0000 mon.a (mon.0) 1862 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:53:24.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:24 smithi177 bash[68594]: audit 2024-09-13T14:53:23.838324+0000 mon.a (mon.0) 1862 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:53:24.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:24 smithi177 bash[68594]: cluster 2024-09-13T14:53:23.842080+0000 mon.a (mon.0) 1863 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2024-09-13T14:53:24.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:24 smithi177 bash[68594]: cluster 2024-09-13T14:53:23.842080+0000 mon.a (mon.0) 1863 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2024-09-13T14:53:24.876 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:24.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:24.979 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:24.999 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:24 smithi099 bash[84576]: audit 2024-09-13T14:53:23.838324+0000 mon.a (mon.0) 1862 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:53:24.999 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:24 smithi099 bash[84576]: audit 2024-09-13T14:53:23.838324+0000 mon.a (mon.0) 1862 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:53:24.999 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:24 smithi099 bash[84576]: cluster 2024-09-13T14:53:23.842080+0000 mon.a (mon.0) 1863 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2024-09-13T14:53:24.999 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:24 smithi099 bash[84576]: cluster 2024-09-13T14:53:23.842080+0000 mon.a (mon.0) 1863 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2024-09-13T14:53:24.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:24 smithi099 bash[87147]: audit 2024-09-13T14:53:23.838324+0000 mon.a (mon.0) 1862 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:53:24.999 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:24 smithi099 bash[87147]: audit 2024-09-13T14:53:23.838324+0000 mon.a (mon.0) 1862 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:53:25.000 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:24 smithi099 bash[87147]: cluster 2024-09-13T14:53:23.842080+0000 mon.a (mon.0) 1863 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2024-09-13T14:53:25.000 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:24 smithi099 bash[87147]: cluster 2024-09-13T14:53:23.842080+0000 mon.a (mon.0) 1863 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2024-09-13T14:53:25.006 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:25.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:25.109 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:25.133 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:25.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:25.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:25.264 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:25.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:25.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:25.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:25.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:25.494 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:25.520 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:25.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:25.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:25.649 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:25.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:25.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:25.780 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:25.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:25.883 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:25.893 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:53:25.893 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:25.908 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:53:25.909 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:25.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:25.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:25.925 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:53:25.925 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:25.942 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:53:25.943 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:25.961 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:53:25.961 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:25.978 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:53:25.979 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:25.995 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:53:25.995 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:26.014 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:53:26.014 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:26.014 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:26.023 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:25 smithi177 bash[68594]: cluster 2024-09-13T14:53:24.618458+0000 mgr.y (mgr.26252) 814 : cluster [DBG] pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:26.024 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:25 smithi177 bash[68594]: cluster 2024-09-13T14:53:24.618458+0000 mgr.y (mgr.26252) 814 : cluster [DBG] pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:26.024 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:25 smithi177 bash[68594]: cluster 2024-09-13T14:53:24.838408+0000 mon.a (mon.0) 1864 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:53:26.024 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:25 smithi177 bash[68594]: cluster 2024-09-13T14:53:24.838408+0000 mon.a (mon.0) 1864 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:53:26.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:25 smithi099 bash[84576]: cluster 2024-09-13T14:53:24.618458+0000 mgr.y (mgr.26252) 814 : cluster [DBG] pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:26.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:25 smithi099 bash[84576]: cluster 2024-09-13T14:53:24.618458+0000 mgr.y (mgr.26252) 814 : cluster [DBG] pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:26.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:25 smithi099 bash[84576]: cluster 2024-09-13T14:53:24.838408+0000 mon.a (mon.0) 1864 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:53:26.025 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:25 smithi099 bash[84576]: cluster 2024-09-13T14:53:24.838408+0000 mon.a (mon.0) 1864 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:53:26.025 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:25 smithi099 bash[87147]: cluster 2024-09-13T14:53:24.618458+0000 mgr.y (mgr.26252) 814 : cluster [DBG] pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:26.025 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:25 smithi099 bash[87147]: cluster 2024-09-13T14:53:24.618458+0000 mgr.y (mgr.26252) 814 : cluster [DBG] pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:26.025 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:25 smithi099 bash[87147]: cluster 2024-09-13T14:53:24.838408+0000 mon.a (mon.0) 1864 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:53:26.025 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:25 smithi099 bash[87147]: cluster 2024-09-13T14:53:24.838408+0000 mon.a (mon.0) 1864 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:53:26.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:26.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:26.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:26.170 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:26.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:26.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:26.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:26.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:26.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:26.429 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:26.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:26.532 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:26.544 DEBUG:teuthology.orchestra.run.smithi099:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-09-13T14:53:26.547 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:53:26.547 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:53:26.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:26.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:26.590 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:hints = 1 2024-09-13T14:53:26.590 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:53:26.590 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object prefix: benchmark_data_smithi099_148651 2024-09-13T14:53:26.590 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:53:26.590 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:53:26.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:26.691 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:26.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:26.823 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:26.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:26.862 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T14:53:26.928 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:26.953 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T14:53:26.953 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T14:53:26.953 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T14:53:26.953 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T14:53:26.953 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.953 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:18082 41.0 41:0d12817f:::benchmark_data_smithi099_126887_object18081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e965)", 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:54.464291+0000", 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 332.48118187900002, 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0119406339999999, 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 18082 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:26.954 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.464291+0000", 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.464291+0000", 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.464293+0000", 2024-09-13T14:53:26.955 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.95e-06 2024-09-13T14:53:26.956 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.956 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.956 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:26.956 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.464457+0000", 2024-09-13T14:53:26.956 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016337099999999999 2024-09-13T14:53:26.956 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.956 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.956 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:26.956 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.464471+0000", 2024-09-13T14:53:26.956 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4255e-05 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.464478+0000", 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1239999999999999e-06 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.464553+0000", 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5232000000000001e-05 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.957 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.958 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:26.958 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.464564+0000", 2024-09-13T14:53:26.958 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1063999999999999e-05 2024-09-13T14:53:26.958 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.958 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.958 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:26.958 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.471435+0000", 2024-09-13T14:53:26.958 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0068707640000001 2024-09-13T14:53:26.958 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.958 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.959 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:26.959 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.471590+0000", 2024-09-13T14:53:26.959 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000154487 2024-09-13T14:53:26.959 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.959 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.959 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:26.959 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.471690+0000", 2024-09-13T14:53:26.959 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9845000000000004e-05 2024-09-13T14:53:26.959 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.475579+0000", 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038894350000000001 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.475717+0000", 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013847999999999999 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:26.960 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.476048+0000", 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033051100000000002 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.476175+0000", 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000127298 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.476232+0000", 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6817999999999997e-05 2024-09-13T14:53:26.961 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:18090 41.0 41:07be1098:::benchmark_data_smithi099_126887_object18089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e965)", 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:54.470822+0000", 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 332.47465108599999, 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0057286489999999, 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:26.962 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 18090 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.470822+0000", 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:26.963 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.470822+0000", 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.470824+0000", 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8619999999999999e-06 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.470879+0000", 2024-09-13T14:53:26.964 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4598000000000001e-05 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.470891+0000", 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2166e-05 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.470896+0000", 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6559999999999997e-06 2024-09-13T14:53:26.965 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.470962+0000", 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5276000000000003e-05 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.470977+0000", 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5342e-05 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.967 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:26.967 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.473610+0000", 2024-09-13T14:53:26.967 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0026329919999999 2024-09-13T14:53:26.967 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.967 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.967 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:26.967 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.473950+0000", 2024-09-13T14:53:26.967 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000339785 2024-09-13T14:53:26.967 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.968 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.968 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:26.968 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.474379+0000", 2024-09-13T14:53:26.968 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00042893299999999999 2024-09-13T14:53:26.968 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.968 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.968 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:26.968 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.476353+0000", 2024-09-13T14:53:26.968 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019744160000000001 2024-09-13T14:53:26.968 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.969 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.969 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:26.969 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.476401+0000", 2024-09-13T14:53:26.969 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8018e-05 2024-09-13T14:53:26.969 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.969 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.969 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:26.969 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.476451+0000", 2024-09-13T14:53:26.969 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0039e-05 2024-09-13T14:53:26.970 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.970 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.970 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:26.970 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.476510+0000", 2024-09-13T14:53:26.970 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8566999999999997e-05 2024-09-13T14:53:26.970 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.970 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.970 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:26.970 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.476551+0000", 2024-09-13T14:53:26.971 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0998999999999998e-05 2024-09-13T14:53:26.971 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:26.971 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:26.971 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:26.971 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.971 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.971 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:18093 41.0 41:0585e538:::benchmark_data_smithi099_126887_object18092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e965)", 2024-09-13T14:53:26.971 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:54.471589+0000", 2024-09-13T14:53:26.971 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 332.473884602, 2024-09-13T14:53:26.972 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0191143110000001, 2024-09-13T14:53:26.972 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:26.972 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:26.972 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:26.972 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:26.972 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:26.972 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:26.972 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 18093 2024-09-13T14:53:26.973 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.973 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:26.973 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.973 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:26.973 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.471589+0000", 2024-09-13T14:53:26.973 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.973 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.973 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.471589+0000", 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.471643+0000", 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3999000000000003e-05 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.471668+0000", 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5145000000000001e-05 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.471678+0000", 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8050000000000001e-06 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:26.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.471683+0000", 2024-09-13T14:53:26.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5659999999999997e-06 2024-09-13T14:53:26.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:26.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.471728+0000", 2024-09-13T14:53:26.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4431000000000002e-05 2024-09-13T14:53:26.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:26.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.471733+0000", 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7780000000000004e-06 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.476651+0000", 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0049176369999999 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.477001+0000", 2024-09-13T14:53:26.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00034988499999999997 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.477387+0000", 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00038579700000000001 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.481929+0000", 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045425020000000003 2024-09-13T14:53:26.978 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.482012+0000", 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2817000000000003e-05 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.490620+0000", 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0086075619999999992 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.490668+0000", 2024-09-13T14:53:26.979 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8838999999999998e-05 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.490703+0000", 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4548000000000002e-05 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:18106 41.0 41:0a9138c2:::benchmark_data_smithi099_126887_object18105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e965)", 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:54.477414+0000", 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 332.468058998, 2024-09-13T14:53:26.980 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0133505540000001, 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 18106 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.477414+0000", 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.981 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.477414+0000", 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.477416+0000", 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3549999999999999e-06 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.477531+0000", 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000115448 2024-09-13T14:53:26.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.477544+0000", 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2746e-05 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.477550+0000", 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6840000000000001e-06 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.477601+0000", 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1706000000000003e-05 2024-09-13T14:53:26.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.477609+0000", 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7389999999999999e-06 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.478767+0000", 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0011579900000001 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.478851+0000", 2024-09-13T14:53:26.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4235999999999998e-05 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.478945+0000", 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4129999999999995e-05 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.481972+0000", 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030265140000000001 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.985 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.485910+0000", 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039382109999999996 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.490714+0000", 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0048040660000000001 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.490741+0000", 2024-09-13T14:53:26.986 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6771000000000002e-05 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.490765+0000", 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3958e-05 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:18156 41.0 41:09aeb125:::benchmark_data_smithi099_126887_object18155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e965)", 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:54.499956+0000", 2024-09-13T14:53:26.987 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 332.44551729900002, 2024-09-13T14:53:26.988 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99285359200000001, 2024-09-13T14:53:26.988 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:26.988 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:26.988 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:26.988 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:26.988 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:26.988 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:26.988 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 18156 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.499956+0000", 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.499956+0000", 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:26.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.499958+0000", 2024-09-13T14:53:26.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.688e-06 2024-09-13T14:53:26.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:26.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.500031+0000", 2024-09-13T14:53:26.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3547000000000006e-05 2024-09-13T14:53:26.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:26.991 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.500044+0000", 2024-09-13T14:53:26.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2779000000000001e-05 2024-09-13T14:53:26.991 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.991 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.991 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:26.991 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.500052+0000", 2024-09-13T14:53:26.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5519999999999999e-06 2024-09-13T14:53:26.991 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.991 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.500829+0000", 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00077723199999999999 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:54.500839+0000", 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0138e-05 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.992 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:26.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.479545+0000", 2024-09-13T14:53:26.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97870638499999996 2024-09-13T14:53:26.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:26.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.479779+0000", 2024-09-13T14:53:26.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023405999999999999 2024-09-13T14:53:26.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.994 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:53:26.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.480104+0000", 2024-09-13T14:53:26.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032464000000000001 2024-09-13T14:53:26.994 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.994 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.994 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:26.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.485965+0000", 2024-09-13T14:53:26.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058608239999999997 2024-09-13T14:53:26.994 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.994 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.995 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:26.995 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.486060+0000", 2024-09-13T14:53:26.995 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5657999999999998e-05 2024-09-13T14:53:26.995 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.995 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:26.995 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.492680+0000", 2024-09-13T14:53:26.995 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0066199199999999996 2024-09-13T14:53:26.995 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:26.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.492725+0000", 2024-09-13T14:53:26.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4898999999999998e-05 2024-09-13T14:53:26.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:26.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:55.492810+0000", 2024-09-13T14:53:26.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4270000000000005e-05 2024-09-13T14:53:26.997 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:26.997 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:26.997 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:26.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.997 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20489 41.6 41:6ac745e6:::benchmark_data_smithi099_126887_object20488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:26.997 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.468463+0000", 2024-09-13T14:53:26.997 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.47701077400001, 2024-09-13T14:53:26.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.022138006, 2024-09-13T14:53:26.998 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:26.998 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:26.998 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T14:53:26.998 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:26.998 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:26.998 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:26.999 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20489 2024-09-13T14:53:26.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:26.999 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:26.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:26.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:26.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468463+0000", 2024-09-13T14:53:26.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:26.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468463+0000", 2024-09-13T14:53:27.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468464+0000", 2024-09-13T14:53:27.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7129999999999999e-06 2024-09-13T14:53:27.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468521+0000", 2024-09-13T14:53:27.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7005000000000001e-05 2024-09-13T14:53:27.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468537+0000", 2024-09-13T14:53:27.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5359999999999999e-05 2024-09-13T14:53:27.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468548+0000", 2024-09-13T14:53:27.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.146e-05 2024-09-13T14:53:27.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468836+0000", 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028758400000000002 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468846+0000", 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.08e-05 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.490529+0000", 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0216827959999999 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.490601+0000", 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1287999999999999e-05 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20493 41.6 41:6e25ce6f:::benchmark_data_smithi099_126887_object20492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.468568+0000", 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.476905687, 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.022614973, 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.004 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20493 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468568+0000", 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468568+0000", 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468568+0000", 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6800000000000001e-07 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468596+0000", 2024-09-13T14:53:27.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7742e-05 2024-09-13T14:53:27.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468602+0000", 2024-09-13T14:53:27.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7030000000000003e-06 2024-09-13T14:53:27.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468605+0000", 2024-09-13T14:53:27.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7969999999999998e-06 2024-09-13T14:53:27.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468888+0000", 2024-09-13T14:53:27.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028260399999999999 2024-09-13T14:53:27.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.468896+0000", 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2379999999999997e-06 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.491133+0000", 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.022236557 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.491183+0000", 2024-09-13T14:53:27.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9864000000000002e-05 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20511 41.6 41:6af8ccf8:::benchmark_data_smithi099_126887_object20510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.475043+0000", 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.47042999500002, 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0244367459999999, 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.010 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.011 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.011 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.011 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.011 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.011 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20511 2024-09-13T14:53:27.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.011 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.475043+0000", 2024-09-13T14:53:27.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.012 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.475043+0000", 2024-09-13T14:53:27.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.012 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.475044+0000", 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.195e-06 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.475120+0000", 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5624000000000006e-05 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.475131+0000", 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.063e-05 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.475144+0000", 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3072e-05 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.475210+0000", 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5738e-05 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.475216+0000", 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.917e-06 2024-09-13T14:53:27.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.491229+0000", 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.016012822 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.491356+0000", 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000126501 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.491467+0000", 2024-09-13T14:53:27.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000111685 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.495544+0000", 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0040766989999999996 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.495611+0000", 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7136000000000002e-05 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.499407+0000", 2024-09-13T14:53:27.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037959109999999999 2024-09-13T14:53:27.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.499444+0000", 2024-09-13T14:53:27.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6807000000000002e-05 2024-09-13T14:53:27.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.499480+0000", 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6009e-05 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20513 41.6 41:66cdf6fc:::benchmark_data_smithi099_126887_object20512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.476887+0000", 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.46858592500001, 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0300919399999999, 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.018 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20513 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.476887+0000", 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.476887+0000", 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.476889+0000", 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.234e-06 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.476921+0000", 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2169000000000003e-05 2024-09-13T14:53:27.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.476936+0000", 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5350999999999999e-05 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.476946+0000", 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8519999999999999e-06 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.476992+0000", 2024-09-13T14:53:27.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6295999999999999e-05 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.476999+0000", 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9539999999999998e-06 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.492241+0000", 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0152420280000001 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.492372+0000", 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013077299999999999 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.492524+0000", 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015192299999999999 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.495550+0000", 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030263579999999998 2024-09-13T14:53:27.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.495682+0000", 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013159599999999999 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.506910+0000", 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011228416999999999 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.506948+0000", 2024-09-13T14:53:27.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8040000000000002e-05 2024-09-13T14:53:27.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.506979+0000", 2024-09-13T14:53:27.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0948999999999998e-05 2024-09-13T14:53:27.025 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.025 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.025 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.026 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20517 41.6 41:69fc92a3:::benchmark_data_smithi099_126887_object20516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:27.026 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.477957+0000", 2024-09-13T14:53:27.026 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.46751650499999, 2024-09-13T14:53:27.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.029128721, 2024-09-13T14:53:27.026 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.026 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.026 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.026 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.026 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.026 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.027 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20517 2024-09-13T14:53:27.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.027 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.477957+0000", 2024-09-13T14:53:27.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.477957+0000", 2024-09-13T14:53:27.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.477958+0000", 2024-09-13T14:53:27.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7900000000000007e-07 2024-09-13T14:53:27.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478035+0000", 2024-09-13T14:53:27.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6750999999999998e-05 2024-09-13T14:53:27.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478044+0000", 2024-09-13T14:53:27.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3689999999999996e-06 2024-09-13T14:53:27.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478049+0000", 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9710000000000003e-06 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478107+0000", 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7939999999999998e-05 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478117+0000", 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0118e-05 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.493373+0000", 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.01525614 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.493490+0000", 2024-09-13T14:53:27.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000117095 2024-09-13T14:53:27.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:27.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.493630+0000", 2024-09-13T14:53:27.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013947699999999999 2024-09-13T14:53:27.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.495695+0000", 2024-09-13T14:53:27.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020654290000000001 2024-09-13T14:53:27.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.497674+0000", 2024-09-13T14:53:27.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001978493 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.507014+0000", 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0093407579999999994 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.507051+0000", 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6885999999999999e-05 2024-09-13T14:53:27.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.507086+0000", 2024-09-13T14:53:27.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4314999999999999e-05 2024-09-13T14:53:27.035 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.035 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.035 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.036 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20518 41.6 41:6e8a748e:::benchmark_data_smithi099_126887_object20517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:27.036 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.478444+0000", 2024-09-13T14:53:27.036 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.467029717, 2024-09-13T14:53:27.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0288277610000001, 2024-09-13T14:53:27.036 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.036 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.036 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.036 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.036 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.037 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.037 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20518 2024-09-13T14:53:27.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.037 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.037 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478444+0000", 2024-09-13T14:53:27.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478444+0000", 2024-09-13T14:53:27.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478444+0000", 2024-09-13T14:53:27.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6899999999999998e-07 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478525+0000", 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0679000000000005e-05 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478531+0000", 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9030000000000001e-06 2024-09-13T14:53:27.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478534+0000", 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8659999999999999e-06 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478590+0000", 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5037000000000002e-05 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.478599+0000", 2024-09-13T14:53:27.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5759999999999995e-06 2024-09-13T14:53:27.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.494477+0000", 2024-09-13T14:53:27.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0158781480000001 2024-09-13T14:53:27.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.494792+0000", 2024-09-13T14:53:27.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00031446600000000001 2024-09-13T14:53:27.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:27.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.494883+0000", 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1285000000000004e-05 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.497630+0000", 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002746971 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.497724+0000", 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4269000000000002e-05 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.507162+0000", 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0094381440000000007 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.507222+0000", 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9963000000000002e-05 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.507271+0000", 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8984999999999999e-05 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20541 41.6 41:61d69916:::benchmark_data_smithi099_126887_object20540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.489642+0000", 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.45583145400002, 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0177439669999999, 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.045 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20541 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.489642+0000", 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.489642+0000", 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.489643+0000", 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5239999999999999e-06 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.489667+0000", 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3552000000000001e-05 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.489675+0000", 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2979999999999992e-06 2024-09-13T14:53:27.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.489679+0000", 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.189e-06 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.489770+0000", 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0272000000000006e-05 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.489778+0000", 2024-09-13T14:53:27.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7110000000000001e-06 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.495738+0000", 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0059599260000001 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.495859+0000", 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120736 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:27.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.496005+0000", 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014634700000000001 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.497687+0000", 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001681133 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.497769+0000", 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2744000000000006e-05 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.507313+0000", 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0095437730000000002 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.507344+0000", 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1072000000000003e-05 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.507386+0000", 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1690000000000002e-05 2024-09-13T14:53:27.051 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20588 41.6 41:67668b5b:::benchmark_data_smithi099_126887_object20587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.514217+0000", 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.43125665399998, 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.009915476, 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.052 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.053 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.053 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.053 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.053 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20588 2024-09-13T14:53:27.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.053 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.514217+0000", 2024-09-13T14:53:27.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.514217+0000", 2024-09-13T14:53:27.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.514222+0000", 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3709999999999999e-06 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.514297+0000", 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4636999999999997e-05 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.514311+0000", 2024-09-13T14:53:27.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4844000000000001e-05 2024-09-13T14:53:27.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.514320+0000", 2024-09-13T14:53:27.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1170000000000004e-06 2024-09-13T14:53:27.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.514391+0000", 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1717000000000005e-05 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.514403+0000", 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2088e-05 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.496800+0000", 2024-09-13T14:53:27.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.98239625100000005 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.496911+0000", 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011087200000000001 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.497003+0000", 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2244000000000007e-05 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.499349+0000", 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002346042 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.499378+0000", 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9573999999999998e-05 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524031+0000", 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024652532000000001 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524074+0000", 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3359999999999998e-05 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524132+0000", 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7827e-05 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20613 41.6 41:6a5830a7:::benchmark_data_smithi099_126887_object20612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.527509+0000", 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.41796440500002, 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.996792648, 2024-09-13T14:53:27.061 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20613 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.527509+0000", 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.527509+0000", 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.527510+0000", 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2330000000000001e-06 2024-09-13T14:53:27.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.527576+0000", 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5495999999999998e-05 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.527586+0000", 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0686000000000001e-05 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.527592+0000", 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3040000000000004e-06 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.527660+0000", 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8036999999999999e-05 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.527672+0000", 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2618e-05 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.497807+0000", 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97013507399999999 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.497935+0000", 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012725200000000001 2024-09-13T14:53:27.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.498072+0000", 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013752799999999999 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.499386+0000", 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001313544 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.501156+0000", 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001770408 2024-09-13T14:53:27.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524166+0000", 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023009762999999999 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524261+0000", 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4920000000000006e-05 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524302+0000", 2024-09-13T14:53:27.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0785000000000003e-05 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20617 41.6 41:6e65744f:::benchmark_data_smithi099_126887_object20616:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.528514+0000", 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.41695893899998, 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99591837400000005, 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.069 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20617 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.528514+0000", 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.528514+0000", 2024-09-13T14:53:27.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.528515+0000", 2024-09-13T14:53:27.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5799999999999998e-07 2024-09-13T14:53:27.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.528555+0000", 2024-09-13T14:53:27.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9678000000000002e-05 2024-09-13T14:53:27.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.528568+0000", 2024-09-13T14:53:27.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3529999999999999e-05 2024-09-13T14:53:27.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.528574+0000", 2024-09-13T14:53:27.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2739999999999998e-06 2024-09-13T14:53:27.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.528635+0000", 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1878999999999995e-05 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.528643+0000", 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3259999999999998e-06 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.498618+0000", 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96997500299999995 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.498712+0000", 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3867000000000001e-05 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.498806+0000", 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4426999999999996e-05 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.500309+0000", 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015032069999999999 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.501205+0000", 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00089534200000000003 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524372+0000", 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023167011000000001 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524411+0000", 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9162999999999998e-05 2024-09-13T14:53:27.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524433+0000", 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1909e-05 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:20627 41.6 41:617c0659:::benchmark_data_smithi099_126887_object20626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e967)", 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:56.534850+0000", 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.410623789, 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.989713603, 2024-09-13T14:53:27.077 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 20627 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.534850+0000", 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.534850+0000", 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.534851+0000", 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6840000000000001e-06 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.534906+0000", 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4441000000000003e-05 2024-09-13T14:53:27.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.534917+0000", 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1379000000000001e-05 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.534922+0000", 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3290000000000001e-06 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.534975+0000", 2024-09-13T14:53:27.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2833000000000001e-05 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:56.534982+0000", 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6170000000000003e-06 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.499501+0000", 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96451969299999996 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.499643+0000", 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000141879 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.499787+0000", 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014397300000000001 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.501251+0000", 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014636880000000001 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.501977+0000", 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00072635899999999999 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524457+0000", 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022480099999999999 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524510+0000", 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2212000000000001e-05 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:57.524563+0000", 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3415999999999998e-05 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:26132 41.a 41:5e4ce099:::benchmark_data_smithi099_126887_object26131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e971)", 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:48:01.322917+0000", 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 325.62255651999999, 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.000142815, 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.084 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 26132 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.322917+0000", 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.322917+0000", 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.322919+0000", 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6469999999999998e-06 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.323131+0000", 2024-09-13T14:53:27.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00021148799999999999 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.323153+0000", 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2529000000000001e-05 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.323171+0000", 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7493999999999999e-05 2024-09-13T14:53:27.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.326732+0000", 2024-09-13T14:53:27.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035606420000000001 2024-09-13T14:53:27.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.326745+0000", 2024-09-13T14:53:27.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.325e-05 2024-09-13T14:53:27.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.318118+0000", 2024-09-13T14:53:27.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.991373371 2024-09-13T14:53:27.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.318260+0000", 2024-09-13T14:53:27.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014135299999999999 2024-09-13T14:53:27.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:53:27.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.318401+0000", 2024-09-13T14:53:27.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014139799999999999 2024-09-13T14:53:27.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.322691+0000", 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042898900000000002 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.322702+0000", 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0927000000000001e-05 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.322944+0000", 2024-09-13T14:53:27.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024228100000000001 2024-09-13T14:53:27.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.323011+0000", 2024-09-13T14:53:27.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7037000000000002e-05 2024-09-13T14:53:27.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.323060+0000", 2024-09-13T14:53:27.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8507999999999997e-05 2024-09-13T14:53:27.093 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.093 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.093 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.093 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:26147 41.a 41:5f3dc495:::benchmark_data_smithi099_126887_object26146:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e971)", 2024-09-13T14:53:27.093 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:48:01.329016+0000", 2024-09-13T14:53:27.093 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 325.61645705799998, 2024-09-13T14:53:27.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99541973800000005, 2024-09-13T14:53:27.094 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.094 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.094 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.094 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.094 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.094 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.094 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 26147 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.329016+0000", 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.329016+0000", 2024-09-13T14:53:27.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.329020+0000", 2024-09-13T14:53:27.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.264e-06 2024-09-13T14:53:27.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.329096+0000", 2024-09-13T14:53:27.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6005000000000002e-05 2024-09-13T14:53:27.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.329141+0000", 2024-09-13T14:53:27.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5728e-05 2024-09-13T14:53:27.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.329176+0000", 2024-09-13T14:53:27.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.481e-05 2024-09-13T14:53:27.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.329362+0000", 2024-09-13T14:53:27.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000185515 2024-09-13T14:53:27.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.329391+0000", 2024-09-13T14:53:27.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8989e-05 2024-09-13T14:53:27.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.319980+0000", 2024-09-13T14:53:27.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99058937599999997 2024-09-13T14:53:27.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.320147+0000", 2024-09-13T14:53:27.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016688400000000001 2024-09-13T14:53:27.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:53:27.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.320290+0000", 2024-09-13T14:53:27.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000143268 2024-09-13T14:53:27.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.324270+0000", 2024-09-13T14:53:27.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039800399999999998 2024-09-13T14:53:27.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.324316+0000", 2024-09-13T14:53:27.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6186000000000001e-05 2024-09-13T14:53:27.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.324363+0000", 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7024999999999998e-05 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.324405+0000", 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1220000000000002e-05 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.324436+0000", 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1427999999999998e-05 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:26150 41.a 41:53549437:::benchmark_data_smithi099_126887_object26149:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e971)", 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:48:01.330569+0000", 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 325.61490398900003, 2024-09-13T14:53:27.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0028162940000001, 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 26150 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.330569+0000", 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.330569+0000", 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.330571+0000", 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5430000000000001e-06 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.330617+0000", 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6561999999999999e-05 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.330637+0000", 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0000000000000002e-05 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.330649+0000", 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1462e-05 2024-09-13T14:53:27.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.330793+0000", 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014416200000000001 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.330822+0000", 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9442999999999999e-05 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.321373+0000", 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99055043200000004 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.321592+0000", 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00021931199999999999 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.321802+0000", 2024-09-13T14:53:27.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020997499999999999 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.324276+0000", 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024739290000000001 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.327757+0000", 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034811260000000002 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.333278+0000", 2024-09-13T14:53:27.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055209980000000001 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.333336+0000", 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7337999999999999e-05 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.333386+0000", 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0012000000000002e-05 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.110 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:26155 41.a 41:535d1c0d:::benchmark_data_smithi099_126887_object26154:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e971)", 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:48:01.333384+0000", 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 325.61208887800001, 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0003987240000001, 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 26155 2024-09-13T14:53:27.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.333384+0000", 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.333384+0000", 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:27.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.333387+0000", 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4880000000000001e-06 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.333446+0000", 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9267e-05 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.333472+0000", 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6100000000000001e-05 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.333490+0000", 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7241999999999999e-05 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.333689+0000", 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019932500000000001 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:01.333715+0000", 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5962e-05 2024-09-13T14:53:27.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.323134+0000", 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.98941907500000004 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.323305+0000", 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017087499999999999 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T14:53:27.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.323497+0000", 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019239800000000001 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.331178+0000", 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0076808800000000002 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.333482+0000", 2024-09-13T14:53:27.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023035360000000001 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.333696+0000", 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000214527 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.333756+0000", 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0027e-05 2024-09-13T14:53:27.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:27.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:27.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:53:27.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:48:02.333783+0000", 2024-09-13T14:53:27.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7022e-05 2024-09-13T14:53:27.121 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.121 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.121 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.121 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:27.121 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:27.121 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T14:53:27.122 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:26 smithi177 bash[68594]: audit 2024-09-13T14:53:25.873074+0000 mon.a (mon.0) 1865 : audit [DBG] from='client.? 172.21.15.99:0/2051498664' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:27.122 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:26 smithi177 bash[68594]: audit 2024-09-13T14:53:25.873074+0000 mon.a (mon.0) 1865 : audit [DBG] from='client.? 172.21.15.99:0/2051498664' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:27.122 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:26 smithi177 bash[68594]: audit 2024-09-13T14:53:25.982011+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.99:0/1009839443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.122 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:26 smithi177 bash[68594]: audit 2024-09-13T14:53:25.982011+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.99:0/1009839443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.122 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:26 smithi177 bash[68594]: audit 2024-09-13T14:53:25.982372+0000 mon.a (mon.0) 1866 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.122 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:26 smithi177 bash[68594]: audit 2024-09-13T14:53:25.982372+0000 mon.a (mon.0) 1866 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.123 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:27.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:27.124 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:53:27] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-13T14:53:27.124 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[84576]: audit 2024-09-13T14:53:25.873074+0000 mon.a (mon.0) 1865 : audit [DBG] from='client.? 172.21.15.99:0/2051498664' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:27.125 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[84576]: audit 2024-09-13T14:53:25.873074+0000 mon.a (mon.0) 1865 : audit [DBG] from='client.? 172.21.15.99:0/2051498664' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:27.125 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[84576]: audit 2024-09-13T14:53:25.982011+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.99:0/1009839443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.125 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[84576]: audit 2024-09-13T14:53:25.982011+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.99:0/1009839443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.125 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[84576]: audit 2024-09-13T14:53:25.982372+0000 mon.a (mon.0) 1866 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.125 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[84576]: audit 2024-09-13T14:53:25.982372+0000 mon.a (mon.0) 1866 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.125 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[87147]: audit 2024-09-13T14:53:25.873074+0000 mon.a (mon.0) 1865 : audit [DBG] from='client.? 172.21.15.99:0/2051498664' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:27.125 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[87147]: audit 2024-09-13T14:53:25.873074+0000 mon.a (mon.0) 1865 : audit [DBG] from='client.? 172.21.15.99:0/2051498664' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:27.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[87147]: audit 2024-09-13T14:53:25.982011+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.99:0/1009839443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[87147]: audit 2024-09-13T14:53:25.982011+0000 mon.c (mon.2) 62 : audit [INF] from='client.? 172.21.15.99:0/1009839443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[87147]: audit 2024-09-13T14:53:25.982372+0000 mon.a (mon.0) 1866 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:26 smithi099 bash[87147]: audit 2024-09-13T14:53:25.982372+0000 mon.a (mon.0) 1866 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:27.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:27.257 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:27.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:27.361 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:27.388 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:27.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:27.491 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:27.515 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:27.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:27.590 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 1 15 2969 2954 184.615 184.625 0.00329991 0.00539729 2024-09-13T14:53:27.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:27.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:27.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:27.746 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:27.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:27.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:27.878 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:27.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:27.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:28.010 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:28.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:27 smithi177 bash[68594]: cluster 2024-09-13T14:53:26.618947+0000 mgr.y (mgr.26252) 815 : cluster [DBG] pgmap v1321: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:28.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:27 smithi177 bash[68594]: cluster 2024-09-13T14:53:26.618947+0000 mgr.y (mgr.26252) 815 : cluster [DBG] pgmap v1321: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:28.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:27 smithi177 bash[68594]: cluster 2024-09-13T14:53:26.852499+0000 mon.a (mon.0) 1867 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:28.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:27 smithi177 bash[68594]: cluster 2024-09-13T14:53:26.852499+0000 mon.a (mon.0) 1867 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:28.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:27 smithi177 bash[68594]: cluster 2024-09-13T14:53:26.852529+0000 mon.a (mon.0) 1868 : cluster [INF] Cluster is now healthy 2024-09-13T14:53:28.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:27 smithi177 bash[68594]: cluster 2024-09-13T14:53:26.852529+0000 mon.a (mon.0) 1868 : cluster [INF] Cluster is now healthy 2024-09-13T14:53:28.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:27 smithi177 bash[68594]: audit 2024-09-13T14:53:26.859896+0000 mon.a (mon.0) 1869 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:28.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:27 smithi177 bash[68594]: audit 2024-09-13T14:53:26.859896+0000 mon.a (mon.0) 1869 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:28.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:27 smithi177 bash[68594]: cluster 2024-09-13T14:53:26.865228+0000 mon.a (mon.0) 1870 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2024-09-13T14:53:28.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:27 smithi177 bash[68594]: cluster 2024-09-13T14:53:26.865228+0000 mon.a (mon.0) 1870 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2024-09-13T14:53:28.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:28.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:28.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:28.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:28.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:28.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:28.292 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:53:28.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:28.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[87147]: cluster 2024-09-13T14:53:26.618947+0000 mgr.y (mgr.26252) 815 : cluster [DBG] pgmap v1321: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:28.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[87147]: cluster 2024-09-13T14:53:26.618947+0000 mgr.y (mgr.26252) 815 : cluster [DBG] pgmap v1321: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:28.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[87147]: cluster 2024-09-13T14:53:26.852499+0000 mon.a (mon.0) 1867 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:28.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[87147]: cluster 2024-09-13T14:53:26.852499+0000 mon.a (mon.0) 1867 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:28.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[87147]: cluster 2024-09-13T14:53:26.852529+0000 mon.a (mon.0) 1868 : cluster [INF] Cluster is now healthy 2024-09-13T14:53:28.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[87147]: cluster 2024-09-13T14:53:26.852529+0000 mon.a (mon.0) 1868 : cluster [INF] Cluster is now healthy 2024-09-13T14:53:28.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[87147]: audit 2024-09-13T14:53:26.859896+0000 mon.a (mon.0) 1869 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:28.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[87147]: audit 2024-09-13T14:53:26.859896+0000 mon.a (mon.0) 1869 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:28.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[87147]: cluster 2024-09-13T14:53:26.865228+0000 mon.a (mon.0) 1870 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2024-09-13T14:53:28.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[87147]: cluster 2024-09-13T14:53:26.865228+0000 mon.a (mon.0) 1870 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2024-09-13T14:53:28.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[84576]: cluster 2024-09-13T14:53:26.618947+0000 mgr.y (mgr.26252) 815 : cluster [DBG] pgmap v1321: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:28.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[84576]: cluster 2024-09-13T14:53:26.618947+0000 mgr.y (mgr.26252) 815 : cluster [DBG] pgmap v1321: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:53:28.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[84576]: cluster 2024-09-13T14:53:26.852499+0000 mon.a (mon.0) 1867 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:28.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[84576]: cluster 2024-09-13T14:53:26.852499+0000 mon.a (mon.0) 1867 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:28.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[84576]: cluster 2024-09-13T14:53:26.852529+0000 mon.a (mon.0) 1868 : cluster [INF] Cluster is now healthy 2024-09-13T14:53:28.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[84576]: cluster 2024-09-13T14:53:26.852529+0000 mon.a (mon.0) 1868 : cluster [INF] Cluster is now healthy 2024-09-13T14:53:28.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[84576]: audit 2024-09-13T14:53:26.859896+0000 mon.a (mon.0) 1869 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:28.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[84576]: audit 2024-09-13T14:53:26.859896+0000 mon.a (mon.0) 1869 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:28.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[84576]: cluster 2024-09-13T14:53:26.865228+0000 mon.a (mon.0) 1870 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2024-09-13T14:53:28.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:27 smithi099 bash[84576]: cluster 2024-09-13T14:53:26.865228+0000 mon.a (mon.0) 1870 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2024-09-13T14:53:28.407 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:28.434 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:28.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:28.544 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:28.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:28.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:28.591 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 2 16 5964 5948 185.863 187.125 0.0062543 0.00536811 2024-09-13T14:53:28.682 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:28.709 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:28.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:28.812 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:28.838 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:28.926 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:28.942 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:28.965 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:28.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:29.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:29.092 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:29.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:29.196 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:29.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:29.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:53:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:29.337 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:29.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:29.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:29.474 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:29.496 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:29.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:29.591 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 3 16 8377 8361 174.171 150.812 0.00179901 0.00534065 2024-09-13T14:53:29.598 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:29.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:29.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:29.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:29.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:29.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:29.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:29.877 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:29.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:29.981 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:30.007 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:30.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:30.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:29 smithi177 bash[68594]: cluster 2024-09-13T14:53:28.619503+0000 mgr.y (mgr.26252) 816 : cluster [DBG] pgmap v1323: 89 pgs: 89 active+clean; 76 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s wr, 150 op/s 2024-09-13T14:53:30.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:29 smithi177 bash[68594]: cluster 2024-09-13T14:53:28.619503+0000 mgr.y (mgr.26252) 816 : cluster [DBG] pgmap v1323: 89 pgs: 89 active+clean; 76 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s wr, 150 op/s 2024-09-13T14:53:30.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:30.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:30.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:29 smithi099 bash[84576]: cluster 2024-09-13T14:53:28.619503+0000 mgr.y (mgr.26252) 816 : cluster [DBG] pgmap v1323: 89 pgs: 89 active+clean; 76 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s wr, 150 op/s 2024-09-13T14:53:30.186 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:29 smithi099 bash[84576]: cluster 2024-09-13T14:53:28.619503+0000 mgr.y (mgr.26252) 816 : cluster [DBG] pgmap v1323: 89 pgs: 89 active+clean; 76 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s wr, 150 op/s 2024-09-13T14:53:30.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:29 smithi099 bash[87147]: cluster 2024-09-13T14:53:28.619503+0000 mgr.y (mgr.26252) 816 : cluster [DBG] pgmap v1323: 89 pgs: 89 active+clean; 76 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s wr, 150 op/s 2024-09-13T14:53:30.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:29 smithi099 bash[87147]: cluster 2024-09-13T14:53:28.619503+0000 mgr.y (mgr.26252) 816 : cluster [DBG] pgmap v1323: 89 pgs: 89 active+clean; 76 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s wr, 150 op/s 2024-09-13T14:53:30.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:30.276 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:30.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:30.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:30.448 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:53:30.512 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:30.522 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:53:30 smithi177 bash[61439]: ts=2024-09-13T14:53:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:30.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:30.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:30.591 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 4 16 9428 9412 147.047 65.6875 0.00441653 0.00679223 2024-09-13T14:53:30.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:30.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:30.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:30.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:30.798 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:30.902 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:30.927 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:30.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:31.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:31.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:31.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:31.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:31.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:31.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:31.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:31.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:31.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:31.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:31.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:31.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:31.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:31.591 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 5 16 12088 12072 150.884 166.25 0.0082408 0.00662052 2024-09-13T14:53:31.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:31.714 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:31.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:31.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:31.848 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:31.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:31.962 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:31 smithi177 bash[68594]: cluster 2024-09-13T14:53:30.622365+0000 mgr.y (mgr.26252) 817 : cluster [DBG] pgmap v1324: 89 pgs: 89 active+clean; 393 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s 2024-09-13T14:53:31.962 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:31 smithi177 bash[68594]: cluster 2024-09-13T14:53:30.622365+0000 mgr.y (mgr.26252) 817 : cluster [DBG] pgmap v1324: 89 pgs: 89 active+clean; 393 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s 2024-09-13T14:53:31.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:31.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:32.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:31 smithi099 bash[84576]: cluster 2024-09-13T14:53:30.622365+0000 mgr.y (mgr.26252) 817 : cluster [DBG] pgmap v1324: 89 pgs: 89 active+clean; 393 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s 2024-09-13T14:53:32.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:31 smithi099 bash[84576]: cluster 2024-09-13T14:53:30.622365+0000 mgr.y (mgr.26252) 817 : cluster [DBG] pgmap v1324: 89 pgs: 89 active+clean; 393 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s 2024-09-13T14:53:32.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:31 smithi099 bash[87147]: cluster 2024-09-13T14:53:30.622365+0000 mgr.y (mgr.26252) 817 : cluster [DBG] pgmap v1324: 89 pgs: 89 active+clean; 393 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s 2024-09-13T14:53:32.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:31 smithi099 bash[87147]: cluster 2024-09-13T14:53:30.622365+0000 mgr.y (mgr.26252) 817 : cluster [DBG] pgmap v1324: 89 pgs: 89 active+clean; 393 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s 2024-09-13T14:53:32.083 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:32.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.114 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:32.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:32.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:32.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:32.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:32.374 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:32.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:32.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:32.507 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:32.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:32.591 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 6 15 14747 14732 153.442 166.25 0.00222177 0.00650964 2024-09-13T14:53:32.612 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:32.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:32.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:32.741 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:32.765 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:32.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:32.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:32.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:32.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:32.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:33.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:33.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:33.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:33.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:33.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:33.171 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:53:33.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:33.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:33.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:33.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:33.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:33.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:33.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:33.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:33.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:33.591 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 7 15 16221 16206 144.682 92.125 0.00873094 0.00690581 2024-09-13T14:53:33.648 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:33.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:33.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:33.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:33.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:33.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:33.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:34.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:34.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:34.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[84576]: cluster 2024-09-13T14:53:32.622918+0000 mgr.y (mgr.26252) 818 : cluster [DBG] pgmap v1325: 89 pgs: 89 active+clean; 484 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T14:53:34.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[84576]: cluster 2024-09-13T14:53:32.622918+0000 mgr.y (mgr.26252) 818 : cluster [DBG] pgmap v1325: 89 pgs: 89 active+clean; 484 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T14:53:34.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[84576]: audit 2024-09-13T14:53:33.286712+0000 mon.a (mon.0) 1871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:34.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[84576]: audit 2024-09-13T14:53:33.286712+0000 mon.a (mon.0) 1871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:34.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[84576]: audit 2024-09-13T14:53:33.288200+0000 mon.a (mon.0) 1872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:34.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[84576]: audit 2024-09-13T14:53:33.288200+0000 mon.a (mon.0) 1872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:34.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[87147]: cluster 2024-09-13T14:53:32.622918+0000 mgr.y (mgr.26252) 818 : cluster [DBG] pgmap v1325: 89 pgs: 89 active+clean; 484 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T14:53:34.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[87147]: cluster 2024-09-13T14:53:32.622918+0000 mgr.y (mgr.26252) 818 : cluster [DBG] pgmap v1325: 89 pgs: 89 active+clean; 484 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T14:53:34.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[87147]: audit 2024-09-13T14:53:33.286712+0000 mon.a (mon.0) 1871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[87147]: audit 2024-09-13T14:53:33.286712+0000 mon.a (mon.0) 1871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[87147]: audit 2024-09-13T14:53:33.288200+0000 mon.a (mon.0) 1872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:33 smithi099 bash[87147]: audit 2024-09-13T14:53:33.288200+0000 mon.a (mon.0) 1872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:33 smithi177 bash[68594]: cluster 2024-09-13T14:53:32.622918+0000 mgr.y (mgr.26252) 818 : cluster [DBG] pgmap v1325: 89 pgs: 89 active+clean; 484 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T14:53:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:33 smithi177 bash[68594]: cluster 2024-09-13T14:53:32.622918+0000 mgr.y (mgr.26252) 818 : cluster [DBG] pgmap v1325: 89 pgs: 89 active+clean; 484 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T14:53:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:33 smithi177 bash[68594]: audit 2024-09-13T14:53:33.286712+0000 mon.a (mon.0) 1871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:33 smithi177 bash[68594]: audit 2024-09-13T14:53:33.286712+0000 mon.a (mon.0) 1871 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:53:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:33 smithi177 bash[68594]: audit 2024-09-13T14:53:33.288200+0000 mon.a (mon.0) 1872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:33 smithi177 bash[68594]: audit 2024-09-13T14:53:33.288200+0000 mon.a (mon.0) 1872 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:34.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:34.144 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:34.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:34.147 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:34.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:34.270 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:34.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:34.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:34.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:34.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:34.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:34.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:34.591 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 8 16 17601 17585 137.369 86.1875 0.0100639 0.00727267 2024-09-13T14:53:34.646 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:34.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:34.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:34.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:34.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:34.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:34.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:34.941 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:34.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:35.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:35.072 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:35.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:35.176 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:35.194 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:35.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:35.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:35.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:35.430 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:35.455 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:35.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:35.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:35.585 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:35.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:35.591 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 9 16 19705 19689 136.716 131.5 0.00826072 0.00730828 2024-09-13T14:53:35.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:35.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:35.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:35.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:35.853 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:35.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:35.956 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:35.965 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:35 smithi099 bash[84576]: cluster 2024-09-13T14:53:34.623646+0000 mgr.y (mgr.26252) 819 : cluster [DBG] pgmap v1326: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:35.966 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:35 smithi099 bash[84576]: cluster 2024-09-13T14:53:34.623646+0000 mgr.y (mgr.26252) 819 : cluster [DBG] pgmap v1326: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:35.966 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:35 smithi099 bash[87147]: cluster 2024-09-13T14:53:34.623646+0000 mgr.y (mgr.26252) 819 : cluster [DBG] pgmap v1326: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:35.966 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:35 smithi099 bash[87147]: cluster 2024-09-13T14:53:34.623646+0000 mgr.y (mgr.26252) 819 : cluster [DBG] pgmap v1326: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:35.982 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:35.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:36.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:36.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:36.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:36.108 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:36.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:36.233 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:36.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:36.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:36.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:36.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:36.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:35 smithi177 bash[68594]: cluster 2024-09-13T14:53:34.623646+0000 mgr.y (mgr.26252) 819 : cluster [DBG] pgmap v1326: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:35 smithi177 bash[68594]: cluster 2024-09-13T14:53:34.623646+0000 mgr.y (mgr.26252) 819 : cluster [DBG] pgmap v1326: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:36.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:36.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:36.582 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:36.591 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 10 16 21254 21238 132.725 96.8125 0.00223269 0.0074142 2024-09-13T14:53:36.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:36.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:36.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:36.732 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:36.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:36.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:36.859 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:36.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:36.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:36.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:37.099 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:37.120 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:37.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:37.161 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:53:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:53:37] "GET /metrics HTTP/1.1" 200 36737 "" "Prometheus/2.51.0" 2024-09-13T14:53:37.222 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:37.251 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:37.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:37.356 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:37.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:37.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:37.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:37.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:37.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:37.591 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 11 16 22734 22718 129.066 92.5 0.00189435 0.00771783 2024-09-13T14:53:37.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:37.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:37.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:37.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:37.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:37.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:37.864 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:37.891 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:37.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:37.994 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:38.007 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:37 smithi177 bash[68594]: cluster 2024-09-13T14:53:36.624003+0000 mgr.y (mgr.26252) 820 : cluster [DBG] pgmap v1327: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:38.008 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:37 smithi177 bash[68594]: cluster 2024-09-13T14:53:36.624003+0000 mgr.y (mgr.26252) 820 : cluster [DBG] pgmap v1327: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:38.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:38.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:38.136 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:38.167 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:38.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:38.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:38.281 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:37 smithi099 bash[84576]: cluster 2024-09-13T14:53:36.624003+0000 mgr.y (mgr.26252) 820 : cluster [DBG] pgmap v1327: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:38.281 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:37 smithi099 bash[84576]: cluster 2024-09-13T14:53:36.624003+0000 mgr.y (mgr.26252) 820 : cluster [DBG] pgmap v1327: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:38.281 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:37 smithi099 bash[87147]: cluster 2024-09-13T14:53:36.624003+0000 mgr.y (mgr.26252) 820 : cluster [DBG] pgmap v1327: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:38.281 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:37 smithi099 bash[87147]: cluster 2024-09-13T14:53:36.624003+0000 mgr.y (mgr.26252) 820 : cluster [DBG] pgmap v1327: 89 pgs: 89 active+clean; 1.0 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-09-13T14:53:38.297 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:38.400 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:38.423 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:38.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:38.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:38.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:38.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:38.592 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 12 16 24126 24110 125.56 87 0.00512679 0.00795825 2024-09-13T14:53:38.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:38.683 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:38.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:38.787 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:38.806 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:38.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:38.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:38.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:38.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:39.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:39.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:39.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:39.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:39.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:39.432 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:39.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:39.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:39.566 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.592 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 13 16 25797 25781 123.935 104.438 0.0114191 0.00806121 2024-09-13T14:53:39.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:39.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:39.730 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:39.834 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.863 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:39.968 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.977 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:39 smithi099 bash[84576]: cluster 2024-09-13T14:53:38.624594+0000 mgr.y (mgr.26252) 821 : cluster [DBG] pgmap v1328: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-13T14:53:39.977 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:39 smithi099 bash[84576]: cluster 2024-09-13T14:53:38.624594+0000 mgr.y (mgr.26252) 821 : cluster [DBG] pgmap v1328: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-13T14:53:39.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:39 smithi099 bash[87147]: cluster 2024-09-13T14:53:38.624594+0000 mgr.y (mgr.26252) 821 : cluster [DBG] pgmap v1328: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-13T14:53:39.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:39 smithi099 bash[87147]: cluster 2024-09-13T14:53:38.624594+0000 mgr.y (mgr.26252) 821 : cluster [DBG] pgmap v1328: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-13T14:53:39.978 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:39 smithi177 bash[68594]: cluster 2024-09-13T14:53:38.624594+0000 mgr.y (mgr.26252) 821 : cluster [DBG] pgmap v1328: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-13T14:53:39.978 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:39 smithi177 bash[68594]: cluster 2024-09-13T14:53:38.624594+0000 mgr.y (mgr.26252) 821 : cluster [DBG] pgmap v1328: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-13T14:53:39.992 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:40.095 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:40.124 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:40.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:40.227 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:40.236 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:53:40 smithi177 bash[61439]: ts=2024-09-13T14:53:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:40.255 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:40.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:40.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:40.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:40.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:40.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:40.592 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 14 16 27764 27748 123.863 122.938 0.00995581 0.00806613 2024-09-13T14:53:40.624 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:40.652 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:40.756 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:40.783 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:40.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:40.886 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:40.897 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:53:40.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:40.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:41.014 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.043 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:41.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:41.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:41.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:40 smithi177 bash[68594]: audit 2024-09-13T14:53:40.500831+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.99:0/2333520338' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:40 smithi177 bash[68594]: audit 2024-09-13T14:53:40.500831+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.99:0/2333520338' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:40 smithi177 bash[68594]: audit 2024-09-13T14:53:40.503397+0000 mon.a (mon.0) 1873 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:40 smithi177 bash[68594]: audit 2024-09-13T14:53:40.503397+0000 mon.a (mon.0) 1873 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:41.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:41.280 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:41.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:41.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:41.415 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:41.423 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[84576]: audit 2024-09-13T14:53:40.500831+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.99:0/2333520338' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.424 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[84576]: audit 2024-09-13T14:53:40.500831+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.99:0/2333520338' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.424 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[84576]: audit 2024-09-13T14:53:40.503397+0000 mon.a (mon.0) 1873 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.424 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[84576]: audit 2024-09-13T14:53:40.503397+0000 mon.a (mon.0) 1873 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.425 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[87147]: audit 2024-09-13T14:53:40.500831+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.99:0/2333520338' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.425 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[87147]: audit 2024-09-13T14:53:40.500831+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.99:0/2333520338' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.425 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[87147]: audit 2024-09-13T14:53:40.503397+0000 mon.a (mon.0) 1873 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.425 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[87147]: audit 2024-09-13T14:53:40.503397+0000 mon.a (mon.0) 1873 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:53:41.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:41.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:41.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:41.592 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 15 16 29720 29704 123.754 122.25 0.00764774 0.00807547 2024-09-13T14:53:41.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:41.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:41.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:41.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:41.821 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:41.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:41.962 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:41.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:42.065 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:42.089 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:42.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:42.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:42.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[84576]: cluster 2024-09-13T14:53:40.625081+0000 mgr.y (mgr.26252) 822 : cluster [DBG] pgmap v1329: 89 pgs: 89 active+clean; 1.5 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T14:53:42.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[84576]: cluster 2024-09-13T14:53:40.625081+0000 mgr.y (mgr.26252) 822 : cluster [DBG] pgmap v1329: 89 pgs: 89 active+clean; 1.5 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T14:53:42.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[84576]: audit 2024-09-13T14:53:40.890919+0000 mon.a (mon.0) 1874 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:53:42.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[84576]: audit 2024-09-13T14:53:40.890919+0000 mon.a (mon.0) 1874 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:53:42.203 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[84576]: cluster 2024-09-13T14:53:40.895112+0000 mon.a (mon.0) 1875 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2024-09-13T14:53:42.203 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[84576]: cluster 2024-09-13T14:53:40.895112+0000 mon.a (mon.0) 1875 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2024-09-13T14:53:42.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[87147]: cluster 2024-09-13T14:53:40.625081+0000 mgr.y (mgr.26252) 822 : cluster [DBG] pgmap v1329: 89 pgs: 89 active+clean; 1.5 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T14:53:42.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[87147]: cluster 2024-09-13T14:53:40.625081+0000 mgr.y (mgr.26252) 822 : cluster [DBG] pgmap v1329: 89 pgs: 89 active+clean; 1.5 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T14:53:42.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[87147]: audit 2024-09-13T14:53:40.890919+0000 mon.a (mon.0) 1874 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:53:42.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[87147]: audit 2024-09-13T14:53:40.890919+0000 mon.a (mon.0) 1874 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:53:42.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[87147]: cluster 2024-09-13T14:53:40.895112+0000 mon.a (mon.0) 1875 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2024-09-13T14:53:42.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:41 smithi099 bash[87147]: cluster 2024-09-13T14:53:40.895112+0000 mon.a (mon.0) 1875 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2024-09-13T14:53:42.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:42.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:42.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:42.339 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:42.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:42.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:42.469 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:42.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:42.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:42.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:42 smithi177 bash[68594]: cluster 2024-09-13T14:53:40.625081+0000 mgr.y (mgr.26252) 822 : cluster [DBG] pgmap v1329: 89 pgs: 89 active+clean; 1.5 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T14:53:42.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:42 smithi177 bash[68594]: cluster 2024-09-13T14:53:40.625081+0000 mgr.y (mgr.26252) 822 : cluster [DBG] pgmap v1329: 89 pgs: 89 active+clean; 1.5 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T14:53:42.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:42 smithi177 bash[68594]: audit 2024-09-13T14:53:40.890919+0000 mon.a (mon.0) 1874 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:53:42.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:42 smithi177 bash[68594]: audit 2024-09-13T14:53:40.890919+0000 mon.a (mon.0) 1874 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:53:42.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:42 smithi177 bash[68594]: cluster 2024-09-13T14:53:40.895112+0000 mon.a (mon.0) 1875 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2024-09-13T14:53:42.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:42 smithi177 bash[68594]: cluster 2024-09-13T14:53:40.895112+0000 mon.a (mon.0) 1875 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2024-09-13T14:53:42.592 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 16 16 30785 30769 120.18 66.5625 0.00868471 0.00831447 2024-09-13T14:53:42.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:42.711 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:42.733 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:42.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:42.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:42.855 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:42.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:42.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:42.983 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:42.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:43.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:43.123 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:43.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:43.227 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:43.258 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:43.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:43.361 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:43.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:43.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:43.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:43.512 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:43.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:43.592 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 17 16 32158 32142 118.157 85.8125 0.00354514 0.0083614 2024-09-13T14:53:43.615 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:43.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:43.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:43.740 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:43.765 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:43.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:43.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:43.896 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:43.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:43.999 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:44.026 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:44.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:44.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.155 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:43 smithi099 bash[84576]: cluster 2024-09-13T14:53:42.625599+0000 mgr.y (mgr.26252) 823 : cluster [DBG] pgmap v1331: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T14:53:44.155 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:43 smithi099 bash[84576]: cluster 2024-09-13T14:53:42.625599+0000 mgr.y (mgr.26252) 823 : cluster [DBG] pgmap v1331: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T14:53:44.156 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:43 smithi099 bash[87147]: cluster 2024-09-13T14:53:42.625599+0000 mgr.y (mgr.26252) 823 : cluster [DBG] pgmap v1331: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T14:53:44.156 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:43 smithi099 bash[87147]: cluster 2024-09-13T14:53:42.625599+0000 mgr.y (mgr.26252) 823 : cluster [DBG] pgmap v1331: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T14:53:44.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:44.254 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:43 smithi177 bash[68594]: cluster 2024-09-13T14:53:42.625599+0000 mgr.y (mgr.26252) 823 : cluster [DBG] pgmap v1331: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T14:53:44.254 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:43 smithi177 bash[68594]: cluster 2024-09-13T14:53:42.625599+0000 mgr.y (mgr.26252) 823 : cluster [DBG] pgmap v1331: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T14:53:44.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:53:44.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:44.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:44.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.557 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:44.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:44.592 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 18 16 33361 33345 115.77 75.1875 0.00785157 0.00863279 2024-09-13T14:53:44.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:44.681 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:44.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:44.816 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:44.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:44.920 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:44.947 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:44.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:45.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:45.081 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:45.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:45.184 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:45.205 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:45.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:45.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:45.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:45.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:45.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:45.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:45.549 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T14:53:45.549 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T14:53:45.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.549 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:34655 42.a 42:5154534b:::benchmark_data_smithi099_148651_object34654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1015)", 2024-09-13T14:53:45.549 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:45.547342+0000", 2024-09-13T14:53:45.549 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0023440280000000002, 2024-09-13T14:53:45.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023780749999999999, 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 34655 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.547342+0000", 2024-09-13T14:53:45.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.547342+0000", 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.547341+0000", 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:45.551 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.547370+0000", 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.885e-05 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.547379+0000", 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1109999999999997e-06 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.547384+0000", 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7899999999999999e-06 2024-09-13T14:53:45.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.547617+0000", 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023272599999999999 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.547745+0000", 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012858100000000001 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:53:45.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.549577+0000", 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001832251 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45350.0:34658 42.3 e1015/1009 42:c5bc2426:::benchmark_data_smithi099_148651_object34657:head v 1015'2171, mlcod=1015'2169)", 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:45.548312+0000", 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0013746330000000001, 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014838679999999999, 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T14:53:45.554 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548312+0000", 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548312+0000", 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:45.555 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548311+0000", 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548352+0000", 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1362000000000001e-05 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548366+0000", 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3461e-05 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.556 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548371+0000", 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9880000000000004e-06 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548480+0000", 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109219 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548505+0000", 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5196000000000001e-05 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.557 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45350.0:34657 42.b e1015/1009 42:de378e07:::benchmark_data_smithi099_148651_object34656:head v 1015'2120, mlcod=1015'2118)", 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:45.548569+0000", 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.001116945, 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012788470000000001, 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548569+0000", 2024-09-13T14:53:45.558 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548569+0000", 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548568+0000", 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:45.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548608+0000", 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0256999999999998e-05 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548631+0000", 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3030000000000001e-05 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548643+0000", 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1844999999999999e-05 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548823+0000", 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000180194 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.548848+0000", 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4712e-05 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:45.561 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.562 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.563 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.563 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:34635 42.5 42:aa5dd1c0:::benchmark_data_smithi099_148651_object34634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1015)", 2024-09-13T14:53:45.563 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:45.532719+0000", 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.016967554999999999, 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017185467999999999, 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 34635 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.564 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.532719+0000", 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.532719+0000", 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.532717+0000", 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T14:53:45.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.532756+0000", 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9351999999999999e-05 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.532770+0000", 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.358e-05 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:45.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.532777+0000", 2024-09-13T14:53:45.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5169999999999996e-06 2024-09-13T14:53:45.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:45.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.532821+0000", 2024-09-13T14:53:45.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4611999999999999e-05 2024-09-13T14:53:45.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:45.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.532944+0000", 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012331399999999999 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.533076+0000", 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000131364 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.538934+0000", 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058584229999999998 2024-09-13T14:53:45.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.539111+0000", 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000176329 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:34647 42.5 42:abbb6dfb:::benchmark_data_smithi099_148651_object34646:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1015)", 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:45.540261+0000", 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0094254130000000005, 2024-09-13T14:53:45.569 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0097201059999999992, 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 34647 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.540261+0000", 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:53:45.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.540261+0000", 2024-09-13T14:53:45.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:53:45.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:53:45.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.540260+0000", 2024-09-13T14:53:45.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:53:45.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.540316+0000", 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6477000000000002e-05 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.540343+0000", 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7399999999999999e-05 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:53:45.572 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.540352+0000", 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0680000000000003e-06 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.540445+0000", 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2763000000000006e-05 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.540598+0000", 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015278899999999999 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:53:45.574 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.540739+0000", 2024-09-13T14:53:45.574 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014143599999999999 2024-09-13T14:53:45.574 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.574 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.574 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:45.574 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.543379+0000", 2024-09-13T14:53:45.574 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026393039999999999 2024-09-13T14:53:45.574 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:53:45.574 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:53:45.574 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:53:45.575 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:45.543454+0000", 2024-09-13T14:53:45.575 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4909999999999999e-05 2024-09-13T14:53:45.575 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.575 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:53:45.575 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.575 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:53:45.575 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2024-09-13T14:53:45.575 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 5 2024-09-13T14:53:45.575 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T14:53:45.575 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:45.592 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 19 16 34668 34652 113.975 81.6875 0.00255871 0.00874605 2024-09-13T14:53:45.593 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:45.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:45.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:45.730 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:45.834 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:45.861 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:45.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:45.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:45.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:45.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:46.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:45 smithi099 bash[84576]: cluster 2024-09-13T14:53:44.626336+0000 mgr.y (mgr.26252) 824 : cluster [DBG] pgmap v1332: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:46.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:45 smithi099 bash[84576]: cluster 2024-09-13T14:53:44.626336+0000 mgr.y (mgr.26252) 824 : cluster [DBG] pgmap v1332: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:46.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:45 smithi099 bash[87147]: cluster 2024-09-13T14:53:44.626336+0000 mgr.y (mgr.26252) 824 : cluster [DBG] pgmap v1332: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:46.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:45 smithi099 bash[87147]: cluster 2024-09-13T14:53:44.626336+0000 mgr.y (mgr.26252) 824 : cluster [DBG] pgmap v1332: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:46.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:46.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:45 smithi177 bash[68594]: cluster 2024-09-13T14:53:44.626336+0000 mgr.y (mgr.26252) 824 : cluster [DBG] pgmap v1332: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:46.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:45 smithi177 bash[68594]: cluster 2024-09-13T14:53:44.626336+0000 mgr.y (mgr.26252) 824 : cluster [DBG] pgmap v1332: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:46.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:46.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:46.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:46.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:46.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:46.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:46.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:46.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:46.476 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:46.503 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:46.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:46.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T14:53:46.596707+0000 min lat: 0.00109161 max lat: 0.809605 avg lat: 0.0090015 2024-09-13T14:53:46.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:53:46.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 20 16 35546 35530 111.02 54.875 0.00761962 0.0090015 2024-09-13T14:53:46.605 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:46.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:46.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:46.742 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:46.766 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:46.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:46.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:46.903 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:46.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:47.006 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:47.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:53:47.031 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:47.136 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:47.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:47.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:47.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:47.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:47.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:47.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:53:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:53:47] "GET /metrics HTTP/1.1" 200 36737 "" "Prometheus/2.51.0" 2024-09-13T14:53:47.408 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:47.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:47.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:47.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:47.589 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:47.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:47.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 21 16 37239 37223 110.771 105.812 0.00749229 0.00902171 2024-09-13T14:53:47.691 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:47.723 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:47.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:47.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:47.854 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:47.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:47.956 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:47.984 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:47.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:48.021 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:53:48.022 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:48.057 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:48.058 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:53:48.070 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:47 smithi099 bash[84576]: cluster 2024-09-13T14:53:46.626769+0000 mgr.y (mgr.26252) 825 : cluster [DBG] pgmap v1333: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:48.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:47 smithi099 bash[84576]: cluster 2024-09-13T14:53:46.626769+0000 mgr.y (mgr.26252) 825 : cluster [DBG] pgmap v1333: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:48.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:47 smithi099 bash[87147]: cluster 2024-09-13T14:53:46.626769+0000 mgr.y (mgr.26252) 825 : cluster [DBG] pgmap v1333: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:48.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:47 smithi099 bash[87147]: cluster 2024-09-13T14:53:46.626769+0000 mgr.y (mgr.26252) 825 : cluster [DBG] pgmap v1333: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:48.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:48.093 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:48.093 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:53:48.113 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:48.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:48.132 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:48.132 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:53:48.165 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:48.166 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:53:48.198 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:48.198 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:53:48.217 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:48.233 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:53:48.234 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:48.249 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:48.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:48.262 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:53:48.263 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:48.353 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:48.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:47 smithi177 bash[68594]: cluster 2024-09-13T14:53:46.626769+0000 mgr.y (mgr.26252) 825 : cluster [DBG] pgmap v1333: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:48.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:47 smithi177 bash[68594]: cluster 2024-09-13T14:53:46.626769+0000 mgr.y (mgr.26252) 825 : cluster [DBG] pgmap v1333: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-13T14:53:48.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:48.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:48.483 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:48.514 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:48.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:48.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 22 16 38880 38864 110.397 102.562 0.00888492 0.00905219 2024-09-13T14:53:48.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:48.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:48.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:48.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:48.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:48.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:48.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:48.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:48.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:48.922 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T14:53:49.009 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:49.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:49.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:49.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[84576]: audit 2024-09-13T14:53:47.991586+0000 mon.a (mon.0) 1876 : audit [DBG] from='client.? 172.21.15.99:0/881020838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:49.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[84576]: audit 2024-09-13T14:53:47.991586+0000 mon.a (mon.0) 1876 : audit [DBG] from='client.? 172.21.15.99:0/881020838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:49.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[84576]: audit 2024-09-13T14:53:48.282419+0000 mon.a (mon.0) 1877 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:49.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[84576]: audit 2024-09-13T14:53:48.282419+0000 mon.a (mon.0) 1877 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:49.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[84576]: audit 2024-09-13T14:53:48.438738+0000 mon.a (mon.0) 1878 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:49.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[84576]: audit 2024-09-13T14:53:48.438738+0000 mon.a (mon.0) 1878 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:49.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[87147]: audit 2024-09-13T14:53:47.991586+0000 mon.a (mon.0) 1876 : audit [DBG] from='client.? 172.21.15.99:0/881020838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:49.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[87147]: audit 2024-09-13T14:53:47.991586+0000 mon.a (mon.0) 1876 : audit [DBG] from='client.? 172.21.15.99:0/881020838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:49.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[87147]: audit 2024-09-13T14:53:48.282419+0000 mon.a (mon.0) 1877 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:49.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[87147]: audit 2024-09-13T14:53:48.282419+0000 mon.a (mon.0) 1877 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:49.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[87147]: audit 2024-09-13T14:53:48.438738+0000 mon.a (mon.0) 1878 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:49.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:48 smithi099 bash[87147]: audit 2024-09-13T14:53:48.438738+0000 mon.a (mon.0) 1878 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:49.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:49.164 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:49.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:49.267 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:49.294 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:49.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:48 smithi177 bash[68594]: audit 2024-09-13T14:53:47.991586+0000 mon.a (mon.0) 1876 : audit [DBG] from='client.? 172.21.15.99:0/881020838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:48 smithi177 bash[68594]: audit 2024-09-13T14:53:47.991586+0000 mon.a (mon.0) 1876 : audit [DBG] from='client.? 172.21.15.99:0/881020838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:48 smithi177 bash[68594]: audit 2024-09-13T14:53:48.282419+0000 mon.a (mon.0) 1877 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:48 smithi177 bash[68594]: audit 2024-09-13T14:53:48.282419+0000 mon.a (mon.0) 1877 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:48 smithi177 bash[68594]: audit 2024-09-13T14:53:48.438738+0000 mon.a (mon.0) 1878 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:48 smithi177 bash[68594]: audit 2024-09-13T14:53:48.438738+0000 mon.a (mon.0) 1878 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:49.400 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:49.424 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:49.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:49.529 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:49.560 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:49.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:49.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 23 16 40510 40494 110.026 101.875 0.00559392 0.00908315 2024-09-13T14:53:49.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:49.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:49.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:49.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:49.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:49.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:49.928 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:49.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:49.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:50.049 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:50.063 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[84576]: cluster 2024-09-13T14:53:48.627274+0000 mgr.y (mgr.26252) 826 : cluster [DBG] pgmap v1334: 89 pgs: 89 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:53:50.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[84576]: cluster 2024-09-13T14:53:48.627274+0000 mgr.y (mgr.26252) 826 : cluster [DBG] pgmap v1334: 89 pgs: 89 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:53:50.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[84576]: cluster 2024-09-13T14:53:48.912791+0000 mon.a (mon.0) 1879 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:50.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[84576]: cluster 2024-09-13T14:53:48.912791+0000 mon.a (mon.0) 1879 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:50.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[84576]: audit 2024-09-13T14:53:48.923372+0000 mon.a (mon.0) 1880 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:50.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[84576]: audit 2024-09-13T14:53:48.923372+0000 mon.a (mon.0) 1880 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:50.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[84576]: cluster 2024-09-13T14:53:48.930317+0000 mon.a (mon.0) 1881 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2024-09-13T14:53:50.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[84576]: cluster 2024-09-13T14:53:48.930317+0000 mon.a (mon.0) 1881 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2024-09-13T14:53:50.065 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[87147]: cluster 2024-09-13T14:53:48.627274+0000 mgr.y (mgr.26252) 826 : cluster [DBG] pgmap v1334: 89 pgs: 89 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:53:50.065 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[87147]: cluster 2024-09-13T14:53:48.627274+0000 mgr.y (mgr.26252) 826 : cluster [DBG] pgmap v1334: 89 pgs: 89 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:53:50.065 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[87147]: cluster 2024-09-13T14:53:48.912791+0000 mon.a (mon.0) 1879 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:50.065 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[87147]: cluster 2024-09-13T14:53:48.912791+0000 mon.a (mon.0) 1879 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:50.065 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[87147]: audit 2024-09-13T14:53:48.923372+0000 mon.a (mon.0) 1880 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:50.065 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[87147]: audit 2024-09-13T14:53:48.923372+0000 mon.a (mon.0) 1880 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:50.065 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[87147]: cluster 2024-09-13T14:53:48.930317+0000 mon.a (mon.0) 1881 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2024-09-13T14:53:50.065 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:49 smithi099 bash[87147]: cluster 2024-09-13T14:53:48.930317+0000 mon.a (mon.0) 1881 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2024-09-13T14:53:50.084 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:50.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:50.187 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:50.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:49 smithi177 bash[68594]: cluster 2024-09-13T14:53:48.627274+0000 mgr.y (mgr.26252) 826 : cluster [DBG] pgmap v1334: 89 pgs: 89 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:53:50.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:49 smithi177 bash[68594]: cluster 2024-09-13T14:53:48.627274+0000 mgr.y (mgr.26252) 826 : cluster [DBG] pgmap v1334: 89 pgs: 89 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:53:50.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:49 smithi177 bash[68594]: cluster 2024-09-13T14:53:48.912791+0000 mon.a (mon.0) 1879 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:50.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:49 smithi177 bash[68594]: cluster 2024-09-13T14:53:48.912791+0000 mon.a (mon.0) 1879 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:50.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:49 smithi177 bash[68594]: audit 2024-09-13T14:53:48.923372+0000 mon.a (mon.0) 1880 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:50.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:49 smithi177 bash[68594]: audit 2024-09-13T14:53:48.923372+0000 mon.a (mon.0) 1880 : audit [INF] from='client.? 172.21.15.99:0/2534790903' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:50.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:49 smithi177 bash[68594]: cluster 2024-09-13T14:53:48.930317+0000 mon.a (mon.0) 1881 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2024-09-13T14:53:50.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:49 smithi177 bash[68594]: cluster 2024-09-13T14:53:48.930317+0000 mon.a (mon.0) 1881 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2024-09-13T14:53:50.220 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:50.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:50.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:50.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:50.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:50.461 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:53:50 smithi177 bash[61439]: ts=2024-09-13T14:53:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:53:50.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:50.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:50.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 24 16 41602 41586 108.286 68.25 0.00437507 0.00922852 2024-09-13T14:53:50.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:50.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:50.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:50.742 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:50.845 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:50.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:50.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:50.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:51.004 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:51.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:51.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:51.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:51.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:51.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:51.268 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:51.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:51.374 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:51.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:51.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:51.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:51.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:51.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 25 16 43251 43235 108.076 103.062 0.011677 0.00924722 2024-09-13T14:53:51.636 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:51.658 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:51.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:51.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:51.789 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:51.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:51.849 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:53:51.893 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:51.918 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:51.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:52.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:52.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:51 smithi177 bash[68594]: cluster 2024-09-13T14:53:50.627932+0000 mgr.y (mgr.26252) 827 : cluster [DBG] pgmap v1336: 89 pgs: 89 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:53:52.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:51 smithi177 bash[68594]: cluster 2024-09-13T14:53:50.627932+0000 mgr.y (mgr.26252) 827 : cluster [DBG] pgmap v1336: 89 pgs: 89 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:53:52.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:52.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:51 smithi099 bash[84576]: cluster 2024-09-13T14:53:50.627932+0000 mgr.y (mgr.26252) 827 : cluster [DBG] pgmap v1336: 89 pgs: 89 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:53:52.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:51 smithi099 bash[84576]: cluster 2024-09-13T14:53:50.627932+0000 mgr.y (mgr.26252) 827 : cluster [DBG] pgmap v1336: 89 pgs: 89 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:53:52.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:51 smithi099 bash[87147]: cluster 2024-09-13T14:53:50.627932+0000 mgr.y (mgr.26252) 827 : cluster [DBG] pgmap v1336: 89 pgs: 89 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:53:52.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:51 smithi099 bash[87147]: cluster 2024-09-13T14:53:50.627932+0000 mgr.y (mgr.26252) 827 : cluster [DBG] pgmap v1336: 89 pgs: 89 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:53:52.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:52.166 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:52.186 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:52.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:52.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:52.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:52.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:52.433 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:52.456 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:52.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:52.560 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:52.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:52.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:52.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 26 16 44827 44811 107.708 98.5 0.00728599 0.00927879 2024-09-13T14:53:52.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:52.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:52.819 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:52.852 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:52.956 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:52.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:53.086 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:53.108 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:53.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:53.211 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:53.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:53.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:53.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:53.383 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:53.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:53.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:53.511 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:53.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:53.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 27 16 46406 46390 107.373 98.6875 0.00434511 0.00930748 2024-09-13T14:53:53.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:53.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:53:53.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:53.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:53.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:53:53.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:53.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:53.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:53.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:53.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:53.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:54.014 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:54.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:54.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:54.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:54.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:54.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:54.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:53 smithi099 bash[84576]: cluster 2024-09-13T14:53:52.628496+0000 mgr.y (mgr.26252) 828 : cluster [DBG] pgmap v1337: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:53:54.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:53 smithi099 bash[84576]: cluster 2024-09-13T14:53:52.628496+0000 mgr.y (mgr.26252) 828 : cluster [DBG] pgmap v1337: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:53:54.262 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:53 smithi099 bash[87147]: cluster 2024-09-13T14:53:52.628496+0000 mgr.y (mgr.26252) 828 : cluster [DBG] pgmap v1337: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:53:54.262 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:53 smithi099 bash[87147]: cluster 2024-09-13T14:53:52.628496+0000 mgr.y (mgr.26252) 828 : cluster [DBG] pgmap v1337: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:53:54.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:54.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:54.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:54.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:53 smithi177 bash[68594]: cluster 2024-09-13T14:53:52.628496+0000 mgr.y (mgr.26252) 828 : cluster [DBG] pgmap v1337: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:53:54.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:53 smithi177 bash[68594]: cluster 2024-09-13T14:53:52.628496+0000 mgr.y (mgr.26252) 828 : cluster [DBG] pgmap v1337: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:53:54.407 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:54.434 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:54.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:54.540 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:54.565 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:54.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:54.593 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 28 16 47898 47882 106.869 93.25 0.0141541 0.00935266 2024-09-13T14:53:54.670 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:54.699 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:54.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:54.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:54.827 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:54.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:54.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:54.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:55.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:55.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:55.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:55.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:55.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:55.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:54 smithi099 bash[84576]: cluster 2024-09-13T14:53:54.629069+0000 mgr.y (mgr.26252) 829 : cluster [DBG] pgmap v1338: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:55.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:55 smithi099 bash[84576]: cluster 2024-09-13T14:53:54.629069+0000 mgr.y (mgr.26252) 829 : cluster [DBG] pgmap v1338: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:55.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:55 smithi099 bash[87147]: cluster 2024-09-13T14:53:54.629069+0000 mgr.y (mgr.26252) 829 : cluster [DBG] pgmap v1338: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:55.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:55 smithi099 bash[87147]: cluster 2024-09-13T14:53:54.629069+0000 mgr.y (mgr.26252) 829 : cluster [DBG] pgmap v1338: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:55.336 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:55.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:54 smithi177 bash[68594]: cluster 2024-09-13T14:53:54.629069+0000 mgr.y (mgr.26252) 829 : cluster [DBG] pgmap v1338: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:55.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:54 smithi177 bash[68594]: cluster 2024-09-13T14:53:54.629069+0000 mgr.y (mgr.26252) 829 : cluster [DBG] pgmap v1338: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:55.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:55.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:55.466 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:55.491 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:55.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:55.593 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:55.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 29 16 49408 49392 106.437 94.375 0.0163184 0.00938964 2024-09-13T14:53:55.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:55.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:55.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:55.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:55.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:55.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:55.881 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:55.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:55.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:56.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:56.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:56.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:56.134 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:56.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:56.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:56.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:56.368 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:56.401 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:56.505 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:56.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:56.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:56.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 30 16 50716 50700 105.614 81.75 0.00793602 0.00946323 2024-09-13T14:53:56.635 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:56.667 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:56.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:56.770 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:56.788 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:56.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:56.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:56.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:57.017 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:57.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:57.146 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:57.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:57.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:57.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:57.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:57.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:53:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:53:57] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.51.0" 2024-09-13T14:53:57.402 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:57.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:57.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:57.541 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:57.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:57 smithi177 bash[68594]: cluster 2024-09-13T14:53:56.629400+0000 mgr.y (mgr.26252) 830 : cluster [DBG] pgmap v1339: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:57.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:57 smithi177 bash[68594]: cluster 2024-09-13T14:53:56.629400+0000 mgr.y (mgr.26252) 830 : cluster [DBG] pgmap v1339: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:57.570 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:57.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:57.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 31 16 51195 51179 103.173 29.9375 0.0105677 0.00968614 2024-09-13T14:53:57.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:57.681 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:57 smithi099 bash[84576]: cluster 2024-09-13T14:53:56.629400+0000 mgr.y (mgr.26252) 830 : cluster [DBG] pgmap v1339: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:57.681 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:57 smithi099 bash[84576]: cluster 2024-09-13T14:53:56.629400+0000 mgr.y (mgr.26252) 830 : cluster [DBG] pgmap v1339: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:57.682 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:57 smithi099 bash[87147]: cluster 2024-09-13T14:53:56.629400+0000 mgr.y (mgr.26252) 830 : cluster [DBG] pgmap v1339: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:57.682 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:57 smithi099 bash[87147]: cluster 2024-09-13T14:53:56.629400+0000 mgr.y (mgr.26252) 830 : cluster [DBG] pgmap v1339: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:53:57.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:57.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:57.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:57.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:57.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:57.962 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:57.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:58.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:58.090 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:58.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:58.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:58.225 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:58.329 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:58.358 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:58.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:58.443 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:58.444 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:58.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:58.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:58.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:58.503 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:58 smithi099 bash[84576]: audit 2024-09-13T14:53:58.445356+0000 mon.c (mon.2) 63 : audit [INF] from='client.? 172.21.15.99:0/583720782' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.503 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:58 smithi099 bash[84576]: audit 2024-09-13T14:53:58.445356+0000 mon.c (mon.2) 63 : audit [INF] from='client.? 172.21.15.99:0/583720782' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.503 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:58 smithi099 bash[84576]: audit 2024-09-13T14:53:58.445722+0000 mon.a (mon.0) 1882 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.503 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:58 smithi099 bash[84576]: audit 2024-09-13T14:53:58.445722+0000 mon.a (mon.0) 1882 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.503 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:53:58.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:58.593 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:58 smithi177 bash[68594]: audit 2024-09-13T14:53:58.445356+0000 mon.c (mon.2) 63 : audit [INF] from='client.? 172.21.15.99:0/583720782' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.594 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:58 smithi177 bash[68594]: audit 2024-09-13T14:53:58.445356+0000 mon.c (mon.2) 63 : audit [INF] from='client.? 172.21.15.99:0/583720782' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.594 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:58 smithi177 bash[68594]: audit 2024-09-13T14:53:58.445722+0000 mon.a (mon.0) 1882 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.594 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:58 smithi177 bash[68594]: audit 2024-09-13T14:53:58.445722+0000 mon.a (mon.0) 1882 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 32 16 52452 52436 102.404 78.5625 0.0139503 0.0097604 2024-09-13T14:53:58.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:58.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:58.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:58.748 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:58.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:58.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:58 smithi099 bash[87147]: audit 2024-09-13T14:53:58.445356+0000 mon.c (mon.2) 63 : audit [INF] from='client.? 172.21.15.99:0/583720782' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:58 smithi099 bash[87147]: audit 2024-09-13T14:53:58.445356+0000 mon.c (mon.2) 63 : audit [INF] from='client.? 172.21.15.99:0/583720782' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:58 smithi099 bash[87147]: audit 2024-09-13T14:53:58.445722+0000 mon.a (mon.0) 1882 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:58 smithi099 bash[87147]: audit 2024-09-13T14:53:58.445722+0000 mon.a (mon.0) 1882 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:58.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:58.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:58.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:59.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:59.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:59.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:59.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:59.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:59.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:59.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:59.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:59.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:59.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:59.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:59.537 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:59.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:59.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 33 16 53595 53579 101.465 71.4375 0.00808457 0.0098501 2024-09-13T14:53:59.641 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:59.650 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:59 smithi177 bash[68594]: audit 2024-09-13T14:53:58.497696+0000 mon.a (mon.0) 1883 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:53:59.650 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:59 smithi177 bash[68594]: audit 2024-09-13T14:53:58.497696+0000 mon.a (mon.0) 1883 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:53:59.650 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:59 smithi177 bash[68594]: cluster 2024-09-13T14:53:58.500971+0000 mon.a (mon.0) 1884 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2024-09-13T14:53:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:59 smithi177 bash[68594]: cluster 2024-09-13T14:53:58.500971+0000 mon.a (mon.0) 1884 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2024-09-13T14:53:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:59 smithi177 bash[68594]: cluster 2024-09-13T14:53:58.631258+0000 mgr.y (mgr.26252) 831 : cluster [DBG] pgmap v1341: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:53:59.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:53:59 smithi177 bash[68594]: cluster 2024-09-13T14:53:58.631258+0000 mgr.y (mgr.26252) 831 : cluster [DBG] pgmap v1341: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:53:59.670 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:59.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:59.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:59.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:59.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:59.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[87147]: audit 2024-09-13T14:53:58.497696+0000 mon.a (mon.0) 1883 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:53:59.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[87147]: audit 2024-09-13T14:53:58.497696+0000 mon.a (mon.0) 1883 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:53:59.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[87147]: cluster 2024-09-13T14:53:58.500971+0000 mon.a (mon.0) 1884 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2024-09-13T14:53:59.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[87147]: cluster 2024-09-13T14:53:58.500971+0000 mon.a (mon.0) 1884 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2024-09-13T14:53:59.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[87147]: cluster 2024-09-13T14:53:58.631258+0000 mgr.y (mgr.26252) 831 : cluster [DBG] pgmap v1341: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:53:59.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[87147]: cluster 2024-09-13T14:53:58.631258+0000 mgr.y (mgr.26252) 831 : cluster [DBG] pgmap v1341: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:53:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[84576]: audit 2024-09-13T14:53:58.497696+0000 mon.a (mon.0) 1883 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:53:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[84576]: audit 2024-09-13T14:53:58.497696+0000 mon.a (mon.0) 1883 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:53:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[84576]: cluster 2024-09-13T14:53:58.500971+0000 mon.a (mon.0) 1884 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2024-09-13T14:53:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[84576]: cluster 2024-09-13T14:53:58.500971+0000 mon.a (mon.0) 1884 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2024-09-13T14:53:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[84576]: cluster 2024-09-13T14:53:58.631258+0000 mgr.y (mgr.26252) 831 : cluster [DBG] pgmap v1341: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:53:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:53:59 smithi099 bash[84576]: cluster 2024-09-13T14:53:58.631258+0000 mgr.y (mgr.26252) 831 : cluster [DBG] pgmap v1341: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:53:59.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:59.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:59.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:00.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:00.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:00.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:00.169 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:00.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:00.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:00.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:00.323 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:00.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:00.427 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:00.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:00.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:00.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 34 16 54378 54362 99.9201 48.9375 0.00834726 0.0100031 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:00.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:00.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:54:00 smithi177 bash[61439]: ts=2024-09-13T14:54:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:00.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:00.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:00.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:00.834 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:00.860 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:00.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:00.917 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:54:00.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:00.982 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:00.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:01.089 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:01.122 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:01.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:01.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:01.255 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:01.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:01.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:01.383 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:01.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:01.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:01.528 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:01.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:01.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 35 16 55756 55740 99.5258 86.125 0.014823 0.0100425 2024-09-13T14:54:01.631 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:01.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:01.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:01.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:01.772 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:01 smithi177 bash[68594]: cluster 2024-09-13T14:54:00.631830+0000 mgr.y (mgr.26252) 832 : cluster [DBG] pgmap v1342: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-13T14:54:01.772 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:01 smithi177 bash[68594]: cluster 2024-09-13T14:54:00.631830+0000 mgr.y (mgr.26252) 832 : cluster [DBG] pgmap v1342: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-13T14:54:01.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:01.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:01.879 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:01.879 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:01.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:01.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:02.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:02.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:02.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:02.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:01 smithi099 bash[84576]: cluster 2024-09-13T14:54:00.631830+0000 mgr.y (mgr.26252) 832 : cluster [DBG] pgmap v1342: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-13T14:54:02.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:01 smithi099 bash[84576]: cluster 2024-09-13T14:54:00.631830+0000 mgr.y (mgr.26252) 832 : cluster [DBG] pgmap v1342: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-13T14:54:02.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:01 smithi099 bash[87147]: cluster 2024-09-13T14:54:00.631830+0000 mgr.y (mgr.26252) 832 : cluster [DBG] pgmap v1342: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-13T14:54:02.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:01 smithi099 bash[87147]: cluster 2024-09-13T14:54:00.631830+0000 mgr.y (mgr.26252) 832 : cluster [DBG] pgmap v1342: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-13T14:54:02.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:02.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:02.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:02.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:02.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:02.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:02.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:02.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:02.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:02.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 36 16 57095 57079 99.0856 83.6875 0.00697848 0.010087 2024-09-13T14:54:02.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:02.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:02.701 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:02.718 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:02.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:02.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:02.853 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:02.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:02.959 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:02.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:02.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:03.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:03.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:03.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:03.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:03.244 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:03.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:03.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:03.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:03.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:03.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:03.528 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:03.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:03.537 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:54:03.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 37 16 58254 58238 98.3653 72.4375 0.00298681 0.0101535 2024-09-13T14:54:03.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:03.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:03.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:03.753 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:03.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:03.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:03.872 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:03.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:03.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:03 smithi177 bash[68594]: cluster 2024-09-13T14:54:02.632348+0000 mgr.y (mgr.26252) 833 : cluster [DBG] pgmap v1343: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:54:03.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:03 smithi177 bash[68594]: cluster 2024-09-13T14:54:02.632348+0000 mgr.y (mgr.26252) 833 : cluster [DBG] pgmap v1343: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:54:03.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:03 smithi177 bash[68594]: audit 2024-09-13T14:54:03.286448+0000 mon.a (mon.0) 1885 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:03.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:03 smithi177 bash[68594]: audit 2024-09-13T14:54:03.286448+0000 mon.a (mon.0) 1885 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:03.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:03 smithi177 bash[68594]: audit 2024-09-13T14:54:03.288788+0000 mon.a (mon.0) 1886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:03.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:03 smithi177 bash[68594]: audit 2024-09-13T14:54:03.288788+0000 mon.a (mon.0) 1886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:03.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:04.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:04.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[84576]: cluster 2024-09-13T14:54:02.632348+0000 mgr.y (mgr.26252) 833 : cluster [DBG] pgmap v1343: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:54:04.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[84576]: cluster 2024-09-13T14:54:02.632348+0000 mgr.y (mgr.26252) 833 : cluster [DBG] pgmap v1343: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:54:04.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[84576]: audit 2024-09-13T14:54:03.286448+0000 mon.a (mon.0) 1885 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:04.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[84576]: audit 2024-09-13T14:54:03.286448+0000 mon.a (mon.0) 1885 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:04.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[84576]: audit 2024-09-13T14:54:03.288788+0000 mon.a (mon.0) 1886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:04.024 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[84576]: audit 2024-09-13T14:54:03.288788+0000 mon.a (mon.0) 1886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:04.024 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[87147]: cluster 2024-09-13T14:54:02.632348+0000 mgr.y (mgr.26252) 833 : cluster [DBG] pgmap v1343: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:54:04.024 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[87147]: cluster 2024-09-13T14:54:02.632348+0000 mgr.y (mgr.26252) 833 : cluster [DBG] pgmap v1343: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:54:04.025 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[87147]: audit 2024-09-13T14:54:03.286448+0000 mon.a (mon.0) 1885 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:04.025 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[87147]: audit 2024-09-13T14:54:03.286448+0000 mon.a (mon.0) 1885 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:04.025 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[87147]: audit 2024-09-13T14:54:03.288788+0000 mon.a (mon.0) 1886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:04.025 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:03 smithi099 bash[87147]: audit 2024-09-13T14:54:03.288788+0000 mon.a (mon.0) 1886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:04.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:04.142 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:04.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.294 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:04.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.424 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:04.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:04.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:04.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 38 16 59120 59104 97.201 54.125 0.0265583 0.0102822 2024-09-13T14:54:04.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.692 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:04.799 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:04.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:04.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:04.929 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:05.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:05.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:05.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:05.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:05.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:05.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:05.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:05.353 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:05.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:05.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:05.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:05.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:05.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:05.595 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 39 16 60443 60427 96.8286 82.6875 0.00556656 0.0103233 2024-09-13T14:54:05.615 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:05.720 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:05.744 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:05.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:05.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:05.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:05 smithi177 bash[68594]: cluster 2024-09-13T14:54:04.632897+0000 mgr.y (mgr.26252) 834 : cluster [DBG] pgmap v1344: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:05 smithi177 bash[68594]: cluster 2024-09-13T14:54:04.632897+0000 mgr.y (mgr.26252) 834 : cluster [DBG] pgmap v1344: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:05.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:05.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:06.006 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:05 smithi099 bash[84576]: cluster 2024-09-13T14:54:04.632897+0000 mgr.y (mgr.26252) 834 : cluster [DBG] pgmap v1344: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:06.006 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:05 smithi099 bash[84576]: cluster 2024-09-13T14:54:04.632897+0000 mgr.y (mgr.26252) 834 : cluster [DBG] pgmap v1344: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:06.006 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:05 smithi099 bash[87147]: cluster 2024-09-13T14:54:04.632897+0000 mgr.y (mgr.26252) 834 : cluster [DBG] pgmap v1344: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:06.006 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:05 smithi099 bash[87147]: cluster 2024-09-13T14:54:04.632897+0000 mgr.y (mgr.26252) 834 : cluster [DBG] pgmap v1344: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:06.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:06.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:06.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:06.150 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:06.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:06.253 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:06.401 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:06.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:06.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:06.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:06.595 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T14:54:06.598568+0000 min lat: 0.00109161 max lat: 0.809605 avg lat: 0.0103668 2024-09-13T14:54:06.595 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:54:06.595 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 40 16 61731 61715 96.4202 80.5 0.0163394 0.0103668 2024-09-13T14:54:06.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:06.686 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:06.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:06.791 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.820 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:06.924 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:06.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:06.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:07.057 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:07.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:07.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:07.188 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:07.196 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:54:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:54:07] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.51.0" 2024-09-13T14:54:07.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:07.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:07.325 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:07.356 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:07.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:07.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:07.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:07.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:07.589 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:07.594 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 41 16 63001 62985 96.0043 79.375 0.0106879 0.010411 2024-09-13T14:54:07.615 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:07.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:07.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:07.729 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:07 smithi099 bash[84576]: cluster 2024-09-13T14:54:06.633247+0000 mgr.y (mgr.26252) 835 : cluster [DBG] pgmap v1345: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:07.729 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:07 smithi099 bash[84576]: cluster 2024-09-13T14:54:06.633247+0000 mgr.y (mgr.26252) 835 : cluster [DBG] pgmap v1345: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:07.730 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:07 smithi099 bash[87147]: cluster 2024-09-13T14:54:06.633247+0000 mgr.y (mgr.26252) 835 : cluster [DBG] pgmap v1345: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:07.730 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:07 smithi099 bash[87147]: cluster 2024-09-13T14:54:06.633247+0000 mgr.y (mgr.26252) 835 : cluster [DBG] pgmap v1345: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:07.746 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:07.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:07.850 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:07.877 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:07.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:07.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:07 smithi177 bash[68594]: cluster 2024-09-13T14:54:06.633247+0000 mgr.y (mgr.26252) 835 : cluster [DBG] pgmap v1345: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:07.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:07 smithi177 bash[68594]: cluster 2024-09-13T14:54:06.633247+0000 mgr.y (mgr.26252) 835 : cluster [DBG] pgmap v1345: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:54:08.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:08.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:08.022 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:08.097 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:54:08.097 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:08.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:08.132 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:08.138 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:54:08.138 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:08.178 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:54:08.178 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:08.219 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:08.220 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:54:08.238 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:08.248 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:54:08.249 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:08.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:08.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:08.277 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:08.278 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:54:08.304 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:08.304 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:54:08.324 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:08.330 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:08.330 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:54:08.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:08.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:08.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:08.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:08.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:08.595 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 42 16 64212 64196 95.5204 75.6875 0.00332891 0.010463 2024-09-13T14:54:08.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:08.644 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:08.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:08.704 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T14:54:08.747 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:08.773 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:08.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:08.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:08.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:09.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.034 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:09.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:08 smithi177 bash[68594]: audit 2024-09-13T14:54:08.064649+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.99:0/4206799385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:09.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:08 smithi177 bash[68594]: audit 2024-09-13T14:54:08.064649+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.99:0/4206799385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:09.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:08 smithi177 bash[68594]: audit 2024-09-13T14:54:08.375452+0000 mon.a (mon.0) 1887 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:09.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:08 smithi177 bash[68594]: audit 2024-09-13T14:54:08.375452+0000 mon.a (mon.0) 1887 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:09.138 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:09.167 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:09.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:09.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: audit 2024-09-13T14:54:08.064649+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.99:0/4206799385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:09.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: audit 2024-09-13T14:54:08.064649+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.99:0/4206799385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:09.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: audit 2024-09-13T14:54:08.375452+0000 mon.a (mon.0) 1887 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:09.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: audit 2024-09-13T14:54:08.375452+0000 mon.a (mon.0) 1887 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:09.287 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: audit 2024-09-13T14:54:08.064649+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.99:0/4206799385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:09.287 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: audit 2024-09-13T14:54:08.064649+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.99:0/4206799385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:09.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: audit 2024-09-13T14:54:08.375452+0000 mon.a (mon.0) 1887 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:09.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: audit 2024-09-13T14:54:08.375452+0000 mon.a (mon.0) 1887 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:09.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:09.408 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.435 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:09.540 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:09.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:09.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:09.595 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 43 16 65445 65429 95.091 77.0625 0.00411367 0.0105103 2024-09-13T14:54:09.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:09.696 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:09.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:09.799 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:09.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:09 smithi177 bash[68594]: cluster 2024-09-13T14:54:08.633878+0000 mgr.y (mgr.26252) 836 : cluster [DBG] pgmap v1346: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:09.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:09 smithi177 bash[68594]: cluster 2024-09-13T14:54:08.633878+0000 mgr.y (mgr.26252) 836 : cluster [DBG] pgmap v1346: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:09.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:09 smithi177 bash[68594]: cluster 2024-09-13T14:54:08.697934+0000 mon.a (mon.0) 1888 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:09.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:09 smithi177 bash[68594]: cluster 2024-09-13T14:54:08.697934+0000 mon.a (mon.0) 1888 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:09.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:09 smithi177 bash[68594]: audit 2024-09-13T14:54:08.707253+0000 mon.a (mon.0) 1889 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:09.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:09 smithi177 bash[68594]: audit 2024-09-13T14:54:08.707253+0000 mon.a (mon.0) 1889 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:09.810 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:09 smithi177 bash[68594]: cluster 2024-09-13T14:54:08.719401+0000 mon.a (mon.0) 1890 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2024-09-13T14:54:09.810 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:09 smithi177 bash[68594]: cluster 2024-09-13T14:54:08.719401+0000 mon.a (mon.0) 1890 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2024-09-13T14:54:09.822 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:09.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.936 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: cluster 2024-09-13T14:54:08.633878+0000 mgr.y (mgr.26252) 836 : cluster [DBG] pgmap v1346: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:09.936 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: cluster 2024-09-13T14:54:08.633878+0000 mgr.y (mgr.26252) 836 : cluster [DBG] pgmap v1346: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:09.936 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: cluster 2024-09-13T14:54:08.697934+0000 mon.a (mon.0) 1888 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:09.936 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: cluster 2024-09-13T14:54:08.697934+0000 mon.a (mon.0) 1888 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:09.936 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: audit 2024-09-13T14:54:08.707253+0000 mon.a (mon.0) 1889 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:09.936 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: audit 2024-09-13T14:54:08.707253+0000 mon.a (mon.0) 1889 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:09.937 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: cluster 2024-09-13T14:54:08.719401+0000 mon.a (mon.0) 1890 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2024-09-13T14:54:09.937 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[84576]: cluster 2024-09-13T14:54:08.719401+0000 mon.a (mon.0) 1890 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2024-09-13T14:54:09.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: cluster 2024-09-13T14:54:08.633878+0000 mgr.y (mgr.26252) 836 : cluster [DBG] pgmap v1346: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:09.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: cluster 2024-09-13T14:54:08.633878+0000 mgr.y (mgr.26252) 836 : cluster [DBG] pgmap v1346: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:09.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: cluster 2024-09-13T14:54:08.697934+0000 mon.a (mon.0) 1888 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:09.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: cluster 2024-09-13T14:54:08.697934+0000 mon.a (mon.0) 1888 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:09.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: audit 2024-09-13T14:54:08.707253+0000 mon.a (mon.0) 1889 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:09.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: audit 2024-09-13T14:54:08.707253+0000 mon.a (mon.0) 1889 : audit [INF] from='client.? 172.21.15.99:0/1092671731' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:09.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: cluster 2024-09-13T14:54:08.719401+0000 mon.a (mon.0) 1890 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2024-09-13T14:54:09.938 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:09 smithi099 bash[87147]: cluster 2024-09-13T14:54:08.719401+0000 mon.a (mon.0) 1890 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2024-09-13T14:54:09.951 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:10.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:10.114 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:10.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:10.220 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:10.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:10.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:10.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:54:10 smithi177 bash[61439]: ts=2024-09-13T14:54:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:10.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:10.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:10.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:10.514 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:10.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:10.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:10.595 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 44 16 66259 66243 94.0858 50.875 0.0130524 0.0106231 2024-09-13T14:54:10.647 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:10.681 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:10.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:10.786 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:10.808 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:10.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:10.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:10 smithi099 bash[84576]: audit 2024-09-13T14:54:09.871090+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:10.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:10 smithi099 bash[84576]: audit 2024-09-13T14:54:09.871090+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:10.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:10 smithi099 bash[87147]: audit 2024-09-13T14:54:09.871090+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:10.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:10 smithi099 bash[87147]: audit 2024-09-13T14:54:09.871090+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:10.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:10.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:11.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:11.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:11.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:11.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:11.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:11.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:11.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:10 smithi177 bash[68594]: audit 2024-09-13T14:54:09.871090+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:11.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:10 smithi177 bash[68594]: audit 2024-09-13T14:54:09.871090+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:54:11.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:11.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:11.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:11.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:11.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:11.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:11.595 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 45 16 67353 67337 93.5143 68.375 0.0104555 0.0106892 2024-09-13T14:54:11.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:11.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:11.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:11.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:11.755 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:11.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:11.757 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:11 smithi177 bash[68594]: cluster 2024-09-13T14:54:10.634531+0000 mgr.y (mgr.26252) 837 : cluster [DBG] pgmap v1348: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:11.757 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:11 smithi177 bash[68594]: cluster 2024-09-13T14:54:10.634531+0000 mgr.y (mgr.26252) 837 : cluster [DBG] pgmap v1348: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:11.858 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:11.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:11.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:11.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:12.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:12.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:12.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:11 smithi099 bash[84576]: cluster 2024-09-13T14:54:10.634531+0000 mgr.y (mgr.26252) 837 : cluster [DBG] pgmap v1348: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:12.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:11 smithi099 bash[84576]: cluster 2024-09-13T14:54:10.634531+0000 mgr.y (mgr.26252) 837 : cluster [DBG] pgmap v1348: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:12.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:11 smithi099 bash[87147]: cluster 2024-09-13T14:54:10.634531+0000 mgr.y (mgr.26252) 837 : cluster [DBG] pgmap v1348: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:12.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:11 smithi099 bash[87147]: cluster 2024-09-13T14:54:10.634531+0000 mgr.y (mgr.26252) 837 : cluster [DBG] pgmap v1348: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:12.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:12.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:12.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:12.245 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:12.265 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:12.368 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:12.394 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:12.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:12.497 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:12.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:12.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:12.595 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 46 16 68047 68031 92.4242 43.375 0.0200732 0.010815 2024-09-13T14:54:12.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:12.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:12.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:12.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:12.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:12.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:12.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:12.904 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:12.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:13.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:13.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:13.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:13.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:13.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:13.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:13.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:13.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:13.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:13.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:13.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:13.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:13.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:13.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:13.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:13.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 47 16 68873 68857 91.5561 51.625 0.0408574 0.010908 2024-09-13T14:54:13.643 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:13.671 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:13.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:13.775 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:13.792 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:13.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:13 smithi177 bash[68594]: cluster 2024-09-13T14:54:12.635117+0000 mgr.y (mgr.26252) 838 : cluster [DBG] pgmap v1349: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:13.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:13 smithi177 bash[68594]: cluster 2024-09-13T14:54:12.635117+0000 mgr.y (mgr.26252) 838 : cluster [DBG] pgmap v1349: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:13.894 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:13.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:13.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:14.021 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:14.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:14.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:14.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:13 smithi099 bash[84576]: cluster 2024-09-13T14:54:12.635117+0000 mgr.y (mgr.26252) 838 : cluster [DBG] pgmap v1349: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:14.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:13 smithi099 bash[84576]: cluster 2024-09-13T14:54:12.635117+0000 mgr.y (mgr.26252) 838 : cluster [DBG] pgmap v1349: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:14.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:13 smithi099 bash[87147]: cluster 2024-09-13T14:54:12.635117+0000 mgr.y (mgr.26252) 838 : cluster [DBG] pgmap v1349: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:14.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:13 smithi099 bash[87147]: cluster 2024-09-13T14:54:12.635117+0000 mgr.y (mgr.26252) 838 : cluster [DBG] pgmap v1349: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:14.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:54:14.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:54:14.161 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:14.189 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:14.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:14.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:14.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:14.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:14.487 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:14.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:14.594 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:14.596 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 48 16 69625 69609 90.6277 47 0.00327007 0.0110251 2024-09-13T14:54:14.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:14.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:14.724 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:14.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:14.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:14.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:14.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:14.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:14.984 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:15.009 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:15.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:15.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:15.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:15.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:15.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:15.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:15.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:15.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:15.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:15.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:15.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:15.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:15.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:15.596 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 49 16 70751 70735 90.2143 70.375 0.0112317 0.0110792 2024-09-13T14:54:15.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:15.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:15.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:15.810 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:15.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:15 smithi099 bash[87147]: cluster 2024-09-13T14:54:14.636919+0000 mgr.y (mgr.26252) 839 : cluster [DBG] pgmap v1350: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:15.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:15 smithi099 bash[87147]: cluster 2024-09-13T14:54:14.636919+0000 mgr.y (mgr.26252) 839 : cluster [DBG] pgmap v1350: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:15.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:15 smithi099 bash[84576]: cluster 2024-09-13T14:54:14.636919+0000 mgr.y (mgr.26252) 839 : cluster [DBG] pgmap v1350: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:15.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:15 smithi099 bash[84576]: cluster 2024-09-13T14:54:14.636919+0000 mgr.y (mgr.26252) 839 : cluster [DBG] pgmap v1350: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:15.822 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:15 smithi177 bash[68594]: cluster 2024-09-13T14:54:14.636919+0000 mgr.y (mgr.26252) 839 : cluster [DBG] pgmap v1350: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:15.822 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:15 smithi177 bash[68594]: cluster 2024-09-13T14:54:14.636919+0000 mgr.y (mgr.26252) 839 : cluster [DBG] pgmap v1350: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:15.840 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:15.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:15.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:15.984 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:15.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:16.089 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:16.112 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:16.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:16.217 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:16.237 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:16.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:16.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:16.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:16.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:16.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:16.596 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 50 16 71719 71703 89.6199 60.5 0.00685853 0.0111523 2024-09-13T14:54:16.611 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T14:54:16.611 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T14:54:16.611 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T14:54:16.611 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T14:54:16.611 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.611 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:7919 41.8 41:178de2fa:::benchmark_data_smithi099_126887_object7918:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e958)", 2024-09-13T14:54:16.611 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:45.688049+0000", 2024-09-13T14:54:16.611 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 390.92065498800002, 2024-09-13T14:54:16.611 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.677281824, 2024-09-13T14:54:16.611 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 7919 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.688049+0000", 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.612 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.688049+0000", 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.688048+0000", 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.613 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.688168+0000", 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000119881 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.688180+0000", 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1671999999999999e-05 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.688185+0000", 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9400000000000001e-06 2024-09-13T14:54:16.614 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.688237+0000", 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2559000000000003e-05 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.688255+0000", 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7751999999999999e-05 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.615 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:46.316153+0000", 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.62789846400000005 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:46.316165+0000", 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1663e-05 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.350110+0000", 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0339450910000001 2024-09-13T14:54:16.616 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.350357+0000", 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024713600000000001 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.350604+0000", 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000246426 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.617 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.356892+0000", 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062881730000000002 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.357111+0000", 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00021875799999999999 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.364399+0000", 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0072879650000000004 2024-09-13T14:54:16.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.364514+0000", 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011523699999999999 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.365330+0000", 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00081648500000000004 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.619 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:7922 41.8 41:10f8507c:::benchmark_data_smithi099_126887_object7921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e958)", 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:45.691481+0000", 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 390.91722295400001, 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.674610127, 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 7922 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.620 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.691481+0000", 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.691481+0000", 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.691480+0000", 2024-09-13T14:54:16.621 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.691508+0000", 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8328000000000001e-05 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.691519+0000", 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1072e-05 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.622 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.691525+0000", 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7579999999999998e-06 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.691560+0000", 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4829000000000001e-05 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.691565+0000", 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9250000000000003e-06 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:46.316191+0000", 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.62462618000000003 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:46.316198+0000", 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8669999999999996e-06 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.352070+0000", 2024-09-13T14:54:16.624 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0358719240000001 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.352467+0000", 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00039753999999999999 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.352917+0000", 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00044983600000000001 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.625 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.356914+0000", 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003996861 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.357264+0000", 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00035027700000000002 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.365445+0000", 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0081810259999999992 2024-09-13T14:54:16.626 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.365563+0000", 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011747399999999999 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.366091+0000", 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00052809999999999999 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.627 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:7926 41.8 41:19baddee:::benchmark_data_smithi099_126887_object7925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e958)", 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:45.692807+0000", 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 390.91589645599998, 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6741400550000001, 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 7926 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.628 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.692807+0000", 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.692807+0000", 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.692806+0000", 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.692925+0000", 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000118819 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.692936+0000", 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0859000000000001e-05 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.630 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.692942+0000", 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2600000000000002e-06 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.692984+0000", 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1980000000000001e-05 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:45.692994+0000", 2024-09-13T14:54:16.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6549999999999999e-06 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:46.316224+0000", 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.62323061499999999 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:46.316230+0000", 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0809999999999999e-06 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.632 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.354074+0000", 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.037843965 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.354405+0000", 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033101199999999999 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.354801+0000", 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00039560700000000002 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.357289+0000", 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002487815 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.357783+0000", 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000494573 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.366272+0000", 2024-09-13T14:54:16.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0084889240000000001 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.366389+0000", 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116675 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:47.366947+0000", 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00055820299999999995 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11764 41.8 41:173a7f39:::benchmark_data_smithi099_126887_object11763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e960)", 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.282602+0000", 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.32610186699998, 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0554135760000001, 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11764 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.636 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.282602+0000", 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.282602+0000", 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.282600+0000", 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:16.637 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.282736+0000", 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000135916 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.282763+0000", 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7636000000000001e-05 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.638 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.282772+0000", 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3210000000000005e-06 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.282833+0000", 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1680000000000006e-05 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.283014+0000", 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000180137 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.639 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.283206+0000", 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019254100000000001 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.284230+0000", 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010234300000000001 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.288617+0000", 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043870890000000003 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.640 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.288627+0000", 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0413e-05 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.314748+0000", 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.026120924 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.315433+0000", 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00068520600000000005 2024-09-13T14:54:16.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.337930+0000", 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022496909999999998 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338015+0000", 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5241999999999995e-05 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.642 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11772 41.8 41:15be3459:::benchmark_data_smithi099_126887_object11771:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e960)", 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.287937+0000", 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.32076649499999, 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0503212980000001, 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11772 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.643 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.287937+0000", 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.287937+0000", 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.287936+0000", 2024-09-13T14:54:16.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:16.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.287968+0000", 2024-09-13T14:54:16.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2514000000000001e-05 2024-09-13T14:54:16.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.287980+0000", 2024-09-13T14:54:16.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1712999999999999e-05 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.287986+0000", 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8860000000000001e-06 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.288937+0000", 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00095140100000000005 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.288944+0000", 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1300000000000003e-06 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.315468+0000", 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0265235939999999 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.315597+0000", 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000129197 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.647 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.315760+0000", 2024-09-13T14:54:16.648 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000162535 2024-09-13T14:54:16.648 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.648 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.648 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.648 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.317898+0000", 2024-09-13T14:54:16.649 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021382079999999999 2024-09-13T14:54:16.649 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.649 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.336683+0000", 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018785050000000001 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338092+0000", 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00140924 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338141+0000", 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8588999999999998e-05 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338258+0000", 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000117751 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11815 41.8 41:1a47ec61:::benchmark_data_smithi099_126887_object11814:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.651 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.310932+0000", 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.29777190999999, 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0275467140000001, 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11815 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.310932+0000", 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.310932+0000", 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.310931+0000", 2024-09-13T14:54:16.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.311002+0000", 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.135e-05 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.311014+0000", 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1800999999999999e-05 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.311018+0000", 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0759999999999996e-06 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.311068+0000", 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0265000000000001e-05 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.311075+0000", 2024-09-13T14:54:16.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.545e-06 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.316392+0000", 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.005317046 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.316524+0000", 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013219600000000001 2024-09-13T14:54:16.656 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.316621+0000", 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7239999999999997e-05 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.318623+0000", 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020014680000000002 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.657 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.336852+0000", 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018229649000000001 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338319+0000", 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001467217 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338372+0000", 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2877999999999998e-05 2024-09-13T14:54:16.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338478+0000", 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000106142 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11829 41.8 41:14876607:::benchmark_data_smithi099_126887_object11828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.316019+0000", 2024-09-13T14:54:16.659 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.29268415799999, 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0227555859999999, 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11829 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.316019+0000", 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.316019+0000", 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.316012+0000", 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999928 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.316059+0000", 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6561999999999999e-05 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.316067+0000", 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1729999999999995e-06 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.316083+0000", 2024-09-13T14:54:16.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6464999999999999e-05 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.316148+0000", 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4558000000000001e-05 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.316157+0000", 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1570000000000006e-06 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.317219+0000", 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.001062119 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.317331+0000", 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000111378 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.317482+0000", 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000151574 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.319309+0000", 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018265 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.336933+0000", 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017624397999999999 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338559+0000", 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001625718 2024-09-13T14:54:16.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338631+0000", 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2634999999999998e-05 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338775+0000", 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000143592 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11842 41.8 41:1382db3f:::benchmark_data_smithi099_126887_object11841:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.325675+0000", 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.28302816299998, 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0132965490000001, 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11842 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.667 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.325675+0000", 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.325675+0000", 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.325674+0000", 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.325726+0000", 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2425e-05 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.325737+0000", 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0482000000000001e-05 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.325740+0000", 2024-09-13T14:54:16.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.676e-06 2024-09-13T14:54:16.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.325777+0000", 2024-09-13T14:54:16.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6943000000000002e-05 2024-09-13T14:54:16.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.325783+0000", 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9959999999999999e-06 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.317925+0000", 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99214180500000004 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.318047+0000", 2024-09-13T14:54:16.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000122197 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.318199+0000", 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000152058 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.320878+0000", 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026785509999999999 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.337138+0000", 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016259673999999998 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338827+0000", 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0016890449999999999 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338899+0000", 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2483000000000004e-05 2024-09-13T14:54:16.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338972+0000", 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2873999999999994e-05 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11852 41.8 41:1b5b6b92:::benchmark_data_smithi099_126887_object11851:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.330221+0000", 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.27848295699999, 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0089581860000001, 2024-09-13T14:54:16.674 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11852 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.330221+0000", 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.330221+0000", 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.330220+0000", 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.330286+0000", 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6736999999999995e-05 2024-09-13T14:54:16.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.330301+0000", 2024-09-13T14:54:16.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4543e-05 2024-09-13T14:54:16.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.330305+0000", 2024-09-13T14:54:16.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5279999999999997e-06 2024-09-13T14:54:16.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.330362+0000", 2024-09-13T14:54:16.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6679999999999999e-05 2024-09-13T14:54:16.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.330370+0000", 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0860000000000002e-06 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.318651+0000", 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.98828070099999998 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.318759+0000", 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108513 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.318907+0000", 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014796899999999999 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.322810+0000", 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039028999999999999 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.337211+0000", 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014401018999999999 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339003+0000", 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0017918610000000001 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339052+0000", 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8594000000000003e-05 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339179+0000", 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000127155 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11857 41.8 41:1fb65dd9:::benchmark_data_smithi099_126887_object11856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.335306+0000", 2024-09-13T14:54:16.684 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.27339748100002, 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.004215222, 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11857 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.335306+0000", 2024-09-13T14:54:16.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.335306+0000", 2024-09-13T14:54:16.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.335306+0000", 2024-09-13T14:54:16.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T14:54:16.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.688 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.335417+0000", 2024-09-13T14:54:16.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110983 2024-09-13T14:54:16.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.335422+0000", 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.524e-06 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.335425+0000", 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0759999999999999e-06 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.335477+0000", 2024-09-13T14:54:16.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1542999999999998e-05 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.335484+0000", 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8020000000000003e-06 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.319356+0000", 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.98387192199999995 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.690 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.319525+0000", 2024-09-13T14:54:16.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016925299999999999 2024-09-13T14:54:16.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.691 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.319862+0000", 2024-09-13T14:54:16.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033678799999999997 2024-09-13T14:54:16.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.324795+0000", 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00493312 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.337269+0000", 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012474704 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339273+0000", 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020036120000000001 2024-09-13T14:54:16.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339336+0000", 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2731999999999995e-05 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339521+0000", 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018554199999999999 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11870 41.8 41:18236522:::benchmark_data_smithi099_126887_object11869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.342379+0000", 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.266325053, 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99743281800000005, 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11870 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.342379+0000", 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.342379+0000", 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.342378+0000", 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T14:54:16.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.342432+0000", 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4837999999999999e-05 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.342444+0000", 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2096000000000001e-05 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.342449+0000", 2024-09-13T14:54:16.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2610000000000004e-06 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.342505+0000", 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6569e-05 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.342516+0000", 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1100999999999999e-05 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.320958+0000", 2024-09-13T14:54:16.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97844127599999997 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.321290+0000", 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033281899999999999 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.321690+0000", 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00039989399999999998 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.326756+0000", 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050658480000000004 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.337329+0000", 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01057262 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339596+0000", 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002266672 2024-09-13T14:54:16.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339665+0000", 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9190000000000007e-05 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339811+0000", 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000146661 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.701 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11920 41.8 41:14df68f4:::benchmark_data_smithi099_126887_object11919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.701 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.369602+0000", 2024-09-13T14:54:16.701 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.23910139899999, 2024-09-13T14:54:16.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97062082800000005, 2024-09-13T14:54:16.701 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.701 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.701 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.701 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.701 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11920 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.369602+0000", 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.369602+0000", 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.369601+0000", 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.369640+0000", 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8930999999999997e-05 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.369653+0000", 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2792999999999999e-05 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.369657+0000", 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3590000000000001e-06 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.369720+0000", 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3258999999999999e-05 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.369730+0000", 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4890000000000001e-06 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.322882+0000", 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95315219600000001 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.323241+0000", 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00035902800000000001 2024-09-13T14:54:16.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.323696+0000", 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00045563599999999999 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.330639+0000", 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0069429510000000002 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.337382+0000", 2024-09-13T14:54:16.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067420709999999997 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339903+0000", 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025210950000000001 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.339980+0000", 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7677999999999997e-05 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.340223+0000", 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024273200000000001 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11955 41.8 41:18ec6ab6:::benchmark_data_smithi099_126887_object11954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.387180+0000", 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.22152368299999, 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95338568000000001, 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.708 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11955 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387180+0000", 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387180+0000", 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387178+0000", 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387216+0000", 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8130000000000003e-05 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387230+0000", 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3927999999999999e-05 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387236+0000", 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8969999999999996e-06 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387305+0000", 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8598999999999998e-05 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387313+0000", 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6219999999999998e-06 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.324929+0000", 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.93761608500000004 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.325282+0000", 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00035296299999999997 2024-09-13T14:54:16.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.325738+0000", 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000456126 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.336528+0000", 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010789592000000001 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.337441+0000", 2024-09-13T14:54:16.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00091267799999999999 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.340346+0000", 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029053500000000001 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.340419+0000", 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2384000000000004e-05 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.340566+0000", 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014708599999999999 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11957 41.8 41:16f43dc0:::benchmark_data_smithi099_126887_object11956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.387251+0000", 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.22145251699999, 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95364669899999999, 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.715 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11957 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387251+0000", 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387251+0000", 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387251+0000", 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387290+0000", 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9110000000000003e-05 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387313+0000", 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3272999999999999e-05 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387322+0000", 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6680000000000007e-06 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387374+0000", 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2775999999999998e-05 2024-09-13T14:54:16.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.387408+0000", 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3980999999999999e-05 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.326880+0000", 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.93947158200000003 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.327232+0000", 2024-09-13T14:54:16.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00035244200000000001 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.327682+0000", 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00045008899999999998 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.336544+0000", 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0088612120000000003 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.337508+0000", 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00096452599999999995 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.340655+0000", 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031464639999999999 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.340719+0000", 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4796999999999996e-05 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.340898+0000", 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017828200000000001 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11973 41.8 41:1c1ce6c6:::benchmark_data_smithi099_126887_object11972:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.396000+0000", 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.21270312199999, 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94605853299999998, 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.722 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11973 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.396000+0000", 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.396000+0000", 2024-09-13T14:54:16.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.395999+0000", 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.396068+0000", 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8674e-05 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.396079+0000", 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1301000000000001e-05 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.396085+0000", 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5799999999999999e-06 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.396136+0000", 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1153999999999998e-05 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.396145+0000", 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1840000000000005e-06 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.329024+0000", 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.93287892100000003 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.329358+0000", 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033389500000000001 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.329777+0000", 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00041926399999999998 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.336746+0000", 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0069684680000000002 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.338063+0000", 2024-09-13T14:54:16.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001317342 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.341768+0000", 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037055069999999998 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.341867+0000", 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9006999999999996e-05 2024-09-13T14:54:16.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.342059+0000", 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019151300000000001 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:11982 41.8 41:19ac0260:::benchmark_data_smithi099_126887_object11981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.401802+0000", 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.20690120500001, 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94059336800000004, 2024-09-13T14:54:16.729 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 11982 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.401802+0000", 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.401802+0000", 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.401801+0000", 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.401854+0000", 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2843000000000002e-05 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.401866+0000", 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2489e-05 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.401872+0000", 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1909999999999999e-06 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.401930+0000", 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8026000000000003e-05 2024-09-13T14:54:16.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.401941+0000", 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1147e-05 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.330741+0000", 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92880020299999999 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.330997+0000", 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000256338 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.331326+0000", 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032831199999999998 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.336763+0000", 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054368970000000004 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.340961+0000", 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041988349999999997 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.342153+0000", 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011913550000000001 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.342236+0000", 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2979999999999995e-05 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.342396+0000", 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016003 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:12003 41.8 41:1f7601ef:::benchmark_data_smithi099_126887_object12002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.414269+0000", 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.194434156, 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92835796999999998, 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.736 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.737 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.737 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.737 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.737 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 12003 2024-09-13T14:54:16.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.737 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.414269+0000", 2024-09-13T14:54:16.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.414269+0000", 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.414268+0000", 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.414326+0000", 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8078000000000002e-05 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.414341+0000", 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4491999999999999e-05 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.414348+0000", 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2640000000000002e-06 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.414396+0000", 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8427999999999998e-05 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.414405+0000", 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.405e-06 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.332043+0000", 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91763848400000003 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.332322+0000", 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00027853099999999999 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.332607+0000", 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028498899999999998 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.336775+0000", 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041685439999999997 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.341402+0000", 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046269859999999996 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.342483+0000", 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010809210000000001 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.342546+0000", 2024-09-13T14:54:16.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2852000000000001e-05 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.342627+0000", 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1358000000000003e-05 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:12005 41.8 41:1bc962cb:::benchmark_data_smithi099_126887_object12004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.416456+0000", 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.19224732599997, 2024-09-13T14:54:16.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92654546400000004, 2024-09-13T14:54:16.744 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.744 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.744 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.744 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.744 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.744 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 12005 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.416456+0000", 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.416456+0000", 2024-09-13T14:54:16.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.416455+0000", 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.416563+0000", 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107533 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.416574+0000", 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1443e-05 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.416580+0000", 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9449999999999998e-06 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.416626+0000", 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6424999999999997e-05 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.416632+0000", 2024-09-13T14:54:16.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1240000000000002e-06 2024-09-13T14:54:16.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.333383+0000", 2024-09-13T14:54:16.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91675023499999997 2024-09-13T14:54:16.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.333602+0000", 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000219075 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.333792+0000", 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019059900000000001 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.336789+0000", 2024-09-13T14:54:16.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029966910000000001 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.341569+0000", 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0047799019999999999 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.342716+0000", 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011475000000000001 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.342818+0000", 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000101859 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.343002+0000", 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000183392 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35268.0:12025 41.8 41:19b2ac28:::benchmark_data_smithi099_126887_object12024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e961)", 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:47:49.447853+0000", 2024-09-13T14:54:16.751 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 387.16085031599999, 2024-09-13T14:54:16.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89544153999999998, 2024-09-13T14:54:16.752 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.752 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.752 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.752 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35268", 2024-09-13T14:54:16.752 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:54:16.752 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 12025 2024-09-13T14:54:16.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.752 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.447853+0000", 2024-09-13T14:54:16.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.447853+0000", 2024-09-13T14:54:16.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.447852+0000", 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.447899+0000", 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.672e-05 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.754 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.447912+0000", 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3423e-05 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.447919+0000", 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0380000000000002e-06 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.447983+0000", 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4241999999999999e-05 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:49.447993+0000", 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6229999999999992e-06 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.334421+0000", 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.88642820700000002 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.334630+0000", 2024-09-13T14:54:16.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000208889 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.334784+0000", 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000153568 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.336801+0000", 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002017179 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.341668+0000", 2024-09-13T14:54:16.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0048674629999999998 2024-09-13T14:54:16.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.343134+0000", 2024-09-13T14:54:16.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014660249999999999 2024-09-13T14:54:16.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.343192+0000", 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7497000000000002e-05 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:47:50.343295+0000", 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102916 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.759 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:8372 42.6 42:631e48f8:::benchmark_data_smithi099_148651_object8371:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1014)", 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:29.387416+0000", 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 47.221287461000003, 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.80537644100000005, 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 8372 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.387416+0000", 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.387416+0000", 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.387415+0000", 2024-09-13T14:54:16.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.387454+0000", 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9167e-05 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.387462+0000", 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1799999999999999e-06 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:16.762 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.387465+0000", 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2590000000000001e-06 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.387508+0000", 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3112999999999999e-05 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.387605+0000", 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6688000000000007e-05 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:54:16.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.387689+0000", 2024-09-13T14:54:16.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4252000000000006e-05 2024-09-13T14:54:16.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T14:54:16.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.388330+0000", 2024-09-13T14:54:16.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00064115899999999998 2024-09-13T14:54:16.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.388885+0000", 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00055544999999999998 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:30.192662+0000", 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.80377627699999998 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:30.192736+0000", 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3947000000000002e-05 2024-09-13T14:54:16.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:16.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:16.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T14:54:16.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:30.192793+0000", 2024-09-13T14:54:16.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6916999999999997e-05 2024-09-13T14:54:16.766 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.766 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.766 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.766 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:16.766 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:16.766 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T14:54:16.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:16.767 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:54:16.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:16.908 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:16.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:17.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:17.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:16 smithi099 bash[84576]: audit 2024-09-13T14:54:15.982217+0000 mon.a (mon.0) 1892 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:17.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:16 smithi099 bash[84576]: audit 2024-09-13T14:54:15.982217+0000 mon.a (mon.0) 1892 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:17.042 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:16 smithi099 bash[87147]: audit 2024-09-13T14:54:15.982217+0000 mon.a (mon.0) 1892 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:17.042 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:16 smithi099 bash[87147]: audit 2024-09-13T14:54:15.982217+0000 mon.a (mon.0) 1892 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:17.057 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:17.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:17.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:16 smithi177 bash[68594]: audit 2024-09-13T14:54:15.982217+0000 mon.a (mon.0) 1892 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:17.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:16 smithi177 bash[68594]: audit 2024-09-13T14:54:15.982217+0000 mon.a (mon.0) 1892 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:17.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:17.168 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:54:17] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.51.0" 2024-09-13T14:54:17.182 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:17.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:17.286 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:17.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:17.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:17.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:17.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:17.541 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:17.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:17.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:17.596 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 51 16 72692 72676 89.055 60.8125 0.0123486 0.0112227 2024-09-13T14:54:17.680 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:17.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:17.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:17.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:17.842 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:17.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:17 smithi177 bash[68594]: cluster 2024-09-13T14:54:16.637250+0000 mgr.y (mgr.26252) 840 : cluster [DBG] pgmap v1351: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:17 smithi177 bash[68594]: cluster 2024-09-13T14:54:16.637250+0000 mgr.y (mgr.26252) 840 : cluster [DBG] pgmap v1351: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:17 smithi177 bash[68594]: audit 2024-09-13T14:54:16.736361+0000 mon.a (mon.0) 1893 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:17 smithi177 bash[68594]: audit 2024-09-13T14:54:16.736361+0000 mon.a (mon.0) 1893 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:17 smithi177 bash[68594]: cluster 2024-09-13T14:54:16.740310+0000 mon.a (mon.0) 1894 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2024-09-13T14:54:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:17 smithi177 bash[68594]: cluster 2024-09-13T14:54:16.740310+0000 mon.a (mon.0) 1894 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2024-09-13T14:54:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:17 smithi177 bash[68594]: audit 2024-09-13T14:54:16.806441+0000 mon.a (mon.0) 1895 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:17 smithi177 bash[68594]: audit 2024-09-13T14:54:16.806441+0000 mon.a (mon.0) 1895 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:17 smithi177 bash[68594]: audit 2024-09-13T14:54:16.818930+0000 mon.a (mon.0) 1896 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:17 smithi177 bash[68594]: audit 2024-09-13T14:54:16.818930+0000 mon.a (mon.0) 1896 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:17.906 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:54:17.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:17.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:17.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[84576]: cluster 2024-09-13T14:54:16.637250+0000 mgr.y (mgr.26252) 840 : cluster [DBG] pgmap v1351: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[84576]: cluster 2024-09-13T14:54:16.637250+0000 mgr.y (mgr.26252) 840 : cluster [DBG] pgmap v1351: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[84576]: audit 2024-09-13T14:54:16.736361+0000 mon.a (mon.0) 1893 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:18.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[84576]: audit 2024-09-13T14:54:16.736361+0000 mon.a (mon.0) 1893 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[84576]: cluster 2024-09-13T14:54:16.740310+0000 mon.a (mon.0) 1894 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2024-09-13T14:54:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[84576]: cluster 2024-09-13T14:54:16.740310+0000 mon.a (mon.0) 1894 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2024-09-13T14:54:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[84576]: audit 2024-09-13T14:54:16.806441+0000 mon.a (mon.0) 1895 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[84576]: audit 2024-09-13T14:54:16.806441+0000 mon.a (mon.0) 1895 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[84576]: audit 2024-09-13T14:54:16.818930+0000 mon.a (mon.0) 1896 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:18.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[84576]: audit 2024-09-13T14:54:16.818930+0000 mon.a (mon.0) 1896 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:18.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[87147]: cluster 2024-09-13T14:54:16.637250+0000 mgr.y (mgr.26252) 840 : cluster [DBG] pgmap v1351: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:18.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[87147]: cluster 2024-09-13T14:54:16.637250+0000 mgr.y (mgr.26252) 840 : cluster [DBG] pgmap v1351: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:54:18.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[87147]: audit 2024-09-13T14:54:16.736361+0000 mon.a (mon.0) 1893 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:18.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[87147]: audit 2024-09-13T14:54:16.736361+0000 mon.a (mon.0) 1893 : audit [INF] from='client.? 172.21.15.99:0/1502344210' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[87147]: cluster 2024-09-13T14:54:16.740310+0000 mon.a (mon.0) 1894 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2024-09-13T14:54:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[87147]: cluster 2024-09-13T14:54:16.740310+0000 mon.a (mon.0) 1894 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2024-09-13T14:54:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[87147]: audit 2024-09-13T14:54:16.806441+0000 mon.a (mon.0) 1895 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[87147]: audit 2024-09-13T14:54:16.806441+0000 mon.a (mon.0) 1895 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[87147]: audit 2024-09-13T14:54:16.818930+0000 mon.a (mon.0) 1896 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:18.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:17 smithi099 bash[87147]: audit 2024-09-13T14:54:16.818930+0000 mon.a (mon.0) 1896 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:18.093 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:18.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:18.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:18.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:18.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:18.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:18.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:54:18.356 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:18.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:18.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:18.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:18.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:18.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:18.596 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 52 16 73788 73772 88.6596 68.5 0.011869 0.0112741 2024-09-13T14:54:18.637 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:18.657 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:18.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:18.761 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:18.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:18 smithi099 bash[84576]: audit 2024-09-13T14:54:18.281666+0000 mon.a (mon.0) 1897 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:18.774 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:18 smithi099 bash[84576]: audit 2024-09-13T14:54:18.281666+0000 mon.a (mon.0) 1897 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:18.786 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:18 smithi099 bash[87147]: audit 2024-09-13T14:54:18.281666+0000 mon.a (mon.0) 1897 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:18.786 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:18 smithi099 bash[87147]: audit 2024-09-13T14:54:18.281666+0000 mon.a (mon.0) 1897 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:18.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:18.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:18.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:18.930 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:19.033 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:19.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:18 smithi177 bash[68594]: audit 2024-09-13T14:54:18.281666+0000 mon.a (mon.0) 1897 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:19.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:18 smithi177 bash[68594]: audit 2024-09-13T14:54:18.281666+0000 mon.a (mon.0) 1897 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:19.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:19.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:19.211 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:19.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:19.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:19.346 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:19.454 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:19.484 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:19.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:19.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:19.596 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 53 16 74883 74867 88.2779 68.4375 0.0203118 0.0113235 2024-09-13T14:54:19.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:19.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:19.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:19.759 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:19.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:19.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:19 smithi177 bash[68594]: cluster 2024-09-13T14:54:18.637770+0000 mgr.y (mgr.26252) 841 : cluster [DBG] pgmap v1353: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:19.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:19 smithi177 bash[68594]: cluster 2024-09-13T14:54:18.637770+0000 mgr.y (mgr.26252) 841 : cluster [DBG] pgmap v1353: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:19.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:19 smithi177 bash[68594]: audit 2024-09-13T14:54:19.692416+0000 mon.a (mon.0) 1898 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:19.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:19 smithi177 bash[68594]: audit 2024-09-13T14:54:19.692416+0000 mon.a (mon.0) 1898 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:19.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:19 smithi177 bash[68594]: audit 2024-09-13T14:54:19.700026+0000 mon.a (mon.0) 1899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:19.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:19 smithi177 bash[68594]: audit 2024-09-13T14:54:19.700026+0000 mon.a (mon.0) 1899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:19.865 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:19.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:19.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:19.996 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:20.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:20.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:20.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[84576]: cluster 2024-09-13T14:54:18.637770+0000 mgr.y (mgr.26252) 841 : cluster [DBG] pgmap v1353: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:20.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[84576]: cluster 2024-09-13T14:54:18.637770+0000 mgr.y (mgr.26252) 841 : cluster [DBG] pgmap v1353: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:20.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[84576]: audit 2024-09-13T14:54:19.692416+0000 mon.a (mon.0) 1898 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[84576]: audit 2024-09-13T14:54:19.692416+0000 mon.a (mon.0) 1898 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[84576]: audit 2024-09-13T14:54:19.700026+0000 mon.a (mon.0) 1899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[84576]: audit 2024-09-13T14:54:19.700026+0000 mon.a (mon.0) 1899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[87147]: cluster 2024-09-13T14:54:18.637770+0000 mgr.y (mgr.26252) 841 : cluster [DBG] pgmap v1353: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:20.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[87147]: cluster 2024-09-13T14:54:18.637770+0000 mgr.y (mgr.26252) 841 : cluster [DBG] pgmap v1353: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:20.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[87147]: audit 2024-09-13T14:54:19.692416+0000 mon.a (mon.0) 1898 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[87147]: audit 2024-09-13T14:54:19.692416+0000 mon.a (mon.0) 1898 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[87147]: audit 2024-09-13T14:54:19.700026+0000 mon.a (mon.0) 1899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:19 smithi099 bash[87147]: audit 2024-09-13T14:54:19.700026+0000 mon.a (mon.0) 1899 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:20.153 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:20.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:20.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:20.289 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:20.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:54:20 smithi177 bash[61439]: ts=2024-09-13T14:54:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:20.392 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:20.423 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:20.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:20.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:20.565 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:20.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:20.596 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 54 16 75992 75976 87.9266 69.3125 0.013798 0.0113678 2024-09-13T14:54:20.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:20.699 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:20.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:20.814 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:20.814 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[84576]: audit 2024-09-13T14:54:20.573616+0000 mon.a (mon.0) 1900 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[84576]: audit 2024-09-13T14:54:20.573616+0000 mon.a (mon.0) 1900 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[84576]: audit 2024-09-13T14:54:20.577707+0000 mon.a (mon.0) 1901 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[84576]: audit 2024-09-13T14:54:20.577707+0000 mon.a (mon.0) 1901 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:20.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[84576]: audit 2024-09-13T14:54:20.586143+0000 mon.a (mon.0) 1902 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[84576]: audit 2024-09-13T14:54:20.586143+0000 mon.a (mon.0) 1902 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[87147]: audit 2024-09-13T14:54:20.573616+0000 mon.a (mon.0) 1900 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[87147]: audit 2024-09-13T14:54:20.573616+0000 mon.a (mon.0) 1900 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[87147]: audit 2024-09-13T14:54:20.577707+0000 mon.a (mon.0) 1901 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[87147]: audit 2024-09-13T14:54:20.577707+0000 mon.a (mon.0) 1901 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[87147]: audit 2024-09-13T14:54:20.586143+0000 mon.a (mon.0) 1902 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:20 smithi099 bash[87147]: audit 2024-09-13T14:54:20.586143+0000 mon.a (mon.0) 1902 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:20.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:20.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:20.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:20.959 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:20.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:21.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:21.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:20 smithi177 bash[68594]: audit 2024-09-13T14:54:20.573616+0000 mon.a (mon.0) 1900 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:21.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:20 smithi177 bash[68594]: audit 2024-09-13T14:54:20.573616+0000 mon.a (mon.0) 1900 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:21.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:20 smithi177 bash[68594]: audit 2024-09-13T14:54:20.577707+0000 mon.a (mon.0) 1901 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:21.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:20 smithi177 bash[68594]: audit 2024-09-13T14:54:20.577707+0000 mon.a (mon.0) 1901 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:21.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:20 smithi177 bash[68594]: audit 2024-09-13T14:54:20.586143+0000 mon.a (mon.0) 1902 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:21.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:20 smithi177 bash[68594]: audit 2024-09-13T14:54:20.586143+0000 mon.a (mon.0) 1902 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:54:21.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:21.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:21.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:21.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:21.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:21.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:21.342 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:21.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:21.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:21.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:21.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:21.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:21.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 55 16 77110 77094 87.5971 69.875 0.0141612 0.0114115 2024-09-13T14:54:21.607 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:21.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:21.710 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:21.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:21.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:21.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:21.852 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:21 smithi099 bash[84576]: cluster 2024-09-13T14:54:20.638386+0000 mgr.y (mgr.26252) 842 : cluster [DBG] pgmap v1354: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T14:54:21.852 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:21 smithi099 bash[84576]: cluster 2024-09-13T14:54:20.638386+0000 mgr.y (mgr.26252) 842 : cluster [DBG] pgmap v1354: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T14:54:21.852 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:21 smithi099 bash[87147]: cluster 2024-09-13T14:54:20.638386+0000 mgr.y (mgr.26252) 842 : cluster [DBG] pgmap v1354: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T14:54:21.852 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:21 smithi099 bash[87147]: cluster 2024-09-13T14:54:20.638386+0000 mgr.y (mgr.26252) 842 : cluster [DBG] pgmap v1354: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T14:54:21.868 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:21.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:21.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:21.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:22.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:21 smithi177 bash[68594]: cluster 2024-09-13T14:54:20.638386+0000 mgr.y (mgr.26252) 842 : cluster [DBG] pgmap v1354: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T14:54:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:21 smithi177 bash[68594]: cluster 2024-09-13T14:54:20.638386+0000 mgr.y (mgr.26252) 842 : cluster [DBG] pgmap v1354: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T14:54:22.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:22.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:22.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:22.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:22.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:22.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:22.388 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:22.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:22.494 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:22.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:22.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:22.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 56 16 78243 78227 87.2973 70.8125 0.00902178 0.0114504 2024-09-13T14:54:22.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:22.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:22.751 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:22.759 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:22.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:22.843 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:22.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:22.937 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:22.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:23.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:23.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:23.174 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:23.200 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:23.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:23.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:23.321 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:23.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:23.423 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:23.463 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:23.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:23.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:23.590 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:23.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:23.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 57 16 79259 79243 86.8797 63.5 0.00728531 0.0115038 2024-09-13T14:54:23.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:23.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:23.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:23.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:23.853 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:23.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:23.956 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:23.984 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:23.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:24.089 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:24.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:23 smithi177 bash[68594]: cluster 2024-09-13T14:54:22.638836+0000 mgr.y (mgr.26252) 843 : cluster [DBG] pgmap v1355: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-09-13T14:54:24.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:23 smithi177 bash[68594]: cluster 2024-09-13T14:54:22.638836+0000 mgr.y (mgr.26252) 843 : cluster [DBG] pgmap v1355: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-09-13T14:54:24.112 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:24.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:24.215 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:24.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:24.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:24.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:23 smithi099 bash[84576]: cluster 2024-09-13T14:54:22.638836+0000 mgr.y (mgr.26252) 843 : cluster [DBG] pgmap v1355: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-09-13T14:54:24.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:23 smithi099 bash[84576]: cluster 2024-09-13T14:54:22.638836+0000 mgr.y (mgr.26252) 843 : cluster [DBG] pgmap v1355: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-09-13T14:54:24.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:23 smithi099 bash[87147]: cluster 2024-09-13T14:54:22.638836+0000 mgr.y (mgr.26252) 843 : cluster [DBG] pgmap v1355: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-09-13T14:54:24.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:23 smithi099 bash[87147]: cluster 2024-09-13T14:54:22.638836+0000 mgr.y (mgr.26252) 843 : cluster [DBG] pgmap v1355: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-09-13T14:54:24.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:24.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:24.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:24.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:24.520 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:24.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:24.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 58 16 80346 80330 86.553 67.9375 0.0206306 0.0115482 2024-09-13T14:54:24.626 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:24.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:24.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:24.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:24.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:24.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:24.887 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:24.923 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:24.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:25.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:25.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:25.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:25.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:25.312 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:25.341 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:25.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:25.469 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:25.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:25.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:25.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 59 16 81433 81417 86.2374 67.9375 0.0135337 0.0115913 2024-09-13T14:54:25.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:25.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:25.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:25.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:25.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:25.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:25.855 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:25 smithi177 bash[68594]: cluster 2024-09-13T14:54:24.639410+0000 mgr.y (mgr.26252) 844 : cluster [DBG] pgmap v1356: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:25.856 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:25 smithi177 bash[68594]: cluster 2024-09-13T14:54:24.639410+0000 mgr.y (mgr.26252) 844 : cluster [DBG] pgmap v1356: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:25.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:25.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:26.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:26.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:25 smithi099 bash[84576]: cluster 2024-09-13T14:54:24.639410+0000 mgr.y (mgr.26252) 844 : cluster [DBG] pgmap v1356: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:26.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:25 smithi099 bash[84576]: cluster 2024-09-13T14:54:24.639410+0000 mgr.y (mgr.26252) 844 : cluster [DBG] pgmap v1356: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:26.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:25 smithi099 bash[87147]: cluster 2024-09-13T14:54:24.639410+0000 mgr.y (mgr.26252) 844 : cluster [DBG] pgmap v1356: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:26.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:25 smithi099 bash[87147]: cluster 2024-09-13T14:54:24.639410+0000 mgr.y (mgr.26252) 844 : cluster [DBG] pgmap v1356: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:26.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:26.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:26.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:26.268 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:26.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:26.371 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:26.394 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:26.497 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:26.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:26.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:26.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T14:54:26.601172+0000 min lat: 0.00109161 max lat: 0.809605 avg lat: 0.011698 2024-09-13T14:54:26.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:54:26.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 60 16 82060 82044 85.4532 39.1875 0.0199252 0.011698 2024-09-13T14:54:26.626 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:26.664 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:26.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:26.767 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:26.796 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:26.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:26.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:26.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:26.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:27.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:27.041 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:27.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:27.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:27.172 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:27.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:27.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:27.297 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:27.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:27.315 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:54:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:54:27] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.51.0" 2024-09-13T14:54:27.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:27.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:27.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:27.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:27.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:27.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:27 smithi099 bash[87147]: cluster 2024-09-13T14:54:26.639802+0000 mgr.y (mgr.26252) 845 : cluster [DBG] pgmap v1357: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:27.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:27 smithi099 bash[87147]: cluster 2024-09-13T14:54:26.639802+0000 mgr.y (mgr.26252) 845 : cluster [DBG] pgmap v1357: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:27.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:27 smithi099 bash[84576]: cluster 2024-09-13T14:54:26.639802+0000 mgr.y (mgr.26252) 845 : cluster [DBG] pgmap v1357: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:27.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:27 smithi099 bash[84576]: cluster 2024-09-13T14:54:26.639802+0000 mgr.y (mgr.26252) 845 : cluster [DBG] pgmap v1357: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:27.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 61 16 82836 82820 84.8473 48.5 0.0120515 0.0117808 2024-09-13T14:54:27.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:27.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:27.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:27.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:27.793 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:27 smithi177 bash[68594]: cluster 2024-09-13T14:54:26.639802+0000 mgr.y (mgr.26252) 845 : cluster [DBG] pgmap v1357: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:27.794 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:27 smithi177 bash[68594]: cluster 2024-09-13T14:54:26.639802+0000 mgr.y (mgr.26252) 845 : cluster [DBG] pgmap v1357: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:54:27.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:27.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:27.906 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:27.935 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:27.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:28.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:28.168 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.211 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:28.316 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:28.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:28.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:28.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:28.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.597 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 62 16 83731 83715 84.381 55.9375 0.018531 0.0118464 2024-09-13T14:54:28.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:28.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:28.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:28.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:28.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:28.991 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:29.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:29.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:29.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:29.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:29.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:29.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:29.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:29.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:29.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:29.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:29.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:29.598 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 63 16 84743 84727 84.0455 63.25 0.00855518 0.0118939 2024-09-13T14:54:29.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:29.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:29.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:29.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:29.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:29.906 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:29.932 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:29.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:30.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:30.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:29 smithi177 bash[68594]: cluster 2024-09-13T14:54:28.640280+0000 mgr.y (mgr.26252) 846 : cluster [DBG] pgmap v1358: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-13T14:54:30.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:29 smithi177 bash[68594]: cluster 2024-09-13T14:54:28.640280+0000 mgr.y (mgr.26252) 846 : cluster [DBG] pgmap v1358: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-13T14:54:30.065 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:30.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:30.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:30.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:29 smithi099 bash[84576]: cluster 2024-09-13T14:54:28.640280+0000 mgr.y (mgr.26252) 846 : cluster [DBG] pgmap v1358: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-13T14:54:30.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:29 smithi099 bash[84576]: cluster 2024-09-13T14:54:28.640280+0000 mgr.y (mgr.26252) 846 : cluster [DBG] pgmap v1358: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-13T14:54:30.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:29 smithi099 bash[87147]: cluster 2024-09-13T14:54:28.640280+0000 mgr.y (mgr.26252) 846 : cluster [DBG] pgmap v1358: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-13T14:54:30.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:29 smithi099 bash[87147]: cluster 2024-09-13T14:54:28.640280+0000 mgr.y (mgr.26252) 846 : cluster [DBG] pgmap v1358: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-13T14:54:30.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:30.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:30.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:30.304 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:54:30 smithi177 bash[61439]: ts=2024-09-13T14:54:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:30.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:30.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:30.435 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:30.469 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:30.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:30.583 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T14:54:30.583 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:85551 42.3 42:c52e06ac:::benchmark_data_smithi099_148651_object85550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1019)", 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:54:30.581180+0000", 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0038948770000000001, 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039292750000000003, 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:54:30.584 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 85551 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581180+0000", 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581180+0000", 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.585 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581179+0000", 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581210+0000", 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1006000000000001e-05 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581225+0000", 2024-09-13T14:54:30.586 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5350999999999999e-05 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581232+0000", 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8569999999999997e-06 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581440+0000", 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020782799999999999 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:30.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581661+0000", 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000221478 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581804+0000", 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014298800000000001 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:85552 42.3 42:cb542dc4:::benchmark_data_smithi099_148651_object85551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1019)", 2024-09-13T14:54:30.588 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:54:30.581903+0000", 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.003172051, 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032980940000000001, 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 85552 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:54:30.589 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581903+0000", 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581903+0000", 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581902+0000", 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.590 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581949+0000", 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6687000000000002e-05 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581958+0000", 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5760000000000006e-06 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:54:30.591 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.581962+0000", 2024-09-13T14:54:30.592 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0230000000000003e-06 2024-09-13T14:54:30.592 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.592 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:54:30.592 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.584839+0000", 2024-09-13T14:54:30.592 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028767699999999998 2024-09-13T14:54:30.592 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:54:30.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:54:30.593 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:54:30.593 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:30.585004+0000", 2024-09-13T14:54:30.593 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000165673 2024-09-13T14:54:30.593 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:30.593 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:54:30.593 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:30.593 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:54:30.593 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2024-09-13T14:54:30.593 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 2 2024-09-13T14:54:30.593 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T14:54:30.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:30.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:30.598 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 64 16 85567 85551 83.5357 51.5 0.00752505 0.0119649 2024-09-13T14:54:30.696 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:30.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:30.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:30.823 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:30.866 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:30.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:30.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:31.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:31.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:31.114 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:31.134 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:31.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:31.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:31.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:31.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:31.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:31.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:31.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:31.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:31.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:31.599 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 65 16 86509 86493 83.1563 58.875 0.01503 0.0120209 2024-09-13T14:54:31.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:31.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:31.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:31.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:31 smithi177 bash[68594]: cluster 2024-09-13T14:54:30.640933+0000 mgr.y (mgr.26252) 847 : cluster [DBG] pgmap v1359: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:31.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:31 smithi177 bash[68594]: cluster 2024-09-13T14:54:30.640933+0000 mgr.y (mgr.26252) 847 : cluster [DBG] pgmap v1359: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:31.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:54:32.033 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:32.060 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:32.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:32.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:31 smithi099 bash[84576]: cluster 2024-09-13T14:54:30.640933+0000 mgr.y (mgr.26252) 847 : cluster [DBG] pgmap v1359: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:32.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:31 smithi099 bash[84576]: cluster 2024-09-13T14:54:30.640933+0000 mgr.y (mgr.26252) 847 : cluster [DBG] pgmap v1359: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:32.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:31 smithi099 bash[87147]: cluster 2024-09-13T14:54:30.640933+0000 mgr.y (mgr.26252) 847 : cluster [DBG] pgmap v1359: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:32.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:31 smithi099 bash[87147]: cluster 2024-09-13T14:54:30.640933+0000 mgr.y (mgr.26252) 847 : cluster [DBG] pgmap v1359: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:54:32.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:32.184 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:32.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:32.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:32.320 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:32.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:32.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:32.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:32.560 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:32.599 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 66 16 87514 87498 82.848 62.8125 0.0188127 0.0120651 2024-09-13T14:54:33.076 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:33.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:33.180 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:33.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:33.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:33.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:33.340 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:33.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:33.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:33.477 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:33.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:33.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:33.599 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 67 16 88446 88430 82.4808 58.25 0.0112612 0.0121187 2024-09-13T14:54:33.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:33.701 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:33 smithi099 bash[87147]: cluster 2024-09-13T14:54:32.641415+0000 mgr.y (mgr.26252) 848 : cluster [DBG] pgmap v1360: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-13T14:54:33.701 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:33 smithi099 bash[87147]: cluster 2024-09-13T14:54:32.641415+0000 mgr.y (mgr.26252) 848 : cluster [DBG] pgmap v1360: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-13T14:54:33.701 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:33 smithi099 bash[87147]: audit 2024-09-13T14:54:33.282036+0000 mon.a (mon.0) 1903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:33.701 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:33 smithi099 bash[87147]: audit 2024-09-13T14:54:33.282036+0000 mon.a (mon.0) 1903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:33.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:33 smithi099 bash[84576]: cluster 2024-09-13T14:54:32.641415+0000 mgr.y (mgr.26252) 848 : cluster [DBG] pgmap v1360: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-13T14:54:33.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:33 smithi099 bash[84576]: cluster 2024-09-13T14:54:32.641415+0000 mgr.y (mgr.26252) 848 : cluster [DBG] pgmap v1360: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-13T14:54:33.701 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:33 smithi099 bash[84576]: audit 2024-09-13T14:54:33.282036+0000 mon.a (mon.0) 1903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:33.702 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:33 smithi099 bash[84576]: audit 2024-09-13T14:54:33.282036+0000 mon.a (mon.0) 1903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:33.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:33.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:33.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:33.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:33 smithi177 bash[68594]: cluster 2024-09-13T14:54:32.641415+0000 mgr.y (mgr.26252) 848 : cluster [DBG] pgmap v1360: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-13T14:54:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:33 smithi177 bash[68594]: cluster 2024-09-13T14:54:32.641415+0000 mgr.y (mgr.26252) 848 : cluster [DBG] pgmap v1360: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-13T14:54:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:33 smithi177 bash[68594]: audit 2024-09-13T14:54:33.282036+0000 mon.a (mon.0) 1903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:33.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:33 smithi177 bash[68594]: audit 2024-09-13T14:54:33.282036+0000 mon.a (mon.0) 1903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:33.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.888 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:33.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:33.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:34.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:34.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:34.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:34.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:34.216 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:54:34.216 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:34.239 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:54:34.240 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:34.271 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:34.272 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:54:34.298 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:34.299 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:54:34.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:34.334 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:54:34.334 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:34.370 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:54:34.370 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:34.407 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:54:34.407 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:34.432 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:54:34.433 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:34.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:34.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:34.599 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 68 16 89459 89443 82.1988 63.3125 0.0395064 0.0121601 2024-09-13T14:54:34.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:34.684 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:34.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:34.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:34.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:34 smithi099 bash[84576]: audit 2024-09-13T14:54:34.196425+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.99:0/3516096875' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:34.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:34 smithi099 bash[84576]: audit 2024-09-13T14:54:34.196425+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.99:0/3516096875' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:34.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:34 smithi099 bash[87147]: audit 2024-09-13T14:54:34.196425+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.99:0/3516096875' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:34.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:34 smithi099 bash[87147]: audit 2024-09-13T14:54:34.196425+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.99:0/3516096875' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:34.842 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:34.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:34.947 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:35.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:34 smithi177 bash[68594]: audit 2024-09-13T14:54:34.196425+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.99:0/3516096875' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:35.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:34 smithi177 bash[68594]: audit 2024-09-13T14:54:34.196425+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.99:0/3516096875' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:35.210 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:35.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:35.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:35.341 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:35.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:35.445 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:35.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:35.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:35.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:35.599 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 69 16 90416 90400 81.8743 59.8125 0.00443229 0.0122077 2024-09-13T14:54:35.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:35.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:35.708 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:54:35.725 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:35.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.745 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:35.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:35.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:35.854 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:35 smithi099 bash[84576]: cluster 2024-09-13T14:54:34.641976+0000 mgr.y (mgr.26252) 849 : cluster [DBG] pgmap v1361: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:35 smithi177 bash[68594]: cluster 2024-09-13T14:54:34.641976+0000 mgr.y (mgr.26252) 849 : cluster [DBG] pgmap v1361: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:35 smithi177 bash[68594]: cluster 2024-09-13T14:54:34.641976+0000 mgr.y (mgr.26252) 849 : cluster [DBG] pgmap v1361: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:35 smithi177 bash[68594]: audit 2024-09-13T14:54:35.157928+0000 mon.c (mon.2) 66 : audit [INF] from='client.? 172.21.15.99:0/843220486' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:35 smithi177 bash[68594]: audit 2024-09-13T14:54:35.157928+0000 mon.c (mon.2) 66 : audit [INF] from='client.? 172.21.15.99:0/843220486' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:35 smithi177 bash[68594]: audit 2024-09-13T14:54:35.159851+0000 mon.a (mon.0) 1904 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:35.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:35 smithi177 bash[68594]: audit 2024-09-13T14:54:35.159851+0000 mon.a (mon.0) 1904 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:36.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:36.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:36.196 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:36.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: cluster 2024-09-13T14:54:34.641976+0000 mgr.y (mgr.26252) 849 : cluster [DBG] pgmap v1361: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:36.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: cluster 2024-09-13T14:54:34.641976+0000 mgr.y (mgr.26252) 849 : cluster [DBG] pgmap v1361: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:36.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: audit 2024-09-13T14:54:35.157928+0000 mon.c (mon.2) 66 : audit [INF] from='client.? 172.21.15.99:0/843220486' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:36.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: audit 2024-09-13T14:54:35.157928+0000 mon.c (mon.2) 66 : audit [INF] from='client.? 172.21.15.99:0/843220486' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:36.422 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: audit 2024-09-13T14:54:35.159851+0000 mon.a (mon.0) 1904 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:36.422 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: audit 2024-09-13T14:54:35.159851+0000 mon.a (mon.0) 1904 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:36.454 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:36.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:36.561 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: cluster 2024-09-13T14:54:34.641976+0000 mgr.y (mgr.26252) 849 : cluster [DBG] pgmap v1361: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T14:54:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: audit 2024-09-13T14:54:35.157928+0000 mon.c (mon.2) 66 : audit [INF] from='client.? 172.21.15.99:0/843220486' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: audit 2024-09-13T14:54:35.157928+0000 mon.c (mon.2) 66 : audit [INF] from='client.? 172.21.15.99:0/843220486' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: audit 2024-09-13T14:54:35.159851+0000 mon.a (mon.0) 1904 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:36.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: audit 2024-09-13T14:54:35.159851+0000 mon.a (mon.0) 1904 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:36.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:36.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:36.599 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 70 16 91260 91244 81.4582 52.75 0.0163236 0.0122711 2024-09-13T14:54:36.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:36.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:36.730 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T14:54:36.818 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:36.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:36.849 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:36.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:36.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:36.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:36.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:36 smithi177 bash[68594]: audit 2024-09-13T14:54:35.708115+0000 mon.a (mon.0) 1905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:36 smithi177 bash[68594]: audit 2024-09-13T14:54:35.708115+0000 mon.a (mon.0) 1905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:36 smithi177 bash[68594]: cluster 2024-09-13T14:54:35.711318+0000 mon.a (mon.0) 1906 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2024-09-13T14:54:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:36 smithi177 bash[68594]: cluster 2024-09-13T14:54:35.711318+0000 mon.a (mon.0) 1906 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2024-09-13T14:54:36.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:36 smithi177 bash[68594]: audit 2024-09-13T14:54:35.874330+0000 mon.c (mon.2) 67 : audit [INF] from='client.? 172.21.15.99:0/2746127760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:36.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:36 smithi177 bash[68594]: audit 2024-09-13T14:54:35.874330+0000 mon.c (mon.2) 67 : audit [INF] from='client.? 172.21.15.99:0/2746127760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:36.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:36 smithi177 bash[68594]: audit 2024-09-13T14:54:35.875238+0000 mon.a (mon.0) 1907 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:36.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:36 smithi177 bash[68594]: audit 2024-09-13T14:54:35.875238+0000 mon.a (mon.0) 1907 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:37.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: audit 2024-09-13T14:54:35.708115+0000 mon.a (mon.0) 1905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:37.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: audit 2024-09-13T14:54:35.708115+0000 mon.a (mon.0) 1905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:37.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: cluster 2024-09-13T14:54:35.711318+0000 mon.a (mon.0) 1906 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2024-09-13T14:54:37.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: cluster 2024-09-13T14:54:35.711318+0000 mon.a (mon.0) 1906 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2024-09-13T14:54:37.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: audit 2024-09-13T14:54:35.874330+0000 mon.c (mon.2) 67 : audit [INF] from='client.? 172.21.15.99:0/2746127760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: audit 2024-09-13T14:54:35.874330+0000 mon.c (mon.2) 67 : audit [INF] from='client.? 172.21.15.99:0/2746127760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: audit 2024-09-13T14:54:35.875238+0000 mon.a (mon.0) 1907 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[84576]: audit 2024-09-13T14:54:35.875238+0000 mon.a (mon.0) 1907 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: audit 2024-09-13T14:54:35.708115+0000 mon.a (mon.0) 1905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:37.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: audit 2024-09-13T14:54:35.708115+0000 mon.a (mon.0) 1905 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:37.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: cluster 2024-09-13T14:54:35.711318+0000 mon.a (mon.0) 1906 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2024-09-13T14:54:37.101 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: cluster 2024-09-13T14:54:35.711318+0000 mon.a (mon.0) 1906 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2024-09-13T14:54:37.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: audit 2024-09-13T14:54:35.874330+0000 mon.c (mon.2) 67 : audit [INF] from='client.? 172.21.15.99:0/2746127760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: audit 2024-09-13T14:54:35.874330+0000 mon.c (mon.2) 67 : audit [INF] from='client.? 172.21.15.99:0/2746127760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: audit 2024-09-13T14:54:35.875238+0000 mon.a (mon.0) 1907 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:36 smithi099 bash[87147]: audit 2024-09-13T14:54:35.875238+0000 mon.a (mon.0) 1907 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.113 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:37.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:37.216 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:37.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:37.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:37.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:37.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:37.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:37.483 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:37.491 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:54:37] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.51.0" 2024-09-13T14:54:37.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.504 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:37.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:37.599 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 71 16 92178 92162 81.119 57.375 0.0135685 0.0123226 2024-09-13T14:54:37.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:37.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:37.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:37.732 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:37.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:37.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:37.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[84576]: cluster 2024-09-13T14:54:36.642330+0000 mgr.y (mgr.26252) 850 : cluster [DBG] pgmap v1363: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s 2024-09-13T14:54:37.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[84576]: cluster 2024-09-13T14:54:36.642330+0000 mgr.y (mgr.26252) 850 : cluster [DBG] pgmap v1363: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s 2024-09-13T14:54:37.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[84576]: cluster 2024-09-13T14:54:36.714810+0000 mon.a (mon.0) 1908 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:37.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[84576]: cluster 2024-09-13T14:54:36.714810+0000 mon.a (mon.0) 1908 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:37.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[84576]: audit 2024-09-13T14:54:36.723974+0000 mon.a (mon.0) 1909 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:54:37.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[84576]: audit 2024-09-13T14:54:36.723974+0000 mon.a (mon.0) 1909 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:54:37.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[84576]: cluster 2024-09-13T14:54:36.727706+0000 mon.a (mon.0) 1910 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2024-09-13T14:54:37.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[84576]: cluster 2024-09-13T14:54:36.727706+0000 mon.a (mon.0) 1910 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2024-09-13T14:54:37.865 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:37.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:37.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[87147]: cluster 2024-09-13T14:54:36.642330+0000 mgr.y (mgr.26252) 850 : cluster [DBG] pgmap v1363: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s 2024-09-13T14:54:37.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[87147]: cluster 2024-09-13T14:54:36.642330+0000 mgr.y (mgr.26252) 850 : cluster [DBG] pgmap v1363: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s 2024-09-13T14:54:37.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[87147]: cluster 2024-09-13T14:54:36.714810+0000 mon.a (mon.0) 1908 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:37.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[87147]: cluster 2024-09-13T14:54:36.714810+0000 mon.a (mon.0) 1908 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:37.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[87147]: audit 2024-09-13T14:54:36.723974+0000 mon.a (mon.0) 1909 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:54:37.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[87147]: audit 2024-09-13T14:54:36.723974+0000 mon.a (mon.0) 1909 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:54:37.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[87147]: cluster 2024-09-13T14:54:36.727706+0000 mon.a (mon.0) 1910 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2024-09-13T14:54:37.977 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:37 smithi099 bash[87147]: cluster 2024-09-13T14:54:36.727706+0000 mon.a (mon.0) 1910 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2024-09-13T14:54:37.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:38.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:37 smithi177 bash[68594]: cluster 2024-09-13T14:54:36.642330+0000 mgr.y (mgr.26252) 850 : cluster [DBG] pgmap v1363: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s 2024-09-13T14:54:38.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:37 smithi177 bash[68594]: cluster 2024-09-13T14:54:36.642330+0000 mgr.y (mgr.26252) 850 : cluster [DBG] pgmap v1363: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s 2024-09-13T14:54:38.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:37 smithi177 bash[68594]: cluster 2024-09-13T14:54:36.714810+0000 mon.a (mon.0) 1908 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:38.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:37 smithi177 bash[68594]: cluster 2024-09-13T14:54:36.714810+0000 mon.a (mon.0) 1908 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:38.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:37 smithi177 bash[68594]: audit 2024-09-13T14:54:36.723974+0000 mon.a (mon.0) 1909 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:54:38.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:37 smithi177 bash[68594]: audit 2024-09-13T14:54:36.723974+0000 mon.a (mon.0) 1909 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:54:38.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:37 smithi177 bash[68594]: cluster 2024-09-13T14:54:36.727706+0000 mon.a (mon.0) 1910 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2024-09-13T14:54:38.090 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:37 smithi177 bash[68594]: cluster 2024-09-13T14:54:36.727706+0000 mon.a (mon.0) 1910 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2024-09-13T14:54:38.103 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:38.207 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:38.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:38.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:38.336 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:38.361 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:38.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:38.465 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:38.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:38.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:38.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.599 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 72 16 93178 93162 80.8603 62.5 0.00708685 0.0123603 2024-09-13T14:54:38.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:38.720 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:38.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:38.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:38.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:39.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:39.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:39.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:39.148 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:39.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:39.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:39.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:39.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:39.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:39.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:39.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:39.554 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:39.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:39.599 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 73 16 94089 94073 80.5325 56.9375 0.0246895 0.012413 2024-09-13T14:54:39.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:39.690 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:54:39.692 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:39.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:39.799 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:39 smithi099 bash[84576]: cluster 2024-09-13T14:54:38.642798+0000 mgr.y (mgr.26252) 851 : cluster [DBG] pgmap v1365: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s 2024-09-13T14:54:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:39 smithi099 bash[84576]: cluster 2024-09-13T14:54:38.642798+0000 mgr.y (mgr.26252) 851 : cluster [DBG] pgmap v1365: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s 2024-09-13T14:54:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:39 smithi099 bash[87147]: cluster 2024-09-13T14:54:38.642798+0000 mgr.y (mgr.26252) 851 : cluster [DBG] pgmap v1365: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s 2024-09-13T14:54:39.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:39 smithi099 bash[87147]: cluster 2024-09-13T14:54:38.642798+0000 mgr.y (mgr.26252) 851 : cluster [DBG] pgmap v1365: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s 2024-09-13T14:54:39.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:39.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:39.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:39.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:39 smithi177 bash[68594]: cluster 2024-09-13T14:54:38.642798+0000 mgr.y (mgr.26252) 851 : cluster [DBG] pgmap v1365: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s 2024-09-13T14:54:39.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:39 smithi177 bash[68594]: cluster 2024-09-13T14:54:38.642798+0000 mgr.y (mgr.26252) 851 : cluster [DBG] pgmap v1365: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s 2024-09-13T14:54:39.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:39.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:40.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:40.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:40.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:40.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:40.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:40.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:40.330 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:40.355 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:40.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:40.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:40.482 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:40.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:40.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:40.602 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 74 16 94704 94688 79.9608 38.4375 0.0321602 0.0125015 2024-09-13T14:54:40.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:40.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:40.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:40.746 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:40.747 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:54:40 smithi177 bash[61439]: ts=2024-09-13T14:54:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:40.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:40.849 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:40.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:40.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:40.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:54:40.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:41.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:41.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:41.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:41.155 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:41.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:41.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:41.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:41.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:41.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:41.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:41.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:41.540 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:41.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:41.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:41.613 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 75 16 95664 95648 79.6946 60 0.0154778 0.0125427 2024-09-13T14:54:41.680 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:41.691 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:54:41.707 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:41.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:41.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:41.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:41 smithi177 bash[68594]: cluster 2024-09-13T14:54:40.643434+0000 mgr.y (mgr.26252) 852 : cluster [DBG] pgmap v1366: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:54:41.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:41 smithi177 bash[68594]: cluster 2024-09-13T14:54:40.643434+0000 mgr.y (mgr.26252) 852 : cluster [DBG] pgmap v1366: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:54:41.847 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:41.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:41.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:41.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:41.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:41 smithi099 bash[84576]: cluster 2024-09-13T14:54:40.643434+0000 mgr.y (mgr.26252) 852 : cluster [DBG] pgmap v1366: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:54:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:41 smithi099 bash[84576]: cluster 2024-09-13T14:54:40.643434+0000 mgr.y (mgr.26252) 852 : cluster [DBG] pgmap v1366: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:54:42.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:41 smithi099 bash[87147]: cluster 2024-09-13T14:54:40.643434+0000 mgr.y (mgr.26252) 852 : cluster [DBG] pgmap v1366: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:54:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:41 smithi099 bash[87147]: cluster 2024-09-13T14:54:40.643434+0000 mgr.y (mgr.26252) 852 : cluster [DBG] pgmap v1366: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T14:54:42.085 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:42.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:42.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:42.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:42.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:42.402 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:42.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:42.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:42.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:42.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:42.614 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 76 16 96598 96582 79.4024 58.375 0.00626625 0.012587 2024-09-13T14:54:42.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:42.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:42.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:42.793 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:42.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:42.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:42.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:42.937 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:42.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:43.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:43.066 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:43.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:43.169 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:43.201 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:43.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:43.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:43.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:43.436 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:43.467 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:43.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:43.573 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:43.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:43.617 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 77 16 97181 97165 78.8406 36.4375 0.0163937 0.0126785 2024-09-13T14:54:43.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:43.742 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:43.746 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:43 smithi177 bash[68594]: cluster 2024-09-13T14:54:42.643866+0000 mgr.y (mgr.26252) 853 : cluster [DBG] pgmap v1367: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 679 op/s 2024-09-13T14:54:43.746 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:43 smithi177 bash[68594]: cluster 2024-09-13T14:54:42.643866+0000 mgr.y (mgr.26252) 853 : cluster [DBG] pgmap v1367: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 679 op/s 2024-09-13T14:54:43.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:43.847 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:43.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.887 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:43.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:43.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:43.997 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:43 smithi099 bash[84576]: cluster 2024-09-13T14:54:42.643866+0000 mgr.y (mgr.26252) 853 : cluster [DBG] pgmap v1367: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 679 op/s 2024-09-13T14:54:43.997 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:43 smithi099 bash[84576]: cluster 2024-09-13T14:54:42.643866+0000 mgr.y (mgr.26252) 853 : cluster [DBG] pgmap v1367: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 679 op/s 2024-09-13T14:54:43.997 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:43 smithi099 bash[87147]: cluster 2024-09-13T14:54:42.643866+0000 mgr.y (mgr.26252) 853 : cluster [DBG] pgmap v1367: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 679 op/s 2024-09-13T14:54:43.997 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:43 smithi099 bash[87147]: cluster 2024-09-13T14:54:42.643866+0000 mgr.y (mgr.26252) 853 : cluster [DBG] pgmap v1367: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 679 op/s 2024-09-13T14:54:44.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:44.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:44.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:44.171 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:44.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:44.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:44.307 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:44.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:44.411 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:44.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:44.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:44.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:44.575 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:44.575 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:44.597 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:44.600 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:44.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:44.617 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 78 16 98100 98084 78.5662 57.4375 0.022039 0.0127221 2024-09-13T14:54:44.705 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:44.728 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:44.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:44.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:44.854 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:44.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:44.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:44.990 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:44.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:45.095 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:45.125 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:45.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:45.230 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:45.265 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:45.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:45.370 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:45.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:45.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:45.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:45.621 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:45.621 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:45.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:54:45.928 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 79 16 98903 98887 77.9507 50.1875 0.00458254 0.0127812 2024-09-13T14:54:46.034 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:45 smithi099 bash[84576]: cluster 2024-09-13T14:54:44.644402+0000 mgr.y (mgr.26252) 854 : cluster [DBG] pgmap v1368: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T14:54:46.034 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:45 smithi099 bash[84576]: cluster 2024-09-13T14:54:44.644402+0000 mgr.y (mgr.26252) 854 : cluster [DBG] pgmap v1368: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T14:54:46.034 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:45 smithi099 bash[87147]: cluster 2024-09-13T14:54:44.644402+0000 mgr.y (mgr.26252) 854 : cluster [DBG] pgmap v1368: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T14:54:46.034 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:45 smithi099 bash[87147]: cluster 2024-09-13T14:54:44.644402+0000 mgr.y (mgr.26252) 854 : cluster [DBG] pgmap v1368: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T14:54:46.049 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:46.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:46.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:45 smithi177 bash[68594]: cluster 2024-09-13T14:54:44.644402+0000 mgr.y (mgr.26252) 854 : cluster [DBG] pgmap v1368: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T14:54:46.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:45 smithi177 bash[68594]: cluster 2024-09-13T14:54:44.644402+0000 mgr.y (mgr.26252) 854 : cluster [DBG] pgmap v1368: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T14:54:46.154 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:46.182 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:46.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:46.284 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:46.319 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:46.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:46.426 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:46.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:46.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:46.555 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:46.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:46.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:46.689 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:46.715 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:46.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:46.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:46.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:46.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:46.988 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:47.091 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:47.101 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:54:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:54:47] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.51.0" 2024-09-13T14:54:47.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:47.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:47.121 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T14:54:46.931594+0000 min lat: 0.00109161 max lat: 0.809605 avg lat: 0.0128676 2024-09-13T14:54:47.121 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:54:47.121 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 80 16 99870 99854 77.6837 60.4375 0.0158323 0.0128676 2024-09-13T14:54:47.218 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:47.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:47.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:47.352 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:47.381 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:47.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:47.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:47.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:47.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:47.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:47.650 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:47.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:47.756 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:47.780 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:47.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:47.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:47.913 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:47.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:48.017 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:48.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:47 smithi099 bash[84576]: cluster 2024-09-13T14:54:46.644800+0000 mgr.y (mgr.26252) 855 : cluster [DBG] pgmap v1369: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2024-09-13T14:54:48.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:47 smithi099 bash[84576]: cluster 2024-09-13T14:54:46.644800+0000 mgr.y (mgr.26252) 855 : cluster [DBG] pgmap v1369: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2024-09-13T14:54:48.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:47 smithi099 bash[87147]: cluster 2024-09-13T14:54:46.644800+0000 mgr.y (mgr.26252) 855 : cluster [DBG] pgmap v1369: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2024-09-13T14:54:48.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:47 smithi099 bash[87147]: cluster 2024-09-13T14:54:46.644800+0000 mgr.y (mgr.26252) 855 : cluster [DBG] pgmap v1369: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2024-09-13T14:54:48.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:47 smithi177 bash[68594]: cluster 2024-09-13T14:54:46.644800+0000 mgr.y (mgr.26252) 855 : cluster [DBG] pgmap v1369: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2024-09-13T14:54:48.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:47 smithi177 bash[68594]: cluster 2024-09-13T14:54:46.644800+0000 mgr.y (mgr.26252) 855 : cluster [DBG] pgmap v1369: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2024-09-13T14:54:48.158 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 81 16 100762 100746 77.2303 55.75 0.0204809 0.0129434 2024-09-13T14:54:48.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:48.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:48.615 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:48.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:48.717 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:48.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:48.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:48.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:48.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:48.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:48.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:49.012 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:49.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:49.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:48 smithi099 bash[84576]: audit 2024-09-13T14:54:48.282171+0000 mon.a (mon.0) 1911 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:49.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:48 smithi099 bash[84576]: audit 2024-09-13T14:54:48.282171+0000 mon.a (mon.0) 1911 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:49.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:48 smithi099 bash[87147]: audit 2024-09-13T14:54:48.282171+0000 mon.a (mon.0) 1911 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:49.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:48 smithi099 bash[87147]: audit 2024-09-13T14:54:48.282171+0000 mon.a (mon.0) 1911 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:49.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:48 smithi177 bash[68594]: audit 2024-09-13T14:54:48.282171+0000 mon.a (mon.0) 1911 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:49.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:48 smithi177 bash[68594]: audit 2024-09-13T14:54:48.282171+0000 mon.a (mon.0) 1911 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:49.121 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:49.153 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:49.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:49.158 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 82 16 101714 101698 76.9812 59.5 0.0209689 0.0129836 2024-09-13T14:54:49.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:49.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:49.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:49.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:49.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:49.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:49.882 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:49.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:49.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:50.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:50.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:49 smithi099 bash[84576]: cluster 2024-09-13T14:54:48.645325+0000 mgr.y (mgr.26252) 856 : cluster [DBG] pgmap v1370: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-09-13T14:54:50.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:49 smithi099 bash[84576]: cluster 2024-09-13T14:54:48.645325+0000 mgr.y (mgr.26252) 856 : cluster [DBG] pgmap v1370: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-09-13T14:54:50.038 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:49 smithi099 bash[87147]: cluster 2024-09-13T14:54:48.645325+0000 mgr.y (mgr.26252) 856 : cluster [DBG] pgmap v1370: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-09-13T14:54:50.039 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:49 smithi099 bash[87147]: cluster 2024-09-13T14:54:48.645325+0000 mgr.y (mgr.26252) 856 : cluster [DBG] pgmap v1370: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-09-13T14:54:50.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:50.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:49 smithi177 bash[68594]: cluster 2024-09-13T14:54:48.645325+0000 mgr.y (mgr.26252) 856 : cluster [DBG] pgmap v1370: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-09-13T14:54:50.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:49 smithi177 bash[68594]: cluster 2024-09-13T14:54:48.645325+0000 mgr.y (mgr.26252) 856 : cluster [DBG] pgmap v1370: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-09-13T14:54:50.158 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 83 16 102502 102486 76.6493 49.25 0.0122287 0.0130415 2024-09-13T14:54:50.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:50.185 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:50.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:50.313 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:50.418 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:50.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:50.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:50.529 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:54:50.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:50.578 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:50.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:50.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:54:50 smithi177 bash[61439]: ts=2024-09-13T14:54:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:54:50.686 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:50.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:50.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:50.809 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:50.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:50.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:50.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:50.972 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:51.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:51.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:51.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:51.158 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 84 16 103414 103398 76.4169 57 0.0149391 0.0130816 2024-09-13T14:54:51.205 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:51.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:51.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:51.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:51.372 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:51.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:51.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:51.501 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:51.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:51.604 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:51.626 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:51.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:51.729 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:51.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:51.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:51.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:51.897 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:52.001 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:52.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:52.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:52.082 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:51 smithi177 bash[68594]: cluster 2024-09-13T14:54:50.645926+0000 mgr.y (mgr.26252) 857 : cluster [DBG] pgmap v1371: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-13T14:54:52.082 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:51 smithi177 bash[68594]: cluster 2024-09-13T14:54:50.645926+0000 mgr.y (mgr.26252) 857 : cluster [DBG] pgmap v1371: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-13T14:54:52.130 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:52.158 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 85 16 104156 104140 76.0658 46.375 0.0100298 0.013142 2024-09-13T14:54:52.375 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:52.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:52.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:52.501 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:52.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:52.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:52.631 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:52.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:52.737 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:52.765 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:52.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:52.869 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:52.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:53.014 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:53.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:53.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:53.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:53.159 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 86 16 105037 105021 75.823 55.0625 0.0134609 0.0131838 2024-09-13T14:54:53.161 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:52 smithi099 bash[87147]: cluster 2024-09-13T14:54:50.645926+0000 mgr.y (mgr.26252) 857 : cluster [DBG] pgmap v1371: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-13T14:54:53.161 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:52 smithi099 bash[87147]: cluster 2024-09-13T14:54:50.645926+0000 mgr.y (mgr.26252) 857 : cluster [DBG] pgmap v1371: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-13T14:54:53.163 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:52 smithi099 bash[84576]: cluster 2024-09-13T14:54:50.645926+0000 mgr.y (mgr.26252) 857 : cluster [DBG] pgmap v1371: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-13T14:54:53.163 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:52 smithi099 bash[84576]: cluster 2024-09-13T14:54:50.645926+0000 mgr.y (mgr.26252) 857 : cluster [DBG] pgmap v1371: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-13T14:54:53.186 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:53.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:53.314 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:53.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:53.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:53.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:53.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:53.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:53.819 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:53 smithi099 bash[84576]: cluster 2024-09-13T14:54:52.646509+0000 mgr.y (mgr.26252) 858 : cluster [DBG] pgmap v1372: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2024-09-13T14:54:53.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:53 smithi099 bash[84576]: cluster 2024-09-13T14:54:52.646509+0000 mgr.y (mgr.26252) 858 : cluster [DBG] pgmap v1372: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2024-09-13T14:54:53.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:53 smithi099 bash[87147]: cluster 2024-09-13T14:54:52.646509+0000 mgr.y (mgr.26252) 858 : cluster [DBG] pgmap v1372: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2024-09-13T14:54:53.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:53 smithi099 bash[87147]: cluster 2024-09-13T14:54:52.646509+0000 mgr.y (mgr.26252) 858 : cluster [DBG] pgmap v1372: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2024-09-13T14:54:53.846 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:53 smithi177 bash[68594]: cluster 2024-09-13T14:54:52.646509+0000 mgr.y (mgr.26252) 858 : cluster [DBG] pgmap v1372: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2024-09-13T14:54:53.846 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:53 smithi177 bash[68594]: cluster 2024-09-13T14:54:52.646509+0000 mgr.y (mgr.26252) 858 : cluster [DBG] pgmap v1372: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2024-09-13T14:54:53.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:53.950 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:54.159 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 87 16 105717 105701 75.4424 42.5 0.0201262 0.0132503 2024-09-13T14:54:54.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:54.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:54.271 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:54.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:54.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:54.405 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:54.424 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:54.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:54.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:54.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:54.659 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:54.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:54.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:54.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:54.969 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:55.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:55.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:55.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:55.159 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 88 16 106583 106567 75.2016 54.125 0.0253134 0.0132925 2024-09-13T14:54:55.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:55.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:55.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:55.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:55.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:55.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:55.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:55.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:55.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:55.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:55.635 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:55.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:55.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:55.770 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:55.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:55.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:55.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:55.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:55 smithi099 bash[87147]: cluster 2024-09-13T14:54:54.647118+0000 mgr.y (mgr.26252) 859 : cluster [DBG] pgmap v1373: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2024-09-13T14:54:55.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:55 smithi099 bash[87147]: cluster 2024-09-13T14:54:54.647118+0000 mgr.y (mgr.26252) 859 : cluster [DBG] pgmap v1373: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2024-09-13T14:54:55.937 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:55 smithi099 bash[84576]: cluster 2024-09-13T14:54:54.647118+0000 mgr.y (mgr.26252) 859 : cluster [DBG] pgmap v1373: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2024-09-13T14:54:55.938 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:55 smithi099 bash[84576]: cluster 2024-09-13T14:54:54.647118+0000 mgr.y (mgr.26252) 859 : cluster [DBG] pgmap v1373: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2024-09-13T14:54:56.009 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:56.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:55 smithi177 bash[68594]: cluster 2024-09-13T14:54:54.647118+0000 mgr.y (mgr.26252) 859 : cluster [DBG] pgmap v1373: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2024-09-13T14:54:56.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:55 smithi177 bash[68594]: cluster 2024-09-13T14:54:54.647118+0000 mgr.y (mgr.26252) 859 : cluster [DBG] pgmap v1373: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s 2024-09-13T14:54:56.038 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:56.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:56.147 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:56.159 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 89 16 107442 107426 74.9613 53.6875 0.00567697 0.0133285 2024-09-13T14:54:56.168 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:56.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:56.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:56.301 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:56.407 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:56.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:56.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:56.537 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:57.032 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:57.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:57.129 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:57.129 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:57.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:57.159 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total time run: 90.0118 2024-09-13T14:54:57.159 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total writes made: 107837 2024-09-13T14:54:57.159 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Write size: 65536 2024-09-13T14:54:57.159 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object size: 65536 2024-09-13T14:54:57.159 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Bandwidth (MB/sec): 74.877 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Bandwidth: 31.0135 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max bandwidth (MB/sec): 187.125 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min bandwidth (MB/sec): 29.9375 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average IOPS: 1198 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev IOPS: 496.216 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max IOPS: 2994 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min IOPS: 479 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average Latency(s): 0.013351 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Latency(s): 0.024368 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max latency(s): 0.809605 2024-09-13T14:54:57.160 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min latency(s): 0.00109161 2024-09-13T14:54:57.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:57.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:57.167 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:54:57.265 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:57.287 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:54:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:54:57] "GET /metrics HTTP/1.1" 200 36749 "" "Prometheus/2.51.0" 2024-09-13T14:54:57.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:57.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:57.399 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:57.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:57.537 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:57.563 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:57.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:57.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:57.698 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:57.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:57.801 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:57.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:57 smithi099 bash[84576]: cluster 2024-09-13T14:54:56.647546+0000 mgr.y (mgr.26252) 860 : cluster [DBG] pgmap v1374: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 665 op/s 2024-09-13T14:54:57.812 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:57 smithi099 bash[84576]: cluster 2024-09-13T14:54:56.647546+0000 mgr.y (mgr.26252) 860 : cluster [DBG] pgmap v1374: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 665 op/s 2024-09-13T14:54:57.813 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:57 smithi099 bash[87147]: cluster 2024-09-13T14:54:56.647546+0000 mgr.y (mgr.26252) 860 : cluster [DBG] pgmap v1374: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 665 op/s 2024-09-13T14:54:57.813 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:57 smithi099 bash[87147]: cluster 2024-09-13T14:54:56.647546+0000 mgr.y (mgr.26252) 860 : cluster [DBG] pgmap v1374: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 665 op/s 2024-09-13T14:54:57.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:57.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:57.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:57.955 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:57.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:58.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:58.071 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:57 smithi177 bash[68594]: cluster 2024-09-13T14:54:56.647546+0000 mgr.y (mgr.26252) 860 : cluster [DBG] pgmap v1374: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 665 op/s 2024-09-13T14:54:58.071 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:57 smithi177 bash[68594]: cluster 2024-09-13T14:54:56.647546+0000 mgr.y (mgr.26252) 860 : cluster [DBG] pgmap v1374: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 665 op/s 2024-09-13T14:54:58.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:58.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:58.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:54:58.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:58.223 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:58.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:58.329 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:58.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:58.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:58.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:58.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:58.670 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:58.696 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:58.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:58.785 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:54:58.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:58.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:58 smithi099 bash[84576]: audit 2024-09-13T14:54:58.691536+0000 mon.a (mon.0) 1912 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:58.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:58 smithi099 bash[84576]: audit 2024-09-13T14:54:58.691536+0000 mon.a (mon.0) 1912 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:58.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:58 smithi099 bash[87147]: audit 2024-09-13T14:54:58.691536+0000 mon.a (mon.0) 1912 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:58.821 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:58 smithi099 bash[87147]: audit 2024-09-13T14:54:58.691536+0000 mon.a (mon.0) 1912 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:58.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:58.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:59.024 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:59.048 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:59.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:58 smithi177 bash[68594]: audit 2024-09-13T14:54:58.691536+0000 mon.a (mon.0) 1912 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:59.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:58 smithi177 bash[68594]: audit 2024-09-13T14:54:58.691536+0000 mon.a (mon.0) 1912 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:59.151 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:59.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:59.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:59.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:59.311 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:59.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:59.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:59.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:59.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:59.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:59.597 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:59.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:59.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:59.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:59.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:59.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[84576]: cluster 2024-09-13T14:54:58.648355+0000 mgr.y (mgr.26252) 861 : cluster [DBG] pgmap v1375: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2024-09-13T14:54:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[84576]: cluster 2024-09-13T14:54:58.648355+0000 mgr.y (mgr.26252) 861 : cluster [DBG] pgmap v1375: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2024-09-13T14:54:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[84576]: audit 2024-09-13T14:54:58.788106+0000 mon.a (mon.0) 1913 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:54:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[84576]: audit 2024-09-13T14:54:58.788106+0000 mon.a (mon.0) 1913 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:54:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[84576]: cluster 2024-09-13T14:54:58.794354+0000 mon.a (mon.0) 1914 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2024-09-13T14:54:59.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[84576]: cluster 2024-09-13T14:54:58.794354+0000 mon.a (mon.0) 1914 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2024-09-13T14:54:59.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[87147]: cluster 2024-09-13T14:54:58.648355+0000 mgr.y (mgr.26252) 861 : cluster [DBG] pgmap v1375: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2024-09-13T14:54:59.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[87147]: cluster 2024-09-13T14:54:58.648355+0000 mgr.y (mgr.26252) 861 : cluster [DBG] pgmap v1375: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2024-09-13T14:54:59.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[87147]: audit 2024-09-13T14:54:58.788106+0000 mon.a (mon.0) 1913 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:54:59.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[87147]: audit 2024-09-13T14:54:58.788106+0000 mon.a (mon.0) 1913 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:54:59.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[87147]: cluster 2024-09-13T14:54:58.794354+0000 mon.a (mon.0) 1914 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2024-09-13T14:54:59.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:54:59 smithi099 bash[87147]: cluster 2024-09-13T14:54:58.794354+0000 mon.a (mon.0) 1914 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2024-09-13T14:54:59.824 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:59.846 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:59.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:59.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:59.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:59.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:00.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:59 smithi177 bash[68594]: cluster 2024-09-13T14:54:58.648355+0000 mgr.y (mgr.26252) 861 : cluster [DBG] pgmap v1375: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2024-09-13T14:55:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:59 smithi177 bash[68594]: cluster 2024-09-13T14:54:58.648355+0000 mgr.y (mgr.26252) 861 : cluster [DBG] pgmap v1375: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2024-09-13T14:55:00.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:59 smithi177 bash[68594]: audit 2024-09-13T14:54:58.788106+0000 mon.a (mon.0) 1913 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:55:00.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:59 smithi177 bash[68594]: audit 2024-09-13T14:54:58.788106+0000 mon.a (mon.0) 1913 : audit [INF] from='client.? 172.21.15.99:0/2440156083' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:55:00.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:59 smithi177 bash[68594]: cluster 2024-09-13T14:54:58.794354+0000 mon.a (mon.0) 1914 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2024-09-13T14:55:00.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:54:59 smithi177 bash[68594]: cluster 2024-09-13T14:54:58.794354+0000 mon.a (mon.0) 1914 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2024-09-13T14:55:00.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:00.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:00.215 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:00.240 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:00.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.368 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:00.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.492 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:00.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:55:00 smithi177 bash[61439]: ts=2024-09-13T14:55:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:00.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:00.728 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:00.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:00.860 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:00.895 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:00.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:01.005 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:01.029 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:01.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:01.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:01.156 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:01.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:01.260 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:01.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:55:01.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:01.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:01.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:01.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:01.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:01.494 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:55:01.494 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:01.540 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:01.546 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:55:01.546 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:01.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:01.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:01.574 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:55:01.574 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:01.610 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:01.610 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:55:01.648 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:55:01.649 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:01.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:01.674 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:55:01.675 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:01.701 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:55:01.701 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:01.704 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:01.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:01.727 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:55:01.728 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:01.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:01.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:01.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:01.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:01 smithi177 bash[68594]: cluster 2024-09-13T14:55:00.648920+0000 mgr.y (mgr.26252) 862 : cluster [DBG] pgmap v1377: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 759 op/s 2024-09-13T14:55:01.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:01 smithi177 bash[68594]: cluster 2024-09-13T14:55:00.648920+0000 mgr.y (mgr.26252) 862 : cluster [DBG] pgmap v1377: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 759 op/s 2024-09-13T14:55:01.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:01 smithi177 bash[68594]: audit 2024-09-13T14:55:01.449697+0000 mon.a (mon.0) 1915 : audit [DBG] from='client.? 172.21.15.99:0/4275574822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:01.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:01 smithi177 bash[68594]: audit 2024-09-13T14:55:01.449697+0000 mon.a (mon.0) 1915 : audit [DBG] from='client.? 172.21.15.99:0/4275574822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:01.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:01.958 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:01 smithi099 bash[84576]: cluster 2024-09-13T14:55:00.648920+0000 mgr.y (mgr.26252) 862 : cluster [DBG] pgmap v1377: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 759 op/s 2024-09-13T14:55:01.958 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:01 smithi099 bash[84576]: cluster 2024-09-13T14:55:00.648920+0000 mgr.y (mgr.26252) 862 : cluster [DBG] pgmap v1377: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 759 op/s 2024-09-13T14:55:01.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:01 smithi099 bash[84576]: audit 2024-09-13T14:55:01.449697+0000 mon.a (mon.0) 1915 : audit [DBG] from='client.? 172.21.15.99:0/4275574822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:01.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:01 smithi099 bash[84576]: audit 2024-09-13T14:55:01.449697+0000 mon.a (mon.0) 1915 : audit [DBG] from='client.? 172.21.15.99:0/4275574822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:01.959 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:01 smithi099 bash[87147]: cluster 2024-09-13T14:55:00.648920+0000 mgr.y (mgr.26252) 862 : cluster [DBG] pgmap v1377: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 759 op/s 2024-09-13T14:55:01.959 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:01 smithi099 bash[87147]: cluster 2024-09-13T14:55:00.648920+0000 mgr.y (mgr.26252) 862 : cluster [DBG] pgmap v1377: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 759 op/s 2024-09-13T14:55:01.959 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:01 smithi099 bash[87147]: audit 2024-09-13T14:55:01.449697+0000 mon.a (mon.0) 1915 : audit [DBG] from='client.? 172.21.15.99:0/4275574822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:01.959 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:01 smithi099 bash[87147]: audit 2024-09-13T14:55:01.449697+0000 mon.a (mon.0) 1915 : audit [DBG] from='client.? 172.21.15.99:0/4275574822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:01.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:02.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:02.123 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:02.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:02.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:02.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:02.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:02.362 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:02.399 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:02.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:02.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:02.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:02.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:02.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:02.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:02.802 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T14:55:02.808 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:02.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:02.922 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:02 smithi177 bash[68594]: audit 2024-09-13T14:55:01.876086+0000 mon.a (mon.0) 1916 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:02.922 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:02 smithi177 bash[68594]: audit 2024-09-13T14:55:01.876086+0000 mon.a (mon.0) 1916 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:02.929 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:02.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:03.036 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:03.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:03.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:03.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:02 smithi099 bash[84576]: audit 2024-09-13T14:55:01.876086+0000 mon.a (mon.0) 1916 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:03.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:02 smithi099 bash[84576]: audit 2024-09-13T14:55:01.876086+0000 mon.a (mon.0) 1916 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:03.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:02 smithi099 bash[87147]: audit 2024-09-13T14:55:01.876086+0000 mon.a (mon.0) 1916 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:03.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:02 smithi099 bash[87147]: audit 2024-09-13T14:55:01.876086+0000 mon.a (mon.0) 1916 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:03.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:03.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:03.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:03.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:03.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.329 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:03.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:03.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:03.452 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:03.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:03.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:03.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:03.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:03.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:03.712 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:03.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:03.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: cluster 2024-09-13T14:55:02.649397+0000 mgr.y (mgr.26252) 863 : cluster [DBG] pgmap v1378: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 749 op/s 2024-09-13T14:55:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: cluster 2024-09-13T14:55:02.649397+0000 mgr.y (mgr.26252) 863 : cluster [DBG] pgmap v1378: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 749 op/s 2024-09-13T14:55:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: cluster 2024-09-13T14:55:02.792067+0000 mon.a (mon.0) 1917 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: cluster 2024-09-13T14:55:02.792067+0000 mon.a (mon.0) 1917 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: cluster 2024-09-13T14:55:02.792092+0000 mon.a (mon.0) 1918 : cluster [INF] Cluster is now healthy 2024-09-13T14:55:03.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: cluster 2024-09-13T14:55:02.792092+0000 mon.a (mon.0) 1918 : cluster [INF] Cluster is now healthy 2024-09-13T14:55:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: audit 2024-09-13T14:55:02.798977+0000 mon.a (mon.0) 1919 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: audit 2024-09-13T14:55:02.798977+0000 mon.a (mon.0) 1919 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: cluster 2024-09-13T14:55:02.804175+0000 mon.a (mon.0) 1920 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2024-09-13T14:55:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: cluster 2024-09-13T14:55:02.804175+0000 mon.a (mon.0) 1920 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2024-09-13T14:55:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: audit 2024-09-13T14:55:03.288355+0000 mon.a (mon.0) 1921 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: audit 2024-09-13T14:55:03.288355+0000 mon.a (mon.0) 1921 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: audit 2024-09-13T14:55:03.290839+0000 mon.a (mon.0) 1922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:03.822 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[84576]: audit 2024-09-13T14:55:03.290839+0000 mon.a (mon.0) 1922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: cluster 2024-09-13T14:55:02.649397+0000 mgr.y (mgr.26252) 863 : cluster [DBG] pgmap v1378: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 749 op/s 2024-09-13T14:55:03.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: cluster 2024-09-13T14:55:02.649397+0000 mgr.y (mgr.26252) 863 : cluster [DBG] pgmap v1378: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 749 op/s 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: cluster 2024-09-13T14:55:02.792067+0000 mon.a (mon.0) 1917 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: cluster 2024-09-13T14:55:02.792067+0000 mon.a (mon.0) 1917 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: cluster 2024-09-13T14:55:02.792092+0000 mon.a (mon.0) 1918 : cluster [INF] Cluster is now healthy 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: cluster 2024-09-13T14:55:02.792092+0000 mon.a (mon.0) 1918 : cluster [INF] Cluster is now healthy 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: audit 2024-09-13T14:55:02.798977+0000 mon.a (mon.0) 1919 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: audit 2024-09-13T14:55:02.798977+0000 mon.a (mon.0) 1919 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: cluster 2024-09-13T14:55:02.804175+0000 mon.a (mon.0) 1920 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: cluster 2024-09-13T14:55:02.804175+0000 mon.a (mon.0) 1920 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: audit 2024-09-13T14:55:03.288355+0000 mon.a (mon.0) 1921 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: audit 2024-09-13T14:55:03.288355+0000 mon.a (mon.0) 1921 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: audit 2024-09-13T14:55:03.290839+0000 mon.a (mon.0) 1922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:03.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:03 smithi099 bash[87147]: audit 2024-09-13T14:55:03.290839+0000 mon.a (mon.0) 1922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:03.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:03.947 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:03.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:03.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:04.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: cluster 2024-09-13T14:55:02.649397+0000 mgr.y (mgr.26252) 863 : cluster [DBG] pgmap v1378: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 749 op/s 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: cluster 2024-09-13T14:55:02.649397+0000 mgr.y (mgr.26252) 863 : cluster [DBG] pgmap v1378: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 749 op/s 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: cluster 2024-09-13T14:55:02.792067+0000 mon.a (mon.0) 1917 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: cluster 2024-09-13T14:55:02.792067+0000 mon.a (mon.0) 1917 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: cluster 2024-09-13T14:55:02.792092+0000 mon.a (mon.0) 1918 : cluster [INF] Cluster is now healthy 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: cluster 2024-09-13T14:55:02.792092+0000 mon.a (mon.0) 1918 : cluster [INF] Cluster is now healthy 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: audit 2024-09-13T14:55:02.798977+0000 mon.a (mon.0) 1919 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: audit 2024-09-13T14:55:02.798977+0000 mon.a (mon.0) 1919 : audit [INF] from='client.? 172.21.15.99:0/510435751' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: cluster 2024-09-13T14:55:02.804175+0000 mon.a (mon.0) 1920 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: cluster 2024-09-13T14:55:02.804175+0000 mon.a (mon.0) 1920 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2024-09-13T14:55:04.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: audit 2024-09-13T14:55:03.288355+0000 mon.a (mon.0) 1921 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:04.086 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: audit 2024-09-13T14:55:03.288355+0000 mon.a (mon.0) 1921 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:04.086 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: audit 2024-09-13T14:55:03.290839+0000 mon.a (mon.0) 1922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:04.086 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:03 smithi177 bash[68594]: audit 2024-09-13T14:55:03.290839+0000 mon.a (mon.0) 1922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:04.104 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:04.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:04.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:04.244 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:04.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:04.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:04.365 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:04.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:04.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:04.493 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:04.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:04.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:04.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:04.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:04.722 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:04.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:04.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:04.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:04.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:04.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:04.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:05.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:05.119 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:05.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:05.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:05.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:55:05.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:05.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:05.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:05.430 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:05.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:05.537 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:05.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:05.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:05.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:05.700 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:05.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:05.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:05.825 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:05.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:05.928 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:05.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:05.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:06.053 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:06.062 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:05 smithi177 bash[68594]: cluster 2024-09-13T14:55:04.650548+0000 mgr.y (mgr.26252) 864 : cluster [DBG] pgmap v1380: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 956 op/s 2024-09-13T14:55:06.062 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:05 smithi177 bash[68594]: cluster 2024-09-13T14:55:04.650548+0000 mgr.y (mgr.26252) 864 : cluster [DBG] pgmap v1380: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 956 op/s 2024-09-13T14:55:06.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:05 smithi099 bash[84576]: cluster 2024-09-13T14:55:04.650548+0000 mgr.y (mgr.26252) 864 : cluster [DBG] pgmap v1380: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 956 op/s 2024-09-13T14:55:06.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:05 smithi099 bash[84576]: cluster 2024-09-13T14:55:04.650548+0000 mgr.y (mgr.26252) 864 : cluster [DBG] pgmap v1380: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 956 op/s 2024-09-13T14:55:06.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:05 smithi099 bash[87147]: cluster 2024-09-13T14:55:04.650548+0000 mgr.y (mgr.26252) 864 : cluster [DBG] pgmap v1380: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 956 op/s 2024-09-13T14:55:06.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:05 smithi099 bash[87147]: cluster 2024-09-13T14:55:04.650548+0000 mgr.y (mgr.26252) 864 : cluster [DBG] pgmap v1380: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 956 op/s 2024-09-13T14:55:06.079 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:06.100 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:55:06.184 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:06.209 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:06.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:06.312 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:06.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:06.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:06.443 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:06.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:06.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:06.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:06.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:06.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:06.777 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:06.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:06.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:06.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:06.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:06.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:07.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:07.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:07.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.313 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:55:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:55:07] "GET /metrics HTTP/1.1" 200 36756 "" "Prometheus/2.51.0" 2024-09-13T14:55:07.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:07.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:07.456 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:07.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:07.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:07.607 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:07.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:07.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.738 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:07.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:07 smithi177 bash[68594]: cluster 2024-09-13T14:55:06.651025+0000 mgr.y (mgr.26252) 865 : cluster [DBG] pgmap v1381: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 14 MiB/s wr, 919 op/s 2024-09-13T14:55:07.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:07 smithi177 bash[68594]: cluster 2024-09-13T14:55:06.651025+0000 mgr.y (mgr.26252) 865 : cluster [DBG] pgmap v1381: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 14 MiB/s wr, 919 op/s 2024-09-13T14:55:07.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:07.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:08.006 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:08.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:08.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:07 smithi099 bash[84576]: cluster 2024-09-13T14:55:06.651025+0000 mgr.y (mgr.26252) 865 : cluster [DBG] pgmap v1381: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 14 MiB/s wr, 919 op/s 2024-09-13T14:55:08.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:07 smithi099 bash[84576]: cluster 2024-09-13T14:55:06.651025+0000 mgr.y (mgr.26252) 865 : cluster [DBG] pgmap v1381: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 14 MiB/s wr, 919 op/s 2024-09-13T14:55:08.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:07 smithi099 bash[87147]: cluster 2024-09-13T14:55:06.651025+0000 mgr.y (mgr.26252) 865 : cluster [DBG] pgmap v1381: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 14 MiB/s wr, 919 op/s 2024-09-13T14:55:08.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:07 smithi099 bash[87147]: cluster 2024-09-13T14:55:06.651025+0000 mgr.y (mgr.26252) 865 : cluster [DBG] pgmap v1381: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 14 MiB/s wr, 919 op/s 2024-09-13T14:55:08.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:08.140 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:08.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:08.243 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:08.268 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:08.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:08.371 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:08.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:08.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:08.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:08.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:08.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:08.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:08.918 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:08.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:09.021 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:09.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:09.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:09.161 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:09.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:09.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:09.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:09.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:09.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:09.430 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:09.452 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:09.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:09.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:09.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:09.658 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T14:55:09.658 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T14:55:09.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.658 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45350.0:117545 42.0 e1023/1009 42:07a6fa9e:::benchmark_data_smithi099_148651_object9705:head v 1023'7463, mlcod=1023'7461)", 2024-09-13T14:55:09.658 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:55:09.645119+0000", 2024-09-13T14:55:09.658 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.012712405, 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012749755, 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645119+0000", 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645119+0000", 2024-09-13T14:55:09.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645118+0000", 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967296 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645122+0000", 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7840000000000001e-06 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645125+0000", 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0759999999999999e-06 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645127+0000", 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.906e-06 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645263+0000", 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000136026 2024-09-13T14:55:09.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645281+0000", 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7991000000000001e-05 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:117559 42.f 42:f3c839cf:::benchmark_data_smithi099_148651_object9719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1023)", 2024-09-13T14:55:09.662 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:55:09.645574+0000", 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.012256853, 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01235937, 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 117559 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:55:09.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645574+0000", 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645574+0000", 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645573+0000", 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:55:09.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645579+0000", 2024-09-13T14:55:09.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2199999999999997e-06 2024-09-13T14:55:09.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:55:09.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645593+0000", 2024-09-13T14:55:09.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3753e-05 2024-09-13T14:55:09.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645599+0000", 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3080000000000004e-06 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.645739+0000", 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013999499999999999 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.646061+0000", 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032202199999999999 2024-09-13T14:55:09.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.646442+0000", 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00038141299999999999 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:117562 42.7 42:ee68bf9f:::benchmark_data_smithi099_148651_object9722:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1023)", 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:55:09.651638+0000", 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0061928010000000004, 2024-09-13T14:55:09.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063659900000000002, 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 117562 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651638+0000", 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651638+0000", 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651637+0000", 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651643+0000", 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3350000000000003e-06 2024-09-13T14:55:09.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651660+0000", 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7133e-05 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651667+0000", 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0380000000000002e-06 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:55:09.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651966+0000", 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00029874300000000002 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.652313+0000", 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00034649099999999999 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.652695+0000", 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00038256100000000001 2024-09-13T14:55:09.671 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:117563 42.f 42:f3755eff:::benchmark_data_smithi099_148651_object9723:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1023)", 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:55:09.651681+0000", 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0061494779999999999, 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063877259999999998, 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:55:09.672 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 117563 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651681+0000", 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651681+0000", 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651681+0000", 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651684+0000", 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2930000000000001e-06 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651692+0000", 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1519999999999992e-06 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651697+0000", 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7090000000000001e-06 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.657080+0000", 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053832059999999998 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.657196+0000", 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000115587 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.657296+0000", 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100614 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:117565 42.7 42.84d0a977 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1023)", 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:55:09.651869+0000", 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0059617200000000002, 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062658849999999997, 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:55:09.676 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "queued for pg", 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 117565 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651869+0000", 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651869+0000", 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651868+0000", 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651873+0000", 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.476e-06 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651885+0000", 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1882000000000001e-05 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.651891+0000", 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7590000000000003e-06 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45350.0:117569 42.9 e1023/1009)", 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:55:09.656966+0000", 2024-09-13T14:55:09.679 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.00086466699999999995, 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001215201, 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "queued for pg", 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.656966+0000", 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.656966+0000", 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.656965+0000", 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.656973+0000", 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3710000000000002e-06 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.656989+0000", 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6025999999999999e-05 2024-09-13T14:55:09.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.656995+0000", 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1360000000000002e-06 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop_reply(client.45350.0:117562 42.7 e1023/1009)", 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:55:09.657122+0000", 2024-09-13T14:55:09.682 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.00070870199999999996, 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001099324, 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "queued for pg", 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.657122+0000", 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.657122+0000", 2024-09-13T14:55:09.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.657122+0000", 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.657135+0000", 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3728e-05 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T14:55:09.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.657147+0000", 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1259e-05 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:09.657152+0000", 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1780000000000002e-06 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 7 2024-09-13T14:55:09.685 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T14:55:09.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:09.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:09.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:09.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:09.819 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:09 smithi177 bash[68594]: cluster 2024-09-13T14:55:08.651864+0000 mgr.y (mgr.26252) 866 : cluster [DBG] pgmap v1382: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 103 B/s rd, 11 MiB/s wr, 769 op/s 2024-09-13T14:55:09.819 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:09 smithi177 bash[68594]: cluster 2024-09-13T14:55:08.651864+0000 mgr.y (mgr.26252) 866 : cluster [DBG] pgmap v1382: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 103 B/s rd, 11 MiB/s wr, 769 op/s 2024-09-13T14:55:09.826 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:09.842 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:09.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:09.946 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:09.969 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:10.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:09 smithi099 bash[84576]: cluster 2024-09-13T14:55:08.651864+0000 mgr.y (mgr.26252) 866 : cluster [DBG] pgmap v1382: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 103 B/s rd, 11 MiB/s wr, 769 op/s 2024-09-13T14:55:10.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:09 smithi099 bash[84576]: cluster 2024-09-13T14:55:08.651864+0000 mgr.y (mgr.26252) 866 : cluster [DBG] pgmap v1382: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 103 B/s rd, 11 MiB/s wr, 769 op/s 2024-09-13T14:55:10.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:09 smithi099 bash[87147]: cluster 2024-09-13T14:55:08.651864+0000 mgr.y (mgr.26252) 866 : cluster [DBG] pgmap v1382: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 103 B/s rd, 11 MiB/s wr, 769 op/s 2024-09-13T14:55:10.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:09 smithi099 bash[87147]: cluster 2024-09-13T14:55:08.651864+0000 mgr.y (mgr.26252) 866 : cluster [DBG] pgmap v1382: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 103 B/s rd, 11 MiB/s wr, 769 op/s 2024-09-13T14:55:10.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:10.121 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:10.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:10.225 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:10.253 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:10.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:10.357 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:10.383 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:10.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:10.476 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:55:10 smithi177 bash[61439]: ts=2024-09-13T14:55:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:10.486 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:10.512 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:10.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:10.615 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:10.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:10.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:10.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:10.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:10.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:10.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:10.907 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:10.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:55:10.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:11.016 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:11.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:11.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:11.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:11.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:11.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:11.322 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:11.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:11.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:11.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:11.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:11.563 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:11.586 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:11.713 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:11.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:11.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:11 smithi099 bash[84576]: cluster 2024-09-13T14:55:10.652652+0000 mgr.y (mgr.26252) 867 : cluster [DBG] pgmap v1383: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:55:11.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:11 smithi099 bash[84576]: cluster 2024-09-13T14:55:10.652652+0000 mgr.y (mgr.26252) 867 : cluster [DBG] pgmap v1383: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:55:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:11 smithi099 bash[87147]: cluster 2024-09-13T14:55:10.652652+0000 mgr.y (mgr.26252) 867 : cluster [DBG] pgmap v1383: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:55:11.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:11 smithi099 bash[87147]: cluster 2024-09-13T14:55:10.652652+0000 mgr.y (mgr.26252) 867 : cluster [DBG] pgmap v1383: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:55:11.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:11.934 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:11.970 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:12.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:12.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:11 smithi177 bash[68594]: cluster 2024-09-13T14:55:10.652652+0000 mgr.y (mgr.26252) 867 : cluster [DBG] pgmap v1383: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:55:12.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:11 smithi177 bash[68594]: cluster 2024-09-13T14:55:10.652652+0000 mgr.y (mgr.26252) 867 : cluster [DBG] pgmap v1383: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2024-09-13T14:55:12.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:12.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:12.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:12.206 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:12.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:12.310 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:12.371 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:12.474 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:12.503 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:12.609 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:12.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:12.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:12.741 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:12.762 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:12.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:12.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:12.889 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:12.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:12.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:13.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:13.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:13.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:13.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:13.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:13.254 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:13.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:13.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:13.380 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:13.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:13.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:13.564 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:13.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:13.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:13.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:13.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:13.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:13.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:13.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:13.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:13.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:14.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:13 smithi099 bash[84576]: cluster 2024-09-13T14:55:12.653173+0000 mgr.y (mgr.26252) 868 : cluster [DBG] pgmap v1384: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-09-13T14:55:14.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:13 smithi099 bash[84576]: cluster 2024-09-13T14:55:12.653173+0000 mgr.y (mgr.26252) 868 : cluster [DBG] pgmap v1384: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-09-13T14:55:14.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:13 smithi099 bash[87147]: cluster 2024-09-13T14:55:12.653173+0000 mgr.y (mgr.26252) 868 : cluster [DBG] pgmap v1384: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-09-13T14:55:14.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:13 smithi099 bash[87147]: cluster 2024-09-13T14:55:12.653173+0000 mgr.y (mgr.26252) 868 : cluster [DBG] pgmap v1384: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-09-13T14:55:14.088 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:14.097 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:13 smithi177 bash[68594]: cluster 2024-09-13T14:55:12.653173+0000 mgr.y (mgr.26252) 868 : cluster [DBG] pgmap v1384: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-09-13T14:55:14.097 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:13 smithi177 bash[68594]: cluster 2024-09-13T14:55:12.653173+0000 mgr.y (mgr.26252) 868 : cluster [DBG] pgmap v1384: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-09-13T14:55:14.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:14.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:14.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:14.272 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:14.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:14.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:14.396 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:14.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:14.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:14.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:14.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:14.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:14.658 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:14.761 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:14.865 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:14.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:14.970 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:15.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:15.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:15.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:15.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:15.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:15.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.288 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:15.394 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:15.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:15.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:15.681 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:15.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:15.784 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:15.785 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.823 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:15.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.969 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:16.066 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:15 smithi099 bash[84576]: cluster 2024-09-13T14:55:14.653812+0000 mgr.y (mgr.26252) 869 : cluster [DBG] pgmap v1385: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:55:16.066 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:15 smithi099 bash[84576]: cluster 2024-09-13T14:55:14.653812+0000 mgr.y (mgr.26252) 869 : cluster [DBG] pgmap v1385: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:55:16.066 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:15 smithi099 bash[87147]: cluster 2024-09-13T14:55:14.653812+0000 mgr.y (mgr.26252) 869 : cluster [DBG] pgmap v1385: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:55:16.066 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:15 smithi099 bash[87147]: cluster 2024-09-13T14:55:14.653812+0000 mgr.y (mgr.26252) 869 : cluster [DBG] pgmap v1385: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:55:16.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:16.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:16.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:16.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:15 smithi177 bash[68594]: cluster 2024-09-13T14:55:14.653812+0000 mgr.y (mgr.26252) 869 : cluster [DBG] pgmap v1385: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:55:16.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:15 smithi177 bash[68594]: cluster 2024-09-13T14:55:14.653812+0000 mgr.y (mgr.26252) 869 : cluster [DBG] pgmap v1385: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-09-13T14:55:16.203 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:16.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:16.338 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:16.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:16.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:16.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:16.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:16.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:16.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:16.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:16.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:16.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:16.842 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:55:16.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:16.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:16.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:17.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:17.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:16 smithi177 bash[68594]: audit 2024-09-13T14:55:16.749684+0000 mon.c (mon.2) 68 : audit [INF] from='client.? 172.21.15.99:0/1905916369' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:16 smithi177 bash[68594]: audit 2024-09-13T14:55:16.749684+0000 mon.c (mon.2) 68 : audit [INF] from='client.? 172.21.15.99:0/1905916369' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:16 smithi177 bash[68594]: audit 2024-09-13T14:55:16.750125+0000 mon.a (mon.0) 1923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:16 smithi177 bash[68594]: audit 2024-09-13T14:55:16.750125+0000 mon.a (mon.0) 1923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:17.153 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:17.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:17.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:17.266 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:55:17] "GET /metrics HTTP/1.1" 200 36756 "" "Prometheus/2.51.0" 2024-09-13T14:55:17.267 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:16 smithi099 bash[84576]: audit 2024-09-13T14:55:16.749684+0000 mon.c (mon.2) 68 : audit [INF] from='client.? 172.21.15.99:0/1905916369' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.267 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:16 smithi099 bash[84576]: audit 2024-09-13T14:55:16.749684+0000 mon.c (mon.2) 68 : audit [INF] from='client.? 172.21.15.99:0/1905916369' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.267 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:16 smithi099 bash[84576]: audit 2024-09-13T14:55:16.750125+0000 mon.a (mon.0) 1923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.267 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:16 smithi099 bash[84576]: audit 2024-09-13T14:55:16.750125+0000 mon.a (mon.0) 1923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.267 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:16 smithi099 bash[87147]: audit 2024-09-13T14:55:16.749684+0000 mon.c (mon.2) 68 : audit [INF] from='client.? 172.21.15.99:0/1905916369' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.268 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:16 smithi099 bash[87147]: audit 2024-09-13T14:55:16.749684+0000 mon.c (mon.2) 68 : audit [INF] from='client.? 172.21.15.99:0/1905916369' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.268 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:16 smithi099 bash[87147]: audit 2024-09-13T14:55:16.750125+0000 mon.a (mon.0) 1923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.268 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:16 smithi099 bash[87147]: audit 2024-09-13T14:55:16.750125+0000 mon.a (mon.0) 1923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:55:17.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:17.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:17.408 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:17.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:17.539 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:17.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:17.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:17.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:17.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:17.827 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:17.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:17.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:17.939 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:17 smithi177 bash[68594]: cluster 2024-09-13T14:55:16.654282+0000 mgr.y (mgr.26252) 870 : cluster [DBG] pgmap v1386: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-13T14:55:17.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:17 smithi177 bash[68594]: cluster 2024-09-13T14:55:16.654282+0000 mgr.y (mgr.26252) 870 : cluster [DBG] pgmap v1386: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-13T14:55:17.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:17 smithi177 bash[68594]: audit 2024-09-13T14:55:16.838756+0000 mon.a (mon.0) 1924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:55:17.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:17 smithi177 bash[68594]: audit 2024-09-13T14:55:16.838756+0000 mon.a (mon.0) 1924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:55:17.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:17 smithi177 bash[68594]: cluster 2024-09-13T14:55:16.843044+0000 mon.a (mon.0) 1925 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2024-09-13T14:55:17.940 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:17 smithi177 bash[68594]: cluster 2024-09-13T14:55:16.843044+0000 mon.a (mon.0) 1925 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2024-09-13T14:55:17.966 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:17.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:18.069 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:18.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[84576]: cluster 2024-09-13T14:55:16.654282+0000 mgr.y (mgr.26252) 870 : cluster [DBG] pgmap v1386: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-13T14:55:18.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[84576]: cluster 2024-09-13T14:55:16.654282+0000 mgr.y (mgr.26252) 870 : cluster [DBG] pgmap v1386: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-13T14:55:18.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[84576]: audit 2024-09-13T14:55:16.838756+0000 mon.a (mon.0) 1924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:55:18.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[84576]: audit 2024-09-13T14:55:16.838756+0000 mon.a (mon.0) 1924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:55:18.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[84576]: cluster 2024-09-13T14:55:16.843044+0000 mon.a (mon.0) 1925 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2024-09-13T14:55:18.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[84576]: cluster 2024-09-13T14:55:16.843044+0000 mon.a (mon.0) 1925 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2024-09-13T14:55:18.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[87147]: cluster 2024-09-13T14:55:16.654282+0000 mgr.y (mgr.26252) 870 : cluster [DBG] pgmap v1386: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-13T14:55:18.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[87147]: cluster 2024-09-13T14:55:16.654282+0000 mgr.y (mgr.26252) 870 : cluster [DBG] pgmap v1386: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-13T14:55:18.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[87147]: audit 2024-09-13T14:55:16.838756+0000 mon.a (mon.0) 1924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:55:18.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[87147]: audit 2024-09-13T14:55:16.838756+0000 mon.a (mon.0) 1924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:55:18.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[87147]: cluster 2024-09-13T14:55:16.843044+0000 mon.a (mon.0) 1925 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2024-09-13T14:55:18.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:17 smithi099 bash[87147]: cluster 2024-09-13T14:55:16.843044+0000 mon.a (mon.0) 1925 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2024-09-13T14:55:18.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:18.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:18.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:18.239 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:18.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:18.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:18.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:18.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:18.482 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:18.513 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:18.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:18.620 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:18.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:18.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:18.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:18.766 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:18.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:18.871 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:18.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:18.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:18.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:19.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:19.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:19.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:18 smithi177 bash[68594]: audit 2024-09-13T14:55:18.282882+0000 mon.a (mon.0) 1926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:19.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:18 smithi177 bash[68594]: audit 2024-09-13T14:55:18.282882+0000 mon.a (mon.0) 1926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:19.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:19.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:19.148 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:18 smithi099 bash[84576]: audit 2024-09-13T14:55:18.282882+0000 mon.a (mon.0) 1926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:19.149 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:18 smithi099 bash[84576]: audit 2024-09-13T14:55:18.282882+0000 mon.a (mon.0) 1926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:19.149 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:18 smithi099 bash[87147]: audit 2024-09-13T14:55:18.282882+0000 mon.a (mon.0) 1926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:19.149 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:18 smithi099 bash[87147]: audit 2024-09-13T14:55:18.282882+0000 mon.a (mon.0) 1926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:19.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:19.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:19.274 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:19.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:19.380 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:19.408 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:19.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:19.513 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:19.543 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:19.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:19.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:19.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:19.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:19.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:19.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:19.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:19.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:19.923 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:19.928 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:19 smithi099 bash[84576]: cluster 2024-09-13T14:55:18.654740+0000 mgr.y (mgr.26252) 871 : cluster [DBG] pgmap v1388: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-09-13T14:55:19.928 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:19 smithi099 bash[84576]: cluster 2024-09-13T14:55:18.654740+0000 mgr.y (mgr.26252) 871 : cluster [DBG] pgmap v1388: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-09-13T14:55:19.928 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:19 smithi099 bash[87147]: cluster 2024-09-13T14:55:18.654740+0000 mgr.y (mgr.26252) 871 : cluster [DBG] pgmap v1388: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-09-13T14:55:19.928 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:19 smithi099 bash[87147]: cluster 2024-09-13T14:55:18.654740+0000 mgr.y (mgr.26252) 871 : cluster [DBG] pgmap v1388: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-09-13T14:55:19.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:20.030 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:20.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:20.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:20.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:19 smithi177 bash[68594]: cluster 2024-09-13T14:55:18.654740+0000 mgr.y (mgr.26252) 871 : cluster [DBG] pgmap v1388: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-09-13T14:55:20.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:19 smithi177 bash[68594]: cluster 2024-09-13T14:55:18.654740+0000 mgr.y (mgr.26252) 871 : cluster [DBG] pgmap v1388: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2024-09-13T14:55:20.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:20.178 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:20.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:20.281 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:20.305 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:20.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:20.411 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:20.420 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:55:20.452 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:20.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:20.556 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:20.566 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:55:20 smithi177 bash[61439]: ts=2024-09-13T14:55:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:20.580 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:20.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:20.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:20.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:20.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:20.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:20.847 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:20.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:20.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:20.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:20 smithi177 bash[68594]: audit 2024-09-13T14:55:20.699214+0000 mon.a (mon.0) 1927 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:20.965 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:20 smithi177 bash[68594]: audit 2024-09-13T14:55:20.699214+0000 mon.a (mon.0) 1927 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:21.006 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:21.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:21.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:20 smithi099 bash[84576]: audit 2024-09-13T14:55:20.699214+0000 mon.a (mon.0) 1927 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:21.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:20 smithi099 bash[84576]: audit 2024-09-13T14:55:20.699214+0000 mon.a (mon.0) 1927 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:21.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:20 smithi099 bash[87147]: audit 2024-09-13T14:55:20.699214+0000 mon.a (mon.0) 1927 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:21.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:20 smithi099 bash[87147]: audit 2024-09-13T14:55:20.699214+0000 mon.a (mon.0) 1927 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:21.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:21.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:21.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:21.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.535 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:21.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:21.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:21.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:21.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:21.920 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:21.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:22.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:22.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:21 smithi099 bash[84576]: cluster 2024-09-13T14:55:20.655342+0000 mgr.y (mgr.26252) 872 : cluster [DBG] pgmap v1389: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:55:22.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:21 smithi099 bash[84576]: cluster 2024-09-13T14:55:20.655342+0000 mgr.y (mgr.26252) 872 : cluster [DBG] pgmap v1389: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:55:22.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:21 smithi099 bash[87147]: cluster 2024-09-13T14:55:20.655342+0000 mgr.y (mgr.26252) 872 : cluster [DBG] pgmap v1389: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:55:22.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:21 smithi099 bash[87147]: cluster 2024-09-13T14:55:20.655342+0000 mgr.y (mgr.26252) 872 : cluster [DBG] pgmap v1389: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:55:22.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:22.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:21 smithi177 bash[68594]: cluster 2024-09-13T14:55:20.655342+0000 mgr.y (mgr.26252) 872 : cluster [DBG] pgmap v1389: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:55:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:21 smithi177 bash[68594]: cluster 2024-09-13T14:55:20.655342+0000 mgr.y (mgr.26252) 872 : cluster [DBG] pgmap v1389: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2024-09-13T14:55:22.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:22.220 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:22.325 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:22.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:22.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:22.465 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:22.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:22.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:22.594 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:22.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:22.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:22.722 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:22.738 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T14:55:22.738 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T14:55:22.738 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T14:55:22.738 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T14:55:22.738 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T14:55:22.742 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:22.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:22.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:22.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:22.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:22.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:23.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:23.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:23.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:23.260 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:23.285 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:23.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:23.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:23.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:23.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:23.529 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:23.557 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:23.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:23.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:23.686 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:23.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:23.812 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:23.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:23.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:24.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:24.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:23 smithi099 bash[84576]: cluster 2024-09-13T14:55:22.655786+0000 mgr.y (mgr.26252) 873 : cluster [DBG] pgmap v1390: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-13T14:55:24.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:23 smithi099 bash[84576]: cluster 2024-09-13T14:55:22.655786+0000 mgr.y (mgr.26252) 873 : cluster [DBG] pgmap v1390: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-13T14:55:24.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:23 smithi099 bash[87147]: cluster 2024-09-13T14:55:22.655786+0000 mgr.y (mgr.26252) 873 : cluster [DBG] pgmap v1390: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-13T14:55:24.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:23 smithi099 bash[87147]: cluster 2024-09-13T14:55:22.655786+0000 mgr.y (mgr.26252) 873 : cluster [DBG] pgmap v1390: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-13T14:55:24.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:24.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:24.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:55:24.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:24.185 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:24.289 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:24.314 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:24.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:24.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:23 smithi177 bash[68594]: cluster 2024-09-13T14:55:22.655786+0000 mgr.y (mgr.26252) 873 : cluster [DBG] pgmap v1390: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-13T14:55:24.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:23 smithi177 bash[68594]: cluster 2024-09-13T14:55:22.655786+0000 mgr.y (mgr.26252) 873 : cluster [DBG] pgmap v1390: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-13T14:55:24.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:24.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:24.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:24.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:24.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:24.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:24.680 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:55:24.681 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:24.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:24.715 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:55:24.715 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:24.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:24.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:24.752 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:55:24.752 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:24.789 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:55:24.789 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:24.817 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:55:24.817 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:24.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:24.850 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:55:24.851 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:24.875 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:55:24.875 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:24.902 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:55:24.903 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:25.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:25.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:24 smithi177 bash[68594]: audit 2024-09-13T14:55:24.656886+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.99:0/2254477264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:25.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:24 smithi177 bash[68594]: audit 2024-09-13T14:55:24.656886+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.99:0/2254477264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:25.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:25.127 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:24 smithi099 bash[84576]: audit 2024-09-13T14:55:24.656886+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.99:0/2254477264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:25.127 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:24 smithi099 bash[84576]: audit 2024-09-13T14:55:24.656886+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.99:0/2254477264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:25.127 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:24 smithi099 bash[87147]: audit 2024-09-13T14:55:24.656886+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.99:0/2254477264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:25.127 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:24 smithi099 bash[87147]: audit 2024-09-13T14:55:24.656886+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.99:0/2254477264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:25.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:25.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:25.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:25.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:25.364 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:25.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:25.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:25.491 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:25.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:25.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:25.632 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:25.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:25.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:25.764 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:25.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:25.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:25.892 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T14:55:25.902 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:25.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:26.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:26.035 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:26.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:25 smithi177 bash[68594]: cluster 2024-09-13T14:55:24.656302+0000 mgr.y (mgr.26252) 874 : cluster [DBG] pgmap v1391: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:25 smithi177 bash[68594]: cluster 2024-09-13T14:55:24.656302+0000 mgr.y (mgr.26252) 874 : cluster [DBG] pgmap v1391: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:25 smithi177 bash[68594]: audit 2024-09-13T14:55:25.133594+0000 mon.c (mon.2) 70 : audit [INF] from='client.? 172.21.15.99:0/3088953953' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:25 smithi177 bash[68594]: audit 2024-09-13T14:55:25.133594+0000 mon.c (mon.2) 70 : audit [INF] from='client.? 172.21.15.99:0/3088953953' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:25 smithi177 bash[68594]: audit 2024-09-13T14:55:25.133927+0000 mon.a (mon.0) 1928 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:25 smithi177 bash[68594]: audit 2024-09-13T14:55:25.133927+0000 mon.a (mon.0) 1928 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.140 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:26.149 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[87147]: cluster 2024-09-13T14:55:24.656302+0000 mgr.y (mgr.26252) 874 : cluster [DBG] pgmap v1391: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:26.149 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[87147]: cluster 2024-09-13T14:55:24.656302+0000 mgr.y (mgr.26252) 874 : cluster [DBG] pgmap v1391: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:26.149 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[87147]: audit 2024-09-13T14:55:25.133594+0000 mon.c (mon.2) 70 : audit [INF] from='client.? 172.21.15.99:0/3088953953' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.149 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[87147]: audit 2024-09-13T14:55:25.133594+0000 mon.c (mon.2) 70 : audit [INF] from='client.? 172.21.15.99:0/3088953953' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.149 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[87147]: audit 2024-09-13T14:55:25.133927+0000 mon.a (mon.0) 1928 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.149 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[87147]: audit 2024-09-13T14:55:25.133927+0000 mon.a (mon.0) 1928 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.149 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[84576]: cluster 2024-09-13T14:55:24.656302+0000 mgr.y (mgr.26252) 874 : cluster [DBG] pgmap v1391: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:26.149 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[84576]: cluster 2024-09-13T14:55:24.656302+0000 mgr.y (mgr.26252) 874 : cluster [DBG] pgmap v1391: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:26.149 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[84576]: audit 2024-09-13T14:55:25.133594+0000 mon.c (mon.2) 70 : audit [INF] from='client.? 172.21.15.99:0/3088953953' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[84576]: audit 2024-09-13T14:55:25.133594+0000 mon.c (mon.2) 70 : audit [INF] from='client.? 172.21.15.99:0/3088953953' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[84576]: audit 2024-09-13T14:55:25.133927+0000 mon.a (mon.0) 1928 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:25 smithi099 bash[84576]: audit 2024-09-13T14:55:25.133927+0000 mon.a (mon.0) 1928 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:26.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:26.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:26.269 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:26.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:26.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:26.394 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:26.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:26.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:26.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:26.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:26.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:26.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:26.673 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:26.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:26.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:26.804 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:26.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:26.975 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:27.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[87147]: cluster 2024-09-13T14:55:25.878667+0000 mon.a (mon.0) 1929 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:27.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[87147]: cluster 2024-09-13T14:55:25.878667+0000 mon.a (mon.0) 1929 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:27.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[87147]: audit 2024-09-13T14:55:25.880124+0000 mon.a (mon.0) 1930 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:27.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[87147]: audit 2024-09-13T14:55:25.880124+0000 mon.a (mon.0) 1930 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:27.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[87147]: cluster 2024-09-13T14:55:25.884439+0000 mon.a (mon.0) 1931 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2024-09-13T14:55:27.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[87147]: cluster 2024-09-13T14:55:25.884439+0000 mon.a (mon.0) 1931 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2024-09-13T14:55:27.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[84576]: cluster 2024-09-13T14:55:25.878667+0000 mon.a (mon.0) 1929 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:27.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[84576]: cluster 2024-09-13T14:55:25.878667+0000 mon.a (mon.0) 1929 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:27.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[84576]: audit 2024-09-13T14:55:25.880124+0000 mon.a (mon.0) 1930 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:27.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[84576]: audit 2024-09-13T14:55:25.880124+0000 mon.a (mon.0) 1930 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:27.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[84576]: cluster 2024-09-13T14:55:25.884439+0000 mon.a (mon.0) 1931 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2024-09-13T14:55:27.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:26 smithi099 bash[84576]: cluster 2024-09-13T14:55:25.884439+0000 mon.a (mon.0) 1931 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2024-09-13T14:55:27.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:27.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:27.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:27.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:27.240 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:55:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:55:27] "GET /metrics HTTP/1.1" 200 36755 "" "Prometheus/2.51.0" 2024-09-13T14:55:27.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:27.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:27.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:26 smithi177 bash[68594]: cluster 2024-09-13T14:55:25.878667+0000 mon.a (mon.0) 1929 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:26 smithi177 bash[68594]: cluster 2024-09-13T14:55:25.878667+0000 mon.a (mon.0) 1929 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:26 smithi177 bash[68594]: audit 2024-09-13T14:55:25.880124+0000 mon.a (mon.0) 1930 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:26 smithi177 bash[68594]: audit 2024-09-13T14:55:25.880124+0000 mon.a (mon.0) 1930 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:26 smithi177 bash[68594]: cluster 2024-09-13T14:55:25.884439+0000 mon.a (mon.0) 1931 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2024-09-13T14:55:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:26 smithi177 bash[68594]: cluster 2024-09-13T14:55:25.884439+0000 mon.a (mon.0) 1931 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2024-09-13T14:55:27.379 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:27.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:27.484 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:27.521 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:27.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:27.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:27.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:27.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:27.751 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:27.770 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:27.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:27.875 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:27.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:27.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:27.914 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:28.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:28.011 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:27 smithi099 bash[87147]: cluster 2024-09-13T14:55:26.656994+0000 mgr.y (mgr.26252) 875 : cluster [DBG] pgmap v1393: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T14:55:28.011 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:27 smithi099 bash[87147]: cluster 2024-09-13T14:55:26.656994+0000 mgr.y (mgr.26252) 875 : cluster [DBG] pgmap v1393: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T14:55:28.014 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:27 smithi099 bash[84576]: cluster 2024-09-13T14:55:26.656994+0000 mgr.y (mgr.26252) 875 : cluster [DBG] pgmap v1393: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T14:55:28.014 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:27 smithi099 bash[84576]: cluster 2024-09-13T14:55:26.656994+0000 mgr.y (mgr.26252) 875 : cluster [DBG] pgmap v1393: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T14:55:28.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:28.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:28.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:28.149 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:27 smithi177 bash[68594]: cluster 2024-09-13T14:55:26.656994+0000 mgr.y (mgr.26252) 875 : cluster [DBG] pgmap v1393: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T14:55:28.149 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:27 smithi177 bash[68594]: cluster 2024-09-13T14:55:26.656994+0000 mgr.y (mgr.26252) 875 : cluster [DBG] pgmap v1393: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T14:55:28.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:28.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:28.294 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:28.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:28.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:28.430 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:28.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:28.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:28.557 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:28.597 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:28.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:28.635 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:28.701 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:28.732 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:28.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:28.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:55:28.837 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:28.877 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:28.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:29.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:29.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:29.027 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[84576]: audit 2024-09-13T14:55:28.029658+0000 mon.a (mon.0) 1932 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.027 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[84576]: audit 2024-09-13T14:55:28.029658+0000 mon.a (mon.0) 1932 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.027 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[84576]: audit 2024-09-13T14:55:28.037138+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:29 smithi177 bash[68594]: audit 2024-09-13T14:55:28.029658+0000 mon.a (mon.0) 1932 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:29 smithi177 bash[68594]: audit 2024-09-13T14:55:28.029658+0000 mon.a (mon.0) 1932 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:29 smithi177 bash[68594]: audit 2024-09-13T14:55:28.037138+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:29 smithi177 bash[68594]: audit 2024-09-13T14:55:28.037138+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:29 smithi177 bash[68594]: cluster 2024-09-13T14:55:28.657404+0000 mgr.y (mgr.26252) 876 : cluster [DBG] pgmap v1394: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:55:29.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:29 smithi177 bash[68594]: cluster 2024-09-13T14:55:28.657404+0000 mgr.y (mgr.26252) 876 : cluster [DBG] pgmap v1394: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:55:29.123 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:29.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:29.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:29.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:29.282 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:29.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[84576]: audit 2024-09-13T14:55:28.037138+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[84576]: cluster 2024-09-13T14:55:28.657404+0000 mgr.y (mgr.26252) 876 : cluster [DBG] pgmap v1394: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:55:29.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[84576]: cluster 2024-09-13T14:55:28.657404+0000 mgr.y (mgr.26252) 876 : cluster [DBG] pgmap v1394: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:55:29.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[87147]: audit 2024-09-13T14:55:28.029658+0000 mon.a (mon.0) 1932 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[87147]: audit 2024-09-13T14:55:28.029658+0000 mon.a (mon.0) 1932 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[87147]: audit 2024-09-13T14:55:28.037138+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[87147]: audit 2024-09-13T14:55:28.037138+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:29.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[87147]: cluster 2024-09-13T14:55:28.657404+0000 mgr.y (mgr.26252) 876 : cluster [DBG] pgmap v1394: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:55:29.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:29 smithi099 bash[87147]: cluster 2024-09-13T14:55:28.657404+0000 mgr.y (mgr.26252) 876 : cluster [DBG] pgmap v1394: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:55:29.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:29.435 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:29.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:29.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:55:29.539 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:29.673 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:29.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:29.813 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:29.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:29.916 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:29.939 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:29.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:30.045 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:30.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:30.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:30.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:30.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:30.313 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:30.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:30.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:55:30 smithi177 bash[61439]: ts=2024-09-13T14:55:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:30.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:30.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:30.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:30.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:30.580 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:30.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:30.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.713 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:30.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:30.945 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.965 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:31.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:31.103 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:31.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:31.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:31.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:31.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: audit 2024-09-13T14:55:30.095777+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: audit 2024-09-13T14:55:30.095777+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: audit 2024-09-13T14:55:30.103380+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: audit 2024-09-13T14:55:30.103380+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: cluster 2024-09-13T14:55:30.657920+0000 mgr.y (mgr.26252) 877 : cluster [DBG] pgmap v1395: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: cluster 2024-09-13T14:55:30.657920+0000 mgr.y (mgr.26252) 877 : cluster [DBG] pgmap v1395: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: audit 2024-09-13T14:55:30.839101+0000 mon.a (mon.0) 1936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: audit 2024-09-13T14:55:30.839101+0000 mon.a (mon.0) 1936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: audit 2024-09-13T14:55:30.840112+0000 mon.a (mon.0) 1937 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: audit 2024-09-13T14:55:30.840112+0000 mon.a (mon.0) 1937 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: audit 2024-09-13T14:55:30.847337+0000 mon.a (mon.0) 1938 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[84576]: audit 2024-09-13T14:55:30.847337+0000 mon.a (mon.0) 1938 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: audit 2024-09-13T14:55:30.095777+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: audit 2024-09-13T14:55:30.095777+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: audit 2024-09-13T14:55:30.103380+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: audit 2024-09-13T14:55:30.103380+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: cluster 2024-09-13T14:55:30.657920+0000 mgr.y (mgr.26252) 877 : cluster [DBG] pgmap v1395: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: cluster 2024-09-13T14:55:30.657920+0000 mgr.y (mgr.26252) 877 : cluster [DBG] pgmap v1395: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: audit 2024-09-13T14:55:30.839101+0000 mon.a (mon.0) 1936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: audit 2024-09-13T14:55:30.839101+0000 mon.a (mon.0) 1936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: audit 2024-09-13T14:55:30.840112+0000 mon.a (mon.0) 1937 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: audit 2024-09-13T14:55:30.840112+0000 mon.a (mon.0) 1937 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: audit 2024-09-13T14:55:30.847337+0000 mon.a (mon.0) 1938 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:31 smithi099 bash[87147]: audit 2024-09-13T14:55:30.847337+0000 mon.a (mon.0) 1938 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:31.351 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: audit 2024-09-13T14:55:30.095777+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: audit 2024-09-13T14:55:30.095777+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: audit 2024-09-13T14:55:30.103380+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: audit 2024-09-13T14:55:30.103380+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: cluster 2024-09-13T14:55:30.657920+0000 mgr.y (mgr.26252) 877 : cluster [DBG] pgmap v1395: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: cluster 2024-09-13T14:55:30.657920+0000 mgr.y (mgr.26252) 877 : cluster [DBG] pgmap v1395: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: audit 2024-09-13T14:55:30.839101+0000 mon.a (mon.0) 1936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: audit 2024-09-13T14:55:30.839101+0000 mon.a (mon.0) 1936 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: audit 2024-09-13T14:55:30.840112+0000 mon.a (mon.0) 1937 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: audit 2024-09-13T14:55:30.840112+0000 mon.a (mon.0) 1937 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: audit 2024-09-13T14:55:30.847337+0000 mon.a (mon.0) 1938 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.352 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:31 smithi177 bash[68594]: audit 2024-09-13T14:55:30.847337+0000 mon.a (mon.0) 1938 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:31.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:31.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:31.492 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:31.519 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:31.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:31.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:31.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:31.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:31.775 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:31.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:31.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:31.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:31.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:32.025 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:32.049 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:32.155 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:32.289 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:32.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:32.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:32.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:32.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:32.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:32.696 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:32.736 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:32.842 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:32.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:32.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:32.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:32.993 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:32.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:33.096 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:33.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:33.234 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:33.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:33.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:33.369 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:33.394 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:33.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:33.497 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:33.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:33.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:33.584 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:33.584 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:33.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:33.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:33.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:33.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:33.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:33 smithi177 bash[68594]: cluster 2024-09-13T14:55:32.658367+0000 mgr.y (mgr.26252) 878 : cluster [DBG] pgmap v1396: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2024-09-13T14:55:33.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:33 smithi177 bash[68594]: cluster 2024-09-13T14:55:32.658367+0000 mgr.y (mgr.26252) 878 : cluster [DBG] pgmap v1396: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2024-09-13T14:55:33.793 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:33 smithi177 bash[68594]: audit 2024-09-13T14:55:33.289205+0000 mon.a (mon.0) 1939 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:33.793 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:33 smithi177 bash[68594]: audit 2024-09-13T14:55:33.289205+0000 mon.a (mon.0) 1939 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:33.793 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:33 smithi177 bash[68594]: audit 2024-09-13T14:55:33.290444+0000 mon.a (mon.0) 1940 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:33.793 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:33 smithi177 bash[68594]: audit 2024-09-13T14:55:33.290444+0000 mon.a (mon.0) 1940 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:33.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:33.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:33.922 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:33.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:33.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:34.046 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:34.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[84576]: cluster 2024-09-13T14:55:32.658367+0000 mgr.y (mgr.26252) 878 : cluster [DBG] pgmap v1396: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2024-09-13T14:55:34.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[84576]: cluster 2024-09-13T14:55:32.658367+0000 mgr.y (mgr.26252) 878 : cluster [DBG] pgmap v1396: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2024-09-13T14:55:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[84576]: audit 2024-09-13T14:55:33.289205+0000 mon.a (mon.0) 1939 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[84576]: audit 2024-09-13T14:55:33.289205+0000 mon.a (mon.0) 1939 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[84576]: audit 2024-09-13T14:55:33.290444+0000 mon.a (mon.0) 1940 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:34.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[84576]: audit 2024-09-13T14:55:33.290444+0000 mon.a (mon.0) 1940 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[87147]: cluster 2024-09-13T14:55:32.658367+0000 mgr.y (mgr.26252) 878 : cluster [DBG] pgmap v1396: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2024-09-13T14:55:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[87147]: cluster 2024-09-13T14:55:32.658367+0000 mgr.y (mgr.26252) 878 : cluster [DBG] pgmap v1396: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2024-09-13T14:55:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[87147]: audit 2024-09-13T14:55:33.289205+0000 mon.a (mon.0) 1939 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[87147]: audit 2024-09-13T14:55:33.289205+0000 mon.a (mon.0) 1939 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:55:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[87147]: audit 2024-09-13T14:55:33.290444+0000 mon.a (mon.0) 1940 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:34.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:33 smithi099 bash[87147]: audit 2024-09-13T14:55:33.290444+0000 mon.a (mon.0) 1940 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:34.082 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:34.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:34.189 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:34.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:34.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:34.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:34.344 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:34.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:34.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:34.478 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:34.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:34.585 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:34.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:34.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:34.715 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:34.723 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:55:34.745 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:34.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:34.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:34.863 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:34 smithi099 bash[84576]: audit 2024-09-13T14:55:33.798350+0000 mon.a (mon.0) 1941 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:34.863 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:34 smithi099 bash[84576]: audit 2024-09-13T14:55:33.798350+0000 mon.a (mon.0) 1941 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:34.863 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:34 smithi099 bash[87147]: audit 2024-09-13T14:55:33.798350+0000 mon.a (mon.0) 1941 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:34.863 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:34 smithi099 bash[87147]: audit 2024-09-13T14:55:33.798350+0000 mon.a (mon.0) 1941 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:34.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:34.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:34.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:35.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:35.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:35.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:34 smithi177 bash[68594]: audit 2024-09-13T14:55:33.798350+0000 mon.a (mon.0) 1941 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:35.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:34 smithi177 bash[68594]: audit 2024-09-13T14:55:33.798350+0000 mon.a (mon.0) 1941 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:55:35.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:35.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:35.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:35.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:35.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:35.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:35.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:35.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:35.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:35.641 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:35.641 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:35.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:35.672 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:35.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:35.775 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:35.794 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:35.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:35.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:35.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:35 smithi177 bash[68594]: cluster 2024-09-13T14:55:34.661265+0000 mgr.y (mgr.26252) 879 : cluster [DBG] pgmap v1397: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T14:55:35.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:35 smithi177 bash[68594]: cluster 2024-09-13T14:55:34.661265+0000 mgr.y (mgr.26252) 879 : cluster [DBG] pgmap v1397: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T14:55:35.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:35 smithi177 bash[68594]: audit 2024-09-13T14:55:34.726540+0000 mon.a (mon.0) 1942 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:55:35.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:35 smithi177 bash[68594]: audit 2024-09-13T14:55:34.726540+0000 mon.a (mon.0) 1942 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:55:35.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:35 smithi177 bash[68594]: cluster 2024-09-13T14:55:34.730885+0000 mon.a (mon.0) 1943 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2024-09-13T14:55:35.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:35 smithi177 bash[68594]: cluster 2024-09-13T14:55:34.730885+0000 mon.a (mon.0) 1943 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2024-09-13T14:55:35.921 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:35.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:36.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:36.049 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:36.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:36.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[84576]: cluster 2024-09-13T14:55:34.661265+0000 mgr.y (mgr.26252) 879 : cluster [DBG] pgmap v1397: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T14:55:36.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[84576]: cluster 2024-09-13T14:55:34.661265+0000 mgr.y (mgr.26252) 879 : cluster [DBG] pgmap v1397: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T14:55:36.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[84576]: audit 2024-09-13T14:55:34.726540+0000 mon.a (mon.0) 1942 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:55:36.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[84576]: audit 2024-09-13T14:55:34.726540+0000 mon.a (mon.0) 1942 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:55:36.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[84576]: cluster 2024-09-13T14:55:34.730885+0000 mon.a (mon.0) 1943 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2024-09-13T14:55:36.096 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[84576]: cluster 2024-09-13T14:55:34.730885+0000 mon.a (mon.0) 1943 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2024-09-13T14:55:36.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[87147]: cluster 2024-09-13T14:55:34.661265+0000 mgr.y (mgr.26252) 879 : cluster [DBG] pgmap v1397: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T14:55:36.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[87147]: cluster 2024-09-13T14:55:34.661265+0000 mgr.y (mgr.26252) 879 : cluster [DBG] pgmap v1397: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T14:55:36.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[87147]: audit 2024-09-13T14:55:34.726540+0000 mon.a (mon.0) 1942 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:55:36.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[87147]: audit 2024-09-13T14:55:34.726540+0000 mon.a (mon.0) 1942 : audit [INF] from='client.? 172.21.15.99:0/1768961046' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:55:36.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[87147]: cluster 2024-09-13T14:55:34.730885+0000 mon.a (mon.0) 1943 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2024-09-13T14:55:36.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:35 smithi099 bash[87147]: cluster 2024-09-13T14:55:34.730885+0000 mon.a (mon.0) 1943 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2024-09-13T14:55:36.153 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:36.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:36.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:36.290 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:36.314 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:36.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:36.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:36.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:36.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:36.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:36.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:36.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:36.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:55:36.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:36.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:36.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:37.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:37.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:37.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:37.168 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:37.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:37.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:37.299 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:37.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:55:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:55:37] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-13T14:55:37.344 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:37.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:37.433 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:55:37.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:37.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:37.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:37.575 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:37.626 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:37.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:37.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:37.754 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:37 smithi099 bash[87147]: cluster 2024-09-13T14:55:36.664517+0000 mgr.y (mgr.26252) 880 : cluster [DBG] pgmap v1399: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-09-13T14:55:37.754 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:37 smithi099 bash[87147]: cluster 2024-09-13T14:55:36.664517+0000 mgr.y (mgr.26252) 880 : cluster [DBG] pgmap v1399: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-09-13T14:55:37.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:37.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:37.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:37.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:37.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:38.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:37 smithi099 bash[84576]: cluster 2024-09-13T14:55:36.664517+0000 mgr.y (mgr.26252) 880 : cluster [DBG] pgmap v1399: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-09-13T14:55:38.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:37 smithi099 bash[84576]: cluster 2024-09-13T14:55:36.664517+0000 mgr.y (mgr.26252) 880 : cluster [DBG] pgmap v1399: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-09-13T14:55:38.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:38.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:37 smithi177 bash[68594]: cluster 2024-09-13T14:55:36.664517+0000 mgr.y (mgr.26252) 880 : cluster [DBG] pgmap v1399: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-09-13T14:55:38.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:37 smithi177 bash[68594]: cluster 2024-09-13T14:55:36.664517+0000 mgr.y (mgr.26252) 880 : cluster [DBG] pgmap v1399: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-09-13T14:55:38.046 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:38.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:38.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:38.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:38.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:38.325 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:38.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:38.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:38.459 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:38.484 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:38.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:38.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:38.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:38.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:38.738 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:38.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:38.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:38.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:38.973 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:38.998 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:39.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:39.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:39.122 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:39.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:39.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:39.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:39.375 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:39.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:39.509 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:39.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:39.617 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:39.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:39.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:39.740 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:39.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:39.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:39 smithi099 bash[84576]: cluster 2024-09-13T14:55:38.665167+0000 mgr.y (mgr.26252) 881 : cluster [DBG] pgmap v1400: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:55:39.821 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:39 smithi099 bash[84576]: cluster 2024-09-13T14:55:38.665167+0000 mgr.y (mgr.26252) 881 : cluster [DBG] pgmap v1400: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:55:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:39 smithi099 bash[87147]: cluster 2024-09-13T14:55:38.665167+0000 mgr.y (mgr.26252) 881 : cluster [DBG] pgmap v1400: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:55:39.822 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:39 smithi099 bash[87147]: cluster 2024-09-13T14:55:38.665167+0000 mgr.y (mgr.26252) 881 : cluster [DBG] pgmap v1400: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:55:39.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:39.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:39.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:40.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:40.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:39 smithi177 bash[68594]: cluster 2024-09-13T14:55:38.665167+0000 mgr.y (mgr.26252) 881 : cluster [DBG] pgmap v1400: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:55:40.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:39 smithi177 bash[68594]: cluster 2024-09-13T14:55:38.665167+0000 mgr.y (mgr.26252) 881 : cluster [DBG] pgmap v1400: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:55:40.027 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:40.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:40.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:40.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:40.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:40.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:40.317 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:40.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:40.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:55:40 smithi177 bash[61439]: ts=2024-09-13T14:55:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:40.425 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:40.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:40.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:40.577 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:40.599 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:40.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:40.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:40.732 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:40.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:40.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:41.022 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:41.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:41.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:41.162 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:41.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:41.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:41.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:41.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:41.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:41.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:41.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:41.533 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:41.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:41.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:41.669 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:41.699 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:41.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:41.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:41.816 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:41 smithi177 bash[68594]: cluster 2024-09-13T14:55:40.668975+0000 mgr.y (mgr.26252) 882 : cluster [DBG] pgmap v1401: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-13T14:55:41.816 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:41 smithi177 bash[68594]: cluster 2024-09-13T14:55:40.668975+0000 mgr.y (mgr.26252) 882 : cluster [DBG] pgmap v1401: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-13T14:55:41.835 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:41.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:41.980 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:41.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:42.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:41 smithi099 bash[84576]: cluster 2024-09-13T14:55:40.668975+0000 mgr.y (mgr.26252) 882 : cluster [DBG] pgmap v1401: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-13T14:55:42.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:41 smithi099 bash[84576]: cluster 2024-09-13T14:55:40.668975+0000 mgr.y (mgr.26252) 882 : cluster [DBG] pgmap v1401: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-13T14:55:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:41 smithi099 bash[87147]: cluster 2024-09-13T14:55:40.668975+0000 mgr.y (mgr.26252) 882 : cluster [DBG] pgmap v1401: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-13T14:55:42.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:41 smithi099 bash[87147]: cluster 2024-09-13T14:55:40.668975+0000 mgr.y (mgr.26252) 882 : cluster [DBG] pgmap v1401: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-13T14:55:42.085 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:42.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:42.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:42.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:42.257 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:42.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:42.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:42.402 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:42.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:42.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:42.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:42.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:42.692 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:42.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:42.796 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:42.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:42.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:42.927 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:42.955 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:43.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:43.097 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:43.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:43.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:43.240 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:43.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:43.319 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:55:43.320 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:43.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:43.380 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:55:43.380 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:43.380 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:43.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:43.419 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:55:43.419 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:43.457 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:55:43.458 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:43.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:43.512 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:55:43.513 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:43.518 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:43.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:43.547 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:55:43.547 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:43.579 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:55:43.579 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:43.604 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:55:43.604 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:43.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:43.624 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:43.626 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:43.656 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:43.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:43.761 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:43.765 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T14:55:43.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:43.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:43.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:43.901 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[84576]: cluster 2024-09-13T14:55:42.669441+0000 mgr.y (mgr.26252) 883 : cluster [DBG] pgmap v1402: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:55:43.902 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[84576]: cluster 2024-09-13T14:55:42.669441+0000 mgr.y (mgr.26252) 883 : cluster [DBG] pgmap v1402: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:55:43.902 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[84576]: audit 2024-09-13T14:55:43.290513+0000 mon.a (mon.0) 1944 : audit [DBG] from='client.? 172.21.15.99:0/4054681000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:43.902 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[84576]: audit 2024-09-13T14:55:43.290513+0000 mon.a (mon.0) 1944 : audit [DBG] from='client.? 172.21.15.99:0/4054681000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:43.902 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[84576]: audit 2024-09-13T14:55:43.645082+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.99:0/4194427764' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.902 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[84576]: audit 2024-09-13T14:55:43.645082+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.99:0/4194427764' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.902 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[84576]: audit 2024-09-13T14:55:43.646265+0000 mon.a (mon.0) 1945 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.902 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[84576]: audit 2024-09-13T14:55:43.646265+0000 mon.a (mon.0) 1945 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.902 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[87147]: cluster 2024-09-13T14:55:42.669441+0000 mgr.y (mgr.26252) 883 : cluster [DBG] pgmap v1402: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:55:43.902 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[87147]: cluster 2024-09-13T14:55:42.669441+0000 mgr.y (mgr.26252) 883 : cluster [DBG] pgmap v1402: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:55:43.903 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[87147]: audit 2024-09-13T14:55:43.290513+0000 mon.a (mon.0) 1944 : audit [DBG] from='client.? 172.21.15.99:0/4054681000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:43.903 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[87147]: audit 2024-09-13T14:55:43.290513+0000 mon.a (mon.0) 1944 : audit [DBG] from='client.? 172.21.15.99:0/4054681000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:43.903 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[87147]: audit 2024-09-13T14:55:43.645082+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.99:0/4194427764' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.903 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[87147]: audit 2024-09-13T14:55:43.645082+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.99:0/4194427764' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.903 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[87147]: audit 2024-09-13T14:55:43.646265+0000 mon.a (mon.0) 1945 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.903 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:43 smithi099 bash[87147]: audit 2024-09-13T14:55:43.646265+0000 mon.a (mon.0) 1945 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:43.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:43.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:44.025 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:44.048 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:44.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:44.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:43 smithi177 bash[68594]: cluster 2024-09-13T14:55:42.669441+0000 mgr.y (mgr.26252) 883 : cluster [DBG] pgmap v1402: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:55:44.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:43 smithi177 bash[68594]: cluster 2024-09-13T14:55:42.669441+0000 mgr.y (mgr.26252) 883 : cluster [DBG] pgmap v1402: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:55:44.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:43 smithi177 bash[68594]: audit 2024-09-13T14:55:43.290513+0000 mon.a (mon.0) 1944 : audit [DBG] from='client.? 172.21.15.99:0/4054681000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:44.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:43 smithi177 bash[68594]: audit 2024-09-13T14:55:43.290513+0000 mon.a (mon.0) 1944 : audit [DBG] from='client.? 172.21.15.99:0/4054681000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:44.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:43 smithi177 bash[68594]: audit 2024-09-13T14:55:43.645082+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.99:0/4194427764' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:44.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:43 smithi177 bash[68594]: audit 2024-09-13T14:55:43.645082+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.99:0/4194427764' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:44.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:43 smithi177 bash[68594]: audit 2024-09-13T14:55:43.646265+0000 mon.a (mon.0) 1945 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:44.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:43 smithi177 bash[68594]: audit 2024-09-13T14:55:43.646265+0000 mon.a (mon.0) 1945 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:44.153 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:44.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:44.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:44.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:44.319 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:44.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:44.425 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:44.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:44.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:44.581 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:44.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:44.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:44.725 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:44.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:44.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:44.871 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:44.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:44.974 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:44.985 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[84576]: cluster 2024-09-13T14:55:43.758479+0000 mon.a (mon.0) 1946 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:44.985 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[84576]: cluster 2024-09-13T14:55:43.758479+0000 mon.a (mon.0) 1946 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:44.985 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[84576]: audit 2024-09-13T14:55:43.760157+0000 mon.a (mon.0) 1947 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:44.985 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[84576]: audit 2024-09-13T14:55:43.760157+0000 mon.a (mon.0) 1947 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:44.985 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[84576]: cluster 2024-09-13T14:55:43.764830+0000 mon.a (mon.0) 1948 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2024-09-13T14:55:44.985 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[84576]: cluster 2024-09-13T14:55:43.764830+0000 mon.a (mon.0) 1948 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2024-09-13T14:55:44.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[87147]: cluster 2024-09-13T14:55:43.758479+0000 mon.a (mon.0) 1946 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:44.986 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[87147]: cluster 2024-09-13T14:55:43.758479+0000 mon.a (mon.0) 1946 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:44.986 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[87147]: audit 2024-09-13T14:55:43.760157+0000 mon.a (mon.0) 1947 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:44.986 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[87147]: audit 2024-09-13T14:55:43.760157+0000 mon.a (mon.0) 1947 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:44.986 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[87147]: cluster 2024-09-13T14:55:43.764830+0000 mon.a (mon.0) 1948 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2024-09-13T14:55:44.986 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:44 smithi099 bash[87147]: cluster 2024-09-13T14:55:43.764830+0000 mon.a (mon.0) 1948 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2024-09-13T14:55:44.996 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:45.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:45.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:45.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:44 smithi177 bash[68594]: cluster 2024-09-13T14:55:43.758479+0000 mon.a (mon.0) 1946 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:45.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:44 smithi177 bash[68594]: cluster 2024-09-13T14:55:43.758479+0000 mon.a (mon.0) 1946 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:45.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:44 smithi177 bash[68594]: audit 2024-09-13T14:55:43.760157+0000 mon.a (mon.0) 1947 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:45.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:44 smithi177 bash[68594]: audit 2024-09-13T14:55:43.760157+0000 mon.a (mon.0) 1947 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:45.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:44 smithi177 bash[68594]: cluster 2024-09-13T14:55:43.764830+0000 mon.a (mon.0) 1948 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2024-09-13T14:55:45.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:44 smithi177 bash[68594]: cluster 2024-09-13T14:55:43.764830+0000 mon.a (mon.0) 1948 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2024-09-13T14:55:45.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:45.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:45.261 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:45.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:45.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:45.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:45.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:45.494 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:45.522 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:45.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:45.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:45.648 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:45.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:45.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:45.770 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:45.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:45.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:45.917 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:45.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:46.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:46.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:45 smithi177 bash[68594]: cluster 2024-09-13T14:55:44.669963+0000 mgr.y (mgr.26252) 884 : cluster [DBG] pgmap v1404: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:46.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:45 smithi177 bash[68594]: cluster 2024-09-13T14:55:44.669963+0000 mgr.y (mgr.26252) 884 : cluster [DBG] pgmap v1404: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:46.056 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:46.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:46.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:45 smithi099 bash[87147]: cluster 2024-09-13T14:55:44.669963+0000 mgr.y (mgr.26252) 884 : cluster [DBG] pgmap v1404: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:46.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:45 smithi099 bash[87147]: cluster 2024-09-13T14:55:44.669963+0000 mgr.y (mgr.26252) 884 : cluster [DBG] pgmap v1404: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:46.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:45 smithi099 bash[84576]: cluster 2024-09-13T14:55:44.669963+0000 mgr.y (mgr.26252) 884 : cluster [DBG] pgmap v1404: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:46.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:45 smithi099 bash[84576]: cluster 2024-09-13T14:55:44.669963+0000 mgr.y (mgr.26252) 884 : cluster [DBG] pgmap v1404: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:46.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:46.184 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:46.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:46.294 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:46.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:46.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:46.424 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:46.446 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:46.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:46.548 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:46.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:46.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:46.683 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:46.726 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:46.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:46.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:46.851 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:46.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:46.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:46.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:46.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:47.100 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:47.127 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:47.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:47.235 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:47.243 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:55:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:55:47] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-13T14:55:47.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:47.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:47.377 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:47.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:47.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:47.502 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:47.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:47.641 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:47.662 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:47.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:47.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:47.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:47.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:47.932 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:48.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:48.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:47 smithi099 bash[84576]: cluster 2024-09-13T14:55:46.670418+0000 mgr.y (mgr.26252) 885 : cluster [DBG] pgmap v1405: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T14:55:48.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:47 smithi099 bash[84576]: cluster 2024-09-13T14:55:46.670418+0000 mgr.y (mgr.26252) 885 : cluster [DBG] pgmap v1405: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T14:55:48.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:47 smithi099 bash[87147]: cluster 2024-09-13T14:55:46.670418+0000 mgr.y (mgr.26252) 885 : cluster [DBG] pgmap v1405: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T14:55:48.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:47 smithi099 bash[87147]: cluster 2024-09-13T14:55:46.670418+0000 mgr.y (mgr.26252) 885 : cluster [DBG] pgmap v1405: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T14:55:48.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:48.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:48.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:47 smithi177 bash[68594]: cluster 2024-09-13T14:55:46.670418+0000 mgr.y (mgr.26252) 885 : cluster [DBG] pgmap v1405: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T14:55:48.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:47 smithi177 bash[68594]: cluster 2024-09-13T14:55:46.670418+0000 mgr.y (mgr.26252) 885 : cluster [DBG] pgmap v1405: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T14:55:48.171 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:48.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:48.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:48.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:48.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:48.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:48.435 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:48.464 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:48.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:48.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:48.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:48.743 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:48.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:48.849 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:48.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:48 smithi177 bash[68594]: audit 2024-09-13T14:55:48.283453+0000 mon.a (mon.0) 1949 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:48.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:48 smithi177 bash[68594]: audit 2024-09-13T14:55:48.283453+0000 mon.a (mon.0) 1949 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:48.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:48.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:49.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:48 smithi099 bash[84576]: audit 2024-09-13T14:55:48.283453+0000 mon.a (mon.0) 1949 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:49.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:48 smithi099 bash[84576]: audit 2024-09-13T14:55:48.283453+0000 mon.a (mon.0) 1949 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:49.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:48 smithi099 bash[87147]: audit 2024-09-13T14:55:48.283453+0000 mon.a (mon.0) 1949 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:49.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:48 smithi099 bash[87147]: audit 2024-09-13T14:55:48.283453+0000 mon.a (mon.0) 1949 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:49.129 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.162 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:49.265 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:49.284 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:49.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:49.388 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:49.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:49.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:49.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:49.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:49.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:49.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:49.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.938 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:49.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:50.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:50.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:49 smithi099 bash[84576]: cluster 2024-09-13T14:55:48.670821+0000 mgr.y (mgr.26252) 886 : cluster [DBG] pgmap v1406: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:50.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:49 smithi099 bash[84576]: cluster 2024-09-13T14:55:48.670821+0000 mgr.y (mgr.26252) 886 : cluster [DBG] pgmap v1406: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:50.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:49 smithi099 bash[87147]: cluster 2024-09-13T14:55:48.670821+0000 mgr.y (mgr.26252) 886 : cluster [DBG] pgmap v1406: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:50.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:49 smithi099 bash[87147]: cluster 2024-09-13T14:55:48.670821+0000 mgr.y (mgr.26252) 886 : cluster [DBG] pgmap v1406: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:50.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:50.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:50.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:49 smithi177 bash[68594]: cluster 2024-09-13T14:55:48.670821+0000 mgr.y (mgr.26252) 886 : cluster [DBG] pgmap v1406: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:50.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:49 smithi177 bash[68594]: cluster 2024-09-13T14:55:48.670821+0000 mgr.y (mgr.26252) 886 : cluster [DBG] pgmap v1406: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:55:50.175 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:50.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:50.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:50.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:55:50.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:50.335 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:50.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:50.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:50.467 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:50.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:50.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:50.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:50.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:55:50 smithi177 bash[61439]: ts=2024-09-13T14:55:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:55:50.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:55:50.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:50.732 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:50.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:50.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:50.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:50.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:51.007 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:51.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:51.111 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:51.132 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:51.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:51.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:51.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:51.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:51.374 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:51.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:51.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:51.505 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:51.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:51.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:51.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:51.687 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:51.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:51.793 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:51.824 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:51.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:51.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:51.962 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:51.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:52.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:52.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:51 smithi099 bash[84576]: cluster 2024-09-13T14:55:50.671460+0000 mgr.y (mgr.26252) 887 : cluster [DBG] pgmap v1407: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:52.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:51 smithi099 bash[84576]: cluster 2024-09-13T14:55:50.671460+0000 mgr.y (mgr.26252) 887 : cluster [DBG] pgmap v1407: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:52.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:51 smithi099 bash[87147]: cluster 2024-09-13T14:55:50.671460+0000 mgr.y (mgr.26252) 887 : cluster [DBG] pgmap v1407: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:52.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:51 smithi099 bash[87147]: cluster 2024-09-13T14:55:50.671460+0000 mgr.y (mgr.26252) 887 : cluster [DBG] pgmap v1407: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:52.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:51 smithi177 bash[68594]: cluster 2024-09-13T14:55:50.671460+0000 mgr.y (mgr.26252) 887 : cluster [DBG] pgmap v1407: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:52.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:51 smithi177 bash[68594]: cluster 2024-09-13T14:55:50.671460+0000 mgr.y (mgr.26252) 887 : cluster [DBG] pgmap v1407: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:52.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:52.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:52.202 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:52.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:52.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:52.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:52.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:52.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:52.465 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:52.497 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:52.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:52.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:52.622 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:52.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:52.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:52.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:52.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:52.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:52.913 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:52.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:53.017 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:53.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:53.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:53.142 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:53.179 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:53.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:53.285 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:53.323 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:53.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:53.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:53.446 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:53.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:53.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:53.584 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:53.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:53.717 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:53.819 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:53.848 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:53.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:53.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:53 smithi177 bash[68594]: cluster 2024-09-13T14:55:52.672000+0000 mgr.y (mgr.26252) 888 : cluster [DBG] pgmap v1408: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:53.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:53 smithi177 bash[68594]: cluster 2024-09-13T14:55:52.672000+0000 mgr.y (mgr.26252) 888 : cluster [DBG] pgmap v1408: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:53.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:53.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:53.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:54.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:53 smithi099 bash[84576]: cluster 2024-09-13T14:55:52.672000+0000 mgr.y (mgr.26252) 888 : cluster [DBG] pgmap v1408: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:54.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:53 smithi099 bash[84576]: cluster 2024-09-13T14:55:52.672000+0000 mgr.y (mgr.26252) 888 : cluster [DBG] pgmap v1408: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:54.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:53 smithi099 bash[87147]: cluster 2024-09-13T14:55:52.672000+0000 mgr.y (mgr.26252) 888 : cluster [DBG] pgmap v1408: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:54.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:53 smithi099 bash[87147]: cluster 2024-09-13T14:55:52.672000+0000 mgr.y (mgr.26252) 888 : cluster [DBG] pgmap v1408: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T14:55:54.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:54.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:54.235 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:54.258 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:54.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:54.368 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:54.408 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:54.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:54.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.547 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:54.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:54.648 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:54.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:54.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:54.741 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:55:54.741 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T14:55:54.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:55:54.741 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45350.0:150960 42.2 e1027/1009 42:456bb65b:::benchmark_data_smithi099_148651_object43120:head v 1027'9338, mlcod=1027'9336)", 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:55:54.733605+0000", 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0095058030000000002, 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095285360000000006, 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:55:54.733605+0000", 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:55:54.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:55:54.733605+0000", 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:55:54.733606+0000", 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0380000000000001e-06 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:55:54.733613+0000", 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.148e-06 2024-09-13T14:55:54.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:55:54.733625+0000", 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2679e-05 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:55:54.733634+0000", 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3259999999999996e-06 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:55:54.733750+0000", 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116674 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:55:54.733769+0000", 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9079000000000001e-05 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 1 2024-09-13T14:55:54.745 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:55:54.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:54.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:54.901 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:54.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:54.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:55.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:55.074 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:55.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:55.128 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:55.128 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:55.185 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:55.207 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:55.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:55.333 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:55.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:55.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:55.465 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:55.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:55.572 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:55.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:55.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:55.720 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:55.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:55.817 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:55:55.825 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:55.868 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:55.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:55.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:55.984 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:55 smithi099 bash[84576]: cluster 2024-09-13T14:55:54.672726+0000 mgr.y (mgr.26252) 889 : cluster [DBG] pgmap v1409: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:55:55.984 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:55 smithi099 bash[84576]: cluster 2024-09-13T14:55:54.672726+0000 mgr.y (mgr.26252) 889 : cluster [DBG] pgmap v1409: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:55:55.984 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:55 smithi099 bash[84576]: audit 2024-09-13T14:55:55.039816+0000 mon.a (mon.0) 1950 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:55:55.984 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:55 smithi099 bash[84576]: audit 2024-09-13T14:55:55.039816+0000 mon.a (mon.0) 1950 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:55:55.984 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:55 smithi099 bash[87147]: cluster 2024-09-13T14:55:54.672726+0000 mgr.y (mgr.26252) 889 : cluster [DBG] pgmap v1409: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:55:55.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:55 smithi099 bash[87147]: cluster 2024-09-13T14:55:54.672726+0000 mgr.y (mgr.26252) 889 : cluster [DBG] pgmap v1409: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:55:55.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:55 smithi099 bash[87147]: audit 2024-09-13T14:55:55.039816+0000 mon.a (mon.0) 1950 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:55:55.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:55 smithi099 bash[87147]: audit 2024-09-13T14:55:55.039816+0000 mon.a (mon.0) 1950 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:55:56.001 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:56.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:56.104 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:56.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:55 smithi177 bash[68594]: cluster 2024-09-13T14:55:54.672726+0000 mgr.y (mgr.26252) 889 : cluster [DBG] pgmap v1409: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:55:56.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:55 smithi177 bash[68594]: cluster 2024-09-13T14:55:54.672726+0000 mgr.y (mgr.26252) 889 : cluster [DBG] pgmap v1409: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:55:56.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:55 smithi177 bash[68594]: audit 2024-09-13T14:55:55.039816+0000 mon.a (mon.0) 1950 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:55:56.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:55 smithi177 bash[68594]: audit 2024-09-13T14:55:55.039816+0000 mon.a (mon.0) 1950 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:55:56.121 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:56.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:56.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:56.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:56.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:56.493 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:56.620 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:56.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:56.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:56.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:56.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:56.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:56.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:55:56.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:57.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:57.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:56 smithi177 bash[68594]: audit 2024-09-13T14:55:55.817784+0000 mon.a (mon.0) 1951 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:55:57.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:56 smithi177 bash[68594]: audit 2024-09-13T14:55:55.817784+0000 mon.a (mon.0) 1951 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:55:57.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:56 smithi177 bash[68594]: cluster 2024-09-13T14:55:55.820730+0000 mon.a (mon.0) 1952 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2024-09-13T14:55:57.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:56 smithi177 bash[68594]: cluster 2024-09-13T14:55:55.820730+0000 mon.a (mon.0) 1952 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2024-09-13T14:55:57.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:57.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:57.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:57.147 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:56 smithi099 bash[84576]: audit 2024-09-13T14:55:55.817784+0000 mon.a (mon.0) 1951 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:55:57.147 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:56 smithi099 bash[84576]: audit 2024-09-13T14:55:55.817784+0000 mon.a (mon.0) 1951 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:55:57.147 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:56 smithi099 bash[84576]: cluster 2024-09-13T14:55:55.820730+0000 mon.a (mon.0) 1952 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2024-09-13T14:55:57.147 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:56 smithi099 bash[84576]: cluster 2024-09-13T14:55:55.820730+0000 mon.a (mon.0) 1952 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2024-09-13T14:55:57.147 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:55:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:55:57] "GET /metrics HTTP/1.1" 200 36757 "" "Prometheus/2.51.0" 2024-09-13T14:55:57.147 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:56 smithi099 bash[87147]: audit 2024-09-13T14:55:55.817784+0000 mon.a (mon.0) 1951 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:55:57.147 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:56 smithi099 bash[87147]: audit 2024-09-13T14:55:55.817784+0000 mon.a (mon.0) 1951 : audit [INF] from='client.? 172.21.15.99:0/2467143298' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:55:57.148 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:56 smithi099 bash[87147]: cluster 2024-09-13T14:55:55.820730+0000 mon.a (mon.0) 1952 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2024-09-13T14:55:57.148 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:56 smithi099 bash[87147]: cluster 2024-09-13T14:55:55.820730+0000 mon.a (mon.0) 1952 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2024-09-13T14:55:57.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:57.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:57.273 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:57.304 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:57.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:57.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:57.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:57.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:57.546 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:57.597 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:57.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:57.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:57.725 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:57.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:57.828 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:57.858 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:57.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:57.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:57.985 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:57.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:58.092 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:58.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:57 smithi177 bash[68594]: cluster 2024-09-13T14:55:56.673237+0000 mgr.y (mgr.26252) 890 : cluster [DBG] pgmap v1411: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:55:58.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:57 smithi177 bash[68594]: cluster 2024-09-13T14:55:56.673237+0000 mgr.y (mgr.26252) 890 : cluster [DBG] pgmap v1411: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:55:58.116 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:58.120 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:57 smithi099 bash[84576]: cluster 2024-09-13T14:55:56.673237+0000 mgr.y (mgr.26252) 890 : cluster [DBG] pgmap v1411: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:55:58.120 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:57 smithi099 bash[84576]: cluster 2024-09-13T14:55:56.673237+0000 mgr.y (mgr.26252) 890 : cluster [DBG] pgmap v1411: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:55:58.120 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:57 smithi099 bash[87147]: cluster 2024-09-13T14:55:56.673237+0000 mgr.y (mgr.26252) 890 : cluster [DBG] pgmap v1411: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:55:58.120 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:57 smithi099 bash[87147]: cluster 2024-09-13T14:55:56.673237+0000 mgr.y (mgr.26252) 890 : cluster [DBG] pgmap v1411: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:55:58.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:58.222 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:58.253 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:58.359 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:58.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:58.493 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:58.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:58.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:58.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:58.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:58.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:58.698 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:55:58.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:55:58.782 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:58.811 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:58.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:58.944 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:58.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:59.046 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:59.068 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:59.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:59.210 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:59.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:59.313 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:59.353 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:59.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:59.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:59.497 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:59.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:59.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:59.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:59.726 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:59.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:59.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:59.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:59.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:59.999 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:00.009 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:59 smithi099 bash[84576]: cluster 2024-09-13T14:55:58.673656+0000 mgr.y (mgr.26252) 891 : cluster [DBG] pgmap v1412: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:56:00.009 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:55:59 smithi099 bash[84576]: cluster 2024-09-13T14:55:58.673656+0000 mgr.y (mgr.26252) 891 : cluster [DBG] pgmap v1412: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:56:00.009 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:59 smithi099 bash[87147]: cluster 2024-09-13T14:55:58.673656+0000 mgr.y (mgr.26252) 891 : cluster [DBG] pgmap v1412: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:56:00.009 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:55:59 smithi099 bash[87147]: cluster 2024-09-13T14:55:58.673656+0000 mgr.y (mgr.26252) 891 : cluster [DBG] pgmap v1412: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:56:00.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:00.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:59 smithi177 bash[68594]: cluster 2024-09-13T14:55:58.673656+0000 mgr.y (mgr.26252) 891 : cluster [DBG] pgmap v1412: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:56:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:55:59 smithi177 bash[68594]: cluster 2024-09-13T14:55:58.673656+0000 mgr.y (mgr.26252) 891 : cluster [DBG] pgmap v1412: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T14:56:00.136 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:00.173 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:00.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:00.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:00.301 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:00.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:00.408 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:00.418 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:56:00 smithi177 bash[61439]: ts=2024-09-13T14:56:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:00.452 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:00.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:00.555 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:00.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.579 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:00.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:00.680 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:00.699 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:00.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:00.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:00.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:00.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:00.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:00.947 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:00.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:01.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:01.081 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:01.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:01.184 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:01.225 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:01.330 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:01.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:01.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:01.471 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:01.495 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:01.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:01.599 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:01.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:01.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:01.734 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:01.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:01.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:01.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:01.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:01.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:02.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:02.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:02.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:02.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:01 smithi099 bash[84576]: cluster 2024-09-13T14:56:00.674341+0000 mgr.y (mgr.26252) 892 : cluster [DBG] pgmap v1413: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:56:02.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:01 smithi099 bash[84576]: cluster 2024-09-13T14:56:00.674341+0000 mgr.y (mgr.26252) 892 : cluster [DBG] pgmap v1413: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:56:02.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:01 smithi099 bash[87147]: cluster 2024-09-13T14:56:00.674341+0000 mgr.y (mgr.26252) 892 : cluster [DBG] pgmap v1413: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:56:02.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:01 smithi099 bash[87147]: cluster 2024-09-13T14:56:00.674341+0000 mgr.y (mgr.26252) 892 : cluster [DBG] pgmap v1413: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:56:02.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:01 smithi177 bash[68594]: cluster 2024-09-13T14:56:00.674341+0000 mgr.y (mgr.26252) 892 : cluster [DBG] pgmap v1413: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:56:02.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:01 smithi177 bash[68594]: cluster 2024-09-13T14:56:00.674341+0000 mgr.y (mgr.26252) 892 : cluster [DBG] pgmap v1413: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:56:02.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:02.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:02.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:02.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:02.335 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:02.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:02.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:02.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:02.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:02.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:02.600 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:02.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:02.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:02.726 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:02.729 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:02.736 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:02.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:02.839 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:02.867 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:02.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:02.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:02.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:02.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:03.100 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:03.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:03.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:03.242 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:03.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:03.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:03.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:03.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:03.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:03.512 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:03.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:03.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:03.643 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:03.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:03.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:03.774 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:03.800 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:03.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:03.908 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:03.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:03 smithi099 bash[84576]: cluster 2024-09-13T14:56:02.674865+0000 mgr.y (mgr.26252) 893 : cluster [DBG] pgmap v1414: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:56:03.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:03 smithi099 bash[84576]: cluster 2024-09-13T14:56:02.674865+0000 mgr.y (mgr.26252) 893 : cluster [DBG] pgmap v1414: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:56:03.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:03 smithi099 bash[84576]: audit 2024-09-13T14:56:03.283785+0000 mon.a (mon.0) 1953 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:03.917 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:03 smithi099 bash[84576]: audit 2024-09-13T14:56:03.283785+0000 mon.a (mon.0) 1953 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:03.917 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:03 smithi099 bash[87147]: cluster 2024-09-13T14:56:02.674865+0000 mgr.y (mgr.26252) 893 : cluster [DBG] pgmap v1414: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:56:03.917 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:03 smithi099 bash[87147]: cluster 2024-09-13T14:56:02.674865+0000 mgr.y (mgr.26252) 893 : cluster [DBG] pgmap v1414: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:56:03.917 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:03 smithi099 bash[87147]: audit 2024-09-13T14:56:03.283785+0000 mon.a (mon.0) 1953 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:03.917 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:03 smithi099 bash[87147]: audit 2024-09-13T14:56:03.283785+0000 mon.a (mon.0) 1953 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:03.933 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:03.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:04.038 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:04.058 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:04.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:04.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:03 smithi177 bash[68594]: cluster 2024-09-13T14:56:02.674865+0000 mgr.y (mgr.26252) 893 : cluster [DBG] pgmap v1414: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:56:04.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:03 smithi177 bash[68594]: cluster 2024-09-13T14:56:02.674865+0000 mgr.y (mgr.26252) 893 : cluster [DBG] pgmap v1414: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:56:04.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:03 smithi177 bash[68594]: audit 2024-09-13T14:56:03.283785+0000 mon.a (mon.0) 1953 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:04.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:03 smithi177 bash[68594]: audit 2024-09-13T14:56:03.283785+0000 mon.a (mon.0) 1953 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:04.161 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:04.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:04.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:04.304 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:04.333 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:04.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:04.436 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:04.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:04.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:04.593 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.612 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:04.715 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:04.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:04.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:04.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:04.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:04.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:04.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:05.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:05.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:05.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:05.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:05.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:05.243 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:05.270 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:05.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:05.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:05.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:05.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:05.571 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:05.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:05.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:05.696 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:05.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:05.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:05.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:05.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:05.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:05.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:06.056 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:06.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:05 smithi099 bash[84576]: cluster 2024-09-13T14:56:04.675482+0000 mgr.y (mgr.26252) 894 : cluster [DBG] pgmap v1415: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-13T14:56:06.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:05 smithi099 bash[84576]: cluster 2024-09-13T14:56:04.675482+0000 mgr.y (mgr.26252) 894 : cluster [DBG] pgmap v1415: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-13T14:56:06.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:05 smithi099 bash[87147]: cluster 2024-09-13T14:56:04.675482+0000 mgr.y (mgr.26252) 894 : cluster [DBG] pgmap v1415: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-13T14:56:06.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:05 smithi099 bash[87147]: cluster 2024-09-13T14:56:04.675482+0000 mgr.y (mgr.26252) 894 : cluster [DBG] pgmap v1415: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-13T14:56:06.091 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:06.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:06.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:05 smithi177 bash[68594]: cluster 2024-09-13T14:56:04.675482+0000 mgr.y (mgr.26252) 894 : cluster [DBG] pgmap v1415: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-13T14:56:06.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:05 smithi177 bash[68594]: cluster 2024-09-13T14:56:04.675482+0000 mgr.y (mgr.26252) 894 : cluster [DBG] pgmap v1415: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-09-13T14:56:06.196 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:06.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:06.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:06.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:06.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:06.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:06.459 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:06.477 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:06.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:06.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:06.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:06.716 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:06.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:06.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:06.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:06.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:06.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:06.977 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:07.012 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:07.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:07.125 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:56:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:56:07] "GET /metrics HTTP/1.1" 200 36752 "" "Prometheus/2.51.0" 2024-09-13T14:56:07.138 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:07.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:07.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:07.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:07.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:07.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:07.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:07.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:07.561 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:07.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:07.667 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:07.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:07.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:07.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:07.852 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:07.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:07.936 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:56:07.936 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:07.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:07.969 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:56:07.969 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:07.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:07.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:07.995 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:07.995 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:56:08.086 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:08.104 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:08.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:08.165 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:07 smithi099 bash[84576]: cluster 2024-09-13T14:56:06.675956+0000 mgr.y (mgr.26252) 895 : cluster [DBG] pgmap v1416: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-13T14:56:08.165 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:07 smithi099 bash[84576]: cluster 2024-09-13T14:56:06.675956+0000 mgr.y (mgr.26252) 895 : cluster [DBG] pgmap v1416: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-13T14:56:08.165 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:07 smithi099 bash[87147]: cluster 2024-09-13T14:56:06.675956+0000 mgr.y (mgr.26252) 895 : cluster [DBG] pgmap v1416: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-13T14:56:08.165 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:07 smithi099 bash[87147]: cluster 2024-09-13T14:56:06.675956+0000 mgr.y (mgr.26252) 895 : cluster [DBG] pgmap v1416: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-13T14:56:08.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:08.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:07 smithi177 bash[68594]: cluster 2024-09-13T14:56:06.675956+0000 mgr.y (mgr.26252) 895 : cluster [DBG] pgmap v1416: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-13T14:56:08.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:07 smithi177 bash[68594]: cluster 2024-09-13T14:56:06.675956+0000 mgr.y (mgr.26252) 895 : cluster [DBG] pgmap v1416: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-13T14:56:08.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:08.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:08.304 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:56:08.304 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:08.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:08.331 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:56:08.331 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:08.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:08.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:08.356 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:56:08.356 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:08.387 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:56:08.388 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:08.424 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:56:08.424 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:08.454 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:08.474 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:08.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:08.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:08.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:08.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:08.709 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:08.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:08.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:08.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:08.876 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:08.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:08.879 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T14:56:08.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:09.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:09.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:09.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:08 smithi099 bash[84576]: audit 2024-09-13T14:56:07.890594+0000 mon.a (mon.0) 1954 : audit [DBG] from='client.? 172.21.15.99:0/2366530292' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:09.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:08 smithi099 bash[84576]: audit 2024-09-13T14:56:07.890594+0000 mon.a (mon.0) 1954 : audit [DBG] from='client.? 172.21.15.99:0/2366530292' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:09.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:08 smithi099 bash[84576]: audit 2024-09-13T14:56:08.179175+0000 mon.a (mon.0) 1955 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:09.064 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:08 smithi099 bash[84576]: audit 2024-09-13T14:56:08.179175+0000 mon.a (mon.0) 1955 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:09.064 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:08 smithi099 bash[87147]: audit 2024-09-13T14:56:07.890594+0000 mon.a (mon.0) 1954 : audit [DBG] from='client.? 172.21.15.99:0/2366530292' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:09.064 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:08 smithi099 bash[87147]: audit 2024-09-13T14:56:07.890594+0000 mon.a (mon.0) 1954 : audit [DBG] from='client.? 172.21.15.99:0/2366530292' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:09.065 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:08 smithi099 bash[87147]: audit 2024-09-13T14:56:08.179175+0000 mon.a (mon.0) 1955 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:09.065 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:08 smithi099 bash[87147]: audit 2024-09-13T14:56:08.179175+0000 mon.a (mon.0) 1955 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:09.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:09.130 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:08 smithi177 bash[68594]: audit 2024-09-13T14:56:07.890594+0000 mon.a (mon.0) 1954 : audit [DBG] from='client.? 172.21.15.99:0/2366530292' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:09.130 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:08 smithi177 bash[68594]: audit 2024-09-13T14:56:07.890594+0000 mon.a (mon.0) 1954 : audit [DBG] from='client.? 172.21.15.99:0/2366530292' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:09.130 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:08 smithi177 bash[68594]: audit 2024-09-13T14:56:08.179175+0000 mon.a (mon.0) 1955 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:09.130 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:08 smithi177 bash[68594]: audit 2024-09-13T14:56:08.179175+0000 mon.a (mon.0) 1955 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:09.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:09.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:09.267 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:09.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:09.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:09.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:09.411 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:09.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:09.541 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:09.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:09.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:09.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:09.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:09.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:09.825 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:09.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:09.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:09.962 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:09.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:10.065 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:10.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[84576]: cluster 2024-09-13T14:56:08.676432+0000 mgr.y (mgr.26252) 896 : cluster [DBG] pgmap v1417: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T14:56:10.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[84576]: cluster 2024-09-13T14:56:08.676432+0000 mgr.y (mgr.26252) 896 : cluster [DBG] pgmap v1417: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T14:56:10.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[84576]: cluster 2024-09-13T14:56:08.867698+0000 mon.a (mon.0) 1956 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:10.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[84576]: cluster 2024-09-13T14:56:08.867698+0000 mon.a (mon.0) 1956 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:10.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[84576]: audit 2024-09-13T14:56:08.877444+0000 mon.a (mon.0) 1957 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:10.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[84576]: audit 2024-09-13T14:56:08.877444+0000 mon.a (mon.0) 1957 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:10.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[84576]: cluster 2024-09-13T14:56:08.882339+0000 mon.a (mon.0) 1958 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2024-09-13T14:56:10.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[84576]: cluster 2024-09-13T14:56:08.882339+0000 mon.a (mon.0) 1958 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2024-09-13T14:56:10.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[87147]: cluster 2024-09-13T14:56:08.676432+0000 mgr.y (mgr.26252) 896 : cluster [DBG] pgmap v1417: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T14:56:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[87147]: cluster 2024-09-13T14:56:08.676432+0000 mgr.y (mgr.26252) 896 : cluster [DBG] pgmap v1417: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T14:56:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[87147]: cluster 2024-09-13T14:56:08.867698+0000 mon.a (mon.0) 1956 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[87147]: cluster 2024-09-13T14:56:08.867698+0000 mon.a (mon.0) 1956 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[87147]: audit 2024-09-13T14:56:08.877444+0000 mon.a (mon.0) 1957 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[87147]: audit 2024-09-13T14:56:08.877444+0000 mon.a (mon.0) 1957 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[87147]: cluster 2024-09-13T14:56:08.882339+0000 mon.a (mon.0) 1958 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2024-09-13T14:56:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:09 smithi099 bash[87147]: cluster 2024-09-13T14:56:08.882339+0000 mon.a (mon.0) 1958 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2024-09-13T14:56:10.094 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:10.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:10.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:09 smithi177 bash[68594]: cluster 2024-09-13T14:56:08.676432+0000 mgr.y (mgr.26252) 896 : cluster [DBG] pgmap v1417: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T14:56:10.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:09 smithi177 bash[68594]: cluster 2024-09-13T14:56:08.676432+0000 mgr.y (mgr.26252) 896 : cluster [DBG] pgmap v1417: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T14:56:10.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:09 smithi177 bash[68594]: cluster 2024-09-13T14:56:08.867698+0000 mon.a (mon.0) 1956 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:09 smithi177 bash[68594]: cluster 2024-09-13T14:56:08.867698+0000 mon.a (mon.0) 1956 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:09 smithi177 bash[68594]: audit 2024-09-13T14:56:08.877444+0000 mon.a (mon.0) 1957 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:09 smithi177 bash[68594]: audit 2024-09-13T14:56:08.877444+0000 mon.a (mon.0) 1957 : audit [INF] from='client.? 172.21.15.99:0/3226979820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:09 smithi177 bash[68594]: cluster 2024-09-13T14:56:08.882339+0000 mon.a (mon.0) 1958 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2024-09-13T14:56:10.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:09 smithi177 bash[68594]: cluster 2024-09-13T14:56:08.882339+0000 mon.a (mon.0) 1958 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2024-09-13T14:56:10.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:10.217 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:10.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:10.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:10.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:10.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:10.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:10.463 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:56:10 smithi177 bash[61439]: ts=2024-09-13T14:56:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:10.484 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:10.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:10.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:10.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:10.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:10.752 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:10.773 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:10.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:10.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:10.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:11.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:11.035 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:11.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:11.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:11.171 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:11.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:11.276 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:11.305 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:11.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:11.409 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:11.428 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:11.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:11.531 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:11.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:11.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:11.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:11.710 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:11.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:11.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:11.835 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:11.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:11.941 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:11.948 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:11 smithi099 bash[84576]: cluster 2024-09-13T14:56:10.677042+0000 mgr.y (mgr.26252) 897 : cluster [DBG] pgmap v1419: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-09-13T14:56:11.949 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:11 smithi099 bash[84576]: cluster 2024-09-13T14:56:10.677042+0000 mgr.y (mgr.26252) 897 : cluster [DBG] pgmap v1419: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-09-13T14:56:11.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:11 smithi099 bash[87147]: cluster 2024-09-13T14:56:10.677042+0000 mgr.y (mgr.26252) 897 : cluster [DBG] pgmap v1419: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-09-13T14:56:11.949 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:11 smithi099 bash[87147]: cluster 2024-09-13T14:56:10.677042+0000 mgr.y (mgr.26252) 897 : cluster [DBG] pgmap v1419: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-09-13T14:56:11.959 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:11.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:12.062 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:12.086 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:12.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:12.190 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.214 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:12.317 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:12.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:11 smithi177 bash[68594]: cluster 2024-09-13T14:56:10.677042+0000 mgr.y (mgr.26252) 897 : cluster [DBG] pgmap v1419: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-09-13T14:56:12.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:11 smithi177 bash[68594]: cluster 2024-09-13T14:56:10.677042+0000 mgr.y (mgr.26252) 897 : cluster [DBG] pgmap v1419: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2024-09-13T14:56:12.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:12.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:12.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:12.597 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.623 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:12.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:12.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:12.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:12.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:13.002 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:13.022 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:13.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:13.149 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:13.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:13.255 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:13.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:13.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:13.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:13.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:13.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:13.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:13.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:13.657 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:13.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:13.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:13.813 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:13.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:13.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:13.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:13.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:13 smithi099 bash[84576]: cluster 2024-09-13T14:56:12.677566+0000 mgr.y (mgr.26252) 898 : cluster [DBG] pgmap v1420: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:56:13.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:13 smithi099 bash[84576]: cluster 2024-09-13T14:56:12.677566+0000 mgr.y (mgr.26252) 898 : cluster [DBG] pgmap v1420: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:56:13.962 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:13 smithi099 bash[87147]: cluster 2024-09-13T14:56:12.677566+0000 mgr.y (mgr.26252) 898 : cluster [DBG] pgmap v1420: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:56:13.962 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:13 smithi099 bash[87147]: cluster 2024-09-13T14:56:12.677566+0000 mgr.y (mgr.26252) 898 : cluster [DBG] pgmap v1420: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:56:13.973 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:13.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:14.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:14.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:14.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:14.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:13 smithi177 bash[68594]: cluster 2024-09-13T14:56:12.677566+0000 mgr.y (mgr.26252) 898 : cluster [DBG] pgmap v1420: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:56:14.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:13 smithi177 bash[68594]: cluster 2024-09-13T14:56:12.677566+0000 mgr.y (mgr.26252) 898 : cluster [DBG] pgmap v1420: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:56:14.206 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:14.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:14.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:14.356 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:14.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:14.405 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:56:14.466 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:14.500 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:14.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:14.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:14.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:14.753 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:14.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:14.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:14.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:14.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:15.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:15.080 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:15.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:15.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:15.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:15.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:15.350 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:15.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:15.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:15.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:15.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:15.591 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:15.617 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:15.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:15.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:15.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:15.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:15.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:15.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:15.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:15.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:16.024 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:16.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:16.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:16.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:16.154 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:15 smithi177 bash[68594]: cluster 2024-09-13T14:56:14.678234+0000 mgr.y (mgr.26252) 899 : cluster [DBG] pgmap v1421: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:56:16.154 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:15 smithi177 bash[68594]: cluster 2024-09-13T14:56:14.678234+0000 mgr.y (mgr.26252) 899 : cluster [DBG] pgmap v1421: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:56:16.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:16.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:16.264 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:15 smithi099 bash[84576]: cluster 2024-09-13T14:56:14.678234+0000 mgr.y (mgr.26252) 899 : cluster [DBG] pgmap v1421: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:56:16.265 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:15 smithi099 bash[84576]: cluster 2024-09-13T14:56:14.678234+0000 mgr.y (mgr.26252) 899 : cluster [DBG] pgmap v1421: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:56:16.265 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:15 smithi099 bash[87147]: cluster 2024-09-13T14:56:14.678234+0000 mgr.y (mgr.26252) 899 : cluster [DBG] pgmap v1421: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:56:16.265 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:15 smithi099 bash[87147]: cluster 2024-09-13T14:56:14.678234+0000 mgr.y (mgr.26252) 899 : cluster [DBG] pgmap v1421: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T14:56:16.287 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:16.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:16.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:16.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:16.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:16.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:16.557 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:16.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:16.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:16.691 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:16.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:16.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:56:16.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:16.832 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:16.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:16.939 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:16.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:16.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:17.082 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:17.116 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:17.126 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T14:56:17.126 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:56:17.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:17.232 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:56:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:56:17] "GET /metrics HTTP/1.1" 200 36752 "" "Prometheus/2.51.0" 2024-09-13T14:56:17.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:17.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:17.352 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.380 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:17.390 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:56:17.391 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:56:17.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.506 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:17.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.632 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:17.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.771 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:17.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:17.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:17.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:18.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:18.017 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:17 smithi099 bash[84576]: cluster 2024-09-13T14:56:16.678633+0000 mgr.y (mgr.26252) 900 : cluster [DBG] pgmap v1422: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:18.017 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:17 smithi099 bash[84576]: cluster 2024-09-13T14:56:16.678633+0000 mgr.y (mgr.26252) 900 : cluster [DBG] pgmap v1422: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:18.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:17 smithi099 bash[87147]: cluster 2024-09-13T14:56:16.678633+0000 mgr.y (mgr.26252) 900 : cluster [DBG] pgmap v1422: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:18.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:17 smithi099 bash[87147]: cluster 2024-09-13T14:56:16.678633+0000 mgr.y (mgr.26252) 900 : cluster [DBG] pgmap v1422: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:18.031 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:18.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:18.136 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:18.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:18.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:18.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:18.276 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:17 smithi177 bash[68594]: cluster 2024-09-13T14:56:16.678633+0000 mgr.y (mgr.26252) 900 : cluster [DBG] pgmap v1422: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:18.276 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:17 smithi177 bash[68594]: cluster 2024-09-13T14:56:16.678633+0000 mgr.y (mgr.26252) 900 : cluster [DBG] pgmap v1422: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:18.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:18.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:18.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:18.422 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:18.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:18.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:18.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:18.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:18.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:18.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:18.790 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:18.832 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:18.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:18.939 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:18.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:18.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:19.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:19.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:18 smithi177 bash[68594]: audit 2024-09-13T14:56:17.382184+0000 mgr.y (mgr.26252) 901 : audit [DBG] from='client.35571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:19.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:18 smithi177 bash[68594]: audit 2024-09-13T14:56:17.382184+0000 mgr.y (mgr.26252) 901 : audit [DBG] from='client.35571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:19.074 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:18 smithi177 bash[68594]: audit 2024-09-13T14:56:18.283950+0000 mon.a (mon.0) 1959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:19.074 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:18 smithi177 bash[68594]: audit 2024-09-13T14:56:18.283950+0000 mon.a (mon.0) 1959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:19.090 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:19.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:18 smithi099 bash[84576]: audit 2024-09-13T14:56:17.382184+0000 mgr.y (mgr.26252) 901 : audit [DBG] from='client.35571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:19.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:18 smithi099 bash[84576]: audit 2024-09-13T14:56:17.382184+0000 mgr.y (mgr.26252) 901 : audit [DBG] from='client.35571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:19.093 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:18 smithi099 bash[84576]: audit 2024-09-13T14:56:18.283950+0000 mon.a (mon.0) 1959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:19.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:19.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:19.225 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:19.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:19.328 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:19.358 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:19.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:19.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:19.475 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:19 smithi099 bash[84576]: audit 2024-09-13T14:56:18.283950+0000 mon.a (mon.0) 1959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:19.476 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:56:19.477 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:19 smithi099 bash[87147]: audit 2024-09-13T14:56:17.382184+0000 mgr.y (mgr.26252) 901 : audit [DBG] from='client.35571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:19.477 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:19 smithi099 bash[87147]: audit 2024-09-13T14:56:17.382184+0000 mgr.y (mgr.26252) 901 : audit [DBG] from='client.35571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:19.477 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:19 smithi099 bash[87147]: audit 2024-09-13T14:56:18.283950+0000 mon.a (mon.0) 1959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:19.477 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:19 smithi099 bash[87147]: audit 2024-09-13T14:56:18.283950+0000 mon.a (mon.0) 1959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:19.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:19.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:19.566 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1422,"stamp":"2024-09-13T14:56:16.678404+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3327465445,"num_objects":50791,"num_object_clones":0,"num_object_copies":152373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50791,"num_whiteouts":0,"num_read":345,"num_read_kb":800,"num_write":165123,"num_write_kb":6903228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":595442,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142175,"ondisk_log_size":142175,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12381120,"kb_used_data":9966592,"kb_used_omap":353,"kb_used_meta":2413918,"kb_avail":737580096,"statfs":{"total":767960285184,"available":755282018304,"internally_reserved":0,"allocated":10205790208,"data_stored":10174297649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":361898,"internal_metadata":2471852630},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-464191488,"num_objects":-7083,"num_object_clones":0,"num_object_copies":-21249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7083,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002744"},"pg_stats":[{"pgid":"42.f","version":"1029'10199","reported_seq":10227,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:14.234743+0000","last_change":"2024-09-13T14:53:02.737967+0000","last_active":"2024-09-13T14:56:14.234743+0000","last_peered":"2024-09-13T14:56:14.234743+0000","last_clean":"2024-09-13T14:56:14.234743+0000","last_became_active":"2024-09-13T14:53:02.737189+0000","last_became_peered":"2024-09-13T14:53:02.737189+0000","last_unstale":"2024-09-13T14:56:14.234743+0000","last_undegraded":"2024-09-13T14:56:14.234743+0000","last_fullsized":"2024-09-13T14:56:14.234743+0000","mapping_epoch":1009,"log_start":"1015'2300","ondisk_log_start":"1015'2300","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7899,"log_dups_size":0,"ondisk_log_size":7899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:43.030910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208601088,"num_objects":3183,"num_object_clones":0,"num_object_copies":9549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10199,"num_write_kb":428224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"42.e","version":"1029'10453","reported_seq":10505,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:13.920550+0000","last_change":"2024-09-13T14:53:03.091056+0000","last_active":"2024-09-13T14:56:13.920550+0000","last_peered":"2024-09-13T14:56:13.920550+0000","last_clean":"2024-09-13T14:56:13.920550+0000","last_became_active":"2024-09-13T14:53:03.090134+0000","last_became_peered":"2024-09-13T14:53:03.090134+0000","last_unstale":"2024-09-13T14:56:13.920550+0000","last_undegraded":"2024-09-13T14:56:13.920550+0000","last_fullsized":"2024-09-13T14:56:13.920550+0000","mapping_epoch":1009,"log_start":"1014'1800","ondisk_log_start":"1014'1800","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":8653,"log_dups_size":0,"ondisk_log_size":8653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:44.438976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210567168,"num_objects":3213,"num_object_clones":0,"num_object_copies":9639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10453,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"42.d","version":"1029'10264","reported_seq":10293,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:14.159987+0000","last_change":"2024-09-13T14:53:02.727418+0000","last_active":"2024-09-13T14:56:14.159987+0000","last_peered":"2024-09-13T14:56:14.159987+0000","last_clean":"2024-09-13T14:56:14.159987+0000","last_became_active":"2024-09-13T14:53:02.726653+0000","last_became_peered":"2024-09-13T14:53:02.726653+0000","last_unstale":"2024-09-13T14:56:14.159987+0000","last_undegraded":"2024-09-13T14:56:14.159987+0000","last_fullsized":"2024-09-13T14:56:14.159987+0000","mapping_epoch":1009,"log_start":"1014'500","ondisk_log_start":"1014'500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9764,"log_dups_size":0,"ondisk_log_size":9764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:30:10.620331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208732184,"num_objects":3186,"num_object_clones":0,"num_object_copies":9558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3186,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10264,"num_write_kb":430337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"42.c","version":"1029'10396","reported_seq":10448,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:13.894077+0000","last_change":"2024-09-13T14:53:02.731439+0000","last_active":"2024-09-13T14:56:13.894077+0000","last_peered":"2024-09-13T14:56:13.894077+0000","last_clean":"2024-09-13T14:56:13.894077+0000","last_became_active":"2024-09-13T14:53:02.730603+0000","last_became_peered":"2024-09-13T14:53:02.730603+0000","last_unstale":"2024-09-13T14:56:13.894077+0000","last_undegraded":"2024-09-13T14:56:13.894077+0000","last_fullsized":"2024-09-13T14:56:13.894077+0000","mapping_epoch":1009,"log_start":"1015'2500","ondisk_log_start":"1015'2500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7896,"log_dups_size":0,"ondisk_log_size":7896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:10.067364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211419136,"num_objects":3226,"num_object_clones":0,"num_object_copies":9678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10396,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"42.b","version":"1029'10264","reported_seq":10316,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:16.579597+0000","last_change":"2024-09-13T14:53:02.735868+0000","last_active":"2024-09-13T14:56:16.579597+0000","last_peered":"2024-09-13T14:56:16.579597+0000","last_clean":"2024-09-13T14:56:16.579597+0000","last_became_active":"2024-09-13T14:53:02.735016+0000","last_became_peered":"2024-09-13T14:53:02.735016+0000","last_unstale":"2024-09-13T14:56:16.579597+0000","last_undegraded":"2024-09-13T14:56:16.579597+0000","last_fullsized":"2024-09-13T14:56:16.579597+0000","mapping_epoch":1009,"log_start":"1016'2500","ondisk_log_start":"1016'2500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7764,"log_dups_size":0,"ondisk_log_size":7764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:44:45.114443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197394432,"num_objects":3012,"num_object_clones":0,"num_object_copies":9036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10264,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"42.a","version":"1029'10209","reported_seq":10237,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:14.418200+0000","last_change":"2024-09-13T14:53:02.730034+0000","last_active":"2024-09-13T14:56:14.418200+0000","last_peered":"2024-09-13T14:56:14.418200+0000","last_clean":"2024-09-13T14:56:14.418200+0000","last_became_active":"2024-09-13T14:53:02.729241+0000","last_became_peered":"2024-09-13T14:53:02.729241+0000","last_unstale":"2024-09-13T14:56:14.418200+0000","last_undegraded":"2024-09-13T14:56:14.418200+0000","last_fullsized":"2024-09-13T14:56:14.418200+0000","mapping_epoch":1009,"log_start":"1014'500","ondisk_log_start":"1014'500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9709,"log_dups_size":0,"ondisk_log_size":9709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:44.133299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206635008,"num_objects":3153,"num_object_clones":0,"num_object_copies":9459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10209,"num_write_kb":427584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"42.9","version":"1029'10308","reported_seq":10336,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:14.178420+0000","last_change":"2024-09-13T14:53:02.737619+0000","last_active":"2024-09-13T14:56:14.178420+0000","last_peered":"2024-09-13T14:56:14.178420+0000","last_clean":"2024-09-13T14:56:14.178420+0000","last_became_active":"2024-09-13T14:53:02.736782+0000","last_became_peered":"2024-09-13T14:53:02.736782+0000","last_unstale":"2024-09-13T14:56:14.178420+0000","last_undegraded":"2024-09-13T14:56:14.178420+0000","last_fullsized":"2024-09-13T14:56:14.178420+0000","mapping_epoch":1009,"log_start":"1014'600","ondisk_log_start":"1014'600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9708,"log_dups_size":0,"ondisk_log_size":9708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:30:17.070735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202768384,"num_objects":3094,"num_object_clones":0,"num_object_copies":9282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10308,"num_write_kb":428864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"42.8","version":"1029'10177","reported_seq":10205,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:14.276897+0000","last_change":"2024-09-13T14:53:03.087618+0000","last_active":"2024-09-13T14:56:14.276897+0000","last_peered":"2024-09-13T14:56:14.276897+0000","last_clean":"2024-09-13T14:56:14.276897+0000","last_became_active":"2024-09-13T14:53:03.086906+0000","last_became_peered":"2024-09-13T14:53:03.086906+0000","last_unstale":"2024-09-13T14:56:14.276897+0000","last_undegraded":"2024-09-13T14:56:14.276897+0000","last_fullsized":"2024-09-13T14:56:14.276897+0000","mapping_epoch":1009,"log_start":"1015'2200","ondisk_log_start":"1015'2200","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7977,"log_dups_size":0,"ondisk_log_size":7977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:05.901169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204013568,"num_objects":3113,"num_object_clones":0,"num_object_copies":9339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10177,"num_write_kb":425280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"42.7","version":"1029'10451","reported_seq":10479,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:14.253247+0000","last_change":"2024-09-13T14:53:02.732343+0000","last_active":"2024-09-13T14:56:14.253247+0000","last_peered":"2024-09-13T14:56:14.253247+0000","last_clean":"2024-09-13T14:56:14.253247+0000","last_became_active":"2024-09-13T14:53:02.731717+0000","last_became_peered":"2024-09-13T14:53:02.731717+0000","last_unstale":"2024-09-13T14:56:14.253247+0000","last_undegraded":"2024-09-13T14:56:14.253247+0000","last_fullsized":"2024-09-13T14:56:14.253247+0000","mapping_epoch":1009,"log_start":"1015'2500","ondisk_log_start":"1015'2500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:37.034264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210829312,"num_objects":3217,"num_object_clones":0,"num_object_copies":9651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10451,"num_write_kb":437376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"42.6","version":"1029'10134","reported_seq":10162,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:14.368939+0000","last_change":"2024-09-13T14:53:02.734127+0000","last_active":"2024-09-13T14:56:14.368939+0000","last_peered":"2024-09-13T14:56:14.368939+0000","last_clean":"2024-09-13T14:56:14.368939+0000","last_became_active":"2024-09-13T14:53:02.733028+0000","last_became_peered":"2024-09-13T14:53:02.733028+0000","last_unstale":"2024-09-13T14:56:14.368939+0000","last_undegraded":"2024-09-13T14:56:14.368939+0000","last_fullsized":"2024-09-13T14:56:14.368939+0000","mapping_epoch":1009,"log_start":"1014'400","ondisk_log_start":"1014'400","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9734,"log_dups_size":0,"ondisk_log_size":9734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:46.557305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202768384,"num_objects":3094,"num_object_clones":0,"num_object_copies":9282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10134,"num_write_kb":423296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"42.5","version":"1029'10365","reported_seq":10393,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:14.278026+0000","last_change":"2024-09-13T14:53:02.730556+0000","last_active":"2024-09-13T14:56:14.278026+0000","last_peered":"2024-09-13T14:56:14.278026+0000","last_clean":"2024-09-13T14:56:14.278026+0000","last_became_active":"2024-09-13T14:53:02.729563+0000","last_became_peered":"2024-09-13T14:53:02.729563+0000","last_unstale":"2024-09-13T14:56:14.278026+0000","last_undegraded":"2024-09-13T14:56:14.278026+0000","last_fullsized":"2024-09-13T14:56:14.278026+0000","mapping_epoch":1009,"log_start":"1014'600","ondisk_log_start":"1014'600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9765,"log_dups_size":0,"ondisk_log_size":9765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:28.870740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207814656,"num_objects":3171,"num_object_clones":0,"num_object_copies":9513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10365,"num_write_kb":433152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"42.4","version":"1029'10346","reported_seq":10398,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:16.575121+0000","last_change":"2024-09-13T14:53:02.737098+0000","last_active":"2024-09-13T14:56:16.575121+0000","last_peered":"2024-09-13T14:56:16.575121+0000","last_clean":"2024-09-13T14:56:16.575121+0000","last_became_active":"2024-09-13T14:53:02.736160+0000","last_became_peered":"2024-09-13T14:53:02.736160+0000","last_unstale":"2024-09-13T14:56:16.575121+0000","last_undegraded":"2024-09-13T14:56:16.575121+0000","last_fullsized":"2024-09-13T14:56:16.575121+0000","mapping_epoch":1009,"log_start":"1016'2600","ondisk_log_start":"1016'2600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7746,"log_dups_size":0,"ondisk_log_size":7746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:48.130549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201064448,"num_objects":3068,"num_object_clones":0,"num_object_copies":9204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10346,"num_write_kb":429248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"42.3","version":"1029'10246","reported_seq":10274,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:14.159613+0000","last_change":"2024-09-13T14:53:03.088642+0000","last_active":"2024-09-13T14:56:14.159613+0000","last_peered":"2024-09-13T14:56:14.159613+0000","last_clean":"2024-09-13T14:56:14.159613+0000","last_became_active":"2024-09-13T14:53:03.087596+0000","last_became_peered":"2024-09-13T14:53:03.087596+0000","last_unstale":"2024-09-13T14:56:14.159613+0000","last_undegraded":"2024-09-13T14:56:14.159613+0000","last_fullsized":"2024-09-13T14:56:14.159613+0000","mapping_epoch":1009,"log_start":"1014'500","ondisk_log_start":"1014'500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9746,"log_dups_size":0,"ondisk_log_size":9746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:30:53.067804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211025920,"num_objects":3220,"num_object_clones":0,"num_object_copies":9660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10246,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"42.2","version":"1029'10151","reported_seq":10203,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:12.304842+0000","last_change":"2024-09-13T14:53:03.090582+0000","last_active":"2024-09-13T14:56:12.304842+0000","last_peered":"2024-09-13T14:56:12.304842+0000","last_clean":"2024-09-13T14:56:12.304842+0000","last_became_active":"2024-09-13T14:53:03.089683+0000","last_became_peered":"2024-09-13T14:53:03.089683+0000","last_unstale":"2024-09-13T14:56:12.304842+0000","last_undegraded":"2024-09-13T14:56:12.304842+0000","last_fullsized":"2024-09-13T14:56:12.304842+0000","mapping_epoch":1009,"log_start":"1014'100","ondisk_log_start":"1014'100","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":10051,"log_dups_size":0,"ondisk_log_size":10051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:13:05.244177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214237184,"num_objects":3269,"num_object_clones":0,"num_object_copies":9807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10151,"num_write_kb":429440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"42.1","version":"1029'10496","reported_seq":10548,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:13.906080+0000","last_change":"2024-09-13T14:53:02.733122+0000","last_active":"2024-09-13T14:56:13.906080+0000","last_peered":"2024-09-13T14:56:13.906080+0000","last_clean":"2024-09-13T14:56:13.906080+0000","last_became_active":"2024-09-13T14:53:02.732272+0000","last_became_peered":"2024-09-13T14:53:02.732272+0000","last_unstale":"2024-09-13T14:56:13.906080+0000","last_undegraded":"2024-09-13T14:56:13.906080+0000","last_fullsized":"2024-09-13T14:56:13.906080+0000","mapping_epoch":1009,"log_start":"1016'2600","ondisk_log_start":"1016'2600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7896,"log_dups_size":0,"ondisk_log_size":7896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:48.079694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212336640,"num_objects":3240,"num_object_clones":0,"num_object_copies":9720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10496,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"42.0","version":"1029'10452","reported_seq":10479,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:14.422204+0000","last_change":"2024-09-13T14:53:02.729978+0000","last_active":"2024-09-13T14:56:14.422204+0000","last_peered":"2024-09-13T14:56:14.422204+0000","last_clean":"2024-09-13T14:56:14.422204+0000","last_became_active":"2024-09-13T14:53:02.728942+0000","last_became_peered":"2024-09-13T14:53:02.728942+0000","last_unstale":"2024-09-13T14:56:14.422204+0000","last_undegraded":"2024-09-13T14:56:14.422204+0000","last_fullsized":"2024-09-13T14:56:14.422204+0000","mapping_epoch":1009,"log_start":"1014'700","ondisk_log_start":"1014'700","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9752,"log_dups_size":0,"ondisk_log_size":9752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:11.356137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216662016,"num_objects":3306,"num_object_clones":0,"num_object_copies":9918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10452,"num_write_kb":440256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":1449,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890208+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T14:56:08.890208+0000","last_peered":"2024-09-13T14:56:08.890208+0000","last_clean":"2024-09-13T14:56:08.890208+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T14:56:08.890208+0000","last_undegraded":"2024-09-13T14:56:08.890208+0000","last_fullsized":"2024-09-13T14:56:08.890208+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884108+0000","last_change":"2024-09-13T14:52:43.599071+0000","last_active":"2024-09-13T14:56:08.884108+0000","last_peered":"2024-09-13T14:56:08.884108+0000","last_clean":"2024-09-13T14:56:08.884108+0000","last_became_active":"2024-09-13T14:52:43.594847+0000","last_became_peered":"2024-09-13T14:52:43.594847+0000","last_unstale":"2024-09-13T14:56:08.884108+0000","last_undegraded":"2024-09-13T14:56:08.884108+0000","last_fullsized":"2024-09-13T14:56:08.884108+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:02.736434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1300,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885476+0000","last_change":"2024-09-13T14:39:34.403170+0000","last_active":"2024-09-13T14:56:08.885476+0000","last_peered":"2024-09-13T14:56:08.885476+0000","last_clean":"2024-09-13T14:56:08.885476+0000","last_became_active":"2024-09-13T14:39:34.389001+0000","last_became_peered":"2024-09-13T14:39:34.389001+0000","last_unstale":"2024-09-13T14:56:08.885476+0000","last_undegraded":"2024-09-13T14:56:08.885476+0000","last_fullsized":"2024-09-13T14:56:08.885476+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:38.542452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":60,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.829168+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T14:55:55.829168+0000","last_peered":"2024-09-13T14:55:55.829168+0000","last_clean":"2024-09-13T14:55:55.829168+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T14:55:55.829168+0000","last_undegraded":"2024-09-13T14:55:55.829168+0000","last_fullsized":"2024-09-13T14:55:55.829168+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1014,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.835885+0000","last_change":"2024-09-13T14:47:46.337827+0000","last_active":"2024-09-13T14:55:55.835885+0000","last_peered":"2024-09-13T14:55:55.835885+0000","last_clean":"2024-09-13T14:55:55.835885+0000","last_became_active":"2024-09-13T14:47:46.321140+0000","last_became_peered":"2024-09-13T14:47:46.321140+0000","last_unstale":"2024-09-13T14:55:55.835885+0000","last_undegraded":"2024-09-13T14:55:55.835885+0000","last_fullsized":"2024-09-13T14:55:55.835885+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:15.997382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":155,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884422+0000","last_change":"2024-09-13T14:47:46.333043+0000","last_active":"2024-09-13T14:56:08.884422+0000","last_peered":"2024-09-13T14:56:08.884422+0000","last_clean":"2024-09-13T14:56:08.884422+0000","last_became_active":"2024-09-13T14:47:46.332292+0000","last_became_peered":"2024-09-13T14:47:46.332292+0000","last_unstale":"2024-09-13T14:56:08.884422+0000","last_undegraded":"2024-09-13T14:56:08.884422+0000","last_fullsized":"2024-09-13T14:56:08.884422+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:38.942334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1059,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.831383+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T14:55:55.831383+0000","last_peered":"2024-09-13T14:55:55.831383+0000","last_clean":"2024-09-13T14:55:55.831383+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T14:55:55.831383+0000","last_undegraded":"2024-09-13T14:55:55.831383+0000","last_fullsized":"2024-09-13T14:55:55.831383+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885269+0000","last_change":"2024-09-13T14:52:43.601798+0000","last_active":"2024-09-13T14:56:08.885269+0000","last_peered":"2024-09-13T14:56:08.885269+0000","last_clean":"2024-09-13T14:56:08.885269+0000","last_became_active":"2024-09-13T14:52:43.600648+0000","last_became_peered":"2024-09-13T14:52:43.600648+0000","last_unstale":"2024-09-13T14:56:08.885269+0000","last_undegraded":"2024-09-13T14:56:08.885269+0000","last_fullsized":"2024-09-13T14:56:08.885269+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:01.021067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1457,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890666+0000","last_change":"2024-09-13T14:47:46.330059+0000","last_active":"2024-09-13T14:56:08.890666+0000","last_peered":"2024-09-13T14:56:08.890666+0000","last_clean":"2024-09-13T14:56:08.890666+0000","last_became_active":"2024-09-13T14:47:46.329344+0000","last_became_peered":"2024-09-13T14:47:46.329344+0000","last_unstale":"2024-09-13T14:56:08.890666+0000","last_undegraded":"2024-09-13T14:56:08.890666+0000","last_fullsized":"2024-09-13T14:56:08.890666+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:58:43.470176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1360,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.879870+0000","last_change":"2024-09-13T14:52:43.594737+0000","last_active":"2024-09-13T14:56:08.879870+0000","last_peered":"2024-09-13T14:56:08.879870+0000","last_clean":"2024-09-13T14:56:08.879870+0000","last_became_active":"2024-09-13T14:52:43.594197+0000","last_became_peered":"2024-09-13T14:52:43.594197+0000","last_unstale":"2024-09-13T14:56:08.879870+0000","last_undegraded":"2024-09-13T14:56:08.879870+0000","last_fullsized":"2024-09-13T14:56:08.879870+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:31.054754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884314+0000","last_change":"2024-09-13T14:52:43.597400+0000","last_active":"2024-09-13T14:56:08.884314+0000","last_peered":"2024-09-13T14:56:08.884314+0000","last_clean":"2024-09-13T14:56:08.884314+0000","last_became_active":"2024-09-13T14:52:43.596294+0000","last_became_peered":"2024-09-13T14:52:43.596294+0000","last_unstale":"2024-09-13T14:56:08.884314+0000","last_undegraded":"2024-09-13T14:56:08.884314+0000","last_fullsized":"2024-09-13T14:56:08.884314+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:35.218171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1346,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.883911+0000","last_change":"2024-09-13T14:52:43.590238+0000","last_active":"2024-09-13T14:56:08.883911+0000","last_peered":"2024-09-13T14:56:08.883911+0000","last_clean":"2024-09-13T14:56:08.883911+0000","last_became_active":"2024-09-13T14:52:43.589713+0000","last_became_peered":"2024-09-13T14:52:43.589713+0000","last_unstale":"2024-09-13T14:56:08.883911+0000","last_undegraded":"2024-09-13T14:56:08.883911+0000","last_fullsized":"2024-09-13T14:56:08.883911+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:19.148195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.897301+0000","last_change":"2024-09-13T14:52:43.591911+0000","last_active":"2024-09-13T14:56:08.897301+0000","last_peered":"2024-09-13T14:56:08.897301+0000","last_clean":"2024-09-13T14:56:08.897301+0000","last_became_active":"2024-09-13T14:52:43.591410+0000","last_became_peered":"2024-09-13T14:52:43.591410+0000","last_unstale":"2024-09-13T14:56:08.897301+0000","last_undegraded":"2024-09-13T14:56:08.897301+0000","last_fullsized":"2024-09-13T14:56:08.897301+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:53.664208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1445,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890560+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T14:56:08.890560+0000","last_peered":"2024-09-13T14:56:08.890560+0000","last_clean":"2024-09-13T14:56:08.890560+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T14:56:08.890560+0000","last_undegraded":"2024-09-13T14:56:08.890560+0000","last_fullsized":"2024-09-13T14:56:08.890560+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1339,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.889494+0000","last_change":"2024-09-13T14:52:43.587987+0000","last_active":"2024-09-13T14:56:08.889494+0000","last_peered":"2024-09-13T14:56:08.889494+0000","last_clean":"2024-09-13T14:56:08.889494+0000","last_became_active":"2024-09-13T14:52:43.587749+0000","last_became_peered":"2024-09-13T14:52:43.587749+0000","last_unstale":"2024-09-13T14:56:08.889494+0000","last_undegraded":"2024-09-13T14:56:08.889494+0000","last_fullsized":"2024-09-13T14:56:08.889494+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:31.924493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1016,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838553+0000","last_change":"2024-09-13T14:52:54.183175+0000","last_active":"2024-09-13T14:55:55.838553+0000","last_peered":"2024-09-13T14:55:55.838553+0000","last_clean":"2024-09-13T14:55:55.838553+0000","last_became_active":"2024-09-13T14:52:54.182514+0000","last_became_peered":"2024-09-13T14:52:54.182514+0000","last_unstale":"2024-09-13T14:55:55.838553+0000","last_undegraded":"2024-09-13T14:55:55.838553+0000","last_fullsized":"2024-09-13T14:55:55.838553+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:55.900845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1320,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.889421+0000","last_change":"2024-09-13T14:52:43.587069+0000","last_active":"2024-09-13T14:56:08.889421+0000","last_peered":"2024-09-13T14:56:08.889421+0000","last_clean":"2024-09-13T14:56:08.889421+0000","last_became_active":"2024-09-13T14:52:43.586196+0000","last_became_peered":"2024-09-13T14:52:43.586196+0000","last_unstale":"2024-09-13T14:56:08.889421+0000","last_undegraded":"2024-09-13T14:56:08.889421+0000","last_fullsized":"2024-09-13T14:56:08.889421+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:50.215660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1437,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890241+0000","last_change":"2024-09-13T14:39:34.318266+0000","last_active":"2024-09-13T14:56:08.890241+0000","last_peered":"2024-09-13T14:56:08.890241+0000","last_clean":"2024-09-13T14:56:08.890241+0000","last_became_active":"2024-09-13T14:39:34.315973+0000","last_became_peered":"2024-09-13T14:39:34.315973+0000","last_unstale":"2024-09-13T14:56:08.890241+0000","last_undegraded":"2024-09-13T14:56:08.890241+0000","last_fullsized":"2024-09-13T14:56:08.890241+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:15.270282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1285,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884122+0000","last_change":"2024-09-13T14:39:34.338378+0000","last_active":"2024-09-13T14:56:08.884122+0000","last_peered":"2024-09-13T14:56:08.884122+0000","last_clean":"2024-09-13T14:56:08.884122+0000","last_became_active":"2024-09-13T14:39:34.338005+0000","last_became_peered":"2024-09-13T14:39:34.338005+0000","last_unstale":"2024-09-13T14:56:08.884122+0000","last_undegraded":"2024-09-13T14:56:08.884122+0000","last_fullsized":"2024-09-13T14:56:08.884122+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:29.120379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1003,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.836420+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T14:55:55.836420+0000","last_peered":"2024-09-13T14:55:55.836420+0000","last_clean":"2024-09-13T14:55:55.836420+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T14:55:55.836420+0000","last_undegraded":"2024-09-13T14:55:55.836420+0000","last_fullsized":"2024-09-13T14:55:55.836420+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1040,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839184+0000","last_change":"2024-09-13T14:47:46.354886+0000","last_active":"2024-09-13T14:55:55.839184+0000","last_peered":"2024-09-13T14:55:55.839184+0000","last_clean":"2024-09-13T14:55:55.839184+0000","last_became_active":"2024-09-13T14:47:46.329947+0000","last_became_peered":"2024-09-13T14:47:46.329947+0000","last_unstale":"2024-09-13T14:55:55.839184+0000","last_undegraded":"2024-09-13T14:55:55.839184+0000","last_fullsized":"2024-09-13T14:55:55.839184+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:22.443920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.894185+0000","last_change":"2024-09-13T14:52:43.586530+0000","last_active":"2024-09-13T14:56:08.894185+0000","last_peered":"2024-09-13T14:56:08.894185+0000","last_clean":"2024-09-13T14:56:08.894185+0000","last_became_active":"2024-09-13T14:52:43.586107+0000","last_became_peered":"2024-09-13T14:52:43.586107+0000","last_unstale":"2024-09-13T14:56:08.894185+0000","last_undegraded":"2024-09-13T14:56:08.894185+0000","last_fullsized":"2024-09-13T14:56:08.894185+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:18.130768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":165,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885616+0000","last_change":"2024-09-13T14:47:46.344536+0000","last_active":"2024-09-13T14:56:08.885616+0000","last_peered":"2024-09-13T14:56:08.885616+0000","last_clean":"2024-09-13T14:56:08.885616+0000","last_became_active":"2024-09-13T14:47:46.321131+0000","last_became_peered":"2024-09-13T14:47:46.321131+0000","last_unstale":"2024-09-13T14:56:08.885616+0000","last_undegraded":"2024-09-13T14:56:08.885616+0000","last_fullsized":"2024-09-13T14:56:08.885616+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:31.114472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885356+0000","last_change":"2024-09-13T14:52:43.600569+0000","last_active":"2024-09-13T14:56:08.885356+0000","last_peered":"2024-09-13T14:56:08.885356+0000","last_clean":"2024-09-13T14:56:08.885356+0000","last_became_active":"2024-09-13T14:52:43.599764+0000","last_became_peered":"2024-09-13T14:52:43.599764+0000","last_unstale":"2024-09-13T14:56:08.885356+0000","last_undegraded":"2024-09-13T14:56:08.885356+0000","last_fullsized":"2024-09-13T14:56:08.885356+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:53.934456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":156,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885116+0000","last_change":"2024-09-13T14:47:46.318785+0000","last_active":"2024-09-13T14:56:08.885116+0000","last_peered":"2024-09-13T14:56:08.885116+0000","last_clean":"2024-09-13T14:56:08.885116+0000","last_became_active":"2024-09-13T14:47:46.318438+0000","last_became_peered":"2024-09-13T14:47:46.318438+0000","last_unstale":"2024-09-13T14:56:08.885116+0000","last_undegraded":"2024-09-13T14:56:08.885116+0000","last_fullsized":"2024-09-13T14:56:08.885116+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:44.745378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":999,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838629+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T14:55:55.838629+0000","last_peered":"2024-09-13T14:55:55.838629+0000","last_clean":"2024-09-13T14:55:55.838629+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T14:55:55.838629+0000","last_undegraded":"2024-09-13T14:55:55.838629+0000","last_fullsized":"2024-09-13T14:55:55.838629+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1498,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890800+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T14:56:08.890800+0000","last_peered":"2024-09-13T14:56:08.890800+0000","last_clean":"2024-09-13T14:56:08.890800+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T14:56:08.890800+0000","last_undegraded":"2024-09-13T14:56:08.890800+0000","last_fullsized":"2024-09-13T14:56:08.890800+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1283,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.889517+0000","last_change":"2024-09-13T14:39:34.349945+0000","last_active":"2024-09-13T14:56:08.889517+0000","last_peered":"2024-09-13T14:56:08.889517+0000","last_clean":"2024-09-13T14:56:08.889517+0000","last_became_active":"2024-09-13T14:39:34.346730+0000","last_became_peered":"2024-09-13T14:39:34.346730+0000","last_unstale":"2024-09-13T14:56:08.889517+0000","last_undegraded":"2024-09-13T14:56:08.889517+0000","last_fullsized":"2024-09-13T14:56:08.889517+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:48.854603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891506+0000","last_change":"2024-09-13T14:52:43.592967+0000","last_active":"2024-09-13T14:56:08.891506+0000","last_peered":"2024-09-13T14:56:08.891506+0000","last_clean":"2024-09-13T14:56:08.891506+0000","last_became_active":"2024-09-13T14:52:43.592431+0000","last_became_peered":"2024-09-13T14:52:43.592431+0000","last_unstale":"2024-09-13T14:56:08.891506+0000","last_undegraded":"2024-09-13T14:56:08.891506+0000","last_fullsized":"2024-09-13T14:56:08.891506+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:56.187568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1286,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884605+0000","last_change":"2024-09-13T14:39:34.401432+0000","last_active":"2024-09-13T14:56:08.884605+0000","last_peered":"2024-09-13T14:56:08.884605+0000","last_clean":"2024-09-13T14:56:08.884605+0000","last_became_active":"2024-09-13T14:39:34.395458+0000","last_became_peered":"2024-09-13T14:39:34.395458+0000","last_unstale":"2024-09-13T14:56:08.884605+0000","last_undegraded":"2024-09-13T14:56:08.884605+0000","last_fullsized":"2024-09-13T14:56:08.884605+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:17.007388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1427,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890453+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T14:56:08.890453+0000","last_peered":"2024-09-13T14:56:08.890453+0000","last_clean":"2024-09-13T14:56:08.890453+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T14:56:08.890453+0000","last_undegraded":"2024-09-13T14:56:08.890453+0000","last_fullsized":"2024-09-13T14:56:08.890453+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1035,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839408+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T14:55:55.839408+0000","last_peered":"2024-09-13T14:55:55.839408+0000","last_clean":"2024-09-13T14:55:55.839408+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T14:55:55.839408+0000","last_undegraded":"2024-09-13T14:55:55.839408+0000","last_fullsized":"2024-09-13T14:55:55.839408+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891074+0000","last_change":"2024-09-13T14:52:43.583860+0000","last_active":"2024-09-13T14:56:08.891074+0000","last_peered":"2024-09-13T14:56:08.891074+0000","last_clean":"2024-09-13T14:56:08.891074+0000","last_became_active":"2024-09-13T14:52:43.583055+0000","last_became_peered":"2024-09-13T14:52:43.583055+0000","last_unstale":"2024-09-13T14:56:08.891074+0000","last_undegraded":"2024-09-13T14:56:08.891074+0000","last_fullsized":"2024-09-13T14:56:08.891074+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:55.113648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.894163+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T14:56:08.894163+0000","last_peered":"2024-09-13T14:56:08.894163+0000","last_clean":"2024-09-13T14:56:08.894163+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T14:56:08.894163+0000","last_undegraded":"2024-09-13T14:56:08.894163+0000","last_fullsized":"2024-09-13T14:56:08.894163+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891249+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T14:56:08.891249+0000","last_peered":"2024-09-13T14:56:08.891249+0000","last_clean":"2024-09-13T14:56:08.891249+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T14:56:08.891249+0000","last_undegraded":"2024-09-13T14:56:08.891249+0000","last_fullsized":"2024-09-13T14:56:08.891249+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1000,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.835951+0000","last_change":"2024-09-13T14:39:34.321388+0000","last_active":"2024-09-13T14:55:55.835951+0000","last_peered":"2024-09-13T14:55:55.835951+0000","last_clean":"2024-09-13T14:55:55.835951+0000","last_became_active":"2024-09-13T14:39:34.321025+0000","last_became_peered":"2024-09-13T14:39:34.321025+0000","last_unstale":"2024-09-13T14:55:55.835951+0000","last_undegraded":"2024-09-13T14:55:55.835951+0000","last_fullsized":"2024-09-13T14:55:55.835951+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:52.593825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891158+0000","last_change":"2024-09-13T14:52:43.583382+0000","last_active":"2024-09-13T14:56:08.891158+0000","last_peered":"2024-09-13T14:56:08.891158+0000","last_clean":"2024-09-13T14:56:08.891158+0000","last_became_active":"2024-09-13T14:52:43.582870+0000","last_became_peered":"2024-09-13T14:52:43.582870+0000","last_unstale":"2024-09-13T14:56:08.891158+0000","last_undegraded":"2024-09-13T14:56:08.891158+0000","last_fullsized":"2024-09-13T14:56:08.891158+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:51.582211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":156,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885474+0000","last_change":"2024-09-13T14:47:46.324072+0000","last_active":"2024-09-13T14:56:08.885474+0000","last_peered":"2024-09-13T14:56:08.885474+0000","last_clean":"2024-09-13T14:56:08.885474+0000","last_became_active":"2024-09-13T14:47:46.322613+0000","last_became_peered":"2024-09-13T14:47:46.322613+0000","last_unstale":"2024-09-13T14:56:08.885474+0000","last_undegraded":"2024-09-13T14:56:08.885474+0000","last_fullsized":"2024-09-13T14:56:08.885474+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:45.314706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1025,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.831558+0000","last_change":"2024-09-13T14:52:43.613106+0000","last_active":"2024-09-13T14:55:55.831558+0000","last_peered":"2024-09-13T14:55:55.831558+0000","last_clean":"2024-09-13T14:55:55.831558+0000","last_became_active":"2024-09-13T14:52:43.594081+0000","last_became_peered":"2024-09-13T14:52:43.594081+0000","last_unstale":"2024-09-13T14:55:55.831558+0000","last_undegraded":"2024-09-13T14:55:55.831558+0000","last_fullsized":"2024-09-13T14:55:55.831558+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:11.121629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891097+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T14:56:08.891097+0000","last_peered":"2024-09-13T14:56:08.891097+0000","last_clean":"2024-09-13T14:56:08.891097+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T14:56:08.891097+0000","last_undegraded":"2024-09-13T14:56:08.891097+0000","last_fullsized":"2024-09-13T14:56:08.891097+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.888004+0000","last_change":"2024-09-13T14:52:43.582732+0000","last_active":"2024-09-13T14:56:08.888004+0000","last_peered":"2024-09-13T14:56:08.888004+0000","last_clean":"2024-09-13T14:56:08.888004+0000","last_became_active":"2024-09-13T14:52:43.582384+0000","last_became_peered":"2024-09-13T14:52:43.582384+0000","last_unstale":"2024-09-13T14:56:08.888004+0000","last_undegraded":"2024-09-13T14:56:08.888004+0000","last_fullsized":"2024-09-13T14:56:08.888004+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:06.715139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1336,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.879632+0000","last_change":"2024-09-13T14:39:34.322533+0000","last_active":"2024-09-13T14:56:08.879632+0000","last_peered":"2024-09-13T14:56:08.879632+0000","last_clean":"2024-09-13T14:56:08.879632+0000","last_became_active":"2024-09-13T14:39:34.322070+0000","last_became_peered":"2024-09-13T14:39:34.322070+0000","last_unstale":"2024-09-13T14:56:08.879632+0000","last_undegraded":"2024-09-13T14:56:08.879632+0000","last_fullsized":"2024-09-13T14:56:08.879632+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:43.746420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1463,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890481+0000","last_change":"2024-09-13T14:52:43.595169+0000","last_active":"2024-09-13T14:56:08.890481+0000","last_peered":"2024-09-13T14:56:08.890481+0000","last_clean":"2024-09-13T14:56:08.890481+0000","last_became_active":"2024-09-13T14:52:43.594586+0000","last_became_peered":"2024-09-13T14:52:43.594586+0000","last_unstale":"2024-09-13T14:56:08.890481+0000","last_undegraded":"2024-09-13T14:56:08.890481+0000","last_fullsized":"2024-09-13T14:56:08.890481+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:53.946025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1463,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.882712+0000","last_change":"2024-09-13T14:47:46.329033+0000","last_active":"2024-09-13T14:56:08.882712+0000","last_peered":"2024-09-13T14:56:08.882712+0000","last_clean":"2024-09-13T14:56:08.882712+0000","last_became_active":"2024-09-13T14:47:46.328203+0000","last_became_peered":"2024-09-13T14:47:46.328203+0000","last_unstale":"2024-09-13T14:56:08.882712+0000","last_undegraded":"2024-09-13T14:56:08.882712+0000","last_fullsized":"2024-09-13T14:56:08.882712+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:23.440178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1035,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839475+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T14:55:55.839475+0000","last_peered":"2024-09-13T14:55:55.839475+0000","last_clean":"2024-09-13T14:55:55.839475+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T14:55:55.839475+0000","last_undegraded":"2024-09-13T14:55:55.839475+0000","last_fullsized":"2024-09-13T14:55:55.839475+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1006,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839241+0000","last_change":"2024-09-13T14:47:46.320830+0000","last_active":"2024-09-13T14:55:55.839241+0000","last_peered":"2024-09-13T14:55:55.839241+0000","last_clean":"2024-09-13T14:55:55.839241+0000","last_became_active":"2024-09-13T14:47:46.320443+0000","last_became_peered":"2024-09-13T14:47:46.320443+0000","last_unstale":"2024-09-13T14:55:55.839241+0000","last_undegraded":"2024-09-13T14:55:55.839241+0000","last_fullsized":"2024-09-13T14:55:55.839241+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:46.279580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1285,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.887397+0000","last_change":"2024-09-13T14:39:34.356129+0000","last_active":"2024-09-13T14:56:08.887397+0000","last_peered":"2024-09-13T14:56:08.887397+0000","last_clean":"2024-09-13T14:56:08.887397+0000","last_became_active":"2024-09-13T14:39:34.355288+0000","last_became_peered":"2024-09-13T14:39:34.355288+0000","last_unstale":"2024-09-13T14:56:08.887397+0000","last_undegraded":"2024-09-13T14:56:08.887397+0000","last_fullsized":"2024-09-13T14:56:08.887397+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.725003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":996,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.835370+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T14:55:55.835370+0000","last_peered":"2024-09-13T14:55:55.835370+0000","last_clean":"2024-09-13T14:55:55.835370+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T14:55:55.835370+0000","last_undegraded":"2024-09-13T14:55:55.835370+0000","last_fullsized":"2024-09-13T14:55:55.835370+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1103,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839553+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T14:55:55.839553+0000","last_peered":"2024-09-13T14:55:55.839553+0000","last_clean":"2024-09-13T14:55:55.839553+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T14:55:55.839553+0000","last_undegraded":"2024-09-13T14:55:55.839553+0000","last_fullsized":"2024-09-13T14:55:55.839553+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891129+0000","last_change":"2024-09-13T14:52:43.591841+0000","last_active":"2024-09-13T14:56:08.891129+0000","last_peered":"2024-09-13T14:56:08.891129+0000","last_clean":"2024-09-13T14:56:08.891129+0000","last_became_active":"2024-09-13T14:52:43.591187+0000","last_became_peered":"2024-09-13T14:52:43.591187+0000","last_unstale":"2024-09-13T14:56:08.891129+0000","last_undegraded":"2024-09-13T14:56:08.891129+0000","last_fullsized":"2024-09-13T14:56:08.891129+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:29.136085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1323,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.889586+0000","last_change":"2024-09-13T14:52:43.589345+0000","last_active":"2024-09-13T14:56:08.889586+0000","last_peered":"2024-09-13T14:56:08.889586+0000","last_clean":"2024-09-13T14:56:08.889586+0000","last_became_active":"2024-09-13T14:52:43.588829+0000","last_became_peered":"2024-09-13T14:52:43.588829+0000","last_unstale":"2024-09-13T14:56:08.889586+0000","last_undegraded":"2024-09-13T14:56:08.889586+0000","last_fullsized":"2024-09-13T14:56:08.889586+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:49.701805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884433+0000","last_change":"2024-09-13T14:52:43.599179+0000","last_active":"2024-09-13T14:56:08.884433+0000","last_peered":"2024-09-13T14:56:08.884433+0000","last_clean":"2024-09-13T14:56:08.884433+0000","last_became_active":"2024-09-13T14:52:43.598166+0000","last_became_peered":"2024-09-13T14:52:43.598166+0000","last_unstale":"2024-09-13T14:56:08.884433+0000","last_undegraded":"2024-09-13T14:56:08.884433+0000","last_fullsized":"2024-09-13T14:56:08.884433+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:08.647786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":155,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885068+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T14:56:08.885068+0000","last_peered":"2024-09-13T14:56:08.885068+0000","last_clean":"2024-09-13T14:56:08.885068+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T14:56:08.885068+0000","last_undegraded":"2024-09-13T14:56:08.885068+0000","last_fullsized":"2024-09-13T14:56:08.885068+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884807+0000","last_change":"2024-09-13T14:52:43.597595+0000","last_active":"2024-09-13T14:56:08.884807+0000","last_peered":"2024-09-13T14:56:08.884807+0000","last_clean":"2024-09-13T14:56:08.884807+0000","last_became_active":"2024-09-13T14:52:43.596592+0000","last_became_peered":"2024-09-13T14:52:43.596592+0000","last_unstale":"2024-09-13T14:56:08.884807+0000","last_undegraded":"2024-09-13T14:56:08.884807+0000","last_fullsized":"2024-09-13T14:56:08.884807+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:56.532603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1044,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838947+0000","last_change":"2024-09-13T14:52:43.596429+0000","last_active":"2024-09-13T14:55:55.838947+0000","last_peered":"2024-09-13T14:55:55.838947+0000","last_clean":"2024-09-13T14:55:55.838947+0000","last_became_active":"2024-09-13T14:52:43.594614+0000","last_became_peered":"2024-09-13T14:52:43.594614+0000","last_unstale":"2024-09-13T14:55:55.838947+0000","last_undegraded":"2024-09-13T14:55:55.838947+0000","last_fullsized":"2024-09-13T14:55:55.838947+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:41.970225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":155,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885520+0000","last_change":"2024-09-13T14:47:46.325496+0000","last_active":"2024-09-13T14:56:08.885520+0000","last_peered":"2024-09-13T14:56:08.885520+0000","last_clean":"2024-09-13T14:56:08.885520+0000","last_became_active":"2024-09-13T14:47:46.325099+0000","last_became_peered":"2024-09-13T14:47:46.325099+0000","last_unstale":"2024-09-13T14:56:08.885520+0000","last_undegraded":"2024-09-13T14:56:08.885520+0000","last_fullsized":"2024-09-13T14:56:08.885520+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:48.803608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1320,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884739+0000","last_change":"2024-09-13T14:47:46.349682+0000","last_active":"2024-09-13T14:56:08.884739+0000","last_peered":"2024-09-13T14:56:08.884739+0000","last_clean":"2024-09-13T14:56:08.884739+0000","last_became_active":"2024-09-13T14:47:46.341656+0000","last_became_peered":"2024-09-13T14:47:46.341656+0000","last_unstale":"2024-09-13T14:56:08.884739+0000","last_undegraded":"2024-09-13T14:56:08.884739+0000","last_fullsized":"2024-09-13T14:56:08.884739+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:02.727169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1483,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890536+0000","last_change":"2024-09-13T14:52:54.185117+0000","last_active":"2024-09-13T14:56:08.890536+0000","last_peered":"2024-09-13T14:56:08.890536+0000","last_clean":"2024-09-13T14:56:08.890536+0000","last_became_active":"2024-09-13T14:52:54.184712+0000","last_became_peered":"2024-09-13T14:52:54.184712+0000","last_unstale":"2024-09-13T14:56:08.890536+0000","last_undegraded":"2024-09-13T14:56:08.890536+0000","last_fullsized":"2024-09-13T14:56:08.890536+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:32.654401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":983,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.831348+0000","last_change":"2024-09-13T14:39:34.393673+0000","last_active":"2024-09-13T14:55:55.831348+0000","last_peered":"2024-09-13T14:55:55.831348+0000","last_clean":"2024-09-13T14:55:55.831348+0000","last_became_active":"2024-09-13T14:39:34.393186+0000","last_became_peered":"2024-09-13T14:39:34.393186+0000","last_unstale":"2024-09-13T14:55:55.831348+0000","last_undegraded":"2024-09-13T14:55:55.831348+0000","last_fullsized":"2024-09-13T14:55:55.831348+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:23.130084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1006,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838571+0000","last_change":"2024-09-13T14:52:43.581604+0000","last_active":"2024-09-13T14:55:55.838571+0000","last_peered":"2024-09-13T14:55:55.838571+0000","last_clean":"2024-09-13T14:55:55.838571+0000","last_became_active":"2024-09-13T14:52:43.579981+0000","last_became_peered":"2024-09-13T14:52:43.579981+0000","last_unstale":"2024-09-13T14:55:55.838571+0000","last_undegraded":"2024-09-13T14:55:55.838571+0000","last_fullsized":"2024-09-13T14:55:55.838571+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:59.366292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":971,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839329+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:55:55.839329+0000","last_peered":"2024-09-13T14:55:55.839329+0000","last_clean":"2024-09-13T14:55:55.839329+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:55:55.839329+0000","last_undegraded":"2024-09-13T14:55:55.839329+0000","last_fullsized":"2024-09-13T14:55:55.839329+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885178+0000","last_change":"2024-09-13T14:52:43.595628+0000","last_active":"2024-09-13T14:56:08.885178+0000","last_peered":"2024-09-13T14:56:08.885178+0000","last_clean":"2024-09-13T14:56:08.885178+0000","last_became_active":"2024-09-13T14:52:43.594675+0000","last_became_peered":"2024-09-13T14:52:43.594675+0000","last_unstale":"2024-09-13T14:56:08.885178+0000","last_undegraded":"2024-09-13T14:56:08.885178+0000","last_fullsized":"2024-09-13T14:56:08.885178+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:40:19.850839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1062,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838056+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T14:55:55.838056+0000","last_peered":"2024-09-13T14:55:55.838056+0000","last_clean":"2024-09-13T14:55:55.838056+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T14:55:55.838056+0000","last_undegraded":"2024-09-13T14:55:55.838056+0000","last_fullsized":"2024-09-13T14:55:55.838056+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":965,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.840054+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:55:55.840054+0000","last_peered":"2024-09-13T14:55:55.840054+0000","last_clean":"2024-09-13T14:55:55.840054+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:55:55.840054+0000","last_undegraded":"2024-09-13T14:55:55.840054+0000","last_fullsized":"2024-09-13T14:55:55.840054+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1033,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.842761+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T14:55:55.842761+0000","last_peered":"2024-09-13T14:55:55.842761+0000","last_clean":"2024-09-13T14:55:55.842761+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T14:55:55.842761+0000","last_undegraded":"2024-09-13T14:55:55.842761+0000","last_fullsized":"2024-09-13T14:55:55.842761+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":999,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838489+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T14:55:55.838489+0000","last_peered":"2024-09-13T14:55:55.838489+0000","last_clean":"2024-09-13T14:55:55.838489+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T14:55:55.838489+0000","last_undegraded":"2024-09-13T14:55:55.838489+0000","last_fullsized":"2024-09-13T14:55:55.838489+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1321,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884456+0000","last_change":"2024-09-13T14:52:43.592945+0000","last_active":"2024-09-13T14:56:08.884456+0000","last_peered":"2024-09-13T14:56:08.884456+0000","last_clean":"2024-09-13T14:56:08.884456+0000","last_became_active":"2024-09-13T14:52:43.592602+0000","last_became_peered":"2024-09-13T14:52:43.592602+0000","last_unstale":"2024-09-13T14:56:08.884456+0000","last_undegraded":"2024-09-13T14:56:08.884456+0000","last_fullsized":"2024-09-13T14:56:08.884456+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:11.719764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1006,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.837915+0000","last_change":"2024-09-13T14:47:46.322428+0000","last_active":"2024-09-13T14:55:55.837915+0000","last_peered":"2024-09-13T14:55:55.837915+0000","last_clean":"2024-09-13T14:55:55.837915+0000","last_became_active":"2024-09-13T14:47:46.321955+0000","last_became_peered":"2024-09-13T14:47:46.321955+0000","last_unstale":"2024-09-13T14:55:55.837915+0000","last_undegraded":"2024-09-13T14:55:55.837915+0000","last_fullsized":"2024-09-13T14:55:55.837915+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:47.614065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1437,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890717+0000","last_change":"2024-09-13T14:39:34.362694+0000","last_active":"2024-09-13T14:56:08.890717+0000","last_peered":"2024-09-13T14:56:08.890717+0000","last_clean":"2024-09-13T14:56:08.890717+0000","last_became_active":"2024-09-13T14:39:34.362435+0000","last_became_peered":"2024-09-13T14:39:34.362435+0000","last_unstale":"2024-09-13T14:56:08.890717+0000","last_undegraded":"2024-09-13T14:56:08.890717+0000","last_fullsized":"2024-09-13T14:56:08.890717+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:33.350328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1437,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.887142+0000","last_change":"2024-09-13T14:39:34.360941+0000","last_active":"2024-09-13T14:56:08.887142+0000","last_peered":"2024-09-13T14:56:08.887142+0000","last_clean":"2024-09-13T14:56:08.887142+0000","last_became_active":"2024-09-13T14:39:34.360383+0000","last_became_peered":"2024-09-13T14:39:34.360383+0000","last_unstale":"2024-09-13T14:56:08.887142+0000","last_undegraded":"2024-09-13T14:56:08.887142+0000","last_fullsized":"2024-09-13T14:56:08.887142+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:51.344296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":982,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.835813+0000","last_change":"2024-09-13T14:39:34.380634+0000","last_active":"2024-09-13T14:55:55.835813+0000","last_peered":"2024-09-13T14:55:55.835813+0000","last_clean":"2024-09-13T14:55:55.835813+0000","last_became_active":"2024-09-13T14:39:34.379785+0000","last_became_peered":"2024-09-13T14:39:34.379785+0000","last_unstale":"2024-09-13T14:55:55.835813+0000","last_undegraded":"2024-09-13T14:55:55.835813+0000","last_fullsized":"2024-09-13T14:55:55.835813+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:02.436193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1039,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.837955+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T14:55:55.837955+0000","last_peered":"2024-09-13T14:55:55.837955+0000","last_clean":"2024-09-13T14:55:55.837955+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T14:55:55.837955+0000","last_undegraded":"2024-09-13T14:55:55.837955+0000","last_fullsized":"2024-09-13T14:55:55.837955+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1329,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884649+0000","last_change":"2024-09-13T14:52:54.156534+0000","last_active":"2024-09-13T14:56:08.884649+0000","last_peered":"2024-09-13T14:56:08.884649+0000","last_clean":"2024-09-13T14:56:08.884649+0000","last_became_active":"2024-09-13T14:52:54.156241+0000","last_became_peered":"2024-09-13T14:52:54.156241+0000","last_unstale":"2024-09-13T14:56:08.884649+0000","last_undegraded":"2024-09-13T14:56:08.884649+0000","last_fullsized":"2024-09-13T14:56:08.884649+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:17.998268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":42,"num_pg":16,"stat_sum":{"num_bytes":3326869528,"num_objects":50765,"num_object_clones":0,"num_object_copies":152295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50765,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":164911,"num_write_kb":6901569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10021646336,"data_stored":10021634120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142011,"ondisk_log_size":142011,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397942,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1963176,"kb_used_data":1653344,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91781976,"statfs":{"total":95995035648,"available":93984743424,"internally_reserved":0,"allocated":1693024256,"data_stored":1689074764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594177,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1133604,"kb_used_data":823772,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92611548,"statfs":{"total":95995035648,"available":94834225152,"internally_reserved":0,"allocated":843542528,"data_stored":839597029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790411,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2158808,"kb_used_data":1857104,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91586344,"statfs":{"total":95995035648,"available":93784416256,"internally_reserved":0,"allocated":1901674496,"data_stored":1897736559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980831940,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1526752,"kb_used_data":1195736,"kb_used_omap":76,"kb_used_meta":330931,"kb_avail":92218400,"statfs":{"total":95995035648,"available":94431641600,"internally_reserved":0,"allocated":1224433664,"data_stored":1220491762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":338874176},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911178,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1541708,"kb_used_data":1253572,"kb_used_omap":77,"kb_used_meta":288050,"kb_avail":92203444,"statfs":{"total":95995035648,"available":94416326656,"internally_reserved":0,"allocated":1283657728,"data_stored":1279731514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":294964021},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023121,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1412752,"kb_used_data":1067400,"kb_used_omap":71,"kb_used_meta":345272,"kb_avail":92332400,"statfs":{"total":95995035648,"available":94548377600,"internally_reserved":0,"allocated":1093017600,"data_stored":1089087812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":353559200},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004251,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1458704,"kb_used_data":1256072,"kb_used_omap":66,"kb_used_meta":202557,"kb_avail":92286448,"statfs":{"total":95995035648,"available":94501322752,"internally_reserved":0,"allocated":1286217728,"data_stored":1282287379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":207418538},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195443,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1185616,"kb_used_data":859592,"kb_used_omap":56,"kb_used_meta":325959,"kb_avail":92559536,"statfs":{"total":95995035648,"available":94780964864,"internally_reserved":0,"allocated":880222208,"data_stored":876290830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":333782159},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":856948736,"data_stored":856948736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1255149568,"data_stored":1255145496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1078132736,"data_stored":1078132736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1263927296,"data_stored":1263927296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1196163072,"data_stored":1196163072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1868894208,"data_stored":1868890136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":830541824,"data_stored":830537752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1671888896,"data_stored":1671888896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:56:19.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:56:19.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:19.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:19.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:19.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:19.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:19.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:19.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:19.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:19.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:20.005 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.032 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:20.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:20.147 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:19 smithi099 bash[84576]: cluster 2024-09-13T14:56:18.679084+0000 mgr.y (mgr.26252) 902 : cluster [DBG] pgmap v1423: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:20.147 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:19 smithi099 bash[84576]: cluster 2024-09-13T14:56:18.679084+0000 mgr.y (mgr.26252) 902 : cluster [DBG] pgmap v1423: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:20.147 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:19 smithi099 bash[87147]: cluster 2024-09-13T14:56:18.679084+0000 mgr.y (mgr.26252) 902 : cluster [DBG] pgmap v1423: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:20.147 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:19 smithi099 bash[87147]: cluster 2024-09-13T14:56:18.679084+0000 mgr.y (mgr.26252) 902 : cluster [DBG] pgmap v1423: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:20.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:20.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:20.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:19 smithi177 bash[68594]: cluster 2024-09-13T14:56:18.679084+0000 mgr.y (mgr.26252) 902 : cluster [DBG] pgmap v1423: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:20.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:19 smithi177 bash[68594]: cluster 2024-09-13T14:56:18.679084+0000 mgr.y (mgr.26252) 902 : cluster [DBG] pgmap v1423: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:56:20.267 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:20.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:20.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:20.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:56:20 smithi177 bash[61439]: ts=2024-09-13T14:56:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:20.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:20.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:20.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:20.521 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:20.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:20.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:20.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:20.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:20.842 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:20.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:20.947 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:20.973 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:20.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:21.039 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:21.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:21.116 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:21.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:21.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:21.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:21.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:21.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:21.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:21.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:21.504 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:21.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:21.609 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:21.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:21.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:21.732 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:21.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:21.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:21.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:21.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:21.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:21.998 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:22.006 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:21 smithi099 bash[84576]: cluster 2024-09-13T14:56:20.679600+0000 mgr.y (mgr.26252) 903 : cluster [DBG] pgmap v1424: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-13T14:56:22.006 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:21 smithi099 bash[84576]: cluster 2024-09-13T14:56:20.679600+0000 mgr.y (mgr.26252) 903 : cluster [DBG] pgmap v1424: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-13T14:56:22.006 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:21 smithi099 bash[87147]: cluster 2024-09-13T14:56:20.679600+0000 mgr.y (mgr.26252) 903 : cluster [DBG] pgmap v1424: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-13T14:56:22.006 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:21 smithi099 bash[87147]: cluster 2024-09-13T14:56:20.679600+0000 mgr.y (mgr.26252) 903 : cluster [DBG] pgmap v1424: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-13T14:56:22.022 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:22.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:22.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:22.156 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:22.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:22.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:22.284 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:22.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:22.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:21 smithi177 bash[68594]: cluster 2024-09-13T14:56:20.679600+0000 mgr.y (mgr.26252) 903 : cluster [DBG] pgmap v1424: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-13T14:56:22.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:21 smithi177 bash[68594]: cluster 2024-09-13T14:56:20.679600+0000 mgr.y (mgr.26252) 903 : cluster [DBG] pgmap v1424: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-13T14:56:22.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:22.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:22.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:22.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:22.547 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:22.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:22.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:22.678 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:22.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:22.804 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:22.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:22.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:22.937 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:22.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:23.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:23.062 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:23.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:23.077 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:23.164 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:23.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:23.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:23.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:23.312 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:23.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:23.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:23.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:23.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:23.542 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:23.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:23.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:23.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:23.714 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:23.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:23.842 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:23.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:23.946 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:23.956 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:23 smithi099 bash[84576]: cluster 2024-09-13T14:56:22.680060+0000 mgr.y (mgr.26252) 904 : cluster [DBG] pgmap v1425: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:56:23.956 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:23 smithi099 bash[84576]: cluster 2024-09-13T14:56:22.680060+0000 mgr.y (mgr.26252) 904 : cluster [DBG] pgmap v1425: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:56:23.956 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:23 smithi099 bash[87147]: cluster 2024-09-13T14:56:22.680060+0000 mgr.y (mgr.26252) 904 : cluster [DBG] pgmap v1425: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:56:23.956 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:23 smithi099 bash[87147]: cluster 2024-09-13T14:56:22.680060+0000 mgr.y (mgr.26252) 904 : cluster [DBG] pgmap v1425: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:56:23.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:23.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:24.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:24.091 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:24.091 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:24.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:24.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:24.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:24.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:24.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:24.336 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:24.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:23 smithi177 bash[68594]: cluster 2024-09-13T14:56:22.680060+0000 mgr.y (mgr.26252) 904 : cluster [DBG] pgmap v1425: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:56:24.347 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:23 smithi177 bash[68594]: cluster 2024-09-13T14:56:22.680060+0000 mgr.y (mgr.26252) 904 : cluster [DBG] pgmap v1425: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:56:24.382 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:24.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:24.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:24.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:24.631 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:24.654 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:24.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:24.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:24.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:24.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:24.912 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:24.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:25.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:25.044 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:25.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:25.146 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:25.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:25.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:25.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:25.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:25.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:25.411 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:25.432 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:25.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:25.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:25.561 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:25.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:25.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:25.697 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:25.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:25.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:25.834 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:25.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:25.943 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:25.968 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:25.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:26.054 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:25 smithi099 bash[84576]: cluster 2024-09-13T14:56:24.680586+0000 mgr.y (mgr.26252) 905 : cluster [DBG] pgmap v1426: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-09-13T14:56:26.054 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:25 smithi099 bash[84576]: cluster 2024-09-13T14:56:24.680586+0000 mgr.y (mgr.26252) 905 : cluster [DBG] pgmap v1426: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-09-13T14:56:26.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:25 smithi099 bash[87147]: cluster 2024-09-13T14:56:24.680586+0000 mgr.y (mgr.26252) 905 : cluster [DBG] pgmap v1426: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-09-13T14:56:26.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:25 smithi099 bash[87147]: cluster 2024-09-13T14:56:24.680586+0000 mgr.y (mgr.26252) 905 : cluster [DBG] pgmap v1426: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-09-13T14:56:26.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:26.099 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:26.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:25 smithi177 bash[68594]: cluster 2024-09-13T14:56:24.680586+0000 mgr.y (mgr.26252) 905 : cluster [DBG] pgmap v1426: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-09-13T14:56:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:25 smithi177 bash[68594]: cluster 2024-09-13T14:56:24.680586+0000 mgr.y (mgr.26252) 905 : cluster [DBG] pgmap v1426: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-09-13T14:56:26.205 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:26.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:26.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:26.330 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:26.362 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:26.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:26.469 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:26.491 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:26.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:26.598 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:26.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:26.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:26.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:26.755 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:26.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:26.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:26.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:26.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:26.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:27.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:27.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:27.122 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:27.153 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:27.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:27.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:27.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:27.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:27.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:56:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:56:27] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.51.0" 2024-09-13T14:56:27.397 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:27.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:27.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:27.565 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:27.591 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:27.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:27.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:27.715 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:27.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:27.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:27.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:27.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:27.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:27.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:27.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:28.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:28.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:27 smithi177 bash[68594]: cluster 2024-09-13T14:56:26.680978+0000 mgr.y (mgr.26252) 906 : cluster [DBG] pgmap v1427: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-13T14:56:28.085 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:27 smithi177 bash[68594]: cluster 2024-09-13T14:56:26.680978+0000 mgr.y (mgr.26252) 906 : cluster [DBG] pgmap v1427: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-13T14:56:28.119 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:28.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:28.225 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:28.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:28.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:28.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:28.366 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:27 smithi099 bash[84576]: cluster 2024-09-13T14:56:26.680978+0000 mgr.y (mgr.26252) 906 : cluster [DBG] pgmap v1427: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-13T14:56:28.366 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:27 smithi099 bash[84576]: cluster 2024-09-13T14:56:26.680978+0000 mgr.y (mgr.26252) 906 : cluster [DBG] pgmap v1427: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-13T14:56:28.366 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:27 smithi099 bash[87147]: cluster 2024-09-13T14:56:26.680978+0000 mgr.y (mgr.26252) 906 : cluster [DBG] pgmap v1427: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-13T14:56:28.367 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:27 smithi099 bash[87147]: cluster 2024-09-13T14:56:26.680978+0000 mgr.y (mgr.26252) 906 : cluster [DBG] pgmap v1427: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2024-09-13T14:56:28.383 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:28.486 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:28.506 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:28.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:28.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:28.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:28.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:28.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:28.771 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:28.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:28.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:28.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:28.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:28.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:29.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:29.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:29.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:29.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:29.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:29.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:29.350 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:29.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:29.455 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:29.474 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:29.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:29.577 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:29.611 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:29.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:29.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:29.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:29.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:29.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:29.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:29.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:29.977 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:30.003 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:30.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:30.006 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:29 smithi099 bash[84576]: cluster 2024-09-13T14:56:28.681486+0000 mgr.y (mgr.26252) 907 : cluster [DBG] pgmap v1428: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2024-09-13T14:56:30.006 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:29 smithi099 bash[84576]: cluster 2024-09-13T14:56:28.681486+0000 mgr.y (mgr.26252) 907 : cluster [DBG] pgmap v1428: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2024-09-13T14:56:30.006 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:29 smithi099 bash[87147]: cluster 2024-09-13T14:56:28.681486+0000 mgr.y (mgr.26252) 907 : cluster [DBG] pgmap v1428: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2024-09-13T14:56:30.006 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:29 smithi099 bash[87147]: cluster 2024-09-13T14:56:28.681486+0000 mgr.y (mgr.26252) 907 : cluster [DBG] pgmap v1428: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2024-09-13T14:56:30.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.134 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:30.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:29 smithi177 bash[68594]: cluster 2024-09-13T14:56:28.681486+0000 mgr.y (mgr.26252) 907 : cluster [DBG] pgmap v1428: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2024-09-13T14:56:30.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:29 smithi177 bash[68594]: cluster 2024-09-13T14:56:28.681486+0000 mgr.y (mgr.26252) 907 : cluster [DBG] pgmap v1428: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2024-09-13T14:56:30.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:30.267 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:30.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:30.373 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:30.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:30.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.516 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:56:30 smithi177 bash[61439]: ts=2024-09-13T14:56:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:30.531 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:30.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:30.671 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:30.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:30.774 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:30.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:30.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:30.908 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:31.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:31.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:31.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:31.166 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:31.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:31.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:31.339 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:31.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:31.442 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:31.464 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:31.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:31.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:31.586 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:31.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:31.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:31.745 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:31.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:31.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:31.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:31.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:31.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:31.999 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:31 smithi177 bash[68594]: cluster 2024-09-13T14:56:30.683234+0000 mgr.y (mgr.26252) 908 : cluster [DBG] pgmap v1429: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2024-09-13T14:56:31.999 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:31 smithi177 bash[68594]: cluster 2024-09-13T14:56:30.683234+0000 mgr.y (mgr.26252) 908 : cluster [DBG] pgmap v1429: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2024-09-13T14:56:31.999 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:31 smithi177 bash[68594]: audit 2024-09-13T14:56:31.235390+0000 mon.a (mon.0) 1960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:31.999 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:31 smithi177 bash[68594]: audit 2024-09-13T14:56:31.235390+0000 mon.a (mon.0) 1960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:32.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:32.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:32.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:32.163 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:32.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:32.269 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:32.287 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:32.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:32.354 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:31 smithi099 bash[84576]: cluster 2024-09-13T14:56:30.683234+0000 mgr.y (mgr.26252) 908 : cluster [DBG] pgmap v1429: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2024-09-13T14:56:32.355 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:31 smithi099 bash[84576]: cluster 2024-09-13T14:56:30.683234+0000 mgr.y (mgr.26252) 908 : cluster [DBG] pgmap v1429: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2024-09-13T14:56:32.355 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:31 smithi099 bash[84576]: audit 2024-09-13T14:56:31.235390+0000 mon.a (mon.0) 1960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:32.355 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:31 smithi099 bash[84576]: audit 2024-09-13T14:56:31.235390+0000 mon.a (mon.0) 1960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:32.355 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:31 smithi099 bash[87147]: cluster 2024-09-13T14:56:30.683234+0000 mgr.y (mgr.26252) 908 : cluster [DBG] pgmap v1429: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2024-09-13T14:56:32.355 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:31 smithi099 bash[87147]: cluster 2024-09-13T14:56:30.683234+0000 mgr.y (mgr.26252) 908 : cluster [DBG] pgmap v1429: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2024-09-13T14:56:32.355 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:31 smithi099 bash[87147]: audit 2024-09-13T14:56:31.235390+0000 mon.a (mon.0) 1960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:32.355 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:31 smithi099 bash[87147]: audit 2024-09-13T14:56:31.235390+0000 mon.a (mon.0) 1960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:32.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:32.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:32.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:32.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:32.541 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:32.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:32.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:32.661 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:32.764 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:32.786 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:32.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:32.890 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:32.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:32.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:33.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:33.046 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:33.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:33.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:33.170 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:33.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:33.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:33.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:33.399 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:33.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:33.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:33.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:33.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:33.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:33.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:33.698 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:33.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:33.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:33.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:33.938 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:33.973 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:33.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:34.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:34.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:34.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:34.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:34.214 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:33 smithi099 bash[84576]: cluster 2024-09-13T14:56:32.683740+0000 mgr.y (mgr.26252) 909 : cluster [DBG] pgmap v1430: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2024-09-13T14:56:34.214 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:33 smithi099 bash[84576]: cluster 2024-09-13T14:56:32.683740+0000 mgr.y (mgr.26252) 909 : cluster [DBG] pgmap v1430: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2024-09-13T14:56:34.214 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:33 smithi099 bash[84576]: audit 2024-09-13T14:56:33.284668+0000 mon.a (mon.0) 1961 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:34.214 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:33 smithi099 bash[84576]: audit 2024-09-13T14:56:33.284668+0000 mon.a (mon.0) 1961 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:34.215 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:33 smithi099 bash[87147]: cluster 2024-09-13T14:56:32.683740+0000 mgr.y (mgr.26252) 909 : cluster [DBG] pgmap v1430: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2024-09-13T14:56:34.215 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:33 smithi099 bash[87147]: cluster 2024-09-13T14:56:32.683740+0000 mgr.y (mgr.26252) 909 : cluster [DBG] pgmap v1430: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2024-09-13T14:56:34.215 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:33 smithi099 bash[87147]: audit 2024-09-13T14:56:33.284668+0000 mon.a (mon.0) 1961 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:34.215 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:33 smithi099 bash[87147]: audit 2024-09-13T14:56:33.284668+0000 mon.a (mon.0) 1961 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:34.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:34.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:34.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:34.359 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:34.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:33 smithi177 bash[68594]: cluster 2024-09-13T14:56:32.683740+0000 mgr.y (mgr.26252) 909 : cluster [DBG] pgmap v1430: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2024-09-13T14:56:34.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:33 smithi177 bash[68594]: cluster 2024-09-13T14:56:32.683740+0000 mgr.y (mgr.26252) 909 : cluster [DBG] pgmap v1430: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2024-09-13T14:56:34.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:33 smithi177 bash[68594]: audit 2024-09-13T14:56:33.284668+0000 mon.a (mon.0) 1961 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:34.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:33 smithi177 bash[68594]: audit 2024-09-13T14:56:33.284668+0000 mon.a (mon.0) 1961 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:34.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:34.466 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:34.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:34.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:34.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:34.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:34.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:34.715 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:34.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:34.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:34.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:34.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:34.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:35.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:35.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:35.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:35.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:35.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:35.225 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:56:35.226 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:35.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:35.253 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:56:35.253 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:35.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:35.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:35.302 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:56:35.303 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:35.335 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:56:35.336 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:35.367 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:56:35.367 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:35.388 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:56:35.389 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:35.398 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:35.417 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:56:35.417 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:35.423 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:35.456 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:56:35.457 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:35.527 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:35.555 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:35.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:35.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:35.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:35.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:35.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:35.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:35.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:35.990 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:35.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:36.096 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:36.117 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:36.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:36.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:36.228 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:35 smithi099 bash[84576]: cluster 2024-09-13T14:56:34.684375+0000 mgr.y (mgr.26252) 910 : cluster [DBG] pgmap v1431: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:56:36.229 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:35 smithi099 bash[84576]: cluster 2024-09-13T14:56:34.684375+0000 mgr.y (mgr.26252) 910 : cluster [DBG] pgmap v1431: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:56:36.229 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:35 smithi099 bash[84576]: audit 2024-09-13T14:56:35.200070+0000 mon.a (mon.0) 1962 : audit [DBG] from='client.? 172.21.15.99:0/3210561683' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:36.229 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:36 smithi099 bash[84576]: audit 2024-09-13T14:56:35.200070+0000 mon.a (mon.0) 1962 : audit [DBG] from='client.? 172.21.15.99:0/3210561683' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:36.229 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:36 smithi099 bash[87147]: cluster 2024-09-13T14:56:34.684375+0000 mgr.y (mgr.26252) 910 : cluster [DBG] pgmap v1431: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:56:36.229 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:36 smithi099 bash[87147]: cluster 2024-09-13T14:56:34.684375+0000 mgr.y (mgr.26252) 910 : cluster [DBG] pgmap v1431: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:56:36.229 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:36 smithi099 bash[87147]: audit 2024-09-13T14:56:35.200070+0000 mon.a (mon.0) 1962 : audit [DBG] from='client.? 172.21.15.99:0/3210561683' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:36.229 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:36 smithi099 bash[87147]: audit 2024-09-13T14:56:35.200070+0000 mon.a (mon.0) 1962 : audit [DBG] from='client.? 172.21.15.99:0/3210561683' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:36.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:36.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:36.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:35 smithi177 bash[68594]: cluster 2024-09-13T14:56:34.684375+0000 mgr.y (mgr.26252) 910 : cluster [DBG] pgmap v1431: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:56:36.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:35 smithi177 bash[68594]: cluster 2024-09-13T14:56:34.684375+0000 mgr.y (mgr.26252) 910 : cluster [DBG] pgmap v1431: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T14:56:36.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:35 smithi177 bash[68594]: audit 2024-09-13T14:56:35.200070+0000 mon.a (mon.0) 1962 : audit [DBG] from='client.? 172.21.15.99:0/3210561683' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:36.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:35 smithi177 bash[68594]: audit 2024-09-13T14:56:35.200070+0000 mon.a (mon.0) 1962 : audit [DBG] from='client.? 172.21.15.99:0/3210561683' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:36.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:36.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:36.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:36.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:36.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:36.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:36.652 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:36.756 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:36.813 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:36.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:36.919 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:36.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:36.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:37.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:37.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:37.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:37.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:37.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:37.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:56:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:56:37] "GET /metrics HTTP/1.1" 200 36768 "" "Prometheus/2.51.0" 2024-09-13T14:56:37.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:37.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:37.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:37.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:37.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:37.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:37.589 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:37.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:37.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:37.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:37.739 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:37.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:37.844 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:37.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:37.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:37.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:38.008 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:37 smithi177 bash[68594]: cluster 2024-09-13T14:56:36.684794+0000 mgr.y (mgr.26252) 911 : cluster [DBG] pgmap v1432: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2024-09-13T14:56:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:37 smithi177 bash[68594]: cluster 2024-09-13T14:56:36.684794+0000 mgr.y (mgr.26252) 911 : cluster [DBG] pgmap v1432: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2024-09-13T14:56:38.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:38.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:38.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:38.156 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:37 smithi099 bash[84576]: cluster 2024-09-13T14:56:36.684794+0000 mgr.y (mgr.26252) 911 : cluster [DBG] pgmap v1432: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2024-09-13T14:56:38.156 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:37 smithi099 bash[84576]: cluster 2024-09-13T14:56:36.684794+0000 mgr.y (mgr.26252) 911 : cluster [DBG] pgmap v1432: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2024-09-13T14:56:38.156 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:37 smithi099 bash[87147]: cluster 2024-09-13T14:56:36.684794+0000 mgr.y (mgr.26252) 911 : cluster [DBG] pgmap v1432: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2024-09-13T14:56:38.156 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:37 smithi099 bash[87147]: cluster 2024-09-13T14:56:36.684794+0000 mgr.y (mgr.26252) 911 : cluster [DBG] pgmap v1432: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2024-09-13T14:56:38.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:38.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:38.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:38.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:38.419 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:38.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:38.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:38.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:38.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:38.683 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:38.702 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:38.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:38.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:38.839 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:38.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:38.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:38.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:38.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:39.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:39.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:39.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:39.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:39.252 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:39.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:39.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[84576]: audit 2024-09-13T14:56:38.066823+0000 mon.a (mon.0) 1963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[84576]: audit 2024-09-13T14:56:38.066823+0000 mon.a (mon.0) 1963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[84576]: audit 2024-09-13T14:56:38.076832+0000 mon.a (mon.0) 1964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[84576]: audit 2024-09-13T14:56:38.076832+0000 mon.a (mon.0) 1964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[84576]: cluster 2024-09-13T14:56:38.685433+0000 mgr.y (mgr.26252) 912 : cluster [DBG] pgmap v1433: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T14:56:39.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[84576]: cluster 2024-09-13T14:56:38.685433+0000 mgr.y (mgr.26252) 912 : cluster [DBG] pgmap v1433: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T14:56:39.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[87147]: audit 2024-09-13T14:56:38.066823+0000 mon.a (mon.0) 1963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[87147]: audit 2024-09-13T14:56:38.066823+0000 mon.a (mon.0) 1963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[87147]: audit 2024-09-13T14:56:38.076832+0000 mon.a (mon.0) 1964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[87147]: audit 2024-09-13T14:56:38.076832+0000 mon.a (mon.0) 1964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[87147]: cluster 2024-09-13T14:56:38.685433+0000 mgr.y (mgr.26252) 912 : cluster [DBG] pgmap v1433: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T14:56:39.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:39 smithi099 bash[87147]: cluster 2024-09-13T14:56:38.685433+0000 mgr.y (mgr.26252) 912 : cluster [DBG] pgmap v1433: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T14:56:39.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:39.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:39 smithi177 bash[68594]: audit 2024-09-13T14:56:38.066823+0000 mon.a (mon.0) 1963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:39 smithi177 bash[68594]: audit 2024-09-13T14:56:38.066823+0000 mon.a (mon.0) 1963 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:39 smithi177 bash[68594]: audit 2024-09-13T14:56:38.076832+0000 mon.a (mon.0) 1964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:39 smithi177 bash[68594]: audit 2024-09-13T14:56:38.076832+0000 mon.a (mon.0) 1964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:39.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:39 smithi177 bash[68594]: cluster 2024-09-13T14:56:38.685433+0000 mgr.y (mgr.26252) 912 : cluster [DBG] pgmap v1433: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T14:56:39.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:39 smithi177 bash[68594]: cluster 2024-09-13T14:56:38.685433+0000 mgr.y (mgr.26252) 912 : cluster [DBG] pgmap v1433: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T14:56:39.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:39.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:39.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:39.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:39.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:39.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:39.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:39.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:39.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:39.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:39.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:39.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:39.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:39.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:40.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:40.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:40.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:40.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:40.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:40.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:40.270 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:40.295 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:40.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:40.399 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:40.425 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:40.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:40.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:40.571 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:40.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:40.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:56:40 smithi177 bash[61439]: ts=2024-09-13T14:56:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:40.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:40.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:40.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:40.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:40.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:40.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:40.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:40.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:40.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:41.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:41.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:41.210 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:41.241 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:41.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:41.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:41.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:41.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:41.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:41.494 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:41.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:41.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:41.631 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:41.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:41.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:56:41.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:41.777 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:41.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:41.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:41.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:42.016 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:42.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:41 smithi177 bash[68594]: cluster 2024-09-13T14:56:40.686440+0000 mgr.y (mgr.26252) 913 : cluster [DBG] pgmap v1434: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:56:42.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:41 smithi177 bash[68594]: cluster 2024-09-13T14:56:40.686440+0000 mgr.y (mgr.26252) 913 : cluster [DBG] pgmap v1434: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:56:42.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:42.153 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:42.162 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:41 smithi099 bash[84576]: cluster 2024-09-13T14:56:40.686440+0000 mgr.y (mgr.26252) 913 : cluster [DBG] pgmap v1434: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:56:42.162 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:41 smithi099 bash[84576]: cluster 2024-09-13T14:56:40.686440+0000 mgr.y (mgr.26252) 913 : cluster [DBG] pgmap v1434: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:56:42.162 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:41 smithi099 bash[87147]: cluster 2024-09-13T14:56:40.686440+0000 mgr.y (mgr.26252) 913 : cluster [DBG] pgmap v1434: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:56:42.162 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:41 smithi099 bash[87147]: cluster 2024-09-13T14:56:40.686440+0000 mgr.y (mgr.26252) 913 : cluster [DBG] pgmap v1434: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:56:42.174 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:42.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:42.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:42.320 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:42.424 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:42.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:42.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:42.555 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:42.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:42.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:42.699 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:42.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:42.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:42.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:42.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:42.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:42.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:42.996 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:43.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:43.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:43.132 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:43.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:43.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:43.276 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:43.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:43.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:43.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:43.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:43.518 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:43.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:43.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:43.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:43.685 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:43.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:43.754 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T14:56:43.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:43.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:43.880 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:56:43.880 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7890 41.0 41:0618b954:::benchmark_data_smithi099_126887_object7889:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314348+0000", 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 537.56488624400004, 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.073313586, 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.881 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7890 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314348+0000", 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314348+0000", 2024-09-13T14:56:43.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314350+0000", 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4739999999999999e-06 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314409+0000", 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8826999999999998e-05 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314428+0000", 2024-09-13T14:56:43.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8879e-05 2024-09-13T14:56:43.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314442+0000", 2024-09-13T14:56:43.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4144999999999999e-05 2024-09-13T14:56:43.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314608+0000", 2024-09-13T14:56:43.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000165737 2024-09-13T14:56:43.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:56:43.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314627+0000", 2024-09-13T14:56:43.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9366000000000001e-05 2024-09-13T14:56:43.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.387501+0000", 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0728740640000001 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.387661+0000", 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000160094 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.888 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7911 41.a 41:50dfa993:::benchmark_data_smithi099_126887_object7910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314466+0000", 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 537.56476754100004, 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0920385480000001, 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7911 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.889 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314466+0000", 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314466+0000", 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314468+0000", 2024-09-13T14:56:43.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1200000000000001e-06 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314505+0000", 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7172000000000003e-05 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314519+0000", 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3875e-05 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314528+0000", 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0580000000000004e-06 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.325392+0000", 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010863892999999999 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.325405+0000", 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3390000000000001e-05 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.395721+0000", 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.070316327 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.396103+0000", 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000381985 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.396584+0000", 2024-09-13T14:56:43.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00048042400000000002 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.399219+0000", 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026351489999999998 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402020+0000", 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028009150000000002 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.405979+0000", 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039590570000000002 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.406038+0000", 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8906999999999997e-05 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.406505+0000", 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00046727599999999998 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.895 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7914 41.a 41:52fc18a2:::benchmark_data_smithi099_126887_object7913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314554+0000", 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 537.56468041200003, 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.092421273, 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.896 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7914 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314554+0000", 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314554+0000", 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314555+0000", 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7999999999999993e-07 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314599+0000", 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4251e-05 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314611+0000", 2024-09-13T14:56:43.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2007999999999999e-05 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314621+0000", 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0375e-05 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327110+0000", 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012489109 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:56:43.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327122+0000", 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1985e-05 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.399315+0000", 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0721924110000001 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.399578+0000", 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000263255 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.399875+0000", 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000297175 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402094+0000", 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022189380000000002 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.403605+0000", 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00151092 2024-09-13T14:56:43.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.406576+0000", 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029710460000000002 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.406651+0000", 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5186999999999997e-05 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.406975+0000", 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00032363299999999998 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7924 41.a 41:5d619088:::benchmark_data_smithi099_126887_object7923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314639+0000", 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 537.56459468499997, 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0982459849999999, 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.903 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7924 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314639+0000", 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314639+0000", 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314640+0000", 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.015e-06 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314677+0000", 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6748999999999997e-05 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314689+0000", 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2133000000000001e-05 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314698+0000", 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8449999999999992e-06 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327158+0000", 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012460404 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327169+0000", 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0023e-05 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.401249+0000", 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0740800580000001 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.401395+0000", 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014684799999999999 2024-09-13T14:56:43.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.401566+0000", 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000170457 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.403524+0000", 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019586009999999999 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.407054+0000", 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003529131 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.412300+0000", 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052460839999999998 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.412491+0000", 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019173899999999999 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.412885+0000", 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000393898 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:7925 41.a 41:50f18a2a:::benchmark_data_smithi099_126887_object7924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e959)", 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:46.314765+0000", 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 537.56446894400005, 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0986208369999999, 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.910 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 7925 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314765+0000", 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314765+0000", 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314766+0000", 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.128e-06 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314806+0000", 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9544e-05 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314820+0000", 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3998e-05 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.314829+0000", 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0499999999999997e-06 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327205+0000", 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012376687000000001 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:46.327215+0000", 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5179999999999993e-06 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402126+0000", 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.074911476 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402298+0000", 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000171891 2024-09-13T14:56:43.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.402506+0000", 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020722000000000001 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.408228+0000", 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005722703 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.408293+0000", 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4697999999999996e-05 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.412947+0000", 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046542809999999997 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.413012+0000", 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4727999999999994e-05 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:47.413386+0000", 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000373915 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13528 41.d 41:b2f0624e:::benchmark_data_smithi099_126887_object13527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e962)", 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.103363+0000", 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 532.77587085799996, 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45605169299999998, 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.917 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13528 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103363+0000", 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103363+0000", 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103364+0000", 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2699999999999999e-06 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103391+0000", 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6205000000000001e-05 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103403+0000", 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2425e-05 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103411+0000", 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9149999999999996e-06 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103552+0000", 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014133200000000001 2024-09-13T14:56:43.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.103723+0000", 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017070599999999999 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106181+0000", 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024575119999999998 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107504+0000", 2024-09-13T14:56:43.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001323219 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108459+0000", 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00095574500000000003 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559286+0000", 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.450826424 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559360+0000", 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4437999999999994e-05 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559415+0000", 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4502000000000001e-05 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13531 41.0 41:09568b39:::benchmark_data_smithi099_126887_object13530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e962)", 2024-09-13T14:56:43.923 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.104399+0000", 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 532.77483551299997, 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45930618299999998, 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13531 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104399+0000", 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104399+0000", 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104400+0000", 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.319e-06 2024-09-13T14:56:43.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104438+0000", 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7682000000000003e-05 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104450+0000", 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2646000000000001e-05 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104457+0000", 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0249999999999997e-06 2024-09-13T14:56:43.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104501+0000", 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3707e-05 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104607+0000", 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010582999999999999 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5,7", 2024-09-13T14:56:43.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104731+0000", 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012387100000000001 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106377+0000", 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001646214 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107174+0000", 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00079693799999999996 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108037+0000", 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00086317000000000002 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563163+0000", 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45512561299999998 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563228+0000", 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5668999999999998e-05 2024-09-13T14:56:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563705+0000", 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00047649900000000002 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13534 41.d 41:b4692011:::benchmark_data_smithi099_126887_object13533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e962)", 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.104475+0000", 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 532.77475932300001, 2024-09-13T14:56:43.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45571292200000002, 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13534 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104475+0000", 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104475+0000", 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104475+0000", 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0100000000000005e-07 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104499+0000", 2024-09-13T14:56:43.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3255999999999999e-05 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104508+0000", 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9110000000000005e-06 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.104514+0000", 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.361e-06 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106804+0000", 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022900260000000001 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106884+0000", 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9691999999999995e-05 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106976+0000", 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2046000000000006e-05 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558992+0000", 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45201617500000002 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559729+0000", 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00073704599999999999 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559988+0000", 2024-09-13T14:56:43.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025961000000000002 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560118+0000", 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012939099999999999 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560188+0000", 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9807000000000004e-05 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.936 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:13540 41.9 41:9360d5c8:::benchmark_data_smithi099_126887_object13539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e962)", 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.106714+0000", 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 532.77251999400005, 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45411328200000001, 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 13540 2024-09-13T14:56:43.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106714+0000", 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106714+0000", 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106716+0000", 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.677e-06 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106751+0000", 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5389000000000003e-05 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106762+0000", 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1367e-05 2024-09-13T14:56:43.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.106992+0000", 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000229583 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107142+0000", 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000150078 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107253+0000", 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011100299999999999 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107385+0000", 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000131839 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560667+0000", 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45328227100000001 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560712+0000", 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4416999999999998e-05 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560736+0000", 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4258999999999999e-05 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560788+0000", 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2485000000000003e-05 2024-09-13T14:56:43.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560827+0000", 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8914e-05 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:13538 41.c e962/953 41:305179fc:::benchmark_data_smithi099_126887_object13537:head v 962'852, mlcod=962'850)", 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.107328+0000", 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 532.77190649800002, 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45097309600000002, 2024-09-13T14:56:43.943 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107328+0000", 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107328+0000", 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107328+0000", 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.13e-07 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107351+0000", 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2756999999999999e-05 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107359+0000", 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4090000000000003e-06 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107366+0000", 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2709999999999998e-06 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107936+0000", 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00057086699999999997 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.107948+0000", 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1112e-05 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108933+0000", 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00098501600000000002 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558301+0000", 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.44936805099999999 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.947 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.35268.0:13531 41.0 e962/959 ondisk, result = 0)", 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.108299+0000", 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 532.77093530900004, 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45542017800000001, 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108299+0000", 2024-09-13T14:56:43.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108299+0000", 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108299+0000", 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0900000000000001e-07 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108302+0000", 2024-09-13T14:56:43.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8150000000000002e-06 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108313+0000", 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0533000000000001e-05 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108319+0000", 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1519999999999997e-06 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563146+0000", 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.454827537 2024-09-13T14:56:43.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563155+0000", 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9029999999999993e-06 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.563719+0000", 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00056362899999999995 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.951 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:13542 41.7 e962/953 41:e044e2cf:::benchmark_data_smithi099_126887_object13541:head v 962'875, mlcod=962'872)", 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.108483+0000", 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 532.77075078400003, 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45160520900000001, 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108483+0000", 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108483+0000", 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108484+0000", 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2900000000000003e-07 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108531+0000", 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7268999999999998e-05 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108537+0000", 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7450000000000001e-06 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108540+0000", 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5159999999999999e-06 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108582+0000", 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1177999999999998e-05 2024-09-13T14:56:43.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108600+0000", 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8212000000000001e-05 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560050+0000", 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45145017900000001 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560088+0000", 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8481e-05 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.35268.0:13540 41.9 e962/953 ondisk, result = 0)", 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.108590+0000", 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 532.77064409699994, 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.452089888, 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.956 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108590+0000", 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108590+0000", 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108591+0000", 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8800000000000002e-07 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108595+0000", 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0210000000000001e-06 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108604+0000", 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4329999999999993e-06 2024-09-13T14:56:43.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108610+0000", 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0360000000000003e-06 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560651+0000", 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45204074599999999 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560661+0000", 2024-09-13T14:56:43.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0307e-05 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560680+0000", 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8657000000000001e-05 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.35268.0:13534 41.d e962/953 ondisk, result = 0)", 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.108637+0000", 2024-09-13T14:56:43.960 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 532.77059699699998, 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45155849999999997, 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108637+0000", 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108637+0000", 2024-09-13T14:56:43.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108638+0000", 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5400000000000001e-07 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108641+0000", 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.625e-06 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108651+0000", 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6719999999999999e-06 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108657+0000", 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8490000000000002e-06 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559973+0000", 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45131629600000001 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.559982+0000", 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.524e-06 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560196+0000", 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021388000000000001 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.964 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:13541 41.c e962/953 41:3de424e3:::benchmark_data_smithi099_126887_object13540:head v 962'853, mlcod=962'850)", 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:47:51.108765+0000", 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 532.77046940000002, 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45157181899999999, 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108765+0000", 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108765+0000", 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108765+0000", 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9800000000000003e-07 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.108790+0000", 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4769e-05 2024-09-13T14:56:43.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558374+0000", 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.44958437800000001 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558386+0000", 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1822e-05 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558458+0000", 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1435999999999998e-05 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.558485+0000", 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7705999999999999e-05 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560297+0000", 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018116829999999999 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:47:51.560336+0000", 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9427e-05 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82467 41.c 41:3feb82e7:::benchmark_data_smithi099_126887_object82466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e975)", 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.296910+0000", 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 474.582324331, 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2643870189999999, 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.969 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82467 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.296910+0000", 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.296910+0000", 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.296912+0000", 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.933e-06 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.296935+0000", 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3615e-05 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.296949+0000", 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3526000000000001e-05 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.296958+0000", 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3670000000000003e-06 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.297287+0000", 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00032909500000000001 2024-09-13T14:56:43.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.297297+0000", 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6360000000000006e-06 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.297983+0000", 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0006866320000001 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.297996+0000", 2024-09-13T14:56:43.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2418999999999999e-05 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.559857+0000", 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.261861468 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.561297+0000", 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014393279999999999 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.974 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:82516 41.c 41:336b3b70:::benchmark_data_smithi099_126887_object82515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e975)", 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:48:49.310554+0000", 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 474.56868029700001, 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2630770629999999, 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 82516 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310554+0000", 2024-09-13T14:56:43.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310554+0000", 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310555+0000", 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.517e-06 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310575+0000", 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9952000000000001e-05 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310584+0000", 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2709999999999998e-06 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310593+0000", 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1340000000000004e-06 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310949+0000", 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00035638800000000002 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:49.310959+0000", 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8830000000000001e-06 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.298026+0000", 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98706671599999996 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T14:56:43.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:50.298034+0000", 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9780000000000005e-06 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.563664+0000", 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.265630107 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.564282+0000", 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00061864600000000004 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:56:43.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.564672+0000", 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00038928699999999999 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.570668+0000", 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0059963209999999998 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.570708+0000", 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9891000000000001e-05 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.570891+0000", 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018345899999999999 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.570956+0000", 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4412999999999995e-05 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:48:51.573631+0000", 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026751000000000001 2024-09-13T14:56:43.981 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137615 41.c 41:37547a0a:::benchmark_data_smithi099_126887_object39827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.525981+0000", 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 391.35325286099999, 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.47123983000000003, 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.982 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137615 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.525981+0000", 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.525981+0000", 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.525982+0000", 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2720000000000001e-06 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.525986+0000", 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8659999999999999e-06 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.530641+0000", 2024-09-13T14:56:43.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004654572 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.530655+0000", 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4019000000000001e-05 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.532691+0000", 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002036328 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.532858+0000", 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016681700000000001 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.533024+0000", 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016638800000000001 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.536550+0000", 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0035253839999999999 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.539073+0000", 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025234620000000002 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.994702+0000", 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45562893100000001 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.994767+0000", 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4647000000000006e-05 2024-09-13T14:56:43.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.997221+0000", 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024541440000000001 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.35268.0:144554 41.4 e981/953 41:2ca54eb2:::benchmark_data_smithi099_126887_object46766:head v 981'9069, mlcod=981'9065)", 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:24.112091+0000", 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 379.76714335499997, 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.438275568, 2024-09-13T14:56:43.988 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.112091+0000", 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.112091+0000", 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.112092+0000", 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8899999999999998e-07 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.112098+0000", 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5400000000000001e-06 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.112108+0000", 2024-09-13T14:56:43.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0118e-05 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.112116+0000", 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3209999999999999e-06 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.115864+0000", 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037486289999999999 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.115882+0000", 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7946000000000001e-05 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.550317+0000", 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.43443448899999998 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.550366+0000", 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9635999999999997e-05 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.992 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:144575 41.c 41:34fa3b70:::benchmark_data_smithi099_126887_object46787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:24.138025+0000", 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 379.74120940300003, 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.43957169499999998, 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 144575 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.993 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.138025+0000", 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.138025+0000", 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.138026+0000", 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.843e-06 2024-09-13T14:56:43.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.138032+0000", 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6550000000000001e-06 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.138044+0000", 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1975e-05 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.138052+0000", 2024-09-13T14:56:43.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1939999999999998e-06 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.550557+0000", 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.41250472100000002 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.550707+0000", 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000149584 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.550852+0000", 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014540500000000001 2024-09-13T14:56:43.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.560990+0000", 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010138367000000001 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.562309+0000", 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001318797 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:43.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.575469+0000", 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013160321000000001 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.575548+0000", 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8663000000000002e-05 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:24.577596+0000", 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020481700000000002 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:56:43.998 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:56:43.999 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:56:43.999 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:44.000 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[84576]: cluster 2024-09-13T14:56:42.689215+0000 mgr.y (mgr.26252) 914 : cluster [DBG] pgmap v1435: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:44.000 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[84576]: cluster 2024-09-13T14:56:42.689215+0000 mgr.y (mgr.26252) 914 : cluster [DBG] pgmap v1435: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:44.000 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[84576]: audit 2024-09-13T14:56:43.080460+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.99:0/3486708451' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.000 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[84576]: audit 2024-09-13T14:56:43.080460+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.99:0/3486708451' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.000 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[84576]: audit 2024-09-13T14:56:43.080830+0000 mon.a (mon.0) 1965 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.000 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[84576]: audit 2024-09-13T14:56:43.080830+0000 mon.a (mon.0) 1965 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.000 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[84576]: audit 2024-09-13T14:56:43.469386+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.000 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[84576]: audit 2024-09-13T14:56:43.469386+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[84576]: audit 2024-09-13T14:56:43.479132+0000 mon.a (mon.0) 1967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[84576]: audit 2024-09-13T14:56:43.479132+0000 mon.a (mon.0) 1967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[87147]: cluster 2024-09-13T14:56:42.689215+0000 mgr.y (mgr.26252) 914 : cluster [DBG] pgmap v1435: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[87147]: cluster 2024-09-13T14:56:42.689215+0000 mgr.y (mgr.26252) 914 : cluster [DBG] pgmap v1435: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[87147]: audit 2024-09-13T14:56:43.080460+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.99:0/3486708451' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[87147]: audit 2024-09-13T14:56:43.080460+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.99:0/3486708451' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[87147]: audit 2024-09-13T14:56:43.080830+0000 mon.a (mon.0) 1965 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[87147]: audit 2024-09-13T14:56:43.080830+0000 mon.a (mon.0) 1965 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[87147]: audit 2024-09-13T14:56:43.469386+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[87147]: audit 2024-09-13T14:56:43.469386+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[87147]: audit 2024-09-13T14:56:43.479132+0000 mon.a (mon.0) 1967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.001 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:43 smithi099 bash[87147]: audit 2024-09-13T14:56:43.479132+0000 mon.a (mon.0) 1967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.002 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:56:44.004 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:56:44.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:43 smithi177 bash[68594]: cluster 2024-09-13T14:56:42.689215+0000 mgr.y (mgr.26252) 914 : cluster [DBG] pgmap v1435: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:44.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:43 smithi177 bash[68594]: cluster 2024-09-13T14:56:42.689215+0000 mgr.y (mgr.26252) 914 : cluster [DBG] pgmap v1435: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:44.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:43 smithi177 bash[68594]: audit 2024-09-13T14:56:43.080460+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.99:0/3486708451' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:43 smithi177 bash[68594]: audit 2024-09-13T14:56:43.080460+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.99:0/3486708451' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:43 smithi177 bash[68594]: audit 2024-09-13T14:56:43.080830+0000 mon.a (mon.0) 1965 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:43 smithi177 bash[68594]: audit 2024-09-13T14:56:43.080830+0000 mon.a (mon.0) 1965 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:44.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:43 smithi177 bash[68594]: audit 2024-09-13T14:56:43.469386+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:43 smithi177 bash[68594]: audit 2024-09-13T14:56:43.469386+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:43 smithi177 bash[68594]: audit 2024-09-13T14:56:43.479132+0000 mon.a (mon.0) 1967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:43 smithi177 bash[68594]: audit 2024-09-13T14:56:43.479132+0000 mon.a (mon.0) 1967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.028 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:44.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:44.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:44.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:44.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:44.306 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:44.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:44.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:44.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:44.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:44.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:44.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:44.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:44.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:44.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:44.839 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:44.860 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: cluster 2024-09-13T14:56:43.748627+0000 mon.a (mon.0) 1968 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: cluster 2024-09-13T14:56:43.748627+0000 mon.a (mon.0) 1968 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: cluster 2024-09-13T14:56:43.748650+0000 mon.a (mon.0) 1969 : cluster [INF] Cluster is now healthy 2024-09-13T14:56:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: cluster 2024-09-13T14:56:43.748650+0000 mon.a (mon.0) 1969 : cluster [INF] Cluster is now healthy 2024-09-13T14:56:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: audit 2024-09-13T14:56:43.750452+0000 mon.a (mon.0) 1970 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: audit 2024-09-13T14:56:43.750452+0000 mon.a (mon.0) 1970 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: cluster 2024-09-13T14:56:43.755447+0000 mon.a (mon.0) 1971 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: cluster 2024-09-13T14:56:43.755447+0000 mon.a (mon.0) 1971 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: audit 2024-09-13T14:56:43.966661+0000 mgr.y (mgr.26252) 915 : audit [DBG] from='client.35583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: audit 2024-09-13T14:56:43.966661+0000 mgr.y (mgr.26252) 915 : audit [DBG] from='client.35583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: audit 2024-09-13T14:56:44.348392+0000 mon.a (mon.0) 1972 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: audit 2024-09-13T14:56:44.348392+0000 mon.a (mon.0) 1972 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: audit 2024-09-13T14:56:44.349745+0000 mon.a (mon.0) 1973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: audit 2024-09-13T14:56:44.349745+0000 mon.a (mon.0) 1973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: audit 2024-09-13T14:56:44.356026+0000 mon.a (mon.0) 1974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:44 smithi177 bash[68594]: audit 2024-09-13T14:56:44.356026+0000 mon.a (mon.0) 1974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:44.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:44.969 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:45.002 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:45.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:45.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:45.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:45.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:45.128 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: cluster 2024-09-13T14:56:43.748627+0000 mon.a (mon.0) 1968 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:45.128 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: cluster 2024-09-13T14:56:43.748627+0000 mon.a (mon.0) 1968 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: cluster 2024-09-13T14:56:43.748650+0000 mon.a (mon.0) 1969 : cluster [INF] Cluster is now healthy 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: cluster 2024-09-13T14:56:43.748650+0000 mon.a (mon.0) 1969 : cluster [INF] Cluster is now healthy 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: audit 2024-09-13T14:56:43.750452+0000 mon.a (mon.0) 1970 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: audit 2024-09-13T14:56:43.750452+0000 mon.a (mon.0) 1970 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: cluster 2024-09-13T14:56:43.755447+0000 mon.a (mon.0) 1971 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: cluster 2024-09-13T14:56:43.755447+0000 mon.a (mon.0) 1971 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: audit 2024-09-13T14:56:43.966661+0000 mgr.y (mgr.26252) 915 : audit [DBG] from='client.35583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: audit 2024-09-13T14:56:43.966661+0000 mgr.y (mgr.26252) 915 : audit [DBG] from='client.35583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: audit 2024-09-13T14:56:44.348392+0000 mon.a (mon.0) 1972 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: audit 2024-09-13T14:56:44.348392+0000 mon.a (mon.0) 1972 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: audit 2024-09-13T14:56:44.349745+0000 mon.a (mon.0) 1973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:45.129 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: audit 2024-09-13T14:56:44.349745+0000 mon.a (mon.0) 1973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:45.130 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: audit 2024-09-13T14:56:44.356026+0000 mon.a (mon.0) 1974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:45.130 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[84576]: audit 2024-09-13T14:56:44.356026+0000 mon.a (mon.0) 1974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:45.130 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: cluster 2024-09-13T14:56:43.748627+0000 mon.a (mon.0) 1968 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:45.130 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: cluster 2024-09-13T14:56:43.748627+0000 mon.a (mon.0) 1968 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:45.130 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: cluster 2024-09-13T14:56:43.748650+0000 mon.a (mon.0) 1969 : cluster [INF] Cluster is now healthy 2024-09-13T14:56:45.130 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: cluster 2024-09-13T14:56:43.748650+0000 mon.a (mon.0) 1969 : cluster [INF] Cluster is now healthy 2024-09-13T14:56:45.130 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: audit 2024-09-13T14:56:43.750452+0000 mon.a (mon.0) 1970 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:45.130 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: audit 2024-09-13T14:56:43.750452+0000 mon.a (mon.0) 1970 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:45.130 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: cluster 2024-09-13T14:56:43.755447+0000 mon.a (mon.0) 1971 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2024-09-13T14:56:45.130 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: cluster 2024-09-13T14:56:43.755447+0000 mon.a (mon.0) 1971 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2024-09-13T14:56:45.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: audit 2024-09-13T14:56:43.966661+0000 mgr.y (mgr.26252) 915 : audit [DBG] from='client.35583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:45.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: audit 2024-09-13T14:56:43.966661+0000 mgr.y (mgr.26252) 915 : audit [DBG] from='client.35583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:45.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: audit 2024-09-13T14:56:44.348392+0000 mon.a (mon.0) 1972 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:45.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: audit 2024-09-13T14:56:44.348392+0000 mon.a (mon.0) 1972 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:56:45.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: audit 2024-09-13T14:56:44.349745+0000 mon.a (mon.0) 1973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:45.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: audit 2024-09-13T14:56:44.349745+0000 mon.a (mon.0) 1973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:56:45.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: audit 2024-09-13T14:56:44.356026+0000 mon.a (mon.0) 1974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:45.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:44 smithi099 bash[87147]: audit 2024-09-13T14:56:44.356026+0000 mon.a (mon.0) 1974 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:45.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:45.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:45.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:45.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:45.366 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:45.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:45.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:45.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:45.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:45.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:45.626 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:45.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:45.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:45.760 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:45.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:45.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:45.898 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:45 smithi177 bash[68594]: cluster 2024-09-13T14:56:44.689855+0000 mgr.y (mgr.26252) 916 : cluster [DBG] pgmap v1437: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2024-09-13T14:56:45.899 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:45 smithi177 bash[68594]: cluster 2024-09-13T14:56:44.689855+0000 mgr.y (mgr.26252) 916 : cluster [DBG] pgmap v1437: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2024-09-13T14:56:45.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:46.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:46.031 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:46.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:46.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:45 smithi099 bash[84576]: cluster 2024-09-13T14:56:44.689855+0000 mgr.y (mgr.26252) 916 : cluster [DBG] pgmap v1437: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2024-09-13T14:56:46.080 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:45 smithi099 bash[84576]: cluster 2024-09-13T14:56:44.689855+0000 mgr.y (mgr.26252) 916 : cluster [DBG] pgmap v1437: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2024-09-13T14:56:46.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:45 smithi099 bash[87147]: cluster 2024-09-13T14:56:44.689855+0000 mgr.y (mgr.26252) 916 : cluster [DBG] pgmap v1437: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2024-09-13T14:56:46.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:46.157 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:46.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:46.292 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:46.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:46.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:46.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:46.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:46.541 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:46.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:46.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:46.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:46 smithi099 bash[87147]: cluster 2024-09-13T14:56:44.689855+0000 mgr.y (mgr.26252) 916 : cluster [DBG] pgmap v1437: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2024-09-13T14:56:46.675 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:46.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:46.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:46.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:46.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:46.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:46.938 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:46.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:46.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:47.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:47.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:47.205 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:47.213 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:56:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:56:47] "GET /metrics HTTP/1.1" 200 36768 "" "Prometheus/2.51.0" 2024-09-13T14:56:47.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:47.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:47.300 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1436,"stamp":"2024-09-13T14:56:43.751305+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2229999589,"num_objects":34045,"num_object_clones":0,"num_object_copies":102135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34045,"num_whiteouts":0,"num_read":345,"num_read_kb":800,"num_write":181869,"num_write_kb":6903228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":595442,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142121,"ondisk_log_size":142121,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9060864,"kb_used_data":6618624,"kb_used_omap":353,"kb_used_meta":2441694,"kb_avail":740900352,"statfs":{"total":767960285184,"available":758681960448,"internally_reserved":0,"allocated":6777470976,"data_stored":6745978417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":361898,"internal_metadata":2500295254},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,4,13,7,6,0,1,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-417792000,"num_objects":-6375,"num_object_clones":0,"num_object_copies":-19125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6375,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.067896"},"pg_stats":[{"pgid":"42.f","version":"1029'11242","reported_seq":11270,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:39.239019+0000","last_change":"2024-09-13T14:53:02.737967+0000","last_active":"2024-09-13T14:56:39.239019+0000","last_peered":"2024-09-13T14:56:39.239019+0000","last_clean":"2024-09-13T14:56:39.239019+0000","last_became_active":"2024-09-13T14:53:02.737189+0000","last_became_peered":"2024-09-13T14:53:02.737189+0000","last_unstale":"2024-09-13T14:56:39.239019+0000","last_undegraded":"2024-09-13T14:56:39.239019+0000","last_fullsized":"2024-09-13T14:56:39.239019+0000","mapping_epoch":1009,"log_start":"1017'3300","ondisk_log_start":"1017'3300","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7942,"log_dups_size":0,"ondisk_log_size":7942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:43.030910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140247040,"num_objects":2140,"num_object_clones":0,"num_object_copies":6420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11242,"num_write_kb":428224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"42.e","version":"1029'11491","reported_seq":11542,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:38.973092+0000","last_change":"2024-09-13T14:53:03.091056+0000","last_active":"2024-09-13T14:56:38.973092+0000","last_peered":"2024-09-13T14:56:38.973092+0000","last_clean":"2024-09-13T14:56:38.973092+0000","last_became_active":"2024-09-13T14:53:03.090134+0000","last_became_peered":"2024-09-13T14:53:03.090134+0000","last_unstale":"2024-09-13T14:56:38.973092+0000","last_undegraded":"2024-09-13T14:56:38.973092+0000","last_fullsized":"2024-09-13T14:56:38.973092+0000","mapping_epoch":1009,"log_start":"1016'2900","ondisk_log_start":"1016'2900","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":8591,"log_dups_size":0,"ondisk_log_size":8591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:44.438976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142540800,"num_objects":2175,"num_object_clones":0,"num_object_copies":6525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11491,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"42.d","version":"1029'11349","reported_seq":11378,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:39.189747+0000","last_change":"2024-09-13T14:53:02.727418+0000","last_active":"2024-09-13T14:56:39.189747+0000","last_peered":"2024-09-13T14:56:39.189747+0000","last_clean":"2024-09-13T14:56:39.189747+0000","last_became_active":"2024-09-13T14:53:02.726653+0000","last_became_peered":"2024-09-13T14:53:02.726653+0000","last_unstale":"2024-09-13T14:56:39.189747+0000","last_undegraded":"2024-09-13T14:56:39.189747+0000","last_fullsized":"2024-09-13T14:56:39.189747+0000","mapping_epoch":1009,"log_start":"1014'1600","ondisk_log_start":"1014'1600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9749,"log_dups_size":0,"ondisk_log_size":9749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:30:10.620331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137625624,"num_objects":2101,"num_object_clones":0,"num_object_copies":6303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2101,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11349,"num_write_kb":430337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"42.c","version":"1029'11473","reported_seq":11525,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:38.967642+0000","last_change":"2024-09-13T14:53:02.731439+0000","last_active":"2024-09-13T14:56:38.967642+0000","last_peered":"2024-09-13T14:56:38.967642+0000","last_clean":"2024-09-13T14:56:38.967642+0000","last_became_active":"2024-09-13T14:53:02.730603+0000","last_became_peered":"2024-09-13T14:53:02.730603+0000","last_unstale":"2024-09-13T14:56:38.967642+0000","last_undegraded":"2024-09-13T14:56:38.967642+0000","last_fullsized":"2024-09-13T14:56:38.967642+0000","mapping_epoch":1009,"log_start":"1017'3500","ondisk_log_start":"1017'3500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7973,"log_dups_size":0,"ondisk_log_size":7973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:10.067364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140836864,"num_objects":2149,"num_object_clones":0,"num_object_copies":6447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11473,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"42.b","version":"1029'11282","reported_seq":11334,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:41.632298+0000","last_change":"2024-09-13T14:53:02.735868+0000","last_active":"2024-09-13T14:56:41.632298+0000","last_peered":"2024-09-13T14:56:41.632298+0000","last_clean":"2024-09-13T14:56:41.632298+0000","last_became_active":"2024-09-13T14:53:02.735016+0000","last_became_peered":"2024-09-13T14:53:02.735016+0000","last_unstale":"2024-09-13T14:56:41.632298+0000","last_undegraded":"2024-09-13T14:56:41.632298+0000","last_fullsized":"2024-09-13T14:56:41.632298+0000","mapping_epoch":1009,"log_start":"1017'3500","ondisk_log_start":"1017'3500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7782,"log_dups_size":0,"ondisk_log_size":7782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:44:45.114443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130678784,"num_objects":1994,"num_object_clones":0,"num_object_copies":5982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11282,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"42.a","version":"1029'11200","reported_seq":11228,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:39.474740+0000","last_change":"2024-09-13T14:53:02.730034+0000","last_active":"2024-09-13T14:56:39.474740+0000","last_peered":"2024-09-13T14:56:39.474740+0000","last_clean":"2024-09-13T14:56:39.474740+0000","last_became_active":"2024-09-13T14:53:02.729241+0000","last_became_peered":"2024-09-13T14:53:02.729241+0000","last_unstale":"2024-09-13T14:56:39.474740+0000","last_undegraded":"2024-09-13T14:56:39.474740+0000","last_fullsized":"2024-09-13T14:56:39.474740+0000","mapping_epoch":1009,"log_start":"1014'1500","ondisk_log_start":"1014'1500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9700,"log_dups_size":0,"ondisk_log_size":9700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:44.133299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141688832,"num_objects":2162,"num_object_clones":0,"num_object_copies":6486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11200,"num_write_kb":427584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"42.9","version":"1029'11333","reported_seq":11361,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:39.175090+0000","last_change":"2024-09-13T14:53:02.737619+0000","last_active":"2024-09-13T14:56:39.175090+0000","last_peered":"2024-09-13T14:56:39.175090+0000","last_clean":"2024-09-13T14:56:39.175090+0000","last_became_active":"2024-09-13T14:53:02.736782+0000","last_became_peered":"2024-09-13T14:53:02.736782+0000","last_unstale":"2024-09-13T14:56:39.175090+0000","last_undegraded":"2024-09-13T14:56:39.175090+0000","last_fullsized":"2024-09-13T14:56:39.175090+0000","mapping_epoch":1009,"log_start":"1014'1600","ondisk_log_start":"1014'1600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9733,"log_dups_size":0,"ondisk_log_size":9733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:30:17.070735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135593984,"num_objects":2069,"num_object_clones":0,"num_object_copies":6207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11333,"num_write_kb":428864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"42.8","version":"1029'11161","reported_seq":11189,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:39.282946+0000","last_change":"2024-09-13T14:53:03.087618+0000","last_active":"2024-09-13T14:56:39.282946+0000","last_peered":"2024-09-13T14:56:39.282946+0000","last_clean":"2024-09-13T14:56:39.282946+0000","last_became_active":"2024-09-13T14:53:03.086906+0000","last_became_peered":"2024-09-13T14:53:03.086906+0000","last_unstale":"2024-09-13T14:56:39.282946+0000","last_undegraded":"2024-09-13T14:56:39.282946+0000","last_fullsized":"2024-09-13T14:56:39.282946+0000","mapping_epoch":1009,"log_start":"1016'3200","ondisk_log_start":"1016'3200","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7961,"log_dups_size":0,"ondisk_log_size":7961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:05.901169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139526144,"num_objects":2129,"num_object_clones":0,"num_object_copies":6387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11161,"num_write_kb":425280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"42.7","version":"1029'11477","reported_seq":11505,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:39.244477+0000","last_change":"2024-09-13T14:53:02.732343+0000","last_active":"2024-09-13T14:56:39.244477+0000","last_peered":"2024-09-13T14:56:39.244477+0000","last_clean":"2024-09-13T14:56:39.244477+0000","last_became_active":"2024-09-13T14:53:02.731717+0000","last_became_peered":"2024-09-13T14:53:02.731717+0000","last_unstale":"2024-09-13T14:56:39.244477+0000","last_undegraded":"2024-09-13T14:56:39.244477+0000","last_fullsized":"2024-09-13T14:56:39.244477+0000","mapping_epoch":1009,"log_start":"1017'3500","ondisk_log_start":"1017'3500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7977,"log_dups_size":0,"ondisk_log_size":7977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:37.034264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143589376,"num_objects":2191,"num_object_clones":0,"num_object_copies":6573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11477,"num_write_kb":437376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"42.6","version":"1029'11136","reported_seq":11163,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:39.515221+0000","last_change":"2024-09-13T14:53:02.734127+0000","last_active":"2024-09-13T14:56:39.515221+0000","last_peered":"2024-09-13T14:56:39.515221+0000","last_clean":"2024-09-13T14:56:39.515221+0000","last_became_active":"2024-09-13T14:53:02.733028+0000","last_became_peered":"2024-09-13T14:53:02.733028+0000","last_unstale":"2024-09-13T14:56:39.515221+0000","last_undegraded":"2024-09-13T14:56:39.515221+0000","last_fullsized":"2024-09-13T14:56:39.515221+0000","mapping_epoch":1009,"log_start":"1014'1400","ondisk_log_start":"1014'1400","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9736,"log_dups_size":0,"ondisk_log_size":9736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:46.557305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137101312,"num_objects":2092,"num_object_clones":0,"num_object_copies":6276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11136,"num_write_kb":423296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"42.5","version":"1029'11392","reported_seq":11420,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:39.537313+0000","last_change":"2024-09-13T14:53:02.730556+0000","last_active":"2024-09-13T14:56:39.537313+0000","last_peered":"2024-09-13T14:56:39.537313+0000","last_clean":"2024-09-13T14:56:39.537313+0000","last_became_active":"2024-09-13T14:53:02.729563+0000","last_became_peered":"2024-09-13T14:53:02.729563+0000","last_unstale":"2024-09-13T14:56:39.537313+0000","last_undegraded":"2024-09-13T14:56:39.537313+0000","last_fullsized":"2024-09-13T14:56:39.537313+0000","mapping_epoch":1009,"log_start":"1014'1700","ondisk_log_start":"1014'1700","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9692,"log_dups_size":0,"ondisk_log_size":9692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:28.870740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140509184,"num_objects":2144,"num_object_clones":0,"num_object_copies":6432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11392,"num_write_kb":433152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"42.4","version":"1029'11373","reported_seq":11425,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:41.590504+0000","last_change":"2024-09-13T14:53:02.737098+0000","last_active":"2024-09-13T14:56:41.590504+0000","last_peered":"2024-09-13T14:56:41.590504+0000","last_clean":"2024-09-13T14:56:41.590504+0000","last_became_active":"2024-09-13T14:53:02.736160+0000","last_became_peered":"2024-09-13T14:53:02.736160+0000","last_unstale":"2024-09-13T14:56:41.590504+0000","last_undegraded":"2024-09-13T14:56:41.590504+0000","last_fullsized":"2024-09-13T14:56:41.590504+0000","mapping_epoch":1009,"log_start":"1017'3600","ondisk_log_start":"1017'3600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7773,"log_dups_size":0,"ondisk_log_size":7773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:48.130549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133758976,"num_objects":2041,"num_object_clones":0,"num_object_copies":6123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11373,"num_write_kb":429248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"42.3","version":"1029'11289","reported_seq":11317,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:39.164210+0000","last_change":"2024-09-13T14:53:03.088642+0000","last_active":"2024-09-13T14:56:39.164210+0000","last_peered":"2024-09-13T14:56:39.164210+0000","last_clean":"2024-09-13T14:56:39.164210+0000","last_became_active":"2024-09-13T14:53:03.087596+0000","last_became_peered":"2024-09-13T14:53:03.087596+0000","last_unstale":"2024-09-13T14:56:39.164210+0000","last_undegraded":"2024-09-13T14:56:39.164210+0000","last_fullsized":"2024-09-13T14:56:39.164210+0000","mapping_epoch":1009,"log_start":"1014'1600","ondisk_log_start":"1014'1600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9689,"log_dups_size":0,"ondisk_log_size":9689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:30:53.067804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142671872,"num_objects":2177,"num_object_clones":0,"num_object_copies":6531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11289,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"42.2","version":"1029'11406","reported_seq":11458,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:42.324199+0000","last_change":"2024-09-13T14:53:03.090582+0000","last_active":"2024-09-13T14:56:42.324199+0000","last_peered":"2024-09-13T14:56:42.324199+0000","last_clean":"2024-09-13T14:56:42.324199+0000","last_became_active":"2024-09-13T14:53:03.089683+0000","last_became_peered":"2024-09-13T14:53:03.089683+0000","last_unstale":"2024-09-13T14:56:42.324199+0000","last_undegraded":"2024-09-13T14:56:42.324199+0000","last_fullsized":"2024-09-13T14:56:42.324199+0000","mapping_epoch":1009,"log_start":"1014'1400","ondisk_log_start":"1014'1400","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":10006,"log_dups_size":0,"ondisk_log_size":10006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:13:05.244177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131989504,"num_objects":2014,"num_object_clones":0,"num_object_copies":6042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11406,"num_write_kb":429440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"42.1","version":"1029'11551","reported_seq":11603,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:38.985387+0000","last_change":"2024-09-13T14:53:02.733122+0000","last_active":"2024-09-13T14:56:38.985387+0000","last_peered":"2024-09-13T14:56:38.985387+0000","last_clean":"2024-09-13T14:56:38.985387+0000","last_became_active":"2024-09-13T14:53:02.732272+0000","last_became_peered":"2024-09-13T14:53:02.732272+0000","last_unstale":"2024-09-13T14:56:38.985387+0000","last_undegraded":"2024-09-13T14:56:38.985387+0000","last_fullsized":"2024-09-13T14:56:38.985387+0000","mapping_epoch":1009,"log_start":"1017'3600","ondisk_log_start":"1017'3600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:48.079694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143196160,"num_objects":2185,"num_object_clones":0,"num_object_copies":6555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11551,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"42.0","version":"1029'11502","reported_seq":11530,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:39.524725+0000","last_change":"2024-09-13T14:53:02.729978+0000","last_active":"2024-09-13T14:56:39.524725+0000","last_peered":"2024-09-13T14:56:39.524725+0000","last_clean":"2024-09-13T14:56:39.524725+0000","last_became_active":"2024-09-13T14:53:02.728942+0000","last_became_peered":"2024-09-13T14:53:02.728942+0000","last_unstale":"2024-09-13T14:56:39.524725+0000","last_undegraded":"2024-09-13T14:56:39.524725+0000","last_fullsized":"2024-09-13T14:56:39.524725+0000","mapping_epoch":1009,"log_start":"1015'1800","ondisk_log_start":"1015'1800","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9702,"log_dups_size":0,"ondisk_log_size":9702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:11.356137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147849216,"num_objects":2256,"num_object_clones":0,"num_object_copies":6768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11502,"num_write_kb":440256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":1449,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890208+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T14:56:08.890208+0000","last_peered":"2024-09-13T14:56:08.890208+0000","last_clean":"2024-09-13T14:56:08.890208+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T14:56:08.890208+0000","last_undegraded":"2024-09-13T14:56:08.890208+0000","last_fullsized":"2024-09-13T14:56:08.890208+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884108+0000","last_change":"2024-09-13T14:52:43.599071+0000","last_active":"2024-09-13T14:56:08.884108+0000","last_peered":"2024-09-13T14:56:08.884108+0000","last_clean":"2024-09-13T14:56:08.884108+0000","last_became_active":"2024-09-13T14:52:43.594847+0000","last_became_peered":"2024-09-13T14:52:43.594847+0000","last_unstale":"2024-09-13T14:56:08.884108+0000","last_undegraded":"2024-09-13T14:56:08.884108+0000","last_fullsized":"2024-09-13T14:56:08.884108+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:02.736434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1300,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885476+0000","last_change":"2024-09-13T14:39:34.403170+0000","last_active":"2024-09-13T14:56:08.885476+0000","last_peered":"2024-09-13T14:56:08.885476+0000","last_clean":"2024-09-13T14:56:08.885476+0000","last_became_active":"2024-09-13T14:39:34.389001+0000","last_became_peered":"2024-09-13T14:39:34.389001+0000","last_unstale":"2024-09-13T14:56:08.885476+0000","last_undegraded":"2024-09-13T14:56:08.885476+0000","last_fullsized":"2024-09-13T14:56:08.885476+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:38.542452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":60,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.829168+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T14:55:55.829168+0000","last_peered":"2024-09-13T14:55:55.829168+0000","last_clean":"2024-09-13T14:55:55.829168+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T14:55:55.829168+0000","last_undegraded":"2024-09-13T14:55:55.829168+0000","last_fullsized":"2024-09-13T14:55:55.829168+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1014,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.835885+0000","last_change":"2024-09-13T14:47:46.337827+0000","last_active":"2024-09-13T14:55:55.835885+0000","last_peered":"2024-09-13T14:55:55.835885+0000","last_clean":"2024-09-13T14:55:55.835885+0000","last_became_active":"2024-09-13T14:47:46.321140+0000","last_became_peered":"2024-09-13T14:47:46.321140+0000","last_unstale":"2024-09-13T14:55:55.835885+0000","last_undegraded":"2024-09-13T14:55:55.835885+0000","last_fullsized":"2024-09-13T14:55:55.835885+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:15.997382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":155,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884422+0000","last_change":"2024-09-13T14:47:46.333043+0000","last_active":"2024-09-13T14:56:08.884422+0000","last_peered":"2024-09-13T14:56:08.884422+0000","last_clean":"2024-09-13T14:56:08.884422+0000","last_became_active":"2024-09-13T14:47:46.332292+0000","last_became_peered":"2024-09-13T14:47:46.332292+0000","last_unstale":"2024-09-13T14:56:08.884422+0000","last_undegraded":"2024-09-13T14:56:08.884422+0000","last_fullsized":"2024-09-13T14:56:08.884422+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:38.942334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1059,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.831383+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T14:55:55.831383+0000","last_peered":"2024-09-13T14:55:55.831383+0000","last_clean":"2024-09-13T14:55:55.831383+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T14:55:55.831383+0000","last_undegraded":"2024-09-13T14:55:55.831383+0000","last_fullsized":"2024-09-13T14:55:55.831383+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885269+0000","last_change":"2024-09-13T14:52:43.601798+0000","last_active":"2024-09-13T14:56:08.885269+0000","last_peered":"2024-09-13T14:56:08.885269+0000","last_clean":"2024-09-13T14:56:08.885269+0000","last_became_active":"2024-09-13T14:52:43.600648+0000","last_became_peered":"2024-09-13T14:52:43.600648+0000","last_unstale":"2024-09-13T14:56:08.885269+0000","last_undegraded":"2024-09-13T14:56:08.885269+0000","last_fullsized":"2024-09-13T14:56:08.885269+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:01.021067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1457,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890666+0000","last_change":"2024-09-13T14:47:46.330059+0000","last_active":"2024-09-13T14:56:08.890666+0000","last_peered":"2024-09-13T14:56:08.890666+0000","last_clean":"2024-09-13T14:56:08.890666+0000","last_became_active":"2024-09-13T14:47:46.329344+0000","last_became_peered":"2024-09-13T14:47:46.329344+0000","last_unstale":"2024-09-13T14:56:08.890666+0000","last_undegraded":"2024-09-13T14:56:08.890666+0000","last_fullsized":"2024-09-13T14:56:08.890666+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:58:43.470176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1361,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.879895+0000","last_change":"2024-09-13T14:52:43.594737+0000","last_active":"2024-09-13T14:56:08.879895+0000","last_peered":"2024-09-13T14:56:08.879895+0000","last_clean":"2024-09-13T14:56:08.879895+0000","last_became_active":"2024-09-13T14:52:43.594197+0000","last_became_peered":"2024-09-13T14:52:43.594197+0000","last_unstale":"2024-09-13T14:56:08.879895+0000","last_undegraded":"2024-09-13T14:56:08.879895+0000","last_fullsized":"2024-09-13T14:56:08.879895+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:31.054754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884314+0000","last_change":"2024-09-13T14:52:43.597400+0000","last_active":"2024-09-13T14:56:08.884314+0000","last_peered":"2024-09-13T14:56:08.884314+0000","last_clean":"2024-09-13T14:56:08.884314+0000","last_became_active":"2024-09-13T14:52:43.596294+0000","last_became_peered":"2024-09-13T14:52:43.596294+0000","last_unstale":"2024-09-13T14:56:08.884314+0000","last_undegraded":"2024-09-13T14:56:08.884314+0000","last_fullsized":"2024-09-13T14:56:08.884314+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:35.218171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1346,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.883911+0000","last_change":"2024-09-13T14:52:43.590238+0000","last_active":"2024-09-13T14:56:08.883911+0000","last_peered":"2024-09-13T14:56:08.883911+0000","last_clean":"2024-09-13T14:56:08.883911+0000","last_became_active":"2024-09-13T14:52:43.589713+0000","last_became_peered":"2024-09-13T14:52:43.589713+0000","last_unstale":"2024-09-13T14:56:08.883911+0000","last_undegraded":"2024-09-13T14:56:08.883911+0000","last_fullsized":"2024-09-13T14:56:08.883911+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:19.148195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.897301+0000","last_change":"2024-09-13T14:52:43.591911+0000","last_active":"2024-09-13T14:56:08.897301+0000","last_peered":"2024-09-13T14:56:08.897301+0000","last_clean":"2024-09-13T14:56:08.897301+0000","last_became_active":"2024-09-13T14:52:43.591410+0000","last_became_peered":"2024-09-13T14:52:43.591410+0000","last_unstale":"2024-09-13T14:56:08.897301+0000","last_undegraded":"2024-09-13T14:56:08.897301+0000","last_fullsized":"2024-09-13T14:56:08.897301+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:53.664208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1446,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890593+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T14:56:08.890593+0000","last_peered":"2024-09-13T14:56:08.890593+0000","last_clean":"2024-09-13T14:56:08.890593+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T14:56:08.890593+0000","last_undegraded":"2024-09-13T14:56:08.890593+0000","last_fullsized":"2024-09-13T14:56:08.890593+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1340,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.889541+0000","last_change":"2024-09-13T14:52:43.587987+0000","last_active":"2024-09-13T14:56:08.889541+0000","last_peered":"2024-09-13T14:56:08.889541+0000","last_clean":"2024-09-13T14:56:08.889541+0000","last_became_active":"2024-09-13T14:52:43.587749+0000","last_became_peered":"2024-09-13T14:52:43.587749+0000","last_unstale":"2024-09-13T14:56:08.889541+0000","last_undegraded":"2024-09-13T14:56:08.889541+0000","last_fullsized":"2024-09-13T14:56:08.889541+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:31.924493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1016,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838553+0000","last_change":"2024-09-13T14:52:54.183175+0000","last_active":"2024-09-13T14:55:55.838553+0000","last_peered":"2024-09-13T14:55:55.838553+0000","last_clean":"2024-09-13T14:55:55.838553+0000","last_became_active":"2024-09-13T14:52:54.182514+0000","last_became_peered":"2024-09-13T14:52:54.182514+0000","last_unstale":"2024-09-13T14:55:55.838553+0000","last_undegraded":"2024-09-13T14:55:55.838553+0000","last_fullsized":"2024-09-13T14:55:55.838553+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:55.900845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1320,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.889421+0000","last_change":"2024-09-13T14:52:43.587069+0000","last_active":"2024-09-13T14:56:08.889421+0000","last_peered":"2024-09-13T14:56:08.889421+0000","last_clean":"2024-09-13T14:56:08.889421+0000","last_became_active":"2024-09-13T14:52:43.586196+0000","last_became_peered":"2024-09-13T14:52:43.586196+0000","last_unstale":"2024-09-13T14:56:08.889421+0000","last_undegraded":"2024-09-13T14:56:08.889421+0000","last_fullsized":"2024-09-13T14:56:08.889421+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:50.215660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1437,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890241+0000","last_change":"2024-09-13T14:39:34.318266+0000","last_active":"2024-09-13T14:56:08.890241+0000","last_peered":"2024-09-13T14:56:08.890241+0000","last_clean":"2024-09-13T14:56:08.890241+0000","last_became_active":"2024-09-13T14:39:34.315973+0000","last_became_peered":"2024-09-13T14:39:34.315973+0000","last_unstale":"2024-09-13T14:56:08.890241+0000","last_undegraded":"2024-09-13T14:56:08.890241+0000","last_fullsized":"2024-09-13T14:56:08.890241+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:15.270282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1285,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884122+0000","last_change":"2024-09-13T14:39:34.338378+0000","last_active":"2024-09-13T14:56:08.884122+0000","last_peered":"2024-09-13T14:56:08.884122+0000","last_clean":"2024-09-13T14:56:08.884122+0000","last_became_active":"2024-09-13T14:39:34.338005+0000","last_became_peered":"2024-09-13T14:39:34.338005+0000","last_unstale":"2024-09-13T14:56:08.884122+0000","last_undegraded":"2024-09-13T14:56:08.884122+0000","last_fullsized":"2024-09-13T14:56:08.884122+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:29.120379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1003,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.836420+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T14:55:55.836420+0000","last_peered":"2024-09-13T14:55:55.836420+0000","last_clean":"2024-09-13T14:55:55.836420+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T14:55:55.836420+0000","last_undegraded":"2024-09-13T14:55:55.836420+0000","last_fullsized":"2024-09-13T14:55:55.836420+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1040,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839184+0000","last_change":"2024-09-13T14:47:46.354886+0000","last_active":"2024-09-13T14:55:55.839184+0000","last_peered":"2024-09-13T14:55:55.839184+0000","last_clean":"2024-09-13T14:55:55.839184+0000","last_became_active":"2024-09-13T14:47:46.329947+0000","last_became_peered":"2024-09-13T14:47:46.329947+0000","last_unstale":"2024-09-13T14:55:55.839184+0000","last_undegraded":"2024-09-13T14:55:55.839184+0000","last_fullsized":"2024-09-13T14:55:55.839184+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:22.443920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.894185+0000","last_change":"2024-09-13T14:52:43.586530+0000","last_active":"2024-09-13T14:56:08.894185+0000","last_peered":"2024-09-13T14:56:08.894185+0000","last_clean":"2024-09-13T14:56:08.894185+0000","last_became_active":"2024-09-13T14:52:43.586107+0000","last_became_peered":"2024-09-13T14:52:43.586107+0000","last_unstale":"2024-09-13T14:56:08.894185+0000","last_undegraded":"2024-09-13T14:56:08.894185+0000","last_fullsized":"2024-09-13T14:56:08.894185+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:18.130768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":165,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885616+0000","last_change":"2024-09-13T14:47:46.344536+0000","last_active":"2024-09-13T14:56:08.885616+0000","last_peered":"2024-09-13T14:56:08.885616+0000","last_clean":"2024-09-13T14:56:08.885616+0000","last_became_active":"2024-09-13T14:47:46.321131+0000","last_became_peered":"2024-09-13T14:47:46.321131+0000","last_unstale":"2024-09-13T14:56:08.885616+0000","last_undegraded":"2024-09-13T14:56:08.885616+0000","last_fullsized":"2024-09-13T14:56:08.885616+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:31.114472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885356+0000","last_change":"2024-09-13T14:52:43.600569+0000","last_active":"2024-09-13T14:56:08.885356+0000","last_peered":"2024-09-13T14:56:08.885356+0000","last_clean":"2024-09-13T14:56:08.885356+0000","last_became_active":"2024-09-13T14:52:43.599764+0000","last_became_peered":"2024-09-13T14:52:43.599764+0000","last_unstale":"2024-09-13T14:56:08.885356+0000","last_undegraded":"2024-09-13T14:56:08.885356+0000","last_fullsized":"2024-09-13T14:56:08.885356+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:53.934456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":156,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885116+0000","last_change":"2024-09-13T14:47:46.318785+0000","last_active":"2024-09-13T14:56:08.885116+0000","last_peered":"2024-09-13T14:56:08.885116+0000","last_clean":"2024-09-13T14:56:08.885116+0000","last_became_active":"2024-09-13T14:47:46.318438+0000","last_became_peered":"2024-09-13T14:47:46.318438+0000","last_unstale":"2024-09-13T14:56:08.885116+0000","last_undegraded":"2024-09-13T14:56:08.885116+0000","last_fullsized":"2024-09-13T14:56:08.885116+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:44.745378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":999,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838629+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T14:55:55.838629+0000","last_peered":"2024-09-13T14:55:55.838629+0000","last_clean":"2024-09-13T14:55:55.838629+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T14:55:55.838629+0000","last_undegraded":"2024-09-13T14:55:55.838629+0000","last_fullsized":"2024-09-13T14:55:55.838629+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1498,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890800+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T14:56:08.890800+0000","last_peered":"2024-09-13T14:56:08.890800+0000","last_clean":"2024-09-13T14:56:08.890800+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T14:56:08.890800+0000","last_undegraded":"2024-09-13T14:56:08.890800+0000","last_fullsized":"2024-09-13T14:56:08.890800+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1283,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.889517+0000","last_change":"2024-09-13T14:39:34.349945+0000","last_active":"2024-09-13T14:56:08.889517+0000","last_peered":"2024-09-13T14:56:08.889517+0000","last_clean":"2024-09-13T14:56:08.889517+0000","last_became_active":"2024-09-13T14:39:34.346730+0000","last_became_peered":"2024-09-13T14:39:34.346730+0000","last_unstale":"2024-09-13T14:56:08.889517+0000","last_undegraded":"2024-09-13T14:56:08.889517+0000","last_fullsized":"2024-09-13T14:56:08.889517+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:48.854603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891506+0000","last_change":"2024-09-13T14:52:43.592967+0000","last_active":"2024-09-13T14:56:08.891506+0000","last_peered":"2024-09-13T14:56:08.891506+0000","last_clean":"2024-09-13T14:56:08.891506+0000","last_became_active":"2024-09-13T14:52:43.592431+0000","last_became_peered":"2024-09-13T14:52:43.592431+0000","last_unstale":"2024-09-13T14:56:08.891506+0000","last_undegraded":"2024-09-13T14:56:08.891506+0000","last_fullsized":"2024-09-13T14:56:08.891506+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:56.187568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1286,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884605+0000","last_change":"2024-09-13T14:39:34.401432+0000","last_active":"2024-09-13T14:56:08.884605+0000","last_peered":"2024-09-13T14:56:08.884605+0000","last_clean":"2024-09-13T14:56:08.884605+0000","last_became_active":"2024-09-13T14:39:34.395458+0000","last_became_peered":"2024-09-13T14:39:34.395458+0000","last_unstale":"2024-09-13T14:56:08.884605+0000","last_undegraded":"2024-09-13T14:56:08.884605+0000","last_fullsized":"2024-09-13T14:56:08.884605+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:17.007388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1427,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890453+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T14:56:08.890453+0000","last_peered":"2024-09-13T14:56:08.890453+0000","last_clean":"2024-09-13T14:56:08.890453+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T14:56:08.890453+0000","last_undegraded":"2024-09-13T14:56:08.890453+0000","last_fullsized":"2024-09-13T14:56:08.890453+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1035,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839408+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T14:55:55.839408+0000","last_peered":"2024-09-13T14:55:55.839408+0000","last_clean":"2024-09-13T14:55:55.839408+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T14:55:55.839408+0000","last_undegraded":"2024-09-13T14:55:55.839408+0000","last_fullsized":"2024-09-13T14:55:55.839408+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891074+0000","last_change":"2024-09-13T14:52:43.583860+0000","last_active":"2024-09-13T14:56:08.891074+0000","last_peered":"2024-09-13T14:56:08.891074+0000","last_clean":"2024-09-13T14:56:08.891074+0000","last_became_active":"2024-09-13T14:52:43.583055+0000","last_became_peered":"2024-09-13T14:52:43.583055+0000","last_unstale":"2024-09-13T14:56:08.891074+0000","last_undegraded":"2024-09-13T14:56:08.891074+0000","last_fullsized":"2024-09-13T14:56:08.891074+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:55.113648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.894163+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T14:56:08.894163+0000","last_peered":"2024-09-13T14:56:08.894163+0000","last_clean":"2024-09-13T14:56:08.894163+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T14:56:08.894163+0000","last_undegraded":"2024-09-13T14:56:08.894163+0000","last_fullsized":"2024-09-13T14:56:08.894163+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891249+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T14:56:08.891249+0000","last_peered":"2024-09-13T14:56:08.891249+0000","last_clean":"2024-09-13T14:56:08.891249+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T14:56:08.891249+0000","last_undegraded":"2024-09-13T14:56:08.891249+0000","last_fullsized":"2024-09-13T14:56:08.891249+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1000,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.835951+0000","last_change":"2024-09-13T14:39:34.321388+0000","last_active":"2024-09-13T14:55:55.835951+0000","last_peered":"2024-09-13T14:55:55.835951+0000","last_clean":"2024-09-13T14:55:55.835951+0000","last_became_active":"2024-09-13T14:39:34.321025+0000","last_became_peered":"2024-09-13T14:39:34.321025+0000","last_unstale":"2024-09-13T14:55:55.835951+0000","last_undegraded":"2024-09-13T14:55:55.835951+0000","last_fullsized":"2024-09-13T14:55:55.835951+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:52.593825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891158+0000","last_change":"2024-09-13T14:52:43.583382+0000","last_active":"2024-09-13T14:56:08.891158+0000","last_peered":"2024-09-13T14:56:08.891158+0000","last_clean":"2024-09-13T14:56:08.891158+0000","last_became_active":"2024-09-13T14:52:43.582870+0000","last_became_peered":"2024-09-13T14:52:43.582870+0000","last_unstale":"2024-09-13T14:56:08.891158+0000","last_undegraded":"2024-09-13T14:56:08.891158+0000","last_fullsized":"2024-09-13T14:56:08.891158+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:51.582211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":156,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885474+0000","last_change":"2024-09-13T14:47:46.324072+0000","last_active":"2024-09-13T14:56:08.885474+0000","last_peered":"2024-09-13T14:56:08.885474+0000","last_clean":"2024-09-13T14:56:08.885474+0000","last_became_active":"2024-09-13T14:47:46.322613+0000","last_became_peered":"2024-09-13T14:47:46.322613+0000","last_unstale":"2024-09-13T14:56:08.885474+0000","last_undegraded":"2024-09-13T14:56:08.885474+0000","last_fullsized":"2024-09-13T14:56:08.885474+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:45.314706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1025,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.831558+0000","last_change":"2024-09-13T14:52:43.613106+0000","last_active":"2024-09-13T14:55:55.831558+0000","last_peered":"2024-09-13T14:55:55.831558+0000","last_clean":"2024-09-13T14:55:55.831558+0000","last_became_active":"2024-09-13T14:52:43.594081+0000","last_became_peered":"2024-09-13T14:52:43.594081+0000","last_unstale":"2024-09-13T14:55:55.831558+0000","last_undegraded":"2024-09-13T14:55:55.831558+0000","last_fullsized":"2024-09-13T14:55:55.831558+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:11.121629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891097+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T14:56:08.891097+0000","last_peered":"2024-09-13T14:56:08.891097+0000","last_clean":"2024-09-13T14:56:08.891097+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T14:56:08.891097+0000","last_undegraded":"2024-09-13T14:56:08.891097+0000","last_fullsized":"2024-09-13T14:56:08.891097+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.888004+0000","last_change":"2024-09-13T14:52:43.582732+0000","last_active":"2024-09-13T14:56:08.888004+0000","last_peered":"2024-09-13T14:56:08.888004+0000","last_clean":"2024-09-13T14:56:08.888004+0000","last_became_active":"2024-09-13T14:52:43.582384+0000","last_became_peered":"2024-09-13T14:52:43.582384+0000","last_unstale":"2024-09-13T14:56:08.888004+0000","last_undegraded":"2024-09-13T14:56:08.888004+0000","last_fullsized":"2024-09-13T14:56:08.888004+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:06.715139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1336,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.879632+0000","last_change":"2024-09-13T14:39:34.322533+0000","last_active":"2024-09-13T14:56:08.879632+0000","last_peered":"2024-09-13T14:56:08.879632+0000","last_clean":"2024-09-13T14:56:08.879632+0000","last_became_active":"2024-09-13T14:39:34.322070+0000","last_became_peered":"2024-09-13T14:39:34.322070+0000","last_unstale":"2024-09-13T14:56:08.879632+0000","last_undegraded":"2024-09-13T14:56:08.879632+0000","last_fullsized":"2024-09-13T14:56:08.879632+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:43.746420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1463,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890481+0000","last_change":"2024-09-13T14:52:43.595169+0000","last_active":"2024-09-13T14:56:08.890481+0000","last_peered":"2024-09-13T14:56:08.890481+0000","last_clean":"2024-09-13T14:56:08.890481+0000","last_became_active":"2024-09-13T14:52:43.594586+0000","last_became_peered":"2024-09-13T14:52:43.594586+0000","last_unstale":"2024-09-13T14:56:08.890481+0000","last_undegraded":"2024-09-13T14:56:08.890481+0000","last_fullsized":"2024-09-13T14:56:08.890481+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:53.946025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1463,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.882712+0000","last_change":"2024-09-13T14:47:46.329033+0000","last_active":"2024-09-13T14:56:08.882712+0000","last_peered":"2024-09-13T14:56:08.882712+0000","last_clean":"2024-09-13T14:56:08.882712+0000","last_became_active":"2024-09-13T14:47:46.328203+0000","last_became_peered":"2024-09-13T14:47:46.328203+0000","last_unstale":"2024-09-13T14:56:08.882712+0000","last_undegraded":"2024-09-13T14:56:08.882712+0000","last_fullsized":"2024-09-13T14:56:08.882712+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:23.440178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1035,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839475+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T14:55:55.839475+0000","last_peered":"2024-09-13T14:55:55.839475+0000","last_clean":"2024-09-13T14:55:55.839475+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T14:55:55.839475+0000","last_undegraded":"2024-09-13T14:55:55.839475+0000","last_fullsized":"2024-09-13T14:55:55.839475+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1006,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839241+0000","last_change":"2024-09-13T14:47:46.320830+0000","last_active":"2024-09-13T14:55:55.839241+0000","last_peered":"2024-09-13T14:55:55.839241+0000","last_clean":"2024-09-13T14:55:55.839241+0000","last_became_active":"2024-09-13T14:47:46.320443+0000","last_became_peered":"2024-09-13T14:47:46.320443+0000","last_unstale":"2024-09-13T14:55:55.839241+0000","last_undegraded":"2024-09-13T14:55:55.839241+0000","last_fullsized":"2024-09-13T14:55:55.839241+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:46.279580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1285,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.887397+0000","last_change":"2024-09-13T14:39:34.356129+0000","last_active":"2024-09-13T14:56:08.887397+0000","last_peered":"2024-09-13T14:56:08.887397+0000","last_clean":"2024-09-13T14:56:08.887397+0000","last_became_active":"2024-09-13T14:39:34.355288+0000","last_became_peered":"2024-09-13T14:39:34.355288+0000","last_unstale":"2024-09-13T14:56:08.887397+0000","last_undegraded":"2024-09-13T14:56:08.887397+0000","last_fullsized":"2024-09-13T14:56:08.887397+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.725003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":996,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.835370+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T14:55:55.835370+0000","last_peered":"2024-09-13T14:55:55.835370+0000","last_clean":"2024-09-13T14:55:55.835370+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T14:55:55.835370+0000","last_undegraded":"2024-09-13T14:55:55.835370+0000","last_fullsized":"2024-09-13T14:55:55.835370+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1103,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839553+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T14:55:55.839553+0000","last_peered":"2024-09-13T14:55:55.839553+0000","last_clean":"2024-09-13T14:55:55.839553+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T14:55:55.839553+0000","last_undegraded":"2024-09-13T14:55:55.839553+0000","last_fullsized":"2024-09-13T14:55:55.839553+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":59,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891129+0000","last_change":"2024-09-13T14:52:43.591841+0000","last_active":"2024-09-13T14:56:08.891129+0000","last_peered":"2024-09-13T14:56:08.891129+0000","last_clean":"2024-09-13T14:56:08.891129+0000","last_became_active":"2024-09-13T14:52:43.591187+0000","last_became_peered":"2024-09-13T14:52:43.591187+0000","last_unstale":"2024-09-13T14:56:08.891129+0000","last_undegraded":"2024-09-13T14:56:08.891129+0000","last_fullsized":"2024-09-13T14:56:08.891129+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:29.136085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1323,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.889586+0000","last_change":"2024-09-13T14:52:43.589345+0000","last_active":"2024-09-13T14:56:08.889586+0000","last_peered":"2024-09-13T14:56:08.889586+0000","last_clean":"2024-09-13T14:56:08.889586+0000","last_became_active":"2024-09-13T14:52:43.588829+0000","last_became_peered":"2024-09-13T14:52:43.588829+0000","last_unstale":"2024-09-13T14:56:08.889586+0000","last_undegraded":"2024-09-13T14:56:08.889586+0000","last_fullsized":"2024-09-13T14:56:08.889586+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:49.701805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884433+0000","last_change":"2024-09-13T14:52:43.599179+0000","last_active":"2024-09-13T14:56:08.884433+0000","last_peered":"2024-09-13T14:56:08.884433+0000","last_clean":"2024-09-13T14:56:08.884433+0000","last_became_active":"2024-09-13T14:52:43.598166+0000","last_became_peered":"2024-09-13T14:52:43.598166+0000","last_unstale":"2024-09-13T14:56:08.884433+0000","last_undegraded":"2024-09-13T14:56:08.884433+0000","last_fullsized":"2024-09-13T14:56:08.884433+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:08.647786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":156,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885091+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T14:56:08.885091+0000","last_peered":"2024-09-13T14:56:08.885091+0000","last_clean":"2024-09-13T14:56:08.885091+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T14:56:08.885091+0000","last_undegraded":"2024-09-13T14:56:08.885091+0000","last_fullsized":"2024-09-13T14:56:08.885091+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":169,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884807+0000","last_change":"2024-09-13T14:52:43.597595+0000","last_active":"2024-09-13T14:56:08.884807+0000","last_peered":"2024-09-13T14:56:08.884807+0000","last_clean":"2024-09-13T14:56:08.884807+0000","last_became_active":"2024-09-13T14:52:43.596592+0000","last_became_peered":"2024-09-13T14:52:43.596592+0000","last_unstale":"2024-09-13T14:56:08.884807+0000","last_undegraded":"2024-09-13T14:56:08.884807+0000","last_fullsized":"2024-09-13T14:56:08.884807+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:56.532603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1044,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838947+0000","last_change":"2024-09-13T14:52:43.596429+0000","last_active":"2024-09-13T14:55:55.838947+0000","last_peered":"2024-09-13T14:55:55.838947+0000","last_clean":"2024-09-13T14:55:55.838947+0000","last_became_active":"2024-09-13T14:52:43.594614+0000","last_became_peered":"2024-09-13T14:52:43.594614+0000","last_unstale":"2024-09-13T14:55:55.838947+0000","last_undegraded":"2024-09-13T14:55:55.838947+0000","last_fullsized":"2024-09-13T14:55:55.838947+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:41.970225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":155,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885520+0000","last_change":"2024-09-13T14:47:46.325496+0000","last_active":"2024-09-13T14:56:08.885520+0000","last_peered":"2024-09-13T14:56:08.885520+0000","last_clean":"2024-09-13T14:56:08.885520+0000","last_became_active":"2024-09-13T14:47:46.325099+0000","last_became_peered":"2024-09-13T14:47:46.325099+0000","last_unstale":"2024-09-13T14:56:08.885520+0000","last_undegraded":"2024-09-13T14:56:08.885520+0000","last_fullsized":"2024-09-13T14:56:08.885520+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:48.803608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1320,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884739+0000","last_change":"2024-09-13T14:47:46.349682+0000","last_active":"2024-09-13T14:56:08.884739+0000","last_peered":"2024-09-13T14:56:08.884739+0000","last_clean":"2024-09-13T14:56:08.884739+0000","last_became_active":"2024-09-13T14:47:46.341656+0000","last_became_peered":"2024-09-13T14:47:46.341656+0000","last_unstale":"2024-09-13T14:56:08.884739+0000","last_undegraded":"2024-09-13T14:56:08.884739+0000","last_fullsized":"2024-09-13T14:56:08.884739+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:02.727169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1484,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890571+0000","last_change":"2024-09-13T14:52:54.185117+0000","last_active":"2024-09-13T14:56:08.890571+0000","last_peered":"2024-09-13T14:56:08.890571+0000","last_clean":"2024-09-13T14:56:08.890571+0000","last_became_active":"2024-09-13T14:52:54.184712+0000","last_became_peered":"2024-09-13T14:52:54.184712+0000","last_unstale":"2024-09-13T14:56:08.890571+0000","last_undegraded":"2024-09-13T14:56:08.890571+0000","last_fullsized":"2024-09-13T14:56:08.890571+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:32.654401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":983,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.831348+0000","last_change":"2024-09-13T14:39:34.393673+0000","last_active":"2024-09-13T14:55:55.831348+0000","last_peered":"2024-09-13T14:55:55.831348+0000","last_clean":"2024-09-13T14:55:55.831348+0000","last_became_active":"2024-09-13T14:39:34.393186+0000","last_became_peered":"2024-09-13T14:39:34.393186+0000","last_unstale":"2024-09-13T14:55:55.831348+0000","last_undegraded":"2024-09-13T14:55:55.831348+0000","last_fullsized":"2024-09-13T14:55:55.831348+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:23.130084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1006,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838571+0000","last_change":"2024-09-13T14:52:43.581604+0000","last_active":"2024-09-13T14:55:55.838571+0000","last_peered":"2024-09-13T14:55:55.838571+0000","last_clean":"2024-09-13T14:55:55.838571+0000","last_became_active":"2024-09-13T14:52:43.579981+0000","last_became_peered":"2024-09-13T14:52:43.579981+0000","last_unstale":"2024-09-13T14:55:55.838571+0000","last_undegraded":"2024-09-13T14:55:55.838571+0000","last_fullsized":"2024-09-13T14:55:55.838571+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:59.366292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":971,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.839329+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:55:55.839329+0000","last_peered":"2024-09-13T14:55:55.839329+0000","last_clean":"2024-09-13T14:55:55.839329+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:55:55.839329+0000","last_undegraded":"2024-09-13T14:55:55.839329+0000","last_fullsized":"2024-09-13T14:55:55.839329+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":170,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885213+0000","last_change":"2024-09-13T14:52:43.595628+0000","last_active":"2024-09-13T14:56:08.885213+0000","last_peered":"2024-09-13T14:56:08.885213+0000","last_clean":"2024-09-13T14:56:08.885213+0000","last_became_active":"2024-09-13T14:52:43.594675+0000","last_became_peered":"2024-09-13T14:52:43.594675+0000","last_unstale":"2024-09-13T14:56:08.885213+0000","last_undegraded":"2024-09-13T14:56:08.885213+0000","last_fullsized":"2024-09-13T14:56:08.885213+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:40:19.850839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1062,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838056+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T14:55:55.838056+0000","last_peered":"2024-09-13T14:55:55.838056+0000","last_clean":"2024-09-13T14:55:55.838056+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T14:55:55.838056+0000","last_undegraded":"2024-09-13T14:55:55.838056+0000","last_fullsized":"2024-09-13T14:55:55.838056+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":965,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.840054+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:55:55.840054+0000","last_peered":"2024-09-13T14:55:55.840054+0000","last_clean":"2024-09-13T14:55:55.840054+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:55:55.840054+0000","last_undegraded":"2024-09-13T14:55:55.840054+0000","last_fullsized":"2024-09-13T14:55:55.840054+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1033,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.842761+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T14:55:55.842761+0000","last_peered":"2024-09-13T14:55:55.842761+0000","last_clean":"2024-09-13T14:55:55.842761+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T14:55:55.842761+0000","last_undegraded":"2024-09-13T14:55:55.842761+0000","last_fullsized":"2024-09-13T14:55:55.842761+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":999,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.838489+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T14:55:55.838489+0000","last_peered":"2024-09-13T14:55:55.838489+0000","last_clean":"2024-09-13T14:55:55.838489+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T14:55:55.838489+0000","last_undegraded":"2024-09-13T14:55:55.838489+0000","last_fullsized":"2024-09-13T14:55:55.838489+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1322,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884493+0000","last_change":"2024-09-13T14:52:43.592945+0000","last_active":"2024-09-13T14:56:08.884493+0000","last_peered":"2024-09-13T14:56:08.884493+0000","last_clean":"2024-09-13T14:56:08.884493+0000","last_became_active":"2024-09-13T14:52:43.592602+0000","last_became_peered":"2024-09-13T14:52:43.592602+0000","last_unstale":"2024-09-13T14:56:08.884493+0000","last_undegraded":"2024-09-13T14:56:08.884493+0000","last_fullsized":"2024-09-13T14:56:08.884493+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:11.719764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1006,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.837915+0000","last_change":"2024-09-13T14:47:46.322428+0000","last_active":"2024-09-13T14:55:55.837915+0000","last_peered":"2024-09-13T14:55:55.837915+0000","last_clean":"2024-09-13T14:55:55.837915+0000","last_became_active":"2024-09-13T14:47:46.321955+0000","last_became_peered":"2024-09-13T14:47:46.321955+0000","last_unstale":"2024-09-13T14:55:55.837915+0000","last_undegraded":"2024-09-13T14:55:55.837915+0000","last_fullsized":"2024-09-13T14:55:55.837915+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:47.614065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1437,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.890717+0000","last_change":"2024-09-13T14:39:34.362694+0000","last_active":"2024-09-13T14:56:08.890717+0000","last_peered":"2024-09-13T14:56:08.890717+0000","last_clean":"2024-09-13T14:56:08.890717+0000","last_became_active":"2024-09-13T14:39:34.362435+0000","last_became_peered":"2024-09-13T14:39:34.362435+0000","last_unstale":"2024-09-13T14:56:08.890717+0000","last_undegraded":"2024-09-13T14:56:08.890717+0000","last_fullsized":"2024-09-13T14:56:08.890717+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:33.350328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1437,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.887142+0000","last_change":"2024-09-13T14:39:34.360941+0000","last_active":"2024-09-13T14:56:08.887142+0000","last_peered":"2024-09-13T14:56:08.887142+0000","last_clean":"2024-09-13T14:56:08.887142+0000","last_became_active":"2024-09-13T14:39:34.360383+0000","last_became_peered":"2024-09-13T14:39:34.360383+0000","last_unstale":"2024-09-13T14:56:08.887142+0000","last_undegraded":"2024-09-13T14:56:08.887142+0000","last_fullsized":"2024-09-13T14:56:08.887142+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:51.344296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":982,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.835813+0000","last_change":"2024-09-13T14:39:34.380634+0000","last_active":"2024-09-13T14:55:55.835813+0000","last_peered":"2024-09-13T14:55:55.835813+0000","last_clean":"2024-09-13T14:55:55.835813+0000","last_became_active":"2024-09-13T14:39:34.379785+0000","last_became_peered":"2024-09-13T14:39:34.379785+0000","last_unstale":"2024-09-13T14:55:55.835813+0000","last_undegraded":"2024-09-13T14:55:55.835813+0000","last_fullsized":"2024-09-13T14:55:55.835813+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:02.436193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1039,"reported_epoch":1028,"state":"active+clean","last_fresh":"2024-09-13T14:55:55.837955+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T14:55:55.837955+0000","last_peered":"2024-09-13T14:55:55.837955+0000","last_clean":"2024-09-13T14:55:55.837955+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T14:55:55.837955+0000","last_undegraded":"2024-09-13T14:55:55.837955+0000","last_fullsized":"2024-09-13T14:55:55.837955+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1329,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884649+0000","last_change":"2024-09-13T14:52:54.156534+0000","last_active":"2024-09-13T14:56:08.884649+0000","last_peered":"2024-09-13T14:56:08.884649+0000","last_clean":"2024-09-13T14:56:08.884649+0000","last_became_active":"2024-09-13T14:52:54.156241+0000","last_became_peered":"2024-09-13T14:52:54.156241+0000","last_unstale":"2024-09-13T14:56:08.884649+0000","last_undegraded":"2024-09-13T14:56:08.884649+0000","last_fullsized":"2024-09-13T14:56:08.884649+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:17.998268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":42,"num_pg":16,"stat_sum":{"num_bytes":2229403672,"num_objects":34019,"num_object_clones":0,"num_object_copies":102057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34019,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181657,"num_write_kb":6901569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6636187648,"data_stored":6636175432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141957,"ondisk_log_size":141957,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397947,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1432232,"kb_used_data":1126560,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92312920,"statfs":{"total":95995035648,"available":94528430080,"internally_reserved":0,"allocated":1153597440,"data_stored":1149647948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594182,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":868388,"kb_used_data":558492,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92876764,"statfs":{"total":95995035648,"available":95105806336,"internally_reserved":0,"allocated":571895808,"data_stored":567950309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790416,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1563288,"kb_used_data":1261776,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92181864,"statfs":{"total":95995035648,"available":94394228736,"internally_reserved":0,"allocated":1292058624,"data_stored":1288120687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,6,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980831945,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1122464,"kb_used_data":781720,"kb_used_omap":76,"kb_used_meta":340659,"kb_avail":92622688,"statfs":{"total":95995035648,"available":94845632512,"internally_reserved":0,"allocated":800481280,"data_stored":796539378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":348835648},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911183,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1149324,"kb_used_data":857028,"kb_used_omap":77,"kb_used_meta":292210,"kb_avail":92595828,"statfs":{"total":95995035648,"available":94818127872,"internally_reserved":0,"allocated":877596672,"data_stored":873670458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":299223861},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023127,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1011728,"kb_used_data":662216,"kb_used_omap":71,"kb_used_meta":349432,"kb_avail":92733424,"statfs":{"total":95995035648,"available":94959026176,"internally_reserved":0,"allocated":678109184,"data_stored":674179396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":357819040},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004256,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1054352,"kb_used_data":841928,"kb_used_omap":66,"kb_used_meta":212285,"kb_avail":92690800,"statfs":{"total":95995035648,"available":94915379200,"internally_reserved":0,"allocated":862134272,"data_stored":858203923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":217380010},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195449,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":859088,"kb_used_data":528904,"kb_used_omap":56,"kb_used_meta":330119,"kb_avail":92886064,"statfs":{"total":95995035648,"available":95115329536,"internally_reserved":0,"allocated":541597696,"data_stored":537666318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":338041999},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":529465344,"data_stored":529465344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":844369920,"data_stored":844365848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":664469504,"data_stored":664469504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":859308032,"data_stored":859308032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":789643264,"data_stored":789643264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1259147264,"data_stored":1259143192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":557125632,"data_stored":557121560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1132658688,"data_stored":1132658688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:56:47.304 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:56:47.304 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:56:47.305 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2024-09-13T14:56:47.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:56:47.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:47.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:56:47.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:47.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:47.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:47.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:47.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:47.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:47.682 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:47.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:47.784 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:47.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:47.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:47 smithi177 bash[68594]: cluster 2024-09-13T14:56:46.690354+0000 mgr.y (mgr.26252) 917 : cluster [DBG] pgmap v1438: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T14:56:47.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:47 smithi177 bash[68594]: cluster 2024-09-13T14:56:46.690354+0000 mgr.y (mgr.26252) 917 : cluster [DBG] pgmap v1438: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T14:56:47.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:47.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:48.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:48.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:47 smithi099 bash[84576]: cluster 2024-09-13T14:56:46.690354+0000 mgr.y (mgr.26252) 917 : cluster [DBG] pgmap v1438: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T14:56:48.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:47 smithi099 bash[84576]: cluster 2024-09-13T14:56:46.690354+0000 mgr.y (mgr.26252) 917 : cluster [DBG] pgmap v1438: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T14:56:48.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:47 smithi099 bash[87147]: cluster 2024-09-13T14:56:46.690354+0000 mgr.y (mgr.26252) 917 : cluster [DBG] pgmap v1438: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T14:56:48.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:47 smithi099 bash[87147]: cluster 2024-09-13T14:56:46.690354+0000 mgr.y (mgr.26252) 917 : cluster [DBG] pgmap v1438: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T14:56:48.064 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:48.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:48.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:48.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:48.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:48.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:48.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:48.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:48.436 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:48.456 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:48.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:48.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:48.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:48.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:48.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:48.723 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:48.826 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:48.850 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:48.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:48.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:48.990 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:49.097 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:49.128 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:49.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:49.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:56:49.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:49.361 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:49.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.516 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[84576]: audit 2024-09-13T14:56:48.293105+0000 mon.a (mon.0) 1975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:49.516 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[84576]: audit 2024-09-13T14:56:48.293105+0000 mon.a (mon.0) 1975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:49.516 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[84576]: audit 2024-09-13T14:56:48.294208+0000 mon.a (mon.0) 1976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:49.516 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[84576]: audit 2024-09-13T14:56:48.294208+0000 mon.a (mon.0) 1976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:49.516 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[84576]: cluster 2024-09-13T14:56:48.690741+0000 mgr.y (mgr.26252) 918 : cluster [DBG] pgmap v1439: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-09-13T14:56:49.516 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[84576]: cluster 2024-09-13T14:56:48.690741+0000 mgr.y (mgr.26252) 918 : cluster [DBG] pgmap v1439: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-09-13T14:56:49.516 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[87147]: audit 2024-09-13T14:56:48.293105+0000 mon.a (mon.0) 1975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:49.516 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[87147]: audit 2024-09-13T14:56:48.293105+0000 mon.a (mon.0) 1975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:49.516 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[87147]: audit 2024-09-13T14:56:48.294208+0000 mon.a (mon.0) 1976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:49.516 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[87147]: audit 2024-09-13T14:56:48.294208+0000 mon.a (mon.0) 1976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:49.517 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[87147]: cluster 2024-09-13T14:56:48.690741+0000 mgr.y (mgr.26252) 918 : cluster [DBG] pgmap v1439: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-09-13T14:56:49.517 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:49 smithi099 bash[87147]: cluster 2024-09-13T14:56:48.690741+0000 mgr.y (mgr.26252) 918 : cluster [DBG] pgmap v1439: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-09-13T14:56:49.539 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:49.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:49 smithi177 bash[68594]: audit 2024-09-13T14:56:48.293105+0000 mon.a (mon.0) 1975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:49.614 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:49 smithi177 bash[68594]: audit 2024-09-13T14:56:48.293105+0000 mon.a (mon.0) 1975 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:56:49.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:49 smithi177 bash[68594]: audit 2024-09-13T14:56:48.294208+0000 mon.a (mon.0) 1976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:49.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:49 smithi177 bash[68594]: audit 2024-09-13T14:56:48.294208+0000 mon.a (mon.0) 1976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:49.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:49 smithi177 bash[68594]: cluster 2024-09-13T14:56:48.690741+0000 mgr.y (mgr.26252) 918 : cluster [DBG] pgmap v1439: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-09-13T14:56:49.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:49 smithi177 bash[68594]: cluster 2024-09-13T14:56:48.690741+0000 mgr.y (mgr.26252) 918 : cluster [DBG] pgmap v1439: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2024-09-13T14:56:49.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:49.672 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:49.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:49.774 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:49.853 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:49.917 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:49.959 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:49.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:50.062 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:50.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:50.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:50.196 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:50.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:50.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:50.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:50.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:50.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:50.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:50.461 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:56:50 smithi177 bash[61439]: ts=2024-09-13T14:56:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:56:50.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:50.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:50.577 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:50.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:50.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:50.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:50.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:50.836 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:50.864 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:50.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:50.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:50.988 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:50.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:51.091 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:51.134 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:51.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:51.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:51.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:51.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:51.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:51.428 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:51.532 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:51.557 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:51.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:51.660 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:51.681 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:51.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:51.790 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:51.799 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:51 smithi177 bash[68594]: cluster 2024-09-13T14:56:50.691705+0000 mgr.y (mgr.26252) 919 : cluster [DBG] pgmap v1440: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-13T14:56:51.799 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:51 smithi177 bash[68594]: cluster 2024-09-13T14:56:50.691705+0000 mgr.y (mgr.26252) 919 : cluster [DBG] pgmap v1440: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-13T14:56:51.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:51.880 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:51.880 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:51.928 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:51.936 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:51 smithi099 bash[84576]: cluster 2024-09-13T14:56:50.691705+0000 mgr.y (mgr.26252) 919 : cluster [DBG] pgmap v1440: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-13T14:56:51.936 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:51 smithi099 bash[84576]: cluster 2024-09-13T14:56:50.691705+0000 mgr.y (mgr.26252) 919 : cluster [DBG] pgmap v1440: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-13T14:56:51.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:51 smithi099 bash[87147]: cluster 2024-09-13T14:56:50.691705+0000 mgr.y (mgr.26252) 919 : cluster [DBG] pgmap v1440: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-13T14:56:51.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:51 smithi099 bash[87147]: cluster 2024-09-13T14:56:50.691705+0000 mgr.y (mgr.26252) 919 : cluster [DBG] pgmap v1440: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-13T14:56:51.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:51.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:52.059 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:52.079 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:52.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:52.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:52.224 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:52.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:52.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:52.373 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:52.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:52.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:52.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:52.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:52.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:52.644 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:52.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:52.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:52.786 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:52.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:52.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:52.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:52.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:53.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:53.059 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:53.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:53.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:53.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:53.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:53.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:53.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:53.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:53.434 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:53.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:53.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:53.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:53.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:53.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:53.698 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:53.729 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:53.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:53.833 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:53.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:53.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:53.906 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:53.970 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:53.983 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:53 smithi099 bash[84576]: cluster 2024-09-13T14:56:52.692257+0000 mgr.y (mgr.26252) 920 : cluster [DBG] pgmap v1441: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:56:53.983 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:53 smithi099 bash[84576]: cluster 2024-09-13T14:56:52.692257+0000 mgr.y (mgr.26252) 920 : cluster [DBG] pgmap v1441: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:56:53.983 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:53 smithi099 bash[87147]: cluster 2024-09-13T14:56:52.692257+0000 mgr.y (mgr.26252) 920 : cluster [DBG] pgmap v1441: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:56:53.983 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:53 smithi099 bash[87147]: cluster 2024-09-13T14:56:52.692257+0000 mgr.y (mgr.26252) 920 : cluster [DBG] pgmap v1441: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:56:54.001 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:54.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:54.104 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:54.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:53 smithi177 bash[68594]: cluster 2024-09-13T14:56:52.692257+0000 mgr.y (mgr.26252) 920 : cluster [DBG] pgmap v1441: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:56:54.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:53 smithi177 bash[68594]: cluster 2024-09-13T14:56:52.692257+0000 mgr.y (mgr.26252) 920 : cluster [DBG] pgmap v1441: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:56:54.129 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:54.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:54.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:54.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:54.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:54.372 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:54.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:54.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:54.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:54.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:54.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:54.659 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:54.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:54.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:54.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:54.825 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:54.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:54.928 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:54.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:54.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:55.065 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:55.097 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:55.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:55.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:55.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:55.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:55.372 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:55.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:55.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:55.497 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:55.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:55.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:55.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:55.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:55.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:55.770 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:55.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:55.901 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:55.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:56.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:56.012 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:55 smithi177 bash[68594]: cluster 2024-09-13T14:56:54.692993+0000 mgr.y (mgr.26252) 921 : cluster [DBG] pgmap v1442: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-13T14:56:56.012 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:55 smithi177 bash[68594]: cluster 2024-09-13T14:56:54.692993+0000 mgr.y (mgr.26252) 921 : cluster [DBG] pgmap v1442: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-13T14:56:56.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:56.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:56.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:55 smithi099 bash[84576]: cluster 2024-09-13T14:56:54.692993+0000 mgr.y (mgr.26252) 921 : cluster [DBG] pgmap v1442: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-13T14:56:56.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:55 smithi099 bash[84576]: cluster 2024-09-13T14:56:54.692993+0000 mgr.y (mgr.26252) 921 : cluster [DBG] pgmap v1442: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-13T14:56:56.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:55 smithi099 bash[87147]: cluster 2024-09-13T14:56:54.692993+0000 mgr.y (mgr.26252) 921 : cluster [DBG] pgmap v1442: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-13T14:56:56.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:55 smithi099 bash[87147]: cluster 2024-09-13T14:56:54.692993+0000 mgr.y (mgr.26252) 921 : cluster [DBG] pgmap v1442: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-13T14:56:56.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:56.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:56.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:56.279 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:56.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:56.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:56.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:56.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:56.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:56.541 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:56.564 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:56.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:56.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:56.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:56.819 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:56.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:56.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:56.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:56.997 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:57.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:57.103 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:57.110 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:56:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:56:57] "GET /metrics HTTP/1.1" 200 36761 "" "Prometheus/2.51.0" 2024-09-13T14:56:57.121 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:57.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:57.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:57.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:57.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:57.383 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:57.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:57.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:57.536 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:57.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:57.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:57.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:57.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:57.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:57.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:57.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:57.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:57.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:58.058 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:58.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:57 smithi099 bash[84576]: cluster 2024-09-13T14:56:56.693343+0000 mgr.y (mgr.26252) 922 : cluster [DBG] pgmap v1443: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:58.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:57 smithi099 bash[84576]: cluster 2024-09-13T14:56:56.693343+0000 mgr.y (mgr.26252) 922 : cluster [DBG] pgmap v1443: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:58.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:57 smithi099 bash[87147]: cluster 2024-09-13T14:56:56.693343+0000 mgr.y (mgr.26252) 922 : cluster [DBG] pgmap v1443: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:58.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:57 smithi099 bash[87147]: cluster 2024-09-13T14:56:56.693343+0000 mgr.y (mgr.26252) 922 : cluster [DBG] pgmap v1443: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:58.083 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:58.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:58.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:57 smithi177 bash[68594]: cluster 2024-09-13T14:56:56.693343+0000 mgr.y (mgr.26252) 922 : cluster [DBG] pgmap v1443: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:58.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:57 smithi177 bash[68594]: cluster 2024-09-13T14:56:56.693343+0000 mgr.y (mgr.26252) 922 : cluster [DBG] pgmap v1443: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2024-09-13T14:56:58.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:58.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:58.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:58.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:58.353 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:58.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:58.478 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:58.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:58.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:58.612 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:58.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:58.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:58.747 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:58.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:58.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:58.874 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:58.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:58.978 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:58.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:58.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:59.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:59.121 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:59.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:59.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:59.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:59.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:59.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:59.369 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:59.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:59.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:59.500 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:59.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:59.606 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:59.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:59.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:59.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:59.789 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:59.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:59.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:59.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:00.019 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:59 smithi099 bash[84576]: cluster 2024-09-13T14:56:58.693833+0000 mgr.y (mgr.26252) 923 : cluster [DBG] pgmap v1444: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2024-09-13T14:57:00.019 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:56:59 smithi099 bash[84576]: cluster 2024-09-13T14:56:58.693833+0000 mgr.y (mgr.26252) 923 : cluster [DBG] pgmap v1444: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2024-09-13T14:57:00.020 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:59 smithi099 bash[87147]: cluster 2024-09-13T14:56:58.693833+0000 mgr.y (mgr.26252) 923 : cluster [DBG] pgmap v1444: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2024-09-13T14:57:00.020 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:56:59 smithi099 bash[87147]: cluster 2024-09-13T14:56:58.693833+0000 mgr.y (mgr.26252) 923 : cluster [DBG] pgmap v1444: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2024-09-13T14:57:00.024 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.072 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:59 smithi177 bash[68594]: cluster 2024-09-13T14:56:58.693833+0000 mgr.y (mgr.26252) 923 : cluster [DBG] pgmap v1444: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2024-09-13T14:57:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:56:59 smithi177 bash[68594]: cluster 2024-09-13T14:56:58.693833+0000 mgr.y (mgr.26252) 923 : cluster [DBG] pgmap v1444: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2024-09-13T14:57:00.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:00.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:00.344 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:00.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:00.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:00.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:00.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:00.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:57:00 smithi177 bash[61439]: ts=2024-09-13T14:57:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:00.615 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:00.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:00.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:00.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.872 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:00.974 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.998 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:01.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:01.102 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:01.123 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:01.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:01.226 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:01.265 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:01.371 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:01.394 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:01.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:01.499 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:01.529 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:01.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:01.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:01.656 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:01.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:01.761 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:01.792 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:01.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:01.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:01.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:01.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:02.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:02.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:01 smithi099 bash[84576]: cluster 2024-09-13T14:57:00.694456+0000 mgr.y (mgr.26252) 924 : cluster [DBG] pgmap v1445: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:57:02.038 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:01 smithi099 bash[84576]: cluster 2024-09-13T14:57:00.694456+0000 mgr.y (mgr.26252) 924 : cluster [DBG] pgmap v1445: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:57:02.042 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:01 smithi099 bash[87147]: cluster 2024-09-13T14:57:00.694456+0000 mgr.y (mgr.26252) 924 : cluster [DBG] pgmap v1445: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:57:02.042 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:01 smithi099 bash[87147]: cluster 2024-09-13T14:57:00.694456+0000 mgr.y (mgr.26252) 924 : cluster [DBG] pgmap v1445: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:57:02.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:02.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:02.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:01 smithi177 bash[68594]: cluster 2024-09-13T14:57:00.694456+0000 mgr.y (mgr.26252) 924 : cluster [DBG] pgmap v1445: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:57:02.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:01 smithi177 bash[68594]: cluster 2024-09-13T14:57:00.694456+0000 mgr.y (mgr.26252) 924 : cluster [DBG] pgmap v1445: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-13T14:57:02.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:02.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:02.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:02.278 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:02.314 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:02.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:02.447 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:02.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:02.583 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:02.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:02.720 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:02.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:02.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:02.866 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:02.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:02.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:03.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:03.127 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:03.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:03.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:03.368 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:03.402 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:03.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:03.508 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.547 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:03.653 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:03.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:03.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:03.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:03.922 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:04.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:04.068 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:03 smithi177 bash[68594]: cluster 2024-09-13T14:57:02.694989+0000 mgr.y (mgr.26252) 925 : cluster [DBG] pgmap v1446: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T14:57:04.068 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:03 smithi177 bash[68594]: cluster 2024-09-13T14:57:02.694989+0000 mgr.y (mgr.26252) 925 : cluster [DBG] pgmap v1446: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T14:57:04.068 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:03 smithi177 bash[68594]: audit 2024-09-13T14:57:03.284919+0000 mon.a (mon.0) 1977 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:04.068 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:03 smithi177 bash[68594]: audit 2024-09-13T14:57:03.284919+0000 mon.a (mon.0) 1977 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:04.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:03 smithi099 bash[84576]: cluster 2024-09-13T14:57:02.694989+0000 mgr.y (mgr.26252) 925 : cluster [DBG] pgmap v1446: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T14:57:04.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:03 smithi099 bash[84576]: cluster 2024-09-13T14:57:02.694989+0000 mgr.y (mgr.26252) 925 : cluster [DBG] pgmap v1446: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T14:57:04.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:03 smithi099 bash[84576]: audit 2024-09-13T14:57:03.284919+0000 mon.a (mon.0) 1977 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:04.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:03 smithi099 bash[84576]: audit 2024-09-13T14:57:03.284919+0000 mon.a (mon.0) 1977 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:04.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:03 smithi099 bash[87147]: cluster 2024-09-13T14:57:02.694989+0000 mgr.y (mgr.26252) 925 : cluster [DBG] pgmap v1446: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T14:57:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:03 smithi099 bash[87147]: cluster 2024-09-13T14:57:02.694989+0000 mgr.y (mgr.26252) 925 : cluster [DBG] pgmap v1446: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T14:57:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:03 smithi099 bash[87147]: audit 2024-09-13T14:57:03.284919+0000 mon.a (mon.0) 1977 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:03 smithi099 bash[87147]: audit 2024-09-13T14:57:03.284919+0000 mon.a (mon.0) 1977 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:04.084 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:04.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:04.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:04.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:04.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:04.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:04.341 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:04.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:04.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:04.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:04.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:04.588 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.617 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:04.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:04.718 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:04.747 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:04.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:04.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:04.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:04.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:04.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:05.006 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:05.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:05.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:05.132 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:05.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:05.236 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:05.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:05.366 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:05.393 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:05.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:05.497 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:05.520 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:05.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:05.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:05.650 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:05.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:05.755 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:05.782 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:05.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:05.887 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:05.908 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:05.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:06.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:06.028 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:05 smithi099 bash[84576]: cluster 2024-09-13T14:57:04.695511+0000 mgr.y (mgr.26252) 926 : cluster [DBG] pgmap v1447: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-09-13T14:57:06.029 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:05 smithi099 bash[84576]: cluster 2024-09-13T14:57:04.695511+0000 mgr.y (mgr.26252) 926 : cluster [DBG] pgmap v1447: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-09-13T14:57:06.029 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:05 smithi099 bash[87147]: cluster 2024-09-13T14:57:04.695511+0000 mgr.y (mgr.26252) 926 : cluster [DBG] pgmap v1447: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-09-13T14:57:06.030 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:05 smithi099 bash[87147]: cluster 2024-09-13T14:57:04.695511+0000 mgr.y (mgr.26252) 926 : cluster [DBG] pgmap v1447: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-09-13T14:57:06.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:06.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:06.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:05 smithi177 bash[68594]: cluster 2024-09-13T14:57:04.695511+0000 mgr.y (mgr.26252) 926 : cluster [DBG] pgmap v1447: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-09-13T14:57:06.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:05 smithi177 bash[68594]: cluster 2024-09-13T14:57:04.695511+0000 mgr.y (mgr.26252) 926 : cluster [DBG] pgmap v1447: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-09-13T14:57:06.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:06.186 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:06.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:06.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:06.325 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:06.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:06.457 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:06.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:06.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:06.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:06.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:06.625 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:57:06.626 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:06.642 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:57:06.643 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:06.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:06.707 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:06.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:06.730 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:57:06.731 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:06.753 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:06.753 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:57:06.790 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:57:06.790 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:06.813 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:06.827 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:06.827 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:57:06.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:06.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:06.847 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:57:06.848 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:06.876 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:57:06.876 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:06.945 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:06.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:07.028 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:06 smithi099 bash[84576]: audit 2024-09-13T14:57:06.607980+0000 mon.a (mon.0) 1978 : audit [DBG] from='client.? 172.21.15.99:0/2308774132' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:07.028 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:06 smithi099 bash[84576]: audit 2024-09-13T14:57:06.607980+0000 mon.a (mon.0) 1978 : audit [DBG] from='client.? 172.21.15.99:0/2308774132' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:07.029 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:06 smithi099 bash[87147]: audit 2024-09-13T14:57:06.607980+0000 mon.a (mon.0) 1978 : audit [DBG] from='client.? 172.21.15.99:0/2308774132' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:07.029 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:06 smithi099 bash[87147]: audit 2024-09-13T14:57:06.607980+0000 mon.a (mon.0) 1978 : audit [DBG] from='client.? 172.21.15.99:0/2308774132' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:07.085 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:07.098 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:06 smithi177 bash[68594]: audit 2024-09-13T14:57:06.607980+0000 mon.a (mon.0) 1978 : audit [DBG] from='client.? 172.21.15.99:0/2308774132' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:07.098 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:06 smithi177 bash[68594]: audit 2024-09-13T14:57:06.607980+0000 mon.a (mon.0) 1978 : audit [DBG] from='client.? 172.21.15.99:0/2308774132' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:07.118 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:07.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:07.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:07.233 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:57:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:57:07] "GET /metrics HTTP/1.1" 200 36761 "" "Prometheus/2.51.0" 2024-09-13T14:57:07.244 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:07.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:07.353 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:07.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:07.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:07.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:07.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:07.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:07.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:07.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:07.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:07.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:07.783 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:07.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:07.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:07.919 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:07.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:08.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:08.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:07 smithi177 bash[68594]: cluster 2024-09-13T14:57:06.698254+0000 mgr.y (mgr.26252) 927 : cluster [DBG] pgmap v1448: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2024-09-13T14:57:08.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:07 smithi177 bash[68594]: cluster 2024-09-13T14:57:06.698254+0000 mgr.y (mgr.26252) 927 : cluster [DBG] pgmap v1448: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2024-09-13T14:57:08.146 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:08.169 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:08.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:08.274 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.295 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:07 smithi099 bash[84576]: cluster 2024-09-13T14:57:06.698254+0000 mgr.y (mgr.26252) 927 : cluster [DBG] pgmap v1448: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2024-09-13T14:57:08.295 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:07 smithi099 bash[84576]: cluster 2024-09-13T14:57:06.698254+0000 mgr.y (mgr.26252) 927 : cluster [DBG] pgmap v1448: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2024-09-13T14:57:08.296 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:07 smithi099 bash[87147]: cluster 2024-09-13T14:57:06.698254+0000 mgr.y (mgr.26252) 927 : cluster [DBG] pgmap v1448: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2024-09-13T14:57:08.296 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:07 smithi099 bash[87147]: cluster 2024-09-13T14:57:06.698254+0000 mgr.y (mgr.26252) 927 : cluster [DBG] pgmap v1448: 89 pgs: 89 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2024-09-13T14:57:08.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:08.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.425 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:08.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:08.663 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:08.696 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:08.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:08.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:08.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:08.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:08.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:08.954 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:08.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:09.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:09.089 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:09.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:09.195 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:09.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:09.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:09.355 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:09.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:09.459 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:09.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:09.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:09.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:09.615 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:09.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:09.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:09.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:09.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:09.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:09 smithi177 bash[68594]: cluster 2024-09-13T14:57:08.698789+0000 mgr.y (mgr.26252) 928 : cluster [DBG] pgmap v1449: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2024-09-13T14:57:09.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:09 smithi177 bash[68594]: cluster 2024-09-13T14:57:08.698789+0000 mgr.y (mgr.26252) 928 : cluster [DBG] pgmap v1449: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2024-09-13T14:57:09.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:09.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:09.986 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:10.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:10.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:10.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:09 smithi099 bash[84576]: cluster 2024-09-13T14:57:08.698789+0000 mgr.y (mgr.26252) 928 : cluster [DBG] pgmap v1449: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2024-09-13T14:57:10.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:09 smithi099 bash[84576]: cluster 2024-09-13T14:57:08.698789+0000 mgr.y (mgr.26252) 928 : cluster [DBG] pgmap v1449: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2024-09-13T14:57:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:09 smithi099 bash[87147]: cluster 2024-09-13T14:57:08.698789+0000 mgr.y (mgr.26252) 928 : cluster [DBG] pgmap v1449: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2024-09-13T14:57:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:09 smithi099 bash[87147]: cluster 2024-09-13T14:57:08.698789+0000 mgr.y (mgr.26252) 928 : cluster [DBG] pgmap v1449: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2024-09-13T14:57:10.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:10.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:10.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:10.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:10.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:10.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:10.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:10.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:10.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:10.540 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:10.548 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:57:10 smithi177 bash[61439]: ts=2024-09-13T14:57:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:10.561 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:10.664 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:10.687 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:10.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:10.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:10.816 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:10.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:10.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:10.945 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:11.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:11.070 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:11.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:11.175 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:11.200 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:11.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:11.328 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:11.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:11.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:11.460 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:11.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:11.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:11.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:11.748 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:11.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:11.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:11.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:11 smithi177 bash[68594]: cluster 2024-09-13T14:57:10.699404+0000 mgr.y (mgr.26252) 929 : cluster [DBG] pgmap v1450: 89 pgs: 89 active+clean; 867 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-09-13T14:57:11.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:11 smithi177 bash[68594]: cluster 2024-09-13T14:57:10.699404+0000 mgr.y (mgr.26252) 929 : cluster [DBG] pgmap v1450: 89 pgs: 89 active+clean; 867 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-09-13T14:57:11.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:11.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:11.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:12.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:12.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:12.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:11 smithi099 bash[84576]: cluster 2024-09-13T14:57:10.699404+0000 mgr.y (mgr.26252) 929 : cluster [DBG] pgmap v1450: 89 pgs: 89 active+clean; 867 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-09-13T14:57:12.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:11 smithi099 bash[84576]: cluster 2024-09-13T14:57:10.699404+0000 mgr.y (mgr.26252) 929 : cluster [DBG] pgmap v1450: 89 pgs: 89 active+clean; 867 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-09-13T14:57:12.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:11 smithi099 bash[87147]: cluster 2024-09-13T14:57:10.699404+0000 mgr.y (mgr.26252) 929 : cluster [DBG] pgmap v1450: 89 pgs: 89 active+clean; 867 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-09-13T14:57:12.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:11 smithi099 bash[87147]: cluster 2024-09-13T14:57:10.699404+0000 mgr.y (mgr.26252) 929 : cluster [DBG] pgmap v1450: 89 pgs: 89 active+clean; 867 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-09-13T14:57:12.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:12.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.161 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:12.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:12.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:12.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:12.372 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T14:57:12.372 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:57:12.397 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:12.434 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:12.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:12.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:12.564 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:12.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:12.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:57:12.668 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:57:12.670 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:57:12.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:12.696 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:12.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:12.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:12.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:12.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:12.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:12.985 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:12.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:13.089 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:13.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:13.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:13.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:13.242 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:13.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:13.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:13.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:13.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:13.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:13.509 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:13.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:13.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:13.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:13.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:13.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:13.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:13.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:13.820 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T14:57:13.882 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[87147]: audit 2024-09-13T14:57:12.661423+0000 mgr.y (mgr.26252) 930 : audit [DBG] from='client.35595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:13.882 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[87147]: audit 2024-09-13T14:57:12.661423+0000 mgr.y (mgr.26252) 930 : audit [DBG] from='client.35595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:13.882 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[87147]: cluster 2024-09-13T14:57:12.699964+0000 mgr.y (mgr.26252) 931 : cluster [DBG] pgmap v1451: 89 pgs: 89 active+clean; 827 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2024-09-13T14:57:13.882 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[87147]: cluster 2024-09-13T14:57:12.699964+0000 mgr.y (mgr.26252) 931 : cluster [DBG] pgmap v1451: 89 pgs: 89 active+clean; 827 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2024-09-13T14:57:13.882 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[87147]: audit 2024-09-13T14:57:12.903511+0000 mon.a (mon.0) 1979 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:13.882 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[87147]: audit 2024-09-13T14:57:12.903511+0000 mon.a (mon.0) 1979 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:13.882 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[84576]: audit 2024-09-13T14:57:12.661423+0000 mgr.y (mgr.26252) 930 : audit [DBG] from='client.35595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:13.882 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[84576]: audit 2024-09-13T14:57:12.661423+0000 mgr.y (mgr.26252) 930 : audit [DBG] from='client.35595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:13.882 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[84576]: cluster 2024-09-13T14:57:12.699964+0000 mgr.y (mgr.26252) 931 : cluster [DBG] pgmap v1451: 89 pgs: 89 active+clean; 827 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2024-09-13T14:57:13.882 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[84576]: cluster 2024-09-13T14:57:12.699964+0000 mgr.y (mgr.26252) 931 : cluster [DBG] pgmap v1451: 89 pgs: 89 active+clean; 827 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2024-09-13T14:57:13.883 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[84576]: audit 2024-09-13T14:57:12.903511+0000 mon.a (mon.0) 1979 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:13.883 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:13 smithi099 bash[84576]: audit 2024-09-13T14:57:12.903511+0000 mon.a (mon.0) 1979 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:13.883 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:13.894 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:13 smithi177 bash[68594]: audit 2024-09-13T14:57:12.661423+0000 mgr.y (mgr.26252) 930 : audit [DBG] from='client.35595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:13.894 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:13 smithi177 bash[68594]: audit 2024-09-13T14:57:12.661423+0000 mgr.y (mgr.26252) 930 : audit [DBG] from='client.35595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:13.894 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:13 smithi177 bash[68594]: cluster 2024-09-13T14:57:12.699964+0000 mgr.y (mgr.26252) 931 : cluster [DBG] pgmap v1451: 89 pgs: 89 active+clean; 827 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2024-09-13T14:57:13.894 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:13 smithi177 bash[68594]: cluster 2024-09-13T14:57:12.699964+0000 mgr.y (mgr.26252) 931 : cluster [DBG] pgmap v1451: 89 pgs: 89 active+clean; 827 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2024-09-13T14:57:13.894 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:13 smithi177 bash[68594]: audit 2024-09-13T14:57:12.903511+0000 mon.a (mon.0) 1979 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:13.894 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:13 smithi177 bash[68594]: audit 2024-09-13T14:57:12.903511+0000 mon.a (mon.0) 1979 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:13.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:13.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:14.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:14.046 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:14.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:14.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:14.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:14.283 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:14.310 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:14.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:14.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:14.434 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:14.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:14.541 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:14.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:14.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:14.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:14.697 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:14.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:14.802 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:14.862 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:14.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:14.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:14.987 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[87147]: cluster 2024-09-13T14:57:13.809315+0000 mon.a (mon.0) 1980 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:14.987 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[87147]: cluster 2024-09-13T14:57:13.809315+0000 mon.a (mon.0) 1980 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:14.988 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[87147]: audit 2024-09-13T14:57:13.821556+0000 mon.a (mon.0) 1981 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:14.988 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[87147]: audit 2024-09-13T14:57:13.821556+0000 mon.a (mon.0) 1981 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:14.988 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[87147]: cluster 2024-09-13T14:57:13.828916+0000 mon.a (mon.0) 1982 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2024-09-13T14:57:14.988 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[87147]: cluster 2024-09-13T14:57:13.828916+0000 mon.a (mon.0) 1982 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2024-09-13T14:57:14.988 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[84576]: cluster 2024-09-13T14:57:13.809315+0000 mon.a (mon.0) 1980 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:14.988 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[84576]: cluster 2024-09-13T14:57:13.809315+0000 mon.a (mon.0) 1980 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:14.988 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[84576]: audit 2024-09-13T14:57:13.821556+0000 mon.a (mon.0) 1981 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:14.988 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[84576]: audit 2024-09-13T14:57:13.821556+0000 mon.a (mon.0) 1981 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:14.988 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[84576]: cluster 2024-09-13T14:57:13.828916+0000 mon.a (mon.0) 1982 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2024-09-13T14:57:14.989 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:14 smithi099 bash[84576]: cluster 2024-09-13T14:57:13.828916+0000 mon.a (mon.0) 1982 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2024-09-13T14:57:15.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:15.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:15.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:14 smithi177 bash[68594]: cluster 2024-09-13T14:57:13.809315+0000 mon.a (mon.0) 1980 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:15.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:14 smithi177 bash[68594]: cluster 2024-09-13T14:57:13.809315+0000 mon.a (mon.0) 1980 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:15.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:14 smithi177 bash[68594]: audit 2024-09-13T14:57:13.821556+0000 mon.a (mon.0) 1981 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:15.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:14 smithi177 bash[68594]: audit 2024-09-13T14:57:13.821556+0000 mon.a (mon.0) 1981 : audit [INF] from='client.? 172.21.15.99:0/461675674' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:15.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:14 smithi177 bash[68594]: cluster 2024-09-13T14:57:13.828916+0000 mon.a (mon.0) 1982 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2024-09-13T14:57:15.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:14 smithi177 bash[68594]: cluster 2024-09-13T14:57:13.828916+0000 mon.a (mon.0) 1982 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2024-09-13T14:57:15.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:15.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:15.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:15.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:15.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.270 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:15.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:15.371 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:15.400 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:15.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:15.503 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:15.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.539 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:15.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:15.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:15.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:15.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:15.760 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:15.794 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:15.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:15.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:15.928 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:15.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:16.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:16.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:15 smithi177 bash[68594]: cluster 2024-09-13T14:57:14.701403+0000 mgr.y (mgr.26252) 932 : cluster [DBG] pgmap v1453: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:16.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:15 smithi177 bash[68594]: cluster 2024-09-13T14:57:14.701403+0000 mgr.y (mgr.26252) 932 : cluster [DBG] pgmap v1453: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:16.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:16.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:16.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:16.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:15 smithi099 bash[84576]: cluster 2024-09-13T14:57:14.701403+0000 mgr.y (mgr.26252) 932 : cluster [DBG] pgmap v1453: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:16.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:15 smithi099 bash[84576]: cluster 2024-09-13T14:57:14.701403+0000 mgr.y (mgr.26252) 932 : cluster [DBG] pgmap v1453: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:16.176 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:15 smithi099 bash[87147]: cluster 2024-09-13T14:57:14.701403+0000 mgr.y (mgr.26252) 932 : cluster [DBG] pgmap v1453: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:16.176 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:15 smithi099 bash[87147]: cluster 2024-09-13T14:57:14.701403+0000 mgr.y (mgr.26252) 932 : cluster [DBG] pgmap v1453: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:16.193 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:16.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:16.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:16.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.343 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:16.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:16.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:16.473 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:16.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:16.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:57:16.577 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:16.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:16.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:16.686 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1450,"stamp":"2024-09-13T14:57:10.698966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":909449189,"num_objects":13895,"num_object_clones":0,"num_object_copies":41685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13895,"num_whiteouts":0,"num_read":345,"num_read_kb":800,"num_write":202019,"num_write_kb":6903228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":595442,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142171,"ondisk_log_size":142171,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5298688,"kb_used_data":2848064,"kb_used_omap":353,"kb_used_meta":2450014,"kb_avail":744662528,"statfs":{"total":767960285184,"available":762534428672,"internally_reserved":0,"allocated":2916417536,"data_stored":2884865017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":361898,"internal_metadata":2508814934},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,5,5,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-634257408,"num_objects":-9678,"num_object_clones":0,"num_object_copies":-29034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9678,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005445"},"pg_stats":[{"pgid":"42.f","version":"1030'12537","reported_seq":12566,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.360082+0000","last_change":"2024-09-13T14:53:02.737967+0000","last_active":"2024-09-13T14:57:09.360082+0000","last_peered":"2024-09-13T14:57:09.360082+0000","last_clean":"2024-09-13T14:57:09.360082+0000","last_became_active":"2024-09-13T14:53:02.737189+0000","last_became_peered":"2024-09-13T14:53:02.737189+0000","last_unstale":"2024-09-13T14:57:09.360082+0000","last_undegraded":"2024-09-13T14:57:09.360082+0000","last_fullsized":"2024-09-13T14:57:09.360082+0000","mapping_epoch":1009,"log_start":"1019'4600","ondisk_log_start":"1019'4600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7937,"log_dups_size":0,"ondisk_log_size":7937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:43.030910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55377920,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12537,"num_write_kb":428224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"42.e","version":"1030'12813","reported_seq":12867,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:08.974229+0000","last_change":"2024-09-13T14:53:03.091056+0000","last_active":"2024-09-13T14:57:08.974229+0000","last_peered":"2024-09-13T14:57:08.974229+0000","last_clean":"2024-09-13T14:57:08.974229+0000","last_became_active":"2024-09-13T14:53:03.090134+0000","last_became_peered":"2024-09-13T14:53:03.090134+0000","last_unstale":"2024-09-13T14:57:08.974229+0000","last_undegraded":"2024-09-13T14:57:08.974229+0000","last_fullsized":"2024-09-13T14:57:08.974229+0000","mapping_epoch":1009,"log_start":"1018'4200","ondisk_log_start":"1018'4200","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":8613,"log_dups_size":0,"ondisk_log_size":8613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:44.438976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55902208,"num_objects":853,"num_object_clones":0,"num_object_copies":2559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12813,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"42.d","version":"1030'12646","reported_seq":12676,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.227534+0000","last_change":"2024-09-13T14:53:02.727418+0000","last_active":"2024-09-13T14:57:09.227534+0000","last_peered":"2024-09-13T14:57:09.227534+0000","last_clean":"2024-09-13T14:57:09.227534+0000","last_became_active":"2024-09-13T14:53:02.726653+0000","last_became_peered":"2024-09-13T14:53:02.726653+0000","last_unstale":"2024-09-13T14:57:09.227534+0000","last_undegraded":"2024-09-13T14:57:09.227534+0000","last_fullsized":"2024-09-13T14:57:09.227534+0000","mapping_epoch":1009,"log_start":"1016'2900","ondisk_log_start":"1016'2900","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9746,"log_dups_size":0,"ondisk_log_size":9746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:30:10.620331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52625432,"num_objects":804,"num_object_clones":0,"num_object_copies":2412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":804,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12646,"num_write_kb":430337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"42.c","version":"1030'12776","reported_seq":12829,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.168829+0000","last_change":"2024-09-13T14:53:02.731439+0000","last_active":"2024-09-13T14:57:09.168829+0000","last_peered":"2024-09-13T14:57:09.168829+0000","last_clean":"2024-09-13T14:57:09.168829+0000","last_became_active":"2024-09-13T14:53:02.730603+0000","last_became_peered":"2024-09-13T14:53:02.730603+0000","last_unstale":"2024-09-13T14:57:09.168829+0000","last_undegraded":"2024-09-13T14:57:09.168829+0000","last_fullsized":"2024-09-13T14:57:09.168829+0000","mapping_epoch":1009,"log_start":"1019'4800","ondisk_log_start":"1019'4800","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7976,"log_dups_size":0,"ondisk_log_size":7976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:10.067364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55443456,"num_objects":846,"num_object_clones":0,"num_object_copies":2538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12776,"num_write_kb":435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"42.b","version":"1030'12337","reported_seq":12391,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:06.582007+0000","last_change":"2024-09-13T14:53:02.735868+0000","last_active":"2024-09-13T14:57:06.582007+0000","last_peered":"2024-09-13T14:57:06.582007+0000","last_clean":"2024-09-13T14:57:06.582007+0000","last_became_active":"2024-09-13T14:53:02.735016+0000","last_became_peered":"2024-09-13T14:53:02.735016+0000","last_unstale":"2024-09-13T14:57:06.582007+0000","last_undegraded":"2024-09-13T14:57:06.582007+0000","last_fullsized":"2024-09-13T14:57:06.582007+0000","mapping_epoch":1009,"log_start":"1019'4600","ondisk_log_start":"1019'4600","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7737,"log_dups_size":0,"ondisk_log_size":7737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:44:45.114443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61538304,"num_objects":939,"num_object_clones":0,"num_object_copies":2817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12337,"num_write_kb":424832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"42.a","version":"1030'12506","reported_seq":12535,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.631119+0000","last_change":"2024-09-13T14:53:02.730034+0000","last_active":"2024-09-13T14:57:09.631119+0000","last_peered":"2024-09-13T14:57:09.631119+0000","last_clean":"2024-09-13T14:57:09.631119+0000","last_became_active":"2024-09-13T14:53:02.729241+0000","last_became_peered":"2024-09-13T14:53:02.729241+0000","last_unstale":"2024-09-13T14:57:09.631119+0000","last_undegraded":"2024-09-13T14:57:09.631119+0000","last_fullsized":"2024-09-13T14:57:09.631119+0000","mapping_epoch":1009,"log_start":"1016'2800","ondisk_log_start":"1016'2800","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9706,"log_dups_size":0,"ondisk_log_size":9706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:56:44.133299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56098816,"num_objects":856,"num_object_clones":0,"num_object_copies":2568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12506,"num_write_kb":427584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"42.9","version":"1030'12571","reported_seq":12600,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.182007+0000","last_change":"2024-09-13T14:53:02.737619+0000","last_active":"2024-09-13T14:57:09.182007+0000","last_peered":"2024-09-13T14:57:09.182007+0000","last_clean":"2024-09-13T14:57:09.182007+0000","last_became_active":"2024-09-13T14:53:02.736782+0000","last_became_peered":"2024-09-13T14:53:02.736782+0000","last_unstale":"2024-09-13T14:57:09.182007+0000","last_undegraded":"2024-09-13T14:57:09.182007+0000","last_fullsized":"2024-09-13T14:57:09.182007+0000","mapping_epoch":1009,"log_start":"1016'2800","ondisk_log_start":"1016'2800","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9771,"log_dups_size":0,"ondisk_log_size":9771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:30:17.070735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12571,"num_write_kb":428864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"42.8","version":"1030'12453","reported_seq":12481,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.356373+0000","last_change":"2024-09-13T14:53:03.087618+0000","last_active":"2024-09-13T14:57:09.356373+0000","last_peered":"2024-09-13T14:57:09.356373+0000","last_clean":"2024-09-13T14:57:09.356373+0000","last_became_active":"2024-09-13T14:53:03.086906+0000","last_became_peered":"2024-09-13T14:53:03.086906+0000","last_unstale":"2024-09-13T14:57:09.356373+0000","last_undegraded":"2024-09-13T14:57:09.356373+0000","last_fullsized":"2024-09-13T14:57:09.356373+0000","mapping_epoch":1009,"log_start":"1019'4500","ondisk_log_start":"1019'4500","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7953,"log_dups_size":0,"ondisk_log_size":7953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:05.901169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54853632,"num_objects":837,"num_object_clones":0,"num_object_copies":2511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12453,"num_write_kb":425280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"42.7","version":"1030'12815","reported_seq":12844,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.376101+0000","last_change":"2024-09-13T14:53:02.732343+0000","last_active":"2024-09-13T14:57:09.376101+0000","last_peered":"2024-09-13T14:57:09.376101+0000","last_clean":"2024-09-13T14:57:09.376101+0000","last_became_active":"2024-09-13T14:53:02.731717+0000","last_became_peered":"2024-09-13T14:53:02.731717+0000","last_unstale":"2024-09-13T14:57:09.376101+0000","last_undegraded":"2024-09-13T14:57:09.376101+0000","last_fullsized":"2024-09-13T14:57:09.376101+0000","mapping_epoch":1009,"log_start":"1019'4900","ondisk_log_start":"1019'4900","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7915,"log_dups_size":0,"ondisk_log_size":7915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:37.034264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55902208,"num_objects":853,"num_object_clones":0,"num_object_copies":2559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12815,"num_write_kb":437376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"42.6","version":"1030'12397","reported_seq":12425,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.628168+0000","last_change":"2024-09-13T14:53:02.734127+0000","last_active":"2024-09-13T14:57:09.628168+0000","last_peered":"2024-09-13T14:57:09.628168+0000","last_clean":"2024-09-13T14:57:09.628168+0000","last_became_active":"2024-09-13T14:53:02.733028+0000","last_became_peered":"2024-09-13T14:53:02.733028+0000","last_unstale":"2024-09-13T14:57:09.628168+0000","last_undegraded":"2024-09-13T14:57:09.628168+0000","last_fullsized":"2024-09-13T14:57:09.628168+0000","mapping_epoch":1009,"log_start":"1016'2700","ondisk_log_start":"1016'2700","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9697,"log_dups_size":0,"ondisk_log_size":9697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:46.557305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12397,"num_write_kb":423296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"42.5","version":"1030'12713","reported_seq":12742,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.610495+0000","last_change":"2024-09-13T14:53:02.730556+0000","last_active":"2024-09-13T14:57:09.610495+0000","last_peered":"2024-09-13T14:57:09.610495+0000","last_clean":"2024-09-13T14:57:09.610495+0000","last_became_active":"2024-09-13T14:53:02.729563+0000","last_became_peered":"2024-09-13T14:53:02.729563+0000","last_unstale":"2024-09-13T14:57:09.610495+0000","last_undegraded":"2024-09-13T14:57:09.610495+0000","last_fullsized":"2024-09-13T14:57:09.610495+0000","mapping_epoch":1009,"log_start":"1016'3000","ondisk_log_start":"1016'3000","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9713,"log_dups_size":0,"ondisk_log_size":9713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:28.870740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53936128,"num_objects":823,"num_object_clones":0,"num_object_copies":2469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12713,"num_write_kb":433152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"42.4","version":"1030'12430","reported_seq":12484,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:06.601457+0000","last_change":"2024-09-13T14:53:02.737098+0000","last_active":"2024-09-13T14:57:06.601457+0000","last_peered":"2024-09-13T14:57:06.601457+0000","last_clean":"2024-09-13T14:57:06.601457+0000","last_became_active":"2024-09-13T14:53:02.736160+0000","last_became_peered":"2024-09-13T14:53:02.736160+0000","last_unstale":"2024-09-13T14:57:06.601457+0000","last_undegraded":"2024-09-13T14:57:06.601457+0000","last_fullsized":"2024-09-13T14:57:06.601457+0000","mapping_epoch":1009,"log_start":"1019'4700","ondisk_log_start":"1019'4700","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7730,"log_dups_size":0,"ondisk_log_size":7730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:48.130549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64487424,"num_objects":984,"num_object_clones":0,"num_object_copies":2952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12430,"num_write_kb":429248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"42.3","version":"1030'12599","reported_seq":12628,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.108707+0000","last_change":"2024-09-13T14:53:03.088642+0000","last_active":"2024-09-13T14:57:09.108707+0000","last_peered":"2024-09-13T14:57:09.108707+0000","last_clean":"2024-09-13T14:57:09.108707+0000","last_became_active":"2024-09-13T14:53:03.087596+0000","last_became_peered":"2024-09-13T14:53:03.087596+0000","last_unstale":"2024-09-13T14:57:09.108707+0000","last_undegraded":"2024-09-13T14:57:09.108707+0000","last_fullsized":"2024-09-13T14:57:09.108707+0000","mapping_epoch":1009,"log_start":"1016'2900","ondisk_log_start":"1016'2900","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9699,"log_dups_size":0,"ondisk_log_size":9699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:30:53.067804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56819712,"num_objects":867,"num_object_clones":0,"num_object_copies":2601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12599,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"42.2","version":"1030'12486","reported_seq":12539,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:07.302214+0000","last_change":"2024-09-13T14:53:03.090582+0000","last_active":"2024-09-13T14:57:07.302214+0000","last_peered":"2024-09-13T14:57:07.302214+0000","last_clean":"2024-09-13T14:57:07.302214+0000","last_became_active":"2024-09-13T14:53:03.089683+0000","last_became_peered":"2024-09-13T14:53:03.089683+0000","last_unstale":"2024-09-13T14:57:07.302214+0000","last_undegraded":"2024-09-13T14:57:07.302214+0000","last_fullsized":"2024-09-13T14:57:07.302214+0000","mapping_epoch":1009,"log_start":"1015'2400","ondisk_log_start":"1015'2400","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":10086,"log_dups_size":0,"ondisk_log_size":10086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:13:05.244177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61210624,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12486,"num_write_kb":429440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"42.1","version":"1030'12861","reported_seq":12915,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.107066+0000","last_change":"2024-09-13T14:53:02.733122+0000","last_active":"2024-09-13T14:57:09.107066+0000","last_peered":"2024-09-13T14:57:09.107066+0000","last_clean":"2024-09-13T14:57:09.107066+0000","last_became_active":"2024-09-13T14:53:02.732272+0000","last_became_peered":"2024-09-13T14:53:02.732272+0000","last_unstale":"2024-09-13T14:57:09.107066+0000","last_undegraded":"2024-09-13T14:57:09.107066+0000","last_fullsized":"2024-09-13T14:57:09.107066+0000","mapping_epoch":1009,"log_start":"1019'4900","ondisk_log_start":"1019'4900","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":7961,"log_dups_size":0,"ondisk_log_size":7961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:48.079694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57344000,"num_objects":875,"num_object_clones":0,"num_object_copies":2625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12861,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"42.0","version":"1030'12867","reported_seq":12895,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:57:09.645170+0000","last_change":"2024-09-13T14:53:02.729978+0000","last_active":"2024-09-13T14:57:09.645170+0000","last_peered":"2024-09-13T14:57:09.645170+0000","last_clean":"2024-09-13T14:57:09.645170+0000","last_became_active":"2024-09-13T14:53:02.728942+0000","last_became_peered":"2024-09-13T14:53:02.728942+0000","last_unstale":"2024-09-13T14:57:09.645170+0000","last_undegraded":"2024-09-13T14:57:09.645170+0000","last_fullsized":"2024-09-13T14:57:09.645170+0000","mapping_epoch":1009,"log_start":"1016'3100","ondisk_log_start":"1016'3100","created":1009,"last_epoch_clean":1010,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:53:01.698029+0000","last_clean_scrub_stamp":"2024-09-13T14:53:01.698029+0000","objects_scrubbed":0,"log_size":9767,"log_dups_size":0,"ondisk_log_size":9767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:34:11.356137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58392576,"num_objects":891,"num_object_clones":0,"num_object_copies":2673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12867,"num_write_kb":440256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":1452,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.774618+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T14:56:43.774618+0000","last_peered":"2024-09-13T14:56:43.774618+0000","last_clean":"2024-09-13T14:56:43.774618+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T14:56:43.774618+0000","last_undegraded":"2024-09-13T14:56:43.774618+0000","last_fullsized":"2024-09-13T14:56:43.774618+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":171,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.752081+0000","last_change":"2024-09-13T14:52:43.599071+0000","last_active":"2024-09-13T14:56:43.752081+0000","last_peered":"2024-09-13T14:56:43.752081+0000","last_clean":"2024-09-13T14:56:43.752081+0000","last_became_active":"2024-09-13T14:52:43.594847+0000","last_became_peered":"2024-09-13T14:52:43.594847+0000","last_unstale":"2024-09-13T14:56:43.752081+0000","last_undegraded":"2024-09-13T14:56:43.752081+0000","last_fullsized":"2024-09-13T14:56:43.752081+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:02.736434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1302,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.786049+0000","last_change":"2024-09-13T14:39:34.403170+0000","last_active":"2024-09-13T14:56:43.786049+0000","last_peered":"2024-09-13T14:56:43.786049+0000","last_clean":"2024-09-13T14:56:43.786049+0000","last_became_active":"2024-09-13T14:39:34.389001+0000","last_became_peered":"2024-09-13T14:39:34.389001+0000","last_unstale":"2024-09-13T14:56:43.786049+0000","last_undegraded":"2024-09-13T14:56:43.786049+0000","last_fullsized":"2024-09-13T14:56:43.786049+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:38.542452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":61,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.880206+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T14:56:08.880206+0000","last_peered":"2024-09-13T14:56:08.880206+0000","last_clean":"2024-09-13T14:56:08.880206+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T14:56:08.880206+0000","last_undegraded":"2024-09-13T14:56:08.880206+0000","last_fullsized":"2024-09-13T14:56:08.880206+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1015,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891416+0000","last_change":"2024-09-13T14:47:46.337827+0000","last_active":"2024-09-13T14:56:08.891416+0000","last_peered":"2024-09-13T14:56:08.891416+0000","last_clean":"2024-09-13T14:56:08.891416+0000","last_became_active":"2024-09-13T14:47:46.321140+0000","last_became_peered":"2024-09-13T14:47:46.321140+0000","last_unstale":"2024-09-13T14:56:08.891416+0000","last_undegraded":"2024-09-13T14:56:08.891416+0000","last_fullsized":"2024-09-13T14:56:08.891416+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:15.997382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":157,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.753172+0000","last_change":"2024-09-13T14:47:46.333043+0000","last_active":"2024-09-13T14:56:43.753172+0000","last_peered":"2024-09-13T14:56:43.753172+0000","last_clean":"2024-09-13T14:56:43.753172+0000","last_became_active":"2024-09-13T14:47:46.332292+0000","last_became_peered":"2024-09-13T14:47:46.332292+0000","last_unstale":"2024-09-13T14:56:43.753172+0000","last_undegraded":"2024-09-13T14:56:43.753172+0000","last_fullsized":"2024-09-13T14:56:43.753172+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:38.942334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1060,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.881236+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T14:56:08.881236+0000","last_peered":"2024-09-13T14:56:08.881236+0000","last_clean":"2024-09-13T14:56:08.881236+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T14:56:08.881236+0000","last_undegraded":"2024-09-13T14:56:08.881236+0000","last_fullsized":"2024-09-13T14:56:08.881236+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":171,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.755078+0000","last_change":"2024-09-13T14:52:43.601798+0000","last_active":"2024-09-13T14:56:43.755078+0000","last_peered":"2024-09-13T14:56:43.755078+0000","last_clean":"2024-09-13T14:56:43.755078+0000","last_became_active":"2024-09-13T14:52:43.600648+0000","last_became_peered":"2024-09-13T14:52:43.600648+0000","last_unstale":"2024-09-13T14:56:43.755078+0000","last_undegraded":"2024-09-13T14:56:43.755078+0000","last_fullsized":"2024-09-13T14:56:43.755078+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:01.021067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1459,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.774979+0000","last_change":"2024-09-13T14:47:46.330059+0000","last_active":"2024-09-13T14:56:43.774979+0000","last_peered":"2024-09-13T14:56:43.774979+0000","last_clean":"2024-09-13T14:56:43.774979+0000","last_became_active":"2024-09-13T14:47:46.329344+0000","last_became_peered":"2024-09-13T14:47:46.329344+0000","last_unstale":"2024-09-13T14:56:43.774979+0000","last_undegraded":"2024-09-13T14:56:43.774979+0000","last_fullsized":"2024-09-13T14:56:43.774979+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:58:43.470176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1363,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.761206+0000","last_change":"2024-09-13T14:52:43.594737+0000","last_active":"2024-09-13T14:56:43.761206+0000","last_peered":"2024-09-13T14:56:43.761206+0000","last_clean":"2024-09-13T14:56:43.761206+0000","last_became_active":"2024-09-13T14:52:43.594197+0000","last_became_peered":"2024-09-13T14:52:43.594197+0000","last_unstale":"2024-09-13T14:56:43.761206+0000","last_undegraded":"2024-09-13T14:56:43.761206+0000","last_fullsized":"2024-09-13T14:56:43.761206+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:31.054754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":171,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.752947+0000","last_change":"2024-09-13T14:52:43.597400+0000","last_active":"2024-09-13T14:56:43.752947+0000","last_peered":"2024-09-13T14:56:43.752947+0000","last_clean":"2024-09-13T14:56:43.752947+0000","last_became_active":"2024-09-13T14:52:43.596294+0000","last_became_peered":"2024-09-13T14:52:43.596294+0000","last_unstale":"2024-09-13T14:56:43.752947+0000","last_undegraded":"2024-09-13T14:56:43.752947+0000","last_fullsized":"2024-09-13T14:56:43.752947+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:35.218171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1349,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.785856+0000","last_change":"2024-09-13T14:52:43.590238+0000","last_active":"2024-09-13T14:56:43.785856+0000","last_peered":"2024-09-13T14:56:43.785856+0000","last_clean":"2024-09-13T14:56:43.785856+0000","last_became_active":"2024-09-13T14:52:43.589713+0000","last_became_peered":"2024-09-13T14:52:43.589713+0000","last_unstale":"2024-09-13T14:56:43.785856+0000","last_undegraded":"2024-09-13T14:56:43.785856+0000","last_fullsized":"2024-09-13T14:56:43.785856+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:19.148195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":61,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.770407+0000","last_change":"2024-09-13T14:52:43.591911+0000","last_active":"2024-09-13T14:56:43.770407+0000","last_peered":"2024-09-13T14:56:43.770407+0000","last_clean":"2024-09-13T14:56:43.770407+0000","last_became_active":"2024-09-13T14:52:43.591410+0000","last_became_peered":"2024-09-13T14:52:43.591410+0000","last_unstale":"2024-09-13T14:56:43.770407+0000","last_undegraded":"2024-09-13T14:56:43.770407+0000","last_fullsized":"2024-09-13T14:56:43.770407+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:53.664208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1448,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.780192+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T14:56:43.780192+0000","last_peered":"2024-09-13T14:56:43.780192+0000","last_clean":"2024-09-13T14:56:43.780192+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T14:56:43.780192+0000","last_undegraded":"2024-09-13T14:56:43.780192+0000","last_fullsized":"2024-09-13T14:56:43.780192+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1342,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.788031+0000","last_change":"2024-09-13T14:52:43.587987+0000","last_active":"2024-09-13T14:56:43.788031+0000","last_peered":"2024-09-13T14:56:43.788031+0000","last_clean":"2024-09-13T14:56:43.788031+0000","last_became_active":"2024-09-13T14:52:43.587749+0000","last_became_peered":"2024-09-13T14:52:43.587749+0000","last_unstale":"2024-09-13T14:56:43.788031+0000","last_undegraded":"2024-09-13T14:56:43.788031+0000","last_fullsized":"2024-09-13T14:56:43.788031+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:31.924493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1017,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885938+0000","last_change":"2024-09-13T14:52:54.183175+0000","last_active":"2024-09-13T14:56:08.885938+0000","last_peered":"2024-09-13T14:56:08.885938+0000","last_clean":"2024-09-13T14:56:08.885938+0000","last_became_active":"2024-09-13T14:52:54.182514+0000","last_became_peered":"2024-09-13T14:52:54.182514+0000","last_unstale":"2024-09-13T14:56:08.885938+0000","last_undegraded":"2024-09-13T14:56:08.885938+0000","last_fullsized":"2024-09-13T14:56:08.885938+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:55.900845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1322,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.788163+0000","last_change":"2024-09-13T14:52:43.587069+0000","last_active":"2024-09-13T14:56:43.788163+0000","last_peered":"2024-09-13T14:56:43.788163+0000","last_clean":"2024-09-13T14:56:43.788163+0000","last_became_active":"2024-09-13T14:52:43.586196+0000","last_became_peered":"2024-09-13T14:52:43.586196+0000","last_unstale":"2024-09-13T14:56:43.788163+0000","last_undegraded":"2024-09-13T14:56:43.788163+0000","last_fullsized":"2024-09-13T14:56:43.788163+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:19:50.215660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1439,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.774665+0000","last_change":"2024-09-13T14:39:34.318266+0000","last_active":"2024-09-13T14:56:43.774665+0000","last_peered":"2024-09-13T14:56:43.774665+0000","last_clean":"2024-09-13T14:56:43.774665+0000","last_became_active":"2024-09-13T14:39:34.315973+0000","last_became_peered":"2024-09-13T14:39:34.315973+0000","last_unstale":"2024-09-13T14:56:43.774665+0000","last_undegraded":"2024-09-13T14:56:43.774665+0000","last_fullsized":"2024-09-13T14:56:43.774665+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:03:15.270282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1287,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.779299+0000","last_change":"2024-09-13T14:39:34.338378+0000","last_active":"2024-09-13T14:56:43.779299+0000","last_peered":"2024-09-13T14:56:43.779299+0000","last_clean":"2024-09-13T14:56:43.779299+0000","last_became_active":"2024-09-13T14:39:34.338005+0000","last_became_peered":"2024-09-13T14:39:34.338005+0000","last_unstale":"2024-09-13T14:56:43.779299+0000","last_undegraded":"2024-09-13T14:56:43.779299+0000","last_fullsized":"2024-09-13T14:56:43.779299+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:29.120379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1004,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.880817+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T14:56:08.880817+0000","last_peered":"2024-09-13T14:56:08.880817+0000","last_clean":"2024-09-13T14:56:08.880817+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T14:56:08.880817+0000","last_undegraded":"2024-09-13T14:56:08.880817+0000","last_fullsized":"2024-09-13T14:56:08.880817+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1041,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884490+0000","last_change":"2024-09-13T14:47:46.354886+0000","last_active":"2024-09-13T14:56:08.884490+0000","last_peered":"2024-09-13T14:56:08.884490+0000","last_clean":"2024-09-13T14:56:08.884490+0000","last_became_active":"2024-09-13T14:47:46.329947+0000","last_became_peered":"2024-09-13T14:47:46.329947+0000","last_unstale":"2024-09-13T14:56:08.884490+0000","last_undegraded":"2024-09-13T14:56:08.884490+0000","last_fullsized":"2024-09-13T14:56:08.884490+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:22.443920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":61,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.767834+0000","last_change":"2024-09-13T14:52:43.586530+0000","last_active":"2024-09-13T14:56:43.767834+0000","last_peered":"2024-09-13T14:56:43.767834+0000","last_clean":"2024-09-13T14:56:43.767834+0000","last_became_active":"2024-09-13T14:52:43.586107+0000","last_became_peered":"2024-09-13T14:52:43.586107+0000","last_unstale":"2024-09-13T14:56:43.767834+0000","last_undegraded":"2024-09-13T14:56:43.767834+0000","last_fullsized":"2024-09-13T14:56:43.767834+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:18.130768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":167,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.755404+0000","last_change":"2024-09-13T14:47:46.344536+0000","last_active":"2024-09-13T14:56:43.755404+0000","last_peered":"2024-09-13T14:56:43.755404+0000","last_clean":"2024-09-13T14:56:43.755404+0000","last_became_active":"2024-09-13T14:47:46.321131+0000","last_became_peered":"2024-09-13T14:47:46.321131+0000","last_unstale":"2024-09-13T14:56:43.755404+0000","last_undegraded":"2024-09-13T14:56:43.755404+0000","last_fullsized":"2024-09-13T14:56:43.755404+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:31.114472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":172,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.755103+0000","last_change":"2024-09-13T14:52:43.600569+0000","last_active":"2024-09-13T14:56:43.755103+0000","last_peered":"2024-09-13T14:56:43.755103+0000","last_clean":"2024-09-13T14:56:43.755103+0000","last_became_active":"2024-09-13T14:52:43.599764+0000","last_became_peered":"2024-09-13T14:52:43.599764+0000","last_unstale":"2024-09-13T14:56:43.755103+0000","last_undegraded":"2024-09-13T14:56:43.755103+0000","last_fullsized":"2024-09-13T14:56:43.755103+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:53.934456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":158,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.752277+0000","last_change":"2024-09-13T14:47:46.318785+0000","last_active":"2024-09-13T14:56:43.752277+0000","last_peered":"2024-09-13T14:56:43.752277+0000","last_clean":"2024-09-13T14:56:43.752277+0000","last_became_active":"2024-09-13T14:47:46.318438+0000","last_became_peered":"2024-09-13T14:47:46.318438+0000","last_unstale":"2024-09-13T14:56:43.752277+0000","last_undegraded":"2024-09-13T14:56:43.752277+0000","last_fullsized":"2024-09-13T14:56:43.752277+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:44.745378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1000,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.886103+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T14:56:08.886103+0000","last_peered":"2024-09-13T14:56:08.886103+0000","last_clean":"2024-09-13T14:56:08.886103+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T14:56:08.886103+0000","last_undegraded":"2024-09-13T14:56:08.886103+0000","last_fullsized":"2024-09-13T14:56:08.886103+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1500,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.781617+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T14:56:43.781617+0000","last_peered":"2024-09-13T14:56:43.781617+0000","last_clean":"2024-09-13T14:56:43.781617+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T14:56:43.781617+0000","last_undegraded":"2024-09-13T14:56:43.781617+0000","last_fullsized":"2024-09-13T14:56:43.781617+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1285,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.788286+0000","last_change":"2024-09-13T14:39:34.349945+0000","last_active":"2024-09-13T14:56:43.788286+0000","last_peered":"2024-09-13T14:56:43.788286+0000","last_clean":"2024-09-13T14:56:43.788286+0000","last_became_active":"2024-09-13T14:39:34.346730+0000","last_became_peered":"2024-09-13T14:39:34.346730+0000","last_unstale":"2024-09-13T14:56:43.788286+0000","last_undegraded":"2024-09-13T14:56:43.788286+0000","last_fullsized":"2024-09-13T14:56:43.788286+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:48:48.854603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":62,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.767875+0000","last_change":"2024-09-13T14:52:43.592967+0000","last_active":"2024-09-13T14:56:43.767875+0000","last_peered":"2024-09-13T14:56:43.767875+0000","last_clean":"2024-09-13T14:56:43.767875+0000","last_became_active":"2024-09-13T14:52:43.592431+0000","last_became_peered":"2024-09-13T14:52:43.592431+0000","last_unstale":"2024-09-13T14:56:43.767875+0000","last_undegraded":"2024-09-13T14:56:43.767875+0000","last_fullsized":"2024-09-13T14:56:43.767875+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:56.187568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1289,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.786365+0000","last_change":"2024-09-13T14:39:34.401432+0000","last_active":"2024-09-13T14:56:43.786365+0000","last_peered":"2024-09-13T14:56:43.786365+0000","last_clean":"2024-09-13T14:56:43.786365+0000","last_became_active":"2024-09-13T14:39:34.395458+0000","last_became_peered":"2024-09-13T14:39:34.395458+0000","last_unstale":"2024-09-13T14:56:43.786365+0000","last_undegraded":"2024-09-13T14:56:43.786365+0000","last_fullsized":"2024-09-13T14:56:43.786365+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:40:17.007388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1430,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.774822+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T14:56:43.774822+0000","last_peered":"2024-09-13T14:56:43.774822+0000","last_clean":"2024-09-13T14:56:43.774822+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T14:56:43.774822+0000","last_undegraded":"2024-09-13T14:56:43.774822+0000","last_fullsized":"2024-09-13T14:56:43.774822+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1036,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884713+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T14:56:08.884713+0000","last_peered":"2024-09-13T14:56:08.884713+0000","last_clean":"2024-09-13T14:56:08.884713+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T14:56:08.884713+0000","last_undegraded":"2024-09-13T14:56:08.884713+0000","last_fullsized":"2024-09-13T14:56:08.884713+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":62,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.779113+0000","last_change":"2024-09-13T14:52:43.583860+0000","last_active":"2024-09-13T14:56:43.779113+0000","last_peered":"2024-09-13T14:56:43.779113+0000","last_clean":"2024-09-13T14:56:43.779113+0000","last_became_active":"2024-09-13T14:52:43.583055+0000","last_became_peered":"2024-09-13T14:52:43.583055+0000","last_unstale":"2024-09-13T14:56:43.779113+0000","last_undegraded":"2024-09-13T14:56:43.779113+0000","last_fullsized":"2024-09-13T14:56:43.779113+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:55.113648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":62,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.767788+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T14:56:43.767788+0000","last_peered":"2024-09-13T14:56:43.767788+0000","last_clean":"2024-09-13T14:56:43.767788+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T14:56:43.767788+0000","last_undegraded":"2024-09-13T14:56:43.767788+0000","last_fullsized":"2024-09-13T14:56:43.767788+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":62,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.779277+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T14:56:43.779277+0000","last_peered":"2024-09-13T14:56:43.779277+0000","last_clean":"2024-09-13T14:56:43.779277+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T14:56:43.779277+0000","last_undegraded":"2024-09-13T14:56:43.779277+0000","last_fullsized":"2024-09-13T14:56:43.779277+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1001,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891483+0000","last_change":"2024-09-13T14:39:34.321388+0000","last_active":"2024-09-13T14:56:08.891483+0000","last_peered":"2024-09-13T14:56:08.891483+0000","last_clean":"2024-09-13T14:56:08.891483+0000","last_became_active":"2024-09-13T14:39:34.321025+0000","last_became_peered":"2024-09-13T14:39:34.321025+0000","last_unstale":"2024-09-13T14:56:08.891483+0000","last_undegraded":"2024-09-13T14:56:08.891483+0000","last_fullsized":"2024-09-13T14:56:08.891483+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:52.593825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.779196+0000","last_change":"2024-09-13T14:52:43.583382+0000","last_active":"2024-09-13T14:56:43.779196+0000","last_peered":"2024-09-13T14:56:43.779196+0000","last_clean":"2024-09-13T14:56:43.779196+0000","last_became_active":"2024-09-13T14:52:43.582870+0000","last_became_peered":"2024-09-13T14:52:43.582870+0000","last_unstale":"2024-09-13T14:56:43.779196+0000","last_undegraded":"2024-09-13T14:56:43.779196+0000","last_fullsized":"2024-09-13T14:56:43.779196+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:51.582211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":158,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.755153+0000","last_change":"2024-09-13T14:47:46.324072+0000","last_active":"2024-09-13T14:56:43.755153+0000","last_peered":"2024-09-13T14:56:43.755153+0000","last_clean":"2024-09-13T14:56:43.755153+0000","last_became_active":"2024-09-13T14:47:46.322613+0000","last_became_peered":"2024-09-13T14:47:46.322613+0000","last_unstale":"2024-09-13T14:56:43.755153+0000","last_undegraded":"2024-09-13T14:56:43.755153+0000","last_fullsized":"2024-09-13T14:56:43.755153+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:45.314706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1026,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.886164+0000","last_change":"2024-09-13T14:52:43.613106+0000","last_active":"2024-09-13T14:56:08.886164+0000","last_peered":"2024-09-13T14:56:08.886164+0000","last_clean":"2024-09-13T14:56:08.886164+0000","last_became_active":"2024-09-13T14:52:43.594081+0000","last_became_peered":"2024-09-13T14:52:43.594081+0000","last_unstale":"2024-09-13T14:56:08.886164+0000","last_undegraded":"2024-09-13T14:56:08.886164+0000","last_fullsized":"2024-09-13T14:56:08.886164+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:11.121629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":62,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.769694+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T14:56:43.769694+0000","last_peered":"2024-09-13T14:56:43.769694+0000","last_clean":"2024-09-13T14:56:43.769694+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T14:56:43.769694+0000","last_undegraded":"2024-09-13T14:56:43.769694+0000","last_fullsized":"2024-09-13T14:56:43.769694+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":62,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.766808+0000","last_change":"2024-09-13T14:52:43.582732+0000","last_active":"2024-09-13T14:56:43.766808+0000","last_peered":"2024-09-13T14:56:43.766808+0000","last_clean":"2024-09-13T14:56:43.766808+0000","last_became_active":"2024-09-13T14:52:43.582384+0000","last_became_peered":"2024-09-13T14:52:43.582384+0000","last_unstale":"2024-09-13T14:56:43.766808+0000","last_undegraded":"2024-09-13T14:56:43.766808+0000","last_fullsized":"2024-09-13T14:56:43.766808+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:06.715139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1338,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.759490+0000","last_change":"2024-09-13T14:39:34.322533+0000","last_active":"2024-09-13T14:56:43.759490+0000","last_peered":"2024-09-13T14:56:43.759490+0000","last_clean":"2024-09-13T14:56:43.759490+0000","last_became_active":"2024-09-13T14:39:34.322070+0000","last_became_peered":"2024-09-13T14:39:34.322070+0000","last_unstale":"2024-09-13T14:56:43.759490+0000","last_undegraded":"2024-09-13T14:56:43.759490+0000","last_fullsized":"2024-09-13T14:56:43.759490+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:43.746420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1465,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.780093+0000","last_change":"2024-09-13T14:52:43.595169+0000","last_active":"2024-09-13T14:56:43.780093+0000","last_peered":"2024-09-13T14:56:43.780093+0000","last_clean":"2024-09-13T14:56:43.780093+0000","last_became_active":"2024-09-13T14:52:43.594586+0000","last_became_peered":"2024-09-13T14:52:43.594586+0000","last_unstale":"2024-09-13T14:56:43.780093+0000","last_undegraded":"2024-09-13T14:56:43.780093+0000","last_fullsized":"2024-09-13T14:56:43.780093+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:53.946025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1466,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.774018+0000","last_change":"2024-09-13T14:47:46.329033+0000","last_active":"2024-09-13T14:56:43.774018+0000","last_peered":"2024-09-13T14:56:43.774018+0000","last_clean":"2024-09-13T14:56:43.774018+0000","last_became_active":"2024-09-13T14:47:46.328203+0000","last_became_peered":"2024-09-13T14:47:46.328203+0000","last_unstale":"2024-09-13T14:56:43.774018+0000","last_undegraded":"2024-09-13T14:56:43.774018+0000","last_fullsized":"2024-09-13T14:56:43.774018+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:23.440178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1036,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.895111+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T14:56:08.895111+0000","last_peered":"2024-09-13T14:56:08.895111+0000","last_clean":"2024-09-13T14:56:08.895111+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T14:56:08.895111+0000","last_undegraded":"2024-09-13T14:56:08.895111+0000","last_fullsized":"2024-09-13T14:56:08.895111+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1007,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.886167+0000","last_change":"2024-09-13T14:47:46.320830+0000","last_active":"2024-09-13T14:56:08.886167+0000","last_peered":"2024-09-13T14:56:08.886167+0000","last_clean":"2024-09-13T14:56:08.886167+0000","last_became_active":"2024-09-13T14:47:46.320443+0000","last_became_peered":"2024-09-13T14:47:46.320443+0000","last_unstale":"2024-09-13T14:56:08.886167+0000","last_undegraded":"2024-09-13T14:56:08.886167+0000","last_fullsized":"2024-09-13T14:56:08.886167+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:46.279580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1287,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.779583+0000","last_change":"2024-09-13T14:39:34.356129+0000","last_active":"2024-09-13T14:56:43.779583+0000","last_peered":"2024-09-13T14:56:43.779583+0000","last_clean":"2024-09-13T14:56:43.779583+0000","last_became_active":"2024-09-13T14:39:34.355288+0000","last_became_peered":"2024-09-13T14:39:34.355288+0000","last_unstale":"2024-09-13T14:56:43.779583+0000","last_undegraded":"2024-09-13T14:56:43.779583+0000","last_fullsized":"2024-09-13T14:56:43.779583+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:30:12.725003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":997,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.882407+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T14:56:08.882407+0000","last_peered":"2024-09-13T14:56:08.882407+0000","last_clean":"2024-09-13T14:56:08.882407+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T14:56:08.882407+0000","last_undegraded":"2024-09-13T14:56:08.882407+0000","last_fullsized":"2024-09-13T14:56:08.882407+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1104,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.894736+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T14:56:08.894736+0000","last_peered":"2024-09-13T14:56:08.894736+0000","last_clean":"2024-09-13T14:56:08.894736+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T14:56:08.894736+0000","last_undegraded":"2024-09-13T14:56:08.894736+0000","last_fullsized":"2024-09-13T14:56:08.894736+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":61,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.769748+0000","last_change":"2024-09-13T14:52:43.591841+0000","last_active":"2024-09-13T14:56:43.769748+0000","last_peered":"2024-09-13T14:56:43.769748+0000","last_clean":"2024-09-13T14:56:43.769748+0000","last_became_active":"2024-09-13T14:52:43.591187+0000","last_became_peered":"2024-09-13T14:52:43.591187+0000","last_unstale":"2024-09-13T14:56:43.769748+0000","last_undegraded":"2024-09-13T14:56:43.769748+0000","last_fullsized":"2024-09-13T14:56:43.769748+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:29.136085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1326,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.788474+0000","last_change":"2024-09-13T14:52:43.589345+0000","last_active":"2024-09-13T14:56:43.788474+0000","last_peered":"2024-09-13T14:56:43.788474+0000","last_clean":"2024-09-13T14:56:43.788474+0000","last_became_active":"2024-09-13T14:52:43.588829+0000","last_became_peered":"2024-09-13T14:52:43.588829+0000","last_unstale":"2024-09-13T14:56:43.788474+0000","last_undegraded":"2024-09-13T14:56:43.788474+0000","last_fullsized":"2024-09-13T14:56:43.788474+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:49.701805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":172,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.755543+0000","last_change":"2024-09-13T14:52:43.599179+0000","last_active":"2024-09-13T14:56:43.755543+0000","last_peered":"2024-09-13T14:56:43.755543+0000","last_clean":"2024-09-13T14:56:43.755543+0000","last_became_active":"2024-09-13T14:52:43.598166+0000","last_became_peered":"2024-09-13T14:52:43.598166+0000","last_unstale":"2024-09-13T14:56:43.755543+0000","last_undegraded":"2024-09-13T14:56:43.755543+0000","last_fullsized":"2024-09-13T14:56:43.755543+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:08.647786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":158,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.752251+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T14:56:43.752251+0000","last_peered":"2024-09-13T14:56:43.752251+0000","last_clean":"2024-09-13T14:56:43.752251+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T14:56:43.752251+0000","last_undegraded":"2024-09-13T14:56:43.752251+0000","last_fullsized":"2024-09-13T14:56:43.752251+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":172,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.755395+0000","last_change":"2024-09-13T14:52:43.597595+0000","last_active":"2024-09-13T14:56:43.755395+0000","last_peered":"2024-09-13T14:56:43.755395+0000","last_clean":"2024-09-13T14:56:43.755395+0000","last_became_active":"2024-09-13T14:52:43.596592+0000","last_became_peered":"2024-09-13T14:52:43.596592+0000","last_unstale":"2024-09-13T14:56:43.755395+0000","last_undegraded":"2024-09-13T14:56:43.755395+0000","last_fullsized":"2024-09-13T14:56:43.755395+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:56.532603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1045,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885775+0000","last_change":"2024-09-13T14:52:43.596429+0000","last_active":"2024-09-13T14:56:08.885775+0000","last_peered":"2024-09-13T14:56:08.885775+0000","last_clean":"2024-09-13T14:56:08.885775+0000","last_became_active":"2024-09-13T14:52:43.594614+0000","last_became_peered":"2024-09-13T14:52:43.594614+0000","last_unstale":"2024-09-13T14:56:08.885775+0000","last_undegraded":"2024-09-13T14:56:08.885775+0000","last_fullsized":"2024-09-13T14:56:08.885775+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:41.970225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.755355+0000","last_change":"2024-09-13T14:47:46.325496+0000","last_active":"2024-09-13T14:56:43.755355+0000","last_peered":"2024-09-13T14:56:43.755355+0000","last_clean":"2024-09-13T14:56:43.755355+0000","last_became_active":"2024-09-13T14:47:46.325099+0000","last_became_peered":"2024-09-13T14:47:46.325099+0000","last_unstale":"2024-09-13T14:56:43.755355+0000","last_undegraded":"2024-09-13T14:56:43.755355+0000","last_fullsized":"2024-09-13T14:56:43.755355+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:53:48.803608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1322,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.795268+0000","last_change":"2024-09-13T14:47:46.349682+0000","last_active":"2024-09-13T14:56:43.795268+0000","last_peered":"2024-09-13T14:56:43.795268+0000","last_clean":"2024-09-13T14:56:43.795268+0000","last_became_active":"2024-09-13T14:47:46.341656+0000","last_became_peered":"2024-09-13T14:47:46.341656+0000","last_unstale":"2024-09-13T14:56:43.795268+0000","last_undegraded":"2024-09-13T14:56:43.795268+0000","last_fullsized":"2024-09-13T14:56:43.795268+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:02.727169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1486,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.774880+0000","last_change":"2024-09-13T14:52:54.185117+0000","last_active":"2024-09-13T14:56:43.774880+0000","last_peered":"2024-09-13T14:56:43.774880+0000","last_clean":"2024-09-13T14:56:43.774880+0000","last_became_active":"2024-09-13T14:52:54.184712+0000","last_became_peered":"2024-09-13T14:52:54.184712+0000","last_unstale":"2024-09-13T14:56:43.774880+0000","last_undegraded":"2024-09-13T14:56:43.774880+0000","last_fullsized":"2024-09-13T14:56:43.774880+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:32.654401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":984,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.881133+0000","last_change":"2024-09-13T14:39:34.393673+0000","last_active":"2024-09-13T14:56:08.881133+0000","last_peered":"2024-09-13T14:56:08.881133+0000","last_clean":"2024-09-13T14:56:08.881133+0000","last_became_active":"2024-09-13T14:39:34.393186+0000","last_became_peered":"2024-09-13T14:39:34.393186+0000","last_unstale":"2024-09-13T14:56:08.881133+0000","last_undegraded":"2024-09-13T14:56:08.881133+0000","last_fullsized":"2024-09-13T14:56:08.881133+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:23.130084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1007,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885460+0000","last_change":"2024-09-13T14:52:43.581604+0000","last_active":"2024-09-13T14:56:08.885460+0000","last_peered":"2024-09-13T14:56:08.885460+0000","last_clean":"2024-09-13T14:56:08.885460+0000","last_became_active":"2024-09-13T14:52:43.579981+0000","last_became_peered":"2024-09-13T14:52:43.579981+0000","last_unstale":"2024-09-13T14:56:08.885460+0000","last_undegraded":"2024-09-13T14:56:08.885460+0000","last_fullsized":"2024-09-13T14:56:08.885460+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:59.366292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":972,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884627+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:56:08.884627+0000","last_peered":"2024-09-13T14:56:08.884627+0000","last_clean":"2024-09-13T14:56:08.884627+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:56:08.884627+0000","last_undegraded":"2024-09-13T14:56:08.884627+0000","last_fullsized":"2024-09-13T14:56:08.884627+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":172,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.752332+0000","last_change":"2024-09-13T14:52:43.595628+0000","last_active":"2024-09-13T14:56:43.752332+0000","last_peered":"2024-09-13T14:56:43.752332+0000","last_clean":"2024-09-13T14:56:43.752332+0000","last_became_active":"2024-09-13T14:52:43.594675+0000","last_became_peered":"2024-09-13T14:52:43.594675+0000","last_unstale":"2024-09-13T14:56:43.752332+0000","last_undegraded":"2024-09-13T14:56:43.752332+0000","last_fullsized":"2024-09-13T14:56:43.752332+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:40:19.850839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1063,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884270+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T14:56:08.884270+0000","last_peered":"2024-09-13T14:56:08.884270+0000","last_clean":"2024-09-13T14:56:08.884270+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T14:56:08.884270+0000","last_undegraded":"2024-09-13T14:56:08.884270+0000","last_fullsized":"2024-09-13T14:56:08.884270+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":966,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.880524+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:56:08.880524+0000","last_peered":"2024-09-13T14:56:08.880524+0000","last_clean":"2024-09-13T14:56:08.880524+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:56:08.880524+0000","last_undegraded":"2024-09-13T14:56:08.880524+0000","last_fullsized":"2024-09-13T14:56:08.880524+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1034,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884702+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T14:56:08.884702+0000","last_peered":"2024-09-13T14:56:08.884702+0000","last_clean":"2024-09-13T14:56:08.884702+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T14:56:08.884702+0000","last_undegraded":"2024-09-13T14:56:08.884702+0000","last_fullsized":"2024-09-13T14:56:08.884702+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":1000,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885555+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T14:56:08.885555+0000","last_peered":"2024-09-13T14:56:08.885555+0000","last_clean":"2024-09-13T14:56:08.885555+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T14:56:08.885555+0000","last_undegraded":"2024-09-13T14:56:08.885555+0000","last_fullsized":"2024-09-13T14:56:08.885555+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1324,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.786157+0000","last_change":"2024-09-13T14:52:43.592945+0000","last_active":"2024-09-13T14:56:43.786157+0000","last_peered":"2024-09-13T14:56:43.786157+0000","last_clean":"2024-09-13T14:56:43.786157+0000","last_became_active":"2024-09-13T14:52:43.592602+0000","last_became_peered":"2024-09-13T14:52:43.592602+0000","last_unstale":"2024-09-13T14:56:43.786157+0000","last_undegraded":"2024-09-13T14:56:43.786157+0000","last_fullsized":"2024-09-13T14:56:43.786157+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:11.719764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1007,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.885208+0000","last_change":"2024-09-13T14:47:46.322428+0000","last_active":"2024-09-13T14:56:08.885208+0000","last_peered":"2024-09-13T14:56:08.885208+0000","last_clean":"2024-09-13T14:56:08.885208+0000","last_became_active":"2024-09-13T14:47:46.321955+0000","last_became_peered":"2024-09-13T14:47:46.321955+0000","last_unstale":"2024-09-13T14:56:08.885208+0000","last_undegraded":"2024-09-13T14:56:08.885208+0000","last_fullsized":"2024-09-13T14:56:08.885208+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:47.614065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1439,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.781504+0000","last_change":"2024-09-13T14:39:34.362694+0000","last_active":"2024-09-13T14:56:43.781504+0000","last_peered":"2024-09-13T14:56:43.781504+0000","last_clean":"2024-09-13T14:56:43.781504+0000","last_became_active":"2024-09-13T14:39:34.362435+0000","last_became_peered":"2024-09-13T14:39:34.362435+0000","last_unstale":"2024-09-13T14:56:43.781504+0000","last_undegraded":"2024-09-13T14:56:43.781504+0000","last_fullsized":"2024-09-13T14:56:43.781504+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:33.350328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1439,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.787307+0000","last_change":"2024-09-13T14:39:34.360941+0000","last_active":"2024-09-13T14:56:43.787307+0000","last_peered":"2024-09-13T14:56:43.787307+0000","last_clean":"2024-09-13T14:56:43.787307+0000","last_became_active":"2024-09-13T14:39:34.360383+0000","last_became_peered":"2024-09-13T14:39:34.360383+0000","last_unstale":"2024-09-13T14:56:43.787307+0000","last_undegraded":"2024-09-13T14:56:43.787307+0000","last_fullsized":"2024-09-13T14:56:43.787307+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:26:51.344296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":983,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.891343+0000","last_change":"2024-09-13T14:39:34.380634+0000","last_active":"2024-09-13T14:56:08.891343+0000","last_peered":"2024-09-13T14:56:08.891343+0000","last_clean":"2024-09-13T14:56:08.891343+0000","last_became_active":"2024-09-13T14:39:34.379785+0000","last_became_peered":"2024-09-13T14:39:34.379785+0000","last_unstale":"2024-09-13T14:56:08.891343+0000","last_undegraded":"2024-09-13T14:56:08.891343+0000","last_fullsized":"2024-09-13T14:56:08.891343+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:02.436193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1040,"reported_epoch":1029,"state":"active+clean","last_fresh":"2024-09-13T14:56:08.884311+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T14:56:08.884311+0000","last_peered":"2024-09-13T14:56:08.884311+0000","last_clean":"2024-09-13T14:56:08.884311+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T14:56:08.884311+0000","last_undegraded":"2024-09-13T14:56:08.884311+0000","last_fullsized":"2024-09-13T14:56:08.884311+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1331,"reported_epoch":1030,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.786634+0000","last_change":"2024-09-13T14:52:54.156534+0000","last_active":"2024-09-13T14:56:43.786634+0000","last_peered":"2024-09-13T14:56:43.786634+0000","last_clean":"2024-09-13T14:56:43.786634+0000","last_became_active":"2024-09-13T14:52:54.156241+0000","last_became_peered":"2024-09-13T14:52:54.156241+0000","last_unstale":"2024-09-13T14:56:43.786634+0000","last_undegraded":"2024-09-13T14:56:43.786634+0000","last_fullsized":"2024-09-13T14:56:43.786634+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:54:17.998268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":42,"num_pg":16,"stat_sum":{"num_bytes":908853272,"num_objects":13869,"num_object_clones":0,"num_object_copies":41607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13869,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":201807,"num_write_kb":6901569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2767990784,"data_stored":2767978568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142007,"ondisk_log_size":142007,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397953,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":767672,"kb_used_data":457840,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92977480,"statfs":{"total":95995035648,"available":95208939520,"internally_reserved":0,"allocated":468828160,"data_stored":464871173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594188,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":527348,"kb_used_data":229996,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93217804,"statfs":{"total":95995035648,"available":95455031296,"internally_reserved":0,"allocated":235515904,"data_stored":231562910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790422,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":802408,"kb_used_data":492448,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92942744,"statfs":{"total":95995035648,"available":95173369856,"internally_reserved":0,"allocated":504266752,"data_stored":500321320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980831950,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":736944,"kb_used_data":391976,"kb_used_omap":76,"kb_used_meta":344819,"kb_avail":93008208,"statfs":{"total":95995035648,"available":95240404992,"internally_reserved":0,"allocated":401383424,"data_stored":397434027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":353095488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911189,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":647324,"kb_used_data":355028,"kb_used_omap":77,"kb_used_meta":292210,"kb_avail":93097828,"statfs":{"total":95995035648,"available":95332175872,"internally_reserved":0,"allocated":363548672,"data_stored":359614963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":299223861},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023132,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":673952,"kb_used_data":316120,"kb_used_omap":71,"kb_used_meta":357752,"kb_avail":93071200,"statfs":{"total":95995035648,"available":95304908800,"internally_reserved":0,"allocated":323706880,"data_stored":319769597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":366338720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004262,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":562592,"kb_used_data":354392,"kb_used_omap":66,"kb_used_meta":208125,"kb_avail":93182560,"statfs":{"total":95995035648,"available":95418941440,"internally_reserved":0,"allocated":362897408,"data_stored":358959564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":213120170},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195454,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":580448,"kb_used_data":250264,"kb_used_omap":56,"kb_used_meta":330119,"kb_avail":93164704,"statfs":{"total":95995035648,"available":95400656896,"internally_reserved":0,"allocated":256270336,"data_stored":252331463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":338041999},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":244383744,"data_stored":244383744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":335417344,"data_stored":335413272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":308150272,"data_stored":308150272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":344915968,"data_stored":344915968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":376176640,"data_stored":376176640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":492376064,"data_stored":492371992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":221384704,"data_stored":221380632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":42,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":445186048,"data_stored":445186048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:57:16.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_1 pgp_num 16 2024-09-13T14:57:16.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:16.787 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:16.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:16.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:16.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.935 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:16.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:17.036 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:17.065 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:17.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:17.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.203 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:17.309 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:17.316 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:57:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:57:17] "GET /metrics HTTP/1.1" 200 36761 "" "Prometheus/2.51.0" 2024-09-13T14:57:17.328 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:17.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:17.433 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.461 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:17.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:17.706 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.729 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:17.833 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:17.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:17.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:17.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:17.988 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:17.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:18.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:18.101 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:17 smithi177 bash[68594]: cluster 2024-09-13T14:57:16.701817+0000 mgr.y (mgr.26252) 933 : cluster [DBG] pgmap v1454: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:18.101 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:17 smithi177 bash[68594]: cluster 2024-09-13T14:57:16.701817+0000 mgr.y (mgr.26252) 933 : cluster [DBG] pgmap v1454: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:18.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:18.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:18.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:17 smithi099 bash[84576]: cluster 2024-09-13T14:57:16.701817+0000 mgr.y (mgr.26252) 933 : cluster [DBG] pgmap v1454: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:18.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:17 smithi099 bash[84576]: cluster 2024-09-13T14:57:16.701817+0000 mgr.y (mgr.26252) 933 : cluster [DBG] pgmap v1454: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:18.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:17 smithi099 bash[87147]: cluster 2024-09-13T14:57:16.701817+0000 mgr.y (mgr.26252) 933 : cluster [DBG] pgmap v1454: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:18.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:17 smithi099 bash[87147]: cluster 2024-09-13T14:57:16.701817+0000 mgr.y (mgr.26252) 933 : cluster [DBG] pgmap v1454: 89 pgs: 89 active+clean; 645 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:57:18.278 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:18.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:18.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:18.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:18.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:18.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:18.540 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:18.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:18.647 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:18.699 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:18.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:18.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:57:18.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:18.838 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:18.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:18.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:18.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:18.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:19.088 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:19.137 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:19.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:19.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:19.274 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:19.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:19.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:19 smithi177 bash[68594]: audit 2024-09-13T14:57:18.290612+0000 mon.a (mon.0) 1983 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:19.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:19 smithi177 bash[68594]: audit 2024-09-13T14:57:18.290612+0000 mon.a (mon.0) 1983 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:19.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:19 smithi177 bash[68594]: audit 2024-09-13T14:57:18.295432+0000 mon.a (mon.0) 1984 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:19.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:19 smithi177 bash[68594]: audit 2024-09-13T14:57:18.295432+0000 mon.a (mon.0) 1984 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:19.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:19 smithi177 bash[68594]: cluster 2024-09-13T14:57:18.702216+0000 mgr.y (mgr.26252) 934 : cluster [DBG] pgmap v1455: 89 pgs: 89 active+clean; 607 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:19.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:19 smithi177 bash[68594]: cluster 2024-09-13T14:57:18.702216+0000 mgr.y (mgr.26252) 934 : cluster [DBG] pgmap v1455: 89 pgs: 89 active+clean; 607 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:19.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:19.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:19.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:19.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:19.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:19.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:19.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[84576]: audit 2024-09-13T14:57:18.290612+0000 mon.a (mon.0) 1983 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[84576]: audit 2024-09-13T14:57:18.290612+0000 mon.a (mon.0) 1983 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[84576]: audit 2024-09-13T14:57:18.295432+0000 mon.a (mon.0) 1984 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[84576]: audit 2024-09-13T14:57:18.295432+0000 mon.a (mon.0) 1984 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[84576]: cluster 2024-09-13T14:57:18.702216+0000 mgr.y (mgr.26252) 934 : cluster [DBG] pgmap v1455: 89 pgs: 89 active+clean; 607 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[84576]: cluster 2024-09-13T14:57:18.702216+0000 mgr.y (mgr.26252) 934 : cluster [DBG] pgmap v1455: 89 pgs: 89 active+clean; 607 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[87147]: audit 2024-09-13T14:57:18.290612+0000 mon.a (mon.0) 1983 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[87147]: audit 2024-09-13T14:57:18.290612+0000 mon.a (mon.0) 1983 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[87147]: audit 2024-09-13T14:57:18.295432+0000 mon.a (mon.0) 1984 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[87147]: audit 2024-09-13T14:57:18.295432+0000 mon.a (mon.0) 1984 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[87147]: cluster 2024-09-13T14:57:18.702216+0000 mgr.y (mgr.26252) 934 : cluster [DBG] pgmap v1455: 89 pgs: 89 active+clean; 607 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:19 smithi099 bash[87147]: cluster 2024-09-13T14:57:18.702216+0000 mgr.y (mgr.26252) 934 : cluster [DBG] pgmap v1455: 89 pgs: 89 active+clean; 607 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:19.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:19.682 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:19.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:19.827 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:19.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:19.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:19.963 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:19.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:20.070 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:20.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:20.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:20.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:20.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:20.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:20.307 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:20.352 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:20.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:20.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:20.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:20.512 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:20.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:57:20 smithi177 bash[61439]: ts=2024-09-13T14:57:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:20.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:20.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:20.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:20.744 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:20.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:20.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:20.874 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:20.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:20.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:20.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:21.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:21.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:21.121 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:21.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:21.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:21.277 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:21.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:21.321 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:21.322 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:21.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:21.415 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:21.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:21.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:21.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:21.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:21.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:21.673 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:21.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:21.778 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:21.787 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:21 smithi099 bash[84576]: cluster 2024-09-13T14:57:20.702899+0000 mgr.y (mgr.26252) 935 : cluster [DBG] pgmap v1456: 89 pgs: 89 active+clean; 425 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:57:21.787 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:21 smithi099 bash[84576]: cluster 2024-09-13T14:57:20.702899+0000 mgr.y (mgr.26252) 935 : cluster [DBG] pgmap v1456: 89 pgs: 89 active+clean; 425 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:57:21.788 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:21 smithi099 bash[87147]: cluster 2024-09-13T14:57:20.702899+0000 mgr.y (mgr.26252) 935 : cluster [DBG] pgmap v1456: 89 pgs: 89 active+clean; 425 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:57:21.788 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:21 smithi099 bash[87147]: cluster 2024-09-13T14:57:20.702899+0000 mgr.y (mgr.26252) 935 : cluster [DBG] pgmap v1456: 89 pgs: 89 active+clean; 425 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:57:21.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:21.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:21.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:21.954 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:21.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:22.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:22.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:22.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:21 smithi177 bash[68594]: cluster 2024-09-13T14:57:20.702899+0000 mgr.y (mgr.26252) 935 : cluster [DBG] pgmap v1456: 89 pgs: 89 active+clean; 425 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:57:22.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:21 smithi177 bash[68594]: cluster 2024-09-13T14:57:20.702899+0000 mgr.y (mgr.26252) 935 : cluster [DBG] pgmap v1456: 89 pgs: 89 active+clean; 425 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T14:57:22.194 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:22.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:22.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:22.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:22.353 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:22.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:22.459 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:22.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:22.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:22.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:22.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:22.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:22.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:22.769 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:22.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:22.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:22.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:22.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:23.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:23.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:23.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:23.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:23.177 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:23.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:23.280 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:23.307 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:23.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:23.345 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:23.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:23.428 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:23.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:23.531 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:23.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:23.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:23.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:23.683 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:23.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:23.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:23.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:23.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:23 smithi099 bash[84576]: cluster 2024-09-13T14:57:22.703412+0000 mgr.y (mgr.26252) 936 : cluster [DBG] pgmap v1457: 89 pgs: 89 active+clean; 382 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:23.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:23 smithi099 bash[84576]: cluster 2024-09-13T14:57:22.703412+0000 mgr.y (mgr.26252) 936 : cluster [DBG] pgmap v1457: 89 pgs: 89 active+clean; 382 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:23.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:23 smithi099 bash[87147]: cluster 2024-09-13T14:57:22.703412+0000 mgr.y (mgr.26252) 936 : cluster [DBG] pgmap v1457: 89 pgs: 89 active+clean; 382 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:23.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:23 smithi099 bash[87147]: cluster 2024-09-13T14:57:22.703412+0000 mgr.y (mgr.26252) 936 : cluster [DBG] pgmap v1457: 89 pgs: 89 active+clean; 382 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:23.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:23 smithi177 bash[68594]: cluster 2024-09-13T14:57:22.703412+0000 mgr.y (mgr.26252) 936 : cluster [DBG] pgmap v1457: 89 pgs: 89 active+clean; 382 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:23.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:23 smithi177 bash[68594]: cluster 2024-09-13T14:57:22.703412+0000 mgr.y (mgr.26252) 936 : cluster [DBG] pgmap v1457: 89 pgs: 89 active+clean; 382 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:57:23.916 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:23.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:23.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:24.053 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:24.086 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:24.191 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:24.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:24.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:24.322 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:24.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:24.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:24.457 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:24.480 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:24.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:24.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:24.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:24.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:24.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:24.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:24.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:24.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:24.880 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:24.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:24.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:25.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:25.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:25.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:25.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:25.265 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:25.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:25.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:25.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:25.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.431 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:25.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:25.532 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:25.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:25.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:25.675 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:25.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:25.807 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:25.834 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:25.925 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:25 smithi099 bash[87147]: cluster 2024-09-13T14:57:24.703942+0000 mgr.y (mgr.26252) 937 : cluster [DBG] pgmap v1458: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T14:57:25.926 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:25 smithi099 bash[87147]: cluster 2024-09-13T14:57:24.703942+0000 mgr.y (mgr.26252) 937 : cluster [DBG] pgmap v1458: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T14:57:25.926 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:25 smithi099 bash[84576]: cluster 2024-09-13T14:57:24.703942+0000 mgr.y (mgr.26252) 937 : cluster [DBG] pgmap v1458: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T14:57:25.926 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:25 smithi099 bash[84576]: cluster 2024-09-13T14:57:24.703942+0000 mgr.y (mgr.26252) 937 : cluster [DBG] pgmap v1458: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T14:57:25.939 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:25.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:25.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:26.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:26.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:26.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:25 smithi177 bash[68594]: cluster 2024-09-13T14:57:24.703942+0000 mgr.y (mgr.26252) 937 : cluster [DBG] pgmap v1458: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T14:57:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:25 smithi177 bash[68594]: cluster 2024-09-13T14:57:24.703942+0000 mgr.y (mgr.26252) 937 : cluster [DBG] pgmap v1458: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T14:57:26.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:26.213 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:26.317 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:26.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:26.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:26.487 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:26.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:26.591 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:26.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:26.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:26.725 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:26.747 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:26.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:26.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:26.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:27.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:27.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:27.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:27.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:27.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:27.258 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:27.285 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:27.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:27.322 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:57:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:57:27] "GET /metrics HTTP/1.1" 200 36761 "" "Prometheus/2.51.0" 2024-09-13T14:57:27.392 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:27.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:27.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:27.521 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:27.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:27.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:27.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:27.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:27.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:27.790 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:27 smithi177 bash[68594]: cluster 2024-09-13T14:57:26.704376+0000 mgr.y (mgr.26252) 938 : cluster [DBG] pgmap v1459: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2024-09-13T14:57:27.790 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:27 smithi177 bash[68594]: cluster 2024-09-13T14:57:26.704376+0000 mgr.y (mgr.26252) 938 : cluster [DBG] pgmap v1459: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2024-09-13T14:57:27.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:27.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:27.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:27.941 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:28.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:27 smithi099 bash[84576]: cluster 2024-09-13T14:57:26.704376+0000 mgr.y (mgr.26252) 938 : cluster [DBG] pgmap v1459: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2024-09-13T14:57:28.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:27 smithi099 bash[84576]: cluster 2024-09-13T14:57:26.704376+0000 mgr.y (mgr.26252) 938 : cluster [DBG] pgmap v1459: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2024-09-13T14:57:28.006 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:27 smithi099 bash[87147]: cluster 2024-09-13T14:57:26.704376+0000 mgr.y (mgr.26252) 938 : cluster [DBG] pgmap v1459: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2024-09-13T14:57:28.006 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:27 smithi099 bash[87147]: cluster 2024-09-13T14:57:26.704376+0000 mgr.y (mgr.26252) 938 : cluster [DBG] pgmap v1459: 89 pgs: 89 active+clean; 215 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2024-09-13T14:57:28.045 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:28.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:28.187 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:28.224 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:28.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:28.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.348 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:28.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.475 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:28.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:28.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:28.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:28.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.739 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:28.847 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:28.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:28.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:28.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:28.993 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:28.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:28.995 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Removed 107837 objects 2024-09-13T14:57:28.995 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Clean up completed and total clean up time :151.826 2024-09-13T14:57:29.096 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:29.119 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:29.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:29.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:29.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:29.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:29.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:29.370 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:29.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:29.473 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:29.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:29.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:29.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:29.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:29.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:29.734 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:29.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:29.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:29.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:29.881 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:29.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:29.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:30.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:30.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:30.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:29 smithi099 bash[84576]: cluster 2024-09-13T14:57:28.704844+0000 mgr.y (mgr.26252) 939 : cluster [DBG] pgmap v1460: 89 pgs: 89 active+clean; 172 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:57:30.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:29 smithi099 bash[84576]: cluster 2024-09-13T14:57:28.704844+0000 mgr.y (mgr.26252) 939 : cluster [DBG] pgmap v1460: 89 pgs: 89 active+clean; 172 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:57:30.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:29 smithi099 bash[87147]: cluster 2024-09-13T14:57:28.704844+0000 mgr.y (mgr.26252) 939 : cluster [DBG] pgmap v1460: 89 pgs: 89 active+clean; 172 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:57:30.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:29 smithi099 bash[87147]: cluster 2024-09-13T14:57:28.704844+0000 mgr.y (mgr.26252) 939 : cluster [DBG] pgmap v1460: 89 pgs: 89 active+clean; 172 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:57:30.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:29 smithi177 bash[68594]: cluster 2024-09-13T14:57:28.704844+0000 mgr.y (mgr.26252) 939 : cluster [DBG] pgmap v1460: 89 pgs: 89 active+clean; 172 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:57:30.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:29 smithi177 bash[68594]: cluster 2024-09-13T14:57:28.704844+0000 mgr.y (mgr.26252) 939 : cluster [DBG] pgmap v1460: 89 pgs: 89 active+clean; 172 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2024-09-13T14:57:30.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:30.140 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:30.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:30.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:30.268 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:30.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:30.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:57:30 smithi177 bash[61439]: ts=2024-09-13T14:57:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:30.370 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:30.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:30.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:30.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:30.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:30.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:30.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:30.658 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:30.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:30.761 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:30.789 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:30.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:30.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:30.918 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:31.021 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:31.049 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:31.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:31.152 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:31.179 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:31.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:31.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:31.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:31.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:31.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:31.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:31.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:31.540 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:31.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:31.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:31.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:31.700 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:31.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:31.780 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:31 smithi099 bash[87147]: cluster 2024-09-13T14:57:30.706011+0000 mgr.y (mgr.26252) 940 : cluster [DBG] pgmap v1461: 89 pgs: 89 active+clean; 15 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-09-13T14:57:31.780 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:31 smithi099 bash[87147]: cluster 2024-09-13T14:57:30.706011+0000 mgr.y (mgr.26252) 940 : cluster [DBG] pgmap v1461: 89 pgs: 89 active+clean; 15 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-09-13T14:57:31.804 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:31.827 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:31.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:31.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:31.941 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:31 smithi099 bash[84576]: cluster 2024-09-13T14:57:30.706011+0000 mgr.y (mgr.26252) 940 : cluster [DBG] pgmap v1461: 89 pgs: 89 active+clean; 15 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-09-13T14:57:31.941 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:31 smithi099 bash[84576]: cluster 2024-09-13T14:57:30.706011+0000 mgr.y (mgr.26252) 940 : cluster [DBG] pgmap v1461: 89 pgs: 89 active+clean; 15 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-09-13T14:57:31.955 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:31.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:32.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:32.070 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:31 smithi177 bash[68594]: cluster 2024-09-13T14:57:30.706011+0000 mgr.y (mgr.26252) 940 : cluster [DBG] pgmap v1461: 89 pgs: 89 active+clean; 15 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-09-13T14:57:32.070 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:31 smithi177 bash[68594]: cluster 2024-09-13T14:57:30.706011+0000 mgr.y (mgr.26252) 940 : cluster [DBG] pgmap v1461: 89 pgs: 89 active+clean; 15 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-09-13T14:57:32.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:32.190 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:32.215 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:32.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:32.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:32.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:32.448 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:32.474 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:32.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:32.577 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:32.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:32.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:32.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:32.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:32.842 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:32.868 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:32.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:32.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:32.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:33.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:33.102 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:33.128 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:33.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:33.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:33.255 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:33.357 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:33.386 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:33.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:33.489 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:33.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:33.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:33.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:33.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:33.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:33.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:33.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:33.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:33.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:33.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:33.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:33.969 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:33 smithi099 bash[87147]: cluster 2024-09-13T14:57:32.706899+0000 mgr.y (mgr.26252) 941 : cluster [DBG] pgmap v1462: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2024-09-13T14:57:33.969 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:33 smithi099 bash[87147]: cluster 2024-09-13T14:57:32.706899+0000 mgr.y (mgr.26252) 941 : cluster [DBG] pgmap v1462: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2024-09-13T14:57:33.969 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:33 smithi099 bash[87147]: audit 2024-09-13T14:57:33.285974+0000 mon.a (mon.0) 1985 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:33.969 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:33 smithi099 bash[87147]: audit 2024-09-13T14:57:33.285974+0000 mon.a (mon.0) 1985 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:33.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:33 smithi099 bash[84576]: cluster 2024-09-13T14:57:32.706899+0000 mgr.y (mgr.26252) 941 : cluster [DBG] pgmap v1462: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2024-09-13T14:57:33.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:33 smithi099 bash[84576]: cluster 2024-09-13T14:57:32.706899+0000 mgr.y (mgr.26252) 941 : cluster [DBG] pgmap v1462: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2024-09-13T14:57:33.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:33 smithi099 bash[84576]: audit 2024-09-13T14:57:33.285974+0000 mon.a (mon.0) 1985 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:33.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:33 smithi099 bash[84576]: audit 2024-09-13T14:57:33.285974+0000 mon.a (mon.0) 1985 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:34.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:34.036 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:34.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:33 smithi177 bash[68594]: cluster 2024-09-13T14:57:32.706899+0000 mgr.y (mgr.26252) 941 : cluster [DBG] pgmap v1462: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2024-09-13T14:57:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:33 smithi177 bash[68594]: cluster 2024-09-13T14:57:32.706899+0000 mgr.y (mgr.26252) 941 : cluster [DBG] pgmap v1462: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2024-09-13T14:57:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:33 smithi177 bash[68594]: audit 2024-09-13T14:57:33.285974+0000 mon.a (mon.0) 1985 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:33 smithi177 bash[68594]: audit 2024-09-13T14:57:33.285974+0000 mon.a (mon.0) 1985 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:34.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:34.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:34.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:34.269 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:34.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:34.401 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:34.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:34.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:34.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:34.557 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:34.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:34.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:34.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:34.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:34.791 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:34.817 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:34.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:34.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:34.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:34.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:35.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:35.079 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:35.181 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:35.207 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:35.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:35.338 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:35.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:35.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:35.469 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:35.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:35.572 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:35.596 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:35.699 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:35.726 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:35.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:35.829 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:35.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:35 smithi099 bash[84576]: cluster 2024-09-13T14:57:34.708058+0000 mgr.y (mgr.26252) 942 : cluster [DBG] pgmap v1463: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 508 op/s 2024-09-13T14:57:35.840 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:35 smithi099 bash[84576]: cluster 2024-09-13T14:57:34.708058+0000 mgr.y (mgr.26252) 942 : cluster [DBG] pgmap v1463: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 508 op/s 2024-09-13T14:57:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:35 smithi099 bash[87147]: cluster 2024-09-13T14:57:34.708058+0000 mgr.y (mgr.26252) 942 : cluster [DBG] pgmap v1463: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 508 op/s 2024-09-13T14:57:35.841 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:35 smithi099 bash[87147]: cluster 2024-09-13T14:57:34.708058+0000 mgr.y (mgr.26252) 942 : cluster [DBG] pgmap v1463: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 508 op/s 2024-09-13T14:57:35.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:35.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:35.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:35.986 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:36.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:36.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:35 smithi177 bash[68594]: cluster 2024-09-13T14:57:34.708058+0000 mgr.y (mgr.26252) 942 : cluster [DBG] pgmap v1463: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 508 op/s 2024-09-13T14:57:36.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:35 smithi177 bash[68594]: cluster 2024-09-13T14:57:34.708058+0000 mgr.y (mgr.26252) 942 : cluster [DBG] pgmap v1463: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 508 op/s 2024-09-13T14:57:36.117 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:36.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:36.221 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:36.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:36.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:36.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:36.377 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:36.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:36.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:36.509 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:36.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:36.612 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:36.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:36.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:36.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:36.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:36.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:36.868 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:36.869 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:57:36.869 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:36.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:36.888 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:57:36.889 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:36.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:36.905 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:57:36.906 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:36.923 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T14:57:36.923 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:36.944 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:57:36.945 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:36.964 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:57:36.965 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:36.985 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:57:36.985 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:36.991 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:37.006 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:57:37.006 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:37.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:37.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:37.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:37.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:37.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:37.207 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:57:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:57:37] "GET /metrics HTTP/1.1" 200 36712 "" "Prometheus/2.51.0" 2024-09-13T14:57:37.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:37.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:37.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:37.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:37.403 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:37.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:37.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:37.534 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:37.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:37.637 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:37.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:37.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:37.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:37.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:37.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:37.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:37.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:38.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:38.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:38.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:38.103 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:37 smithi099 bash[84576]: cluster 2024-09-13T14:57:36.708578+0000 mgr.y (mgr.26252) 943 : cluster [DBG] pgmap v1464: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:38.104 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:37 smithi099 bash[84576]: cluster 2024-09-13T14:57:36.708578+0000 mgr.y (mgr.26252) 943 : cluster [DBG] pgmap v1464: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:38.104 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:37 smithi099 bash[84576]: audit 2024-09-13T14:57:36.851393+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.99:0/738449310' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:38.104 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:37 smithi099 bash[84576]: audit 2024-09-13T14:57:36.851393+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.99:0/738449310' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:38.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:37 smithi099 bash[87147]: cluster 2024-09-13T14:57:36.708578+0000 mgr.y (mgr.26252) 943 : cluster [DBG] pgmap v1464: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:38.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:37 smithi099 bash[87147]: cluster 2024-09-13T14:57:36.708578+0000 mgr.y (mgr.26252) 943 : cluster [DBG] pgmap v1464: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:38.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:37 smithi099 bash[87147]: audit 2024-09-13T14:57:36.851393+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.99:0/738449310' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:38.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:37 smithi099 bash[87147]: audit 2024-09-13T14:57:36.851393+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.99:0/738449310' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:37 smithi177 bash[68594]: cluster 2024-09-13T14:57:36.708578+0000 mgr.y (mgr.26252) 943 : cluster [DBG] pgmap v1464: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:37 smithi177 bash[68594]: cluster 2024-09-13T14:57:36.708578+0000 mgr.y (mgr.26252) 943 : cluster [DBG] pgmap v1464: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:37 smithi177 bash[68594]: audit 2024-09-13T14:57:36.851393+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.99:0/738449310' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:37 smithi177 bash[68594]: audit 2024-09-13T14:57:36.851393+0000 mon.c (mon.2) 71 : audit [DBG] from='client.? 172.21.15.99:0/738449310' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:38.157 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:38.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:38.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:38.315 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:38.418 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:38.445 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:38.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:38.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:38.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:38.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:38.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:38.704 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:38.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:38.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:38.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:38.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:38.937 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:38.967 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:38.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:39.070 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:39.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:39.202 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:39.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:39.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:39.330 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:39.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:39.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:39.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:39.488 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:39.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:39.591 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:39.619 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:39.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:39.721 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:39.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:39.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:39.881 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:39.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:39.984 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:40.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:40.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:39 smithi177 bash[68594]: cluster 2024-09-13T14:57:38.709490+0000 mgr.y (mgr.26252) 944 : cluster [DBG] pgmap v1465: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:40.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:39 smithi177 bash[68594]: cluster 2024-09-13T14:57:38.709490+0000 mgr.y (mgr.26252) 944 : cluster [DBG] pgmap v1465: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:40.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:40.141 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:40.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:39 smithi099 bash[84576]: cluster 2024-09-13T14:57:38.709490+0000 mgr.y (mgr.26252) 944 : cluster [DBG] pgmap v1465: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:40.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:39 smithi099 bash[84576]: cluster 2024-09-13T14:57:38.709490+0000 mgr.y (mgr.26252) 944 : cluster [DBG] pgmap v1465: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:40.142 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:39 smithi099 bash[87147]: cluster 2024-09-13T14:57:38.709490+0000 mgr.y (mgr.26252) 944 : cluster [DBG] pgmap v1465: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:40.143 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:39 smithi099 bash[87147]: cluster 2024-09-13T14:57:38.709490+0000 mgr.y (mgr.26252) 944 : cluster [DBG] pgmap v1465: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 285 op/s 2024-09-13T14:57:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:40.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:40.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:40.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:40.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:40.402 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:40.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:40.505 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:40.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:40.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:40.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:57:40 smithi177 bash[61439]: ts=2024-09-13T14:57:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:40.636 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:40.664 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:40.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:40.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:40.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:40.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:40.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:41.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.056 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:41.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.187 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.275 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:57:41.275 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T14:57:41.275 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T14:57:41.275 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T14:57:41.275 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:57:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:41.377 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:41.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:41.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:41.500 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:41.529 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:41.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:41.632 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:41.654 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:41.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:41.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:41.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:41.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:41.802 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T14:57:41.884 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:41.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[84576]: cluster 2024-09-13T14:57:40.710609+0000 mgr.y (mgr.26252) 945 : cluster [DBG] pgmap v1466: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 228 op/s 2024-09-13T14:57:41.992 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[84576]: cluster 2024-09-13T14:57:40.710609+0000 mgr.y (mgr.26252) 945 : cluster [DBG] pgmap v1466: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 228 op/s 2024-09-13T14:57:41.992 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[84576]: audit 2024-09-13T14:57:41.670444+0000 mon.c (mon.2) 72 : audit [INF] from='client.? 172.21.15.99:0/80420780' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:41.992 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[84576]: audit 2024-09-13T14:57:41.670444+0000 mon.c (mon.2) 72 : audit [INF] from='client.? 172.21.15.99:0/80420780' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:41.992 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[84576]: audit 2024-09-13T14:57:41.670741+0000 mon.a (mon.0) 1986 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:41.992 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[84576]: audit 2024-09-13T14:57:41.670741+0000 mon.a (mon.0) 1986 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:41.992 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[87147]: cluster 2024-09-13T14:57:40.710609+0000 mgr.y (mgr.26252) 945 : cluster [DBG] pgmap v1466: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 228 op/s 2024-09-13T14:57:41.992 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[87147]: cluster 2024-09-13T14:57:40.710609+0000 mgr.y (mgr.26252) 945 : cluster [DBG] pgmap v1466: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 228 op/s 2024-09-13T14:57:41.993 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[87147]: audit 2024-09-13T14:57:41.670444+0000 mon.c (mon.2) 72 : audit [INF] from='client.? 172.21.15.99:0/80420780' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:41.993 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[87147]: audit 2024-09-13T14:57:41.670444+0000 mon.c (mon.2) 72 : audit [INF] from='client.? 172.21.15.99:0/80420780' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:41.993 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[87147]: audit 2024-09-13T14:57:41.670741+0000 mon.a (mon.0) 1986 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:41.993 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:41 smithi099 bash[87147]: audit 2024-09-13T14:57:41.670741+0000 mon.a (mon.0) 1986 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:42.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:42.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:42.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:42.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:41 smithi177 bash[68594]: cluster 2024-09-13T14:57:40.710609+0000 mgr.y (mgr.26252) 945 : cluster [DBG] pgmap v1466: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 228 op/s 2024-09-13T14:57:42.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:41 smithi177 bash[68594]: cluster 2024-09-13T14:57:40.710609+0000 mgr.y (mgr.26252) 945 : cluster [DBG] pgmap v1466: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 228 op/s 2024-09-13T14:57:42.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:41 smithi177 bash[68594]: audit 2024-09-13T14:57:41.670444+0000 mon.c (mon.2) 72 : audit [INF] from='client.? 172.21.15.99:0/80420780' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:42.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:41 smithi177 bash[68594]: audit 2024-09-13T14:57:41.670444+0000 mon.c (mon.2) 72 : audit [INF] from='client.? 172.21.15.99:0/80420780' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:42.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:41 smithi177 bash[68594]: audit 2024-09-13T14:57:41.670741+0000 mon.a (mon.0) 1986 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:42.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:41 smithi177 bash[68594]: audit 2024-09-13T14:57:41.670741+0000 mon.a (mon.0) 1986 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:42.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:42.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:42.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:42.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:42.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:42.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:42.405 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:42.427 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:42.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:42.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:42.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:42.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:42.653 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:42.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:42.678 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:57:42.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:42.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:42.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:42.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:42.814 INFO:teuthology.orchestra.run.smithi099.stderr:set pool 42 pgp_num to 16 2024-09-13T14:57:42.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:42.922 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:42 smithi177 bash[68594]: cluster 2024-09-13T14:57:41.798064+0000 mon.a (mon.0) 1987 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:42.922 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:42 smithi177 bash[68594]: cluster 2024-09-13T14:57:41.798064+0000 mon.a (mon.0) 1987 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:42.922 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:42 smithi177 bash[68594]: audit 2024-09-13T14:57:41.799619+0000 mon.a (mon.0) 1988 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:42.922 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:42 smithi177 bash[68594]: audit 2024-09-13T14:57:41.799619+0000 mon.a (mon.0) 1988 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:42.922 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:42 smithi177 bash[68594]: cluster 2024-09-13T14:57:41.802894+0000 mon.a (mon.0) 1989 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2024-09-13T14:57:42.923 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:42 smithi177 bash[68594]: cluster 2024-09-13T14:57:41.802894+0000 mon.a (mon.0) 1989 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2024-09-13T14:57:42.923 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:42 smithi177 bash[68594]: audit 2024-09-13T14:57:42.742938+0000 mon.a (mon.0) 1990 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:57:42.923 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:42 smithi177 bash[68594]: audit 2024-09-13T14:57:42.742938+0000 mon.a (mon.0) 1990 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:57:42.939 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:43.018 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[87147]: cluster 2024-09-13T14:57:41.798064+0000 mon.a (mon.0) 1987 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:43.018 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[87147]: cluster 2024-09-13T14:57:41.798064+0000 mon.a (mon.0) 1987 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:43.018 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[87147]: audit 2024-09-13T14:57:41.799619+0000 mon.a (mon.0) 1988 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:43.018 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[87147]: audit 2024-09-13T14:57:41.799619+0000 mon.a (mon.0) 1988 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:43.018 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[87147]: cluster 2024-09-13T14:57:41.802894+0000 mon.a (mon.0) 1989 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2024-09-13T14:57:43.019 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[87147]: cluster 2024-09-13T14:57:41.802894+0000 mon.a (mon.0) 1989 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2024-09-13T14:57:43.019 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[87147]: audit 2024-09-13T14:57:42.742938+0000 mon.a (mon.0) 1990 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:57:43.020 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[87147]: audit 2024-09-13T14:57:42.742938+0000 mon.a (mon.0) 1990 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:57:43.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[84576]: cluster 2024-09-13T14:57:41.798064+0000 mon.a (mon.0) 1987 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:43.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[84576]: cluster 2024-09-13T14:57:41.798064+0000 mon.a (mon.0) 1987 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:43.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[84576]: audit 2024-09-13T14:57:41.799619+0000 mon.a (mon.0) 1988 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:43.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[84576]: audit 2024-09-13T14:57:41.799619+0000 mon.a (mon.0) 1988 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:43.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[84576]: cluster 2024-09-13T14:57:41.802894+0000 mon.a (mon.0) 1989 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2024-09-13T14:57:43.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[84576]: cluster 2024-09-13T14:57:41.802894+0000 mon.a (mon.0) 1989 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2024-09-13T14:57:43.021 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[84576]: audit 2024-09-13T14:57:42.742938+0000 mon.a (mon.0) 1990 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:57:43.021 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:42 smithi099 bash[84576]: audit 2024-09-13T14:57:42.742938+0000 mon.a (mon.0) 1990 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:57:43.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:43.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:43.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:43.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:43.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:43.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:43.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:43.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:43.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:43.434 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:43.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:43.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:43.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:43.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:43.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:43.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:43.720 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:43.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:43.823 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:43.850 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:43.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:43.965 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:43 smithi177 bash[68594]: cluster 2024-09-13T14:57:42.711154+0000 mgr.y (mgr.26252) 946 : cluster [DBG] pgmap v1468: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:43.965 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:43 smithi177 bash[68594]: cluster 2024-09-13T14:57:42.711154+0000 mgr.y (mgr.26252) 946 : cluster [DBG] pgmap v1468: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:43.965 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:43 smithi177 bash[68594]: audit 2024-09-13T14:57:42.815917+0000 mon.a (mon.0) 1991 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:57:43.965 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:43 smithi177 bash[68594]: audit 2024-09-13T14:57:42.815917+0000 mon.a (mon.0) 1991 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:57:43.965 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:43 smithi177 bash[68594]: cluster 2024-09-13T14:57:42.825402+0000 mon.a (mon.0) 1992 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2024-09-13T14:57:43.965 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:43 smithi177 bash[68594]: cluster 2024-09-13T14:57:42.825402+0000 mon.a (mon.0) 1992 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2024-09-13T14:57:43.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:43.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:43.991 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:57:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[84576]: cluster 2024-09-13T14:57:42.711154+0000 mgr.y (mgr.26252) 946 : cluster [DBG] pgmap v1468: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[84576]: cluster 2024-09-13T14:57:42.711154+0000 mgr.y (mgr.26252) 946 : cluster [DBG] pgmap v1468: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[84576]: audit 2024-09-13T14:57:42.815917+0000 mon.a (mon.0) 1991 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:57:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[84576]: audit 2024-09-13T14:57:42.815917+0000 mon.a (mon.0) 1991 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:57:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[84576]: cluster 2024-09-13T14:57:42.825402+0000 mon.a (mon.0) 1992 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2024-09-13T14:57:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[84576]: cluster 2024-09-13T14:57:42.825402+0000 mon.a (mon.0) 1992 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2024-09-13T14:57:44.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[87147]: cluster 2024-09-13T14:57:42.711154+0000 mgr.y (mgr.26252) 946 : cluster [DBG] pgmap v1468: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:44.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[87147]: cluster 2024-09-13T14:57:42.711154+0000 mgr.y (mgr.26252) 946 : cluster [DBG] pgmap v1468: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:44.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[87147]: audit 2024-09-13T14:57:42.815917+0000 mon.a (mon.0) 1991 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:57:44.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[87147]: audit 2024-09-13T14:57:42.815917+0000 mon.a (mon.0) 1991 : audit [INF] from='client.? 172.21.15.99:0/500107271' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:57:44.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[87147]: cluster 2024-09-13T14:57:42.825402+0000 mon.a (mon.0) 1992 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2024-09-13T14:57:44.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:43 smithi099 bash[87147]: cluster 2024-09-13T14:57:42.825402+0000 mon.a (mon.0) 1992 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2024-09-13T14:57:44.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:44.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:44.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:44.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:44.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:44.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:44.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:44.362 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:44.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:44.465 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:44.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:44.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:44.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:44.623 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:44.726 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:44.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:44.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:44.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:44 smithi177 bash[68594]: audit 2024-09-13T14:57:44.468064+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:44.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:44 smithi177 bash[68594]: audit 2024-09-13T14:57:44.468064+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:44.878 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:44.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:45.004 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:45.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:45.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:44 smithi099 bash[84576]: audit 2024-09-13T14:57:44.468064+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:45.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:44 smithi099 bash[84576]: audit 2024-09-13T14:57:44.468064+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:45.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:44 smithi099 bash[87147]: audit 2024-09-13T14:57:44.468064+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:45.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:44 smithi099 bash[87147]: audit 2024-09-13T14:57:44.468064+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:57:45.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:45.133 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:45.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:45.200 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-09-13T14:57:45.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-09-13T14:57:45.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:45.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:45.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:45.359 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:45.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:45.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:45.489 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:45.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:45.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:45.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:45.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:45.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:45.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:45.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:45.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:45.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:45.905 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:46.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:46.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:45 smithi177 bash[68594]: cluster 2024-09-13T14:57:44.711786+0000 mgr.y (mgr.26252) 947 : cluster [DBG] pgmap v1470: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:46.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:45 smithi177 bash[68594]: cluster 2024-09-13T14:57:44.711786+0000 mgr.y (mgr.26252) 947 : cluster [DBG] pgmap v1470: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:46.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:46.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:46.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:46.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:45 smithi099 bash[84576]: cluster 2024-09-13T14:57:44.711786+0000 mgr.y (mgr.26252) 947 : cluster [DBG] pgmap v1470: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:46.151 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:45 smithi099 bash[84576]: cluster 2024-09-13T14:57:44.711786+0000 mgr.y (mgr.26252) 947 : cluster [DBG] pgmap v1470: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:46.151 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:45 smithi099 bash[87147]: cluster 2024-09-13T14:57:44.711786+0000 mgr.y (mgr.26252) 947 : cluster [DBG] pgmap v1470: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:46.151 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:45 smithi099 bash[87147]: cluster 2024-09-13T14:57:44.711786+0000 mgr.y (mgr.26252) 947 : cluster [DBG] pgmap v1470: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:46.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:46.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:46.269 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:46.297 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:46.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:46.400 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:46.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:46.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:46.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:46.554 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:46.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:46.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:46.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:46.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:46.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:46.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:46.940 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:46.965 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:57:47.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:47.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:47.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:47.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:47.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:47.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:47.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:47.309 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:57:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:57:47] "GET /metrics HTTP/1.1" 200 36712 "" "Prometheus/2.51.0" 2024-09-13T14:57:47.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:47.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:47.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:47.440 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:47.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:47.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:47.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:47.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:47.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:47.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:47.708 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:47.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:47.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:47.836 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:47.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:47.939 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:47.965 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:47.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:48.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:48.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:48.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:48.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:47 smithi177 bash[68594]: cluster 2024-09-13T14:57:46.712261+0000 mgr.y (mgr.26252) 948 : cluster [DBG] pgmap v1471: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:48.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:47 smithi177 bash[68594]: cluster 2024-09-13T14:57:46.712261+0000 mgr.y (mgr.26252) 948 : cluster [DBG] pgmap v1471: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:48.197 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:48.223 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:48.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:48.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:48 smithi099 bash[87147]: cluster 2024-09-13T14:57:46.712261+0000 mgr.y (mgr.26252) 948 : cluster [DBG] pgmap v1471: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:48.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:48 smithi099 bash[87147]: cluster 2024-09-13T14:57:46.712261+0000 mgr.y (mgr.26252) 948 : cluster [DBG] pgmap v1471: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:48.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:48 smithi099 bash[84576]: cluster 2024-09-13T14:57:46.712261+0000 mgr.y (mgr.26252) 948 : cluster [DBG] pgmap v1471: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:48.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:48 smithi099 bash[84576]: cluster 2024-09-13T14:57:46.712261+0000 mgr.y (mgr.26252) 948 : cluster [DBG] pgmap v1471: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:48.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:48.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:48.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:48.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:48.462 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:48.484 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:48.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:48.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:48.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:48.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:48.744 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:48.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:48.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:48.872 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:48.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:48.975 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:49.003 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:49.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:49.014 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:48 smithi099 bash[84576]: audit 2024-09-13T14:57:48.286837+0000 mon.a (mon.0) 1994 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:49.014 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:48 smithi099 bash[84576]: audit 2024-09-13T14:57:48.286837+0000 mon.a (mon.0) 1994 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:49.015 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:48 smithi099 bash[87147]: audit 2024-09-13T14:57:48.286837+0000 mon.a (mon.0) 1994 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:49.015 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:48 smithi099 bash[87147]: audit 2024-09-13T14:57:48.286837+0000 mon.a (mon.0) 1994 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:49.106 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:49.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:48 smithi177 bash[68594]: audit 2024-09-13T14:57:48.286837+0000 mon.a (mon.0) 1994 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:49.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:48 smithi177 bash[68594]: audit 2024-09-13T14:57:48.286837+0000 mon.a (mon.0) 1994 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:49.133 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:49.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:49.236 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:49.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:49.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:49.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:49.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:49.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:49.481 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:49.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:49.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:49.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:49.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:49.644 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:49.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:49.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:49.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:49.879 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:49 smithi099 bash[84576]: cluster 2024-09-13T14:57:48.713175+0000 mgr.y (mgr.26252) 949 : cluster [DBG] pgmap v1472: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:49.879 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:49 smithi099 bash[84576]: cluster 2024-09-13T14:57:48.713175+0000 mgr.y (mgr.26252) 949 : cluster [DBG] pgmap v1472: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:49.880 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:49 smithi099 bash[87147]: cluster 2024-09-13T14:57:48.713175+0000 mgr.y (mgr.26252) 949 : cluster [DBG] pgmap v1472: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:49.880 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:49 smithi099 bash[87147]: cluster 2024-09-13T14:57:48.713175+0000 mgr.y (mgr.26252) 949 : cluster [DBG] pgmap v1472: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:49.891 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:49.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:49.994 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:50.002 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:49 smithi177 bash[68594]: cluster 2024-09-13T14:57:48.713175+0000 mgr.y (mgr.26252) 949 : cluster [DBG] pgmap v1472: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:50.002 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:49 smithi177 bash[68594]: cluster 2024-09-13T14:57:48.713175+0000 mgr.y (mgr.26252) 949 : cluster [DBG] pgmap v1472: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:50.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.139 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.201 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:57:50.201 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:57:50.201 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:57:50.201 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T14:57:50.202 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.3 2024-09-13T14:57:50.242 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.267 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:57:50 smithi177 bash[61439]: ts=2024-09-13T14:57:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:57:50.370 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:50.393 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:50.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.505 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:57:50 smithi099 systemd[1]: Stopping Ceph osd.3 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T14:57:50.519 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.646 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.749 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.773 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:50.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:50.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:50.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:51.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:51.020 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:57:50 smithi099 bash[111000]: debug 2024-09-13T14:57:50.883+0000 7f247ab78640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T14:57:51.020 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:57:50 smithi099 bash[111000]: debug 2024-09-13T14:57:50.883+0000 7f247ab78640 -1 osd.3 1033 *** Got signal Terminated *** 2024-09-13T14:57:51.020 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:57:50 smithi099 bash[111000]: debug 2024-09-13T14:57:50.883+0000 7f247ab78640 -1 osd.3 1033 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:57:51.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:51.123 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:51.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:51.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:51.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:51.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:51.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:51.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:50.248369+0000 mon.a (mon.0) 1995 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:50.248369+0000 mon.a (mon.0) 1995 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:50.253263+0000 mon.a (mon.0) 1996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:50.253263+0000 mon.a (mon.0) 1996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:50.487696+0000 mon.a (mon.0) 1997 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:50.487696+0000 mon.a (mon.0) 1997 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:50.493790+0000 mon.a (mon.0) 1998 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:50.493790+0000 mon.a (mon.0) 1998 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: cluster 2024-09-13T14:57:50.713911+0000 mgr.y (mgr.26252) 950 : cluster [DBG] pgmap v1473: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: cluster 2024-09-13T14:57:50.713911+0000 mgr.y (mgr.26252) 950 : cluster [DBG] pgmap v1473: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: cluster 2024-09-13T14:57:50.889675+0000 mon.a (mon.0) 1999 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:57:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: cluster 2024-09-13T14:57:50.889675+0000 mon.a (mon.0) 1999 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:57:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:51.083820+0000 mon.a (mon.0) 2000 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:51.083820+0000 mon.a (mon.0) 2000 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:51.084559+0000 mon.a (mon.0) 2001 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:51.084559+0000 mon.a (mon.0) 2001 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:51.089371+0000 mon.a (mon.0) 2002 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[84576]: audit 2024-09-13T14:57:51.089371+0000 mon.a (mon.0) 2002 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:50.248369+0000 mon.a (mon.0) 1995 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:50.248369+0000 mon.a (mon.0) 1995 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:50.253263+0000 mon.a (mon.0) 1996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:50.253263+0000 mon.a (mon.0) 1996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:50.487696+0000 mon.a (mon.0) 1997 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:50.487696+0000 mon.a (mon.0) 1997 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:50.493790+0000 mon.a (mon.0) 1998 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:50.493790+0000 mon.a (mon.0) 1998 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: cluster 2024-09-13T14:57:50.713911+0000 mgr.y (mgr.26252) 950 : cluster [DBG] pgmap v1473: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:51.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: cluster 2024-09-13T14:57:50.713911+0000 mgr.y (mgr.26252) 950 : cluster [DBG] pgmap v1473: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:51.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: cluster 2024-09-13T14:57:50.889675+0000 mon.a (mon.0) 1999 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:57:51.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: cluster 2024-09-13T14:57:50.889675+0000 mon.a (mon.0) 1999 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:57:51.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:51.083820+0000 mon.a (mon.0) 2000 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:51.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:51.083820+0000 mon.a (mon.0) 2000 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:51.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:51.084559+0000 mon.a (mon.0) 2001 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:51.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:51.084559+0000 mon.a (mon.0) 2001 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:51.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:51.089371+0000 mon.a (mon.0) 2002 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:51 smithi099 bash[87147]: audit 2024-09-13T14:57:51.089371+0000 mon.a (mon.0) 2002 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.377 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:51.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:50.248369+0000 mon.a (mon.0) 1995 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:50.248369+0000 mon.a (mon.0) 1995 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:50.253263+0000 mon.a (mon.0) 1996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:50.253263+0000 mon.a (mon.0) 1996 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:50.487696+0000 mon.a (mon.0) 1997 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:50.487696+0000 mon.a (mon.0) 1997 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:50.493790+0000 mon.a (mon.0) 1998 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:50.493790+0000 mon.a (mon.0) 1998 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: cluster 2024-09-13T14:57:50.713911+0000 mgr.y (mgr.26252) 950 : cluster [DBG] pgmap v1473: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: cluster 2024-09-13T14:57:50.713911+0000 mgr.y (mgr.26252) 950 : cluster [DBG] pgmap v1473: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: cluster 2024-09-13T14:57:50.889675+0000 mon.a (mon.0) 1999 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: cluster 2024-09-13T14:57:50.889675+0000 mon.a (mon.0) 1999 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:51.083820+0000 mon.a (mon.0) 2000 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:51.083820+0000 mon.a (mon.0) 2000 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:51.084559+0000 mon.a (mon.0) 2001 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:51.084559+0000 mon.a (mon.0) 2001 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:51.089371+0000 mon.a (mon.0) 2002 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:51 smithi177 bash[68594]: audit 2024-09-13T14:57:51.089371+0000 mon.a (mon.0) 2002 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:57:51.399 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:51.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:51.502 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:51.509 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:51.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:51.636 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:51.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:51.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:51.792 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:51.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:51.895 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:51.922 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:51.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:52.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:52.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:52.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:52.154 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:52.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:52.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:52.277 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:52.305 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:52.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:52.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:52 smithi099 bash[84576]: cluster 2024-09-13T14:57:51.493326+0000 mon.a (mon.0) 2003 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:57:52.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:52 smithi099 bash[87147]: cluster 2024-09-13T14:57:51.493326+0000 mon.a (mon.0) 2003 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:57:52.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:52.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:52.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:52.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:52.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:52 smithi177 bash[68594]: cluster 2024-09-13T14:57:51.493326+0000 mon.a (mon.0) 2003 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:57:52.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:52 smithi177 bash[68594]: cluster 2024-09-13T14:57:51.493326+0000 mon.a (mon.0) 2003 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:57:52.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:52 smithi177 bash[68594]: cluster 2024-09-13T14:57:51.508374+0000 mon.a (mon.0) 2004 : cluster [DBG] osdmap e1034: 8 total, 7 up, 8 in 2024-09-13T14:57:52.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:52 smithi177 bash[68594]: cluster 2024-09-13T14:57:51.508374+0000 mon.a (mon.0) 2004 : cluster [DBG] osdmap e1034: 8 total, 7 up, 8 in 2024-09-13T14:57:52.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:52.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:52.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:52.699 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:52.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:52.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:52.828 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:52.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:52.867 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:52 smithi099 bash[87147]: cluster 2024-09-13T14:57:51.493326+0000 mon.a (mon.0) 2003 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:57:52.867 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:52 smithi099 bash[87147]: cluster 2024-09-13T14:57:51.508374+0000 mon.a (mon.0) 2004 : cluster [DBG] osdmap e1034: 8 total, 7 up, 8 in 2024-09-13T14:57:52.867 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:52 smithi099 bash[87147]: cluster 2024-09-13T14:57:51.508374+0000 mon.a (mon.0) 2004 : cluster [DBG] osdmap e1034: 8 total, 7 up, 8 in 2024-09-13T14:57:52.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:52 smithi099 bash[84576]: cluster 2024-09-13T14:57:51.493326+0000 mon.a (mon.0) 2003 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:57:52.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:52 smithi099 bash[84576]: cluster 2024-09-13T14:57:51.508374+0000 mon.a (mon.0) 2004 : cluster [DBG] osdmap e1034: 8 total, 7 up, 8 in 2024-09-13T14:57:52.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:52 smithi099 bash[84576]: cluster 2024-09-13T14:57:51.508374+0000 mon.a (mon.0) 2004 : cluster [DBG] osdmap e1034: 8 total, 7 up, 8 in 2024-09-13T14:57:52.932 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:52.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:52.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:53.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:53.092 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:53.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:53.194 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:53.223 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:53.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:53.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:53.353 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:53.457 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:53.484 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:53.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:53.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:53.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:53.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:53.730 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:53 smithi177 bash[68594]: cluster 2024-09-13T14:57:52.523550+0000 mon.a (mon.0) 2005 : cluster [DBG] osdmap e1035: 8 total, 7 up, 8 in 2024-09-13T14:57:53.730 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:53 smithi177 bash[68594]: cluster 2024-09-13T14:57:52.523550+0000 mon.a (mon.0) 2005 : cluster [DBG] osdmap e1035: 8 total, 7 up, 8 in 2024-09-13T14:57:53.730 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:53 smithi177 bash[68594]: cluster 2024-09-13T14:57:52.714519+0000 mgr.y (mgr.26252) 951 : cluster [DBG] pgmap v1476: 89 pgs: 1 activating+undersized+degraded, 5 active+undersized, 8 peering, 11 stale+active+clean, 64 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-13T14:57:53.731 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:53 smithi177 bash[68594]: cluster 2024-09-13T14:57:52.714519+0000 mgr.y (mgr.26252) 951 : cluster [DBG] pgmap v1476: 89 pgs: 1 activating+undersized+degraded, 5 active+undersized, 8 peering, 11 stale+active+clean, 64 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-13T14:57:53.747 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:53.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:53.795 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:53 smithi099 bash[84576]: cluster 2024-09-13T14:57:52.523550+0000 mon.a (mon.0) 2005 : cluster [DBG] osdmap e1035: 8 total, 7 up, 8 in 2024-09-13T14:57:53.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:53 smithi099 bash[84576]: cluster 2024-09-13T14:57:52.523550+0000 mon.a (mon.0) 2005 : cluster [DBG] osdmap e1035: 8 total, 7 up, 8 in 2024-09-13T14:57:53.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:53 smithi099 bash[84576]: cluster 2024-09-13T14:57:52.714519+0000 mgr.y (mgr.26252) 951 : cluster [DBG] pgmap v1476: 89 pgs: 1 activating+undersized+degraded, 5 active+undersized, 8 peering, 11 stale+active+clean, 64 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-13T14:57:53.796 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:53 smithi099 bash[84576]: cluster 2024-09-13T14:57:52.714519+0000 mgr.y (mgr.26252) 951 : cluster [DBG] pgmap v1476: 89 pgs: 1 activating+undersized+degraded, 5 active+undersized, 8 peering, 11 stale+active+clean, 64 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-13T14:57:53.796 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:53 smithi099 bash[87147]: cluster 2024-09-13T14:57:52.523550+0000 mon.a (mon.0) 2005 : cluster [DBG] osdmap e1035: 8 total, 7 up, 8 in 2024-09-13T14:57:53.796 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:53 smithi099 bash[87147]: cluster 2024-09-13T14:57:52.523550+0000 mon.a (mon.0) 2005 : cluster [DBG] osdmap e1035: 8 total, 7 up, 8 in 2024-09-13T14:57:53.796 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:53 smithi099 bash[87147]: cluster 2024-09-13T14:57:52.714519+0000 mgr.y (mgr.26252) 951 : cluster [DBG] pgmap v1476: 89 pgs: 1 activating+undersized+degraded, 5 active+undersized, 8 peering, 11 stale+active+clean, 64 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-13T14:57:53.797 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:53 smithi099 bash[87147]: cluster 2024-09-13T14:57:52.714519+0000 mgr.y (mgr.26252) 951 : cluster [DBG] pgmap v1476: 89 pgs: 1 activating+undersized+degraded, 5 active+undersized, 8 peering, 11 stale+active+clean, 64 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-09-13T14:57:53.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:53.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:53.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:54.009 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:54.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:54.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:54.138 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:54.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:54.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:54.270 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:54.373 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:54.401 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:54.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:54.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:54.531 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:54.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:54.634 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:54.645 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:54 smithi099 bash[84576]: cluster 2024-09-13T14:57:53.505930+0000 mon.a (mon.0) 2006 : cluster [WRN] Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:57:54.646 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:54 smithi099 bash[84576]: cluster 2024-09-13T14:57:53.505930+0000 mon.a (mon.0) 2006 : cluster [WRN] Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:57:54.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:54 smithi099 bash[87147]: cluster 2024-09-13T14:57:53.505930+0000 mon.a (mon.0) 2006 : cluster [WRN] Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:57:54.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:54 smithi099 bash[87147]: cluster 2024-09-13T14:57:53.505930+0000 mon.a (mon.0) 2006 : cluster [WRN] Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:57:54.660 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:54.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:54.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:54.774 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:54 smithi177 bash[68594]: cluster 2024-09-13T14:57:53.505930+0000 mon.a (mon.0) 2006 : cluster [WRN] Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:57:54.774 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:54 smithi177 bash[68594]: cluster 2024-09-13T14:57:53.505930+0000 mon.a (mon.0) 2006 : cluster [WRN] Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:57:54.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:54.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:54.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:54.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:54.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:55.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:55.052 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:55.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:55.155 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:55.179 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:55.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:55.282 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:55.311 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:55.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:55.322 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:57:55 smithi099 bash[165976]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-3 2024-09-13T14:57:55.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:55.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:55.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:55.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:55.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:55 smithi177 bash[68594]: cluster 2024-09-13T14:57:54.715245+0000 mgr.y (mgr.26252) 952 : cluster [DBG] pgmap v1477: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:55.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:55 smithi177 bash[68594]: cluster 2024-09-13T14:57:54.715245+0000 mgr.y (mgr.26252) 952 : cluster [DBG] pgmap v1477: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:55.571 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:55.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:55.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:55.700 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:55.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:55.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:55.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:55 smithi099 bash[84576]: cluster 2024-09-13T14:57:54.715245+0000 mgr.y (mgr.26252) 952 : cluster [DBG] pgmap v1477: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:55.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:55 smithi099 bash[84576]: cluster 2024-09-13T14:57:54.715245+0000 mgr.y (mgr.26252) 952 : cluster [DBG] pgmap v1477: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:55.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:55 smithi099 bash[87147]: cluster 2024-09-13T14:57:54.715245+0000 mgr.y (mgr.26252) 952 : cluster [DBG] pgmap v1477: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:55.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:55 smithi099 bash[87147]: cluster 2024-09-13T14:57:54.715245+0000 mgr.y (mgr.26252) 952 : cluster [DBG] pgmap v1477: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:55.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:55.934 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:55.960 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:55.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:56.063 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:56.091 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:56.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:56.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:56.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:56.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:56.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:56.352 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:56.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:56.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:56.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:56.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:56.584 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:56.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:56.713 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:56.718 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:57:56 smithi099 bash[166333]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-3 2024-09-13T14:57:56.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:56.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:56.842 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:56.871 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:56.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:56.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:56.997 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:56.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:57.099 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:57.127 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:57.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:57.258 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:57.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:57.361 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:57.388 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:57.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:57.486 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:57:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:57:57] "GET /metrics HTTP/1.1" 200 36714 "" "Prometheus/2.51.0" 2024-09-13T14:57:57.491 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:57.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:57.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:57.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:57.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:57.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:57.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:57.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:57.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:57.878 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:57.890 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:57 smithi177 bash[68594]: cluster 2024-09-13T14:57:56.715948+0000 mgr.y (mgr.26252) 953 : cluster [DBG] pgmap v1478: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:57.890 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:57 smithi177 bash[68594]: cluster 2024-09-13T14:57:56.715948+0000 mgr.y (mgr.26252) 953 : cluster [DBG] pgmap v1478: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:57.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:57.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:58.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:58.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:57 smithi099 bash[84576]: cluster 2024-09-13T14:57:56.715948+0000 mgr.y (mgr.26252) 953 : cluster [DBG] pgmap v1478: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:58.020 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:57 smithi099 bash[84576]: cluster 2024-09-13T14:57:56.715948+0000 mgr.y (mgr.26252) 953 : cluster [DBG] pgmap v1478: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:58.020 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:57 smithi099 bash[87147]: cluster 2024-09-13T14:57:56.715948+0000 mgr.y (mgr.26252) 953 : cluster [DBG] pgmap v1478: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:58.020 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:57 smithi099 bash[87147]: cluster 2024-09-13T14:57:56.715948+0000 mgr.y (mgr.26252) 953 : cluster [DBG] pgmap v1478: 89 pgs: 1 activating+undersized+degraded, 24 active+undersized, 8 peering, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:58.032 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:58.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:58.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.160 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:58.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.290 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:58.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:58.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:58.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:58.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:58.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:58.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:58.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:58.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:58.787 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:58.816 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:58.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:58.944 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:58.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:59.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:59.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:59.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:59.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:59.205 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:59.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:59.307 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:59.334 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:59.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:59.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:59.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:59.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:59.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:59.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:59.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:59.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:59.724 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:59.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:59.827 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:59.855 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:59.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:59.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:59.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:59 smithi177 bash[68594]: cluster 2024-09-13T14:57:58.716950+0000 mgr.y (mgr.26252) 954 : cluster [DBG] pgmap v1479: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:59.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:57:59 smithi177 bash[68594]: cluster 2024-09-13T14:57:58.716950+0000 mgr.y (mgr.26252) 954 : cluster [DBG] pgmap v1479: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:57:59.984 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:59.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:00.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:59 smithi099 bash[84576]: cluster 2024-09-13T14:57:58.716950+0000 mgr.y (mgr.26252) 954 : cluster [DBG] pgmap v1479: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:00.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:57:59 smithi099 bash[84576]: cluster 2024-09-13T14:57:58.716950+0000 mgr.y (mgr.26252) 954 : cluster [DBG] pgmap v1479: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:00.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:59 smithi099 bash[87147]: cluster 2024-09-13T14:57:58.716950+0000 mgr.y (mgr.26252) 954 : cluster [DBG] pgmap v1479: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:00.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:57:59 smithi099 bash[87147]: cluster 2024-09-13T14:57:58.716950+0000 mgr.y (mgr.26252) 954 : cluster [DBG] pgmap v1479: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:00.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:00.108 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:00.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:00.212 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:00.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:00.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:00.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:00.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:58:00 smithi177 bash[61439]: ts=2024-09-13T14:58:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:00.364 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:00.467 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:00.494 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:00.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:00.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:00.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:00.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:00.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:00.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:00.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:00.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:00.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:01.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:01.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:01.137 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:01.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:01.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:01.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:01.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:01.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:01.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:01.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:01.500 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:01.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:01.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:01.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:01.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:01.760 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:01.784 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:01.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:01.887 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:01.909 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:01.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:02.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:02.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:02.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:02.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:01 smithi177 bash[68594]: cluster 2024-09-13T14:58:00.718122+0000 mgr.y (mgr.26252) 955 : cluster [DBG] pgmap v1480: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:02.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:01 smithi177 bash[68594]: cluster 2024-09-13T14:58:00.718122+0000 mgr.y (mgr.26252) 955 : cluster [DBG] pgmap v1480: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:02.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:01 smithi177 bash[68594]: cluster 2024-09-13T14:58:01.425330+0000 mon.a (mon.0) 2007 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:02.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:01 smithi177 bash[68594]: cluster 2024-09-13T14:58:01.425330+0000 mon.a (mon.0) 2007 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:02.130 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:01 smithi099 bash[84576]: cluster 2024-09-13T14:58:00.718122+0000 mgr.y (mgr.26252) 955 : cluster [DBG] pgmap v1480: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:02.130 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:01 smithi099 bash[84576]: cluster 2024-09-13T14:58:00.718122+0000 mgr.y (mgr.26252) 955 : cluster [DBG] pgmap v1480: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:02.130 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:01 smithi099 bash[84576]: cluster 2024-09-13T14:58:01.425330+0000 mon.a (mon.0) 2007 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:02.130 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:01 smithi099 bash[84576]: cluster 2024-09-13T14:58:01.425330+0000 mon.a (mon.0) 2007 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:02.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:01 smithi099 bash[87147]: cluster 2024-09-13T14:58:00.718122+0000 mgr.y (mgr.26252) 955 : cluster [DBG] pgmap v1480: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:02.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:01 smithi099 bash[87147]: cluster 2024-09-13T14:58:00.718122+0000 mgr.y (mgr.26252) 955 : cluster [DBG] pgmap v1480: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:02.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:01 smithi099 bash[87147]: cluster 2024-09-13T14:58:01.425330+0000 mon.a (mon.0) 2007 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:02.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:01 smithi099 bash[87147]: cluster 2024-09-13T14:58:01.425330+0000 mon.a (mon.0) 2007 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:02.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:02.172 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:02.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:02.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:02.300 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:02.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:02.403 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:02.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:02.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:02.533 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:02.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:02.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:02.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:02.689 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:02.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:02.793 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:02.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:02.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:02.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:02.951 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:03.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:03.186 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:03.318 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:03.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:03.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:03.448 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:03.474 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:03.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:03.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:03.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:03.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:03.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:03.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:03.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:03 smithi177 bash[68594]: cluster 2024-09-13T14:58:02.718763+0000 mgr.y (mgr.26252) 956 : cluster [DBG] pgmap v1481: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:03.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:03 smithi177 bash[68594]: cluster 2024-09-13T14:58:02.718763+0000 mgr.y (mgr.26252) 956 : cluster [DBG] pgmap v1481: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:03.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:03 smithi177 bash[68594]: audit 2024-09-13T14:58:03.293678+0000 mon.a (mon.0) 2008 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:03.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:03 smithi177 bash[68594]: audit 2024-09-13T14:58:03.293678+0000 mon.a (mon.0) 2008 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:03.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:03 smithi177 bash[68594]: audit 2024-09-13T14:58:03.295877+0000 mon.a (mon.0) 2009 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:03.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:03 smithi177 bash[68594]: audit 2024-09-13T14:58:03.295877+0000 mon.a (mon.0) 2009 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:03.864 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:03.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:03.967 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:03.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:04.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[84576]: cluster 2024-09-13T14:58:02.718763+0000 mgr.y (mgr.26252) 956 : cluster [DBG] pgmap v1481: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:04.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[84576]: cluster 2024-09-13T14:58:02.718763+0000 mgr.y (mgr.26252) 956 : cluster [DBG] pgmap v1481: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:04.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[84576]: audit 2024-09-13T14:58:03.293678+0000 mon.a (mon.0) 2008 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:04.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[84576]: audit 2024-09-13T14:58:03.293678+0000 mon.a (mon.0) 2008 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:04.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[84576]: audit 2024-09-13T14:58:03.295877+0000 mon.a (mon.0) 2009 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:04.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[84576]: audit 2024-09-13T14:58:03.295877+0000 mon.a (mon.0) 2009 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:04.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[87147]: cluster 2024-09-13T14:58:02.718763+0000 mgr.y (mgr.26252) 956 : cluster [DBG] pgmap v1481: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:04.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[87147]: cluster 2024-09-13T14:58:02.718763+0000 mgr.y (mgr.26252) 956 : cluster [DBG] pgmap v1481: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:04.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[87147]: audit 2024-09-13T14:58:03.293678+0000 mon.a (mon.0) 2008 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:04.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[87147]: audit 2024-09-13T14:58:03.293678+0000 mon.a (mon.0) 2008 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:04.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[87147]: audit 2024-09-13T14:58:03.295877+0000 mon.a (mon.0) 2009 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:04.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:03 smithi099 bash[87147]: audit 2024-09-13T14:58:03.295877+0000 mon.a (mon.0) 2009 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:04.097 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:04.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:04.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:04.227 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:04.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:04.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:04.356 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:04.382 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:04.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:04.485 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:04.513 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:04.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:04.616 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:04.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:04.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:04.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:04.766 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:04.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:04.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:04.896 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:04.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:04.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:05.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:05.128 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:05.156 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:05.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:05.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:05.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:05.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:05.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:05.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:05.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:05.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:05.543 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:05.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:05.590 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:58:05.590 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:05.607 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:58:05.607 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:05.624 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:58:05.624 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:05.625 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T14:58:05.645 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:58:05.645 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:05.647 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:05.665 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:58:05.666 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:05.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:05.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:05.685 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:58:05.686 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:05.705 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:58:05.705 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:05.771 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:05.797 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:05.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:05.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:05.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:05.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:05.937 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:05 smithi099 bash[84576]: cluster 2024-09-13T14:58:04.719329+0000 mgr.y (mgr.26252) 957 : cluster [DBG] pgmap v1482: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:05.937 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:05 smithi099 bash[84576]: cluster 2024-09-13T14:58:04.719329+0000 mgr.y (mgr.26252) 957 : cluster [DBG] pgmap v1482: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:05.937 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:05 smithi099 bash[84576]: audit 2024-09-13T14:58:05.572362+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.99:0/3394293195' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:05.937 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:05 smithi099 bash[84576]: audit 2024-09-13T14:58:05.572362+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.99:0/3394293195' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:05.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:05 smithi099 bash[87147]: cluster 2024-09-13T14:58:04.719329+0000 mgr.y (mgr.26252) 957 : cluster [DBG] pgmap v1482: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:05.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:05 smithi099 bash[87147]: cluster 2024-09-13T14:58:04.719329+0000 mgr.y (mgr.26252) 957 : cluster [DBG] pgmap v1482: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:05.938 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:05 smithi099 bash[87147]: audit 2024-09-13T14:58:05.572362+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.99:0/3394293195' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:05.938 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:05 smithi099 bash[87147]: audit 2024-09-13T14:58:05.572362+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.99:0/3394293195' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:06.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:06.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:06.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:05 smithi177 bash[68594]: cluster 2024-09-13T14:58:04.719329+0000 mgr.y (mgr.26252) 957 : cluster [DBG] pgmap v1482: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:06.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:05 smithi177 bash[68594]: cluster 2024-09-13T14:58:04.719329+0000 mgr.y (mgr.26252) 957 : cluster [DBG] pgmap v1482: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:06.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:05 smithi177 bash[68594]: audit 2024-09-13T14:58:05.572362+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.99:0/3394293195' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:06.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:05 smithi177 bash[68594]: audit 2024-09-13T14:58:05.572362+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.99:0/3394293195' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:06.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:06.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:06.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:06.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:06.321 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:06.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:06.424 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:06.451 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:06.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:06.580 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:06.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:06.683 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:06.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:06.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:06.814 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:06.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:06.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:06.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:06.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:06.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:07.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:07.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:07.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:07.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:07.230 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:07.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:07.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:58:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:58:07] "GET /metrics HTTP/1.1" 200 36716 "" "Prometheus/2.51.0" 2024-09-13T14:58:07.333 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:07.361 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:07.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:07.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:07.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:07.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:07.623 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:07.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:07.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:07.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:07.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:07.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:07.867 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:07 smithi099 bash[84576]: cluster 2024-09-13T14:58:06.719924+0000 mgr.y (mgr.26252) 958 : cluster [DBG] pgmap v1483: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:07.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:07 smithi099 bash[84576]: cluster 2024-09-13T14:58:06.719924+0000 mgr.y (mgr.26252) 958 : cluster [DBG] pgmap v1483: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:07.868 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:07 smithi099 bash[87147]: cluster 2024-09-13T14:58:06.719924+0000 mgr.y (mgr.26252) 958 : cluster [DBG] pgmap v1483: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:07.868 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:07 smithi099 bash[87147]: cluster 2024-09-13T14:58:06.719924+0000 mgr.y (mgr.26252) 958 : cluster [DBG] pgmap v1483: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:07.885 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:07.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:07.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:07.998 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:07 smithi177 bash[68594]: cluster 2024-09-13T14:58:06.719924+0000 mgr.y (mgr.26252) 958 : cluster [DBG] pgmap v1483: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:07.999 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:07 smithi177 bash[68594]: cluster 2024-09-13T14:58:06.719924+0000 mgr.y (mgr.26252) 958 : cluster [DBG] pgmap v1483: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:08.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:08.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:08.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:08.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:08.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:08.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:08.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:08.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:08.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:08.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:08.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:08.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:08.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:08.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:08.638 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:08.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:08.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:08.768 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:08.792 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:08.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:08.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:08.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:08.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:09.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:09.057 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:09.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:09.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:09.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:09.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:09.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:09.319 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:09.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:09.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:09.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:09.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:09.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:09.577 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:09.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T14:58:09.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:09.682 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:09.709 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:09.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:09.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:09.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:09 smithi099 bash[84576]: cluster 2024-09-13T14:58:08.720477+0000 mgr.y (mgr.26252) 959 : cluster [DBG] pgmap v1484: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:09.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:09 smithi099 bash[84576]: cluster 2024-09-13T14:58:08.720477+0000 mgr.y (mgr.26252) 959 : cluster [DBG] pgmap v1484: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:09 smithi099 bash[87147]: cluster 2024-09-13T14:58:08.720477+0000 mgr.y (mgr.26252) 959 : cluster [DBG] pgmap v1484: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:09.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:09 smithi099 bash[87147]: cluster 2024-09-13T14:58:08.720477+0000 mgr.y (mgr.26252) 959 : cluster [DBG] pgmap v1484: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:09.824 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:58:09 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T14:58:09.824 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 14:58:09 smithi099 systemd[1]: Stopped Ceph osd.3 for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T14:58:09.840 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:09.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:09.920 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:58:09.920 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T14:58:09.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:09.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:09.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:10.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:10.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:10.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:10.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:09 smithi177 bash[68594]: cluster 2024-09-13T14:58:08.720477+0000 mgr.y (mgr.26252) 959 : cluster [DBG] pgmap v1484: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:10.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:09 smithi177 bash[68594]: cluster 2024-09-13T14:58:08.720477+0000 mgr.y (mgr.26252) 959 : cluster [DBG] pgmap v1484: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:10.205 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:10.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:10.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:10.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:10.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:10.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:10.482 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:10.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:10.495 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:58:10.496 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:58:10.505 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:10.505 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:10.586 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:10.597 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:58:10 smithi177 bash[61439]: ts=2024-09-13T14:58:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:10.614 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:10.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:10.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:10.745 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:10.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:10.848 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:10.875 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:10.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:10.979 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:11.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:11.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:11.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:11.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:11.265 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:11.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:11.368 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:11.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:11.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:11.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:11.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:11.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:11.657 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:11.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:11.761 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:11.788 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:11.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:11.902 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:11 smithi099 bash[84576]: cluster 2024-09-13T14:58:10.721793+0000 mgr.y (mgr.26252) 960 : cluster [DBG] pgmap v1485: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:11.952 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:11 smithi099 bash[84576]: cluster 2024-09-13T14:58:10.721793+0000 mgr.y (mgr.26252) 960 : cluster [DBG] pgmap v1485: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:11.953 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:11 smithi099 bash[87147]: cluster 2024-09-13T14:58:10.721793+0000 mgr.y (mgr.26252) 960 : cluster [DBG] pgmap v1485: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:11.953 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:11 smithi099 bash[87147]: cluster 2024-09-13T14:58:10.721793+0000 mgr.y (mgr.26252) 960 : cluster [DBG] pgmap v1485: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:11.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.954 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:11.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:12.055 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:12.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:12.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:12.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:11 smithi177 bash[68594]: cluster 2024-09-13T14:58:10.721793+0000 mgr.y (mgr.26252) 960 : cluster [DBG] pgmap v1485: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:12.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:11 smithi177 bash[68594]: cluster 2024-09-13T14:58:10.721793+0000 mgr.y (mgr.26252) 960 : cluster [DBG] pgmap v1485: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:12.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:12.225 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:12.328 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:12.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:12.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:12.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:12.477 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:12.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:12.497 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:58:12.580 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:12.607 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:12.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:12.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:12.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:12.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:12.808 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_1' removed 2024-09-13T14:58:12.844 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:12.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:12 smithi177 bash[68594]: audit 2024-09-13T14:58:12.446150+0000 mon.a (mon.0) 2010 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:12.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:12 smithi177 bash[68594]: audit 2024-09-13T14:58:12.446150+0000 mon.a (mon.0) 2010 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:12.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:12.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:12.991 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:13.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:13.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:13.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:12 smithi099 bash[84576]: audit 2024-09-13T14:58:12.446150+0000 mon.a (mon.0) 2010 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:13.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:12 smithi099 bash[84576]: audit 2024-09-13T14:58:12.446150+0000 mon.a (mon.0) 2010 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:13.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:12 smithi099 bash[87147]: audit 2024-09-13T14:58:12.446150+0000 mon.a (mon.0) 2010 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:13.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:12 smithi099 bash[87147]: audit 2024-09-13T14:58:12.446150+0000 mon.a (mon.0) 2010 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:58:13.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:13.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:13.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:13.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:13.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:13.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:13.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:13.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:13.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:13.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:13.537 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:13.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:13.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:13.667 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:13.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:13.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:13.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:13.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:13.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:58:13.907 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:13.930 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:13.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:14.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:14.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:14.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:14.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:13 smithi177 bash[68594]: cluster 2024-09-13T14:58:12.722357+0000 mgr.y (mgr.26252) 961 : cluster [DBG] pgmap v1486: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:14.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:13 smithi177 bash[68594]: cluster 2024-09-13T14:58:12.722357+0000 mgr.y (mgr.26252) 961 : cluster [DBG] pgmap v1486: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:14.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:13 smithi177 bash[68594]: audit 2024-09-13T14:58:12.809339+0000 mon.a (mon.0) 2011 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:58:14.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:13 smithi177 bash[68594]: audit 2024-09-13T14:58:12.809339+0000 mon.a (mon.0) 2011 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:58:14.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:13 smithi177 bash[68594]: cluster 2024-09-13T14:58:12.817751+0000 mon.a (mon.0) 2012 : cluster [DBG] osdmap e1036: 8 total, 7 up, 8 in 2024-09-13T14:58:14.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:13 smithi177 bash[68594]: cluster 2024-09-13T14:58:12.817751+0000 mon.a (mon.0) 2012 : cluster [DBG] osdmap e1036: 8 total, 7 up, 8 in 2024-09-13T14:58:14.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:14.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:14.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:14.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[87147]: cluster 2024-09-13T14:58:12.722357+0000 mgr.y (mgr.26252) 961 : cluster [DBG] pgmap v1486: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:14.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[87147]: cluster 2024-09-13T14:58:12.722357+0000 mgr.y (mgr.26252) 961 : cluster [DBG] pgmap v1486: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:14.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[87147]: audit 2024-09-13T14:58:12.809339+0000 mon.a (mon.0) 2011 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:58:14.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[87147]: audit 2024-09-13T14:58:12.809339+0000 mon.a (mon.0) 2011 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:58:14.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[87147]: cluster 2024-09-13T14:58:12.817751+0000 mon.a (mon.0) 2012 : cluster [DBG] osdmap e1036: 8 total, 7 up, 8 in 2024-09-13T14:58:14.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[87147]: cluster 2024-09-13T14:58:12.817751+0000 mon.a (mon.0) 2012 : cluster [DBG] osdmap e1036: 8 total, 7 up, 8 in 2024-09-13T14:58:14.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[84576]: cluster 2024-09-13T14:58:12.722357+0000 mgr.y (mgr.26252) 961 : cluster [DBG] pgmap v1486: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:14.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[84576]: cluster 2024-09-13T14:58:12.722357+0000 mgr.y (mgr.26252) 961 : cluster [DBG] pgmap v1486: 89 pgs: 32 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:14.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[84576]: audit 2024-09-13T14:58:12.809339+0000 mon.a (mon.0) 2011 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:58:14.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[84576]: audit 2024-09-13T14:58:12.809339+0000 mon.a (mon.0) 2011 : audit [INF] from='client.? 172.21.15.99:0/2693034421' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:58:14.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[84576]: cluster 2024-09-13T14:58:12.817751+0000 mon.a (mon.0) 2012 : cluster [DBG] osdmap e1036: 8 total, 7 up, 8 in 2024-09-13T14:58:14.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:13 smithi099 bash[84576]: cluster 2024-09-13T14:58:12.817751+0000 mon.a (mon.0) 2012 : cluster [DBG] osdmap e1036: 8 total, 7 up, 8 in 2024-09-13T14:58:14.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:14.315 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:14.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:14.418 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:14.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:14.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:14.552 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:14.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:14.680 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:14.707 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:14.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:14.781 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:58:14.781 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:58:14.782 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-09-13T14:58:14.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-09-13T14:58:14.810 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:14.819 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T14:58:14.822 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:14 smithi177 bash[68594]: audit 2024-09-13T14:58:14.244873+0000 mon.a (mon.0) 2013 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:14.822 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:14 smithi177 bash[68594]: audit 2024-09-13T14:58:14.244873+0000 mon.a (mon.0) 2013 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:14.835 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:14.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:14.920 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T14:58:14.920 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:58:14.920 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-13T14:58:14.921 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:58:14.921 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:58:14.921 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-13T14:58:14.939 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:14.988 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:15.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:14 smithi099 bash[84576]: audit 2024-09-13T14:58:14.244873+0000 mon.a (mon.0) 2013 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:15.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:14 smithi099 bash[84576]: audit 2024-09-13T14:58:14.244873+0000 mon.a (mon.0) 2013 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:15.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:14 smithi099 bash[87147]: audit 2024-09-13T14:58:14.244873+0000 mon.a (mon.0) 2013 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:15.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:14 smithi099 bash[87147]: audit 2024-09-13T14:58:14.244873+0000 mon.a (mon.0) 2013 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:58:15.092 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.117 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:15.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:15.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:15.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:15.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:15.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:15.507 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:15.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:15.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:15.740 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:15.768 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:15.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:15.871 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:15.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:15.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:16.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:16.029 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:16.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:16.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[84576]: cluster 2024-09-13T14:58:14.723260+0000 mgr.y (mgr.26252) 962 : cluster [DBG] pgmap v1488: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:16.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[84576]: cluster 2024-09-13T14:58:14.723260+0000 mgr.y (mgr.26252) 962 : cluster [DBG] pgmap v1488: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:16.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[84576]: cluster 2024-09-13T14:58:14.812056+0000 mon.a (mon.0) 2014 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:16.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[84576]: cluster 2024-09-13T14:58:14.812056+0000 mon.a (mon.0) 2014 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:16.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[84576]: audit 2024-09-13T14:58:14.820226+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:16.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[84576]: audit 2024-09-13T14:58:14.820226+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:16.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[84576]: cluster 2024-09-13T14:58:14.825247+0000 mon.a (mon.0) 2016 : cluster [DBG] osdmap e1037: 8 total, 7 up, 8 in 2024-09-13T14:58:16.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[84576]: cluster 2024-09-13T14:58:14.825247+0000 mon.a (mon.0) 2016 : cluster [DBG] osdmap e1037: 8 total, 7 up, 8 in 2024-09-13T14:58:16.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[87147]: cluster 2024-09-13T14:58:14.723260+0000 mgr.y (mgr.26252) 962 : cluster [DBG] pgmap v1488: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:16.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[87147]: cluster 2024-09-13T14:58:14.723260+0000 mgr.y (mgr.26252) 962 : cluster [DBG] pgmap v1488: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:16.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[87147]: cluster 2024-09-13T14:58:14.812056+0000 mon.a (mon.0) 2014 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:16.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[87147]: cluster 2024-09-13T14:58:14.812056+0000 mon.a (mon.0) 2014 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:16.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[87147]: audit 2024-09-13T14:58:14.820226+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:16.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[87147]: audit 2024-09-13T14:58:14.820226+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:16.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[87147]: cluster 2024-09-13T14:58:14.825247+0000 mon.a (mon.0) 2016 : cluster [DBG] osdmap e1037: 8 total, 7 up, 8 in 2024-09-13T14:58:16.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:15 smithi099 bash[87147]: cluster 2024-09-13T14:58:14.825247+0000 mon.a (mon.0) 2016 : cluster [DBG] osdmap e1037: 8 total, 7 up, 8 in 2024-09-13T14:58:16.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:15 smithi177 bash[68594]: cluster 2024-09-13T14:58:14.723260+0000 mgr.y (mgr.26252) 962 : cluster [DBG] pgmap v1488: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:16.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:15 smithi177 bash[68594]: cluster 2024-09-13T14:58:14.723260+0000 mgr.y (mgr.26252) 962 : cluster [DBG] pgmap v1488: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:16.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:15 smithi177 bash[68594]: cluster 2024-09-13T14:58:14.812056+0000 mon.a (mon.0) 2014 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:16.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:15 smithi177 bash[68594]: cluster 2024-09-13T14:58:14.812056+0000 mon.a (mon.0) 2014 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:16.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:15 smithi177 bash[68594]: audit 2024-09-13T14:58:14.820226+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:16.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:15 smithi177 bash[68594]: audit 2024-09-13T14:58:14.820226+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? 172.21.15.99:0/2948984500' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:58:16.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:15 smithi177 bash[68594]: cluster 2024-09-13T14:58:14.825247+0000 mon.a (mon.0) 2016 : cluster [DBG] osdmap e1037: 8 total, 7 up, 8 in 2024-09-13T14:58:16.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:15 smithi177 bash[68594]: cluster 2024-09-13T14:58:14.825247+0000 mon.a (mon.0) 2016 : cluster [DBG] osdmap e1037: 8 total, 7 up, 8 in 2024-09-13T14:58:16.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:16.158 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:16.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:16.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:16.288 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:16.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:16.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:16.419 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:16.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:16.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:16.551 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:16.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:16.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:16.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:16.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:16.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:16.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:16.917 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:16.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:16.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:17.046 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:17.073 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:17.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:17.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:17.205 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:17.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:17.308 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:17.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:58:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:58:17] "GET /metrics HTTP/1.1" 200 36716 "" "Prometheus/2.51.0" 2024-09-13T14:58:17.336 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:17.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:17.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:17.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:17.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:17.589 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:17.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:17.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:17.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:17.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:17.822 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:17.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:17.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:17.932 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:58:17.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:17.993 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:17 smithi099 bash[84576]: cluster 2024-09-13T14:58:16.723966+0000 mgr.y (mgr.26252) 963 : cluster [DBG] pgmap v1490: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:17.994 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:17 smithi099 bash[84576]: cluster 2024-09-13T14:58:16.723966+0000 mgr.y (mgr.26252) 963 : cluster [DBG] pgmap v1490: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:17.994 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:17 smithi099 bash[87147]: cluster 2024-09-13T14:58:16.723966+0000 mgr.y (mgr.26252) 963 : cluster [DBG] pgmap v1490: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:17.994 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:17 smithi099 bash[87147]: cluster 2024-09-13T14:58:16.723966+0000 mgr.y (mgr.26252) 963 : cluster [DBG] pgmap v1490: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:18.006 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:18.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:18.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:18.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:17 smithi177 bash[68594]: cluster 2024-09-13T14:58:16.723966+0000 mgr.y (mgr.26252) 963 : cluster [DBG] pgmap v1490: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:18.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:17 smithi177 bash[68594]: cluster 2024-09-13T14:58:16.723966+0000 mgr.y (mgr.26252) 963 : cluster [DBG] pgmap v1490: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:18.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:18.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:18.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:18.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:18.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:18.403 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:18.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:18.430 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:18.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:18.530 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:18.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:18.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:18.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:18.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:18.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:18.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:18.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:18.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:18.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:19.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:19.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:18 smithi177 bash[68594]: audit 2024-09-13T14:58:18.287356+0000 mon.a (mon.0) 2017 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:19.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:18 smithi177 bash[68594]: audit 2024-09-13T14:58:18.287356+0000 mon.a (mon.0) 2017 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:19.055 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:19.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:19.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:18 smithi099 bash[84576]: audit 2024-09-13T14:58:18.287356+0000 mon.a (mon.0) 2017 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:19.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:18 smithi099 bash[84576]: audit 2024-09-13T14:58:18.287356+0000 mon.a (mon.0) 2017 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:19.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:18 smithi099 bash[87147]: audit 2024-09-13T14:58:18.287356+0000 mon.a (mon.0) 2017 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:19.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:18 smithi099 bash[87147]: audit 2024-09-13T14:58:18.287356+0000 mon.a (mon.0) 2017 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:19.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:19.185 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:19.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:19.289 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:19.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:19.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:19.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:19.441 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:19.442 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:19.446 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:19.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:19.576 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:19.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:19.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:19.689 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:19.704 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:19.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:19.807 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:19.835 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:19.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:19.970 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:19.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:20.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:20.084 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:19 smithi177 bash[68594]: cluster 2024-09-13T14:58:18.724911+0000 mgr.y (mgr.26252) 964 : cluster [DBG] pgmap v1491: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:20.084 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:19 smithi177 bash[68594]: cluster 2024-09-13T14:58:18.724911+0000 mgr.y (mgr.26252) 964 : cluster [DBG] pgmap v1491: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:20.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:20.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:20.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:20.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:20.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:19 smithi099 bash[84576]: cluster 2024-09-13T14:58:18.724911+0000 mgr.y (mgr.26252) 964 : cluster [DBG] pgmap v1491: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:20.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:19 smithi099 bash[84576]: cluster 2024-09-13T14:58:18.724911+0000 mgr.y (mgr.26252) 964 : cluster [DBG] pgmap v1491: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:20.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:19 smithi099 bash[87147]: cluster 2024-09-13T14:58:18.724911+0000 mgr.y (mgr.26252) 964 : cluster [DBG] pgmap v1491: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:20.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:19 smithi099 bash[87147]: cluster 2024-09-13T14:58:18.724911+0000 mgr.y (mgr.26252) 964 : cluster [DBG] pgmap v1491: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:20.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:20.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:20.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:20.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:58:20 smithi177 bash[61439]: ts=2024-09-13T14:58:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:20.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:20.488 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:20.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:20.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:20.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:20.721 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:20.749 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:20.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:20.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:20.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:20.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:20.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.009 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:21.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:21.139 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:21.242 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:21.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:21.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:21.376 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:21.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:21.499 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:21.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:21.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:21.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:21.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:21.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:21.753 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.774 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:21.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:22.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:22.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:21 smithi177 bash[68594]: cluster 2024-09-13T14:58:20.726222+0000 mgr.y (mgr.26252) 965 : cluster [DBG] pgmap v1492: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:22.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:21 smithi177 bash[68594]: cluster 2024-09-13T14:58:20.726222+0000 mgr.y (mgr.26252) 965 : cluster [DBG] pgmap v1492: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:22.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:22.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:22.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:21 smithi099 bash[84576]: cluster 2024-09-13T14:58:20.726222+0000 mgr.y (mgr.26252) 965 : cluster [DBG] pgmap v1492: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:22.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:21 smithi099 bash[84576]: cluster 2024-09-13T14:58:20.726222+0000 mgr.y (mgr.26252) 965 : cluster [DBG] pgmap v1492: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:22.151 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:21 smithi099 bash[87147]: cluster 2024-09-13T14:58:20.726222+0000 mgr.y (mgr.26252) 965 : cluster [DBG] pgmap v1492: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:22.151 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:21 smithi099 bash[87147]: cluster 2024-09-13T14:58:20.726222+0000 mgr.y (mgr.26252) 965 : cluster [DBG] pgmap v1492: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:22.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:22.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:22.294 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:22.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:22.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:22.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:22.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:22.547 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:22.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:22.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:22.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:22.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:22.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:22.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:22.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:22.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:22.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:23.043 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:23.070 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:23.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:23.204 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:23.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:23.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:23.335 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:23.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:23.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:23.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.469 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:23.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:23.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:23.597 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:23.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:23.700 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:23.726 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:23.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:23.828 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:23.856 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:23.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:23.959 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:23.983 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:23.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:24.086 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:24.094 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:23 smithi099 bash[84576]: cluster 2024-09-13T14:58:22.726820+0000 mgr.y (mgr.26252) 966 : cluster [DBG] pgmap v1493: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:24.095 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:23 smithi099 bash[84576]: cluster 2024-09-13T14:58:22.726820+0000 mgr.y (mgr.26252) 966 : cluster [DBG] pgmap v1493: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:24.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:23 smithi099 bash[87147]: cluster 2024-09-13T14:58:22.726820+0000 mgr.y (mgr.26252) 966 : cluster [DBG] pgmap v1493: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:24.095 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:23 smithi099 bash[87147]: cluster 2024-09-13T14:58:22.726820+0000 mgr.y (mgr.26252) 966 : cluster [DBG] pgmap v1493: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:24.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:24.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:23 smithi177 bash[68594]: cluster 2024-09-13T14:58:22.726820+0000 mgr.y (mgr.26252) 966 : cluster [DBG] pgmap v1493: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:24.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:23 smithi177 bash[68594]: cluster 2024-09-13T14:58:22.726820+0000 mgr.y (mgr.26252) 966 : cluster [DBG] pgmap v1493: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:24.149 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T14:58:24.149 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:58:24.149 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T14:58:24.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:24.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:24.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:24.259 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:24.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:24.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:24.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:24.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:24.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:24.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:24.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:24.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:24.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:24.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:24.747 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:24.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:24.850 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:24.876 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:24.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:24.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:25.009 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:25.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:25.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:25.140 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:25.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:25.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:25.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:25.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:25.375 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:25.402 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:25.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:25.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:25.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:25.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:25.637 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:25.666 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:25.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:25.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:25.792 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:25.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:25.895 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:25.922 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:25.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:26.025 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:26.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:25 smithi099 bash[84576]: cluster 2024-09-13T14:58:24.727512+0000 mgr.y (mgr.26252) 967 : cluster [DBG] pgmap v1494: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:26.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:25 smithi099 bash[84576]: cluster 2024-09-13T14:58:24.727512+0000 mgr.y (mgr.26252) 967 : cluster [DBG] pgmap v1494: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:26.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:25 smithi099 bash[87147]: cluster 2024-09-13T14:58:24.727512+0000 mgr.y (mgr.26252) 967 : cluster [DBG] pgmap v1494: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:26.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:25 smithi099 bash[87147]: cluster 2024-09-13T14:58:24.727512+0000 mgr.y (mgr.26252) 967 : cluster [DBG] pgmap v1494: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:25 smithi177 bash[68594]: cluster 2024-09-13T14:58:24.727512+0000 mgr.y (mgr.26252) 967 : cluster [DBG] pgmap v1494: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:25 smithi177 bash[68594]: cluster 2024-09-13T14:58:24.727512+0000 mgr.y (mgr.26252) 967 : cluster [DBG] pgmap v1494: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:26.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:26.185 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:26.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:26.289 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:26.419 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:26.447 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:26.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:26.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.579 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:26.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:26.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:26.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:26.811 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:26.939 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.966 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:27.070 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:27.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:27.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:27.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:27.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:27.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:27.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:58:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:58:27] "GET /metrics HTTP/1.1" 200 34346 "" "Prometheus/2.51.0" 2024-09-13T14:58:27.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:27.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:27.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:27.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:27.487 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:27.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:27.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:27.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:27.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:27.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:27.734 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:27 smithi177 bash[68594]: cluster 2024-09-13T14:58:26.728198+0000 mgr.y (mgr.26252) 968 : cluster [DBG] pgmap v1495: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:27.734 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:27 smithi177 bash[68594]: cluster 2024-09-13T14:58:26.728198+0000 mgr.y (mgr.26252) 968 : cluster [DBG] pgmap v1495: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:27.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:27.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:27.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:27 smithi099 bash[87147]: cluster 2024-09-13T14:58:26.728198+0000 mgr.y (mgr.26252) 968 : cluster [DBG] pgmap v1495: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:27.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:27 smithi099 bash[87147]: cluster 2024-09-13T14:58:26.728198+0000 mgr.y (mgr.26252) 968 : cluster [DBG] pgmap v1495: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:27.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:27 smithi099 bash[84576]: cluster 2024-09-13T14:58:26.728198+0000 mgr.y (mgr.26252) 968 : cluster [DBG] pgmap v1495: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:27.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:27 smithi099 bash[84576]: cluster 2024-09-13T14:58:26.728198+0000 mgr.y (mgr.26252) 968 : cluster [DBG] pgmap v1495: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:27.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:27.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:27.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:28.012 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:28.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:28.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:28.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:28.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:28.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:28.377 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:28.402 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:28.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:28.505 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:28.532 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:28.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:28.636 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.664 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:28.767 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:28.794 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:28.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:28.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:29.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:29.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.057 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:29.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:29.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:29.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:29.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:29.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.314 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:29.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.444 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:29.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:29.574 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:29.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:29.705 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:29.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:29.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:29 smithi099 bash[87147]: cluster 2024-09-13T14:58:28.729143+0000 mgr.y (mgr.26252) 969 : cluster [DBG] pgmap v1496: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:29.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:29 smithi099 bash[87147]: cluster 2024-09-13T14:58:28.729143+0000 mgr.y (mgr.26252) 969 : cluster [DBG] pgmap v1496: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:29.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:29 smithi099 bash[84576]: cluster 2024-09-13T14:58:28.729143+0000 mgr.y (mgr.26252) 969 : cluster [DBG] pgmap v1496: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:29.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:29 smithi099 bash[84576]: cluster 2024-09-13T14:58:28.729143+0000 mgr.y (mgr.26252) 969 : cluster [DBG] pgmap v1496: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:29.835 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:29.939 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.967 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:29.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:30.039 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T14:58:30.039 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:58:30.070 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:30.081 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:29 smithi177 bash[68594]: cluster 2024-09-13T14:58:28.729143+0000 mgr.y (mgr.26252) 969 : cluster [DBG] pgmap v1496: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:30.081 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:29 smithi177 bash[68594]: cluster 2024-09-13T14:58:28.729143+0000 mgr.y (mgr.26252) 969 : cluster [DBG] pgmap v1496: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:30.097 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:30.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:30.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:30.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:30.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:30.273 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:30.273 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:58:30.291 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:58:30.291 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:30.307 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:58:30.308 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:30.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T14:58:30.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:30.329 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:58:30.329 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:30.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:30.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:30.350 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:30.350 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:58:30.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:58:30 smithi177 bash[61439]: ts=2024-09-13T14:58:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:30.369 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:58:30.370 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:30.391 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:58:30.391 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:30.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:30.473 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:30.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:30.576 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:30.597 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:30.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:30.699 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:30.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:30.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:30.830 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:30.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:30.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:30.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:30.971 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:30 smithi099 bash[84576]: audit 2024-09-13T14:58:30.255943+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.99:0/1092461127' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:30.971 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:30 smithi099 bash[84576]: audit 2024-09-13T14:58:30.255943+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.99:0/1092461127' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:30.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:30 smithi099 bash[87147]: audit 2024-09-13T14:58:30.255943+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.99:0/1092461127' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:30.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:30 smithi099 bash[87147]: audit 2024-09-13T14:58:30.255943+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.99:0/1092461127' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:30.986 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:30.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:31.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:31.100 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:30 smithi177 bash[68594]: audit 2024-09-13T14:58:30.255943+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.99:0/1092461127' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:31.100 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:30 smithi177 bash[68594]: audit 2024-09-13T14:58:30.255943+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.99:0/1092461127' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:31.115 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:31.218 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:31.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:31.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:31.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:31.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:31.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:31.504 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:31.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:31.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:31.635 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:31.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:31.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:31.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:31.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:31.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:31.877 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:31 smithi177 bash[68594]: cluster 2024-09-13T14:58:30.730299+0000 mgr.y (mgr.26252) 970 : cluster [DBG] pgmap v1497: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:31.877 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:31 smithi177 bash[68594]: cluster 2024-09-13T14:58:30.730299+0000 mgr.y (mgr.26252) 970 : cluster [DBG] pgmap v1497: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:31.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:31.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:31.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:32.007 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:31 smithi099 bash[84576]: cluster 2024-09-13T14:58:30.730299+0000 mgr.y (mgr.26252) 970 : cluster [DBG] pgmap v1497: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:32.007 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:31 smithi099 bash[84576]: cluster 2024-09-13T14:58:30.730299+0000 mgr.y (mgr.26252) 970 : cluster [DBG] pgmap v1497: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:32.007 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:31 smithi099 bash[87147]: cluster 2024-09-13T14:58:30.730299+0000 mgr.y (mgr.26252) 970 : cluster [DBG] pgmap v1497: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:32.007 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:31 smithi099 bash[87147]: cluster 2024-09-13T14:58:30.730299+0000 mgr.y (mgr.26252) 970 : cluster [DBG] pgmap v1497: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:32.022 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:32.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:32.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:32.153 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:32.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:32.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:32.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:32.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:32.388 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:32.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:32.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:32.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:32.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:32.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:32.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:32.674 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:32.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:32.777 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:32.805 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:32.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:32.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:32.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:33.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:33.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:33.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:33.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:33.194 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:33.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:33.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:33.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:33.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:33.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:33.455 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:33.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:33.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:33.582 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:33.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:33.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:33.713 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:33.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:33.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:33 smithi099 bash[84576]: cluster 2024-09-13T14:58:32.731352+0000 mgr.y (mgr.26252) 971 : cluster [DBG] pgmap v1498: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:33.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:33 smithi099 bash[84576]: cluster 2024-09-13T14:58:32.731352+0000 mgr.y (mgr.26252) 971 : cluster [DBG] pgmap v1498: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:33.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:33 smithi099 bash[84576]: audit 2024-09-13T14:58:33.287851+0000 mon.a (mon.0) 2018 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:33.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:33 smithi099 bash[84576]: audit 2024-09-13T14:58:33.287851+0000 mon.a (mon.0) 2018 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:33 smithi099 bash[87147]: cluster 2024-09-13T14:58:32.731352+0000 mgr.y (mgr.26252) 971 : cluster [DBG] pgmap v1498: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:33 smithi099 bash[87147]: cluster 2024-09-13T14:58:32.731352+0000 mgr.y (mgr.26252) 971 : cluster [DBG] pgmap v1498: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:33 smithi099 bash[87147]: audit 2024-09-13T14:58:33.287851+0000 mon.a (mon.0) 2018 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:33.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:33 smithi099 bash[87147]: audit 2024-09-13T14:58:33.287851+0000 mon.a (mon.0) 2018 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:33.844 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:33.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:33.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:33.975 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:34.078 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:34.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:33 smithi177 bash[68594]: cluster 2024-09-13T14:58:32.731352+0000 mgr.y (mgr.26252) 971 : cluster [DBG] pgmap v1498: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:34.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:33 smithi177 bash[68594]: cluster 2024-09-13T14:58:32.731352+0000 mgr.y (mgr.26252) 971 : cluster [DBG] pgmap v1498: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:34.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:33 smithi177 bash[68594]: audit 2024-09-13T14:58:33.287851+0000 mon.a (mon.0) 2018 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:34.089 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:33 smithi177 bash[68594]: audit 2024-09-13T14:58:33.287851+0000 mon.a (mon.0) 2018 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:34.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:34.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:34.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:34.237 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:34.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:34.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:34.367 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:34.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:34.370 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:58:34.371 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:58:34.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:34.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:34.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:34.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:34.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:34.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:34.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:34.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:34.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:34.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:34.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:35.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:35.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:35.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:35.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:35.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:35.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:35.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:35.321 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:58:35.321 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T14:58:35.321 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2024-09-13T14:58:35.322 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:58:35.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:35.403 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:35.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:35.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:35.534 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:35.637 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:35.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:35.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:35.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:35.792 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:35.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:35.812 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_2' created 2024-09-13T14:58:35.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:35.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:35.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:36.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:35 smithi177 bash[68594]: cluster 2024-09-13T14:58:34.731998+0000 mgr.y (mgr.26252) 972 : cluster [DBG] pgmap v1499: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:35 smithi177 bash[68594]: cluster 2024-09-13T14:58:34.731998+0000 mgr.y (mgr.26252) 972 : cluster [DBG] pgmap v1499: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:35 smithi177 bash[68594]: audit 2024-09-13T14:58:35.030204+0000 mon.a (mon.0) 2019 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:58:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:35 smithi177 bash[68594]: audit 2024-09-13T14:58:35.030204+0000 mon.a (mon.0) 2019 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:58:36.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:36.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:36.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:36.151 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:35 smithi099 bash[84576]: cluster 2024-09-13T14:58:34.731998+0000 mgr.y (mgr.26252) 972 : cluster [DBG] pgmap v1499: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:36.151 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:35 smithi099 bash[84576]: cluster 2024-09-13T14:58:34.731998+0000 mgr.y (mgr.26252) 972 : cluster [DBG] pgmap v1499: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:36.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:35 smithi099 bash[84576]: audit 2024-09-13T14:58:35.030204+0000 mon.a (mon.0) 2019 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:58:36.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:35 smithi099 bash[84576]: audit 2024-09-13T14:58:35.030204+0000 mon.a (mon.0) 2019 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:58:36.152 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:35 smithi099 bash[87147]: cluster 2024-09-13T14:58:34.731998+0000 mgr.y (mgr.26252) 972 : cluster [DBG] pgmap v1499: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:36.152 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:35 smithi099 bash[87147]: cluster 2024-09-13T14:58:34.731998+0000 mgr.y (mgr.26252) 972 : cluster [DBG] pgmap v1499: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:36.152 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:35 smithi099 bash[87147]: audit 2024-09-13T14:58:35.030204+0000 mon.a (mon.0) 2019 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:58:36.152 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:35 smithi099 bash[87147]: audit 2024-09-13T14:58:35.030204+0000 mon.a (mon.0) 2019 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:58:36.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:36.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:36.284 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:36.312 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:36.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:36.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:58:36.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:36.444 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:36.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:36.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:36.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:36.698 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:36.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:36.801 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:36.828 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:36.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:36.932 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:36.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:36.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:36.984 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:36 smithi099 bash[84576]: audit 2024-09-13T14:58:35.813481+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:58:36.984 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:36 smithi099 bash[84576]: audit 2024-09-13T14:58:35.813481+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:58:36.984 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:36 smithi099 bash[84576]: cluster 2024-09-13T14:58:35.821788+0000 mon.a (mon.0) 2021 : cluster [DBG] osdmap e1038: 8 total, 7 up, 8 in 2024-09-13T14:58:36.984 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:36 smithi099 bash[84576]: cluster 2024-09-13T14:58:35.821788+0000 mon.a (mon.0) 2021 : cluster [DBG] osdmap e1038: 8 total, 7 up, 8 in 2024-09-13T14:58:36.984 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:36 smithi099 bash[87147]: audit 2024-09-13T14:58:35.813481+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:58:36.984 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:36 smithi099 bash[87147]: audit 2024-09-13T14:58:35.813481+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:58:36.984 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:36 smithi099 bash[87147]: cluster 2024-09-13T14:58:35.821788+0000 mon.a (mon.0) 2021 : cluster [DBG] osdmap e1038: 8 total, 7 up, 8 in 2024-09-13T14:58:36.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:36 smithi099 bash[87147]: cluster 2024-09-13T14:58:35.821788+0000 mon.a (mon.0) 2021 : cluster [DBG] osdmap e1038: 8 total, 7 up, 8 in 2024-09-13T14:58:37.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:37.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:36 smithi177 bash[68594]: audit 2024-09-13T14:58:35.813481+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:58:37.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:36 smithi177 bash[68594]: audit 2024-09-13T14:58:35.813481+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? 172.21.15.99:0/692561111' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:58:37.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:36 smithi177 bash[68594]: cluster 2024-09-13T14:58:35.821788+0000 mon.a (mon.0) 2021 : cluster [DBG] osdmap e1038: 8 total, 7 up, 8 in 2024-09-13T14:58:37.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:36 smithi177 bash[68594]: cluster 2024-09-13T14:58:35.821788+0000 mon.a (mon.0) 2021 : cluster [DBG] osdmap e1038: 8 total, 7 up, 8 in 2024-09-13T14:58:37.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:37.189 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:37.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:37.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:37.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:37.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:58:37] "GET /metrics HTTP/1.1" 200 34353 "" "Prometheus/2.51.0" 2024-09-13T14:58:37.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:37.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:37.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:37.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:37.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:37.580 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:37.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:37.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:37.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:37.738 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:37.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:37.853 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[84576]: cluster 2024-09-13T14:58:36.732524+0000 mgr.y (mgr.26252) 973 : cluster [DBG] pgmap v1501: 89 pgs: 16 unknown, 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:37.853 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[84576]: cluster 2024-09-13T14:58:36.732524+0000 mgr.y (mgr.26252) 973 : cluster [DBG] pgmap v1501: 89 pgs: 16 unknown, 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:37.853 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[84576]: cluster 2024-09-13T14:58:36.813471+0000 mon.a (mon.0) 2022 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:58:37.853 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[84576]: cluster 2024-09-13T14:58:36.813471+0000 mon.a (mon.0) 2022 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:58:37.853 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[84576]: cluster 2024-09-13T14:58:36.827256+0000 mon.a (mon.0) 2023 : cluster [DBG] osdmap e1039: 8 total, 7 up, 8 in 2024-09-13T14:58:37.853 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[84576]: cluster 2024-09-13T14:58:36.827256+0000 mon.a (mon.0) 2023 : cluster [DBG] osdmap e1039: 8 total, 7 up, 8 in 2024-09-13T14:58:37.854 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[87147]: cluster 2024-09-13T14:58:36.732524+0000 mgr.y (mgr.26252) 973 : cluster [DBG] pgmap v1501: 89 pgs: 16 unknown, 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:37.854 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[87147]: cluster 2024-09-13T14:58:36.732524+0000 mgr.y (mgr.26252) 973 : cluster [DBG] pgmap v1501: 89 pgs: 16 unknown, 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:37.854 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[87147]: cluster 2024-09-13T14:58:36.813471+0000 mon.a (mon.0) 2022 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:58:37.854 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[87147]: cluster 2024-09-13T14:58:36.813471+0000 mon.a (mon.0) 2022 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:58:37.854 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[87147]: cluster 2024-09-13T14:58:36.827256+0000 mon.a (mon.0) 2023 : cluster [DBG] osdmap e1039: 8 total, 7 up, 8 in 2024-09-13T14:58:37.854 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:37 smithi099 bash[87147]: cluster 2024-09-13T14:58:36.827256+0000 mon.a (mon.0) 2023 : cluster [DBG] osdmap e1039: 8 total, 7 up, 8 in 2024-09-13T14:58:37.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:37.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:37.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:37.999 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:38.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:38.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:37 smithi177 bash[68594]: cluster 2024-09-13T14:58:36.732524+0000 mgr.y (mgr.26252) 973 : cluster [DBG] pgmap v1501: 89 pgs: 16 unknown, 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:37 smithi177 bash[68594]: cluster 2024-09-13T14:58:36.732524+0000 mgr.y (mgr.26252) 973 : cluster [DBG] pgmap v1501: 89 pgs: 16 unknown, 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:37 smithi177 bash[68594]: cluster 2024-09-13T14:58:36.813471+0000 mon.a (mon.0) 2022 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:58:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:37 smithi177 bash[68594]: cluster 2024-09-13T14:58:36.813471+0000 mon.a (mon.0) 2022 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:58:38.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:37 smithi177 bash[68594]: cluster 2024-09-13T14:58:36.827256+0000 mon.a (mon.0) 2023 : cluster [DBG] osdmap e1039: 8 total, 7 up, 8 in 2024-09-13T14:58:38.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:37 smithi177 bash[68594]: cluster 2024-09-13T14:58:36.827256+0000 mon.a (mon.0) 2023 : cluster [DBG] osdmap e1039: 8 total, 7 up, 8 in 2024-09-13T14:58:38.128 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:38.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:38.232 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:38.261 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:38.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:38.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:38.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:38.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:38.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:38.520 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:38.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:38.623 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:38.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:38.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:38.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:38.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:38.884 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:38.896 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:38 smithi177 bash[68594]: cluster 2024-09-13T14:58:37.840215+0000 mon.a (mon.0) 2024 : cluster [DBG] osdmap e1040: 8 total, 7 up, 8 in 2024-09-13T14:58:38.896 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:38 smithi177 bash[68594]: cluster 2024-09-13T14:58:37.840215+0000 mon.a (mon.0) 2024 : cluster [DBG] osdmap e1040: 8 total, 7 up, 8 in 2024-09-13T14:58:38.913 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:38.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:39.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:39.027 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:38 smithi099 bash[84576]: cluster 2024-09-13T14:58:37.840215+0000 mon.a (mon.0) 2024 : cluster [DBG] osdmap e1040: 8 total, 7 up, 8 in 2024-09-13T14:58:39.028 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:38 smithi099 bash[84576]: cluster 2024-09-13T14:58:37.840215+0000 mon.a (mon.0) 2024 : cluster [DBG] osdmap e1040: 8 total, 7 up, 8 in 2024-09-13T14:58:39.028 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:38 smithi099 bash[87147]: cluster 2024-09-13T14:58:37.840215+0000 mon.a (mon.0) 2024 : cluster [DBG] osdmap e1040: 8 total, 7 up, 8 in 2024-09-13T14:58:39.028 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:38 smithi099 bash[87147]: cluster 2024-09-13T14:58:37.840215+0000 mon.a (mon.0) 2024 : cluster [DBG] osdmap e1040: 8 total, 7 up, 8 in 2024-09-13T14:58:39.042 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:39.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:39.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-09-13T14:58:39.146 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:39.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:39.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:39.184 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:58:39.275 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:39.303 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:39.407 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:39.432 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:39.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:39.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:39.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:39.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:39.667 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:39.695 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:39.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:39.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:39.823 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:39.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:39.842 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T14:58:39.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:39.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:39 smithi177 bash[68594]: cluster 2024-09-13T14:58:38.733294+0000 mgr.y (mgr.26252) 974 : cluster [DBG] pgmap v1504: 89 pgs: 13 unknown, 27 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:39.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:39 smithi177 bash[68594]: cluster 2024-09-13T14:58:38.733294+0000 mgr.y (mgr.26252) 974 : cluster [DBG] pgmap v1504: 89 pgs: 13 unknown, 27 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:39.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:39 smithi177 bash[68594]: audit 2024-09-13T14:58:39.635127+0000 mon.a (mon.0) 2025 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:39.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:39 smithi177 bash[68594]: audit 2024-09-13T14:58:39.635127+0000 mon.a (mon.0) 2025 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:39.952 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:39.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:40.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:40.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:39 smithi099 bash[84576]: cluster 2024-09-13T14:58:38.733294+0000 mgr.y (mgr.26252) 974 : cluster [DBG] pgmap v1504: 89 pgs: 13 unknown, 27 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:40.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:39 smithi099 bash[84576]: cluster 2024-09-13T14:58:38.733294+0000 mgr.y (mgr.26252) 974 : cluster [DBG] pgmap v1504: 89 pgs: 13 unknown, 27 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:40.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:39 smithi099 bash[84576]: audit 2024-09-13T14:58:39.635127+0000 mon.a (mon.0) 2025 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:40.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:39 smithi099 bash[84576]: audit 2024-09-13T14:58:39.635127+0000 mon.a (mon.0) 2025 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:40.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:39 smithi099 bash[87147]: cluster 2024-09-13T14:58:38.733294+0000 mgr.y (mgr.26252) 974 : cluster [DBG] pgmap v1504: 89 pgs: 13 unknown, 27 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:40.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:39 smithi099 bash[87147]: cluster 2024-09-13T14:58:38.733294+0000 mgr.y (mgr.26252) 974 : cluster [DBG] pgmap v1504: 89 pgs: 13 unknown, 27 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:40.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:39 smithi099 bash[87147]: audit 2024-09-13T14:58:39.635127+0000 mon.a (mon.0) 2025 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:40.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:39 smithi099 bash[87147]: audit 2024-09-13T14:58:39.635127+0000 mon.a (mon.0) 2025 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:40.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:40.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:40.186 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:40.211 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:40.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:40.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:40.341 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:40.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:40.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:40.455 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:58:40 smithi177 bash[61439]: ts=2024-09-13T14:58:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:40.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:40.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:40.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:40.601 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:40.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:40.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:40.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:40.833 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:40.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:40.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:40.964 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:40.976 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:40 smithi177 bash[68594]: cluster 2024-09-13T14:58:39.841908+0000 mon.a (mon.0) 2026 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:40.976 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:40 smithi177 bash[68594]: cluster 2024-09-13T14:58:39.841908+0000 mon.a (mon.0) 2026 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:40.976 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:40 smithi177 bash[68594]: audit 2024-09-13T14:58:39.843722+0000 mon.a (mon.0) 2027 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:40.976 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:40 smithi177 bash[68594]: audit 2024-09-13T14:58:39.843722+0000 mon.a (mon.0) 2027 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:40.976 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:40 smithi177 bash[68594]: cluster 2024-09-13T14:58:39.847119+0000 mon.a (mon.0) 2028 : cluster [DBG] osdmap e1041: 8 total, 7 up, 8 in 2024-09-13T14:58:40.976 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:40 smithi177 bash[68594]: cluster 2024-09-13T14:58:39.847119+0000 mon.a (mon.0) 2028 : cluster [DBG] osdmap e1041: 8 total, 7 up, 8 in 2024-09-13T14:58:40.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:40.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:41.095 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:41.106 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[84576]: cluster 2024-09-13T14:58:39.841908+0000 mon.a (mon.0) 2026 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:41.106 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[84576]: cluster 2024-09-13T14:58:39.841908+0000 mon.a (mon.0) 2026 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:41.106 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[84576]: audit 2024-09-13T14:58:39.843722+0000 mon.a (mon.0) 2027 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:41.106 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[84576]: audit 2024-09-13T14:58:39.843722+0000 mon.a (mon.0) 2027 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:41.106 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[84576]: cluster 2024-09-13T14:58:39.847119+0000 mon.a (mon.0) 2028 : cluster [DBG] osdmap e1041: 8 total, 7 up, 8 in 2024-09-13T14:58:41.106 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[84576]: cluster 2024-09-13T14:58:39.847119+0000 mon.a (mon.0) 2028 : cluster [DBG] osdmap e1041: 8 total, 7 up, 8 in 2024-09-13T14:58:41.106 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[87147]: cluster 2024-09-13T14:58:39.841908+0000 mon.a (mon.0) 2026 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:41.106 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[87147]: cluster 2024-09-13T14:58:39.841908+0000 mon.a (mon.0) 2026 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:41.107 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[87147]: audit 2024-09-13T14:58:39.843722+0000 mon.a (mon.0) 2027 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:41.107 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[87147]: audit 2024-09-13T14:58:39.843722+0000 mon.a (mon.0) 2027 : audit [INF] from='client.? 172.21.15.99:0/3064224399' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:41.107 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[87147]: cluster 2024-09-13T14:58:39.847119+0000 mon.a (mon.0) 2028 : cluster [DBG] osdmap e1041: 8 total, 7 up, 8 in 2024-09-13T14:58:41.107 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:40 smithi099 bash[87147]: cluster 2024-09-13T14:58:39.847119+0000 mon.a (mon.0) 2028 : cluster [DBG] osdmap e1041: 8 total, 7 up, 8 in 2024-09-13T14:58:41.121 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:41.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:41.225 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:41.252 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:41.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:41.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:41.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:41.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:41.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:41.512 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:41.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:41.615 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:41.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:41.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:41.744 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:41.770 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:41.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:41.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:41.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:41.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:42.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:42.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:42.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:42.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:41 smithi099 bash[84576]: cluster 2024-09-13T14:58:40.734089+0000 mgr.y (mgr.26252) 975 : cluster [DBG] pgmap v1506: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:42.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:41 smithi099 bash[84576]: cluster 2024-09-13T14:58:40.734089+0000 mgr.y (mgr.26252) 975 : cluster [DBG] pgmap v1506: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:42.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:41 smithi099 bash[87147]: cluster 2024-09-13T14:58:40.734089+0000 mgr.y (mgr.26252) 975 : cluster [DBG] pgmap v1506: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:42.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:41 smithi099 bash[87147]: cluster 2024-09-13T14:58:40.734089+0000 mgr.y (mgr.26252) 975 : cluster [DBG] pgmap v1506: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:42.135 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:42.146 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:41 smithi177 bash[68594]: cluster 2024-09-13T14:58:40.734089+0000 mgr.y (mgr.26252) 975 : cluster [DBG] pgmap v1506: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:42.146 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:41 smithi177 bash[68594]: cluster 2024-09-13T14:58:40.734089+0000 mgr.y (mgr.26252) 975 : cluster [DBG] pgmap v1506: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:42.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:42.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:42.293 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:42.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:42.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:42.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:42.512 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:42.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:42.547 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:42.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:42.703 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:42.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:42.806 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:42.835 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:42.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:42.939 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:42.967 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:43.030 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:58:43.070 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:43.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:43.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:43.198 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:43.223 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:43.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:43.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:43.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:43.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:43.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:43.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:43.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:43.528 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:43.528 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:43.589 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:43.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:43.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:43.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:43.742 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:43.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:43.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:43.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:43.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:43.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:44.004 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:44.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:43 smithi099 bash[84576]: cluster 2024-09-13T14:58:42.735047+0000 mgr.y (mgr.26252) 976 : cluster [DBG] pgmap v1507: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:44.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:43 smithi099 bash[84576]: cluster 2024-09-13T14:58:42.735047+0000 mgr.y (mgr.26252) 976 : cluster [DBG] pgmap v1507: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:44.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:43 smithi099 bash[87147]: cluster 2024-09-13T14:58:42.735047+0000 mgr.y (mgr.26252) 976 : cluster [DBG] pgmap v1507: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:44.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:43 smithi099 bash[87147]: cluster 2024-09-13T14:58:42.735047+0000 mgr.y (mgr.26252) 976 : cluster [DBG] pgmap v1507: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:44.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:44.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:43 smithi177 bash[68594]: cluster 2024-09-13T14:58:42.735047+0000 mgr.y (mgr.26252) 976 : cluster [DBG] pgmap v1507: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:44.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:43 smithi177 bash[68594]: cluster 2024-09-13T14:58:42.735047+0000 mgr.y (mgr.26252) 976 : cluster [DBG] pgmap v1507: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:44.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:44.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:44.235 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:44.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:44.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:44.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:44.396 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:44.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:44.499 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:44.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:44.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:44.658 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:44.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:44.761 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:44.788 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:44.890 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:44.918 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:44.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:45.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:45.048 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:45.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:45.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:45.178 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:45.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:45.282 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:45.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:45.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:45.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:45.439 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:45.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:45.542 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:45.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:45.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:45.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:45.697 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:45.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:45.800 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:45.828 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:45.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:45.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:45.960 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:46.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:46.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:45 smithi099 bash[84576]: cluster 2024-09-13T14:58:44.736216+0000 mgr.y (mgr.26252) 977 : cluster [DBG] pgmap v1508: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:46.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:45 smithi099 bash[84576]: cluster 2024-09-13T14:58:44.736216+0000 mgr.y (mgr.26252) 977 : cluster [DBG] pgmap v1508: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:46.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:45 smithi099 bash[87147]: cluster 2024-09-13T14:58:44.736216+0000 mgr.y (mgr.26252) 977 : cluster [DBG] pgmap v1508: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:46.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:45 smithi099 bash[87147]: cluster 2024-09-13T14:58:44.736216+0000 mgr.y (mgr.26252) 977 : cluster [DBG] pgmap v1508: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:46.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:46.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:46.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:45 smithi177 bash[68594]: cluster 2024-09-13T14:58:44.736216+0000 mgr.y (mgr.26252) 977 : cluster [DBG] pgmap v1508: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:46.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:45 smithi177 bash[68594]: cluster 2024-09-13T14:58:44.736216+0000 mgr.y (mgr.26252) 977 : cluster [DBG] pgmap v1508: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:46.196 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:46.220 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:46.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:46.323 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:46.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:46.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:46.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:46.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:46.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:46.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:46.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:46.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:46.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:46.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:46.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:46.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:46.881 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:46.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.012 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:47.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:47.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:47.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:58:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:58:47] "GET /metrics HTTP/1.1" 200 34353 "" "Prometheus/2.51.0" 2024-09-13T14:58:47.375 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:47.402 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:47.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:47.505 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:47.532 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:47.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:47.635 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:47.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:47.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:47.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:47.794 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:47.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:47.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:47.908 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:47 smithi177 bash[68594]: cluster 2024-09-13T14:58:46.736979+0000 mgr.y (mgr.26252) 978 : cluster [DBG] pgmap v1509: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:47.909 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:47 smithi177 bash[68594]: cluster 2024-09-13T14:58:46.736979+0000 mgr.y (mgr.26252) 978 : cluster [DBG] pgmap v1509: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:47.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:47.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:47.962 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:48.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:48.055 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:47 smithi099 bash[84576]: cluster 2024-09-13T14:58:46.736979+0000 mgr.y (mgr.26252) 978 : cluster [DBG] pgmap v1509: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:47 smithi099 bash[84576]: cluster 2024-09-13T14:58:46.736979+0000 mgr.y (mgr.26252) 978 : cluster [DBG] pgmap v1509: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:48.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:47 smithi099 bash[87147]: cluster 2024-09-13T14:58:46.736979+0000 mgr.y (mgr.26252) 978 : cluster [DBG] pgmap v1509: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:48.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:47 smithi099 bash[87147]: cluster 2024-09-13T14:58:46.736979+0000 mgr.y (mgr.26252) 978 : cluster [DBG] pgmap v1509: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:48.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:48.187 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:48.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:48.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:48.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:48.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:48.420 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:48.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:48.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:48.552 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:48.579 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:48.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:48.683 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:48.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:48.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:48.839 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:48.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:48.942 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:48.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:49.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:49.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:49.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:49.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:49.230 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:49.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:49.333 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:49.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:49.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:49.464 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:49.490 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:49.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[84576]: audit 2024-09-13T14:58:48.295796+0000 mon.a (mon.0) 2029 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[84576]: audit 2024-09-13T14:58:48.295796+0000 mon.a (mon.0) 2029 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[84576]: audit 2024-09-13T14:58:48.297017+0000 mon.a (mon.0) 2030 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[84576]: audit 2024-09-13T14:58:48.297017+0000 mon.a (mon.0) 2030 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[84576]: cluster 2024-09-13T14:58:48.737992+0000 mgr.y (mgr.26252) 979 : cluster [DBG] pgmap v1510: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[84576]: cluster 2024-09-13T14:58:48.737992+0000 mgr.y (mgr.26252) 979 : cluster [DBG] pgmap v1510: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[87147]: audit 2024-09-13T14:58:48.295796+0000 mon.a (mon.0) 2029 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[87147]: audit 2024-09-13T14:58:48.295796+0000 mon.a (mon.0) 2029 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[87147]: audit 2024-09-13T14:58:48.297017+0000 mon.a (mon.0) 2030 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[87147]: audit 2024-09-13T14:58:48.297017+0000 mon.a (mon.0) 2030 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[87147]: cluster 2024-09-13T14:58:48.737992+0000 mgr.y (mgr.26252) 979 : cluster [DBG] pgmap v1510: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:49 smithi099 bash[87147]: cluster 2024-09-13T14:58:48.737992+0000 mgr.y (mgr.26252) 979 : cluster [DBG] pgmap v1510: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:49.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:49.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:49 smithi177 bash[68594]: audit 2024-09-13T14:58:48.295796+0000 mon.a (mon.0) 2029 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:49.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:49 smithi177 bash[68594]: audit 2024-09-13T14:58:48.295796+0000 mon.a (mon.0) 2029 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:49.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:49 smithi177 bash[68594]: audit 2024-09-13T14:58:48.297017+0000 mon.a (mon.0) 2030 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:49.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:49 smithi177 bash[68594]: audit 2024-09-13T14:58:48.297017+0000 mon.a (mon.0) 2030 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:49.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:49 smithi177 bash[68594]: cluster 2024-09-13T14:58:48.737992+0000 mgr.y (mgr.26252) 979 : cluster [DBG] pgmap v1510: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:49.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:49 smithi177 bash[68594]: cluster 2024-09-13T14:58:48.737992+0000 mgr.y (mgr.26252) 979 : cluster [DBG] pgmap v1510: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:49.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:49.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:49.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:49.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:49.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:49.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:49.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:49.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:49.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:50.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:50.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:50.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:50.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:50.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:50.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:50.374 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:50.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:50.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:50.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:50.534 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:50.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:50.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:58:50 smithi177 bash[61439]: ts=2024-09-13T14:58:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:58:50.637 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:50.663 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:50.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.794 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:50.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:51.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:51.055 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:51.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:51.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:51.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:51.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:51.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:51.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:51.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:51.412 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:51.436 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:51.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:51.539 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:51.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:51.669 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:51.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:51.791 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:51.802 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:51 smithi099 bash[84576]: cluster 2024-09-13T14:58:50.738778+0000 mgr.y (mgr.26252) 980 : cluster [DBG] pgmap v1511: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:51.802 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:51 smithi099 bash[84576]: cluster 2024-09-13T14:58:50.738778+0000 mgr.y (mgr.26252) 980 : cluster [DBG] pgmap v1511: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:51.802 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:51 smithi099 bash[84576]: audit 2024-09-13T14:58:51.162227+0000 mon.a (mon.0) 2031 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:51.802 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:51 smithi099 bash[84576]: audit 2024-09-13T14:58:51.162227+0000 mon.a (mon.0) 2031 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:51.802 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:51 smithi099 bash[87147]: cluster 2024-09-13T14:58:50.738778+0000 mgr.y (mgr.26252) 980 : cluster [DBG] pgmap v1511: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:51.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:51 smithi099 bash[87147]: cluster 2024-09-13T14:58:50.738778+0000 mgr.y (mgr.26252) 980 : cluster [DBG] pgmap v1511: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:51.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:51 smithi099 bash[87147]: audit 2024-09-13T14:58:51.162227+0000 mon.a (mon.0) 2031 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:51.803 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:51 smithi099 bash[87147]: audit 2024-09-13T14:58:51.162227+0000 mon.a (mon.0) 2031 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:51.816 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:51.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:51.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:52.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:52.073 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:52.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:52.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:51 smithi177 bash[68594]: cluster 2024-09-13T14:58:50.738778+0000 mgr.y (mgr.26252) 980 : cluster [DBG] pgmap v1511: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:52.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:51 smithi177 bash[68594]: cluster 2024-09-13T14:58:50.738778+0000 mgr.y (mgr.26252) 980 : cluster [DBG] pgmap v1511: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:52.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:51 smithi177 bash[68594]: audit 2024-09-13T14:58:51.162227+0000 mon.a (mon.0) 2031 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:52.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:51 smithi177 bash[68594]: audit 2024-09-13T14:58:51.162227+0000 mon.a (mon.0) 2031 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:52.176 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:52.202 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:52.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:52.332 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:52.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:52.435 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:52.455 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:52.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:52.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:52.587 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:52.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:52.690 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:52.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:52.819 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:52.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:52.948 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:52.975 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:52.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:53.078 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:53.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:53.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:52 smithi177 bash[68594]: cluster 2024-09-13T14:58:52.796003+0000 mon.a (mon.0) 2032 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:58:53.106 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:52 smithi177 bash[68594]: cluster 2024-09-13T14:58:52.796003+0000 mon.a (mon.0) 2032 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:58:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:53.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:53.234 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:53.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:52 smithi099 bash[84576]: cluster 2024-09-13T14:58:52.796003+0000 mon.a (mon.0) 2032 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:58:53.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:52 smithi099 bash[84576]: cluster 2024-09-13T14:58:52.796003+0000 mon.a (mon.0) 2032 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:58:53.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:52 smithi099 bash[87147]: cluster 2024-09-13T14:58:52.796003+0000 mon.a (mon.0) 2032 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:58:53.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:52 smithi099 bash[87147]: cluster 2024-09-13T14:58:52.796003+0000 mon.a (mon.0) 2032 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:58:53.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:53.365 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:53.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:53.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:53.495 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:53.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:53.598 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:53.622 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:53.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:53.725 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:53.749 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:53.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:53.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:53.877 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:53.925 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:53 smithi099 bash[87147]: cluster 2024-09-13T14:58:52.739512+0000 mgr.y (mgr.26252) 981 : cluster [DBG] pgmap v1512: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:53.925 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:53 smithi099 bash[87147]: cluster 2024-09-13T14:58:52.739512+0000 mgr.y (mgr.26252) 981 : cluster [DBG] pgmap v1512: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:53.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:54.007 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:54.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:54.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:54.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:53 smithi177 bash[68594]: cluster 2024-09-13T14:58:52.739512+0000 mgr.y (mgr.26252) 981 : cluster [DBG] pgmap v1512: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:54.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:53 smithi177 bash[68594]: cluster 2024-09-13T14:58:52.739512+0000 mgr.y (mgr.26252) 981 : cluster [DBG] pgmap v1512: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:54.137 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:54.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:54.237 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:53 smithi099 bash[84576]: cluster 2024-09-13T14:58:52.739512+0000 mgr.y (mgr.26252) 981 : cluster [DBG] pgmap v1512: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:54.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:53 smithi099 bash[84576]: cluster 2024-09-13T14:58:52.739512+0000 mgr.y (mgr.26252) 981 : cluster [DBG] pgmap v1512: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:54.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:54.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:54.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:54.362 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:54.380 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:58:54.380 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:58:54.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:54.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:54.398 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:58:54.398 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:54.415 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:58:54.415 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:54.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T14:58:54.435 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:58:54.435 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:54.455 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:58:54.455 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:54.475 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:58:54.475 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:54.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:54.492 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:58:54.493 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:54.515 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:54.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:54.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:54.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:54.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:54.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:54.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:54.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:54.878 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:54.889 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:54 smithi177 bash[68594]: audit 2024-09-13T14:58:54.363574+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.99:0/277099391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:54.889 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:54 smithi177 bash[68594]: audit 2024-09-13T14:58:54.363574+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.99:0/277099391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:54.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:55.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:55.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:55.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:55.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:55.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:55.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:55.262 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:55.274 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:54 smithi099 bash[84576]: audit 2024-09-13T14:58:54.363574+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.99:0/277099391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:55.274 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:54 smithi099 bash[84576]: audit 2024-09-13T14:58:54.363574+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.99:0/277099391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:55.274 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:54 smithi099 bash[87147]: audit 2024-09-13T14:58:54.363574+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.99:0/277099391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:55.274 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:54 smithi099 bash[87147]: audit 2024-09-13T14:58:54.363574+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.99:0/277099391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:55.290 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:55.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:55.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:55.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:55.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:55.653 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:55.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:55.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:55.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:55.803 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:55.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:55.908 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:55.920 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:55 smithi177 bash[68594]: cluster 2024-09-13T14:58:54.741023+0000 mgr.y (mgr.26252) 982 : cluster [DBG] pgmap v1513: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:55.921 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:55 smithi177 bash[68594]: cluster 2024-09-13T14:58:54.741023+0000 mgr.y (mgr.26252) 982 : cluster [DBG] pgmap v1513: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:55.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:55.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:56.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:56.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:56.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:56.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:55 smithi099 bash[84576]: cluster 2024-09-13T14:58:54.741023+0000 mgr.y (mgr.26252) 982 : cluster [DBG] pgmap v1513: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:56.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:55 smithi099 bash[84576]: cluster 2024-09-13T14:58:54.741023+0000 mgr.y (mgr.26252) 982 : cluster [DBG] pgmap v1513: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:56.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:55 smithi099 bash[87147]: cluster 2024-09-13T14:58:54.741023+0000 mgr.y (mgr.26252) 982 : cluster [DBG] pgmap v1513: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:56.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:55 smithi099 bash[87147]: cluster 2024-09-13T14:58:54.741023+0000 mgr.y (mgr.26252) 982 : cluster [DBG] pgmap v1513: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:56.166 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:56.195 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:56.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:56.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:56.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:56.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:56.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:56.452 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:56.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:56.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:56.575 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:56.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:56.678 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:56.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:56.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:56.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:56.828 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:56.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:56.931 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:56.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:56.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:57.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:57.079 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:57.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:57.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:57.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:57.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:57.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:57.323 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:57.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:58:57] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.51.0" 2024-09-13T14:58:57.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:57.425 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:57.452 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:57.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:57.555 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:57.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:57.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:57.680 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:57.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:57.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:57.809 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:57.837 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:57.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:57.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[84576]: cluster 2024-09-13T14:58:56.741980+0000 mgr.y (mgr.26252) 983 : cluster [DBG] pgmap v1514: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:57.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[84576]: cluster 2024-09-13T14:58:56.741980+0000 mgr.y (mgr.26252) 983 : cluster [DBG] pgmap v1514: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:57.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[84576]: audit 2024-09-13T14:58:57.201189+0000 mon.a (mon.0) 2033 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:57.951 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[84576]: audit 2024-09-13T14:58:57.201189+0000 mon.a (mon.0) 2033 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:57.952 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[84576]: audit 2024-09-13T14:58:57.213466+0000 mon.a (mon.0) 2034 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:57.952 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[84576]: audit 2024-09-13T14:58:57.213466+0000 mon.a (mon.0) 2034 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:57.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[87147]: cluster 2024-09-13T14:58:56.741980+0000 mgr.y (mgr.26252) 983 : cluster [DBG] pgmap v1514: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:57.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[87147]: cluster 2024-09-13T14:58:56.741980+0000 mgr.y (mgr.26252) 983 : cluster [DBG] pgmap v1514: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:57.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[87147]: audit 2024-09-13T14:58:57.201189+0000 mon.a (mon.0) 2033 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:57.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[87147]: audit 2024-09-13T14:58:57.201189+0000 mon.a (mon.0) 2033 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:57.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[87147]: audit 2024-09-13T14:58:57.213466+0000 mon.a (mon.0) 2034 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:57.952 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:57 smithi099 bash[87147]: audit 2024-09-13T14:58:57.213466+0000 mon.a (mon.0) 2034 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:57.966 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:57.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:58.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:58.096 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:58.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:58.199 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:58.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:58:58.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:58:58.330 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:58.342 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:57 smithi177 bash[68594]: cluster 2024-09-13T14:58:56.741980+0000 mgr.y (mgr.26252) 983 : cluster [DBG] pgmap v1514: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:58.342 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:57 smithi177 bash[68594]: cluster 2024-09-13T14:58:56.741980+0000 mgr.y (mgr.26252) 983 : cluster [DBG] pgmap v1514: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:58.342 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:57 smithi177 bash[68594]: audit 2024-09-13T14:58:57.201189+0000 mon.a (mon.0) 2033 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:58.342 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:57 smithi177 bash[68594]: audit 2024-09-13T14:58:57.201189+0000 mon.a (mon.0) 2033 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:58.342 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:57 smithi177 bash[68594]: audit 2024-09-13T14:58:57.213466+0000 mon.a (mon.0) 2034 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:58.342 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:57 smithi177 bash[68594]: audit 2024-09-13T14:58:57.213466+0000 mon.a (mon.0) 2034 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:58:58.358 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:58.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:58.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:58.487 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:58.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:58.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:58.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:58.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:58.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:58.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:58.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:58.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:58.876 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:58.979 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:59.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:59.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:59.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:59.137 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:59.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:59.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:59.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:59.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:58:59.305 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:58:59.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:59.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:59.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:59.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:59.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:59.628 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:59.649 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:59.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:59.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:59.779 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:59.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:59.882 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:59.883 INFO:teuthology.orchestra.run.smithi099.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-09-13T14:58:59.896 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[84576]: cluster 2024-09-13T14:58:58.742760+0000 mgr.y (mgr.26252) 984 : cluster [DBG] pgmap v1515: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:59.896 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[84576]: cluster 2024-09-13T14:58:58.742760+0000 mgr.y (mgr.26252) 984 : cluster [DBG] pgmap v1515: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:59.896 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[84576]: audit 2024-09-13T14:58:59.849545+0000 mon.c (mon.2) 76 : audit [INF] from='client.? 172.21.15.99:0/499586270' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:59.896 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[84576]: audit 2024-09-13T14:58:59.849545+0000 mon.c (mon.2) 76 : audit [INF] from='client.? 172.21.15.99:0/499586270' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:59.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[84576]: audit 2024-09-13T14:58:59.850208+0000 mon.a (mon.0) 2035 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:59.897 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[84576]: audit 2024-09-13T14:58:59.850208+0000 mon.a (mon.0) 2035 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:59.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[87147]: cluster 2024-09-13T14:58:58.742760+0000 mgr.y (mgr.26252) 984 : cluster [DBG] pgmap v1515: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:59.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[87147]: cluster 2024-09-13T14:58:58.742760+0000 mgr.y (mgr.26252) 984 : cluster [DBG] pgmap v1515: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:58:59.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[87147]: audit 2024-09-13T14:58:59.849545+0000 mon.c (mon.2) 76 : audit [INF] from='client.? 172.21.15.99:0/499586270' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:59.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[87147]: audit 2024-09-13T14:58:59.849545+0000 mon.c (mon.2) 76 : audit [INF] from='client.? 172.21.15.99:0/499586270' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:59.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[87147]: audit 2024-09-13T14:58:59.850208+0000 mon.a (mon.0) 2035 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:59.897 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:58:59 smithi099 bash[87147]: audit 2024-09-13T14:58:59.850208+0000 mon.a (mon.0) 2035 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:58:59.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:59.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:00.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:00.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:59 smithi177 bash[68594]: cluster 2024-09-13T14:58:58.742760+0000 mgr.y (mgr.26252) 984 : cluster [DBG] pgmap v1515: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:00.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:59 smithi177 bash[68594]: cluster 2024-09-13T14:58:58.742760+0000 mgr.y (mgr.26252) 984 : cluster [DBG] pgmap v1515: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:00.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:59 smithi177 bash[68594]: audit 2024-09-13T14:58:59.849545+0000 mon.c (mon.2) 76 : audit [INF] from='client.? 172.21.15.99:0/499586270' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:59:00.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:59 smithi177 bash[68594]: audit 2024-09-13T14:58:59.849545+0000 mon.c (mon.2) 76 : audit [INF] from='client.? 172.21.15.99:0/499586270' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:59:00.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:59 smithi177 bash[68594]: audit 2024-09-13T14:58:59.850208+0000 mon.a (mon.0) 2035 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:59:00.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:58:59 smithi177 bash[68594]: audit 2024-09-13T14:58:59.850208+0000 mon.a (mon.0) 2035 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:59:00.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:00.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:00.140 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T14:59:00.140 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:59:00.142 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:00.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:00.265 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:00.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:59:00 smithi177 bash[61439]: ts=2024-09-13T14:59:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:00.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:00.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:00.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:00.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:00.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:00.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:00.553 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:00.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:00.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:00.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:00.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:00.805 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:00.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:00.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:00.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:00.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:01.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:01.065 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:01.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:01.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:00 smithi099 bash[84576]: audit 2024-09-13T14:58:59.876748+0000 mon.a (mon.0) 2036 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:59:01.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:00 smithi099 bash[84576]: audit 2024-09-13T14:58:59.876748+0000 mon.a (mon.0) 2036 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:59:01.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:00 smithi099 bash[84576]: cluster 2024-09-13T14:58:59.883522+0000 mon.a (mon.0) 2037 : cluster [DBG] osdmap e1042: 8 total, 7 up, 8 in 2024-09-13T14:59:01.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:00 smithi099 bash[84576]: cluster 2024-09-13T14:58:59.883522+0000 mon.a (mon.0) 2037 : cluster [DBG] osdmap e1042: 8 total, 7 up, 8 in 2024-09-13T14:59:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:00 smithi099 bash[87147]: audit 2024-09-13T14:58:59.876748+0000 mon.a (mon.0) 2036 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:59:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:00 smithi099 bash[87147]: audit 2024-09-13T14:58:59.876748+0000 mon.a (mon.0) 2036 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:59:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:00 smithi099 bash[87147]: cluster 2024-09-13T14:58:59.883522+0000 mon.a (mon.0) 2037 : cluster [DBG] osdmap e1042: 8 total, 7 up, 8 in 2024-09-13T14:59:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:00 smithi099 bash[87147]: cluster 2024-09-13T14:58:59.883522+0000 mon.a (mon.0) 2037 : cluster [DBG] osdmap e1042: 8 total, 7 up, 8 in 2024-09-13T14:59:01.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:01.179 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:00 smithi177 bash[68594]: audit 2024-09-13T14:58:59.876748+0000 mon.a (mon.0) 2036 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:59:01.179 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:00 smithi177 bash[68594]: audit 2024-09-13T14:58:59.876748+0000 mon.a (mon.0) 2036 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:59:01.179 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:00 smithi177 bash[68594]: cluster 2024-09-13T14:58:59.883522+0000 mon.a (mon.0) 2037 : cluster [DBG] osdmap e1042: 8 total, 7 up, 8 in 2024-09-13T14:59:01.179 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:00 smithi177 bash[68594]: cluster 2024-09-13T14:58:59.883522+0000 mon.a (mon.0) 2037 : cluster [DBG] osdmap e1042: 8 total, 7 up, 8 in 2024-09-13T14:59:01.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:01.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:01.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:01.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:59:01.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:01.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:01.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:01.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:01.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:01.556 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:01.576 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:01.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:01.707 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:01.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:01.810 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:01.838 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:01.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:01.941 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:01.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:01.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:02.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:02.084 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:02.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:02.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:01 smithi177 bash[68594]: cluster 2024-09-13T14:59:00.743765+0000 mgr.y (mgr.26252) 985 : cluster [DBG] pgmap v1517: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:02.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:01 smithi177 bash[68594]: cluster 2024-09-13T14:59:00.743765+0000 mgr.y (mgr.26252) 985 : cluster [DBG] pgmap v1517: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:02.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:01 smithi177 bash[68594]: cluster 2024-09-13T14:59:00.885309+0000 mon.a (mon.0) 2038 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:59:02.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:01 smithi177 bash[68594]: cluster 2024-09-13T14:59:00.885309+0000 mon.a (mon.0) 2038 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:59:02.187 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:02.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:02.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:02.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:02.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:01 smithi099 bash[84576]: cluster 2024-09-13T14:59:00.743765+0000 mgr.y (mgr.26252) 985 : cluster [DBG] pgmap v1517: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:02.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:01 smithi099 bash[84576]: cluster 2024-09-13T14:59:00.743765+0000 mgr.y (mgr.26252) 985 : cluster [DBG] pgmap v1517: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:02.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:01 smithi099 bash[84576]: cluster 2024-09-13T14:59:00.885309+0000 mon.a (mon.0) 2038 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:59:02.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:01 smithi099 bash[84576]: cluster 2024-09-13T14:59:00.885309+0000 mon.a (mon.0) 2038 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:59:02.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:01 smithi099 bash[87147]: cluster 2024-09-13T14:59:00.743765+0000 mgr.y (mgr.26252) 985 : cluster [DBG] pgmap v1517: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:02.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:01 smithi099 bash[87147]: cluster 2024-09-13T14:59:00.743765+0000 mgr.y (mgr.26252) 985 : cluster [DBG] pgmap v1517: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:02.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:01 smithi099 bash[87147]: cluster 2024-09-13T14:59:00.885309+0000 mon.a (mon.0) 2038 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:59:02.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:01 smithi099 bash[87147]: cluster 2024-09-13T14:59:00.885309+0000 mon.a (mon.0) 2038 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:59:02.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:02.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:02.449 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:02.471 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:02.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:02.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:02.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:02.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:02.696 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:02.724 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:02.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:02.827 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:02.855 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:02.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:02.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:02.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:02.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:03.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:03.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:03.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:03.233 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:03.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:03.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:03.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:03.393 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:03.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:59:03.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:03.523 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:03.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:03.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:03.638 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:02.477309+0000 mon.a (mon.0) 2039 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.638 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:02.477309+0000 mon.a (mon.0) 2039 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.638 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:02.485147+0000 mon.a (mon.0) 2040 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.638 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:02.485147+0000 mon.a (mon.0) 2040 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.638 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: cluster 2024-09-13T14:59:02.745031+0000 mgr.y (mgr.26252) 986 : cluster [DBG] pgmap v1518: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:03.638 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: cluster 2024-09-13T14:59:02.745031+0000 mgr.y (mgr.26252) 986 : cluster [DBG] pgmap v1518: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:03.638 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:03.109979+0000 mon.a (mon.0) 2041 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:03.638 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:03.109979+0000 mon.a (mon.0) 2041 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:03.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:03.111518+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:03.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:03.111518+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:03.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:03.121604+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:03.121604+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:03.297009+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:03.297009+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:03.299053+0000 mon.a (mon.0) 2045 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:03.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:03 smithi177 bash[68594]: audit 2024-09-13T14:59:03.299053+0000 mon.a (mon.0) 2045 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:03.652 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:03.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:03.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:02.477309+0000 mon.a (mon.0) 2039 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:02.477309+0000 mon.a (mon.0) 2039 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:02.485147+0000 mon.a (mon.0) 2040 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:02.485147+0000 mon.a (mon.0) 2040 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: cluster 2024-09-13T14:59:02.745031+0000 mgr.y (mgr.26252) 986 : cluster [DBG] pgmap v1518: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:03.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: cluster 2024-09-13T14:59:02.745031+0000 mgr.y (mgr.26252) 986 : cluster [DBG] pgmap v1518: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:03.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:03.109979+0000 mon.a (mon.0) 2041 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:03.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:03.109979+0000 mon.a (mon.0) 2041 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:03.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:03.111518+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:03.671 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:03.111518+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:03.671 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:03.121604+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.671 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:03.121604+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.671 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:03.297009+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.671 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:03.297009+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.671 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:03.299053+0000 mon.a (mon.0) 2045 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:03.671 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[84576]: audit 2024-09-13T14:59:03.299053+0000 mon.a (mon.0) 2045 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:03.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:02.477309+0000 mon.a (mon.0) 2039 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.672 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:02.477309+0000 mon.a (mon.0) 2039 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.672 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:02.485147+0000 mon.a (mon.0) 2040 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.672 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:02.485147+0000 mon.a (mon.0) 2040 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.672 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: cluster 2024-09-13T14:59:02.745031+0000 mgr.y (mgr.26252) 986 : cluster [DBG] pgmap v1518: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:03.672 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: cluster 2024-09-13T14:59:02.745031+0000 mgr.y (mgr.26252) 986 : cluster [DBG] pgmap v1518: 89 pgs: 31 active+undersized, 3 active+undersized+degraded, 55 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:59:03.672 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:03.109979+0000 mon.a (mon.0) 2041 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:03.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:03.109979+0000 mon.a (mon.0) 2041 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:03.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:03.111518+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:03.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:03.111518+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:03.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:03.121604+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:03.121604+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:03.297009+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:03.297009+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:03.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:03.299053+0000 mon.a (mon.0) 2045 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:03.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:03 smithi099 bash[87147]: audit 2024-09-13T14:59:03.299053+0000 mon.a (mon.0) 2045 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:03.755 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:03.783 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:03.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:03.887 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:03.908 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:03.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:04.011 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:04.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:04.142 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:04.169 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:04.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:04.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:04.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:04.395 DEBUG:teuthology.orchestra.run.smithi099:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-09-13T14:59:04.396 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2024-09-13T14:59:04.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:59:04.399 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:59:04.399 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:59:04.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:04.430 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:04.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:04.469 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:hints = 1 2024-09-13T14:59:04.469 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:59:04.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object prefix: benchmark_data_smithi099_171278 2024-09-13T14:59:04.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:59:04.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:59:04.503 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T14:59:04.532 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:04.564 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:04.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:04 smithi099 bash[87147]: audit 2024-09-13T14:59:03.973250+0000 mon.a (mon.0) 2046 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:04.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:04 smithi099 bash[87147]: audit 2024-09-13T14:59:03.973250+0000 mon.a (mon.0) 2046 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:04.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:04.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:04 smithi099 bash[84576]: audit 2024-09-13T14:59:03.973250+0000 mon.a (mon.0) 2046 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:04.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:04 smithi099 bash[84576]: audit 2024-09-13T14:59:03.973250+0000 mon.a (mon.0) 2046 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:04.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:04.713 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:04.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:04.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:04.846 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:04.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:04.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:04 smithi177 bash[68594]: audit 2024-09-13T14:59:03.973250+0000 mon.a (mon.0) 2046 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:04.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:04 smithi177 bash[68594]: audit 2024-09-13T14:59:03.973250+0000 mon.a (mon.0) 2046 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:04.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:04.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:04.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:05.083 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:05.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:05.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:05.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:05.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:05.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:05.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:05.359 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:05.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:05.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:05.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 1 16 1874 1858 116.119 116.125 0.46289 0.00807543 2024-09-13T14:59:05.496 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:05.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:05.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:05.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:05.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:05.734 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:05.746 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[84576]: cluster 2024-09-13T14:59:04.488499+0000 mon.a (mon.0) 2047 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:05.746 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[84576]: cluster 2024-09-13T14:59:04.488499+0000 mon.a (mon.0) 2047 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:05.746 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[84576]: audit 2024-09-13T14:59:04.499532+0000 mon.a (mon.0) 2048 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:59:05.746 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[84576]: audit 2024-09-13T14:59:04.499532+0000 mon.a (mon.0) 2048 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:59:05.746 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[84576]: cluster 2024-09-13T14:59:04.504427+0000 mon.a (mon.0) 2049 : cluster [DBG] osdmap e1043: 8 total, 7 up, 8 in 2024-09-13T14:59:05.746 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[84576]: cluster 2024-09-13T14:59:04.504427+0000 mon.a (mon.0) 2049 : cluster [DBG] osdmap e1043: 8 total, 7 up, 8 in 2024-09-13T14:59:05.746 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[84576]: cluster 2024-09-13T14:59:04.745677+0000 mgr.y (mgr.26252) 987 : cluster [DBG] pgmap v1520: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:05.746 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[84576]: cluster 2024-09-13T14:59:04.745677+0000 mgr.y (mgr.26252) 987 : cluster [DBG] pgmap v1520: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:05.747 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[87147]: cluster 2024-09-13T14:59:04.488499+0000 mon.a (mon.0) 2047 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:05.747 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[87147]: cluster 2024-09-13T14:59:04.488499+0000 mon.a (mon.0) 2047 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:05.747 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[87147]: audit 2024-09-13T14:59:04.499532+0000 mon.a (mon.0) 2048 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:59:05.747 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[87147]: audit 2024-09-13T14:59:04.499532+0000 mon.a (mon.0) 2048 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:59:05.747 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[87147]: cluster 2024-09-13T14:59:04.504427+0000 mon.a (mon.0) 2049 : cluster [DBG] osdmap e1043: 8 total, 7 up, 8 in 2024-09-13T14:59:05.747 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[87147]: cluster 2024-09-13T14:59:04.504427+0000 mon.a (mon.0) 2049 : cluster [DBG] osdmap e1043: 8 total, 7 up, 8 in 2024-09-13T14:59:05.747 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[87147]: cluster 2024-09-13T14:59:04.745677+0000 mgr.y (mgr.26252) 987 : cluster [DBG] pgmap v1520: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:05.747 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:05 smithi099 bash[87147]: cluster 2024-09-13T14:59:04.745677+0000 mgr.y (mgr.26252) 987 : cluster [DBG] pgmap v1520: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:05.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:05.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:05 smithi177 bash[68594]: cluster 2024-09-13T14:59:04.488499+0000 mon.a (mon.0) 2047 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:05.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:05 smithi177 bash[68594]: cluster 2024-09-13T14:59:04.488499+0000 mon.a (mon.0) 2047 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:05 smithi177 bash[68594]: audit 2024-09-13T14:59:04.499532+0000 mon.a (mon.0) 2048 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:59:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:05 smithi177 bash[68594]: audit 2024-09-13T14:59:04.499532+0000 mon.a (mon.0) 2048 : audit [INF] from='client.? 172.21.15.99:0/380216146' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:59:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:05 smithi177 bash[68594]: cluster 2024-09-13T14:59:04.504427+0000 mon.a (mon.0) 2049 : cluster [DBG] osdmap e1043: 8 total, 7 up, 8 in 2024-09-13T14:59:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:05 smithi177 bash[68594]: cluster 2024-09-13T14:59:04.504427+0000 mon.a (mon.0) 2049 : cluster [DBG] osdmap e1043: 8 total, 7 up, 8 in 2024-09-13T14:59:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:05 smithi177 bash[68594]: cluster 2024-09-13T14:59:04.745677+0000 mgr.y (mgr.26252) 987 : cluster [DBG] pgmap v1520: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:05 smithi177 bash[68594]: cluster 2024-09-13T14:59:04.745677+0000 mgr.y (mgr.26252) 987 : cluster [DBG] pgmap v1520: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:05.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:05.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:05.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:06.000 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:06.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:06.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:06.129 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:06.155 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:06.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:06.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:06.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:06.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:06.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:06.416 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:06.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:06.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 2 16 5380 5364 167.612 219.125 0.00191801 0.00595397 2024-09-13T14:59:06.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:06.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:06 smithi099 bash[84576]: cluster 2024-09-13T14:59:05.505224+0000 mon.a (mon.0) 2050 : cluster [WRN] Health check update: Degraded data redundancy: 53/492 objects degraded (10.772%), 4 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:06.531 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:06 smithi099 bash[84576]: cluster 2024-09-13T14:59:05.505224+0000 mon.a (mon.0) 2050 : cluster [WRN] Health check update: Degraded data redundancy: 53/492 objects degraded (10.772%), 4 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:06.531 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:06 smithi099 bash[87147]: cluster 2024-09-13T14:59:05.505224+0000 mon.a (mon.0) 2050 : cluster [WRN] Health check update: Degraded data redundancy: 53/492 objects degraded (10.772%), 4 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:06.531 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:06 smithi099 bash[87147]: cluster 2024-09-13T14:59:05.505224+0000 mon.a (mon.0) 2050 : cluster [WRN] Health check update: Degraded data redundancy: 53/492 objects degraded (10.772%), 4 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:06.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:06.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:06.648 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:06.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:06.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:06.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:06.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:06.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:06.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:06 smithi177 bash[68594]: cluster 2024-09-13T14:59:05.505224+0000 mon.a (mon.0) 2050 : cluster [WRN] Health check update: Degraded data redundancy: 53/492 objects degraded (10.772%), 4 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:06.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:06 smithi177 bash[68594]: cluster 2024-09-13T14:59:05.505224+0000 mon.a (mon.0) 2050 : cluster [WRN] Health check update: Degraded data redundancy: 53/492 objects degraded (10.772%), 4 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:06.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:06.938 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:06.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:07.042 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:07.070 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:07.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:07.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:07.201 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:07.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:07.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:07.316 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:59:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:59:07] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.51.0" 2024-09-13T14:59:07.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:07.432 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:07.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:07.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:07.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 3 16 8790 8774 182.777 213.125 0.00622395 0.00546179 2024-09-13T14:59:07.480 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:07.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:07.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:07 smithi099 bash[84576]: cluster 2024-09-13T14:59:06.746480+0000 mgr.y (mgr.26252) 988 : cluster [DBG] pgmap v1521: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:07.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:07 smithi099 bash[84576]: cluster 2024-09-13T14:59:06.746480+0000 mgr.y (mgr.26252) 988 : cluster [DBG] pgmap v1521: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:07.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:07 smithi099 bash[87147]: cluster 2024-09-13T14:59:06.746480+0000 mgr.y (mgr.26252) 988 : cluster [DBG] pgmap v1521: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:07.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:07 smithi099 bash[87147]: cluster 2024-09-13T14:59:06.746480+0000 mgr.y (mgr.26252) 988 : cluster [DBG] pgmap v1521: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:07.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:07.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:07.690 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:07.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:07.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:07.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:07.837 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:07 smithi177 bash[68594]: cluster 2024-09-13T14:59:06.746480+0000 mgr.y (mgr.26252) 988 : cluster [DBG] pgmap v1521: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:07.837 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:07 smithi177 bash[68594]: cluster 2024-09-13T14:59:06.746480+0000 mgr.y (mgr.26252) 988 : cluster [DBG] pgmap v1521: 89 pgs: 30 active+undersized, 4 active+undersized+degraded, 55 active+clean; 9.2 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 17 op/s; 53/492 objects degraded (10.772%) 2024-09-13T14:59:07.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:07.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:07.961 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:07.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:07.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:07.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:59:08.094 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:08.120 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:08.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:08.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:08.251 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:08.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:08.354 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:08.382 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:08.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 4 16 11860 11844 185.048 191.875 0.00566298 0.00539673 2024-09-13T14:59:08.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:08.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:08.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:08.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:08.646 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:08.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:08.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:08.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:08.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:08.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:08.903 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:08.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:09.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:09.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:09.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:09.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:09.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:09.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:09.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:09.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:09.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:09.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:09.423 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:09.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:09.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 5 16 14751 14735 184.173 180.688 0.00534295 0.00542193 2024-09-13T14:59:09.513 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:09.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:09.564 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:09.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:09.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:09.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:09.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:09.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:09.830 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:09.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:09.959 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:09.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:10.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:10.072 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:09 smithi177 bash[68594]: cluster 2024-09-13T14:59:08.747335+0000 mgr.y (mgr.26252) 989 : cluster [DBG] pgmap v1522: 89 pgs: 29 active+undersized, 5 active+undersized+degraded, 55 active+clean; 108 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s wr, 193 op/s; 575/5214 objects degraded (11.028%) 2024-09-13T14:59:10.072 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:09 smithi177 bash[68594]: cluster 2024-09-13T14:59:08.747335+0000 mgr.y (mgr.26252) 989 : cluster [DBG] pgmap v1522: 89 pgs: 29 active+undersized, 5 active+undersized+degraded, 55 active+clean; 108 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s wr, 193 op/s; 575/5214 objects degraded (11.028%) 2024-09-13T14:59:10.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:09 smithi099 bash[84576]: cluster 2024-09-13T14:59:08.747335+0000 mgr.y (mgr.26252) 989 : cluster [DBG] pgmap v1522: 89 pgs: 29 active+undersized, 5 active+undersized+degraded, 55 active+clean; 108 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s wr, 193 op/s; 575/5214 objects degraded (11.028%) 2024-09-13T14:59:10.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:09 smithi099 bash[84576]: cluster 2024-09-13T14:59:08.747335+0000 mgr.y (mgr.26252) 989 : cluster [DBG] pgmap v1522: 89 pgs: 29 active+undersized, 5 active+undersized+degraded, 55 active+clean; 108 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s wr, 193 op/s; 575/5214 objects degraded (11.028%) 2024-09-13T14:59:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:09 smithi099 bash[87147]: cluster 2024-09-13T14:59:08.747335+0000 mgr.y (mgr.26252) 989 : cluster [DBG] pgmap v1522: 89 pgs: 29 active+undersized, 5 active+undersized+degraded, 55 active+clean; 108 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s wr, 193 op/s; 575/5214 objects degraded (11.028%) 2024-09-13T14:59:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:09 smithi099 bash[87147]: cluster 2024-09-13T14:59:08.747335+0000 mgr.y (mgr.26252) 989 : cluster [DBG] pgmap v1522: 89 pgs: 29 active+undersized, 5 active+undersized+degraded, 55 active+clean; 108 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s wr, 193 op/s; 575/5214 objects degraded (11.028%) 2024-09-13T14:59:10.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:10.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:10.198 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:10.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:10.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:10.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:59:10 smithi177 bash[61439]: ts=2024-09-13T14:59:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:10.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:10.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:10.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 6 16 17294 17278 179.965 158.938 0.00494101 0.00555051 2024-09-13T14:59:10.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:10.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:10.522 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:10.596 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:10.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:10.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:10.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:10.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:10.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:10.855 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:10.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:10.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:10.998 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:10 smithi177 bash[68594]: cluster 2024-09-13T14:59:10.802843+0000 mon.a (mon.0) 2051 : cluster [WRN] Health check update: Degraded data redundancy: 4138/40311 objects degraded (10.265%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:10.998 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:10 smithi177 bash[68594]: cluster 2024-09-13T14:59:10.802843+0000 mon.a (mon.0) 2051 : cluster [WRN] Health check update: Degraded data redundancy: 4138/40311 objects degraded (10.265%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:11.018 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:11.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:11.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:10 smithi099 bash[84576]: cluster 2024-09-13T14:59:10.802843+0000 mon.a (mon.0) 2051 : cluster [WRN] Health check update: Degraded data redundancy: 4138/40311 objects degraded (10.265%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:11.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:10 smithi099 bash[84576]: cluster 2024-09-13T14:59:10.802843+0000 mon.a (mon.0) 2051 : cluster [WRN] Health check update: Degraded data redundancy: 4138/40311 objects degraded (10.265%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:11.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:10 smithi099 bash[87147]: cluster 2024-09-13T14:59:10.802843+0000 mon.a (mon.0) 2051 : cluster [WRN] Health check update: Degraded data redundancy: 4138/40311 objects degraded (10.265%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:11.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:10 smithi099 bash[87147]: cluster 2024-09-13T14:59:10.802843+0000 mon.a (mon.0) 2051 : cluster [WRN] Health check update: Degraded data redundancy: 4138/40311 objects degraded (10.265%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:11.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:11.149 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:11.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:11.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:11.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:11.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:11.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:11.408 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:11.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:11.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 7 16 19523 19507 174.155 139.312 0.00274261 0.0057348 2024-09-13T14:59:11.512 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:11.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:11.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:11.632 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:11.656 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:11.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:11.759 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:11.781 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:11.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:11.883 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:11.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:12.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:12.042 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:12.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:12.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:11 smithi177 bash[68594]: cluster 2024-09-13T14:59:10.748096+0000 mgr.y (mgr.26252) 990 : cluster [DBG] pgmap v1523: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 839 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 4138/40311 objects degraded (10.265%) 2024-09-13T14:59:12.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:11 smithi177 bash[68594]: cluster 2024-09-13T14:59:10.748096+0000 mgr.y (mgr.26252) 990 : cluster [DBG] pgmap v1523: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 839 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 4138/40311 objects degraded (10.265%) 2024-09-13T14:59:12.147 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:12.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:12.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:12.275 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:12.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:12.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:12 smithi099 bash[84576]: cluster 2024-09-13T14:59:10.748096+0000 mgr.y (mgr.26252) 990 : cluster [DBG] pgmap v1523: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 839 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 4138/40311 objects degraded (10.265%) 2024-09-13T14:59:12.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:12 smithi099 bash[84576]: cluster 2024-09-13T14:59:10.748096+0000 mgr.y (mgr.26252) 990 : cluster [DBG] pgmap v1523: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 839 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 4138/40311 objects degraded (10.265%) 2024-09-13T14:59:12.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:12 smithi099 bash[87147]: cluster 2024-09-13T14:59:10.748096+0000 mgr.y (mgr.26252) 990 : cluster [DBG] pgmap v1523: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 839 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 4138/40311 objects degraded (10.265%) 2024-09-13T14:59:12.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:12 smithi099 bash[87147]: cluster 2024-09-13T14:59:10.748096+0000 mgr.y (mgr.26252) 990 : cluster [DBG] pgmap v1523: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 839 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 4138/40311 objects degraded (10.265%) 2024-09-13T14:59:12.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:12.433 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:12.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:12.470 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 8 16 20097 20081 156.87 35.875 0.0134856 0.00636921 2024-09-13T14:59:12.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:12.554 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:12.563 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:12.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:12.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:12.693 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:12.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:12.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:12.825 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:12.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:12.928 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:12.956 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:12.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:13.059 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:13.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:13.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:13.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:13.206 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:13.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:13.310 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:13.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:13.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:13.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:13.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:13.471 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 9 16 22230 22214 154.251 133.312 0.0136766 0.00647678 2024-09-13T14:59:13.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:13.575 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:13.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:13.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:13.730 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:13.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:13.833 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:13.858 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:13.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:13.962 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:13.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:13 smithi099 bash[84576]: cluster 2024-09-13T14:59:12.748929+0000 mgr.y (mgr.26252) 991 : cluster [DBG] pgmap v1524: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 973 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 4846/46737 objects degraded (10.369%) 2024-09-13T14:59:13.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:13 smithi099 bash[84576]: cluster 2024-09-13T14:59:12.748929+0000 mgr.y (mgr.26252) 991 : cluster [DBG] pgmap v1524: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 973 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 4846/46737 objects degraded (10.369%) 2024-09-13T14:59:13.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:13 smithi099 bash[87147]: cluster 2024-09-13T14:59:12.748929+0000 mgr.y (mgr.26252) 991 : cluster [DBG] pgmap v1524: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 973 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 4846/46737 objects degraded (10.369%) 2024-09-13T14:59:13.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:13 smithi099 bash[87147]: cluster 2024-09-13T14:59:12.748929+0000 mgr.y (mgr.26252) 991 : cluster [DBG] pgmap v1524: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 973 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 4846/46737 objects degraded (10.369%) 2024-09-13T14:59:13.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:14.092 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:14.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:14.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:14.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:14.225 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:13 smithi177 bash[68594]: cluster 2024-09-13T14:59:12.748929+0000 mgr.y (mgr.26252) 991 : cluster [DBG] pgmap v1524: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 973 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 4846/46737 objects degraded (10.369%) 2024-09-13T14:59:14.225 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:13 smithi177 bash[68594]: cluster 2024-09-13T14:59:12.748929+0000 mgr.y (mgr.26252) 991 : cluster [DBG] pgmap v1524: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 973 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 4846/46737 objects degraded (10.369%) 2024-09-13T14:59:14.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:14.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:14.349 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:14.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:14.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 10 16 24441 24425 152.643 138.188 0.00252576 0.00654107 2024-09-13T14:59:14.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:14.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:14.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:14.604 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:14.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:14.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:14.737 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:14.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:14.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:14.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:14.886 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:14.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:14.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:15.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:15.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:15.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:15.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:15.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:15.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:15.263 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:15.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:15.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:15.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:15.471 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 11 16 25361 25345 143.993 57.5 0.00504579 0.00693797 2024-09-13T14:59:15.497 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:15.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.533 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:15.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:15.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:15.670 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:15.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:15.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:15.792 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:15.895 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:15.923 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:15.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:16.026 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:16.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:15 smithi177 bash[68594]: cluster 2024-09-13T14:59:14.750258+0000 mgr.y (mgr.26252) 992 : cluster [DBG] pgmap v1525: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:16.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:15 smithi177 bash[68594]: cluster 2024-09-13T14:59:14.750258+0000 mgr.y (mgr.26252) 992 : cluster [DBG] pgmap v1525: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:16.056 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:16.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:16.187 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:16.292 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:16.322 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:16.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:15 smithi099 bash[84576]: cluster 2024-09-13T14:59:14.750258+0000 mgr.y (mgr.26252) 992 : cluster [DBG] pgmap v1525: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:16.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:15 smithi099 bash[84576]: cluster 2024-09-13T14:59:14.750258+0000 mgr.y (mgr.26252) 992 : cluster [DBG] pgmap v1525: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:16.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:15 smithi099 bash[87147]: cluster 2024-09-13T14:59:14.750258+0000 mgr.y (mgr.26252) 992 : cluster [DBG] pgmap v1525: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:16.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:15 smithi099 bash[87147]: cluster 2024-09-13T14:59:14.750258+0000 mgr.y (mgr.26252) 992 : cluster [DBG] pgmap v1525: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:16.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:16.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:16.451 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:16.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:16.471 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 12 16 27390 27374 142.561 126.812 0.00629471 0.00700858 2024-09-13T14:59:16.555 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:16.589 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:16.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:16.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:16.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:16.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:16.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:16.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:16.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:16.978 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:16.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:17.082 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:17.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:16 smithi177 bash[68594]: cluster 2024-09-13T14:59:16.443134+0000 mon.a (mon.0) 2052 : cluster [WRN] Health check update: Degraded data redundancy: 7364/70305 objects degraded (10.474%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:17.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:16 smithi177 bash[68594]: cluster 2024-09-13T14:59:16.443134+0000 mon.a (mon.0) 2052 : cluster [WRN] Health check update: Degraded data redundancy: 7364/70305 objects degraded (10.474%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:17.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:17.143 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:16 smithi099 bash[84576]: cluster 2024-09-13T14:59:16.443134+0000 mon.a (mon.0) 2052 : cluster [WRN] Health check update: Degraded data redundancy: 7364/70305 objects degraded (10.474%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:17.143 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:16 smithi099 bash[84576]: cluster 2024-09-13T14:59:16.443134+0000 mon.a (mon.0) 2052 : cluster [WRN] Health check update: Degraded data redundancy: 7364/70305 objects degraded (10.474%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:17.143 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:16 smithi099 bash[87147]: cluster 2024-09-13T14:59:16.443134+0000 mon.a (mon.0) 2052 : cluster [WRN] Health check update: Degraded data redundancy: 7364/70305 objects degraded (10.474%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:17.144 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:16 smithi099 bash[87147]: cluster 2024-09-13T14:59:16.443134+0000 mon.a (mon.0) 2052 : cluster [WRN] Health check update: Degraded data redundancy: 7364/70305 objects degraded (10.474%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:17.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:17.222 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:59:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:59:17] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.51.0" 2024-09-13T14:59:17.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:17.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:17.338 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:17.361 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:17.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:17.467 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:17.471 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 13 16 29633 29617 142.377 140.188 0.00401969 0.00701947 2024-09-13T14:59:17.492 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:17.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:17.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:17.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:17.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:17.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:17.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:17.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:17.887 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:17.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:17.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:18.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:18.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:18.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:18.135 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:17 smithi099 bash[84576]: cluster 2024-09-13T14:59:16.750910+0000 mgr.y (mgr.26252) 993 : cluster [DBG] pgmap v1526: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:18.135 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:17 smithi099 bash[84576]: cluster 2024-09-13T14:59:16.750910+0000 mgr.y (mgr.26252) 993 : cluster [DBG] pgmap v1526: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:18.135 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:17 smithi099 bash[87147]: cluster 2024-09-13T14:59:16.750910+0000 mgr.y (mgr.26252) 993 : cluster [DBG] pgmap v1526: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:18.135 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:17 smithi099 bash[87147]: cluster 2024-09-13T14:59:16.750910+0000 mgr.y (mgr.26252) 993 : cluster [DBG] pgmap v1526: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:18.152 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:18.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:18.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:18.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:18.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:18.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:17 smithi177 bash[68594]: cluster 2024-09-13T14:59:16.750910+0000 mgr.y (mgr.26252) 993 : cluster [DBG] pgmap v1526: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:18.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:17 smithi177 bash[68594]: cluster 2024-09-13T14:59:16.750910+0000 mgr.y (mgr.26252) 993 : cluster [DBG] pgmap v1526: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s; 7364/70305 objects degraded (10.474%) 2024-09-13T14:59:18.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:18.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:18.471 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 14 16 31824 31808 141.988 136.938 0.0027634 0.00703736 2024-09-13T14:59:18.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:18.547 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:18.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:18.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:18.675 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:18.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:18.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:18.803 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:18.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:18.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:18.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:18.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:19.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:19.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:19.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:19.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:19.302 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:19.325 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:19.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:19.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:19.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:19.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:19.471 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 15 16 33810 33794 140.796 124.125 0.00643653 0.00709803 2024-09-13T14:59:19.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:19.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:19 smithi177 bash[68594]: audit 2024-09-13T14:59:18.295327+0000 mon.a (mon.0) 2053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:19.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:19 smithi177 bash[68594]: audit 2024-09-13T14:59:18.295327+0000 mon.a (mon.0) 2053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:19.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:19 smithi177 bash[68594]: audit 2024-09-13T14:59:18.298205+0000 mon.a (mon.0) 2054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:19.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:19 smithi177 bash[68594]: audit 2024-09-13T14:59:18.298205+0000 mon.a (mon.0) 2054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:19.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:19 smithi177 bash[68594]: cluster 2024-09-13T14:59:18.751629+0000 mgr.y (mgr.26252) 994 : cluster [DBG] pgmap v1527: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 7982/75816 objects degraded (10.528%) 2024-09-13T14:59:19.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:19 smithi177 bash[68594]: cluster 2024-09-13T14:59:18.751629+0000 mgr.y (mgr.26252) 994 : cluster [DBG] pgmap v1527: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 7982/75816 objects degraded (10.528%) 2024-09-13T14:59:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[84576]: audit 2024-09-13T14:59:18.295327+0000 mon.a (mon.0) 2053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[84576]: audit 2024-09-13T14:59:18.295327+0000 mon.a (mon.0) 2053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[84576]: audit 2024-09-13T14:59:18.298205+0000 mon.a (mon.0) 2054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[84576]: audit 2024-09-13T14:59:18.298205+0000 mon.a (mon.0) 2054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[84576]: cluster 2024-09-13T14:59:18.751629+0000 mgr.y (mgr.26252) 994 : cluster [DBG] pgmap v1527: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 7982/75816 objects degraded (10.528%) 2024-09-13T14:59:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[84576]: cluster 2024-09-13T14:59:18.751629+0000 mgr.y (mgr.26252) 994 : cluster [DBG] pgmap v1527: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 7982/75816 objects degraded (10.528%) 2024-09-13T14:59:19.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[87147]: audit 2024-09-13T14:59:18.295327+0000 mon.a (mon.0) 2053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[87147]: audit 2024-09-13T14:59:18.295327+0000 mon.a (mon.0) 2053 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T14:59:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[87147]: audit 2024-09-13T14:59:18.298205+0000 mon.a (mon.0) 2054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[87147]: audit 2024-09-13T14:59:18.298205+0000 mon.a (mon.0) 2054 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[87147]: cluster 2024-09-13T14:59:18.751629+0000 mgr.y (mgr.26252) 994 : cluster [DBG] pgmap v1527: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 7982/75816 objects degraded (10.528%) 2024-09-13T14:59:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:19 smithi099 bash[87147]: cluster 2024-09-13T14:59:18.751629+0000 mgr.y (mgr.26252) 994 : cluster [DBG] pgmap v1527: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 7982/75816 objects degraded (10.528%) 2024-09-13T14:59:19.576 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:19.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:19.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:19.705 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:19.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:19.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:19.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:19.939 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:19.970 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:19.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:20.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:20.106 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:20.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:20.212 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:20.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:20.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:20.352 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:20.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:20.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:59:20 smithi177 bash[61439]: ts=2024-09-13T14:59:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:20.455 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:20.471 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 16 16 35508 35492 138.629 106.125 0.00712673 0.00720871 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:20.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:20.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:20.602 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:20.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:20.705 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:20.733 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:20.837 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:20.866 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:20.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:20.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:20.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:21.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:21.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:21.134 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:21.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:21.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:21.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:21.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:21.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:21.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:21.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:21.471 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 17 16 36814 36798 135.275 81.625 0.0136689 0.00736615 2024-09-13T14:59:21.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:21.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:21.617 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:21.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:21.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:21.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:21.770 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:21.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:21.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:21.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:21.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:22.009 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:22.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:22.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:22.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:21 smithi099 bash[84576]: cluster 2024-09-13T14:59:20.752299+0000 mgr.y (mgr.26252) 995 : cluster [DBG] pgmap v1528: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 162 MiB/s wr, 2.60k op/s; 10309/98832 objects degraded (10.431%) 2024-09-13T14:59:22.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:21 smithi099 bash[84576]: cluster 2024-09-13T14:59:20.752299+0000 mgr.y (mgr.26252) 995 : cluster [DBG] pgmap v1528: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 162 MiB/s wr, 2.60k op/s; 10309/98832 objects degraded (10.431%) 2024-09-13T14:59:22.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:21 smithi099 bash[84576]: cluster 2024-09-13T14:59:21.443848+0000 mon.a (mon.0) 2055 : cluster [WRN] Health check update: Degraded data redundancy: 10309/98832 objects degraded (10.431%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:22.088 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:21 smithi099 bash[84576]: cluster 2024-09-13T14:59:21.443848+0000 mon.a (mon.0) 2055 : cluster [WRN] Health check update: Degraded data redundancy: 10309/98832 objects degraded (10.431%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:22.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:21 smithi099 bash[87147]: cluster 2024-09-13T14:59:20.752299+0000 mgr.y (mgr.26252) 995 : cluster [DBG] pgmap v1528: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 162 MiB/s wr, 2.60k op/s; 10309/98832 objects degraded (10.431%) 2024-09-13T14:59:22.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:21 smithi099 bash[87147]: cluster 2024-09-13T14:59:20.752299+0000 mgr.y (mgr.26252) 995 : cluster [DBG] pgmap v1528: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 162 MiB/s wr, 2.60k op/s; 10309/98832 objects degraded (10.431%) 2024-09-13T14:59:22.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:21 smithi099 bash[87147]: cluster 2024-09-13T14:59:21.443848+0000 mon.a (mon.0) 2055 : cluster [WRN] Health check update: Degraded data redundancy: 10309/98832 objects degraded (10.431%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:22.089 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:21 smithi099 bash[87147]: cluster 2024-09-13T14:59:21.443848+0000 mon.a (mon.0) 2055 : cluster [WRN] Health check update: Degraded data redundancy: 10309/98832 objects degraded (10.431%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:21 smithi177 bash[68594]: cluster 2024-09-13T14:59:20.752299+0000 mgr.y (mgr.26252) 995 : cluster [DBG] pgmap v1528: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 162 MiB/s wr, 2.60k op/s; 10309/98832 objects degraded (10.431%) 2024-09-13T14:59:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:21 smithi177 bash[68594]: cluster 2024-09-13T14:59:20.752299+0000 mgr.y (mgr.26252) 995 : cluster [DBG] pgmap v1528: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 162 MiB/s wr, 2.60k op/s; 10309/98832 objects degraded (10.431%) 2024-09-13T14:59:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:21 smithi177 bash[68594]: cluster 2024-09-13T14:59:21.443848+0000 mon.a (mon.0) 2055 : cluster [WRN] Health check update: Degraded data redundancy: 10309/98832 objects degraded (10.431%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:21 smithi177 bash[68594]: cluster 2024-09-13T14:59:21.443848+0000 mon.a (mon.0) 2055 : cluster [WRN] Health check update: Degraded data redundancy: 10309/98832 objects degraded (10.431%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:22.131 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.167 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:22.273 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:22.296 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:22.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:22.401 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:22.429 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:22.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 18 16 38271 38255 132.818 91.0625 0.0148895 0.00752382 2024-09-13T14:59:22.533 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:22.659 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:22.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:22.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:22.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:22.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:22.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:22.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:23.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:23.067 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:23.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:23.194 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:23.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:23.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:23.319 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:23.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:23.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:23.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:23.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 19 16 39328 39312 129.304 66.0625 0.0109999 0.00769021 2024-09-13T14:59:23.548 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:23.582 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:23.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:23.689 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:23.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:23.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:23.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:23.835 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:23.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:23.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:23.962 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:23.964 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:23 smithi099 bash[84576]: cluster 2024-09-13T14:59:22.753035+0000 mgr.y (mgr.26252) 996 : cluster [DBG] pgmap v1529: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 10898/103806 objects degraded (10.498%) 2024-09-13T14:59:23.964 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:23 smithi099 bash[84576]: cluster 2024-09-13T14:59:22.753035+0000 mgr.y (mgr.26252) 996 : cluster [DBG] pgmap v1529: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 10898/103806 objects degraded (10.498%) 2024-09-13T14:59:23.965 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:23 smithi099 bash[87147]: cluster 2024-09-13T14:59:22.753035+0000 mgr.y (mgr.26252) 996 : cluster [DBG] pgmap v1529: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 10898/103806 objects degraded (10.498%) 2024-09-13T14:59:23.965 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:23 smithi099 bash[87147]: cluster 2024-09-13T14:59:22.753035+0000 mgr.y (mgr.26252) 996 : cluster [DBG] pgmap v1529: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 10898/103806 objects degraded (10.498%) 2024-09-13T14:59:23.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:24.066 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:24.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:24.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:24.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:23 smithi177 bash[68594]: cluster 2024-09-13T14:59:22.753035+0000 mgr.y (mgr.26252) 996 : cluster [DBG] pgmap v1529: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 10898/103806 objects degraded (10.498%) 2024-09-13T14:59:24.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:23 smithi177 bash[68594]: cluster 2024-09-13T14:59:22.753035+0000 mgr.y (mgr.26252) 996 : cluster [DBG] pgmap v1529: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 10898/103806 objects degraded (10.498%) 2024-09-13T14:59:24.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:24.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:24.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:24.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:24.358 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:24.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:24.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:24.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T14:59:24.473528+0000 min lat: 0.00108369 max lat: 0.468934 avg lat: 0.00787613 2024-09-13T14:59:24.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:59:24.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 20 16 40621 40605 126.879 80.8125 0.00300832 0.00787613 2024-09-13T14:59:24.485 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:24.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:24.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:24.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:24.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:24.728 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:24.752 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:24.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:24.882 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:24.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:24.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:25.009 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:25.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:25.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:25.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:25.249 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:25.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:25.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:25.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:25.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:25.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:25.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 21 16 42068 42052 125.144 90.4375 0.00416624 0.00798472 2024-09-13T14:59:25.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:25.535 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:25.638 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:25.660 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:25.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:25.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:25.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:25.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:25.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:25.906 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:25 smithi099 bash[84576]: cluster 2024-09-13T14:59:24.754147+0000 mgr.y (mgr.26252) 997 : cluster [DBG] pgmap v1530: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:25.906 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:25 smithi099 bash[84576]: cluster 2024-09-13T14:59:24.754147+0000 mgr.y (mgr.26252) 997 : cluster [DBG] pgmap v1530: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:25.906 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:25 smithi099 bash[87147]: cluster 2024-09-13T14:59:24.754147+0000 mgr.y (mgr.26252) 997 : cluster [DBG] pgmap v1530: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:25.906 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:25 smithi099 bash[87147]: cluster 2024-09-13T14:59:24.754147+0000 mgr.y (mgr.26252) 997 : cluster [DBG] pgmap v1530: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:25.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:25.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:26.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:26.058 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:26.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:25 smithi177 bash[68594]: cluster 2024-09-13T14:59:24.754147+0000 mgr.y (mgr.26252) 997 : cluster [DBG] pgmap v1530: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:26.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:25 smithi177 bash[68594]: cluster 2024-09-13T14:59:24.754147+0000 mgr.y (mgr.26252) 997 : cluster [DBG] pgmap v1530: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:26.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:26.181 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:26.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:26.283 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:26.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:26.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:26.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:26.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:26.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:26.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 22 15 43128 43113 122.469 66.3125 0.0112592 0.00816068 2024-09-13T14:59:26.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:26.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:26.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:26.683 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:26.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:26.814 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:26.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:26.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:26.947 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:26.956 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:26 smithi099 bash[84576]: cluster 2024-09-13T14:59:26.448185+0000 mon.a (mon.0) 2056 : cluster [WRN] Health check update: Degraded data redundancy: 12664/120297 objects degraded (10.527%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:26.956 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:26 smithi099 bash[84576]: cluster 2024-09-13T14:59:26.448185+0000 mon.a (mon.0) 2056 : cluster [WRN] Health check update: Degraded data redundancy: 12664/120297 objects degraded (10.527%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:26.957 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:26 smithi099 bash[87147]: cluster 2024-09-13T14:59:26.448185+0000 mon.a (mon.0) 2056 : cluster [WRN] Health check update: Degraded data redundancy: 12664/120297 objects degraded (10.527%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:26.957 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:26 smithi099 bash[87147]: cluster 2024-09-13T14:59:26.448185+0000 mon.a (mon.0) 2056 : cluster [WRN] Health check update: Degraded data redundancy: 12664/120297 objects degraded (10.527%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:26.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:26.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:27.078 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:27.087 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:26 smithi177 bash[68594]: cluster 2024-09-13T14:59:26.448185+0000 mon.a (mon.0) 2056 : cluster [WRN] Health check update: Degraded data redundancy: 12664/120297 objects degraded (10.527%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:27.088 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:26 smithi177 bash[68594]: cluster 2024-09-13T14:59:26.448185+0000 mon.a (mon.0) 2056 : cluster [WRN] Health check update: Degraded data redundancy: 12664/120297 objects degraded (10.527%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:27.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:27.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:27.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:27.221 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:59:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:59:27] "GET /metrics HTTP/1.1" 200 36763 "" "Prometheus/2.51.0" 2024-09-13T14:59:27.237 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:27.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:27.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:27.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:27.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:27.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:27.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 23 16 44698 44682 121.407 98.0625 0.00217429 0.00819277 2024-09-13T14:59:27.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:27.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:27.593 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:27.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:27.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:27.721 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:27.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:27.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:27.889 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:27.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:27.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:28.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:28.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:27 smithi099 bash[84576]: cluster 2024-09-13T14:59:26.754868+0000 mgr.y (mgr.26252) 998 : cluster [DBG] pgmap v1531: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:28.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:27 smithi099 bash[84576]: cluster 2024-09-13T14:59:26.754868+0000 mgr.y (mgr.26252) 998 : cluster [DBG] pgmap v1531: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:28.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:27 smithi099 bash[87147]: cluster 2024-09-13T14:59:26.754868+0000 mgr.y (mgr.26252) 998 : cluster [DBG] pgmap v1531: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:28.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:27 smithi099 bash[87147]: cluster 2024-09-13T14:59:26.754868+0000 mgr.y (mgr.26252) 998 : cluster [DBG] pgmap v1531: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:28.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:27 smithi177 bash[68594]: cluster 2024-09-13T14:59:26.754868+0000 mgr.y (mgr.26252) 998 : cluster [DBG] pgmap v1531: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:28.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:27 smithi177 bash[68594]: cluster 2024-09-13T14:59:26.754868+0000 mgr.y (mgr.26252) 998 : cluster [DBG] pgmap v1531: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 12664/120297 objects degraded (10.527%) 2024-09-13T14:59:28.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:28.149 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:28.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:28.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:28.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.408 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:28.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 24 16 45841 45825 119.325 71.4375 0.00613898 0.00837518 2024-09-13T14:59:28.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:28.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:28.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:28.667 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:28.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:28.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:28.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:29.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:29.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.074 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:29.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:29.175 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:29.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:29.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:29.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:29.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:29.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:29.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:29.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:29.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:29.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 25 16 47470 47454 118.624 101.812 0.00601237 0.00842429 2024-09-13T14:59:29.573 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:29.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:29.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:29.709 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:29.728 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:29.830 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:29.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:29.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:29.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:30.004 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:30.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:30.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:29 smithi099 bash[84576]: cluster 2024-09-13T14:59:28.755593+0000 mgr.y (mgr.26252) 999 : cluster [DBG] pgmap v1532: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 13094/124218 objects degraded (10.541%) 2024-09-13T14:59:30.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:29 smithi099 bash[84576]: cluster 2024-09-13T14:59:28.755593+0000 mgr.y (mgr.26252) 999 : cluster [DBG] pgmap v1532: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 13094/124218 objects degraded (10.541%) 2024-09-13T14:59:30.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:29 smithi099 bash[87147]: cluster 2024-09-13T14:59:28.755593+0000 mgr.y (mgr.26252) 999 : cluster [DBG] pgmap v1532: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 13094/124218 objects degraded (10.541%) 2024-09-13T14:59:30.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:29 smithi099 bash[87147]: cluster 2024-09-13T14:59:28.755593+0000 mgr.y (mgr.26252) 999 : cluster [DBG] pgmap v1532: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 13094/124218 objects degraded (10.541%) 2024-09-13T14:59:30.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:30.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:29 smithi177 bash[68594]: cluster 2024-09-13T14:59:28.755593+0000 mgr.y (mgr.26252) 999 : cluster [DBG] pgmap v1532: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 13094/124218 objects degraded (10.541%) 2024-09-13T14:59:30.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:29 smithi177 bash[68594]: cluster 2024-09-13T14:59:28.755593+0000 mgr.y (mgr.26252) 999 : cluster [DBG] pgmap v1532: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 13094/124218 objects degraded (10.541%) 2024-09-13T14:59:30.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:30.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:30.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:30.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:30.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:30.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:30.385 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:30.472 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 26 16 48925 48909 117.559 90.9375 0.0132899 0.00850119 2024-09-13T14:59:30.491 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:30.519 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:30.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:30.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:59:30 smithi177 bash[61439]: ts=2024-09-13T14:59:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:30.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:30.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:30.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:30.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:30.779 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:30.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:30.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:30.912 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:30.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:31.017 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:31.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:31.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:31.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:31.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:31.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:31.275 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:31.303 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:31.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:31.408 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:31.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:31.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:31.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 27 16 50200 50184 116.156 79.6875 0.0177101 0.00855041 2024-09-13T14:59:31.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:31.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:31.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:31.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:31.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:31.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:31.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:31.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:31.943 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:31.972 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:32.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:32.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:32.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:32.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:31 smithi177 bash[68594]: cluster 2024-09-13T14:59:30.756361+0000 mgr.y (mgr.26252) 1000 : cluster [DBG] pgmap v1533: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s; 14756/140631 objects degraded (10.493%) 2024-09-13T14:59:32.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:31 smithi177 bash[68594]: cluster 2024-09-13T14:59:30.756361+0000 mgr.y (mgr.26252) 1000 : cluster [DBG] pgmap v1533: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s; 14756/140631 objects degraded (10.493%) 2024-09-13T14:59:32.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:31 smithi177 bash[68594]: cluster 2024-09-13T14:59:31.450016+0000 mon.a (mon.0) 2057 : cluster [WRN] Health check update: Degraded data redundancy: 14756/140631 objects degraded (10.493%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:32.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:31 smithi177 bash[68594]: cluster 2024-09-13T14:59:31.450016+0000 mon.a (mon.0) 2057 : cluster [WRN] Health check update: Degraded data redundancy: 14756/140631 objects degraded (10.493%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:32.114 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:31 smithi099 bash[84576]: cluster 2024-09-13T14:59:30.756361+0000 mgr.y (mgr.26252) 1000 : cluster [DBG] pgmap v1533: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s; 14756/140631 objects degraded (10.493%) 2024-09-13T14:59:32.117 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:31 smithi099 bash[84576]: cluster 2024-09-13T14:59:30.756361+0000 mgr.y (mgr.26252) 1000 : cluster [DBG] pgmap v1533: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s; 14756/140631 objects degraded (10.493%) 2024-09-13T14:59:32.117 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:31 smithi099 bash[84576]: cluster 2024-09-13T14:59:31.450016+0000 mon.a (mon.0) 2057 : cluster [WRN] Health check update: Degraded data redundancy: 14756/140631 objects degraded (10.493%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:32.117 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:31 smithi099 bash[84576]: cluster 2024-09-13T14:59:31.450016+0000 mon.a (mon.0) 2057 : cluster [WRN] Health check update: Degraded data redundancy: 14756/140631 objects degraded (10.493%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:32.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:31 smithi099 bash[87147]: cluster 2024-09-13T14:59:30.756361+0000 mgr.y (mgr.26252) 1000 : cluster [DBG] pgmap v1533: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s; 14756/140631 objects degraded (10.493%) 2024-09-13T14:59:32.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:31 smithi099 bash[87147]: cluster 2024-09-13T14:59:30.756361+0000 mgr.y (mgr.26252) 1000 : cluster [DBG] pgmap v1533: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s; 14756/140631 objects degraded (10.493%) 2024-09-13T14:59:32.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:31 smithi099 bash[87147]: cluster 2024-09-13T14:59:31.450016+0000 mon.a (mon.0) 2057 : cluster [WRN] Health check update: Degraded data redundancy: 14756/140631 objects degraded (10.493%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:32.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:31 smithi099 bash[87147]: cluster 2024-09-13T14:59:31.450016+0000 mon.a (mon.0) 2057 : cluster [WRN] Health check update: Degraded data redundancy: 14756/140631 objects degraded (10.493%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:32.196 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:32.215 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:32.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:32.344 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:32.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:32.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:32.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 28 16 51302 51286 114.467 68.875 0.0104365 0.00873111 2024-09-13T14:59:32.493 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:32.597 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:32.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:32.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:32.721 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:32.755 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:32.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:32.860 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:32.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:32.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:32.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:33.032 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:33.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:33.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:33.157 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:33.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:33.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:33.287 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:33.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:33.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:33.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:33.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:33.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 29 16 52719 52703 113.574 88.5625 0.00996426 0.00880002 2024-09-13T14:59:33.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:33.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:33.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:33.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:33.683 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:33.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:33.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:33.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:33.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:33.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:33.942 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:33.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:33.974 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:59:33.975 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T14:59:34.046 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.056 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:33 smithi177 bash[68594]: cluster 2024-09-13T14:59:32.757076+0000 mgr.y (mgr.26252) 1001 : cluster [DBG] pgmap v1534: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 15169/144603 objects degraded (10.490%) 2024-09-13T14:59:34.056 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:33 smithi177 bash[68594]: cluster 2024-09-13T14:59:32.757076+0000 mgr.y (mgr.26252) 1001 : cluster [DBG] pgmap v1534: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 15169/144603 objects degraded (10.490%) 2024-09-13T14:59:34.056 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:33 smithi177 bash[68594]: audit 2024-09-13T14:59:33.289371+0000 mon.a (mon.0) 2058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:34.056 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:33 smithi177 bash[68594]: audit 2024-09-13T14:59:33.289371+0000 mon.a (mon.0) 2058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:34.067 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:34.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:34.199 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T14:59:34.200 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:59:34.217 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:34.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:34.234 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T14:59:34.234 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:59:34.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T14:59:34.259 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T14:59:34.260 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:59:34.291 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T14:59:34.291 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:59:34.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:33 smithi099 bash[84576]: cluster 2024-09-13T14:59:32.757076+0000 mgr.y (mgr.26252) 1001 : cluster [DBG] pgmap v1534: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 15169/144603 objects degraded (10.490%) 2024-09-13T14:59:34.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:33 smithi099 bash[84576]: cluster 2024-09-13T14:59:32.757076+0000 mgr.y (mgr.26252) 1001 : cluster [DBG] pgmap v1534: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 15169/144603 objects degraded (10.490%) 2024-09-13T14:59:34.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:33 smithi099 bash[84576]: audit 2024-09-13T14:59:33.289371+0000 mon.a (mon.0) 2058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:34.304 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:33 smithi099 bash[84576]: audit 2024-09-13T14:59:33.289371+0000 mon.a (mon.0) 2058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:34.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:33 smithi099 bash[87147]: cluster 2024-09-13T14:59:32.757076+0000 mgr.y (mgr.26252) 1001 : cluster [DBG] pgmap v1534: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 15169/144603 objects degraded (10.490%) 2024-09-13T14:59:34.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:33 smithi099 bash[87147]: cluster 2024-09-13T14:59:32.757076+0000 mgr.y (mgr.26252) 1001 : cluster [DBG] pgmap v1534: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 15169/144603 objects degraded (10.490%) 2024-09-13T14:59:34.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:33 smithi099 bash[87147]: audit 2024-09-13T14:59:33.289371+0000 mon.a (mon.0) 2058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:34.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:33 smithi099 bash[87147]: audit 2024-09-13T14:59:33.289371+0000 mon.a (mon.0) 2058 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:34.322 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T14:59:34.322 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:59:34.324 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:34.347 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T14:59:34.347 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:59:34.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:34.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:34.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:34.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 30 16 53943 53927 112.337 76.5 0.0108491 0.00889612 2024-09-13T14:59:34.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:34.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:34.580 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:34.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:34.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:34.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.747 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:34.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:34.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:34.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:34.937 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:34 smithi099 bash[84576]: audit 2024-09-13T14:59:33.958756+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.99:0/1915842629' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:34.937 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:34 smithi099 bash[87147]: audit 2024-09-13T14:59:33.958756+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.99:0/1915842629' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:34.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:35.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:35.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:35.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:34 smithi177 bash[68594]: audit 2024-09-13T14:59:33.958756+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.99:0/1915842629' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:35.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:34 smithi177 bash[68594]: audit 2024-09-13T14:59:33.958756+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.99:0/1915842629' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:35.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:35.125 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:34 smithi099 bash[84576]: audit 2024-09-13T14:59:33.958756+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.99:0/1915842629' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:35.125 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:34 smithi099 bash[87147]: audit 2024-09-13T14:59:33.958756+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.99:0/1915842629' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:35.139 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:35.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:35.242 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:35.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:35.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:35.389 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:35.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:35.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 31 16 55030 55014 110.905 67.9375 0.00500881 0.00901046 2024-09-13T14:59:35.492 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:35.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:35.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:35.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:35.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:35.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:35.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:35.780 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:35.883 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:35.902 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:35.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:36.009 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:36.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:35 smithi177 bash[68594]: cluster 2024-09-13T14:59:34.757846+0000 mgr.y (mgr.26252) 1002 : cluster [DBG] pgmap v1535: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:36.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:35 smithi177 bash[68594]: cluster 2024-09-13T14:59:34.757846+0000 mgr.y (mgr.26252) 1002 : cluster [DBG] pgmap v1535: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:36.043 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:36.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:36.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:36.172 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:36.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:36.275 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:36.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:36.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:36.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:35 smithi099 bash[84576]: cluster 2024-09-13T14:59:34.757846+0000 mgr.y (mgr.26252) 1002 : cluster [DBG] pgmap v1535: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:36.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:35 smithi099 bash[84576]: cluster 2024-09-13T14:59:34.757846+0000 mgr.y (mgr.26252) 1002 : cluster [DBG] pgmap v1535: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:36.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:35 smithi099 bash[87147]: cluster 2024-09-13T14:59:34.757846+0000 mgr.y (mgr.26252) 1002 : cluster [DBG] pgmap v1535: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:36.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:35 smithi099 bash[87147]: cluster 2024-09-13T14:59:34.757846+0000 mgr.y (mgr.26252) 1002 : cluster [DBG] pgmap v1535: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:36.402 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:36.429 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:36.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:36.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 32 16 55986 55970 109.306 59.75 0.00999653 0.00914412 2024-09-13T14:59:36.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:36.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:36.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:36.661 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:36.687 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:36.791 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:36.823 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:36.927 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:36.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:37.062 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:37.086 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:37.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:37.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:36 smithi177 bash[68594]: cluster 2024-09-13T14:59:36.450642+0000 mon.a (mon.0) 2059 : cluster [WRN] Health check update: Degraded data redundancy: 16737/160245 objects degraded (10.445%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:37.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:36 smithi177 bash[68594]: cluster 2024-09-13T14:59:36.450642+0000 mon.a (mon.0) 2059 : cluster [WRN] Health check update: Degraded data redundancy: 16737/160245 objects degraded (10.445%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:37.195 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:37.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:37.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:37.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:59:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:59:37] "GET /metrics HTTP/1.1" 200 36766 "" "Prometheus/2.51.0" 2024-09-13T14:59:37.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:36 smithi099 bash[84576]: cluster 2024-09-13T14:59:36.450642+0000 mon.a (mon.0) 2059 : cluster [WRN] Health check update: Degraded data redundancy: 16737/160245 objects degraded (10.445%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:37.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:36 smithi099 bash[84576]: cluster 2024-09-13T14:59:36.450642+0000 mon.a (mon.0) 2059 : cluster [WRN] Health check update: Degraded data redundancy: 16737/160245 objects degraded (10.445%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:37.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:36 smithi099 bash[87147]: cluster 2024-09-13T14:59:36.450642+0000 mon.a (mon.0) 2059 : cluster [WRN] Health check update: Degraded data redundancy: 16737/160245 objects degraded (10.445%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:37.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:36 smithi099 bash[87147]: cluster 2024-09-13T14:59:36.450642+0000 mon.a (mon.0) 2059 : cluster [WRN] Health check update: Degraded data redundancy: 16737/160245 objects degraded (10.445%), 8 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:37.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:37.360 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:37.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:37.463 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:37.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 33 16 57460 57444 108.785 92.125 0.00682249 0.00918687 2024-09-13T14:59:37.484 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T14:59:37.488 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T14:59:37.493 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:37.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:37.601 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:37.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:37.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:37.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:37.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:37.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:37.864 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:37.902 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:37.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:38.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:38.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:37 smithi099 bash[84576]: cluster 2024-09-13T14:59:36.758465+0000 mgr.y (mgr.26252) 1003 : cluster [DBG] pgmap v1536: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:38.016 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:37 smithi099 bash[84576]: cluster 2024-09-13T14:59:36.758465+0000 mgr.y (mgr.26252) 1003 : cluster [DBG] pgmap v1536: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:38.016 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:37 smithi099 bash[87147]: cluster 2024-09-13T14:59:36.758465+0000 mgr.y (mgr.26252) 1003 : cluster [DBG] pgmap v1536: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:38.017 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:37 smithi099 bash[87147]: cluster 2024-09-13T14:59:36.758465+0000 mgr.y (mgr.26252) 1003 : cluster [DBG] pgmap v1536: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:38.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:38.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:37 smithi177 bash[68594]: cluster 2024-09-13T14:59:36.758465+0000 mgr.y (mgr.26252) 1003 : cluster [DBG] pgmap v1536: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:38.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:37 smithi177 bash[68594]: cluster 2024-09-13T14:59:36.758465+0000 mgr.y (mgr.26252) 1003 : cluster [DBG] pgmap v1536: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 16737/160245 objects degraded (10.445%) 2024-09-13T14:59:38.134 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:38.158 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:38.262 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:38.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:38.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:38.365 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:59:38.365 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:59:38.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:38.424 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:38.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:38.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 34 16 58976 58960 108.372 94.75 0.0055727 0.00922268 2024-09-13T14:59:38.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:38.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:38.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:38.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:38.688 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:38.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:38.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:38.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:38.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:38.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:38.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:39.037 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:39.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:38 smithi099 bash[84576]: audit 2024-09-13T14:59:37.477446+0000 mgr.y (mgr.26252) 1004 : audit [DBG] from='client.35694 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:39.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:38 smithi099 bash[84576]: audit 2024-09-13T14:59:37.477446+0000 mgr.y (mgr.26252) 1004 : audit [DBG] from='client.35694 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:39.048 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:38 smithi099 bash[87147]: audit 2024-09-13T14:59:37.477446+0000 mgr.y (mgr.26252) 1004 : audit [DBG] from='client.35694 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:39.048 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:38 smithi099 bash[87147]: audit 2024-09-13T14:59:37.477446+0000 mgr.y (mgr.26252) 1004 : audit [DBG] from='client.35694 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:39.063 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:39.145 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T14:59:39.145 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T14:59:39.145 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137611 41.f 41:f3e29f70:::benchmark_data_smithi099_126887_object39823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.521997+0000", 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 566.62195561399994, 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.46795233000000003, 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:59:39.146 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137611 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.521997+0000", 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.521997+0000", 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.147 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.521999+0000", 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.609e-06 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522002+0000", 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3380000000000001e-06 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522011+0000", 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3259999999999996e-06 2024-09-13T14:59:39.148 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522016+0000", 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2970000000000003e-06 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.526224+0000", 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0042076049999999997 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.526371+0000", 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014755700000000001 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.526519+0000", 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014766599999999999 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.531915+0000", 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0053958069999999999 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.150 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.549794+0000", 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017879692999999999 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.985725+0000", 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.43593075399999998 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.986090+0000", 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00036506800000000002 2024-09-13T14:59:39.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.989950+0000", 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0038596099999999999 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137613 41.f 41:f4b91036:::benchmark_data_smithi099_126887_object39825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.522025+0000", 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 566.62192821400004, 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.49653631599999998, 2024-09-13T14:59:39.152 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137613 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522025+0000", 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.153 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522025+0000", 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522025+0000", 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.27e-07 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522027+0000", 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.717e-06 2024-09-13T14:59:39.154 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522032+0000", 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.916e-06 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.522036+0000", 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9249999999999997e-06 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.531941+0000", 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0099056219999999993 2024-09-13T14:59:39.155 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.532041+0000", 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9648000000000005e-05 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.532127+0000", 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5906e-05 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.156 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.536271+0000", 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004143847 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.549818+0000", 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013547201 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.017054+0000", 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.46723604499999999 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.157 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.017107+0000", 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3396999999999997e-05 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.018561+0000", 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014537650000000001 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.158 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.35268.0:137628 41.f 41:f2dde755:::benchmark_data_smithi099_126887_object39840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e981)", 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:50:12.543546+0000", 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 566.60040683700004, 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.475842716, 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.35268", 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3011686466", 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137628 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.159 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543546+0000", 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543546+0000", 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.160 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543547+0000", 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.237e-06 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543553+0000", 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0250000000000002e-06 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543564+0000", 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1265e-05 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.543570+0000", 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1750000000000002e-06 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.546441+0000", 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028709289999999999 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.546593+0000", 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000151669 2024-09-13T14:59:39.162 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.546748+0000", 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015586800000000001 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.550002+0000", 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032534450000000002 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:12.561684+0000", 2024-09-13T14:59:39.163 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011681881999999999 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.018608+0000", 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.45692447000000003 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.018659+0000", 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0742000000000003e-05 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.164 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:50:13.019389+0000", 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000730009 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45350.0:8373 42.2 42:4985d3e5:::benchmark_data_smithi099_148651_object8372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1014)", 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:53:29.390976+0000", 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 369.752977337, 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.80169195699999996, 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.165 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45350", 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 8373 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.390976+0000", 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.166 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.390976+0000", 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.390978+0000", 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8899999999999999e-06 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.391001+0000", 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3794e-05 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.167 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.391016+0000", 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4243000000000001e-05 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.391024+0000", 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1119999999999996e-06 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.391088+0000", 2024-09-13T14:59:39.168 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4314999999999997e-05 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.391223+0000", 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000135158 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.391366+0000", 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143084 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.169 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.392154+0000", 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00078806500000000001 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:29.392333+0000", 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000178927 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:30.192211+0000", 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.79987746400000004 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.170 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:30.192582+0000", 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00037102599999999999 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:53:30.192668+0000", 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5878999999999995e-05 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45350.0:189121 42.2 42:40df2aca:::benchmark_data_smithi099_148651_object81281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1030)", 2024-09-13T14:59:39.171 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:56:50.319515+0000", 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 168.82443816599999, 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59625237399999997, 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45350", 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 189121 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.172 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.319515+0000", 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.319515+0000", 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.319516+0000", 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3489999999999999e-06 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.173 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.319521+0000", 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8189999999999999e-06 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.319532+0000", 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1248e-05 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.174 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.319538+0000", 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6980000000000003e-06 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.319584+0000", 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6051999999999999e-05 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.319655+0000", 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1058999999999998e-05 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:39.175 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.319726+0000", 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1322999999999995e-05 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.324381+0000", 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046548880000000003 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.324454+0000", 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3022999999999995e-05 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.176 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.915630+0000", 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59117546200000004 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.915706+0000", 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6242999999999995e-05 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.915767+0000", 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1210000000000005e-05 2024-09-13T14:59:39.177 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45350.0:189125 42.2 42:4c89d1b8:::benchmark_data_smithi099_148651_object81285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1030)", 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:56:50.328030+0000", 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 168.81592329399999, 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58788266, 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45350", 2024-09-13T14:59:39.178 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 189125 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.328030+0000", 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.328030+0000", 2024-09-13T14:59:39.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.328031+0000", 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.438e-06 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.328036+0000", 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7859999999999996e-06 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.328045+0000", 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8060000000000001e-06 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.328050+0000", 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8289999999999998e-06 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.472011+0000", 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.14396167600000001 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.472331+0000", 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00031930700000000001 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.472958+0000", 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00062777500000000001 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.491982+0000", 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019023571 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.492007+0000", 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4613e-05 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.915814+0000", 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.42380715499999999 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.915868+0000", 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4657999999999997e-05 2024-09-13T14:59:39.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.915912+0000", 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4045999999999998e-05 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45350.0:189137 42.2 42:45e3dd17:::benchmark_data_smithi099_148651_object81297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1030)", 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:56:50.348079+0000", 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 168.79587419200001, 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.56799033799999998, 2024-09-13T14:59:39.184 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45350", 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 189137 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.348079+0000", 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.348079+0000", 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.348080+0000", 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3379999999999999e-06 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.348086+0000", 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2689999999999999e-06 2024-09-13T14:59:39.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.472095+0000", 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.124009464 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.472126+0000", 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1312000000000001e-05 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.483587+0000", 2024-09-13T14:59:39.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011460724 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.483817+0000", 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000230144 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.484044+0000", 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000226868 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.188 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.492107+0000", 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0080631030000000003 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.493662+0000", 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001554494 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.915959+0000", 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.42229773100000001 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.916023+0000", 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3291000000000001e-05 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:56:50.916069+0000", 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6600000000000001e-05 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45572.0:54978 43.2 43:4f5057f2:::benchmark_data_smithi099_171278_object54977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1043)", 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:35.160264+0000", 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 3.9836890249999999, 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28037303000000002, 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45572", 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 54978 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.192 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.192 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.160264+0000", 2024-09-13T14:59:39.192 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.192 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.192 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.192 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.192 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.160264+0000", 2024-09-13T14:59:39.192 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.160265+0000", 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0529999999999999e-06 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.160315+0000", 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9679000000000003e-05 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.193 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.160324+0000", 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9260000000000006e-06 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.160329+0000", 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8210000000000001e-06 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.160389+0000", 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0452000000000003e-05 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.160492+0000", 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102718 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.160600+0000", 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010851199999999999 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.162463+0000", 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001862442 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.164985+0000", 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025220780000000001 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440444+0000", 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27545926500000001 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440555+0000", 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011099000000000001 2024-09-13T14:59:39.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440637+0000", 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2094000000000004e-05 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45572.0:54980 43.7 43:e79b1488:::benchmark_data_smithi099_171278_object54979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1043)", 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:35.162440+0000", 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 3.9815132129999999, 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28752585200000003, 2024-09-13T14:59:39.197 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45572", 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 54980 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.162440+0000", 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.198 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.162440+0000", 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.162441+0000", 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2559999999999999e-06 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.162489+0000", 2024-09-13T14:59:39.199 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7941000000000001e-05 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.162498+0000", 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1710000000000008e-06 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.162503+0000", 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8280000000000002e-06 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.162555+0000", 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1736000000000001e-05 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.162690+0000", 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013483700000000001 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.162798+0000", 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010801 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.164380+0000", 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001582373 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.165198+0000", 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00081844199999999995 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.449874+0000", 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.284676023 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.449926+0000", 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1836000000000003e-05 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.449966+0000", 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9399e-05 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:54985 43.b e1043/1038 43:da309605:::benchmark_data_smithi099_171278_object54984:head v 1043'3480, mlcod=1043'3477)", 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:35.167503+0000", 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 3.976449761, 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27552562400000002, 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.167503+0000", 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.167503+0000", 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.167504+0000", 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7400000000000003e-07 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.167560+0000", 2024-09-13T14:59:39.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6474000000000002e-05 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.167568+0000", 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7330000000000003e-06 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440365+0000", 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27279721400000001 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.206 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440454+0000", 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8886000000000002e-05 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440480+0000", 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.601e-05 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.442990+0000", 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025100560000000001 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.443029+0000", 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8577e-05 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.45572.0:54978 43.2 e1043/1038 ondisk, result = 0)", 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:35.168152+0000", 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 3.9758013239999999, 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27249615999999999, 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.208 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.168152+0000", 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.168152+0000", 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.168152+0000", 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2399999999999998e-07 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.168156+0000", 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.019e-06 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440022+0000", 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.271865578 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440047+0000", 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5454000000000001e-05 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440395+0000", 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00034798899999999999 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440432+0000", 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6452000000000003e-05 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:35.440648+0000", 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021604400000000001 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:55737 43.9 e1043/1038 43:9e0568c5:::benchmark_data_smithi099_171278_object55736:head v 1043'3446, mlcod=1043'3443)", 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:36.046397+0000", 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 3.0975561410000001, 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28141959900000002, 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.212 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.046397+0000", 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.046397+0000", 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.213 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.046398+0000", 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1999999999999998e-07 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.046461+0000", 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2750999999999996e-05 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.046469+0000", 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4910000000000005e-06 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.046475+0000", 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6890000000000001e-06 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.046519+0000", 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3988e-05 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.046537+0000", 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8355999999999998e-05 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.048177+0000", 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001640131 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.327817+0000", 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27963937300000002 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45572.0:55740 43.2 43:4957dd07:::benchmark_data_smithi099_171278_object55739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1043)", 2024-09-13T14:59:39.216 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:36.048090+0000", 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 3.0958627500000002, 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.298384767, 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45572", 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 55740 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.217 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.048090+0000", 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.048090+0000", 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.048091+0000", 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.111e-06 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.048120+0000", 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8136999999999998e-05 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.328528+0000", 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28040880299999998 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.328553+0000", 2024-09-13T14:59:39.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4368000000000001e-05 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.328736+0000", 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018355499999999999 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.329055+0000", 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00031818999999999999 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.220 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.329852+0000", 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00079772299999999997 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.337573+0000", 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0077209210000000004 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.339573+0000", 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019998239999999999 2024-09-13T14:59:39.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.346054+0000", 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006480937 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.346419+0000", 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000365015 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.346475+0000", 2024-09-13T14:59:39.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6183000000000001e-05 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:55751 43.9 e1043/1038 43:929c2eb1:::benchmark_data_smithi099_171278_object55750:head v 1043'3447, mlcod=1043'3444)", 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:36.058526+0000", 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 3.0854268060000001, 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.275692139, 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.223 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.058526+0000", 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.058526+0000", 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.224 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.058527+0000", 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8000000000000004e-07 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.058579+0000", 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1941000000000003e-05 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.327179+0000", 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.26860014399999999 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.327213+0000", 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4063000000000002e-05 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.329307+0000", 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020934339999999999 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.329358+0000", 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1635999999999998e-05 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.334133+0000", 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0047745790000000001 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:36.334218+0000", 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5462000000000003e-05 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.227 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45572.0:59400 43.7 43:ed55388f:::benchmark_data_smithi099_171278_object59399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1043)", 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:38.774245+0000", 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.36970776599999999, 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.19013492200000001, 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45572", 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 59400 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.774245+0000", 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.774245+0000", 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.774247+0000", 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4559999999999999e-06 2024-09-13T14:59:39.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.774284+0000", 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7051999999999997e-05 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.774294+0000", 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0485999999999999e-05 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.774302+0000", 2024-09-13T14:59:39.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2840000000000001e-06 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.774358+0000", 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6802999999999998e-05 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.774507+0000", 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014843899999999999 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:59:39.231 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.774641+0000", 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013438100000000001 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.776380+0000", 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017389630000000001 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.779711+0000", 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033310610000000002 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.232 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.964207+0000", 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.18449545000000001 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.964315+0000", 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108496 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.964380+0000", 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5050999999999998e-05 2024-09-13T14:59:39.233 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45572.0:59408 43.7 43:e41fa42c:::benchmark_data_smithi099_171278_object59407:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1043)", 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:38.777929+0000", 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.36602436100000002, 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.194997794, 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45572", 2024-09-13T14:59:39.234 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 59408 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.777929+0000", 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.777929+0000", 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.777930+0000", 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0419999999999999e-06 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.777959+0000", 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9614000000000001e-05 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.777969+0000", 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6840000000000008e-06 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.777975+0000", 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8919999999999997e-06 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.778021+0000", 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6535000000000001e-05 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.778109+0000", 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7591999999999998e-05 2024-09-13T14:59:39.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.778196+0000", 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6620999999999996e-05 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.782220+0000", 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040246780000000003 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.786372+0000", 2024-09-13T14:59:39.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041511630000000002 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.972804+0000", 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.186432771 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.972869+0000", 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4296999999999998e-05 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.972927+0000", 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7905000000000002e-05 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:59414 43.b e1043/1038 43:dc99eb86:::benchmark_data_smithi099_171278_object59413:head v 1043'3745, mlcod=1043'3740)", 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:38.788091+0000", 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.35586254, 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.17126708800000001, 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.240 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.788091+0000", 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.788091+0000", 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.788091+0000", 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.37e-07 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.788152+0000", 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0748000000000002e-05 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.788160+0000", 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6529999999999994e-06 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.788164+0000", 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6040000000000003e-06 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.788194+0000", 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0079000000000001e-05 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.788206+0000", 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1481000000000001e-05 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.790286+0000", 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020804019999999999 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.959358+0000", 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.16907148399999999 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:59416 43.b e1043/1038 43:d5f15a83:::benchmark_data_smithi099_171278_object59415:head v 1043'3746, mlcod=1043'3743)", 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:38.789006+0000", 2024-09-13T14:59:39.244 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.35494662799999999, 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.17608757899999999, 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.789006+0000", 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.789006+0000", 2024-09-13T14:59:39.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.789007+0000", 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7999999999999995e-07 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.789066+0000", 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8770000000000001e-05 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.789075+0000", 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4120000000000007e-06 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.789079+0000", 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0779999999999997e-06 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.789383+0000", 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00030419000000000003 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.789398+0000", 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4876e-05 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.964903+0000", 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.175504717 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.965094+0000", 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019095599999999999 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.248 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:59417 43.b e1043/1038 43:da891e12:::benchmark_data_smithi099_171278_object59416:head v 1043'3747, mlcod=1043'3743)", 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:38.790782+0000", 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.35317126900000001, 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.176682016, 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.790782+0000", 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.790782+0000", 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.790782+0000", 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4099999999999998e-07 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.790847+0000", 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4648999999999997e-05 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.958874+0000", 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.168026649 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.958896+0000", 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.226e-05 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.959430+0000", 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00053421499999999999 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.962348+0000", 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029174069999999999 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.965101+0000", 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027532659999999999 2024-09-13T14:59:39.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.967464+0000", 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023629290000000002 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:59423 43.9 e1043/1038 43:97f3fb0f:::benchmark_data_smithi099_171278_object59422:head v 1043'3695, mlcod=1043'3693)", 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:38.793673+0000", 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.350279695, 2024-09-13T14:59:39.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.17034523500000001, 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.793673+0000", 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.793673+0000", 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T14:59:39.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.793674+0000", 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5499999999999997e-07 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.793780+0000", 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106261 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.954705+0000", 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.160924866 2024-09-13T14:59:39.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.954735+0000", 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9955999999999999e-05 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.959953+0000", 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005218182 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.959998+0000", 2024-09-13T14:59:39.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4926000000000003e-05 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.963959+0000", 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039601690000000004 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:38.964019+0000", 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.012e-05 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.257 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.258 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T14:59:39.258 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T14:59:39.258 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T14:59:39.258 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:39.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:38 smithi177 bash[68594]: audit 2024-09-13T14:59:37.477446+0000 mgr.y (mgr.26252) 1004 : audit [DBG] from='client.35694 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:39.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:38 smithi177 bash[68594]: audit 2024-09-13T14:59:37.477446+0000 mgr.y (mgr.26252) 1004 : audit [DBG] from='client.35694 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:39.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:39.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:39.401 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:39.431 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:39.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:39.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 35 16 60102 60086 107.287 70.375 0.00786178 0.00931532 2024-09-13T14:59:39.538 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:39.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.581 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:39.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:39.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:39.714 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:39.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:39.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:39.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:39.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:39.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:39.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:39 smithi099 bash[84576]: cluster 2024-09-13T14:59:38.759141+0000 mgr.y (mgr.26252) 1005 : cluster [DBG] pgmap v1537: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 17137/163734 objects degraded (10.466%) 2024-09-13T14:59:39.961 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:39 smithi099 bash[84576]: cluster 2024-09-13T14:59:38.759141+0000 mgr.y (mgr.26252) 1005 : cluster [DBG] pgmap v1537: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 17137/163734 objects degraded (10.466%) 2024-09-13T14:59:39.961 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:39 smithi099 bash[87147]: cluster 2024-09-13T14:59:38.759141+0000 mgr.y (mgr.26252) 1005 : cluster [DBG] pgmap v1537: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 17137/163734 objects degraded (10.466%) 2024-09-13T14:59:39.961 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:39 smithi099 bash[87147]: cluster 2024-09-13T14:59:38.759141+0000 mgr.y (mgr.26252) 1005 : cluster [DBG] pgmap v1537: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 17137/163734 objects degraded (10.466%) 2024-09-13T14:59:39.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:39.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:40.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:40.108 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:40.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:39 smithi177 bash[68594]: cluster 2024-09-13T14:59:38.759141+0000 mgr.y (mgr.26252) 1005 : cluster [DBG] pgmap v1537: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 17137/163734 objects degraded (10.466%) 2024-09-13T14:59:40.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:39 smithi177 bash[68594]: cluster 2024-09-13T14:59:38.759141+0000 mgr.y (mgr.26252) 1005 : cluster [DBG] pgmap v1537: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 17137/163734 objects degraded (10.466%) 2024-09-13T14:59:40.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:40.242 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:40.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:40.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:59:40.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:40.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 36 16 61290 61274 106.369 74.25 0.0115669 0.00939697 2024-09-13T14:59:40.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:40.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:40.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:40.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:59:40 smithi177 bash[61439]: ts=2024-09-13T14:59:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:40.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:40.652 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:40.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:40.794 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:40.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:40.927 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:40.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:41.033 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:41.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:41.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:41.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:41.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:41.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:41.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:41.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:41.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:41.421 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:41.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:41.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:41.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 37 16 62466 62450 105.48 73.5 0.0141835 0.0094756 2024-09-13T14:59:41.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:41.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:41.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:41.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:41.709 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:41.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:41.735 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1536,"stamp":"2024-09-13T14:59:36.758025+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3499497421,"num_objects":53415,"num_object_clones":0,"num_object_copies":160245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16737,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53415,"num_whiteouts":0,"num_read":344,"num_read_kb":799,"num_write":53601,"num_write_kb":3418555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":595442,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":53553,"ondisk_log_size":53553,"up":233,"acting":233,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11699380,"kb_used_data":9078192,"kb_used_omap":353,"kb_used_meta":2620766,"kb_avail":738261836,"statfs":{"total":767960285184,"available":755980120064,"internally_reserved":0,"allocated":9296068608,"data_stored":9263808456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":361898,"internal_metadata":2683664982},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":872677376,"num_objects":13316,"num_object_clones":0,"num_object_copies":39948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4073,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13316,"num_write_kb":852224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004648"},"pg_stats":[{"pgid":"43.f","version":"1043'3347","reported_seq":3368,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:34.280551+0000","last_change":"2024-09-13T14:58:36.839129+0000","last_active":"2024-09-13T14:59:34.280551+0000","last_peered":"2024-09-13T14:59:34.280551+0000","last_clean":"2024-09-13T14:59:34.280551+0000","last_became_active":"2024-09-13T14:58:36.838830+0000","last_became_peered":"2024-09-13T14:58:36.838830+0000","last_unstale":"2024-09-13T14:59:34.280551+0000","last_undegraded":"2024-09-13T14:59:34.280551+0000","last_fullsized":"2024-09-13T14:59:34.280551+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3347,"log_dups_size":0,"ondisk_log_size":3347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:32:54.324207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219348992,"num_objects":3347,"num_object_clones":0,"num_object_copies":10041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3347,"num_write_kb":214208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"43.e","version":"1043'3363","reported_seq":3373,"reported_epoch":1043,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:34.244815+0000","last_change":"2024-09-13T14:59:04.484378+0000","last_active":"2024-09-13T14:59:34.244815+0000","last_peered":"2024-09-13T14:59:34.244815+0000","last_clean":"2024-09-13T14:58:35.801479+0000","last_became_active":"2024-09-13T14:58:37.301143+0000","last_became_peered":"2024-09-13T14:58:37.301143+0000","last_unstale":"2024-09-13T14:59:34.244815+0000","last_undegraded":"2024-09-13T14:59:04.478457+0000","last_fullsized":"2024-09-13T14:58:36.828536+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3363,"log_dups_size":0,"ondisk_log_size":3363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:29:00.244187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220397568,"num_objects":3363,"num_object_clones":0,"num_object_copies":10089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3363,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3363,"num_write_kb":215232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":3363}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"43.d","version":"1043'3267","reported_seq":3280,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:34.433260+0000","last_change":"2024-09-13T14:58:36.840334+0000","last_active":"2024-09-13T14:59:34.433260+0000","last_peered":"2024-09-13T14:59:34.433260+0000","last_clean":"2024-09-13T14:59:34.433260+0000","last_became_active":"2024-09-13T14:58:36.839804+0000","last_became_peered":"2024-09-13T14:58:36.839804+0000","last_unstale":"2024-09-13T14:59:34.433260+0000","last_undegraded":"2024-09-13T14:59:34.433260+0000","last_fullsized":"2024-09-13T14:59:34.433260+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3267,"log_dups_size":0,"ondisk_log_size":3267,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:01:13.585532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214106112,"num_objects":3267,"num_object_clones":0,"num_object_copies":9801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3267,"num_write_kb":209088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"43.c","version":"1043'3365","reported_seq":3378,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:34.440191+0000","last_change":"2024-09-13T14:58:37.298659+0000","last_active":"2024-09-13T14:59:34.440191+0000","last_peered":"2024-09-13T14:59:34.440191+0000","last_clean":"2024-09-13T14:59:34.440191+0000","last_became_active":"2024-09-13T14:58:37.297884+0000","last_became_peered":"2024-09-13T14:58:37.297884+0000","last_unstale":"2024-09-13T14:59:34.440191+0000","last_undegraded":"2024-09-13T14:59:34.440191+0000","last_fullsized":"2024-09-13T14:59:34.440191+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3365,"log_dups_size":0,"ondisk_log_size":3365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:44.700798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220528640,"num_objects":3365,"num_object_clones":0,"num_object_copies":10095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3365,"num_write_kb":215360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"43.b","version":"1043'3388","reported_seq":3398,"reported_epoch":1043,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:34.262062+0000","last_change":"2024-09-13T14:59:04.479624+0000","last_active":"2024-09-13T14:59:34.262062+0000","last_peered":"2024-09-13T14:59:34.262062+0000","last_clean":"2024-09-13T14:58:35.801479+0000","last_became_active":"2024-09-13T14:58:36.835534+0000","last_became_peered":"2024-09-13T14:58:36.835534+0000","last_unstale":"2024-09-13T14:59:34.262062+0000","last_undegraded":"2024-09-13T14:59:04.479310+0000","last_fullsized":"2024-09-13T14:58:36.828102+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3388,"log_dups_size":0,"ondisk_log_size":3388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:55.719487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222035968,"num_objects":3388,"num_object_clones":0,"num_object_copies":10164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3388,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3388,"num_write_kb":216832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":3388}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"43.a","version":"1043'3350","reported_seq":3363,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:34.419115+0000","last_change":"2024-09-13T14:58:37.298748+0000","last_active":"2024-09-13T14:59:34.419115+0000","last_peered":"2024-09-13T14:59:34.419115+0000","last_clean":"2024-09-13T14:59:34.419115+0000","last_became_active":"2024-09-13T14:58:37.297401+0000","last_became_peered":"2024-09-13T14:58:37.297401+0000","last_unstale":"2024-09-13T14:59:34.419115+0000","last_undegraded":"2024-09-13T14:59:34.419115+0000","last_fullsized":"2024-09-13T14:59:34.419115+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3350,"log_dups_size":0,"ondisk_log_size":3350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:15:53.728195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219545600,"num_objects":3350,"num_object_clones":0,"num_object_copies":10050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3350,"num_write_kb":214400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"43.9","version":"1043'3366","reported_seq":3376,"reported_epoch":1043,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:34.690970+0000","last_change":"2024-09-13T14:59:04.479971+0000","last_active":"2024-09-13T14:59:34.690970+0000","last_peered":"2024-09-13T14:59:34.690970+0000","last_clean":"2024-09-13T14:58:35.801479+0000","last_became_active":"2024-09-13T14:58:36.837580+0000","last_became_peered":"2024-09-13T14:58:36.837580+0000","last_unstale":"2024-09-13T14:59:34.690970+0000","last_undegraded":"2024-09-13T14:59:04.479794+0000","last_fullsized":"2024-09-13T14:58:36.834594+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3366,"log_dups_size":0,"ondisk_log_size":3366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:31.608841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220594176,"num_objects":3366,"num_object_clones":0,"num_object_copies":10098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3366,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3366,"num_write_kb":215424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":3366}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"43.8","version":"1043'3385","reported_seq":3398,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:34.429624+0000","last_change":"2024-09-13T14:58:37.299352+0000","last_active":"2024-09-13T14:59:34.429624+0000","last_peered":"2024-09-13T14:59:34.429624+0000","last_clean":"2024-09-13T14:59:34.429624+0000","last_became_active":"2024-09-13T14:58:37.298528+0000","last_became_peered":"2024-09-13T14:58:37.298528+0000","last_unstale":"2024-09-13T14:59:34.429624+0000","last_undegraded":"2024-09-13T14:59:34.429624+0000","last_fullsized":"2024-09-13T14:59:34.429624+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3385,"log_dups_size":0,"ondisk_log_size":3385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:46:32.493987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221839360,"num_objects":3385,"num_object_clones":0,"num_object_copies":10155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3385,"num_write_kb":216640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"43.7","version":"1043'3197","reported_seq":3218,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:32.565532+0000","last_change":"2024-09-13T14:58:36.837280+0000","last_active":"2024-09-13T14:59:32.565532+0000","last_peered":"2024-09-13T14:59:32.565532+0000","last_clean":"2024-09-13T14:59:32.565532+0000","last_became_active":"2024-09-13T14:58:36.836990+0000","last_became_peered":"2024-09-13T14:58:36.836990+0000","last_unstale":"2024-09-13T14:59:32.565532+0000","last_undegraded":"2024-09-13T14:59:32.565532+0000","last_fullsized":"2024-09-13T14:59:32.565532+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3197,"log_dups_size":0,"ondisk_log_size":3197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:00:51.019568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209518592,"num_objects":3197,"num_object_clones":0,"num_object_copies":9591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3197,"num_write_kb":204608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"43.6","version":"1043'3383","reported_seq":3396,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:34.705649+0000","last_change":"2024-09-13T14:58:37.302198+0000","last_active":"2024-09-13T14:59:34.705649+0000","last_peered":"2024-09-13T14:59:34.705649+0000","last_clean":"2024-09-13T14:59:34.705649+0000","last_became_active":"2024-09-13T14:58:37.301538+0000","last_became_peered":"2024-09-13T14:58:37.301538+0000","last_unstale":"2024-09-13T14:59:34.705649+0000","last_undegraded":"2024-09-13T14:59:34.705649+0000","last_fullsized":"2024-09-13T14:59:34.705649+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3383,"log_dups_size":0,"ondisk_log_size":3383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:17.144575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221708288,"num_objects":3383,"num_object_clones":0,"num_object_copies":10149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3383,"num_write_kb":216512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"43.5","version":"1043'3339","reported_seq":3360,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:34.287480+0000","last_change":"2024-09-13T14:58:36.841038+0000","last_active":"2024-09-13T14:59:34.287480+0000","last_peered":"2024-09-13T14:59:34.287480+0000","last_clean":"2024-09-13T14:59:34.287480+0000","last_became_active":"2024-09-13T14:58:36.840716+0000","last_became_peered":"2024-09-13T14:58:36.840716+0000","last_unstale":"2024-09-13T14:59:34.287480+0000","last_undegraded":"2024-09-13T14:59:34.287480+0000","last_fullsized":"2024-09-13T14:59:34.287480+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3339,"log_dups_size":0,"ondisk_log_size":3339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:41:22.890999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218824704,"num_objects":3339,"num_object_clones":0,"num_object_copies":10017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3339,"num_write_kb":213696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"43.4","version":"1043'3336","reported_seq":3354,"reported_epoch":1043,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:34.292997+0000","last_change":"2024-09-13T14:59:04.484804+0000","last_active":"2024-09-13T14:59:34.292997+0000","last_peered":"2024-09-13T14:59:34.292997+0000","last_clean":"2024-09-13T14:58:35.801479+0000","last_became_active":"2024-09-13T14:58:36.834656+0000","last_became_peered":"2024-09-13T14:58:36.834656+0000","last_unstale":"2024-09-13T14:59:34.292997+0000","last_undegraded":"2024-09-13T14:59:04.484557+0000","last_fullsized":"2024-09-13T14:58:36.828614+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3336,"log_dups_size":0,"ondisk_log_size":3336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:53:41.666021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218628096,"num_objects":3336,"num_object_clones":0,"num_object_copies":10008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3336,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3336,"num_write_kb":213504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":3336}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"43.3","version":"1043'3280","reported_seq":3298,"reported_epoch":1043,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:32.550398+0000","last_change":"2024-09-13T14:59:04.508558+0000","last_active":"2024-09-13T14:59:32.550398+0000","last_peered":"2024-09-13T14:59:32.550398+0000","last_clean":"2024-09-13T14:58:35.801479+0000","last_became_active":"2024-09-13T14:58:36.833207+0000","last_became_peered":"2024-09-13T14:58:36.833207+0000","last_unstale":"2024-09-13T14:59:32.550398+0000","last_undegraded":"2024-09-13T14:59:04.476309+0000","last_fullsized":"2024-09-13T14:58:36.824463+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3280,"log_dups_size":0,"ondisk_log_size":3280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:05:53.577004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214958080,"num_objects":3280,"num_object_clones":0,"num_object_copies":9840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3280,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3280,"num_write_kb":209920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":3280}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"43.2","version":"1043'3365","reported_seq":3384,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:32.583790+0000","last_change":"2024-09-13T14:58:36.839202+0000","last_active":"2024-09-13T14:59:32.583790+0000","last_peered":"2024-09-13T14:59:32.583790+0000","last_clean":"2024-09-13T14:59:32.583790+0000","last_became_active":"2024-09-13T14:58:36.838843+0000","last_became_peered":"2024-09-13T14:58:36.838843+0000","last_unstale":"2024-09-13T14:59:32.583790+0000","last_undegraded":"2024-09-13T14:59:32.583790+0000","last_fullsized":"2024-09-13T14:59:32.583790+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3365,"log_dups_size":0,"ondisk_log_size":3365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:59.568232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220528640,"num_objects":3365,"num_object_clones":0,"num_object_copies":10095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3365,"num_write_kb":215360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"43.1","version":"1043'3280","reported_seq":3293,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:34.705137+0000","last_change":"2024-09-13T14:58:37.299013+0000","last_active":"2024-09-13T14:59:34.705137+0000","last_peered":"2024-09-13T14:59:34.705137+0000","last_clean":"2024-09-13T14:59:34.705137+0000","last_became_active":"2024-09-13T14:58:37.298126+0000","last_became_peered":"2024-09-13T14:58:37.298126+0000","last_unstale":"2024-09-13T14:59:34.705137+0000","last_undegraded":"2024-09-13T14:59:34.705137+0000","last_fullsized":"2024-09-13T14:59:34.705137+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3280,"log_dups_size":0,"ondisk_log_size":3280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:50.443279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214958080,"num_objects":3280,"num_object_clones":0,"num_object_copies":9840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3280,"num_write_kb":209920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"43.0","version":"1043'3378","reported_seq":3391,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:34.249055+0000","last_change":"2024-09-13T14:58:37.299460+0000","last_active":"2024-09-13T14:59:34.249055+0000","last_peered":"2024-09-13T14:59:34.249055+0000","last_clean":"2024-09-13T14:59:34.249055+0000","last_became_active":"2024-09-13T14:58:37.298568+0000","last_became_peered":"2024-09-13T14:58:37.298568+0000","last_unstale":"2024-09-13T14:59:34.249055+0000","last_undegraded":"2024-09-13T14:59:34.249055+0000","last_fullsized":"2024-09-13T14:59:34.249055+0000","mapping_epoch":1038,"log_start":"0'0","ondisk_log_start":"0'0","created":1038,"last_epoch_clean":1039,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:58:35.801479+0000","last_clean_scrub_stamp":"2024-09-13T14:58:35.801479+0000","objects_scrubbed":0,"log_size":3378,"log_dups_size":0,"ondisk_log_size":3378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:11.498613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221380608,"num_objects":3378,"num_object_clones":0,"num_object_copies":10134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3378,"num_write_kb":216192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":1479,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.503278+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T14:59:04.503278+0000","last_peered":"2024-09-13T14:59:04.503278+0000","last_clean":"2024-09-13T14:59:04.503278+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T14:59:04.503278+0000","last_undegraded":"2024-09-13T14:59:04.503278+0000","last_fullsized":"2024-09-13T14:59:04.503278+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":198,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.504716+0000","last_change":"2024-09-13T14:52:43.599071+0000","last_active":"2024-09-13T14:59:04.504716+0000","last_peered":"2024-09-13T14:59:04.504716+0000","last_clean":"2024-09-13T14:59:04.504716+0000","last_became_active":"2024-09-13T14:52:43.594847+0000","last_became_peered":"2024-09-13T14:52:43.594847+0000","last_unstale":"2024-09-13T14:59:04.504716+0000","last_undegraded":"2024-09-13T14:59:04.504716+0000","last_fullsized":"2024-09-13T14:59:04.504716+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:02.736434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":41,"reported_epoch":1042,"state":"active+undersized","last_fresh":"2024-09-13T14:59:00.154520+0000","last_change":"2024-09-13T14:57:52.527498+0000","last_active":"2024-09-13T14:59:00.154520+0000","last_peered":"2024-09-13T14:59:00.154520+0000","last_clean":"2024-09-13T14:24:16.175691+0000","last_became_active":"2024-09-13T14:57:52.527498+0000","last_became_peered":"2024-09-13T14:57:52.527498+0000","last_unstale":"2024-09-13T14:59:00.154520+0000","last_undegraded":"2024-09-13T14:59:00.154520+0000","last_fullsized":"2024-09-13T14:57:52.516910+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:20.231670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":74,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.154072+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T14:59:00.154072+0000","last_peered":"2024-09-13T14:59:00.154072+0000","last_clean":"2024-09-13T14:59:00.154072+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T14:59:00.154072+0000","last_undegraded":"2024-09-13T14:59:00.154072+0000","last_fullsized":"2024-09-13T14:59:00.154072+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1028,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.154474+0000","last_change":"2024-09-13T14:47:46.337827+0000","last_active":"2024-09-13T14:59:00.154474+0000","last_peered":"2024-09-13T14:59:00.154474+0000","last_clean":"2024-09-13T14:59:00.154474+0000","last_became_active":"2024-09-13T14:47:46.321140+0000","last_became_peered":"2024-09-13T14:47:46.321140+0000","last_unstale":"2024-09-13T14:59:00.154474+0000","last_undegraded":"2024-09-13T14:59:00.154474+0000","last_fullsized":"2024-09-13T14:59:00.154474+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:15.997382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":188,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.505529+0000","last_change":"2024-09-13T14:57:52.524068+0000","last_active":"2024-09-13T14:59:04.505529+0000","last_peered":"2024-09-13T14:59:04.505529+0000","last_clean":"2024-09-13T14:57:46.744631+0000","last_became_active":"2024-09-13T14:57:52.524068+0000","last_became_peered":"2024-09-13T14:57:52.524068+0000","last_unstale":"2024-09-13T14:59:04.505529+0000","last_undegraded":"2024-09-13T14:59:04.505529+0000","last_fullsized":"2024-09-13T14:57:52.510427+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:10.935170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1073,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.156632+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T14:59:00.156632+0000","last_peered":"2024-09-13T14:59:00.156632+0000","last_clean":"2024-09-13T14:59:00.156632+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T14:59:00.156632+0000","last_undegraded":"2024-09-13T14:59:00.156632+0000","last_fullsized":"2024-09-13T14:59:00.156632+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":202,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.505986+0000","last_change":"2024-09-13T14:57:52.608274+0000","last_active":"2024-09-13T14:59:04.505986+0000","last_peered":"2024-09-13T14:59:04.505986+0000","last_clean":"2024-09-13T14:57:46.744660+0000","last_became_active":"2024-09-13T14:57:52.608274+0000","last_became_peered":"2024-09-13T14:57:52.608274+0000","last_unstale":"2024-09-13T14:59:04.505986+0000","last_undegraded":"2024-09-13T14:59:04.505986+0000","last_fullsized":"2024-09-13T14:57:52.510976+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:30.239753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1486,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.502523+0000","last_change":"2024-09-13T14:47:46.330059+0000","last_active":"2024-09-13T14:59:04.502523+0000","last_peered":"2024-09-13T14:59:04.502523+0000","last_clean":"2024-09-13T14:59:04.502523+0000","last_became_active":"2024-09-13T14:47:46.329344+0000","last_became_peered":"2024-09-13T14:47:46.329344+0000","last_unstale":"2024-09-13T14:59:04.502523+0000","last_undegraded":"2024-09-13T14:59:04.502523+0000","last_fullsized":"2024-09-13T14:59:04.502523+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:58:43.470176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1394,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.508196+0000","last_change":"2024-09-13T14:57:52.531642+0000","last_active":"2024-09-13T14:59:04.508196+0000","last_peered":"2024-09-13T14:59:04.508196+0000","last_clean":"2024-09-13T14:57:42.823387+0000","last_became_active":"2024-09-13T14:57:52.531642+0000","last_became_peered":"2024-09-13T14:57:52.531642+0000","last_unstale":"2024-09-13T14:59:04.508196+0000","last_undegraded":"2024-09-13T14:59:04.508196+0000","last_fullsized":"2024-09-13T14:57:52.523032+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:46.509860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":198,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.505473+0000","last_change":"2024-09-13T14:52:43.597400+0000","last_active":"2024-09-13T14:59:04.505473+0000","last_peered":"2024-09-13T14:59:04.505473+0000","last_clean":"2024-09-13T14:59:04.505473+0000","last_became_active":"2024-09-13T14:52:43.596294+0000","last_became_peered":"2024-09-13T14:52:43.596294+0000","last_unstale":"2024-09-13T14:59:04.505473+0000","last_undegraded":"2024-09-13T14:59:04.505473+0000","last_fullsized":"2024-09-13T14:59:04.505473+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:35.218171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1315,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.506054+0000","last_change":"2024-09-13T14:57:52.537575+0000","last_active":"2024-09-13T14:59:04.506054+0000","last_peered":"2024-09-13T14:59:04.506054+0000","last_clean":"2024-09-13T14:52:40.121284+0000","last_became_active":"2024-09-13T14:57:52.537575+0000","last_became_peered":"2024-09-13T14:57:52.537575+0000","last_unstale":"2024-09-13T14:59:04.506054+0000","last_undegraded":"2024-09-13T14:59:04.506054+0000","last_fullsized":"2024-09-13T14:57:52.526034+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.838090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":92,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.506405+0000","last_change":"2024-09-13T14:57:52.538971+0000","last_active":"2024-09-13T14:59:04.506405+0000","last_peered":"2024-09-13T14:59:04.506405+0000","last_clean":"2024-09-13T14:57:43.384926+0000","last_became_active":"2024-09-13T14:57:52.538971+0000","last_became_peered":"2024-09-13T14:57:52.538971+0000","last_unstale":"2024-09-13T14:59:04.506405+0000","last_undegraded":"2024-09-13T14:59:04.506405+0000","last_fullsized":"2024-09-13T14:57:52.518582+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:08:15.593435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1475,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.502167+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T14:59:04.502167+0000","last_peered":"2024-09-13T14:59:04.502167+0000","last_clean":"2024-09-13T14:59:04.502167+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T14:59:04.502167+0000","last_undegraded":"2024-09-13T14:59:04.502167+0000","last_fullsized":"2024-09-13T14:59:04.502167+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1308,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.501979+0000","last_change":"2024-09-13T14:57:52.532766+0000","last_active":"2024-09-13T14:59:04.501979+0000","last_peered":"2024-09-13T14:59:04.501979+0000","last_clean":"2024-09-13T14:52:40.120556+0000","last_became_active":"2024-09-13T14:57:52.532766+0000","last_became_peered":"2024-09-13T14:57:52.532766+0000","last_unstale":"2024-09-13T14:59:04.501979+0000","last_undegraded":"2024-09-13T14:59:04.501979+0000","last_fullsized":"2024-09-13T14:57:52.515253+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:16:40.443577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1030,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.154836+0000","last_change":"2024-09-13T14:52:54.183175+0000","last_active":"2024-09-13T14:59:00.154836+0000","last_peered":"2024-09-13T14:59:00.154836+0000","last_clean":"2024-09-13T14:59:00.154836+0000","last_became_active":"2024-09-13T14:52:54.182514+0000","last_became_peered":"2024-09-13T14:52:54.182514+0000","last_unstale":"2024-09-13T14:59:00.154836+0000","last_undegraded":"2024-09-13T14:59:00.154836+0000","last_fullsized":"2024-09-13T14:59:00.154836+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:55.900845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1288,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.506602+0000","last_change":"2024-09-13T14:57:52.529977+0000","last_active":"2024-09-13T14:59:04.506602+0000","last_peered":"2024-09-13T14:59:04.506602+0000","last_clean":"2024-09-13T14:52:40.125209+0000","last_became_active":"2024-09-13T14:57:52.529977+0000","last_became_peered":"2024-09-13T14:57:52.529977+0000","last_unstale":"2024-09-13T14:59:04.506602+0000","last_undegraded":"2024-09-13T14:59:04.506602+0000","last_fullsized":"2024-09-13T14:57:52.515419+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:11.447938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1470,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.503366+0000","last_change":"2024-09-13T14:57:52.524642+0000","last_active":"2024-09-13T14:59:04.503366+0000","last_peered":"2024-09-13T14:59:04.503366+0000","last_clean":"2024-09-13T14:57:42.887589+0000","last_became_active":"2024-09-13T14:57:52.524642+0000","last_became_peered":"2024-09-13T14:57:52.524642+0000","last_unstale":"2024-09-13T14:59:04.503366+0000","last_undegraded":"2024-09-13T14:59:04.503366+0000","last_fullsized":"2024-09-13T14:57:52.514403+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:37.383913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":645,"reported_epoch":1042,"state":"active+undersized","last_fresh":"2024-09-13T14:59:00.152268+0000","last_change":"2024-09-13T14:57:52.526180+0000","last_active":"2024-09-13T14:59:00.152268+0000","last_peered":"2024-09-13T14:59:00.152268+0000","last_clean":"2024-09-13T14:24:17.188743+0000","last_became_active":"2024-09-13T14:57:52.526180+0000","last_became_peered":"2024-09-13T14:57:52.526180+0000","last_unstale":"2024-09-13T14:59:00.152268+0000","last_undegraded":"2024-09-13T14:59:00.152268+0000","last_fullsized":"2024-09-13T14:57:52.516434+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:47:35.364999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1017,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.155038+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T14:59:00.155038+0000","last_peered":"2024-09-13T14:59:00.155038+0000","last_clean":"2024-09-13T14:59:00.155038+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T14:59:00.155038+0000","last_undegraded":"2024-09-13T14:59:00.155038+0000","last_fullsized":"2024-09-13T14:59:00.155038+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1061,"reported_epoch":1042,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:00.156673+0000","last_change":"2024-09-13T14:57:52.546472+0000","last_active":"2024-09-13T14:59:00.156673+0000","last_peered":"2024-09-13T14:59:00.156673+0000","last_clean":"2024-09-13T14:57:42.894880+0000","last_became_active":"2024-09-13T14:57:52.546472+0000","last_became_peered":"2024-09-13T14:57:52.546472+0000","last_unstale":"2024-09-13T14:59:00.156673+0000","last_undegraded":"2024-09-13T14:57:52.514987+0000","last_fullsized":"2024-09-13T14:57:52.514866+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:32.888371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":88,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.506069+0000","last_change":"2024-09-13T14:52:43.586530+0000","last_active":"2024-09-13T14:59:04.506069+0000","last_peered":"2024-09-13T14:59:04.506069+0000","last_clean":"2024-09-13T14:59:04.506069+0000","last_became_active":"2024-09-13T14:52:43.586107+0000","last_became_peered":"2024-09-13T14:52:43.586107+0000","last_unstale":"2024-09-13T14:59:04.506069+0000","last_undegraded":"2024-09-13T14:59:04.506069+0000","last_fullsized":"2024-09-13T14:59:04.506069+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:18.130768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":195,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.504901+0000","last_change":"2024-09-13T14:47:46.344536+0000","last_active":"2024-09-13T14:59:04.504901+0000","last_peered":"2024-09-13T14:59:04.504901+0000","last_clean":"2024-09-13T14:59:04.504901+0000","last_became_active":"2024-09-13T14:47:46.321131+0000","last_became_peered":"2024-09-13T14:47:46.321131+0000","last_unstale":"2024-09-13T14:59:04.504901+0000","last_undegraded":"2024-09-13T14:59:04.504901+0000","last_fullsized":"2024-09-13T14:59:04.504901+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:31.114472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":199,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.506227+0000","last_change":"2024-09-13T14:52:43.600569+0000","last_active":"2024-09-13T14:59:04.506227+0000","last_peered":"2024-09-13T14:59:04.506227+0000","last_clean":"2024-09-13T14:59:04.506227+0000","last_became_active":"2024-09-13T14:52:43.599764+0000","last_became_peered":"2024-09-13T14:52:43.599764+0000","last_unstale":"2024-09-13T14:59:04.506227+0000","last_undegraded":"2024-09-13T14:59:04.506227+0000","last_fullsized":"2024-09-13T14:59:04.506227+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:53.934456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":185,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.505177+0000","last_change":"2024-09-13T14:47:46.318785+0000","last_active":"2024-09-13T14:59:04.505177+0000","last_peered":"2024-09-13T14:59:04.505177+0000","last_clean":"2024-09-13T14:59:04.505177+0000","last_became_active":"2024-09-13T14:47:46.318438+0000","last_became_peered":"2024-09-13T14:47:46.318438+0000","last_unstale":"2024-09-13T14:59:04.505177+0000","last_undegraded":"2024-09-13T14:59:04.505177+0000","last_fullsized":"2024-09-13T14:59:04.505177+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:44.745378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1013,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.155018+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T14:59:00.155018+0000","last_peered":"2024-09-13T14:59:00.155018+0000","last_clean":"2024-09-13T14:59:00.155018+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T14:59:00.155018+0000","last_undegraded":"2024-09-13T14:59:00.155018+0000","last_fullsized":"2024-09-13T14:59:00.155018+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1527,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.502439+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T14:59:04.502439+0000","last_peered":"2024-09-13T14:59:04.502439+0000","last_clean":"2024-09-13T14:59:04.502439+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T14:59:04.502439+0000","last_undegraded":"2024-09-13T14:59:04.502439+0000","last_fullsized":"2024-09-13T14:59:04.502439+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":642,"reported_epoch":1042,"state":"active+undersized","last_fresh":"2024-09-13T14:59:00.152507+0000","last_change":"2024-09-13T14:57:52.538584+0000","last_active":"2024-09-13T14:59:00.152507+0000","last_peered":"2024-09-13T14:59:00.152507+0000","last_clean":"2024-09-13T14:35:50.051972+0000","last_became_active":"2024-09-13T14:57:52.538584+0000","last_became_peered":"2024-09-13T14:57:52.538584+0000","last_unstale":"2024-09-13T14:59:00.152507+0000","last_undegraded":"2024-09-13T14:59:00.152507+0000","last_fullsized":"2024-09-13T14:57:52.515637+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:52.637609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":89,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.506502+0000","last_change":"2024-09-13T14:52:43.592967+0000","last_active":"2024-09-13T14:59:04.506502+0000","last_peered":"2024-09-13T14:59:04.506502+0000","last_clean":"2024-09-13T14:59:04.506502+0000","last_became_active":"2024-09-13T14:52:43.592431+0000","last_became_peered":"2024-09-13T14:52:43.592431+0000","last_unstale":"2024-09-13T14:59:04.506502+0000","last_undegraded":"2024-09-13T14:59:04.506502+0000","last_fullsized":"2024-09-13T14:59:04.506502+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:56.187568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":52,"reported_epoch":1042,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:00.157519+0000","last_change":"2024-09-13T14:57:52.608972+0000","last_active":"2024-09-13T14:59:00.157519+0000","last_peered":"2024-09-13T14:59:00.157519+0000","last_clean":"2024-09-13T14:24:20.313173+0000","last_became_active":"2024-09-13T14:57:52.608972+0000","last_became_peered":"2024-09-13T14:57:52.608972+0000","last_unstale":"2024-09-13T14:59:00.157519+0000","last_undegraded":"2024-09-13T14:57:52.519236+0000","last_fullsized":"2024-09-13T14:57:52.518945+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:56.976455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1457,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.502397+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T14:59:04.502397+0000","last_peered":"2024-09-13T14:59:04.502397+0000","last_clean":"2024-09-13T14:59:04.502397+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T14:59:04.502397+0000","last_undegraded":"2024-09-13T14:59:04.502397+0000","last_fullsized":"2024-09-13T14:59:04.502397+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1049,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.157115+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T14:59:00.157115+0000","last_peered":"2024-09-13T14:59:00.157115+0000","last_clean":"2024-09-13T14:59:00.157115+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T14:59:00.157115+0000","last_undegraded":"2024-09-13T14:59:00.157115+0000","last_fullsized":"2024-09-13T14:59:00.157115+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":89,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.506290+0000","last_change":"2024-09-13T14:52:43.583860+0000","last_active":"2024-09-13T14:59:04.506290+0000","last_peered":"2024-09-13T14:59:04.506290+0000","last_clean":"2024-09-13T14:59:04.506290+0000","last_became_active":"2024-09-13T14:52:43.583055+0000","last_became_peered":"2024-09-13T14:52:43.583055+0000","last_unstale":"2024-09-13T14:59:04.506290+0000","last_undegraded":"2024-09-13T14:59:04.506290+0000","last_fullsized":"2024-09-13T14:59:04.506290+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:55.113648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":89,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.505955+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T14:59:04.505955+0000","last_peered":"2024-09-13T14:59:04.505955+0000","last_clean":"2024-09-13T14:59:04.505955+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T14:59:04.505955+0000","last_undegraded":"2024-09-13T14:59:04.505955+0000","last_fullsized":"2024-09-13T14:59:04.505955+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":89,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.506589+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T14:59:04.506589+0000","last_peered":"2024-09-13T14:59:04.506589+0000","last_clean":"2024-09-13T14:59:04.506589+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T14:59:04.506589+0000","last_undegraded":"2024-09-13T14:59:04.506589+0000","last_fullsized":"2024-09-13T14:59:04.506589+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1021,"reported_epoch":1042,"state":"active+undersized","last_fresh":"2024-09-13T14:59:00.155272+0000","last_change":"2024-09-13T14:57:52.528372+0000","last_active":"2024-09-13T14:59:00.155272+0000","last_peered":"2024-09-13T14:59:00.155272+0000","last_clean":"2024-09-13T14:57:42.822230+0000","last_became_active":"2024-09-13T14:57:52.528372+0000","last_became_peered":"2024-09-13T14:57:52.528372+0000","last_unstale":"2024-09-13T14:59:00.155272+0000","last_undegraded":"2024-09-13T14:59:00.155272+0000","last_fullsized":"2024-09-13T14:57:52.515808+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:43.143322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.506402+0000","last_change":"2024-09-13T14:57:52.535638+0000","last_active":"2024-09-13T14:59:04.506402+0000","last_peered":"2024-09-13T14:59:04.506402+0000","last_clean":"2024-09-13T14:57:43.385736+0000","last_became_active":"2024-09-13T14:57:52.535638+0000","last_became_peered":"2024-09-13T14:57:52.535638+0000","last_unstale":"2024-09-13T14:59:04.506402+0000","last_undegraded":"2024-09-13T14:59:04.506402+0000","last_fullsized":"2024-09-13T14:57:52.526096+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:43.910330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":185,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.506292+0000","last_change":"2024-09-13T14:47:46.324072+0000","last_active":"2024-09-13T14:59:04.506292+0000","last_peered":"2024-09-13T14:59:04.506292+0000","last_clean":"2024-09-13T14:59:04.506292+0000","last_became_active":"2024-09-13T14:47:46.322613+0000","last_became_peered":"2024-09-13T14:47:46.322613+0000","last_unstale":"2024-09-13T14:59:04.506292+0000","last_undegraded":"2024-09-13T14:59:04.506292+0000","last_fullsized":"2024-09-13T14:59:04.506292+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:45.314706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1039,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.152949+0000","last_change":"2024-09-13T14:52:43.613106+0000","last_active":"2024-09-13T14:59:00.152949+0000","last_peered":"2024-09-13T14:59:00.152949+0000","last_clean":"2024-09-13T14:59:00.152949+0000","last_became_active":"2024-09-13T14:52:43.594081+0000","last_became_peered":"2024-09-13T14:52:43.594081+0000","last_unstale":"2024-09-13T14:59:00.152949+0000","last_undegraded":"2024-09-13T14:59:00.152949+0000","last_fullsized":"2024-09-13T14:59:00.152949+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:11.121629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.507943+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T14:59:04.507943+0000","last_peered":"2024-09-13T14:59:04.507943+0000","last_clean":"2024-09-13T14:59:04.507943+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T14:59:04.507943+0000","last_undegraded":"2024-09-13T14:59:04.507943+0000","last_fullsized":"2024-09-13T14:59:04.507943+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":89,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.506766+0000","last_change":"2024-09-13T14:52:43.582732+0000","last_active":"2024-09-13T14:59:04.506766+0000","last_peered":"2024-09-13T14:59:04.506766+0000","last_clean":"2024-09-13T14:59:04.506766+0000","last_became_active":"2024-09-13T14:52:43.582384+0000","last_became_peered":"2024-09-13T14:52:43.582384+0000","last_unstale":"2024-09-13T14:59:04.506766+0000","last_undegraded":"2024-09-13T14:59:04.506766+0000","last_fullsized":"2024-09-13T14:59:04.506766+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:06.715139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1369,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.508196+0000","last_change":"2024-09-13T14:57:52.533226+0000","last_active":"2024-09-13T14:59:04.508196+0000","last_peered":"2024-09-13T14:59:04.508196+0000","last_clean":"2024-09-13T14:57:42.822755+0000","last_became_active":"2024-09-13T14:57:52.533226+0000","last_became_peered":"2024-09-13T14:57:52.533226+0000","last_unstale":"2024-09-13T14:59:04.508196+0000","last_undegraded":"2024-09-13T14:59:04.508196+0000","last_fullsized":"2024-09-13T14:57:52.518902+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:16:10.437860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1492,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.502072+0000","last_change":"2024-09-13T14:52:43.595169+0000","last_active":"2024-09-13T14:59:04.502072+0000","last_peered":"2024-09-13T14:59:04.502072+0000","last_clean":"2024-09-13T14:59:04.502072+0000","last_became_active":"2024-09-13T14:52:43.594586+0000","last_became_peered":"2024-09-13T14:52:43.594586+0000","last_unstale":"2024-09-13T14:59:04.502072+0000","last_undegraded":"2024-09-13T14:59:04.502072+0000","last_fullsized":"2024-09-13T14:59:04.502072+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:53.946025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1497,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.503703+0000","last_change":"2024-09-13T14:57:52.526901+0000","last_active":"2024-09-13T14:59:04.503703+0000","last_peered":"2024-09-13T14:59:04.503703+0000","last_clean":"2024-09-13T14:57:42.887232+0000","last_became_active":"2024-09-13T14:57:52.526901+0000","last_became_peered":"2024-09-13T14:57:52.526901+0000","last_unstale":"2024-09-13T14:59:04.503703+0000","last_undegraded":"2024-09-13T14:59:04.503703+0000","last_fullsized":"2024-09-13T14:57:52.514561+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:47:39.792742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1049,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.153127+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T14:59:00.153127+0000","last_peered":"2024-09-13T14:59:00.153127+0000","last_clean":"2024-09-13T14:59:00.153127+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T14:59:00.153127+0000","last_undegraded":"2024-09-13T14:59:00.153127+0000","last_fullsized":"2024-09-13T14:59:00.153127+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1020,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.155353+0000","last_change":"2024-09-13T14:47:46.320830+0000","last_active":"2024-09-13T14:59:00.155353+0000","last_peered":"2024-09-13T14:59:00.155353+0000","last_clean":"2024-09-13T14:59:00.155353+0000","last_became_active":"2024-09-13T14:47:46.320443+0000","last_became_peered":"2024-09-13T14:47:46.320443+0000","last_unstale":"2024-09-13T14:59:00.155353+0000","last_undegraded":"2024-09-13T14:59:00.155353+0000","last_fullsized":"2024-09-13T14:59:00.155353+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:46.279580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":644,"reported_epoch":1042,"state":"active+undersized","last_fresh":"2024-09-13T14:59:00.153216+0000","last_change":"2024-09-13T14:57:52.522034+0000","last_active":"2024-09-13T14:59:00.153216+0000","last_peered":"2024-09-13T14:59:00.153216+0000","last_clean":"2024-09-13T14:24:16.175691+0000","last_became_active":"2024-09-13T14:57:52.522034+0000","last_became_peered":"2024-09-13T14:57:52.522034+0000","last_unstale":"2024-09-13T14:59:00.153216+0000","last_undegraded":"2024-09-13T14:59:00.153216+0000","last_fullsized":"2024-09-13T14:57:52.514676+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:15.687612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1010,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.152289+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T14:59:00.152289+0000","last_peered":"2024-09-13T14:59:00.152289+0000","last_clean":"2024-09-13T14:59:00.152289+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T14:59:00.152289+0000","last_undegraded":"2024-09-13T14:59:00.152289+0000","last_fullsized":"2024-09-13T14:59:00.152289+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1117,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.153212+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T14:59:00.153212+0000","last_peered":"2024-09-13T14:59:00.153212+0000","last_clean":"2024-09-13T14:59:00.153212+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T14:59:00.153212+0000","last_undegraded":"2024-09-13T14:59:00.153212+0000","last_fullsized":"2024-09-13T14:59:00.153212+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":93,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.508081+0000","last_change":"2024-09-13T14:57:52.537590+0000","last_active":"2024-09-13T14:59:04.508081+0000","last_peered":"2024-09-13T14:59:04.508081+0000","last_clean":"2024-09-13T14:57:43.386126+0000","last_became_active":"2024-09-13T14:57:52.537590+0000","last_became_peered":"2024-09-13T14:57:52.537590+0000","last_unstale":"2024-09-13T14:59:04.508081+0000","last_undegraded":"2024-09-13T14:59:04.508081+0000","last_fullsized":"2024-09-13T14:57:52.527356+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:03.099081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1292,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.505502+0000","last_change":"2024-09-13T14:57:52.532557+0000","last_active":"2024-09-13T14:59:04.505502+0000","last_peered":"2024-09-13T14:59:04.505502+0000","last_clean":"2024-09-13T14:52:40.121056+0000","last_became_active":"2024-09-13T14:57:52.532557+0000","last_became_peered":"2024-09-13T14:57:52.532557+0000","last_unstale":"2024-09-13T14:59:04.505502+0000","last_undegraded":"2024-09-13T14:59:04.505502+0000","last_fullsized":"2024-09-13T14:57:52.514161+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:26:26.835138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":199,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.506894+0000","last_change":"2024-09-13T14:52:43.599179+0000","last_active":"2024-09-13T14:59:04.506894+0000","last_peered":"2024-09-13T14:59:04.506894+0000","last_clean":"2024-09-13T14:59:04.506894+0000","last_became_active":"2024-09-13T14:52:43.598166+0000","last_became_peered":"2024-09-13T14:52:43.598166+0000","last_unstale":"2024-09-13T14:59:04.506894+0000","last_undegraded":"2024-09-13T14:59:04.506894+0000","last_fullsized":"2024-09-13T14:59:04.506894+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:08.647786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":185,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.505105+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T14:59:04.505105+0000","last_peered":"2024-09-13T14:59:04.505105+0000","last_clean":"2024-09-13T14:59:04.505105+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T14:59:04.505105+0000","last_undegraded":"2024-09-13T14:59:04.505105+0000","last_fullsized":"2024-09-13T14:59:04.505105+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":199,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.506644+0000","last_change":"2024-09-13T14:52:43.597595+0000","last_active":"2024-09-13T14:59:04.506644+0000","last_peered":"2024-09-13T14:59:04.506644+0000","last_clean":"2024-09-13T14:59:04.506644+0000","last_became_active":"2024-09-13T14:52:43.596592+0000","last_became_peered":"2024-09-13T14:52:43.596592+0000","last_unstale":"2024-09-13T14:59:04.506644+0000","last_undegraded":"2024-09-13T14:59:04.506644+0000","last_fullsized":"2024-09-13T14:59:04.506644+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:56.532603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1058,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.152542+0000","last_change":"2024-09-13T14:52:43.596429+0000","last_active":"2024-09-13T14:59:00.152542+0000","last_peered":"2024-09-13T14:59:00.152542+0000","last_clean":"2024-09-13T14:59:00.152542+0000","last_became_active":"2024-09-13T14:52:43.594614+0000","last_became_peered":"2024-09-13T14:52:43.594614+0000","last_unstale":"2024-09-13T14:59:00.152542+0000","last_undegraded":"2024-09-13T14:59:00.152542+0000","last_fullsized":"2024-09-13T14:59:00.152542+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:41.970225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":189,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.504782+0000","last_change":"2024-09-13T14:57:52.541208+0000","last_active":"2024-09-13T14:59:04.504782+0000","last_peered":"2024-09-13T14:59:04.504782+0000","last_clean":"2024-09-13T14:57:42.819963+0000","last_became_active":"2024-09-13T14:57:52.541208+0000","last_became_peered":"2024-09-13T14:57:52.541208+0000","last_unstale":"2024-09-13T14:59:04.504782+0000","last_undegraded":"2024-09-13T14:59:04.504782+0000","last_fullsized":"2024-09-13T14:57:52.511806+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:01.484625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":74,"reported_epoch":1043,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:04.502669+0000","last_change":"2024-09-13T14:57:52.608759+0000","last_active":"2024-09-13T14:59:04.502669+0000","last_peered":"2024-09-13T14:59:04.502669+0000","last_clean":"2024-09-13T14:24:20.312402+0000","last_became_active":"2024-09-13T14:57:52.608759+0000","last_became_peered":"2024-09-13T14:57:52.608759+0000","last_unstale":"2024-09-13T14:59:04.502669+0000","last_undegraded":"2024-09-13T14:57:52.520273+0000","last_fullsized":"2024-09-13T14:57:52.520078+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:53.804929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1513,"reported_epoch":1043,"state":"active+clean","last_fresh":"2024-09-13T14:59:04.502442+0000","last_change":"2024-09-13T14:52:54.185117+0000","last_active":"2024-09-13T14:59:04.502442+0000","last_peered":"2024-09-13T14:59:04.502442+0000","last_clean":"2024-09-13T14:59:04.502442+0000","last_became_active":"2024-09-13T14:52:54.184712+0000","last_became_peered":"2024-09-13T14:52:54.184712+0000","last_unstale":"2024-09-13T14:59:04.502442+0000","last_undegraded":"2024-09-13T14:59:04.502442+0000","last_fullsized":"2024-09-13T14:59:04.502442+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:32.654401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1004,"reported_epoch":1042,"state":"active+undersized","last_fresh":"2024-09-13T14:59:00.156370+0000","last_change":"2024-09-13T14:57:52.523302+0000","last_active":"2024-09-13T14:59:00.156370+0000","last_peered":"2024-09-13T14:59:00.156370+0000","last_clean":"2024-09-13T14:57:42.821860+0000","last_became_active":"2024-09-13T14:57:52.523302+0000","last_became_peered":"2024-09-13T14:57:52.523302+0000","last_unstale":"2024-09-13T14:59:00.156370+0000","last_undegraded":"2024-09-13T14:59:00.156370+0000","last_fullsized":"2024-09-13T14:57:52.514457+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.134976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1020,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.152292+0000","last_change":"2024-09-13T14:52:43.581604+0000","last_active":"2024-09-13T14:59:00.152292+0000","last_peered":"2024-09-13T14:59:00.152292+0000","last_clean":"2024-09-13T14:59:00.152292+0000","last_became_active":"2024-09-13T14:52:43.579981+0000","last_became_peered":"2024-09-13T14:52:43.579981+0000","last_unstale":"2024-09-13T14:59:00.152292+0000","last_undegraded":"2024-09-13T14:59:00.152292+0000","last_fullsized":"2024-09-13T14:59:00.152292+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:59.366292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":985,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.156898+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T14:59:00.156898+0000","last_peered":"2024-09-13T14:59:00.156898+0000","last_clean":"2024-09-13T14:59:00.156898+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T14:59:00.156898+0000","last_undegraded":"2024-09-13T14:59:00.156898+0000","last_fullsized":"2024-09-13T14:59:00.156898+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":203,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.505232+0000","last_change":"2024-09-13T14:57:52.538489+0000","last_active":"2024-09-13T14:59:04.505232+0000","last_peered":"2024-09-13T14:59:04.505232+0000","last_clean":"2024-09-13T14:57:42.820043+0000","last_became_active":"2024-09-13T14:57:52.538489+0000","last_became_peered":"2024-09-13T14:57:52.538489+0000","last_unstale":"2024-09-13T14:59:04.505232+0000","last_undegraded":"2024-09-13T14:59:04.505232+0000","last_fullsized":"2024-09-13T14:57:52.512057+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:24.504837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1076,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.157134+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T14:59:00.157134+0000","last_peered":"2024-09-13T14:59:00.157134+0000","last_clean":"2024-09-13T14:59:00.157134+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T14:59:00.157134+0000","last_undegraded":"2024-09-13T14:59:00.157134+0000","last_fullsized":"2024-09-13T14:59:00.157134+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":979,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.154587+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T14:59:00.154587+0000","last_peered":"2024-09-13T14:59:00.154587+0000","last_clean":"2024-09-13T14:59:00.154587+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T14:59:00.154587+0000","last_undegraded":"2024-09-13T14:59:00.154587+0000","last_fullsized":"2024-09-13T14:59:00.154587+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1047,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.157259+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T14:59:00.157259+0000","last_peered":"2024-09-13T14:59:00.157259+0000","last_clean":"2024-09-13T14:59:00.157259+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T14:59:00.157259+0000","last_undegraded":"2024-09-13T14:59:00.157259+0000","last_fullsized":"2024-09-13T14:59:00.157259+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":1013,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.154656+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T14:59:00.154656+0000","last_peered":"2024-09-13T14:59:00.154656+0000","last_clean":"2024-09-13T14:59:00.154656+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T14:59:00.154656+0000","last_undegraded":"2024-09-13T14:59:00.154656+0000","last_fullsized":"2024-09-13T14:59:00.154656+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1282,"reported_epoch":1042,"state":"active+undersized","last_fresh":"2024-09-13T14:59:00.157255+0000","last_change":"2024-09-13T14:57:52.531820+0000","last_active":"2024-09-13T14:59:00.157255+0000","last_peered":"2024-09-13T14:59:00.157255+0000","last_clean":"2024-09-13T14:52:40.124168+0000","last_became_active":"2024-09-13T14:57:52.531820+0000","last_became_peered":"2024-09-13T14:57:52.531820+0000","last_unstale":"2024-09-13T14:59:00.157255+0000","last_undegraded":"2024-09-13T14:59:00.157255+0000","last_fullsized":"2024-09-13T14:57:52.517742+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:16.785361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1020,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.152786+0000","last_change":"2024-09-13T14:47:46.322428+0000","last_active":"2024-09-13T14:59:00.152786+0000","last_peered":"2024-09-13T14:59:00.152786+0000","last_clean":"2024-09-13T14:59:00.152786+0000","last_became_active":"2024-09-13T14:47:46.321955+0000","last_became_peered":"2024-09-13T14:47:46.321955+0000","last_unstale":"2024-09-13T14:59:00.152786+0000","last_undegraded":"2024-09-13T14:59:00.152786+0000","last_fullsized":"2024-09-13T14:59:00.152786+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:47.614065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1470,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.502253+0000","last_change":"2024-09-13T14:57:52.542172+0000","last_active":"2024-09-13T14:59:04.502253+0000","last_peered":"2024-09-13T14:59:04.502253+0000","last_clean":"2024-09-13T14:57:42.887703+0000","last_became_active":"2024-09-13T14:57:52.542172+0000","last_became_peered":"2024-09-13T14:57:52.542172+0000","last_unstale":"2024-09-13T14:59:04.502253+0000","last_undegraded":"2024-09-13T14:59:04.502253+0000","last_fullsized":"2024-09-13T14:57:52.522300+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:26.613968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1471,"reported_epoch":1043,"state":"active+undersized","last_fresh":"2024-09-13T14:59:04.502240+0000","last_change":"2024-09-13T14:57:52.534865+0000","last_active":"2024-09-13T14:59:04.502240+0000","last_peered":"2024-09-13T14:59:04.502240+0000","last_clean":"2024-09-13T14:57:42.887357+0000","last_became_active":"2024-09-13T14:57:52.534865+0000","last_became_peered":"2024-09-13T14:57:52.534865+0000","last_unstale":"2024-09-13T14:59:04.502240+0000","last_undegraded":"2024-09-13T14:59:04.502240+0000","last_fullsized":"2024-09-13T14:57:52.517042+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:17.508856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1003,"reported_epoch":1042,"state":"active+undersized","last_fresh":"2024-09-13T14:59:00.154155+0000","last_change":"2024-09-13T14:57:52.523699+0000","last_active":"2024-09-13T14:59:00.154155+0000","last_peered":"2024-09-13T14:59:00.154155+0000","last_clean":"2024-09-13T14:57:42.822145+0000","last_became_active":"2024-09-13T14:57:52.523699+0000","last_became_peered":"2024-09-13T14:57:52.523699+0000","last_unstale":"2024-09-13T14:59:00.154155+0000","last_undegraded":"2024-09-13T14:59:00.154155+0000","last_fullsized":"2024-09-13T14:57:52.515367+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:46:32.001179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1053,"reported_epoch":1042,"state":"active+clean","last_fresh":"2024-09-13T14:59:00.155890+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T14:59:00.155890+0000","last_peered":"2024-09-13T14:59:00.155890+0000","last_clean":"2024-09-13T14:59:00.155890+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T14:59:00.155890+0000","last_undegraded":"2024-09-13T14:59:00.155890+0000","last_fullsized":"2024-09-13T14:59:00.155890+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1293,"reported_epoch":1042,"state":"active+undersized","last_fresh":"2024-09-13T14:59:00.156143+0000","last_change":"2024-09-13T14:57:52.525160+0000","last_active":"2024-09-13T14:59:00.156143+0000","last_peered":"2024-09-13T14:59:00.156143+0000","last_clean":"2024-09-13T14:52:43.593692+0000","last_became_active":"2024-09-13T14:57:52.525160+0000","last_became_peered":"2024-09-13T14:57:52.525160+0000","last_unstale":"2024-09-13T14:59:00.156143+0000","last_undegraded":"2024-09-13T14:59:00.156143+0000","last_fullsized":"2024-09-13T14:57:52.514147+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:40.347221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":43,"num_pg":16,"stat_sum":{"num_bytes":3498901504,"num_objects":53389,"num_object_clones":0,"num_object_copies":160167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16733,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":53389,"num_write_kb":3416896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9299099648,"data_stored":9299099648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":53389,"ondisk_log_size":53389,"up":43,"acting":43,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":23,"acting":23,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":83,"acting":83,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":81,"acting":81,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397982,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1825356,"kb_used_data":1515524,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91919796,"statfs":{"total":95995035648,"available":94125871104,"internally_reserved":0,"allocated":1551896576,"data_stored":1547839917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594217,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1789316,"kb_used_data":1483644,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91955836,"statfs":{"total":95995035648,"available":94162776064,"internally_reserved":0,"allocated":1519251456,"data_stored":1515202862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790451,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1593848,"kb_used_data":1288176,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92151304,"statfs":{"total":95995035648,"available":94362935296,"internally_reserved":0,"allocated":1319092224,"data_stored":1315051192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980831979,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1174788,"kb_used_data":807676,"kb_used_omap":76,"kb_used_meta":367027,"kb_avail":92570364,"statfs":{"total":95995035648,"available":94792052736,"internally_reserved":0,"allocated":827060224,"data_stored":823011155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":375836480},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911197,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307024,"kb_used_data":10564,"kb_used_omap":77,"kb_used_meta":296370,"kb_avail":93438128,"statfs":{"total":95995035648,"available":95680643072,"internally_reserved":0,"allocated":10817536,"data_stored":6861682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":303483701},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023161,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1656308,"kb_used_data":1256812,"kb_used_omap":71,"kb_used_meta":399416,"kb_avail":92088844,"statfs":{"total":95995035648,"available":94298976256,"internally_reserved":0,"allocated":1286975488,"data_stored":1282938533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":409002656},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004291,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1543664,"kb_used_data":1268840,"kb_used_omap":66,"kb_used_meta":274749,"kb_avail":92201488,"statfs":{"total":95995035648,"available":94414323712,"internally_reserved":0,"allocated":1299292160,"data_stored":1295258716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":281343146},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195483,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1809076,"kb_used_data":1446956,"kb_used_omap":56,"kb_used_meta":362055,"kb_avail":91936076,"statfs":{"total":95995035648,"available":94142541824,"internally_reserved":0,"allocated":1481682944,"data_stored":1477644399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":370744463},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":43,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1471873024,"data_stored":1471873024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":43,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1301020672,"data_stored":1301020672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":43,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1280901120,"data_stored":1280901120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":43,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":827129856,"data_stored":827129856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":43,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1330380800,"data_stored":1330380800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":43,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1541734400,"data_stored":1541734400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":43,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1546059776,"data_stored":1546059776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:59:41.741 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:59:41.741 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_2 pg_num 26 2024-09-13T14:59:41.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:41.837 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:41.865 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T14:59:41.940 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:41.965 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:41.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:42.067 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[84576]: cluster 2024-09-13T14:59:40.760777+0000 mgr.y (mgr.26252) 1006 : cluster [DBG] pgmap v1538: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 18673/178839 objects degraded (10.441%) 2024-09-13T14:59:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[84576]: cluster 2024-09-13T14:59:40.760777+0000 mgr.y (mgr.26252) 1006 : cluster [DBG] pgmap v1538: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 18673/178839 objects degraded (10.441%) 2024-09-13T14:59:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[84576]: audit 2024-09-13T14:59:41.429752+0000 mon.a (mon.0) 2060 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[84576]: audit 2024-09-13T14:59:41.429752+0000 mon.a (mon.0) 2060 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[84576]: cluster 2024-09-13T14:59:41.451495+0000 mon.a (mon.0) 2061 : cluster [WRN] Health check update: Degraded data redundancy: 18673/178839 objects degraded (10.441%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[84576]: cluster 2024-09-13T14:59:41.451495+0000 mon.a (mon.0) 2061 : cluster [WRN] Health check update: Degraded data redundancy: 18673/178839 objects degraded (10.441%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[87147]: cluster 2024-09-13T14:59:40.760777+0000 mgr.y (mgr.26252) 1006 : cluster [DBG] pgmap v1538: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 18673/178839 objects degraded (10.441%) 2024-09-13T14:59:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[87147]: cluster 2024-09-13T14:59:40.760777+0000 mgr.y (mgr.26252) 1006 : cluster [DBG] pgmap v1538: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 18673/178839 objects degraded (10.441%) 2024-09-13T14:59:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[87147]: audit 2024-09-13T14:59:41.429752+0000 mon.a (mon.0) 2060 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[87147]: audit 2024-09-13T14:59:41.429752+0000 mon.a (mon.0) 2060 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[87147]: cluster 2024-09-13T14:59:41.451495+0000 mon.a (mon.0) 2061 : cluster [WRN] Health check update: Degraded data redundancy: 18673/178839 objects degraded (10.441%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:41 smithi099 bash[87147]: cluster 2024-09-13T14:59:41.451495+0000 mon.a (mon.0) 2061 : cluster [WRN] Health check update: Degraded data redundancy: 18673/178839 objects degraded (10.441%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:42.095 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:42.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:42.198 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:42.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:42.325 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:42.335 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:41 smithi177 bash[68594]: cluster 2024-09-13T14:59:40.760777+0000 mgr.y (mgr.26252) 1006 : cluster [DBG] pgmap v1538: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 18673/178839 objects degraded (10.441%) 2024-09-13T14:59:42.335 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:41 smithi177 bash[68594]: cluster 2024-09-13T14:59:40.760777+0000 mgr.y (mgr.26252) 1006 : cluster [DBG] pgmap v1538: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 18673/178839 objects degraded (10.441%) 2024-09-13T14:59:42.336 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:41 smithi177 bash[68594]: audit 2024-09-13T14:59:41.429752+0000 mon.a (mon.0) 2060 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:42.336 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:41 smithi177 bash[68594]: audit 2024-09-13T14:59:41.429752+0000 mon.a (mon.0) 2060 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:42.336 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:41 smithi177 bash[68594]: cluster 2024-09-13T14:59:41.451495+0000 mon.a (mon.0) 2061 : cluster [WRN] Health check update: Degraded data redundancy: 18673/178839 objects degraded (10.441%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:42.336 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:41 smithi177 bash[68594]: cluster 2024-09-13T14:59:41.451495+0000 mon.a (mon.0) 2061 : cluster [WRN] Health check update: Degraded data redundancy: 18673/178839 objects degraded (10.441%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:42.356 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:42.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:42.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:42.473 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 38 16 63457 63441 104.334 61.9375 0.0194076 0.0095795 2024-09-13T14:59:42.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:42.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:42.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:42.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:42.718 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:42.743 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:42.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:42.791 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:59:42.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:42.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:42.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:42.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:42.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:42 smithi177 bash[68594]: cluster 2024-09-13T14:59:41.865007+0000 mon.a (mon.0) 2062 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:42.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:42 smithi177 bash[68594]: cluster 2024-09-13T14:59:41.865007+0000 mon.a (mon.0) 2062 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:42.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:42 smithi177 bash[68594]: audit 2024-09-13T14:59:41.866432+0000 mon.a (mon.0) 2063 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:42.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:42 smithi177 bash[68594]: audit 2024-09-13T14:59:41.866432+0000 mon.a (mon.0) 2063 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:42.985 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:42 smithi177 bash[68594]: cluster 2024-09-13T14:59:41.870968+0000 mon.a (mon.0) 2064 : cluster [DBG] osdmap e1044: 8 total, 7 up, 8 in 2024-09-13T14:59:42.985 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:42 smithi177 bash[68594]: cluster 2024-09-13T14:59:41.870968+0000 mon.a (mon.0) 2064 : cluster [DBG] osdmap e1044: 8 total, 7 up, 8 in 2024-09-13T14:59:43.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:43.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[87147]: cluster 2024-09-13T14:59:41.865007+0000 mon.a (mon.0) 2062 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[87147]: cluster 2024-09-13T14:59:41.865007+0000 mon.a (mon.0) 2062 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:43.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[87147]: audit 2024-09-13T14:59:41.866432+0000 mon.a (mon.0) 2063 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:43.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[87147]: audit 2024-09-13T14:59:41.866432+0000 mon.a (mon.0) 2063 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:43.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[87147]: cluster 2024-09-13T14:59:41.870968+0000 mon.a (mon.0) 2064 : cluster [DBG] osdmap e1044: 8 total, 7 up, 8 in 2024-09-13T14:59:43.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[87147]: cluster 2024-09-13T14:59:41.870968+0000 mon.a (mon.0) 2064 : cluster [DBG] osdmap e1044: 8 total, 7 up, 8 in 2024-09-13T14:59:43.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[84576]: cluster 2024-09-13T14:59:41.865007+0000 mon.a (mon.0) 2062 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:43.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[84576]: cluster 2024-09-13T14:59:41.865007+0000 mon.a (mon.0) 2062 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:43.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[84576]: audit 2024-09-13T14:59:41.866432+0000 mon.a (mon.0) 2063 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:43.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[84576]: audit 2024-09-13T14:59:41.866432+0000 mon.a (mon.0) 2063 : audit [INF] from='client.? 172.21.15.99:0/1367925642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:43.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[84576]: cluster 2024-09-13T14:59:41.870968+0000 mon.a (mon.0) 2064 : cluster [DBG] osdmap e1044: 8 total, 7 up, 8 in 2024-09-13T14:59:43.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:42 smithi099 bash[84576]: cluster 2024-09-13T14:59:41.870968+0000 mon.a (mon.0) 2064 : cluster [DBG] osdmap e1044: 8 total, 7 up, 8 in 2024-09-13T14:59:43.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:43.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:43.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:43.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:43.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:43.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:43.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:43.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:43.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:43.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 39 16 64757 64741 103.742 81.25 0.00878291 0.00963431 2024-09-13T14:59:43.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:43.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:43.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:43.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:43.678 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:43.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:43.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:43.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:43.917 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.954 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:43.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:44.055 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:44.065 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:43 smithi099 bash[84576]: cluster 2024-09-13T14:59:42.761741+0000 mgr.y (mgr.26252) 1007 : cluster [DBG] pgmap v1540: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 19039/182091 objects degraded (10.456%) 2024-09-13T14:59:44.066 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:43 smithi099 bash[84576]: cluster 2024-09-13T14:59:42.761741+0000 mgr.y (mgr.26252) 1007 : cluster [DBG] pgmap v1540: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 19039/182091 objects degraded (10.456%) 2024-09-13T14:59:44.066 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:43 smithi099 bash[87147]: cluster 2024-09-13T14:59:42.761741+0000 mgr.y (mgr.26252) 1007 : cluster [DBG] pgmap v1540: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 19039/182091 objects degraded (10.456%) 2024-09-13T14:59:44.066 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:43 smithi099 bash[87147]: cluster 2024-09-13T14:59:42.761741+0000 mgr.y (mgr.26252) 1007 : cluster [DBG] pgmap v1540: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 19039/182091 objects degraded (10.456%) 2024-09-13T14:59:44.079 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:44.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:44.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:43 smithi177 bash[68594]: cluster 2024-09-13T14:59:42.761741+0000 mgr.y (mgr.26252) 1007 : cluster [DBG] pgmap v1540: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 19039/182091 objects degraded (10.456%) 2024-09-13T14:59:44.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:43 smithi177 bash[68594]: cluster 2024-09-13T14:59:42.761741+0000 mgr.y (mgr.26252) 1007 : cluster [DBG] pgmap v1540: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 19039/182091 objects degraded (10.456%) 2024-09-13T14:59:44.185 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:44.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:44.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:44.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:44.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:44.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:44.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T14:59:44.475443+0000 min lat: 0.00108369 max lat: 0.468934 avg lat: 0.00967339 2024-09-13T14:59:44.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:59:44.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 40 16 66146 66130 103.319 86.8125 0.00944511 0.00967339 2024-09-13T14:59:44.493 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:44.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:44.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:44.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:44.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:44.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:44.761 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:44.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:44.866 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:44.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:44.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:45.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:45.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:45.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:45.163 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:45.271 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:45.304 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:45.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:45.408 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:45.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:45.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:45.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 41 16 67520 67504 102.893 85.875 0.00863847 0.00971453 2024-09-13T14:59:45.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:45.562 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:45.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:45.665 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:45.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:45.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:45.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:59:45.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:45.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:45.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:45.932 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:45.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:45.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:46.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:46.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:45 smithi099 bash[84576]: cluster 2024-09-13T14:59:44.763127+0000 mgr.y (mgr.26252) 1008 : cluster [DBG] pgmap v1541: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:46.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:45 smithi099 bash[84576]: cluster 2024-09-13T14:59:44.763127+0000 mgr.y (mgr.26252) 1008 : cluster [DBG] pgmap v1541: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:46.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:45 smithi099 bash[87147]: cluster 2024-09-13T14:59:44.763127+0000 mgr.y (mgr.26252) 1008 : cluster [DBG] pgmap v1541: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:46.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:45 smithi099 bash[87147]: cluster 2024-09-13T14:59:44.763127+0000 mgr.y (mgr.26252) 1008 : cluster [DBG] pgmap v1541: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:46.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:46.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:46.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:45 smithi177 bash[68594]: cluster 2024-09-13T14:59:44.763127+0000 mgr.y (mgr.26252) 1008 : cluster [DBG] pgmap v1541: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:46.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:45 smithi177 bash[68594]: cluster 2024-09-13T14:59:44.763127+0000 mgr.y (mgr.26252) 1008 : cluster [DBG] pgmap v1541: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:46.221 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:46.234 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:46.338 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.363 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:46.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 42 16 68826 68810 102.386 81.625 0.013455 0.00976269 2024-09-13T14:59:46.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:46.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:46.634 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:46.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:46.739 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.763 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:46.869 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:46.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:46.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:47.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:47.022 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:46 smithi177 bash[68594]: cluster 2024-09-13T14:59:46.452214+0000 mon.a (mon.0) 2065 : cluster [WRN] Health check update: Degraded data redundancy: 20528/196899 objects degraded (10.426%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:47.023 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:46 smithi177 bash[68594]: cluster 2024-09-13T14:59:46.452214+0000 mon.a (mon.0) 2065 : cluster [WRN] Health check update: Degraded data redundancy: 20528/196899 objects degraded (10.426%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:47.033 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:47.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:47.137 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:47.155 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:47.236 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:47.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:47.266 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:59:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:59:47] "GET /metrics HTTP/1.1" 200 36766 "" "Prometheus/2.51.0" 2024-09-13T14:59:47.266 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:46 smithi099 bash[84576]: cluster 2024-09-13T14:59:46.452214+0000 mon.a (mon.0) 2065 : cluster [WRN] Health check update: Degraded data redundancy: 20528/196899 objects degraded (10.426%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:47.266 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:46 smithi099 bash[84576]: cluster 2024-09-13T14:59:46.452214+0000 mon.a (mon.0) 2065 : cluster [WRN] Health check update: Degraded data redundancy: 20528/196899 objects degraded (10.426%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:47.266 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:46 smithi099 bash[87147]: cluster 2024-09-13T14:59:46.452214+0000 mon.a (mon.0) 2065 : cluster [WRN] Health check update: Degraded data redundancy: 20528/196899 objects degraded (10.426%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:47.266 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:46 smithi099 bash[87147]: cluster 2024-09-13T14:59:46.452214+0000 mon.a (mon.0) 2065 : cluster [WRN] Health check update: Degraded data redundancy: 20528/196899 objects degraded (10.426%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:47.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:47.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:47.402 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:47.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:47.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 43 16 69697 69681 101.271 54.4375 0.00288708 0.009815 2024-09-13T14:59:47.505 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:47.536 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:47.625 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:47.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:47.677 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:47.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:47.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:47.801 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:47.904 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:47.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:47.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:48.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:48.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:47 smithi177 bash[68594]: cluster 2024-09-13T14:59:46.763748+0000 mgr.y (mgr.26252) 1009 : cluster [DBG] pgmap v1542: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:48.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:47 smithi177 bash[68594]: cluster 2024-09-13T14:59:46.763748+0000 mgr.y (mgr.26252) 1009 : cluster [DBG] pgmap v1542: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:48.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:48.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:47 smithi099 bash[84576]: cluster 2024-09-13T14:59:46.763748+0000 mgr.y (mgr.26252) 1009 : cluster [DBG] pgmap v1542: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:47 smithi099 bash[84576]: cluster 2024-09-13T14:59:46.763748+0000 mgr.y (mgr.26252) 1009 : cluster [DBG] pgmap v1542: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:48.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:47 smithi099 bash[87147]: cluster 2024-09-13T14:59:46.763748+0000 mgr.y (mgr.26252) 1009 : cluster [DBG] pgmap v1542: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:48.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:47 smithi099 bash[87147]: cluster 2024-09-13T14:59:46.763748+0000 mgr.y (mgr.26252) 1009 : cluster [DBG] pgmap v1542: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20528/196899 objects degraded (10.426%) 2024-09-13T14:59:48.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:48.194 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:48.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:48.297 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:48.330 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:48.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:48.435 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:48.469 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:48.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:48.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 44 16 70836 70820 100.587 71.1875 0.0186261 0.00993549 2024-09-13T14:59:48.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:48.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:48.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:48.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:48.736 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:48.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:48.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:48.871 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:48.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:48.978 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:48.988 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:48 smithi177 bash[68594]: audit 2024-09-13T14:59:48.289511+0000 mon.a (mon.0) 2066 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:48.988 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:48 smithi177 bash[68594]: audit 2024-09-13T14:59:48.289511+0000 mon.a (mon.0) 2066 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:49.003 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:49.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:49.111 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:49.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:49.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:49.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:49.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:49.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:49.377 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:49.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:49.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:49.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 45 16 72154 72138 100.182 82.375 0.00696525 0.00997684 2024-09-13T14:59:49.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:49.526 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:49 smithi099 bash[84576]: audit 2024-09-13T14:59:48.289511+0000 mon.a (mon.0) 2066 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:49.526 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:49 smithi099 bash[84576]: audit 2024-09-13T14:59:48.289511+0000 mon.a (mon.0) 2066 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:49.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:49 smithi099 bash[87147]: audit 2024-09-13T14:59:48.289511+0000 mon.a (mon.0) 2066 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:49.526 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:49 smithi099 bash[87147]: audit 2024-09-13T14:59:48.289511+0000 mon.a (mon.0) 2066 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:49.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:49.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:49.647 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:49.674 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:49.777 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:49.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:49.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:49.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:49.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:49.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:50.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:50.055 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:49 smithi099 bash[84576]: cluster 2024-09-13T14:59:48.764440+0000 mgr.y (mgr.26252) 1010 : cluster [DBG] pgmap v1543: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20890/200388 objects degraded (10.425%) 2024-09-13T14:59:50.055 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:49 smithi099 bash[84576]: cluster 2024-09-13T14:59:48.764440+0000 mgr.y (mgr.26252) 1010 : cluster [DBG] pgmap v1543: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20890/200388 objects degraded (10.425%) 2024-09-13T14:59:50.056 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:49 smithi099 bash[87147]: cluster 2024-09-13T14:59:48.764440+0000 mgr.y (mgr.26252) 1010 : cluster [DBG] pgmap v1543: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20890/200388 objects degraded (10.425%) 2024-09-13T14:59:50.056 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:49 smithi099 bash[87147]: cluster 2024-09-13T14:59:48.764440+0000 mgr.y (mgr.26252) 1010 : cluster [DBG] pgmap v1543: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20890/200388 objects degraded (10.425%) 2024-09-13T14:59:50.076 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:50.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:50.180 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:50.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:49 smithi177 bash[68594]: cluster 2024-09-13T14:59:48.764440+0000 mgr.y (mgr.26252) 1010 : cluster [DBG] pgmap v1543: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20890/200388 objects degraded (10.425%) 2024-09-13T14:59:50.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:49 smithi177 bash[68594]: cluster 2024-09-13T14:59:48.764440+0000 mgr.y (mgr.26252) 1010 : cluster [DBG] pgmap v1543: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 20890/200388 objects degraded (10.425%) 2024-09-13T14:59:50.209 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:50.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:50.281 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:50.314 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:50.336 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:50.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:50.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:50.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:50.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:50.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 46 16 73223 73207 99.4567 66.8125 0.00325182 0.0100228 2024-09-13T14:59:50.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:50.588 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:50.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:50.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 14:59:50 smithi177 bash[61439]: ts=2024-09-13T14:59:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T14:59:50.691 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:50.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:50.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:50.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:50.857 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:50.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:50.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:50.987 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:50.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:51.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:51.119 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:51.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:51.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:51.242 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:51.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:51.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:51.378 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:51.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:51.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 47 16 74068 74052 98.4642 52.8125 0.0154958 0.0101513 2024-09-13T14:59:51.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:51.512 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:51.615 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:51.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:51.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:51.742 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:51.761 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:51.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:51.863 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:51.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:51.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:51.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:52.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:52.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:52.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:51 smithi177 bash[68594]: cluster 2024-09-13T14:59:50.765379+0000 mgr.y (mgr.26252) 1011 : cluster [DBG] pgmap v1544: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 22389/215094 objects degraded (10.409%) 2024-09-13T14:59:52.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:51 smithi177 bash[68594]: cluster 2024-09-13T14:59:50.765379+0000 mgr.y (mgr.26252) 1011 : cluster [DBG] pgmap v1544: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 22389/215094 objects degraded (10.409%) 2024-09-13T14:59:52.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:51 smithi177 bash[68594]: cluster 2024-09-13T14:59:51.453056+0000 mon.a (mon.0) 2067 : cluster [WRN] Health check update: Degraded data redundancy: 22389/215094 objects degraded (10.409%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:52.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:51 smithi177 bash[68594]: cluster 2024-09-13T14:59:51.453056+0000 mon.a (mon.0) 2067 : cluster [WRN] Health check update: Degraded data redundancy: 22389/215094 objects degraded (10.409%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:52.124 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:52.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:52.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:52.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:52.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:52.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:52.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:52.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:52.474 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 48 16 75002 74986 97.6289 58.375 0.00706369 0.0102388 2024-09-13T14:59:52.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:52.539 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:52.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:52.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:52 smithi099 bash[84576]: cluster 2024-09-13T14:59:50.765379+0000 mgr.y (mgr.26252) 1011 : cluster [DBG] pgmap v1544: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 22389/215094 objects degraded (10.409%) 2024-09-13T14:59:52.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:52 smithi099 bash[84576]: cluster 2024-09-13T14:59:50.765379+0000 mgr.y (mgr.26252) 1011 : cluster [DBG] pgmap v1544: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 22389/215094 objects degraded (10.409%) 2024-09-13T14:59:52.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:52 smithi099 bash[84576]: cluster 2024-09-13T14:59:51.453056+0000 mon.a (mon.0) 2067 : cluster [WRN] Health check update: Degraded data redundancy: 22389/215094 objects degraded (10.409%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:52.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:52 smithi099 bash[84576]: cluster 2024-09-13T14:59:51.453056+0000 mon.a (mon.0) 2067 : cluster [WRN] Health check update: Degraded data redundancy: 22389/215094 objects degraded (10.409%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:52.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:52 smithi099 bash[87147]: cluster 2024-09-13T14:59:50.765379+0000 mgr.y (mgr.26252) 1011 : cluster [DBG] pgmap v1544: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 22389/215094 objects degraded (10.409%) 2024-09-13T14:59:52.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:52 smithi099 bash[87147]: cluster 2024-09-13T14:59:50.765379+0000 mgr.y (mgr.26252) 1011 : cluster [DBG] pgmap v1544: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 22389/215094 objects degraded (10.409%) 2024-09-13T14:59:52.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:52 smithi099 bash[87147]: cluster 2024-09-13T14:59:51.453056+0000 mon.a (mon.0) 2067 : cluster [WRN] Health check update: Degraded data redundancy: 22389/215094 objects degraded (10.409%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:52.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:52 smithi099 bash[87147]: cluster 2024-09-13T14:59:51.453056+0000 mon.a (mon.0) 2067 : cluster [WRN] Health check update: Degraded data redundancy: 22389/215094 objects degraded (10.409%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:52.643 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:52.670 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:52.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:52.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:52.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:52.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:52.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:52.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:52.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:53.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:53.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:53.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:53.166 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:53.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:53.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:53.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:53.435 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:53.475 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 49 16 75800 75784 96.6542 49.875 0.00709142 0.0103321 2024-09-13T14:59:53.556 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:53.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:53.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:53.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:53.710 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:53.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:53.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:53.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:53.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:53.950 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:54.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:53 smithi099 bash[87147]: cluster 2024-09-13T14:59:52.768726+0000 mgr.y (mgr.26252) 1012 : cluster [DBG] pgmap v1545: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 22722/218133 objects degraded (10.417%) 2024-09-13T14:59:54.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:53 smithi099 bash[87147]: cluster 2024-09-13T14:59:52.768726+0000 mgr.y (mgr.26252) 1012 : cluster [DBG] pgmap v1545: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 22722/218133 objects degraded (10.417%) 2024-09-13T14:59:54.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:53 smithi099 bash[84576]: cluster 2024-09-13T14:59:52.768726+0000 mgr.y (mgr.26252) 1012 : cluster [DBG] pgmap v1545: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 22722/218133 objects degraded (10.417%) 2024-09-13T14:59:54.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:53 smithi099 bash[84576]: cluster 2024-09-13T14:59:52.768726+0000 mgr.y (mgr.26252) 1012 : cluster [DBG] pgmap v1545: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 22722/218133 objects degraded (10.417%) 2024-09-13T14:59:54.083 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:54.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:54.218 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:54.227 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:53 smithi177 bash[68594]: cluster 2024-09-13T14:59:52.768726+0000 mgr.y (mgr.26252) 1012 : cluster [DBG] pgmap v1545: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 22722/218133 objects degraded (10.417%) 2024-09-13T14:59:54.227 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:53 smithi177 bash[68594]: cluster 2024-09-13T14:59:52.768726+0000 mgr.y (mgr.26252) 1012 : cluster [DBG] pgmap v1545: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 22722/218133 objects degraded (10.417%) 2024-09-13T14:59:54.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:54.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:54.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:54.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:54.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:54.475 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 50 16 76826 76810 96.0035 64.125 0.0227452 0.0104126 2024-09-13T14:59:54.484 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:54.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:54.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:54.623 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:54.645 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:54.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:54.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:54.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:54.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:54.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:54.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:54.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:55.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:55.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:55.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:55.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:55.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:55.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:55.330 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:55.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:55.433 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:55.469 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:55.475 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 51 16 77483 77467 94.9262 41.0625 0.0242385 0.0105295 2024-09-13T14:59:55.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:55.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:55.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:55.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:55.711 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:55.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.746 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:55.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:55.847 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:55.897 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:55.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:56.003 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:56.026 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:56.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:56.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:55 smithi177 bash[68594]: cluster 2024-09-13T14:59:54.770225+0000 mgr.y (mgr.26252) 1013 : cluster [DBG] pgmap v1546: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:56.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:55 smithi177 bash[68594]: cluster 2024-09-13T14:59:54.770225+0000 mgr.y (mgr.26252) 1013 : cluster [DBG] pgmap v1546: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:56.128 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:56.137 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:55 smithi099 bash[84576]: cluster 2024-09-13T14:59:54.770225+0000 mgr.y (mgr.26252) 1013 : cluster [DBG] pgmap v1546: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:56.137 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:55 smithi099 bash[84576]: cluster 2024-09-13T14:59:54.770225+0000 mgr.y (mgr.26252) 1013 : cluster [DBG] pgmap v1546: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:56.137 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:55 smithi099 bash[87147]: cluster 2024-09-13T14:59:54.770225+0000 mgr.y (mgr.26252) 1013 : cluster [DBG] pgmap v1546: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:56.137 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:55 smithi099 bash[87147]: cluster 2024-09-13T14:59:54.770225+0000 mgr.y (mgr.26252) 1013 : cluster [DBG] pgmap v1546: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:56.156 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:56.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:56.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:56.282 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:56.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:56.372 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T14:59:56.372 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T14:59:56.372 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2024-09-13T14:59:56.372 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T14:59:56.385 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:56.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:56.475 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 52 16 78691 78675 94.5524 75.5 0.00701594 0.0105716 2024-09-13T14:59:56.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:56.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:56.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:56.641 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:56.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:56.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:56.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:56.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:56.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:56.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:56.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:56.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:57.053 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:57.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:57.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:57.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:56 smithi177 bash[68594]: cluster 2024-09-13T14:59:56.453787+0000 mon.a (mon.0) 2068 : cluster [WRN] Health check update: Degraded data redundancy: 23863/229389 objects degraded (10.403%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:57.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:56 smithi177 bash[68594]: cluster 2024-09-13T14:59:56.453787+0000 mon.a (mon.0) 2068 : cluster [WRN] Health check update: Degraded data redundancy: 23863/229389 objects degraded (10.403%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:57.177 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:57.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:57.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:57.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:56 smithi099 bash[84576]: cluster 2024-09-13T14:59:56.453787+0000 mon.a (mon.0) 2068 : cluster [WRN] Health check update: Degraded data redundancy: 23863/229389 objects degraded (10.403%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:57.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:56 smithi099 bash[84576]: cluster 2024-09-13T14:59:56.453787+0000 mon.a (mon.0) 2068 : cluster [WRN] Health check update: Degraded data redundancy: 23863/229389 objects degraded (10.403%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:57.248 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 14:59:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:14:59:57] "GET /metrics HTTP/1.1" 200 36763 "" "Prometheus/2.51.0" 2024-09-13T14:59:57.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:56 smithi099 bash[87147]: cluster 2024-09-13T14:59:56.453787+0000 mon.a (mon.0) 2068 : cluster [WRN] Health check update: Degraded data redundancy: 23863/229389 objects degraded (10.403%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:57.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:56 smithi099 bash[87147]: cluster 2024-09-13T14:59:56.453787+0000 mon.a (mon.0) 2068 : cluster [WRN] Health check update: Degraded data redundancy: 23863/229389 objects degraded (10.403%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:57.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:57.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:57.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:57.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:57.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:57.475 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 53 16 79500 79484 93.7224 50.5625 0.0113199 0.0106643 2024-09-13T14:59:57.573 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:57.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.615 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:57.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:57.717 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:57.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:59:57.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:57.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:57.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:57.905 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:57.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:58.010 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:58.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:58.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:58.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:57 smithi177 bash[68594]: cluster 2024-09-13T14:59:56.770782+0000 mgr.y (mgr.26252) 1014 : cluster [DBG] pgmap v1547: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:58.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:57 smithi177 bash[68594]: cluster 2024-09-13T14:59:56.770782+0000 mgr.y (mgr.26252) 1014 : cluster [DBG] pgmap v1547: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:58.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:58.140 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:57 smithi099 bash[84576]: cluster 2024-09-13T14:59:56.770782+0000 mgr.y (mgr.26252) 1014 : cluster [DBG] pgmap v1547: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:58.140 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 14:59:57 smithi099 bash[84576]: cluster 2024-09-13T14:59:56.770782+0000 mgr.y (mgr.26252) 1014 : cluster [DBG] pgmap v1547: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:58.141 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:57 smithi099 bash[87147]: cluster 2024-09-13T14:59:56.770782+0000 mgr.y (mgr.26252) 1014 : cluster [DBG] pgmap v1547: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:58.144 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 14:59:57 smithi099 bash[87147]: cluster 2024-09-13T14:59:56.770782+0000 mgr.y (mgr.26252) 1014 : cluster [DBG] pgmap v1547: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 23863/229389 objects degraded (10.403%) 2024-09-13T14:59:58.158 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:58.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:58.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:58.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:58.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:58.445 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:58.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:58.475 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 54 16 80534 80518 93.1834 64.625 0.00470392 0.010726 2024-09-13T14:59:58.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:58.572 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:58.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:58.675 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:58.699 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:58.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:58.802 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:58.838 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:58.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:58.941 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:58.966 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:59.071 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:59.103 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:59.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:59.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:59.238 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:59.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:59.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:59.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:59.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:59.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:59.475 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 55 16 81324 81308 92.3868 49.375 0.00700294 0.0108192 2024-09-13T14:59:59.497 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:59.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:59.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:59.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:59.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:59.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:59.763 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:59.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:59.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:59.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:59:59.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:59:59.999 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:00.017 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:00.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:00.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:59 smithi177 bash[68594]: cluster 2024-09-13T14:59:58.772086+0000 mgr.y (mgr.26252) 1015 : cluster [DBG] pgmap v1548: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s; 24146/231843 objects degraded (10.415%) 2024-09-13T15:00:00.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 14:59:59 smithi177 bash[68594]: cluster 2024-09-13T14:59:58.772086+0000 mgr.y (mgr.26252) 1015 : cluster [DBG] pgmap v1548: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s; 24146/231843 objects degraded (10.415%) 2024-09-13T15:00:00.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:00.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T14:59:58.772086+0000 mgr.y (mgr.26252) 1015 : cluster [DBG] pgmap v1548: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s; 24146/231843 objects degraded (10.415%) 2024-09-13T15:00:00.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T14:59:58.772086+0000 mgr.y (mgr.26252) 1015 : cluster [DBG] pgmap v1548: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s; 24146/231843 objects degraded (10.415%) 2024-09-13T15:00:00.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T14:59:58.772086+0000 mgr.y (mgr.26252) 1015 : cluster [DBG] pgmap v1548: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s; 24146/231843 objects degraded (10.415%) 2024-09-13T15:00:00.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T14:59:58.772086+0000 mgr.y (mgr.26252) 1015 : cluster [DBG] pgmap v1548: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s; 24146/231843 objects degraded (10.415%) 2024-09-13T15:00:00.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:00.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:00.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:00.475 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 56 16 82454 82438 91.9981 70.625 0.00499921 0.0108653 2024-09-13T15:00:00.512 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:00.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[61439]: ts=2024-09-13T15:00:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:00.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:00.661 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:00.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:00.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.793 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:00.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:00.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:01.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:01.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:01.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000131+0000 mon.a (mon.0) 2069 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000131+0000 mon.a (mon.0) 2069 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000151+0000 mon.a (mon.0) 2070 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000151+0000 mon.a (mon.0) 2070 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000158+0000 mon.a (mon.0) 2071 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000158+0000 mon.a (mon.0) 2071 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000164+0000 mon.a (mon.0) 2072 : cluster [WRN] osd.3 (root=default,host=smithi099) is down 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000164+0000 mon.a (mon.0) 2072 : cluster [WRN] osd.3 (root=default,host=smithi099) is down 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000170+0000 mon.a (mon.0) 2073 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000170+0000 mon.a (mon.0) 2073 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000177+0000 mon.a (mon.0) 2074 : cluster [WRN] pg 2.1 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000177+0000 mon.a (mon.0) 2074 : cluster [WRN] pg 2.1 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000184+0000 mon.a (mon.0) 2075 : cluster [WRN] pg 3.1 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000184+0000 mon.a (mon.0) 2075 : cluster [WRN] pg 3.1 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2024-09-13T15:00:01.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000191+0000 mon.a (mon.0) 2076 : cluster [WRN] pg 3.2 is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000191+0000 mon.a (mon.0) 2076 : cluster [WRN] pg 3.2 is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000200+0000 mon.a (mon.0) 2077 : cluster [WRN] pg 3.5 is stuck undersized for 2m, current state active+undersized, last acting [5,2] 2024-09-13T15:00:01.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000200+0000 mon.a (mon.0) 2077 : cluster [WRN] pg 3.5 is stuck undersized for 2m, current state active+undersized, last acting [5,2] 2024-09-13T15:00:01.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000216+0000 mon.a (mon.0) 2078 : cluster [WRN] pg 3.7 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,0] 2024-09-13T15:00:01.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000216+0000 mon.a (mon.0) 2078 : cluster [WRN] pg 3.7 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,0] 2024-09-13T15:00:01.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000131+0000 mon.a (mon.0) 2069 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000131+0000 mon.a (mon.0) 2069 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000151+0000 mon.a (mon.0) 2070 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000151+0000 mon.a (mon.0) 2070 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000158+0000 mon.a (mon.0) 2071 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000158+0000 mon.a (mon.0) 2071 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000164+0000 mon.a (mon.0) 2072 : cluster [WRN] osd.3 (root=default,host=smithi099) is down 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000164+0000 mon.a (mon.0) 2072 : cluster [WRN] osd.3 (root=default,host=smithi099) is down 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000170+0000 mon.a (mon.0) 2073 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000170+0000 mon.a (mon.0) 2073 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000177+0000 mon.a (mon.0) 2074 : cluster [WRN] pg 2.1 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000177+0000 mon.a (mon.0) 2074 : cluster [WRN] pg 2.1 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000184+0000 mon.a (mon.0) 2075 : cluster [WRN] pg 3.1 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000184+0000 mon.a (mon.0) 2075 : cluster [WRN] pg 3.1 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2024-09-13T15:00:01.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000191+0000 mon.a (mon.0) 2076 : cluster [WRN] pg 3.2 is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000191+0000 mon.a (mon.0) 2076 : cluster [WRN] pg 3.2 is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000200+0000 mon.a (mon.0) 2077 : cluster [WRN] pg 3.5 is stuck undersized for 2m, current state active+undersized, last acting [5,2] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000200+0000 mon.a (mon.0) 2077 : cluster [WRN] pg 3.5 is stuck undersized for 2m, current state active+undersized, last acting [5,2] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000216+0000 mon.a (mon.0) 2078 : cluster [WRN] pg 3.7 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,0] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000216+0000 mon.a (mon.0) 2078 : cluster [WRN] pg 3.7 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,0] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000223+0000 mon.a (mon.0) 2079 : cluster [WRN] pg 3.8 is stuck undersized for 2m, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000223+0000 mon.a (mon.0) 2079 : cluster [WRN] pg 3.8 is stuck undersized for 2m, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000230+0000 mon.a (mon.0) 2080 : cluster [WRN] pg 3.b is stuck undersized for 2m, current state active+undersized+degraded, last acting [0,4] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000230+0000 mon.a (mon.0) 2080 : cluster [WRN] pg 3.b is stuck undersized for 2m, current state active+undersized+degraded, last acting [0,4] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000236+0000 mon.a (mon.0) 2081 : cluster [WRN] pg 3.c is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000236+0000 mon.a (mon.0) 2081 : cluster [WRN] pg 3.c is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000245+0000 mon.a (mon.0) 2082 : cluster [WRN] pg 3.12 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000245+0000 mon.a (mon.0) 2082 : cluster [WRN] pg 3.12 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000252+0000 mon.a (mon.0) 2083 : cluster [WRN] pg 3.15 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,4] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000252+0000 mon.a (mon.0) 2083 : cluster [WRN] pg 3.15 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,4] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000259+0000 mon.a (mon.0) 2084 : cluster [WRN] pg 3.17 is stuck undersized for 2m, current state active+undersized, last acting [0,5] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000259+0000 mon.a (mon.0) 2084 : cluster [WRN] pg 3.17 is stuck undersized for 2m, current state active+undersized, last acting [0,5] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000266+0000 mon.a (mon.0) 2085 : cluster [WRN] pg 3.18 is stuck undersized for 2m, current state active+undersized, last acting [0,1] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000266+0000 mon.a (mon.0) 2085 : cluster [WRN] pg 3.18 is stuck undersized for 2m, current state active+undersized, last acting [0,1] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000273+0000 mon.a (mon.0) 2086 : cluster [WRN] pg 3.19 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000273+0000 mon.a (mon.0) 2086 : cluster [WRN] pg 3.19 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000280+0000 mon.a (mon.0) 2087 : cluster [WRN] pg 3.1e is stuck undersized for 2m, current state active+undersized, last acting [6,2] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000280+0000 mon.a (mon.0) 2087 : cluster [WRN] pg 3.1e is stuck undersized for 2m, current state active+undersized, last acting [6,2] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000287+0000 mon.a (mon.0) 2088 : cluster [WRN] pg 4.0 is stuck undersized for 2m, current state active+undersized, last acting [7,0] 2024-09-13T15:00:01.078 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000287+0000 mon.a (mon.0) 2088 : cluster [WRN] pg 4.0 is stuck undersized for 2m, current state active+undersized, last acting [7,0] 2024-09-13T15:00:01.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000223+0000 mon.a (mon.0) 2079 : cluster [WRN] pg 3.8 is stuck undersized for 2m, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000223+0000 mon.a (mon.0) 2079 : cluster [WRN] pg 3.8 is stuck undersized for 2m, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000230+0000 mon.a (mon.0) 2080 : cluster [WRN] pg 3.b is stuck undersized for 2m, current state active+undersized+degraded, last acting [0,4] 2024-09-13T15:00:01.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000230+0000 mon.a (mon.0) 2080 : cluster [WRN] pg 3.b is stuck undersized for 2m, current state active+undersized+degraded, last acting [0,4] 2024-09-13T15:00:01.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000236+0000 mon.a (mon.0) 2081 : cluster [WRN] pg 3.c is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000236+0000 mon.a (mon.0) 2081 : cluster [WRN] pg 3.c is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000245+0000 mon.a (mon.0) 2082 : cluster [WRN] pg 3.12 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000245+0000 mon.a (mon.0) 2082 : cluster [WRN] pg 3.12 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000252+0000 mon.a (mon.0) 2083 : cluster [WRN] pg 3.15 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,4] 2024-09-13T15:00:01.079 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000252+0000 mon.a (mon.0) 2083 : cluster [WRN] pg 3.15 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,4] 2024-09-13T15:00:01.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000259+0000 mon.a (mon.0) 2084 : cluster [WRN] pg 3.17 is stuck undersized for 2m, current state active+undersized, last acting [0,5] 2024-09-13T15:00:01.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000259+0000 mon.a (mon.0) 2084 : cluster [WRN] pg 3.17 is stuck undersized for 2m, current state active+undersized, last acting [0,5] 2024-09-13T15:00:01.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000266+0000 mon.a (mon.0) 2085 : cluster [WRN] pg 3.18 is stuck undersized for 2m, current state active+undersized, last acting [0,1] 2024-09-13T15:00:01.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000266+0000 mon.a (mon.0) 2085 : cluster [WRN] pg 3.18 is stuck undersized for 2m, current state active+undersized, last acting [0,1] 2024-09-13T15:00:01.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000273+0000 mon.a (mon.0) 2086 : cluster [WRN] pg 3.19 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000273+0000 mon.a (mon.0) 2086 : cluster [WRN] pg 3.19 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000280+0000 mon.a (mon.0) 2087 : cluster [WRN] pg 3.1e is stuck undersized for 2m, current state active+undersized, last acting [6,2] 2024-09-13T15:00:01.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000280+0000 mon.a (mon.0) 2087 : cluster [WRN] pg 3.1e is stuck undersized for 2m, current state active+undersized, last acting [6,2] 2024-09-13T15:00:01.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000287+0000 mon.a (mon.0) 2088 : cluster [WRN] pg 4.0 is stuck undersized for 2m, current state active+undersized, last acting [7,0] 2024-09-13T15:00:01.080 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000287+0000 mon.a (mon.0) 2088 : cluster [WRN] pg 4.0 is stuck undersized for 2m, current state active+undersized, last acting [7,0] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000295+0000 mon.a (mon.0) 2089 : cluster [WRN] pg 4.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000295+0000 mon.a (mon.0) 2089 : cluster [WRN] pg 4.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000301+0000 mon.a (mon.0) 2090 : cluster [WRN] pg 4.9 is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000301+0000 mon.a (mon.0) 2090 : cluster [WRN] pg 4.9 is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000310+0000 mon.a (mon.0) 2091 : cluster [WRN] pg 4.c is stuck undersized for 2m, current state active+undersized, last acting [4,6] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000310+0000 mon.a (mon.0) 2091 : cluster [WRN] pg 4.c is stuck undersized for 2m, current state active+undersized, last acting [4,6] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000318+0000 mon.a (mon.0) 2092 : cluster [WRN] pg 4.f is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000318+0000 mon.a (mon.0) 2092 : cluster [WRN] pg 4.f is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000326+0000 mon.a (mon.0) 2093 : cluster [WRN] pg 4.10 is stuck undersized for 2m, current state active+undersized, last acting [1,6] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000326+0000 mon.a (mon.0) 2093 : cluster [WRN] pg 4.10 is stuck undersized for 2m, current state active+undersized, last acting [1,6] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000333+0000 mon.a (mon.0) 2094 : cluster [WRN] pg 4.11 is stuck undersized for 2m, current state active+undersized, last acting [7,6] 2024-09-13T15:00:01.081 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000333+0000 mon.a (mon.0) 2094 : cluster [WRN] pg 4.11 is stuck undersized for 2m, current state active+undersized, last acting [7,6] 2024-09-13T15:00:01.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000341+0000 mon.a (mon.0) 2095 : cluster [WRN] pg 4.14 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000341+0000 mon.a (mon.0) 2095 : cluster [WRN] pg 4.14 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000348+0000 mon.a (mon.0) 2096 : cluster [WRN] pg 4.15 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000348+0000 mon.a (mon.0) 2096 : cluster [WRN] pg 4.15 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000355+0000 mon.a (mon.0) 2097 : cluster [WRN] pg 4.16 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000355+0000 mon.a (mon.0) 2097 : cluster [WRN] pg 4.16 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000362+0000 mon.a (mon.0) 2098 : cluster [WRN] pg 4.17 is stuck undersized for 2m, current state active+undersized, last acting [7,1] 2024-09-13T15:00:01.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000362+0000 mon.a (mon.0) 2098 : cluster [WRN] pg 4.17 is stuck undersized for 2m, current state active+undersized, last acting [7,1] 2024-09-13T15:00:01.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000295+0000 mon.a (mon.0) 2089 : cluster [WRN] pg 4.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-09-13T15:00:01.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000295+0000 mon.a (mon.0) 2089 : cluster [WRN] pg 4.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-09-13T15:00:01.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000301+0000 mon.a (mon.0) 2090 : cluster [WRN] pg 4.9 is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000301+0000 mon.a (mon.0) 2090 : cluster [WRN] pg 4.9 is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000310+0000 mon.a (mon.0) 2091 : cluster [WRN] pg 4.c is stuck undersized for 2m, current state active+undersized, last acting [4,6] 2024-09-13T15:00:01.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000310+0000 mon.a (mon.0) 2091 : cluster [WRN] pg 4.c is stuck undersized for 2m, current state active+undersized, last acting [4,6] 2024-09-13T15:00:01.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000318+0000 mon.a (mon.0) 2092 : cluster [WRN] pg 4.f is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000318+0000 mon.a (mon.0) 2092 : cluster [WRN] pg 4.f is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000326+0000 mon.a (mon.0) 2093 : cluster [WRN] pg 4.10 is stuck undersized for 2m, current state active+undersized, last acting [1,6] 2024-09-13T15:00:01.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000326+0000 mon.a (mon.0) 2093 : cluster [WRN] pg 4.10 is stuck undersized for 2m, current state active+undersized, last acting [1,6] 2024-09-13T15:00:01.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000333+0000 mon.a (mon.0) 2094 : cluster [WRN] pg 4.11 is stuck undersized for 2m, current state active+undersized, last acting [7,6] 2024-09-13T15:00:01.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000333+0000 mon.a (mon.0) 2094 : cluster [WRN] pg 4.11 is stuck undersized for 2m, current state active+undersized, last acting [7,6] 2024-09-13T15:00:01.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000341+0000 mon.a (mon.0) 2095 : cluster [WRN] pg 4.14 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000341+0000 mon.a (mon.0) 2095 : cluster [WRN] pg 4.14 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000348+0000 mon.a (mon.0) 2096 : cluster [WRN] pg 4.15 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000348+0000 mon.a (mon.0) 2096 : cluster [WRN] pg 4.15 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000355+0000 mon.a (mon.0) 2097 : cluster [WRN] pg 4.16 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000355+0000 mon.a (mon.0) 2097 : cluster [WRN] pg 4.16 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000362+0000 mon.a (mon.0) 2098 : cluster [WRN] pg 4.17 is stuck undersized for 2m, current state active+undersized, last acting [7,1] 2024-09-13T15:00:01.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000362+0000 mon.a (mon.0) 2098 : cluster [WRN] pg 4.17 is stuck undersized for 2m, current state active+undersized, last acting [7,1] 2024-09-13T15:00:01.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000371+0000 mon.a (mon.0) 2099 : cluster [WRN] pg 4.1a is stuck undersized for 2m, current state active+undersized, last acting [4,0] 2024-09-13T15:00:01.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000371+0000 mon.a (mon.0) 2099 : cluster [WRN] pg 4.1a is stuck undersized for 2m, current state active+undersized, last acting [4,0] 2024-09-13T15:00:01.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000381+0000 mon.a (mon.0) 2100 : cluster [WRN] pg 4.1b is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000381+0000 mon.a (mon.0) 2100 : cluster [WRN] pg 4.1b is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000388+0000 mon.a (mon.0) 2101 : cluster [WRN] pg 4.1c is stuck undersized for 2m, current state active+undersized, last acting [2,1] 2024-09-13T15:00:01.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000388+0000 mon.a (mon.0) 2101 : cluster [WRN] pg 4.1c is stuck undersized for 2m, current state active+undersized, last acting [2,1] 2024-09-13T15:00:01.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000395+0000 mon.a (mon.0) 2102 : cluster [WRN] pg 4.1d is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000395+0000 mon.a (mon.0) 2102 : cluster [WRN] pg 4.1d is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000402+0000 mon.a (mon.0) 2103 : cluster [WRN] pg 43.3 is stuck undersized for 81s, current state active+undersized+degraded, last acting [2,4] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000402+0000 mon.a (mon.0) 2103 : cluster [WRN] pg 43.3 is stuck undersized for 81s, current state active+undersized+degraded, last acting [2,4] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000410+0000 mon.a (mon.0) 2104 : cluster [WRN] pg 43.4 is stuck undersized for 81s, current state active+undersized+degraded, last acting [1,6] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000410+0000 mon.a (mon.0) 2104 : cluster [WRN] pg 43.4 is stuck undersized for 81s, current state active+undersized+degraded, last acting [1,6] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000417+0000 mon.a (mon.0) 2105 : cluster [WRN] pg 43.9 is stuck undersized for 81s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000417+0000 mon.a (mon.0) 2105 : cluster [WRN] pg 43.9 is stuck undersized for 81s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000424+0000 mon.a (mon.0) 2106 : cluster [WRN] pg 43.b is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,2] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000424+0000 mon.a (mon.0) 2106 : cluster [WRN] pg 43.b is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,2] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000431+0000 mon.a (mon.0) 2107 : cluster [WRN] pg 43.e is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,0] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.000431+0000 mon.a (mon.0) 2107 : cluster [WRN] pg 43.e is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,0] 2024-09-13T15:00:01.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000371+0000 mon.a (mon.0) 2099 : cluster [WRN] pg 4.1a is stuck undersized for 2m, current state active+undersized, last acting [4,0] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000371+0000 mon.a (mon.0) 2099 : cluster [WRN] pg 4.1a is stuck undersized for 2m, current state active+undersized, last acting [4,0] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000381+0000 mon.a (mon.0) 2100 : cluster [WRN] pg 4.1b is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000381+0000 mon.a (mon.0) 2100 : cluster [WRN] pg 4.1b is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000388+0000 mon.a (mon.0) 2101 : cluster [WRN] pg 4.1c is stuck undersized for 2m, current state active+undersized, last acting [2,1] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000388+0000 mon.a (mon.0) 2101 : cluster [WRN] pg 4.1c is stuck undersized for 2m, current state active+undersized, last acting [2,1] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000395+0000 mon.a (mon.0) 2102 : cluster [WRN] pg 4.1d is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000395+0000 mon.a (mon.0) 2102 : cluster [WRN] pg 4.1d is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000402+0000 mon.a (mon.0) 2103 : cluster [WRN] pg 43.3 is stuck undersized for 81s, current state active+undersized+degraded, last acting [2,4] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000402+0000 mon.a (mon.0) 2103 : cluster [WRN] pg 43.3 is stuck undersized for 81s, current state active+undersized+degraded, last acting [2,4] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000410+0000 mon.a (mon.0) 2104 : cluster [WRN] pg 43.4 is stuck undersized for 81s, current state active+undersized+degraded, last acting [1,6] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000410+0000 mon.a (mon.0) 2104 : cluster [WRN] pg 43.4 is stuck undersized for 81s, current state active+undersized+degraded, last acting [1,6] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000417+0000 mon.a (mon.0) 2105 : cluster [WRN] pg 43.9 is stuck undersized for 81s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000417+0000 mon.a (mon.0) 2105 : cluster [WRN] pg 43.9 is stuck undersized for 81s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000424+0000 mon.a (mon.0) 2106 : cluster [WRN] pg 43.b is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,2] 2024-09-13T15:00:01.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000424+0000 mon.a (mon.0) 2106 : cluster [WRN] pg 43.b is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,2] 2024-09-13T15:00:01.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000431+0000 mon.a (mon.0) 2107 : cluster [WRN] pg 43.e is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,0] 2024-09-13T15:00:01.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:00 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.000431+0000 mon.a (mon.0) 2107 : cluster [WRN] pg 43.e is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,0] 2024-09-13T15:00:01.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:01.192 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:01.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:01.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:01.319 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:01.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:01.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000131+0000 mon.a (mon.0) 2069 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000131+0000 mon.a (mon.0) 2069 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; 1 osds down; Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000151+0000 mon.a (mon.0) 2070 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:00:01.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000151+0000 mon.a (mon.0) 2070 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:00:01.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000158+0000 mon.a (mon.0) 2071 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T15:00:01.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000158+0000 mon.a (mon.0) 2071 : cluster [WRN] [WRN] OSD_DOWN: 1 osds down 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000164+0000 mon.a (mon.0) 2072 : cluster [WRN] osd.3 (root=default,host=smithi099) is down 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000164+0000 mon.a (mon.0) 2072 : cluster [WRN] osd.3 (root=default,host=smithi099) is down 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000170+0000 mon.a (mon.0) 2073 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000170+0000 mon.a (mon.0) 2073 : cluster [WRN] [WRN] PG_DEGRADED: Degraded data redundancy: 24146/231843 objects degraded (10.415%), 8 pgs degraded, 34 pgs undersized 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000177+0000 mon.a (mon.0) 2074 : cluster [WRN] pg 2.1 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000177+0000 mon.a (mon.0) 2074 : cluster [WRN] pg 2.1 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000184+0000 mon.a (mon.0) 2075 : cluster [WRN] pg 3.1 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000184+0000 mon.a (mon.0) 2075 : cluster [WRN] pg 3.1 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000191+0000 mon.a (mon.0) 2076 : cluster [WRN] pg 3.2 is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000191+0000 mon.a (mon.0) 2076 : cluster [WRN] pg 3.2 is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000200+0000 mon.a (mon.0) 2077 : cluster [WRN] pg 3.5 is stuck undersized for 2m, current state active+undersized, last acting [5,2] 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000200+0000 mon.a (mon.0) 2077 : cluster [WRN] pg 3.5 is stuck undersized for 2m, current state active+undersized, last acting [5,2] 2024-09-13T15:00:01.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000216+0000 mon.a (mon.0) 2078 : cluster [WRN] pg 3.7 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,0] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000216+0000 mon.a (mon.0) 2078 : cluster [WRN] pg 3.7 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,0] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000223+0000 mon.a (mon.0) 2079 : cluster [WRN] pg 3.8 is stuck undersized for 2m, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000223+0000 mon.a (mon.0) 2079 : cluster [WRN] pg 3.8 is stuck undersized for 2m, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000230+0000 mon.a (mon.0) 2080 : cluster [WRN] pg 3.b is stuck undersized for 2m, current state active+undersized+degraded, last acting [0,4] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000230+0000 mon.a (mon.0) 2080 : cluster [WRN] pg 3.b is stuck undersized for 2m, current state active+undersized+degraded, last acting [0,4] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000236+0000 mon.a (mon.0) 2081 : cluster [WRN] pg 3.c is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000236+0000 mon.a (mon.0) 2081 : cluster [WRN] pg 3.c is stuck undersized for 2m, current state active+undersized, last acting [5,6] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000245+0000 mon.a (mon.0) 2082 : cluster [WRN] pg 3.12 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000245+0000 mon.a (mon.0) 2082 : cluster [WRN] pg 3.12 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000252+0000 mon.a (mon.0) 2083 : cluster [WRN] pg 3.15 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,4] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000252+0000 mon.a (mon.0) 2083 : cluster [WRN] pg 3.15 is stuck undersized for 2m, current state active+undersized+degraded, last acting [7,4] 2024-09-13T15:00:01.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000259+0000 mon.a (mon.0) 2084 : cluster [WRN] pg 3.17 is stuck undersized for 2m, current state active+undersized, last acting [0,5] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000259+0000 mon.a (mon.0) 2084 : cluster [WRN] pg 3.17 is stuck undersized for 2m, current state active+undersized, last acting [0,5] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000266+0000 mon.a (mon.0) 2085 : cluster [WRN] pg 3.18 is stuck undersized for 2m, current state active+undersized, last acting [0,1] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000266+0000 mon.a (mon.0) 2085 : cluster [WRN] pg 3.18 is stuck undersized for 2m, current state active+undersized, last acting [0,1] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000273+0000 mon.a (mon.0) 2086 : cluster [WRN] pg 3.19 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000273+0000 mon.a (mon.0) 2086 : cluster [WRN] pg 3.19 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000280+0000 mon.a (mon.0) 2087 : cluster [WRN] pg 3.1e is stuck undersized for 2m, current state active+undersized, last acting [6,2] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000280+0000 mon.a (mon.0) 2087 : cluster [WRN] pg 3.1e is stuck undersized for 2m, current state active+undersized, last acting [6,2] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000287+0000 mon.a (mon.0) 2088 : cluster [WRN] pg 4.0 is stuck undersized for 2m, current state active+undersized, last acting [7,0] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000287+0000 mon.a (mon.0) 2088 : cluster [WRN] pg 4.0 is stuck undersized for 2m, current state active+undersized, last acting [7,0] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000295+0000 mon.a (mon.0) 2089 : cluster [WRN] pg 4.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000295+0000 mon.a (mon.0) 2089 : cluster [WRN] pg 4.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000301+0000 mon.a (mon.0) 2090 : cluster [WRN] pg 4.9 is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000301+0000 mon.a (mon.0) 2090 : cluster [WRN] pg 4.9 is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000310+0000 mon.a (mon.0) 2091 : cluster [WRN] pg 4.c is stuck undersized for 2m, current state active+undersized, last acting [4,6] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000310+0000 mon.a (mon.0) 2091 : cluster [WRN] pg 4.c is stuck undersized for 2m, current state active+undersized, last acting [4,6] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000318+0000 mon.a (mon.0) 2092 : cluster [WRN] pg 4.f is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000318+0000 mon.a (mon.0) 2092 : cluster [WRN] pg 4.f is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000326+0000 mon.a (mon.0) 2093 : cluster [WRN] pg 4.10 is stuck undersized for 2m, current state active+undersized, last acting [1,6] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000326+0000 mon.a (mon.0) 2093 : cluster [WRN] pg 4.10 is stuck undersized for 2m, current state active+undersized, last acting [1,6] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000333+0000 mon.a (mon.0) 2094 : cluster [WRN] pg 4.11 is stuck undersized for 2m, current state active+undersized, last acting [7,6] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000333+0000 mon.a (mon.0) 2094 : cluster [WRN] pg 4.11 is stuck undersized for 2m, current state active+undersized, last acting [7,6] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000341+0000 mon.a (mon.0) 2095 : cluster [WRN] pg 4.14 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000341+0000 mon.a (mon.0) 2095 : cluster [WRN] pg 4.14 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000348+0000 mon.a (mon.0) 2096 : cluster [WRN] pg 4.15 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000348+0000 mon.a (mon.0) 2096 : cluster [WRN] pg 4.15 is stuck undersized for 2m, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000355+0000 mon.a (mon.0) 2097 : cluster [WRN] pg 4.16 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000355+0000 mon.a (mon.0) 2097 : cluster [WRN] pg 4.16 is stuck undersized for 2m, current state active+undersized, last acting [0,7] 2024-09-13T15:00:01.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000362+0000 mon.a (mon.0) 2098 : cluster [WRN] pg 4.17 is stuck undersized for 2m, current state active+undersized, last acting [7,1] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000362+0000 mon.a (mon.0) 2098 : cluster [WRN] pg 4.17 is stuck undersized for 2m, current state active+undersized, last acting [7,1] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000371+0000 mon.a (mon.0) 2099 : cluster [WRN] pg 4.1a is stuck undersized for 2m, current state active+undersized, last acting [4,0] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000371+0000 mon.a (mon.0) 2099 : cluster [WRN] pg 4.1a is stuck undersized for 2m, current state active+undersized, last acting [4,0] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000381+0000 mon.a (mon.0) 2100 : cluster [WRN] pg 4.1b is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000381+0000 mon.a (mon.0) 2100 : cluster [WRN] pg 4.1b is stuck undersized for 2m, current state active+undersized, last acting [4,1] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000388+0000 mon.a (mon.0) 2101 : cluster [WRN] pg 4.1c is stuck undersized for 2m, current state active+undersized, last acting [2,1] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000388+0000 mon.a (mon.0) 2101 : cluster [WRN] pg 4.1c is stuck undersized for 2m, current state active+undersized, last acting [2,1] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000395+0000 mon.a (mon.0) 2102 : cluster [WRN] pg 4.1d is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000395+0000 mon.a (mon.0) 2102 : cluster [WRN] pg 4.1d is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000402+0000 mon.a (mon.0) 2103 : cluster [WRN] pg 43.3 is stuck undersized for 81s, current state active+undersized+degraded, last acting [2,4] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000402+0000 mon.a (mon.0) 2103 : cluster [WRN] pg 43.3 is stuck undersized for 81s, current state active+undersized+degraded, last acting [2,4] 2024-09-13T15:00:01.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000410+0000 mon.a (mon.0) 2104 : cluster [WRN] pg 43.4 is stuck undersized for 81s, current state active+undersized+degraded, last acting [1,6] 2024-09-13T15:00:01.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000410+0000 mon.a (mon.0) 2104 : cluster [WRN] pg 43.4 is stuck undersized for 81s, current state active+undersized+degraded, last acting [1,6] 2024-09-13T15:00:01.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000417+0000 mon.a (mon.0) 2105 : cluster [WRN] pg 43.9 is stuck undersized for 81s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000417+0000 mon.a (mon.0) 2105 : cluster [WRN] pg 43.9 is stuck undersized for 81s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000424+0000 mon.a (mon.0) 2106 : cluster [WRN] pg 43.b is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,2] 2024-09-13T15:00:01.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000424+0000 mon.a (mon.0) 2106 : cluster [WRN] pg 43.b is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,2] 2024-09-13T15:00:01.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000431+0000 mon.a (mon.0) 2107 : cluster [WRN] pg 43.e is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,0] 2024-09-13T15:00:01.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:00 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.000431+0000 mon.a (mon.0) 2107 : cluster [WRN] pg 43.e is stuck undersized for 81s, current state active+undersized+degraded, last acting [6,0] 2024-09-13T15:00:01.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:01.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:01.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:01.475 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 57 16 83598 83582 91.6384 71.5 0.0200065 0.0109073 2024-09-13T15:00:01.570 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:01.597 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:01.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:01.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:01.728 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:01.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:01.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:01.853 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:01.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:01.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:01.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:01 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.773070+0000 mgr.y (mgr.26252) 1016 : cluster [DBG] pgmap v1549: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 25276/243021 objects degraded (10.401%) 2024-09-13T15:00:01.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:01 smithi177 bash[68594]: cluster 2024-09-13T15:00:00.773070+0000 mgr.y (mgr.26252) 1016 : cluster [DBG] pgmap v1549: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 25276/243021 objects degraded (10.401%) 2024-09-13T15:00:01.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:01 smithi177 bash[68594]: cluster 2024-09-13T15:00:01.454492+0000 mon.a (mon.0) 2108 : cluster [WRN] Health check update: Degraded data redundancy: 25276/243021 objects degraded (10.401%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:01.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:01 smithi177 bash[68594]: cluster 2024-09-13T15:00:01.454492+0000 mon.a (mon.0) 2108 : cluster [WRN] Health check update: Degraded data redundancy: 25276/243021 objects degraded (10.401%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:01.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:02.083 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:02.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:02.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:02.205 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:02.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:02.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:02.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:01 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.773070+0000 mgr.y (mgr.26252) 1016 : cluster [DBG] pgmap v1549: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 25276/243021 objects degraded (10.401%) 2024-09-13T15:00:02.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:01 smithi099 bash[84576]: cluster 2024-09-13T15:00:00.773070+0000 mgr.y (mgr.26252) 1016 : cluster [DBG] pgmap v1549: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 25276/243021 objects degraded (10.401%) 2024-09-13T15:00:02.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:01 smithi099 bash[84576]: cluster 2024-09-13T15:00:01.454492+0000 mon.a (mon.0) 2108 : cluster [WRN] Health check update: Degraded data redundancy: 25276/243021 objects degraded (10.401%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:02.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:01 smithi099 bash[84576]: cluster 2024-09-13T15:00:01.454492+0000 mon.a (mon.0) 2108 : cluster [WRN] Health check update: Degraded data redundancy: 25276/243021 objects degraded (10.401%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:01 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.773070+0000 mgr.y (mgr.26252) 1016 : cluster [DBG] pgmap v1549: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 25276/243021 objects degraded (10.401%) 2024-09-13T15:00:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:01 smithi099 bash[87147]: cluster 2024-09-13T15:00:00.773070+0000 mgr.y (mgr.26252) 1016 : cluster [DBG] pgmap v1549: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 25276/243021 objects degraded (10.401%) 2024-09-13T15:00:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:01 smithi099 bash[87147]: cluster 2024-09-13T15:00:01.454492+0000 mon.a (mon.0) 2108 : cluster [WRN] Health check update: Degraded data redundancy: 25276/243021 objects degraded (10.401%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:01 smithi099 bash[87147]: cluster 2024-09-13T15:00:01.454492+0000 mon.a (mon.0) 2108 : cluster [WRN] Health check update: Degraded data redundancy: 25276/243021 objects degraded (10.401%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:02.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:02.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:02.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:02.476 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 58 16 84424 84408 90.9484 51.625 0.0110536 0.0109907 2024-09-13T15:00:02.489 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:02.513 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:02.595 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:02.616 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:02.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:02.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:02.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:02.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:02.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:02.887 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:02.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:02.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:03.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:03.046 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:03.154 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:03.177 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:03.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:03.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:03.314 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:03.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:03.423 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:03.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:03.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:03.476 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 59 16 85460 85444 90.5042 64.75 0.00525073 0.0110368 2024-09-13T15:00:03.544 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:03.565 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:03.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:03.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:03.696 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:03.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:03.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:03.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:03.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:03.937 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:03.966 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:03.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:04.070 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:04.109 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:04.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:04.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:04.227 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[84576]: cluster 2024-09-13T15:00:02.773875+0000 mgr.y (mgr.26252) 1017 : cluster [DBG] pgmap v1550: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 25573/245706 objects degraded (10.408%) 2024-09-13T15:00:04.227 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[84576]: cluster 2024-09-13T15:00:02.773875+0000 mgr.y (mgr.26252) 1017 : cluster [DBG] pgmap v1550: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 25573/245706 objects degraded (10.408%) 2024-09-13T15:00:04.227 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[84576]: audit 2024-09-13T15:00:03.232223+0000 mon.a (mon.0) 2109 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:04.227 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[84576]: audit 2024-09-13T15:00:03.232223+0000 mon.a (mon.0) 2109 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:04.227 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[84576]: audit 2024-09-13T15:00:03.291215+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:04.227 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[84576]: audit 2024-09-13T15:00:03.291215+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:04.227 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[87147]: cluster 2024-09-13T15:00:02.773875+0000 mgr.y (mgr.26252) 1017 : cluster [DBG] pgmap v1550: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 25573/245706 objects degraded (10.408%) 2024-09-13T15:00:04.227 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[87147]: cluster 2024-09-13T15:00:02.773875+0000 mgr.y (mgr.26252) 1017 : cluster [DBG] pgmap v1550: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 25573/245706 objects degraded (10.408%) 2024-09-13T15:00:04.228 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[87147]: audit 2024-09-13T15:00:03.232223+0000 mon.a (mon.0) 2109 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:04.228 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[87147]: audit 2024-09-13T15:00:03.232223+0000 mon.a (mon.0) 2109 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:04.228 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[87147]: audit 2024-09-13T15:00:03.291215+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:04.228 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:03 smithi099 bash[87147]: audit 2024-09-13T15:00:03.291215+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:04.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:04.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:04.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:04.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:04 smithi177 bash[68594]: cluster 2024-09-13T15:00:02.773875+0000 mgr.y (mgr.26252) 1017 : cluster [DBG] pgmap v1550: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 25573/245706 objects degraded (10.408%) 2024-09-13T15:00:04.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:04 smithi177 bash[68594]: cluster 2024-09-13T15:00:02.773875+0000 mgr.y (mgr.26252) 1017 : cluster [DBG] pgmap v1550: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s; 25573/245706 objects degraded (10.408%) 2024-09-13T15:00:04.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:04 smithi177 bash[68594]: audit 2024-09-13T15:00:03.232223+0000 mon.a (mon.0) 2109 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:04 smithi177 bash[68594]: audit 2024-09-13T15:00:03.232223+0000 mon.a (mon.0) 2109 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:04 smithi177 bash[68594]: audit 2024-09-13T15:00:03.291215+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:04 smithi177 bash[68594]: audit 2024-09-13T15:00:03.291215+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:04.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:04.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:04.476 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:00:04.477364+0000 min lat: 0.00108369 max lat: 0.468934 avg lat: 0.0111087 2024-09-13T15:00:04.476 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:00:04.476 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 60 16 86224 86208 89.7916 47.75 0.00357072 0.0111087 2024-09-13T15:00:04.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:04.499 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:04.602 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:04.632 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:04.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:04.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:04.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:04.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:04.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:04.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:04.939 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:00:04.940 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:00:04.981 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:04.981 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:00:05.000 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:05.020 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:00:05.020 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:05.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:00:05.032 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:04 smithi099 bash[84576]: audit 2024-09-13T15:00:04.895531+0000 mon.a (mon.0) 2111 : audit [DBG] from='client.? 172.21.15.99:0/523034734' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:05.032 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:04 smithi099 bash[84576]: audit 2024-09-13T15:00:04.895531+0000 mon.a (mon.0) 2111 : audit [DBG] from='client.? 172.21.15.99:0/523034734' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:05.032 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:04 smithi099 bash[87147]: audit 2024-09-13T15:00:04.895531+0000 mon.a (mon.0) 2111 : audit [DBG] from='client.? 172.21.15.99:0/523034734' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:05.032 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:04 smithi099 bash[87147]: audit 2024-09-13T15:00:04.895531+0000 mon.a (mon.0) 2111 : audit [DBG] from='client.? 172.21.15.99:0/523034734' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:05.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:05.060 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:00:05.651 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:05.652 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:00:05.652 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:05.653 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:00:05.653 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:05.654 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:05.654 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:00:05.654 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 61 16 87057 87041 89.1729 52.0625 0.00805877 0.0112088 2024-09-13T15:00:05.655 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:05.656 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:04 smithi177 bash[68594]: audit 2024-09-13T15:00:04.895531+0000 mon.a (mon.0) 2111 : audit [DBG] from='client.? 172.21.15.99:0/523034734' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:05.656 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:04 smithi177 bash[68594]: audit 2024-09-13T15:00:04.895531+0000 mon.a (mon.0) 2111 : audit [DBG] from='client.? 172.21.15.99:0/523034734' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:05.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:05.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:05.787 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:05.816 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:05.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:05.921 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:05.949 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:05.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:05.968 INFO:teuthology.orchestra.run.smithi099.stderr:set pool 43 pg_num to 26 2024-09-13T15:00:05.977 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:00:06.053 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:06.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[87147]: cluster 2024-09-13T15:00:04.775086+0000 mgr.y (mgr.26252) 1018 : cluster [DBG] pgmap v1551: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:06.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[87147]: cluster 2024-09-13T15:00:04.775086+0000 mgr.y (mgr.26252) 1018 : cluster [DBG] pgmap v1551: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:06.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[87147]: audit 2024-09-13T15:00:05.232545+0000 mon.a (mon.0) 2112 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:00:06.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[87147]: audit 2024-09-13T15:00:05.232545+0000 mon.a (mon.0) 2112 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:00:06.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[87147]: audit 2024-09-13T15:00:05.413656+0000 mon.a (mon.0) 2113 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[87147]: audit 2024-09-13T15:00:05.413656+0000 mon.a (mon.0) 2113 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[87147]: audit 2024-09-13T15:00:05.413938+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.99:0/125741132' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[87147]: audit 2024-09-13T15:00:05.413938+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.99:0/125741132' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[84576]: cluster 2024-09-13T15:00:04.775086+0000 mgr.y (mgr.26252) 1018 : cluster [DBG] pgmap v1551: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:06.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[84576]: cluster 2024-09-13T15:00:04.775086+0000 mgr.y (mgr.26252) 1018 : cluster [DBG] pgmap v1551: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:06.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[84576]: audit 2024-09-13T15:00:05.232545+0000 mon.a (mon.0) 2112 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:00:06.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[84576]: audit 2024-09-13T15:00:05.232545+0000 mon.a (mon.0) 2112 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:00:06.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[84576]: audit 2024-09-13T15:00:05.413656+0000 mon.a (mon.0) 2113 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[84576]: audit 2024-09-13T15:00:05.413656+0000 mon.a (mon.0) 2113 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[84576]: audit 2024-09-13T15:00:05.413938+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.99:0/125741132' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:05 smithi099 bash[84576]: audit 2024-09-13T15:00:05.413938+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.99:0/125741132' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.075 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:06.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:06.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:06.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:06.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:06.307 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:06.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:05 smithi177 bash[68594]: cluster 2024-09-13T15:00:04.775086+0000 mgr.y (mgr.26252) 1018 : cluster [DBG] pgmap v1551: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:06.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:05 smithi177 bash[68594]: cluster 2024-09-13T15:00:04.775086+0000 mgr.y (mgr.26252) 1018 : cluster [DBG] pgmap v1551: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:06.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:05 smithi177 bash[68594]: audit 2024-09-13T15:00:05.232545+0000 mon.a (mon.0) 2112 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:00:06.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:05 smithi177 bash[68594]: audit 2024-09-13T15:00:05.232545+0000 mon.a (mon.0) 2112 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:00:06.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:05 smithi177 bash[68594]: audit 2024-09-13T15:00:05.413656+0000 mon.a (mon.0) 2113 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:05 smithi177 bash[68594]: audit 2024-09-13T15:00:05.413656+0000 mon.a (mon.0) 2113 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:05 smithi177 bash[68594]: audit 2024-09-13T15:00:05.413938+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.99:0/125741132' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:05 smithi177 bash[68594]: audit 2024-09-13T15:00:05.413938+0000 mon.b (mon.1) 14 : audit [INF] from='client.? 172.21.15.99:0/125741132' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:06.333 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:06.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:06.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:06.462 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:06.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:06.476 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 62 16 88164 88148 88.8505 69.1875 0.00965707 0.0112506 2024-09-13T15:00:06.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:06.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:06.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:06.699 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:06.729 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:06.771 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:06.771 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:06.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:06.853 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:06.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:06.956 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:06.982 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:07.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: cluster 2024-09-13T15:00:05.965194+0000 mon.a (mon.0) 2114 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:07.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: cluster 2024-09-13T15:00:05.965194+0000 mon.a (mon.0) 2114 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:07.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: audit 2024-09-13T15:00:05.969485+0000 mon.a (mon.0) 2115 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:00:07.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: audit 2024-09-13T15:00:05.969485+0000 mon.a (mon.0) 2115 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:00:07.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: audit 2024-09-13T15:00:05.969538+0000 mon.a (mon.0) 2116 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:07.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: audit 2024-09-13T15:00:05.969538+0000 mon.a (mon.0) 2116 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:07.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: cluster 2024-09-13T15:00:05.975282+0000 mon.a (mon.0) 2117 : cluster [DBG] osdmap e1045: 8 total, 7 up, 8 in 2024-09-13T15:00:07.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: cluster 2024-09-13T15:00:05.975282+0000 mon.a (mon.0) 2117 : cluster [DBG] osdmap e1045: 8 total, 7 up, 8 in 2024-09-13T15:00:07.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: cluster 2024-09-13T15:00:06.455238+0000 mon.a (mon.0) 2118 : cluster [WRN] Health check update: Degraded data redundancy: 26869/257748 objects degraded (10.425%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:07.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: cluster 2024-09-13T15:00:06.455238+0000 mon.a (mon.0) 2118 : cluster [WRN] Health check update: Degraded data redundancy: 26869/257748 objects degraded (10.425%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:07.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: audit 2024-09-13T15:00:06.776653+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:00:07.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[84576]: audit 2024-09-13T15:00:06.776653+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: cluster 2024-09-13T15:00:05.965194+0000 mon.a (mon.0) 2114 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: cluster 2024-09-13T15:00:05.965194+0000 mon.a (mon.0) 2114 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: audit 2024-09-13T15:00:05.969485+0000 mon.a (mon.0) 2115 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: audit 2024-09-13T15:00:05.969485+0000 mon.a (mon.0) 2115 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: audit 2024-09-13T15:00:05.969538+0000 mon.a (mon.0) 2116 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: audit 2024-09-13T15:00:05.969538+0000 mon.a (mon.0) 2116 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: cluster 2024-09-13T15:00:05.975282+0000 mon.a (mon.0) 2117 : cluster [DBG] osdmap e1045: 8 total, 7 up, 8 in 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: cluster 2024-09-13T15:00:05.975282+0000 mon.a (mon.0) 2117 : cluster [DBG] osdmap e1045: 8 total, 7 up, 8 in 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: cluster 2024-09-13T15:00:06.455238+0000 mon.a (mon.0) 2118 : cluster [WRN] Health check update: Degraded data redundancy: 26869/257748 objects degraded (10.425%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: cluster 2024-09-13T15:00:06.455238+0000 mon.a (mon.0) 2118 : cluster [WRN] Health check update: Degraded data redundancy: 26869/257748 objects degraded (10.425%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: audit 2024-09-13T15:00:06.776653+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:00:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:06 smithi099 bash[87147]: audit 2024-09-13T15:00:06.776653+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:00:07.085 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:07.169 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:07.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:07.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:07.291 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:07.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:07.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:00:07] "GET /metrics HTTP/1.1" 200 36770 "" "Prometheus/2.51.0" 2024-09-13T15:00:07.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:07.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:07.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:07.454 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: cluster 2024-09-13T15:00:05.965194+0000 mon.a (mon.0) 2114 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:07.454 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: cluster 2024-09-13T15:00:05.965194+0000 mon.a (mon.0) 2114 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:07.454 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: audit 2024-09-13T15:00:05.969485+0000 mon.a (mon.0) 2115 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:00:07.454 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: audit 2024-09-13T15:00:05.969485+0000 mon.a (mon.0) 2115 : audit [INF] from='client.? 172.21.15.99:0/3796379240' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:00:07.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: audit 2024-09-13T15:00:05.969538+0000 mon.a (mon.0) 2116 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:07.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: audit 2024-09-13T15:00:05.969538+0000 mon.a (mon.0) 2116 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:07.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: cluster 2024-09-13T15:00:05.975282+0000 mon.a (mon.0) 2117 : cluster [DBG] osdmap e1045: 8 total, 7 up, 8 in 2024-09-13T15:00:07.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: cluster 2024-09-13T15:00:05.975282+0000 mon.a (mon.0) 2117 : cluster [DBG] osdmap e1045: 8 total, 7 up, 8 in 2024-09-13T15:00:07.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: cluster 2024-09-13T15:00:06.455238+0000 mon.a (mon.0) 2118 : cluster [WRN] Health check update: Degraded data redundancy: 26869/257748 objects degraded (10.425%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:07.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: cluster 2024-09-13T15:00:06.455238+0000 mon.a (mon.0) 2118 : cluster [WRN] Health check update: Degraded data redundancy: 26869/257748 objects degraded (10.425%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:07.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: audit 2024-09-13T15:00:06.776653+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:00:07.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:06 smithi177 bash[68594]: audit 2024-09-13T15:00:06.776653+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:00:07.476 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 63 16 88543 88527 87.8161 23.6875 0.00344254 0.0112833 2024-09-13T15:00:07.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:07.545 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:07.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:07.647 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:07.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:07.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:07.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:07.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:07.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:07.901 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:07.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:07.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:08.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:08.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:07 smithi177 bash[68594]: cluster 2024-09-13T15:00:06.775949+0000 mgr.y (mgr.26252) 1019 : cluster [DBG] pgmap v1553: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:08.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:07 smithi177 bash[68594]: cluster 2024-09-13T15:00:06.775949+0000 mgr.y (mgr.26252) 1019 : cluster [DBG] pgmap v1553: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:08.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:07 smithi177 bash[68594]: audit 2024-09-13T15:00:06.988954+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:00:08.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:07 smithi177 bash[68594]: audit 2024-09-13T15:00:06.988954+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:00:08.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:07 smithi177 bash[68594]: cluster 2024-09-13T15:00:06.998196+0000 mon.a (mon.0) 2121 : cluster [DBG] osdmap e1046: 8 total, 7 up, 8 in 2024-09-13T15:00:08.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:07 smithi177 bash[68594]: cluster 2024-09-13T15:00:06.998196+0000 mon.a (mon.0) 2121 : cluster [DBG] osdmap e1046: 8 total, 7 up, 8 in 2024-09-13T15:00:08.051 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:08.154 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:08.186 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:08.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:08.289 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:08.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[84576]: cluster 2024-09-13T15:00:06.775949+0000 mgr.y (mgr.26252) 1019 : cluster [DBG] pgmap v1553: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:08.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[84576]: cluster 2024-09-13T15:00:06.775949+0000 mgr.y (mgr.26252) 1019 : cluster [DBG] pgmap v1553: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:08.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[84576]: audit 2024-09-13T15:00:06.988954+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:00:08.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[84576]: audit 2024-09-13T15:00:06.988954+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:00:08.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[84576]: cluster 2024-09-13T15:00:06.998196+0000 mon.a (mon.0) 2121 : cluster [DBG] osdmap e1046: 8 total, 7 up, 8 in 2024-09-13T15:00:08.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[84576]: cluster 2024-09-13T15:00:06.998196+0000 mon.a (mon.0) 2121 : cluster [DBG] osdmap e1046: 8 total, 7 up, 8 in 2024-09-13T15:00:08.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[87147]: cluster 2024-09-13T15:00:06.775949+0000 mgr.y (mgr.26252) 1019 : cluster [DBG] pgmap v1553: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:08.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[87147]: cluster 2024-09-13T15:00:06.775949+0000 mgr.y (mgr.26252) 1019 : cluster [DBG] pgmap v1553: 89 pgs: 26 active+undersized, 8 active+undersized+degraded, 55 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s; 26869/257748 objects degraded (10.425%) 2024-09-13T15:00:08.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[87147]: audit 2024-09-13T15:00:06.988954+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:00:08.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[87147]: audit 2024-09-13T15:00:06.988954+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:00:08.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[87147]: cluster 2024-09-13T15:00:06.998196+0000 mon.a (mon.0) 2121 : cluster [DBG] osdmap e1046: 8 total, 7 up, 8 in 2024-09-13T15:00:08.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:07 smithi099 bash[87147]: cluster 2024-09-13T15:00:06.998196+0000 mon.a (mon.0) 2121 : cluster [DBG] osdmap e1046: 8 total, 7 up, 8 in 2024-09-13T15:00:08.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:08.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:08.423 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:08.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:08.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:08.476 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 64 16 89043 89027 86.9322 31.25 0.00300683 0.0114976 2024-09-13T15:00:08.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:08.598 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:08.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:08.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:08.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:08.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:00:08.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:08.860 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:08.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:08.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:08.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:09.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:09.103 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:09.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:09.206 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:09.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:09.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:09 smithi099 bash[84576]: cluster 2024-09-13T15:00:08.000325+0000 mon.a (mon.0) 2122 : cluster [DBG] osdmap e1047: 8 total, 7 up, 8 in 2024-09-13T15:00:09.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:09 smithi099 bash[84576]: cluster 2024-09-13T15:00:08.000325+0000 mon.a (mon.0) 2122 : cluster [DBG] osdmap e1047: 8 total, 7 up, 8 in 2024-09-13T15:00:09.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:09 smithi099 bash[87147]: cluster 2024-09-13T15:00:08.000325+0000 mon.a (mon.0) 2122 : cluster [DBG] osdmap e1047: 8 total, 7 up, 8 in 2024-09-13T15:00:09.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:09 smithi099 bash[87147]: cluster 2024-09-13T15:00:08.000325+0000 mon.a (mon.0) 2122 : cluster [DBG] osdmap e1047: 8 total, 7 up, 8 in 2024-09-13T15:00:09.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:09.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:09 smithi177 bash[68594]: cluster 2024-09-13T15:00:08.000325+0000 mon.a (mon.0) 2122 : cluster [DBG] osdmap e1047: 8 total, 7 up, 8 in 2024-09-13T15:00:09.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:09 smithi177 bash[68594]: cluster 2024-09-13T15:00:08.000325+0000 mon.a (mon.0) 2122 : cluster [DBG] osdmap e1047: 8 total, 7 up, 8 in 2024-09-13T15:00:09.371 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:09.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:09.477 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 65 16 89848 89832 86.3688 50.3125 0.0095338 0.0115729 2024-09-13T15:00:09.500 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:09.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:09.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:09.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:09.732 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:09.757 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:09.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:09.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:09.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:09.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:09.945 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:00:09.945 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:00:09.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:10.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:10.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:10.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:10.158 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:10.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:10.265 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:10.274 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[84576]: cluster 2024-09-13T15:00:08.311121+0000 osd.6 (osd.6) 11 : cluster [DBG] 43.0 scrub starts 2024-09-13T15:00:10.274 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[84576]: cluster 2024-09-13T15:00:08.311121+0000 osd.6 (osd.6) 11 : cluster [DBG] 43.0 scrub starts 2024-09-13T15:00:10.274 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[84576]: cluster 2024-09-13T15:00:08.776502+0000 mgr.y (mgr.26252) 1020 : cluster [DBG] pgmap v1556: 99 pgs: 3 peering, 10 unknown, 26 active+undersized, 7 active+undersized+degraded, 53 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 21548/259992 objects degraded (8.288%) 2024-09-13T15:00:10.274 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[84576]: cluster 2024-09-13T15:00:08.776502+0000 mgr.y (mgr.26252) 1020 : cluster [DBG] pgmap v1556: 99 pgs: 3 peering, 10 unknown, 26 active+undersized, 7 active+undersized+degraded, 53 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 21548/259992 objects degraded (8.288%) 2024-09-13T15:00:10.274 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[84576]: audit 2024-09-13T15:00:09.724927+0000 mon.a (mon.0) 2123 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.274 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[84576]: audit 2024-09-13T15:00:09.724927+0000 mon.a (mon.0) 2123 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.275 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[84576]: audit 2024-09-13T15:00:09.735411+0000 mon.a (mon.0) 2124 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.275 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[84576]: audit 2024-09-13T15:00:09.735411+0000 mon.a (mon.0) 2124 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.275 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[87147]: cluster 2024-09-13T15:00:08.311121+0000 osd.6 (osd.6) 11 : cluster [DBG] 43.0 scrub starts 2024-09-13T15:00:10.275 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[87147]: cluster 2024-09-13T15:00:08.311121+0000 osd.6 (osd.6) 11 : cluster [DBG] 43.0 scrub starts 2024-09-13T15:00:10.275 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[87147]: cluster 2024-09-13T15:00:08.776502+0000 mgr.y (mgr.26252) 1020 : cluster [DBG] pgmap v1556: 99 pgs: 3 peering, 10 unknown, 26 active+undersized, 7 active+undersized+degraded, 53 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 21548/259992 objects degraded (8.288%) 2024-09-13T15:00:10.275 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[87147]: cluster 2024-09-13T15:00:08.776502+0000 mgr.y (mgr.26252) 1020 : cluster [DBG] pgmap v1556: 99 pgs: 3 peering, 10 unknown, 26 active+undersized, 7 active+undersized+degraded, 53 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 21548/259992 objects degraded (8.288%) 2024-09-13T15:00:10.275 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[87147]: audit 2024-09-13T15:00:09.724927+0000 mon.a (mon.0) 2123 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.275 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[87147]: audit 2024-09-13T15:00:09.724927+0000 mon.a (mon.0) 2123 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.275 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[87147]: audit 2024-09-13T15:00:09.735411+0000 mon.a (mon.0) 2124 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.275 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:10 smithi099 bash[87147]: audit 2024-09-13T15:00:09.735411+0000 mon.a (mon.0) 2124 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.289 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:10.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:00:10 smithi177 bash[61439]: ts=2024-09-13T15:00:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:10 smithi177 bash[68594]: cluster 2024-09-13T15:00:08.311121+0000 osd.6 (osd.6) 11 : cluster [DBG] 43.0 scrub starts 2024-09-13T15:00:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:10 smithi177 bash[68594]: cluster 2024-09-13T15:00:08.311121+0000 osd.6 (osd.6) 11 : cluster [DBG] 43.0 scrub starts 2024-09-13T15:00:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:10 smithi177 bash[68594]: cluster 2024-09-13T15:00:08.776502+0000 mgr.y (mgr.26252) 1020 : cluster [DBG] pgmap v1556: 99 pgs: 3 peering, 10 unknown, 26 active+undersized, 7 active+undersized+degraded, 53 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 21548/259992 objects degraded (8.288%) 2024-09-13T15:00:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:10 smithi177 bash[68594]: cluster 2024-09-13T15:00:08.776502+0000 mgr.y (mgr.26252) 1020 : cluster [DBG] pgmap v1556: 99 pgs: 3 peering, 10 unknown, 26 active+undersized, 7 active+undersized+degraded, 53 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 21548/259992 objects degraded (8.288%) 2024-09-13T15:00:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:10 smithi177 bash[68594]: audit 2024-09-13T15:00:09.724927+0000 mon.a (mon.0) 2123 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:10 smithi177 bash[68594]: audit 2024-09-13T15:00:09.724927+0000 mon.a (mon.0) 2123 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:10 smithi177 bash[68594]: audit 2024-09-13T15:00:09.735411+0000 mon.a (mon.0) 2124 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:10 smithi177 bash[68594]: audit 2024-09-13T15:00:09.735411+0000 mon.a (mon.0) 2124 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:10.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:10.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:10.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:10.476 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 66 16 90976 90960 86.1282 70.5 0.0117695 0.0116059 2024-09-13T15:00:10.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:10.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:10.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:10.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:10.685 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:10.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:00:10.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:10.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:10.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:10.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:10.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:10.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:10.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:11.070 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:11.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:11 smithi099 bash[84576]: cluster 2024-09-13T15:00:09.275914+0000 osd.6 (osd.6) 12 : cluster [DBG] 43.10 scrub starts 2024-09-13T15:00:11.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:11 smithi099 bash[84576]: cluster 2024-09-13T15:00:09.275914+0000 osd.6 (osd.6) 12 : cluster [DBG] 43.10 scrub starts 2024-09-13T15:00:11.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:11 smithi099 bash[84576]: cluster 2024-09-13T15:00:10.036076+0000 osd.6 (osd.6) 13 : cluster [DBG] 43.0 scrub ok 2024-09-13T15:00:11.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:11 smithi099 bash[84576]: cluster 2024-09-13T15:00:10.036076+0000 osd.6 (osd.6) 13 : cluster [DBG] 43.0 scrub ok 2024-09-13T15:00:11.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:11 smithi099 bash[87147]: cluster 2024-09-13T15:00:09.275914+0000 osd.6 (osd.6) 12 : cluster [DBG] 43.10 scrub starts 2024-09-13T15:00:11.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:11 smithi099 bash[87147]: cluster 2024-09-13T15:00:09.275914+0000 osd.6 (osd.6) 12 : cluster [DBG] 43.10 scrub starts 2024-09-13T15:00:11.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:11 smithi099 bash[87147]: cluster 2024-09-13T15:00:10.036076+0000 osd.6 (osd.6) 13 : cluster [DBG] 43.0 scrub ok 2024-09-13T15:00:11.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:11 smithi099 bash[87147]: cluster 2024-09-13T15:00:10.036076+0000 osd.6 (osd.6) 13 : cluster [DBG] 43.0 scrub ok 2024-09-13T15:00:11.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:11.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:11.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:11.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:11 smithi177 bash[68594]: cluster 2024-09-13T15:00:09.275914+0000 osd.6 (osd.6) 12 : cluster [DBG] 43.10 scrub starts 2024-09-13T15:00:11.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:11 smithi177 bash[68594]: cluster 2024-09-13T15:00:09.275914+0000 osd.6 (osd.6) 12 : cluster [DBG] 43.10 scrub starts 2024-09-13T15:00:11.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:11 smithi177 bash[68594]: cluster 2024-09-13T15:00:10.036076+0000 osd.6 (osd.6) 13 : cluster [DBG] 43.0 scrub ok 2024-09-13T15:00:11.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:11 smithi177 bash[68594]: cluster 2024-09-13T15:00:10.036076+0000 osd.6 (osd.6) 13 : cluster [DBG] 43.0 scrub ok 2024-09-13T15:00:11.246 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:11.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:11.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:11.384 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:11.409 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:00:11.477 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 67 16 92150 92134 85.9374 73.375 0.0193503 0.0116311 2024-09-13T15:00:11.492 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:11.519 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:11.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:11.625 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:11.654 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:11.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:11.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:11.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:11.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:11.932 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:12.037 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:12.064 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:12.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:12.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:10.397858+0000 osd.1 (osd.1) 1 : cluster [DBG] 43.5 scrub starts 2024-09-13T15:00:12.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:10.397858+0000 osd.1 (osd.1) 1 : cluster [DBG] 43.5 scrub starts 2024-09-13T15:00:12.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:10.536046+0000 osd.6 (osd.6) 14 : cluster [DBG] 43.10 scrub ok 2024-09-13T15:00:12.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:10.536046+0000 osd.6 (osd.6) 14 : cluster [DBG] 43.10 scrub ok 2024-09-13T15:00:12.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:10.688532+0000 osd.2 (osd.2) 18 : cluster [DBG] 43.2 scrub starts 2024-09-13T15:00:12.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:10.688532+0000 osd.2 (osd.2) 18 : cluster [DBG] 43.2 scrub starts 2024-09-13T15:00:12.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:10.777499+0000 mgr.y (mgr.26252) 1021 : cluster [DBG] pgmap v1557: 99 pgs: 2 active+clean+scrubbing, 3 peering, 3 unknown, 26 active+undersized, 9 active+undersized+degraded, 56 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 22464/268770 objects degraded (8.358%) 2024-09-13T15:00:12.179 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:10.777499+0000 mgr.y (mgr.26252) 1021 : cluster [DBG] pgmap v1557: 99 pgs: 2 active+clean+scrubbing, 3 peering, 3 unknown, 26 active+undersized, 9 active+undersized+degraded, 56 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 22464/268770 objects degraded (8.358%) 2024-09-13T15:00:12.179 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:11.161743+0000 osd.1 (osd.1) 2 : cluster [DBG] 43.5 scrub ok 2024-09-13T15:00:12.179 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:11.161743+0000 osd.1 (osd.1) 2 : cluster [DBG] 43.5 scrub ok 2024-09-13T15:00:12.179 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:11.456095+0000 mon.a (mon.0) 2125 : cluster [WRN] Health check update: Degraded data redundancy: 22464/268770 objects degraded (8.358%), 9 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:12.179 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[84576]: cluster 2024-09-13T15:00:11.456095+0000 mon.a (mon.0) 2125 : cluster [WRN] Health check update: Degraded data redundancy: 22464/268770 objects degraded (8.358%), 9 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:12.179 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:10.397858+0000 osd.1 (osd.1) 1 : cluster [DBG] 43.5 scrub starts 2024-09-13T15:00:12.179 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:10.397858+0000 osd.1 (osd.1) 1 : cluster [DBG] 43.5 scrub starts 2024-09-13T15:00:12.179 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:10.536046+0000 osd.6 (osd.6) 14 : cluster [DBG] 43.10 scrub ok 2024-09-13T15:00:12.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:10.536046+0000 osd.6 (osd.6) 14 : cluster [DBG] 43.10 scrub ok 2024-09-13T15:00:12.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:10.688532+0000 osd.2 (osd.2) 18 : cluster [DBG] 43.2 scrub starts 2024-09-13T15:00:12.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:10.688532+0000 osd.2 (osd.2) 18 : cluster [DBG] 43.2 scrub starts 2024-09-13T15:00:12.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:10.777499+0000 mgr.y (mgr.26252) 1021 : cluster [DBG] pgmap v1557: 99 pgs: 2 active+clean+scrubbing, 3 peering, 3 unknown, 26 active+undersized, 9 active+undersized+degraded, 56 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 22464/268770 objects degraded (8.358%) 2024-09-13T15:00:12.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:10.777499+0000 mgr.y (mgr.26252) 1021 : cluster [DBG] pgmap v1557: 99 pgs: 2 active+clean+scrubbing, 3 peering, 3 unknown, 26 active+undersized, 9 active+undersized+degraded, 56 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 22464/268770 objects degraded (8.358%) 2024-09-13T15:00:12.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:11.161743+0000 osd.1 (osd.1) 2 : cluster [DBG] 43.5 scrub ok 2024-09-13T15:00:12.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:11.161743+0000 osd.1 (osd.1) 2 : cluster [DBG] 43.5 scrub ok 2024-09-13T15:00:12.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:11.456095+0000 mon.a (mon.0) 2125 : cluster [WRN] Health check update: Degraded data redundancy: 22464/268770 objects degraded (8.358%), 9 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:12.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:12 smithi099 bash[87147]: cluster 2024-09-13T15:00:11.456095+0000 mon.a (mon.0) 2125 : cluster [WRN] Health check update: Degraded data redundancy: 22464/268770 objects degraded (8.358%), 9 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:12.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:12.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:12.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:12.332 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:12.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:12.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:10.397858+0000 osd.1 (osd.1) 1 : cluster [DBG] 43.5 scrub starts 2024-09-13T15:00:12.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:10.397858+0000 osd.1 (osd.1) 1 : cluster [DBG] 43.5 scrub starts 2024-09-13T15:00:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:10.536046+0000 osd.6 (osd.6) 14 : cluster [DBG] 43.10 scrub ok 2024-09-13T15:00:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:10.536046+0000 osd.6 (osd.6) 14 : cluster [DBG] 43.10 scrub ok 2024-09-13T15:00:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:10.688532+0000 osd.2 (osd.2) 18 : cluster [DBG] 43.2 scrub starts 2024-09-13T15:00:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:10.688532+0000 osd.2 (osd.2) 18 : cluster [DBG] 43.2 scrub starts 2024-09-13T15:00:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:10.777499+0000 mgr.y (mgr.26252) 1021 : cluster [DBG] pgmap v1557: 99 pgs: 2 active+clean+scrubbing, 3 peering, 3 unknown, 26 active+undersized, 9 active+undersized+degraded, 56 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 22464/268770 objects degraded (8.358%) 2024-09-13T15:00:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:10.777499+0000 mgr.y (mgr.26252) 1021 : cluster [DBG] pgmap v1557: 99 pgs: 2 active+clean+scrubbing, 3 peering, 3 unknown, 26 active+undersized, 9 active+undersized+degraded, 56 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 22464/268770 objects degraded (8.358%) 2024-09-13T15:00:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:11.161743+0000 osd.1 (osd.1) 2 : cluster [DBG] 43.5 scrub ok 2024-09-13T15:00:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:11.161743+0000 osd.1 (osd.1) 2 : cluster [DBG] 43.5 scrub ok 2024-09-13T15:00:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:11.456095+0000 mon.a (mon.0) 2125 : cluster [WRN] Health check update: Degraded data redundancy: 22464/268770 objects degraded (8.358%), 9 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:12 smithi177 bash[68594]: cluster 2024-09-13T15:00:11.456095+0000 mon.a (mon.0) 2125 : cluster [WRN] Health check update: Degraded data redundancy: 22464/268770 objects degraded (8.358%), 9 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:12.436 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:12.461 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:12.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:12.477 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 68 16 93322 93306 85.7507 73.25 0.0211259 0.0116569 2024-09-13T15:00:12.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:12.605 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:12.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:12.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:12.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:12.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:12.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:12.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:13.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:13.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:13.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:13.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:13.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:13.228 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:13.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:13.251 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:11.431156+0000 osd.1 (osd.1) 3 : cluster [DBG] 43.15 scrub starts 2024-09-13T15:00:13.251 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:11.431156+0000 osd.1 (osd.1) 3 : cluster [DBG] 43.15 scrub starts 2024-09-13T15:00:13.251 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:11.432195+0000 osd.2 (osd.2) 19 : cluster [DBG] 43.2 scrub ok 2024-09-13T15:00:13.251 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:11.432195+0000 osd.2 (osd.2) 19 : cluster [DBG] 43.2 scrub ok 2024-09-13T15:00:13.251 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:11.717121+0000 osd.2 (osd.2) 20 : cluster [DBG] 43.12 scrub starts 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:11.717121+0000 osd.2 (osd.2) 20 : cluster [DBG] 43.12 scrub starts 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:12.221912+0000 osd.1 (osd.1) 4 : cluster [DBG] 43.15 scrub ok 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:12.221912+0000 osd.1 (osd.1) 4 : cluster [DBG] 43.15 scrub ok 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: audit 2024-09-13T15:00:12.224471+0000 mon.a (mon.0) 2126 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: audit 2024-09-13T15:00:12.224471+0000 mon.a (mon.0) 2126 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: audit 2024-09-13T15:00:12.231939+0000 mon.a (mon.0) 2127 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: audit 2024-09-13T15:00:12.231939+0000 mon.a (mon.0) 2127 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:12.465271+0000 osd.7 (osd.7) 3 : cluster [DBG] 43.8 scrub starts 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:12.465271+0000 osd.7 (osd.7) 3 : cluster [DBG] 43.8 scrub starts 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:12.778587+0000 mgr.y (mgr.26252) 1022 : cluster [DBG] pgmap v1558: 99 pgs: 2 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 60 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 28337/271665 objects degraded (10.431%); 19 KiB/s, 0 objects/s recovering 2024-09-13T15:00:13.252 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:13 smithi177 bash[68594]: cluster 2024-09-13T15:00:12.778587+0000 mgr.y (mgr.26252) 1022 : cluster [DBG] pgmap v1558: 99 pgs: 2 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 60 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 28337/271665 objects degraded (10.431%); 19 KiB/s, 0 objects/s recovering 2024-09-13T15:00:13.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:13.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:11.431156+0000 osd.1 (osd.1) 3 : cluster [DBG] 43.15 scrub starts 2024-09-13T15:00:13.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:11.431156+0000 osd.1 (osd.1) 3 : cluster [DBG] 43.15 scrub starts 2024-09-13T15:00:13.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:11.432195+0000 osd.2 (osd.2) 19 : cluster [DBG] 43.2 scrub ok 2024-09-13T15:00:13.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:11.432195+0000 osd.2 (osd.2) 19 : cluster [DBG] 43.2 scrub ok 2024-09-13T15:00:13.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:11.717121+0000 osd.2 (osd.2) 20 : cluster [DBG] 43.12 scrub starts 2024-09-13T15:00:13.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:11.717121+0000 osd.2 (osd.2) 20 : cluster [DBG] 43.12 scrub starts 2024-09-13T15:00:13.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:12.221912+0000 osd.1 (osd.1) 4 : cluster [DBG] 43.15 scrub ok 2024-09-13T15:00:13.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:12.221912+0000 osd.1 (osd.1) 4 : cluster [DBG] 43.15 scrub ok 2024-09-13T15:00:13.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: audit 2024-09-13T15:00:12.224471+0000 mon.a (mon.0) 2126 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: audit 2024-09-13T15:00:12.224471+0000 mon.a (mon.0) 2126 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: audit 2024-09-13T15:00:12.231939+0000 mon.a (mon.0) 2127 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: audit 2024-09-13T15:00:12.231939+0000 mon.a (mon.0) 2127 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:12.465271+0000 osd.7 (osd.7) 3 : cluster [DBG] 43.8 scrub starts 2024-09-13T15:00:13.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:12.465271+0000 osd.7 (osd.7) 3 : cluster [DBG] 43.8 scrub starts 2024-09-13T15:00:13.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:12.778587+0000 mgr.y (mgr.26252) 1022 : cluster [DBG] pgmap v1558: 99 pgs: 2 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 60 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 28337/271665 objects degraded (10.431%); 19 KiB/s, 0 objects/s recovering 2024-09-13T15:00:13.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[84576]: cluster 2024-09-13T15:00:12.778587+0000 mgr.y (mgr.26252) 1022 : cluster [DBG] pgmap v1558: 99 pgs: 2 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 60 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 28337/271665 objects degraded (10.431%); 19 KiB/s, 0 objects/s recovering 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:11.431156+0000 osd.1 (osd.1) 3 : cluster [DBG] 43.15 scrub starts 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:11.431156+0000 osd.1 (osd.1) 3 : cluster [DBG] 43.15 scrub starts 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:11.432195+0000 osd.2 (osd.2) 19 : cluster [DBG] 43.2 scrub ok 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:11.432195+0000 osd.2 (osd.2) 19 : cluster [DBG] 43.2 scrub ok 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:11.717121+0000 osd.2 (osd.2) 20 : cluster [DBG] 43.12 scrub starts 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:11.717121+0000 osd.2 (osd.2) 20 : cluster [DBG] 43.12 scrub starts 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:12.221912+0000 osd.1 (osd.1) 4 : cluster [DBG] 43.15 scrub ok 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:12.221912+0000 osd.1 (osd.1) 4 : cluster [DBG] 43.15 scrub ok 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: audit 2024-09-13T15:00:12.224471+0000 mon.a (mon.0) 2126 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: audit 2024-09-13T15:00:12.224471+0000 mon.a (mon.0) 2126 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: audit 2024-09-13T15:00:12.231939+0000 mon.a (mon.0) 2127 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: audit 2024-09-13T15:00:12.231939+0000 mon.a (mon.0) 2127 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:13.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:12.465271+0000 osd.7 (osd.7) 3 : cluster [DBG] 43.8 scrub starts 2024-09-13T15:00:13.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:12.465271+0000 osd.7 (osd.7) 3 : cluster [DBG] 43.8 scrub starts 2024-09-13T15:00:13.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:12.778587+0000 mgr.y (mgr.26252) 1022 : cluster [DBG] pgmap v1558: 99 pgs: 2 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 60 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 28337/271665 objects degraded (10.431%); 19 KiB/s, 0 objects/s recovering 2024-09-13T15:00:13.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:13 smithi099 bash[87147]: cluster 2024-09-13T15:00:12.778587+0000 mgr.y (mgr.26252) 1022 : cluster [DBG] pgmap v1558: 99 pgs: 2 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 60 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 28337/271665 objects degraded (10.431%); 19 KiB/s, 0 objects/s recovering 2024-09-13T15:00:13.371 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:13.396 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:13.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:13.477 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 69 16 94521 94505 85.5939 74.9375 0.0166286 0.0116786 2024-09-13T15:00:13.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:13.527 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:13.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:13.643 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T15:00:13.643 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:00:13.643 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-09-13T15:00:13.643 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-09-13T15:00:13.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:13.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:13.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:13.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:13.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:13.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:13.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:13.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:14.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:14.065 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:14.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:14.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:14.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:14.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:14.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:14.312 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: cluster 2024-09-13T15:00:12.597329+0000 osd.2 (osd.2) 21 : cluster [DBG] 43.12 scrub ok 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: cluster 2024-09-13T15:00:12.597329+0000 osd.2 (osd.2) 21 : cluster [DBG] 43.12 scrub ok 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: cluster 2024-09-13T15:00:12.673592+0000 osd.2 (osd.2) 22 : cluster [DBG] 43.7 scrub starts 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: cluster 2024-09-13T15:00:12.673592+0000 osd.2 (osd.2) 22 : cluster [DBG] 43.7 scrub starts 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: audit 2024-09-13T15:00:13.101299+0000 mon.a (mon.0) 2128 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: audit 2024-09-13T15:00:13.101299+0000 mon.a (mon.0) 2128 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: audit 2024-09-13T15:00:13.104167+0000 mon.a (mon.0) 2129 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: audit 2024-09-13T15:00:13.104167+0000 mon.a (mon.0) 2129 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: audit 2024-09-13T15:00:13.112270+0000 mon.a (mon.0) 2130 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: audit 2024-09-13T15:00:13.112270+0000 mon.a (mon.0) 2130 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: cluster 2024-09-13T15:00:13.607276+0000 osd.7 (osd.7) 4 : cluster [DBG] 43.8 scrub ok 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: cluster 2024-09-13T15:00:13.607276+0000 osd.7 (osd.7) 4 : cluster [DBG] 43.8 scrub ok 2024-09-13T15:00:14.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: cluster 2024-09-13T15:00:13.774790+0000 osd.5 (osd.5) 8 : cluster [DBG] 43.1 scrub starts 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[84576]: cluster 2024-09-13T15:00:13.774790+0000 osd.5 (osd.5) 8 : cluster [DBG] 43.1 scrub starts 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: cluster 2024-09-13T15:00:12.597329+0000 osd.2 (osd.2) 21 : cluster [DBG] 43.12 scrub ok 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: cluster 2024-09-13T15:00:12.597329+0000 osd.2 (osd.2) 21 : cluster [DBG] 43.12 scrub ok 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: cluster 2024-09-13T15:00:12.673592+0000 osd.2 (osd.2) 22 : cluster [DBG] 43.7 scrub starts 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: cluster 2024-09-13T15:00:12.673592+0000 osd.2 (osd.2) 22 : cluster [DBG] 43.7 scrub starts 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: audit 2024-09-13T15:00:13.101299+0000 mon.a (mon.0) 2128 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: audit 2024-09-13T15:00:13.101299+0000 mon.a (mon.0) 2128 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: audit 2024-09-13T15:00:13.104167+0000 mon.a (mon.0) 2129 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: audit 2024-09-13T15:00:13.104167+0000 mon.a (mon.0) 2129 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: audit 2024-09-13T15:00:13.112270+0000 mon.a (mon.0) 2130 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: audit 2024-09-13T15:00:13.112270+0000 mon.a (mon.0) 2130 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:14.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: cluster 2024-09-13T15:00:13.607276+0000 osd.7 (osd.7) 4 : cluster [DBG] 43.8 scrub ok 2024-09-13T15:00:14.315 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: cluster 2024-09-13T15:00:13.607276+0000 osd.7 (osd.7) 4 : cluster [DBG] 43.8 scrub ok 2024-09-13T15:00:14.315 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: cluster 2024-09-13T15:00:13.774790+0000 osd.5 (osd.5) 8 : cluster [DBG] 43.1 scrub starts 2024-09-13T15:00:14.315 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:14 smithi099 bash[87147]: cluster 2024-09-13T15:00:13.774790+0000 osd.5 (osd.5) 8 : cluster [DBG] 43.1 scrub starts 2024-09-13T15:00:14.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:14.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:14.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: cluster 2024-09-13T15:00:12.597329+0000 osd.2 (osd.2) 21 : cluster [DBG] 43.12 scrub ok 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: cluster 2024-09-13T15:00:12.597329+0000 osd.2 (osd.2) 21 : cluster [DBG] 43.12 scrub ok 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: cluster 2024-09-13T15:00:12.673592+0000 osd.2 (osd.2) 22 : cluster [DBG] 43.7 scrub starts 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: cluster 2024-09-13T15:00:12.673592+0000 osd.2 (osd.2) 22 : cluster [DBG] 43.7 scrub starts 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: audit 2024-09-13T15:00:13.101299+0000 mon.a (mon.0) 2128 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: audit 2024-09-13T15:00:13.101299+0000 mon.a (mon.0) 2128 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: audit 2024-09-13T15:00:13.104167+0000 mon.a (mon.0) 2129 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: audit 2024-09-13T15:00:13.104167+0000 mon.a (mon.0) 2129 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: audit 2024-09-13T15:00:13.112270+0000 mon.a (mon.0) 2130 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: audit 2024-09-13T15:00:13.112270+0000 mon.a (mon.0) 2130 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: cluster 2024-09-13T15:00:13.607276+0000 osd.7 (osd.7) 4 : cluster [DBG] 43.8 scrub ok 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: cluster 2024-09-13T15:00:13.607276+0000 osd.7 (osd.7) 4 : cluster [DBG] 43.8 scrub ok 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: cluster 2024-09-13T15:00:13.774790+0000 osd.5 (osd.5) 8 : cluster [DBG] 43.1 scrub starts 2024-09-13T15:00:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[68594]: cluster 2024-09-13T15:00:13.774790+0000 osd.5 (osd.5) 8 : cluster [DBG] 43.1 scrub starts 2024-09-13T15:00:14.435 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:14.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.464 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:14.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:14.477 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 70 16 95782 95766 85.4969 78.8125 0.00897335 0.0116916 2024-09-13T15:00:14.565 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:14.589 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:14.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:14.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:14.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:14.765 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:00:14 smithi177 bash[60145]: logger=cleanup t=2024-09-13T15:00:14.508187572Z level=info msg="Completed cleanup jobs" duration=66.176926ms 2024-09-13T15:00:14.865 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:14.889 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:14.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:14.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:15.020 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:15.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:15.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:15.132 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[84576]: cluster 2024-09-13T15:00:13.351470+0000 osd.2 (osd.2) 23 : cluster [DBG] 43.7 scrub ok 2024-09-13T15:00:15.132 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[84576]: cluster 2024-09-13T15:00:13.351470+0000 osd.2 (osd.2) 23 : cluster [DBG] 43.7 scrub ok 2024-09-13T15:00:15.132 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[84576]: cluster 2024-09-13T15:00:13.641647+0000 osd.2 (osd.2) 24 : cluster [DBG] 43.17 scrub starts 2024-09-13T15:00:15.132 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[84576]: cluster 2024-09-13T15:00:13.641647+0000 osd.2 (osd.2) 24 : cluster [DBG] 43.17 scrub starts 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[84576]: cluster 2024-09-13T15:00:14.674987+0000 osd.5 (osd.5) 9 : cluster [DBG] 43.1 scrub ok 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[84576]: cluster 2024-09-13T15:00:14.674987+0000 osd.5 (osd.5) 9 : cluster [DBG] 43.1 scrub ok 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[84576]: cluster 2024-09-13T15:00:14.725896+0000 osd.5 (osd.5) 10 : cluster [DBG] 43.6 scrub starts 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[84576]: cluster 2024-09-13T15:00:14.725896+0000 osd.5 (osd.5) 10 : cluster [DBG] 43.6 scrub starts 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[84576]: cluster 2024-09-13T15:00:14.779526+0000 mgr.y (mgr.26252) 1023 : cluster [DBG] pgmap v1559: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 16 KiB/s, 0 objects/s recovering 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[84576]: cluster 2024-09-13T15:00:14.779526+0000 mgr.y (mgr.26252) 1023 : cluster [DBG] pgmap v1559: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 16 KiB/s, 0 objects/s recovering 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[87147]: cluster 2024-09-13T15:00:13.351470+0000 osd.2 (osd.2) 23 : cluster [DBG] 43.7 scrub ok 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[87147]: cluster 2024-09-13T15:00:13.351470+0000 osd.2 (osd.2) 23 : cluster [DBG] 43.7 scrub ok 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[87147]: cluster 2024-09-13T15:00:13.641647+0000 osd.2 (osd.2) 24 : cluster [DBG] 43.17 scrub starts 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[87147]: cluster 2024-09-13T15:00:13.641647+0000 osd.2 (osd.2) 24 : cluster [DBG] 43.17 scrub starts 2024-09-13T15:00:15.133 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[87147]: cluster 2024-09-13T15:00:14.674987+0000 osd.5 (osd.5) 9 : cluster [DBG] 43.1 scrub ok 2024-09-13T15:00:15.134 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[87147]: cluster 2024-09-13T15:00:14.674987+0000 osd.5 (osd.5) 9 : cluster [DBG] 43.1 scrub ok 2024-09-13T15:00:15.134 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[87147]: cluster 2024-09-13T15:00:14.725896+0000 osd.5 (osd.5) 10 : cluster [DBG] 43.6 scrub starts 2024-09-13T15:00:15.134 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[87147]: cluster 2024-09-13T15:00:14.725896+0000 osd.5 (osd.5) 10 : cluster [DBG] 43.6 scrub starts 2024-09-13T15:00:15.134 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[87147]: cluster 2024-09-13T15:00:14.779526+0000 mgr.y (mgr.26252) 1023 : cluster [DBG] pgmap v1559: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 16 KiB/s, 0 objects/s recovering 2024-09-13T15:00:15.134 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:15 smithi099 bash[87147]: cluster 2024-09-13T15:00:14.779526+0000 mgr.y (mgr.26252) 1023 : cluster [DBG] pgmap v1559: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 16 KiB/s, 0 objects/s recovering 2024-09-13T15:00:15.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:15.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:15.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:15.258 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:15.268 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:15.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:15 smithi177 bash[68594]: cluster 2024-09-13T15:00:13.351470+0000 osd.2 (osd.2) 23 : cluster [DBG] 43.7 scrub ok 2024-09-13T15:00:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:15 smithi177 bash[68594]: cluster 2024-09-13T15:00:13.351470+0000 osd.2 (osd.2) 23 : cluster [DBG] 43.7 scrub ok 2024-09-13T15:00:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:15 smithi177 bash[68594]: cluster 2024-09-13T15:00:13.641647+0000 osd.2 (osd.2) 24 : cluster [DBG] 43.17 scrub starts 2024-09-13T15:00:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:15 smithi177 bash[68594]: cluster 2024-09-13T15:00:13.641647+0000 osd.2 (osd.2) 24 : cluster [DBG] 43.17 scrub starts 2024-09-13T15:00:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:15 smithi177 bash[68594]: cluster 2024-09-13T15:00:14.674987+0000 osd.5 (osd.5) 9 : cluster [DBG] 43.1 scrub ok 2024-09-13T15:00:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:15 smithi177 bash[68594]: cluster 2024-09-13T15:00:14.674987+0000 osd.5 (osd.5) 9 : cluster [DBG] 43.1 scrub ok 2024-09-13T15:00:15.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:15 smithi177 bash[68594]: cluster 2024-09-13T15:00:14.725896+0000 osd.5 (osd.5) 10 : cluster [DBG] 43.6 scrub starts 2024-09-13T15:00:15.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:15 smithi177 bash[68594]: cluster 2024-09-13T15:00:14.725896+0000 osd.5 (osd.5) 10 : cluster [DBG] 43.6 scrub starts 2024-09-13T15:00:15.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:15 smithi177 bash[68594]: cluster 2024-09-13T15:00:14.779526+0000 mgr.y (mgr.26252) 1023 : cluster [DBG] pgmap v1559: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 16 KiB/s, 0 objects/s recovering 2024-09-13T15:00:15.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:15 smithi177 bash[68594]: cluster 2024-09-13T15:00:14.779526+0000 mgr.y (mgr.26252) 1023 : cluster [DBG] pgmap v1559: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 16 KiB/s, 0 objects/s recovering 2024-09-13T15:00:15.370 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:15.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:15.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:15.477 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 71 16 96825 96809 85.2108 65.1875 0.0233447 0.0117313 2024-09-13T15:00:15.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:15.524 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:15.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:15.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:15.654 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:15.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:15.782 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:15.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:15.886 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:15.913 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:15.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:16.016 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:16.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:16.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:16.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:16.175 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:16.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:16.244 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:16.281 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:16.311 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:16.333 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:16 smithi177 bash[68594]: cluster 2024-09-13T15:00:14.353248+0000 osd.2 (osd.2) 25 : cluster [DBG] 43.17 scrub ok 2024-09-13T15:00:16.333 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:16 smithi177 bash[68594]: cluster 2024-09-13T15:00:14.353248+0000 osd.2 (osd.2) 25 : cluster [DBG] 43.17 scrub ok 2024-09-13T15:00:16.383 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:16 smithi099 bash[84576]: cluster 2024-09-13T15:00:14.353248+0000 osd.2 (osd.2) 25 : cluster [DBG] 43.17 scrub ok 2024-09-13T15:00:16.383 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:16 smithi099 bash[84576]: cluster 2024-09-13T15:00:14.353248+0000 osd.2 (osd.2) 25 : cluster [DBG] 43.17 scrub ok 2024-09-13T15:00:16.383 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:16 smithi099 bash[87147]: cluster 2024-09-13T15:00:14.353248+0000 osd.2 (osd.2) 25 : cluster [DBG] 43.17 scrub ok 2024-09-13T15:00:16.383 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:16 smithi099 bash[87147]: cluster 2024-09-13T15:00:14.353248+0000 osd.2 (osd.2) 25 : cluster [DBG] 43.17 scrub ok 2024-09-13T15:00:16.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:16.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:16.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:16.477 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 72 16 98129 98113 85.1591 81.5 0.00997551 0.011738 2024-09-13T15:00:16.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:16.582 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:16.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:16.687 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:16.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:16.814 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:16.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:16.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:16.948 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:16.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:16.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:17.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:17.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:17.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:17.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:17.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:17 smithi177 bash[68594]: cluster 2024-09-13T15:00:15.896553+0000 osd.5 (osd.5) 11 : cluster [DBG] 43.6 scrub ok 2024-09-13T15:00:17.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:17 smithi177 bash[68594]: cluster 2024-09-13T15:00:15.896553+0000 osd.5 (osd.5) 11 : cluster [DBG] 43.6 scrub ok 2024-09-13T15:00:17.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:17 smithi177 bash[68594]: cluster 2024-09-13T15:00:16.456905+0000 mon.a (mon.0) 2131 : cluster [WRN] Health check update: Degraded data redundancy: 29824/286182 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:17.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:17 smithi177 bash[68594]: cluster 2024-09-13T15:00:16.456905+0000 mon.a (mon.0) 2131 : cluster [WRN] Health check update: Degraded data redundancy: 29824/286182 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:17.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:17 smithi177 bash[68594]: cluster 2024-09-13T15:00:16.780233+0000 mgr.y (mgr.26252) 1024 : cluster [DBG] pgmap v1560: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 13 KiB/s, 0 objects/s recovering 2024-09-13T15:00:17.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:17 smithi177 bash[68594]: cluster 2024-09-13T15:00:16.780233+0000 mgr.y (mgr.26252) 1024 : cluster [DBG] pgmap v1560: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 13 KiB/s, 0 objects/s recovering 2024-09-13T15:00:17.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:17.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:17.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:00:17] "GET /metrics HTTP/1.1" 200 36770 "" "Prometheus/2.51.0" 2024-09-13T15:00:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[84576]: cluster 2024-09-13T15:00:15.896553+0000 osd.5 (osd.5) 11 : cluster [DBG] 43.6 scrub ok 2024-09-13T15:00:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[84576]: cluster 2024-09-13T15:00:15.896553+0000 osd.5 (osd.5) 11 : cluster [DBG] 43.6 scrub ok 2024-09-13T15:00:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[84576]: cluster 2024-09-13T15:00:16.456905+0000 mon.a (mon.0) 2131 : cluster [WRN] Health check update: Degraded data redundancy: 29824/286182 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[84576]: cluster 2024-09-13T15:00:16.456905+0000 mon.a (mon.0) 2131 : cluster [WRN] Health check update: Degraded data redundancy: 29824/286182 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[84576]: cluster 2024-09-13T15:00:16.780233+0000 mgr.y (mgr.26252) 1024 : cluster [DBG] pgmap v1560: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 13 KiB/s, 0 objects/s recovering 2024-09-13T15:00:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[84576]: cluster 2024-09-13T15:00:16.780233+0000 mgr.y (mgr.26252) 1024 : cluster [DBG] pgmap v1560: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 13 KiB/s, 0 objects/s recovering 2024-09-13T15:00:17.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[87147]: cluster 2024-09-13T15:00:15.896553+0000 osd.5 (osd.5) 11 : cluster [DBG] 43.6 scrub ok 2024-09-13T15:00:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[87147]: cluster 2024-09-13T15:00:15.896553+0000 osd.5 (osd.5) 11 : cluster [DBG] 43.6 scrub ok 2024-09-13T15:00:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[87147]: cluster 2024-09-13T15:00:16.456905+0000 mon.a (mon.0) 2131 : cluster [WRN] Health check update: Degraded data redundancy: 29824/286182 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[87147]: cluster 2024-09-13T15:00:16.456905+0000 mon.a (mon.0) 2131 : cluster [WRN] Health check update: Degraded data redundancy: 29824/286182 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[87147]: cluster 2024-09-13T15:00:16.780233+0000 mgr.y (mgr.26252) 1024 : cluster [DBG] pgmap v1560: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 13 KiB/s, 0 objects/s recovering 2024-09-13T15:00:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:17 smithi099 bash[87147]: cluster 2024-09-13T15:00:16.780233+0000 mgr.y (mgr.26252) 1024 : cluster [DBG] pgmap v1560: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29824/286182 objects degraded (10.421%); 13 KiB/s, 0 objects/s recovering 2024-09-13T15:00:17.347 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:17.374 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:17.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:17.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 73 16 99443 99427 85.1175 82.125 0.00659884 0.0117437 2024-09-13T15:00:17.479 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:17.497 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:17.599 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:17.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:17.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:17.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:17.766 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:17.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:17.871 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:17.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:17.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:18.005 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:18.025 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:18.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:18.128 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:18.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:18.253 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:18.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:18.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:18 smithi099 bash[84576]: cluster 2024-09-13T15:00:17.647237+0000 osd.5 (osd.5) 12 : cluster [DBG] 43.16 scrub starts 2024-09-13T15:00:18.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:18 smithi099 bash[84576]: cluster 2024-09-13T15:00:17.647237+0000 osd.5 (osd.5) 12 : cluster [DBG] 43.16 scrub starts 2024-09-13T15:00:18.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:18 smithi099 bash[87147]: cluster 2024-09-13T15:00:17.647237+0000 osd.5 (osd.5) 12 : cluster [DBG] 43.16 scrub starts 2024-09-13T15:00:18.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:18 smithi099 bash[87147]: cluster 2024-09-13T15:00:17.647237+0000 osd.5 (osd.5) 12 : cluster [DBG] 43.16 scrub starts 2024-09-13T15:00:18.304 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:18.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:18 smithi177 bash[68594]: cluster 2024-09-13T15:00:17.647237+0000 osd.5 (osd.5) 12 : cluster [DBG] 43.16 scrub starts 2024-09-13T15:00:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:18 smithi177 bash[68594]: cluster 2024-09-13T15:00:17.647237+0000 osd.5 (osd.5) 12 : cluster [DBG] 43.16 scrub starts 2024-09-13T15:00:18.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:18.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:18.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:18.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 74 16 100189 100173 84.5972 46.625 0.0133709 0.0118161 2024-09-13T15:00:18.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:18.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:18.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:18.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:18.679 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:18.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:18.782 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:18.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:18.922 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:18.946 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:18.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:19.052 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:19.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:19.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:19.184 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:19.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:19.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:19.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:19.328 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:19 smithi177 bash[68594]: audit 2024-09-13T15:00:18.290988+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:19.328 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:19 smithi177 bash[68594]: audit 2024-09-13T15:00:18.290988+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:19.328 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:19 smithi177 bash[68594]: cluster 2024-09-13T15:00:18.609372+0000 osd.5 (osd.5) 13 : cluster [DBG] 43.11 scrub starts 2024-09-13T15:00:19.328 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:19 smithi177 bash[68594]: cluster 2024-09-13T15:00:18.609372+0000 osd.5 (osd.5) 13 : cluster [DBG] 43.11 scrub starts 2024-09-13T15:00:19.328 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:19 smithi177 bash[68594]: cluster 2024-09-13T15:00:18.780976+0000 mgr.y (mgr.26252) 1025 : cluster [DBG] pgmap v1561: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 30176/289431 objects degraded (10.426%) 2024-09-13T15:00:19.329 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:19 smithi177 bash[68594]: cluster 2024-09-13T15:00:18.780976+0000 mgr.y (mgr.26252) 1025 : cluster [DBG] pgmap v1561: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 30176/289431 objects degraded (10.426%) 2024-09-13T15:00:19.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:19.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:19.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:19.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 75 16 101395 101379 84.4742 75.375 0.00847301 0.0118328 2024-09-13T15:00:19.478 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:19.581 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:19.603 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[84576]: audit 2024-09-13T15:00:18.290988+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:19.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[84576]: audit 2024-09-13T15:00:18.290988+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:19.605 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:19.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:19.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:19.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:19.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:19.864 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:19.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:19.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[84576]: cluster 2024-09-13T15:00:18.609372+0000 osd.5 (osd.5) 13 : cluster [DBG] 43.11 scrub starts 2024-09-13T15:00:19.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[84576]: cluster 2024-09-13T15:00:18.609372+0000 osd.5 (osd.5) 13 : cluster [DBG] 43.11 scrub starts 2024-09-13T15:00:19.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[84576]: cluster 2024-09-13T15:00:18.780976+0000 mgr.y (mgr.26252) 1025 : cluster [DBG] pgmap v1561: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 30176/289431 objects degraded (10.426%) 2024-09-13T15:00:19.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[84576]: cluster 2024-09-13T15:00:18.780976+0000 mgr.y (mgr.26252) 1025 : cluster [DBG] pgmap v1561: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 30176/289431 objects degraded (10.426%) 2024-09-13T15:00:19.868 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[87147]: audit 2024-09-13T15:00:18.290988+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:19.869 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[87147]: audit 2024-09-13T15:00:18.290988+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:19.869 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[87147]: cluster 2024-09-13T15:00:18.609372+0000 osd.5 (osd.5) 13 : cluster [DBG] 43.11 scrub starts 2024-09-13T15:00:19.869 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[87147]: cluster 2024-09-13T15:00:18.609372+0000 osd.5 (osd.5) 13 : cluster [DBG] 43.11 scrub starts 2024-09-13T15:00:19.869 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[87147]: cluster 2024-09-13T15:00:18.780976+0000 mgr.y (mgr.26252) 1025 : cluster [DBG] pgmap v1561: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 30176/289431 objects degraded (10.426%) 2024-09-13T15:00:19.869 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:19 smithi099 bash[87147]: cluster 2024-09-13T15:00:18.780976+0000 mgr.y (mgr.26252) 1025 : cluster [DBG] pgmap v1561: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 30176/289431 objects degraded (10.426%) 2024-09-13T15:00:19.968 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:19.998 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:20.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:20.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.127 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:20.230 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:20.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:20 smithi099 bash[84576]: cluster 2024-09-13T15:00:19.080686+0000 osd.5 (osd.5) 14 : cluster [DBG] 43.16 scrub ok 2024-09-13T15:00:20.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:20 smithi099 bash[84576]: cluster 2024-09-13T15:00:19.080686+0000 osd.5 (osd.5) 14 : cluster [DBG] 43.16 scrub ok 2024-09-13T15:00:20.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:20 smithi099 bash[84576]: cluster 2024-09-13T15:00:19.814913+0000 osd.5 (osd.5) 15 : cluster [DBG] 43.11 scrub ok 2024-09-13T15:00:20.240 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:20 smithi099 bash[84576]: cluster 2024-09-13T15:00:19.814913+0000 osd.5 (osd.5) 15 : cluster [DBG] 43.11 scrub ok 2024-09-13T15:00:20.240 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:20 smithi099 bash[87147]: cluster 2024-09-13T15:00:19.080686+0000 osd.5 (osd.5) 14 : cluster [DBG] 43.16 scrub ok 2024-09-13T15:00:20.240 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:20 smithi099 bash[87147]: cluster 2024-09-13T15:00:19.080686+0000 osd.5 (osd.5) 14 : cluster [DBG] 43.16 scrub ok 2024-09-13T15:00:20.240 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:20 smithi099 bash[87147]: cluster 2024-09-13T15:00:19.814913+0000 osd.5 (osd.5) 15 : cluster [DBG] 43.11 scrub ok 2024-09-13T15:00:20.240 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:20 smithi099 bash[87147]: cluster 2024-09-13T15:00:19.814913+0000 osd.5 (osd.5) 15 : cluster [DBG] 43.11 scrub ok 2024-09-13T15:00:20.252 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:20.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:20.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:20.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:20.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:20.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:20 smithi177 bash[68594]: cluster 2024-09-13T15:00:19.080686+0000 osd.5 (osd.5) 14 : cluster [DBG] 43.16 scrub ok 2024-09-13T15:00:20.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:20 smithi177 bash[68594]: cluster 2024-09-13T15:00:19.080686+0000 osd.5 (osd.5) 14 : cluster [DBG] 43.16 scrub ok 2024-09-13T15:00:20.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:20 smithi177 bash[68594]: cluster 2024-09-13T15:00:19.814913+0000 osd.5 (osd.5) 15 : cluster [DBG] 43.11 scrub ok 2024-09-13T15:00:20.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:20 smithi177 bash[68594]: cluster 2024-09-13T15:00:19.814913+0000 osd.5 (osd.5) 15 : cluster [DBG] 43.11 scrub ok 2024-09-13T15:00:20.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 76 16 102052 102036 83.9029 41.0625 0.356591 0.0118974 2024-09-13T15:00:20.482 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:20.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.632 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:20.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.744 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:00:20 smithi177 bash[61439]: ts=2024-09-13T15:00:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:20.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:20.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:20.901 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:20.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:21.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:21.038 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:21.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:21.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:21 smithi099 bash[84576]: cluster 2024-09-13T15:00:20.782545+0000 mgr.y (mgr.26252) 1026 : cluster [DBG] pgmap v1562: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31611/303243 objects degraded (10.424%) 2024-09-13T15:00:21.150 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:21 smithi099 bash[84576]: cluster 2024-09-13T15:00:20.782545+0000 mgr.y (mgr.26252) 1026 : cluster [DBG] pgmap v1562: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31611/303243 objects degraded (10.424%) 2024-09-13T15:00:21.151 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:21 smithi099 bash[87147]: cluster 2024-09-13T15:00:20.782545+0000 mgr.y (mgr.26252) 1026 : cluster [DBG] pgmap v1562: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31611/303243 objects degraded (10.424%) 2024-09-13T15:00:21.151 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:21 smithi099 bash[87147]: cluster 2024-09-13T15:00:20.782545+0000 mgr.y (mgr.26252) 1026 : cluster [DBG] pgmap v1562: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31611/303243 objects degraded (10.424%) 2024-09-13T15:00:21.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:21.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:21.269 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:21.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:21.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:21 smithi177 bash[68594]: cluster 2024-09-13T15:00:20.782545+0000 mgr.y (mgr.26252) 1026 : cluster [DBG] pgmap v1562: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31611/303243 objects degraded (10.424%) 2024-09-13T15:00:21.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:21 smithi177 bash[68594]: cluster 2024-09-13T15:00:20.782545+0000 mgr.y (mgr.26252) 1026 : cluster [DBG] pgmap v1562: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31611/303243 objects degraded (10.424%) 2024-09-13T15:00:21.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:21.443 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:21.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 77 16 102945 102929 83.538 55.8125 0.00951278 0.011966 2024-09-13T15:00:21.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:21.580 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:21.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:21.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:21.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:21.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:21.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:21.834 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:21.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:21.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:21.968 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:21.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:22.071 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:22.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:22 smithi099 bash[84576]: cluster 2024-09-13T15:00:21.457594+0000 mon.a (mon.0) 2133 : cluster [WRN] Health check update: Degraded data redundancy: 31611/303243 objects degraded (10.424%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:22.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:22 smithi099 bash[84576]: cluster 2024-09-13T15:00:21.457594+0000 mon.a (mon.0) 2133 : cluster [WRN] Health check update: Degraded data redundancy: 31611/303243 objects degraded (10.424%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:22.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:22 smithi099 bash[87147]: cluster 2024-09-13T15:00:21.457594+0000 mon.a (mon.0) 2133 : cluster [WRN] Health check update: Degraded data redundancy: 31611/303243 objects degraded (10.424%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:22.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:22 smithi099 bash[87147]: cluster 2024-09-13T15:00:21.457594+0000 mon.a (mon.0) 2133 : cluster [WRN] Health check update: Degraded data redundancy: 31611/303243 objects degraded (10.424%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:22.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:22.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:22.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:22.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:22.325 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:22.333 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:22 smithi177 bash[68594]: cluster 2024-09-13T15:00:21.457594+0000 mon.a (mon.0) 2133 : cluster [WRN] Health check update: Degraded data redundancy: 31611/303243 objects degraded (10.424%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:22.334 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:22 smithi177 bash[68594]: cluster 2024-09-13T15:00:21.457594+0000 mon.a (mon.0) 2133 : cluster [WRN] Health check update: Degraded data redundancy: 31611/303243 objects degraded (10.424%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:22.344 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:22.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:22.447 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:22.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:22.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:22.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 78 16 103838 103822 83.1824 55.8125 0.00206445 0.0119767 2024-09-13T15:00:22.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:22.592 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:22.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:22.715 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:22.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:22.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:22.839 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:22.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:22.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:22.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:22.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:23.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:23.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:23.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:23.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:23.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:23 smithi099 bash[84576]: cluster 2024-09-13T15:00:22.783837+0000 mgr.y (mgr.26252) 1027 : cluster [DBG] pgmap v1563: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31869/305598 objects degraded (10.428%) 2024-09-13T15:00:23.220 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:23 smithi099 bash[84576]: cluster 2024-09-13T15:00:22.783837+0000 mgr.y (mgr.26252) 1027 : cluster [DBG] pgmap v1563: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31869/305598 objects degraded (10.428%) 2024-09-13T15:00:23.220 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:23 smithi099 bash[87147]: cluster 2024-09-13T15:00:22.783837+0000 mgr.y (mgr.26252) 1027 : cluster [DBG] pgmap v1563: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31869/305598 objects degraded (10.428%) 2024-09-13T15:00:23.220 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:23 smithi099 bash[87147]: cluster 2024-09-13T15:00:22.783837+0000 mgr.y (mgr.26252) 1027 : cluster [DBG] pgmap v1563: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31869/305598 objects degraded (10.428%) 2024-09-13T15:00:23.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:23.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:23.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:23.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:23 smithi177 bash[68594]: cluster 2024-09-13T15:00:22.783837+0000 mgr.y (mgr.26252) 1027 : cluster [DBG] pgmap v1563: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31869/305598 objects degraded (10.428%) 2024-09-13T15:00:23.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:23 smithi177 bash[68594]: cluster 2024-09-13T15:00:22.783837+0000 mgr.y (mgr.26252) 1027 : cluster [DBG] pgmap v1563: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 31869/305598 objects degraded (10.428%) 2024-09-13T15:00:23.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:23.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:23.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:23.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 79 15 104658 104643 82.779 51.3125 0.00687161 0.0120758 2024-09-13T15:00:23.491 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:23.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:23.597 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:23.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:23.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:23.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:23.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:23.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:23.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:23.891 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:23.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:23.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:24.022 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:24.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:24.138 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:24 smithi099 bash[84576]: cluster 2024-09-13T15:00:23.427372+0000 osd.7 (osd.7) 5 : cluster [DBG] 43.18 scrub starts 2024-09-13T15:00:24.138 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:24 smithi099 bash[84576]: cluster 2024-09-13T15:00:23.427372+0000 osd.7 (osd.7) 5 : cluster [DBG] 43.18 scrub starts 2024-09-13T15:00:24.138 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:24 smithi099 bash[87147]: cluster 2024-09-13T15:00:23.427372+0000 osd.7 (osd.7) 5 : cluster [DBG] 43.18 scrub starts 2024-09-13T15:00:24.138 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:24 smithi099 bash[87147]: cluster 2024-09-13T15:00:23.427372+0000 osd.7 (osd.7) 5 : cluster [DBG] 43.18 scrub starts 2024-09-13T15:00:24.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:24.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:24.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:24.288 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:24.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:24 smithi177 bash[68594]: cluster 2024-09-13T15:00:23.427372+0000 osd.7 (osd.7) 5 : cluster [DBG] 43.18 scrub starts 2024-09-13T15:00:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:24 smithi177 bash[68594]: cluster 2024-09-13T15:00:23.427372+0000 osd.7 (osd.7) 5 : cluster [DBG] 43.18 scrub starts 2024-09-13T15:00:24.392 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:24.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.419 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:24.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:24.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:00:24.479682+0000 min lat: 0.00108369 max lat: 1.34488 avg lat: 0.012113 2024-09-13T15:00:24.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:00:24.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 80 16 105659 105643 82.5254 62.5 0.0076443 0.012113 2024-09-13T15:00:24.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:24.560 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:24.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:24.664 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:24.686 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:24.757 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:00:24.757 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:00:24.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:24.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:24.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:24.925 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:24.954 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:24.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:25.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:25.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:25.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:25.197 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:25.220 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:25.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:25.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:25 smithi099 bash[84576]: cluster 2024-09-13T15:00:24.784624+0000 mgr.y (mgr.26252) 1028 : cluster [DBG] pgmap v1564: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:25.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:25 smithi099 bash[84576]: cluster 2024-09-13T15:00:24.784624+0000 mgr.y (mgr.26252) 1028 : cluster [DBG] pgmap v1564: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:25.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:25 smithi099 bash[87147]: cluster 2024-09-13T15:00:24.784624+0000 mgr.y (mgr.26252) 1028 : cluster [DBG] pgmap v1564: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:25.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:25 smithi099 bash[87147]: cluster 2024-09-13T15:00:24.784624+0000 mgr.y (mgr.26252) 1028 : cluster [DBG] pgmap v1564: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:25.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:25.350 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:25.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:25.395 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:25 smithi177 bash[68594]: cluster 2024-09-13T15:00:24.784624+0000 mgr.y (mgr.26252) 1028 : cluster [DBG] pgmap v1564: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:25.395 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:25 smithi177 bash[68594]: cluster 2024-09-13T15:00:24.784624+0000 mgr.y (mgr.26252) 1028 : cluster [DBG] pgmap v1564: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:25.455 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:25.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 81 16 106813 106797 82.3969 72.125 0.00799195 0.0121321 2024-09-13T15:00:25.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:25.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:25.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:25.612 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:25.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:25.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:25.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:25.883 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:25.903 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:25.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:26.006 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:26.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:26.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:26.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:26.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:00:26.147 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:26 smithi099 bash[84576]: cluster 2024-09-13T15:00:25.045269+0000 osd.7 (osd.7) 6 : cluster [DBG] 43.18 scrub ok 2024-09-13T15:00:26.148 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:26 smithi099 bash[84576]: cluster 2024-09-13T15:00:25.045269+0000 osd.7 (osd.7) 6 : cluster [DBG] 43.18 scrub ok 2024-09-13T15:00:26.148 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:26 smithi099 bash[87147]: cluster 2024-09-13T15:00:25.045269+0000 osd.7 (osd.7) 6 : cluster [DBG] 43.18 scrub ok 2024-09-13T15:00:26.148 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:26 smithi099 bash[87147]: cluster 2024-09-13T15:00:25.045269+0000 osd.7 (osd.7) 6 : cluster [DBG] 43.18 scrub ok 2024-09-13T15:00:26.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:26.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:26.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:26.290 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:26.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:26.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:26 smithi177 bash[68594]: cluster 2024-09-13T15:00:25.045269+0000 osd.7 (osd.7) 6 : cluster [DBG] 43.18 scrub ok 2024-09-13T15:00:26.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:26 smithi177 bash[68594]: cluster 2024-09-13T15:00:25.045269+0000 osd.7 (osd.7) 6 : cluster [DBG] 43.18 scrub ok 2024-09-13T15:00:26.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:26.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:26.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:26.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 82 16 107802 107786 82.1458 61.8125 0.00415802 0.0121601 2024-09-13T15:00:26.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:26.541 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:26.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:26.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:26.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:26.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:26.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:26.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:26.906 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:27.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:27.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.061 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:27.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:27.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:27.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:27.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:27.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:27.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:27.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:27 smithi177 bash[68594]: cluster 2024-09-13T15:00:26.458579+0000 mon.a (mon.0) 2134 : cluster [WRN] Health check update: Degraded data redundancy: 32921/316149 objects degraded (10.413%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:27 smithi177 bash[68594]: cluster 2024-09-13T15:00:26.458579+0000 mon.a (mon.0) 2134 : cluster [WRN] Health check update: Degraded data redundancy: 32921/316149 objects degraded (10.413%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:27 smithi177 bash[68594]: cluster 2024-09-13T15:00:26.785122+0000 mgr.y (mgr.26252) 1029 : cluster [DBG] pgmap v1565: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:27 smithi177 bash[68594]: cluster 2024-09-13T15:00:26.785122+0000 mgr.y (mgr.26252) 1029 : cluster [DBG] pgmap v1565: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:27.433 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:27.443 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:27 smithi099 bash[84576]: cluster 2024-09-13T15:00:26.458579+0000 mon.a (mon.0) 2134 : cluster [WRN] Health check update: Degraded data redundancy: 32921/316149 objects degraded (10.413%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:27.443 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:27 smithi099 bash[84576]: cluster 2024-09-13T15:00:26.458579+0000 mon.a (mon.0) 2134 : cluster [WRN] Health check update: Degraded data redundancy: 32921/316149 objects degraded (10.413%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:27.444 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:27 smithi099 bash[84576]: cluster 2024-09-13T15:00:26.785122+0000 mgr.y (mgr.26252) 1029 : cluster [DBG] pgmap v1565: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:27.444 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:27 smithi099 bash[84576]: cluster 2024-09-13T15:00:26.785122+0000 mgr.y (mgr.26252) 1029 : cluster [DBG] pgmap v1565: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:27.444 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:00:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:00:27] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:00:27.445 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:27 smithi099 bash[87147]: cluster 2024-09-13T15:00:26.458579+0000 mon.a (mon.0) 2134 : cluster [WRN] Health check update: Degraded data redundancy: 32921/316149 objects degraded (10.413%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:27.445 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:27 smithi099 bash[87147]: cluster 2024-09-13T15:00:26.458579+0000 mon.a (mon.0) 2134 : cluster [WRN] Health check update: Degraded data redundancy: 32921/316149 objects degraded (10.413%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:27.445 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:27 smithi099 bash[87147]: cluster 2024-09-13T15:00:26.785122+0000 mgr.y (mgr.26252) 1029 : cluster [DBG] pgmap v1565: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:27.445 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:27 smithi099 bash[87147]: cluster 2024-09-13T15:00:26.785122+0000 mgr.y (mgr.26252) 1029 : cluster [DBG] pgmap v1565: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32921/316149 objects degraded (10.413%) 2024-09-13T15:00:27.457 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:27.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:27.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 83 16 108846 108830 81.9422 65.25 0.0064687 0.0121996 2024-09-13T15:00:27.563 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:27.591 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:27.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:27.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:27.725 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:27.830 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:27.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:27.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:27.960 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:27.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:27.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:28.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:28.114 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:28.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:28.217 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:28.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:28.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:28.356 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:28.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:28.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 84 16 109796 109780 81.6735 59.375 0.014538 0.0122391 2024-09-13T15:00:28.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:28.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:28.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:28.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:28.670 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:28.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:28.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:28.797 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:28.901 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:28.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:28.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:29.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:29.041 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:29.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:29.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:29.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:29.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:29.305 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:29.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:29.409 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:29.438 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:29.478 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 85 16 110964 110948 81.5713 73 0.014187 0.0122542 2024-09-13T15:00:29.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:29.575 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:29.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:29.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:29.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:29.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:29.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:29.837 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:29.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:29.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:29.950 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:29 smithi177 bash[68594]: cluster 2024-09-13T15:00:28.785972+0000 mgr.y (mgr.26252) 1030 : cluster [DBG] pgmap v1566: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33237/319125 objects degraded (10.415%) 2024-09-13T15:00:29.950 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:29 smithi177 bash[68594]: cluster 2024-09-13T15:00:28.785972+0000 mgr.y (mgr.26252) 1030 : cluster [DBG] pgmap v1566: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33237/319125 objects degraded (10.415%) 2024-09-13T15:00:29.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:29.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:30.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:30.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:30.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:30.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:30.236 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:30.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:29 smithi099 bash[84576]: cluster 2024-09-13T15:00:28.785972+0000 mgr.y (mgr.26252) 1030 : cluster [DBG] pgmap v1566: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33237/319125 objects degraded (10.415%) 2024-09-13T15:00:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:29 smithi099 bash[84576]: cluster 2024-09-13T15:00:28.785972+0000 mgr.y (mgr.26252) 1030 : cluster [DBG] pgmap v1566: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33237/319125 objects degraded (10.415%) 2024-09-13T15:00:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:29 smithi099 bash[87147]: cluster 2024-09-13T15:00:28.785972+0000 mgr.y (mgr.26252) 1030 : cluster [DBG] pgmap v1566: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33237/319125 objects degraded (10.415%) 2024-09-13T15:00:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:29 smithi099 bash[87147]: cluster 2024-09-13T15:00:28.785972+0000 mgr.y (mgr.26252) 1030 : cluster [DBG] pgmap v1566: 99 pgs: 1 active+clean+scrubbing, 26 active+undersized, 11 active+undersized+degraded, 61 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 33237/319125 objects degraded (10.415%) 2024-09-13T15:00:30.342 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:30.366 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:30.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:30.470 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:30.478 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:00:30 smithi177 bash[61439]: ts=2024-09-13T15:00:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:30.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 86 16 112131 112115 81.4709 72.9375 0.00546772 0.0122687 2024-09-13T15:00:30.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:30.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:30.598 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:30.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:30.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:30.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:30.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.766 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:30.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:30.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:30.889 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:30.974 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:30.991 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:31.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:31.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:31.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:31.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:31.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:31.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:31.282 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:31.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:31.411 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:31.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:31.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 87 16 113073 113057 81.2111 58.875 0.0123108 0.0123093 2024-09-13T15:00:31.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:31.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:31.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:31.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:31.672 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:31.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:31.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:31.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:31.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:31.919 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:31.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:32.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:31 smithi099 bash[84576]: cluster 2024-09-13T15:00:30.786691+0000 mgr.y (mgr.26252) 1031 : cluster [DBG] pgmap v1567: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8.7 MiB/s wr, 138 op/s; 34582/331890 objects degraded (10.420%) 2024-09-13T15:00:32.034 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:31 smithi099 bash[84576]: cluster 2024-09-13T15:00:30.786691+0000 mgr.y (mgr.26252) 1031 : cluster [DBG] pgmap v1567: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8.7 MiB/s wr, 138 op/s; 34582/331890 objects degraded (10.420%) 2024-09-13T15:00:32.034 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:31 smithi099 bash[84576]: cluster 2024-09-13T15:00:31.459361+0000 mon.a (mon.0) 2135 : cluster [WRN] Health check update: Degraded data redundancy: 34582/331890 objects degraded (10.420%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:32.034 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:31 smithi099 bash[84576]: cluster 2024-09-13T15:00:31.459361+0000 mon.a (mon.0) 2135 : cluster [WRN] Health check update: Degraded data redundancy: 34582/331890 objects degraded (10.420%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:32.034 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:31 smithi099 bash[87147]: cluster 2024-09-13T15:00:30.786691+0000 mgr.y (mgr.26252) 1031 : cluster [DBG] pgmap v1567: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8.7 MiB/s wr, 138 op/s; 34582/331890 objects degraded (10.420%) 2024-09-13T15:00:32.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:31 smithi099 bash[87147]: cluster 2024-09-13T15:00:30.786691+0000 mgr.y (mgr.26252) 1031 : cluster [DBG] pgmap v1567: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8.7 MiB/s wr, 138 op/s; 34582/331890 objects degraded (10.420%) 2024-09-13T15:00:32.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:31 smithi099 bash[87147]: cluster 2024-09-13T15:00:31.459361+0000 mon.a (mon.0) 2135 : cluster [WRN] Health check update: Degraded data redundancy: 34582/331890 objects degraded (10.420%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:32.035 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:31 smithi099 bash[87147]: cluster 2024-09-13T15:00:31.459361+0000 mon.a (mon.0) 2135 : cluster [WRN] Health check update: Degraded data redundancy: 34582/331890 objects degraded (10.420%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:32.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:32.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:31 smithi177 bash[68594]: cluster 2024-09-13T15:00:30.786691+0000 mgr.y (mgr.26252) 1031 : cluster [DBG] pgmap v1567: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8.7 MiB/s wr, 138 op/s; 34582/331890 objects degraded (10.420%) 2024-09-13T15:00:32.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:31 smithi177 bash[68594]: cluster 2024-09-13T15:00:30.786691+0000 mgr.y (mgr.26252) 1031 : cluster [DBG] pgmap v1567: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8.7 MiB/s wr, 138 op/s; 34582/331890 objects degraded (10.420%) 2024-09-13T15:00:32.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:31 smithi177 bash[68594]: cluster 2024-09-13T15:00:31.459361+0000 mon.a (mon.0) 2135 : cluster [WRN] Health check update: Degraded data redundancy: 34582/331890 objects degraded (10.420%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:32.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:31 smithi177 bash[68594]: cluster 2024-09-13T15:00:31.459361+0000 mon.a (mon.0) 2135 : cluster [WRN] Health check update: Degraded data redundancy: 34582/331890 objects degraded (10.420%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:32.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.186 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:32.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.319 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:32.424 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:32.444 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:32.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:32.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 88 16 114028 114012 80.9665 59.6875 0.0036901 0.0123324 2024-09-13T15:00:32.547 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:32.570 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:32.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:32.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:32.708 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:32.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:32.831 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:32.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.060 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:33.217 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:33.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:33.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.344 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.447 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:33.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:33.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:33.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 89 16 114840 114824 80.6269 50.75 0.00729901 0.012398 2024-09-13T15:00:33.573 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:33.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:33.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:33.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.831 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:33.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:33.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:33.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:33.972 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:33 smithi099 bash[84576]: cluster 2024-09-13T15:00:32.787462+0000 mgr.y (mgr.26252) 1032 : cluster [DBG] pgmap v1568: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 21 MiB/s wr, 340 op/s; 34882/334578 objects degraded (10.426%) 2024-09-13T15:00:33.972 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:33 smithi099 bash[84576]: cluster 2024-09-13T15:00:32.787462+0000 mgr.y (mgr.26252) 1032 : cluster [DBG] pgmap v1568: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 21 MiB/s wr, 340 op/s; 34882/334578 objects degraded (10.426%) 2024-09-13T15:00:33.972 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:33 smithi099 bash[84576]: audit 2024-09-13T15:00:33.290919+0000 mon.a (mon.0) 2136 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:33.972 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:33 smithi099 bash[84576]: audit 2024-09-13T15:00:33.290919+0000 mon.a (mon.0) 2136 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:33.972 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:33 smithi099 bash[87147]: cluster 2024-09-13T15:00:32.787462+0000 mgr.y (mgr.26252) 1032 : cluster [DBG] pgmap v1568: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 21 MiB/s wr, 340 op/s; 34882/334578 objects degraded (10.426%) 2024-09-13T15:00:33.972 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:33 smithi099 bash[87147]: cluster 2024-09-13T15:00:32.787462+0000 mgr.y (mgr.26252) 1032 : cluster [DBG] pgmap v1568: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 21 MiB/s wr, 340 op/s; 34882/334578 objects degraded (10.426%) 2024-09-13T15:00:33.973 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:33 smithi099 bash[87147]: audit 2024-09-13T15:00:33.290919+0000 mon.a (mon.0) 2136 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:33.973 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:33 smithi099 bash[87147]: audit 2024-09-13T15:00:33.290919+0000 mon.a (mon.0) 2136 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:33.996 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:34.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:34.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:33 smithi177 bash[68594]: cluster 2024-09-13T15:00:32.787462+0000 mgr.y (mgr.26252) 1032 : cluster [DBG] pgmap v1568: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 21 MiB/s wr, 340 op/s; 34882/334578 objects degraded (10.426%) 2024-09-13T15:00:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:33 smithi177 bash[68594]: cluster 2024-09-13T15:00:32.787462+0000 mgr.y (mgr.26252) 1032 : cluster [DBG] pgmap v1568: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 21 MiB/s wr, 340 op/s; 34882/334578 objects degraded (10.426%) 2024-09-13T15:00:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:33 smithi177 bash[68594]: audit 2024-09-13T15:00:33.290919+0000 mon.a (mon.0) 2136 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:33 smithi177 bash[68594]: audit 2024-09-13T15:00:33.290919+0000 mon.a (mon.0) 2136 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:34.130 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:00:34.133 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:34.135 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:00:34.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:34.171 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:00:34.171 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:34.196 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:34.196 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:00:34.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:00:34.233 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:00:34.233 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:34.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:34.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:34.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:34.271 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:34.271 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:00:34.297 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:00:34.297 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:34.327 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:34.327 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:00:34.366 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:34.396 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:34.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:34.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total time run: 90.0072 2024-09-13T15:00:34.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total writes made: 116009 2024-09-13T15:00:34.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Write size: 65536 2024-09-13T15:00:34.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object size: 65536 2024-09-13T15:00:34.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Bandwidth (MB/sec): 80.5554 2024-09-13T15:00:34.479 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Bandwidth: 36.7869 2024-09-13T15:00:34.480 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max bandwidth (MB/sec): 219.125 2024-09-13T15:00:34.480 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min bandwidth (MB/sec): 23.6875 2024-09-13T15:00:34.480 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average IOPS: 1288 2024-09-13T15:00:34.480 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev IOPS: 588.59 2024-09-13T15:00:34.480 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max IOPS: 3506 2024-09-13T15:00:34.480 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min IOPS: 379 2024-09-13T15:00:34.480 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average Latency(s): 0.0124101 2024-09-13T15:00:34.480 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Latency(s): 0.0265734 2024-09-13T15:00:34.480 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max latency(s): 1.34488 2024-09-13T15:00:34.480 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min latency(s): 0.00108369 2024-09-13T15:00:34.483 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T15:00:34.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:34.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:34.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:34.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:34.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:34.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:34.758 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:34.789 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2024-09-13T15:00:34.789 INFO:teuthology.orchestra.run.smithi099.stderr:0 2024-09-13T15:00:34.789 INFO:teuthology.orchestra.run.smithi099.stderr:1 2024-09-13T15:00:34.790 INFO:teuthology.orchestra.run.smithi099.stderr:2 2024-09-13T15:00:34.790 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2024-09-13T15:00:34.790 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2024-09-13T15:00:34.790 INFO:teuthology.orchestra.run.smithi099.stderr:bench [] [] [] [] 2024-09-13T15:00:34.790 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs debug_inject_read_zeros 2024-09-13T15:00:34.790 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs files list 2024-09-13T15:00:34.790 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs stats 2024-09-13T15:00:34.790 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator dump block 2024-09-13T15:00:34.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error EINVAL: invalid command 2024-09-13T15:00:34.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:34.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:34.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:34.896 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:34 smithi177 bash[68594]: audit 2024-09-13T15:00:34.101706+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.99:0/507720507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:34.896 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:34 smithi177 bash[68594]: audit 2024-09-13T15:00:34.101706+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.99:0/507720507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:34.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:34.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:35.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:35.027 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:34 smithi099 bash[84576]: audit 2024-09-13T15:00:34.101706+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.99:0/507720507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:35.027 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:34 smithi099 bash[84576]: audit 2024-09-13T15:00:34.101706+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.99:0/507720507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:35.027 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:34 smithi099 bash[87147]: audit 2024-09-13T15:00:34.101706+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.99:0/507720507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:35.027 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:34 smithi099 bash[87147]: audit 2024-09-13T15:00:34.101706+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.99:0/507720507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:35.042 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:35.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:35.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:35.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:35.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:35.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:35.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:35.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:35.407 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:35.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:35.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:35.550 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:35.568 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:35.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:35.671 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:35.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:35.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:35.791 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:35.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:35.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:35.865 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:00:35.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:35.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:35.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:36.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:36.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:36.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:36.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:36.216 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[84576]: cluster 2024-09-13T15:00:34.788568+0000 mgr.y (mgr.26252) 1033 : cluster [DBG] pgmap v1569: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 622 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:36.216 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[84576]: cluster 2024-09-13T15:00:34.788568+0000 mgr.y (mgr.26252) 1033 : cluster [DBG] pgmap v1569: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 622 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:36.216 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[84576]: audit 2024-09-13T15:00:34.855461+0000 mon.c (mon.2) 79 : audit [INF] from='client.? 172.21.15.99:0/1036719274' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.216 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[84576]: audit 2024-09-13T15:00:34.855461+0000 mon.c (mon.2) 79 : audit [INF] from='client.? 172.21.15.99:0/1036719274' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.217 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[84576]: audit 2024-09-13T15:00:34.862086+0000 mon.a (mon.0) 2137 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.217 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[84576]: audit 2024-09-13T15:00:34.862086+0000 mon.a (mon.0) 2137 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.217 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[87147]: cluster 2024-09-13T15:00:34.788568+0000 mgr.y (mgr.26252) 1033 : cluster [DBG] pgmap v1569: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 622 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:36.217 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[87147]: cluster 2024-09-13T15:00:34.788568+0000 mgr.y (mgr.26252) 1033 : cluster [DBG] pgmap v1569: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 622 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:36.217 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[87147]: audit 2024-09-13T15:00:34.855461+0000 mon.c (mon.2) 79 : audit [INF] from='client.? 172.21.15.99:0/1036719274' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.217 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[87147]: audit 2024-09-13T15:00:34.855461+0000 mon.c (mon.2) 79 : audit [INF] from='client.? 172.21.15.99:0/1036719274' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.217 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[87147]: audit 2024-09-13T15:00:34.862086+0000 mon.a (mon.0) 2137 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.218 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:35 smithi099 bash[87147]: audit 2024-09-13T15:00:34.862086+0000 mon.a (mon.0) 2137 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:36.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:36.345 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:35 smithi177 bash[68594]: cluster 2024-09-13T15:00:34.788568+0000 mgr.y (mgr.26252) 1033 : cluster [DBG] pgmap v1569: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 622 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:35 smithi177 bash[68594]: cluster 2024-09-13T15:00:34.788568+0000 mgr.y (mgr.26252) 1033 : cluster [DBG] pgmap v1569: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 622 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:35 smithi177 bash[68594]: audit 2024-09-13T15:00:34.855461+0000 mon.c (mon.2) 79 : audit [INF] from='client.? 172.21.15.99:0/1036719274' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:35 smithi177 bash[68594]: audit 2024-09-13T15:00:34.855461+0000 mon.c (mon.2) 79 : audit [INF] from='client.? 172.21.15.99:0/1036719274' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:35 smithi177 bash[68594]: audit 2024-09-13T15:00:34.862086+0000 mon.a (mon.0) 2137 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:35 smithi177 bash[68594]: audit 2024-09-13T15:00:34.862086+0000 mon.a (mon.0) 2137 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:36.364 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:36.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:36.470 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:36.494 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:36.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:36.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:36.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:36.750 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:36.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:36.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:36.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:36.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:36.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:37.009 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:37.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[84576]: cluster 2024-09-13T15:00:35.857297+0000 mon.a (mon.0) 2138 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[84576]: cluster 2024-09-13T15:00:35.857297+0000 mon.a (mon.0) 2138 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[84576]: audit 2024-09-13T15:00:35.859174+0000 mon.a (mon.0) 2139 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[84576]: audit 2024-09-13T15:00:35.859174+0000 mon.a (mon.0) 2139 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[84576]: cluster 2024-09-13T15:00:35.866487+0000 mon.a (mon.0) 2140 : cluster [DBG] osdmap e1048: 8 total, 7 up, 8 in 2024-09-13T15:00:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[84576]: cluster 2024-09-13T15:00:35.866487+0000 mon.a (mon.0) 2140 : cluster [DBG] osdmap e1048: 8 total, 7 up, 8 in 2024-09-13T15:00:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[84576]: cluster 2024-09-13T15:00:36.460228+0000 mon.a (mon.0) 2141 : cluster [WRN] Health check update: Degraded data redundancy: 36121/346668 objects degraded (10.419%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[84576]: cluster 2024-09-13T15:00:36.460228+0000 mon.a (mon.0) 2141 : cluster [WRN] Health check update: Degraded data redundancy: 36121/346668 objects degraded (10.419%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:37.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[87147]: cluster 2024-09-13T15:00:35.857297+0000 mon.a (mon.0) 2138 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:37.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[87147]: cluster 2024-09-13T15:00:35.857297+0000 mon.a (mon.0) 2138 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:37.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[87147]: audit 2024-09-13T15:00:35.859174+0000 mon.a (mon.0) 2139 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:37.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[87147]: audit 2024-09-13T15:00:35.859174+0000 mon.a (mon.0) 2139 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:37.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[87147]: cluster 2024-09-13T15:00:35.866487+0000 mon.a (mon.0) 2140 : cluster [DBG] osdmap e1048: 8 total, 7 up, 8 in 2024-09-13T15:00:37.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[87147]: cluster 2024-09-13T15:00:35.866487+0000 mon.a (mon.0) 2140 : cluster [DBG] osdmap e1048: 8 total, 7 up, 8 in 2024-09-13T15:00:37.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[87147]: cluster 2024-09-13T15:00:36.460228+0000 mon.a (mon.0) 2141 : cluster [WRN] Health check update: Degraded data redundancy: 36121/346668 objects degraded (10.419%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:37.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:36 smithi099 bash[87147]: cluster 2024-09-13T15:00:36.460228+0000 mon.a (mon.0) 2141 : cluster [WRN] Health check update: Degraded data redundancy: 36121/346668 objects degraded (10.419%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:37.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:37.125 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:36 smithi177 bash[68594]: cluster 2024-09-13T15:00:35.857297+0000 mon.a (mon.0) 2138 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:37.125 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:36 smithi177 bash[68594]: cluster 2024-09-13T15:00:35.857297+0000 mon.a (mon.0) 2138 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:37.125 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:36 smithi177 bash[68594]: audit 2024-09-13T15:00:35.859174+0000 mon.a (mon.0) 2139 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:37.125 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:36 smithi177 bash[68594]: audit 2024-09-13T15:00:35.859174+0000 mon.a (mon.0) 2139 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:37.125 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:36 smithi177 bash[68594]: cluster 2024-09-13T15:00:35.866487+0000 mon.a (mon.0) 2140 : cluster [DBG] osdmap e1048: 8 total, 7 up, 8 in 2024-09-13T15:00:37.125 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:36 smithi177 bash[68594]: cluster 2024-09-13T15:00:35.866487+0000 mon.a (mon.0) 2140 : cluster [DBG] osdmap e1048: 8 total, 7 up, 8 in 2024-09-13T15:00:37.125 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:36 smithi177 bash[68594]: cluster 2024-09-13T15:00:36.460228+0000 mon.a (mon.0) 2141 : cluster [WRN] Health check update: Degraded data redundancy: 36121/346668 objects degraded (10.419%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:37.125 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:36 smithi177 bash[68594]: cluster 2024-09-13T15:00:36.460228+0000 mon.a (mon.0) 2141 : cluster [WRN] Health check update: Degraded data redundancy: 36121/346668 objects degraded (10.419%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:37.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:37.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:37.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:37.247 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:00:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:00:37] "GET /metrics HTTP/1.1" 200 36766 "" "Prometheus/2.51.0" 2024-09-13T15:00:37.267 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:37.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:37.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:37.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:37.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:37.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:37.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:37.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:37.667 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:37.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:37.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:37.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:37.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:37.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:37.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:37.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:38.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:38.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:37 smithi177 bash[68594]: cluster 2024-09-13T15:00:36.789300+0000 mgr.y (mgr.26252) 1034 : cluster [DBG] pgmap v1571: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 708 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:38.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:37 smithi177 bash[68594]: cluster 2024-09-13T15:00:36.789300+0000 mgr.y (mgr.26252) 1034 : cluster [DBG] pgmap v1571: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 708 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:38.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:38.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:38.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:37 smithi099 bash[84576]: cluster 2024-09-13T15:00:36.789300+0000 mgr.y (mgr.26252) 1034 : cluster [DBG] pgmap v1571: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 708 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:38.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:37 smithi099 bash[84576]: cluster 2024-09-13T15:00:36.789300+0000 mgr.y (mgr.26252) 1034 : cluster [DBG] pgmap v1571: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 708 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:38.176 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:37 smithi099 bash[87147]: cluster 2024-09-13T15:00:36.789300+0000 mgr.y (mgr.26252) 1034 : cluster [DBG] pgmap v1571: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 708 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:38.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:37 smithi099 bash[87147]: cluster 2024-09-13T15:00:36.789300+0000 mgr.y (mgr.26252) 1034 : cluster [DBG] pgmap v1571: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 708 op/s; 36121/346668 objects degraded (10.419%) 2024-09-13T15:00:38.193 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:38.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:38.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:38.330 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:38.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:38.436 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:38.437 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:38.437 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:38.457 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:38.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:38.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:38.585 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:120171 43.6 43:60d0311a:::benchmark_data_smithi099_171278_object4159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:38.578924+0000", 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.003110074, 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031486090000000001, 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:00:38.586 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:00:38.587 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120171 2024-09-13T15:00:38.587 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.587 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:00:38.587 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:00:38.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.578924+0000", 2024-09-13T15:00:38.588 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:00:38.588 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.588 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.588 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:00:38.588 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.578924+0000", 2024-09-13T15:00:38.588 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:00:38.588 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.588 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.588 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:00:38.588 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.578923+0000", 2024-09-13T15:00:38.589 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:38.589 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.589 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.589 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:00:38.589 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.578929+0000", 2024-09-13T15:00:38.590 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0950000000000001e-06 2024-09-13T15:00:38.590 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.590 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.590 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:00:38.590 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.578940+0000", 2024-09-13T15:00:38.590 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0964999999999999e-05 2024-09-13T15:00:38.590 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.590 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.590 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:00:38.591 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.578945+0000", 2024-09-13T15:00:38.591 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2660000000000001e-06 2024-09-13T15:00:38.591 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.591 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.591 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:00:38.591 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.578990+0000", 2024-09-13T15:00:38.591 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4608999999999999e-05 2024-09-13T15:00:38.591 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.579100+0000", 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109994 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.579195+0000", 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5030000000000003e-05 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.592 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.581954+0000", 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00275903 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.581960+0000", 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0619999999999997e-06 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.593 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45572.0:120159 43.a e1048/1038 43:56bf3789:::benchmark_data_smithi099_171278_object4147:head v 1048'7638, mlcod=1048'7635)", 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:38.581446+0000", 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.00058827, 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000710178, 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.581446+0000", 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.594 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.581446+0000", 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.581444+0000", 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.581453+0000", 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7090000000000008e-06 2024-09-13T15:00:38.595 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.581466+0000", 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2728000000000001e-05 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.581472+0000", 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9789999999999998e-06 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:00:38.596 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.581550+0000", 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7830000000000005e-05 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:38.581571+0000", 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1750999999999999e-05 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 2 2024-09-13T15:00:38.597 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:00:38.605 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:38.709 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:38.732 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:38.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:38.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:38.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:38.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:38.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:39.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:39.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:39.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:39.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:39.242 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:39.277 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:39.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:39.343 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T15:00:39.343 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T15:00:39.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-09-13T15:00:39.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:39.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:39.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:39.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:39.537 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:39.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:39.643 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:39.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:39.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:39.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:39.828 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:39.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:39.934 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:39.959 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:39.960 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:00:39.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:40.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:40.082 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:40.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:40.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:39 smithi177 bash[68594]: cluster 2024-09-13T15:00:38.790127+0000 mgr.y (mgr.26252) 1035 : cluster [DBG] pgmap v1572: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 697 op/s; 36055/346128 objects degraded (10.417%) 2024-09-13T15:00:40.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:39 smithi177 bash[68594]: cluster 2024-09-13T15:00:38.790127+0000 mgr.y (mgr.26252) 1035 : cluster [DBG] pgmap v1572: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 697 op/s; 36055/346128 objects degraded (10.417%) 2024-09-13T15:00:40.187 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:40.220 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:40.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:40.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:39 smithi099 bash[84576]: cluster 2024-09-13T15:00:38.790127+0000 mgr.y (mgr.26252) 1035 : cluster [DBG] pgmap v1572: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 697 op/s; 36055/346128 objects degraded (10.417%) 2024-09-13T15:00:40.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:39 smithi099 bash[84576]: cluster 2024-09-13T15:00:38.790127+0000 mgr.y (mgr.26252) 1035 : cluster [DBG] pgmap v1572: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 697 op/s; 36055/346128 objects degraded (10.417%) 2024-09-13T15:00:40.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:39 smithi099 bash[87147]: cluster 2024-09-13T15:00:38.790127+0000 mgr.y (mgr.26252) 1035 : cluster [DBG] pgmap v1572: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 697 op/s; 36055/346128 objects degraded (10.417%) 2024-09-13T15:00:40.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:39 smithi099 bash[87147]: cluster 2024-09-13T15:00:38.790127+0000 mgr.y (mgr.26252) 1035 : cluster [DBG] pgmap v1572: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 697 op/s; 36055/346128 objects degraded (10.417%) 2024-09-13T15:00:40.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:40.348 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:40.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:40.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:00:40.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:40.506 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:40.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:40.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:00:40 smithi177 bash[61439]: ts=2024-09-13T15:00:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:40.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:40.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:40.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:40.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:40.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:40.884 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:40.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:40.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:41.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:41.046 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:41.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:41.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:41.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:41.329 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:41.415 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:00:41.436 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:41.487 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:41.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:41.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:41.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:41.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:41.860 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.888 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:41.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:42.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:42.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:42.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:42.134 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:41 smithi177 bash[68594]: cluster 2024-09-13T15:00:40.790927+0000 mgr.y (mgr.26252) 1036 : cluster [DBG] pgmap v1573: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 1.02k op/s; 34809/333570 objects degraded (10.435%) 2024-09-13T15:00:42.135 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:41 smithi177 bash[68594]: cluster 2024-09-13T15:00:40.790927+0000 mgr.y (mgr.26252) 1036 : cluster [DBG] pgmap v1573: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 1.02k op/s; 34809/333570 objects degraded (10.435%) 2024-09-13T15:00:42.135 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:41 smithi177 bash[68594]: cluster 2024-09-13T15:00:41.461465+0000 mon.a (mon.0) 2142 : cluster [WRN] Health check update: Degraded data redundancy: 34809/333570 objects degraded (10.435%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:42.135 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:41 smithi177 bash[68594]: cluster 2024-09-13T15:00:41.461465+0000 mon.a (mon.0) 2142 : cluster [WRN] Health check update: Degraded data redundancy: 34809/333570 objects degraded (10.435%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:42.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:42.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:42.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:42.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:41 smithi099 bash[84576]: cluster 2024-09-13T15:00:40.790927+0000 mgr.y (mgr.26252) 1036 : cluster [DBG] pgmap v1573: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 1.02k op/s; 34809/333570 objects degraded (10.435%) 2024-09-13T15:00:42.272 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:41 smithi099 bash[84576]: cluster 2024-09-13T15:00:40.790927+0000 mgr.y (mgr.26252) 1036 : cluster [DBG] pgmap v1573: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 1.02k op/s; 34809/333570 objects degraded (10.435%) 2024-09-13T15:00:42.272 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:41 smithi099 bash[84576]: cluster 2024-09-13T15:00:41.461465+0000 mon.a (mon.0) 2142 : cluster [WRN] Health check update: Degraded data redundancy: 34809/333570 objects degraded (10.435%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:42.272 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:41 smithi099 bash[84576]: cluster 2024-09-13T15:00:41.461465+0000 mon.a (mon.0) 2142 : cluster [WRN] Health check update: Degraded data redundancy: 34809/333570 objects degraded (10.435%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:42.272 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:41 smithi099 bash[87147]: cluster 2024-09-13T15:00:40.790927+0000 mgr.y (mgr.26252) 1036 : cluster [DBG] pgmap v1573: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 1.02k op/s; 34809/333570 objects degraded (10.435%) 2024-09-13T15:00:42.272 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:41 smithi099 bash[87147]: cluster 2024-09-13T15:00:40.790927+0000 mgr.y (mgr.26252) 1036 : cluster [DBG] pgmap v1573: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 1.02k op/s; 34809/333570 objects degraded (10.435%) 2024-09-13T15:00:42.272 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:41 smithi099 bash[87147]: cluster 2024-09-13T15:00:41.461465+0000 mon.a (mon.0) 2142 : cluster [WRN] Health check update: Degraded data redundancy: 34809/333570 objects degraded (10.435%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:42.272 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:41 smithi099 bash[87147]: cluster 2024-09-13T15:00:41.461465+0000 mon.a (mon.0) 2142 : cluster [WRN] Health check update: Degraded data redundancy: 34809/333570 objects degraded (10.435%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:42.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:42.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:42.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:42.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:42.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:42.515 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:42.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:42.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:42.647 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:42.671 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:42.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:42.774 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:42.798 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:42.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:42.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:42.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:42.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:43.030 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:43.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:43.159 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:43.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:43.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:43.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:43.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:43.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:43.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:43.444 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:43.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:43.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:43.576 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:43.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:43.679 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:43.707 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:43.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:43.811 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:43.838 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:43.942 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:43.966 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:43.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:44.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:44.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:44.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:44.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:43 smithi177 bash[68594]: cluster 2024-09-13T15:00:42.791900+0000 mgr.y (mgr.26252) 1037 : cluster [DBG] pgmap v1574: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 1.05k op/s; 34423/330099 objects degraded (10.428%) 2024-09-13T15:00:44.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:43 smithi177 bash[68594]: cluster 2024-09-13T15:00:42.791900+0000 mgr.y (mgr.26252) 1037 : cluster [DBG] pgmap v1574: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 1.05k op/s; 34423/330099 objects degraded (10.428%) 2024-09-13T15:00:44.154 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:44.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:44.236 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:44.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:44.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:43 smithi099 bash[84576]: cluster 2024-09-13T15:00:42.791900+0000 mgr.y (mgr.26252) 1037 : cluster [DBG] pgmap v1574: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 1.05k op/s; 34423/330099 objects degraded (10.428%) 2024-09-13T15:00:44.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:43 smithi099 bash[84576]: cluster 2024-09-13T15:00:42.791900+0000 mgr.y (mgr.26252) 1037 : cluster [DBG] pgmap v1574: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 1.05k op/s; 34423/330099 objects degraded (10.428%) 2024-09-13T15:00:44.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:43 smithi099 bash[87147]: cluster 2024-09-13T15:00:42.791900+0000 mgr.y (mgr.26252) 1037 : cluster [DBG] pgmap v1574: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 1.05k op/s; 34423/330099 objects degraded (10.428%) 2024-09-13T15:00:44.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:43 smithi099 bash[87147]: cluster 2024-09-13T15:00:42.791900+0000 mgr.y (mgr.26252) 1037 : cluster [DBG] pgmap v1574: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 1.05k op/s; 34423/330099 objects degraded (10.428%) 2024-09-13T15:00:44.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:44.366 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:44.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:44.469 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:44.495 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:44.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:44.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:44.633 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:44.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:44.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:44.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:44.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:44.832 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:44.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:44.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:44.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:44.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:45.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:45.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:45.158 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:45.169 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:45.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:45.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:45.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:45.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:45.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:45.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:45.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:45.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:45.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:45.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:45.672 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:45.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:45.775 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:45.805 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:45.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:45.908 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:45.917 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:45 smithi177 bash[68594]: cluster 2024-09-13T15:00:44.792752+0000 mgr.y (mgr.26252) 1038 : cluster [DBG] pgmap v1575: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.6 MiB/s wr, 1.14k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:45.917 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:45 smithi177 bash[68594]: cluster 2024-09-13T15:00:44.792752+0000 mgr.y (mgr.26252) 1038 : cluster [DBG] pgmap v1575: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.6 MiB/s wr, 1.14k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:45.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:45.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:46.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:46.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:46.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:46.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:46.186 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:45 smithi099 bash[84576]: cluster 2024-09-13T15:00:44.792752+0000 mgr.y (mgr.26252) 1038 : cluster [DBG] pgmap v1575: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.6 MiB/s wr, 1.14k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:46.186 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:45 smithi099 bash[84576]: cluster 2024-09-13T15:00:44.792752+0000 mgr.y (mgr.26252) 1038 : cluster [DBG] pgmap v1575: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.6 MiB/s wr, 1.14k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:46.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:45 smithi099 bash[87147]: cluster 2024-09-13T15:00:44.792752+0000 mgr.y (mgr.26252) 1038 : cluster [DBG] pgmap v1575: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.6 MiB/s wr, 1.14k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:46.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:45 smithi099 bash[87147]: cluster 2024-09-13T15:00:44.792752+0000 mgr.y (mgr.26252) 1038 : cluster [DBG] pgmap v1575: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.6 MiB/s wr, 1.14k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:46.190 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:46.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:46.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:46.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:46.330 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:46.433 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:46.458 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:46.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:46.561 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:46.585 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:46.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:46.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:46.714 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:46.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:46.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:46.842 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:46.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:46.945 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:46.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:46.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:47.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:47.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:46 smithi099 bash[84576]: cluster 2024-09-13T15:00:46.462247+0000 mon.a (mon.0) 2143 : cluster [WRN] Health check update: Degraded data redundancy: 32865/314976 objects degraded (10.434%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:47.083 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:46 smithi099 bash[84576]: cluster 2024-09-13T15:00:46.462247+0000 mon.a (mon.0) 2143 : cluster [WRN] Health check update: Degraded data redundancy: 32865/314976 objects degraded (10.434%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:47.083 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:00:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:00:47] "GET /metrics HTTP/1.1" 200 36766 "" "Prometheus/2.51.0" 2024-09-13T15:00:47.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:46 smithi099 bash[87147]: cluster 2024-09-13T15:00:46.462247+0000 mon.a (mon.0) 2143 : cluster [WRN] Health check update: Degraded data redundancy: 32865/314976 objects degraded (10.434%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:47.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:46 smithi099 bash[87147]: cluster 2024-09-13T15:00:46.462247+0000 mon.a (mon.0) 2143 : cluster [WRN] Health check update: Degraded data redundancy: 32865/314976 objects degraded (10.434%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:47.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:47.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:47.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:47.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:47.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:47.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:47.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:47.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:47.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:46 smithi177 bash[68594]: cluster 2024-09-13T15:00:46.462247+0000 mon.a (mon.0) 2143 : cluster [WRN] Health check update: Degraded data redundancy: 32865/314976 objects degraded (10.434%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:47.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:46 smithi177 bash[68594]: cluster 2024-09-13T15:00:46.462247+0000 mon.a (mon.0) 2143 : cluster [WRN] Health check update: Degraded data redundancy: 32865/314976 objects degraded (10.434%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:47.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:47.502 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:47.609 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:47.633 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:47.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:47.737 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:47.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:47.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:47.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:47.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:47.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:47.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:48.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:48.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:48.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:48.134 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:48.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:48.173 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:47 smithi099 bash[84576]: cluster 2024-09-13T15:00:46.793301+0000 mgr.y (mgr.26252) 1039 : cluster [DBG] pgmap v1576: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.3 MiB/s wr, 1.04k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:48.173 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:47 smithi099 bash[84576]: cluster 2024-09-13T15:00:46.793301+0000 mgr.y (mgr.26252) 1039 : cluster [DBG] pgmap v1576: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.3 MiB/s wr, 1.04k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:48.174 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:47 smithi099 bash[87147]: cluster 2024-09-13T15:00:46.793301+0000 mgr.y (mgr.26252) 1039 : cluster [DBG] pgmap v1576: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.3 MiB/s wr, 1.04k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:48.174 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:47 smithi099 bash[87147]: cluster 2024-09-13T15:00:46.793301+0000 mgr.y (mgr.26252) 1039 : cluster [DBG] pgmap v1576: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.3 MiB/s wr, 1.04k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:48.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:48.247 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:47 smithi177 bash[68594]: cluster 2024-09-13T15:00:46.793301+0000 mgr.y (mgr.26252) 1039 : cluster [DBG] pgmap v1576: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.3 MiB/s wr, 1.04k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:48.247 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:47 smithi177 bash[68594]: cluster 2024-09-13T15:00:46.793301+0000 mgr.y (mgr.26252) 1039 : cluster [DBG] pgmap v1576: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.3 MiB/s wr, 1.04k op/s; 32865/314976 objects degraded (10.434%) 2024-09-13T15:00:48.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:48.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:48.391 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:48.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:48.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:48.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:48.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:48.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:48.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:48.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:48.791 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:48.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:48.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:48.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:48.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:48.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:49.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:49.085 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:49.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:49.191 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:49.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:49.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:49.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[84576]: audit 2024-09-13T15:00:48.294215+0000 mon.a (mon.0) 2144 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:49.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[84576]: audit 2024-09-13T15:00:48.294215+0000 mon.a (mon.0) 2144 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:49.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[84576]: audit 2024-09-13T15:00:48.295732+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:49.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[84576]: audit 2024-09-13T15:00:48.295732+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:49.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[84576]: cluster 2024-09-13T15:00:48.793934+0000 mgr.y (mgr.26252) 1040 : cluster [DBG] pgmap v1577: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.1 MiB/s wr, 1.03k op/s; 32545/312042 objects degraded (10.430%) 2024-09-13T15:00:49.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[84576]: cluster 2024-09-13T15:00:48.793934+0000 mgr.y (mgr.26252) 1040 : cluster [DBG] pgmap v1577: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.1 MiB/s wr, 1.03k op/s; 32545/312042 objects degraded (10.430%) 2024-09-13T15:00:49.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[87147]: audit 2024-09-13T15:00:48.294215+0000 mon.a (mon.0) 2144 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:49.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[87147]: audit 2024-09-13T15:00:48.294215+0000 mon.a (mon.0) 2144 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:49.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[87147]: audit 2024-09-13T15:00:48.295732+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:49.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[87147]: audit 2024-09-13T15:00:48.295732+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:49.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[87147]: cluster 2024-09-13T15:00:48.793934+0000 mgr.y (mgr.26252) 1040 : cluster [DBG] pgmap v1577: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.1 MiB/s wr, 1.03k op/s; 32545/312042 objects degraded (10.430%) 2024-09-13T15:00:49.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:49 smithi099 bash[87147]: cluster 2024-09-13T15:00:48.793934+0000 mgr.y (mgr.26252) 1040 : cluster [DBG] pgmap v1577: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.1 MiB/s wr, 1.03k op/s; 32545/312042 objects degraded (10.430%) 2024-09-13T15:00:49.356 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:49.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:49.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:49.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:49.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:49.590 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:49 smithi177 bash[68594]: audit 2024-09-13T15:00:48.294215+0000 mon.a (mon.0) 2144 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:49 smithi177 bash[68594]: audit 2024-09-13T15:00:48.294215+0000 mon.a (mon.0) 2144 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:00:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:49 smithi177 bash[68594]: audit 2024-09-13T15:00:48.295732+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:49 smithi177 bash[68594]: audit 2024-09-13T15:00:48.295732+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:49 smithi177 bash[68594]: cluster 2024-09-13T15:00:48.793934+0000 mgr.y (mgr.26252) 1040 : cluster [DBG] pgmap v1577: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.1 MiB/s wr, 1.03k op/s; 32545/312042 objects degraded (10.430%) 2024-09-13T15:00:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:49 smithi177 bash[68594]: cluster 2024-09-13T15:00:48.793934+0000 mgr.y (mgr.26252) 1040 : cluster [DBG] pgmap v1577: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 2.1 MiB/s wr, 1.03k op/s; 32545/312042 objects degraded (10.430%) 2024-09-13T15:00:49.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:49.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:49.717 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:49.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:49.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:49.847 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:49.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:49.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:49.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:50.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:50.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:50.109 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:50.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:50.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:50.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:50.249 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:00:50 smithi177 bash[61439]: ts=2024-09-13T15:00:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:00:50.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:50.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:50.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:50.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:50.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:50.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:50.524 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:50.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:50.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:50.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:50.758 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:50.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:50.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:50.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:50.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:51.011 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:51.033 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:51.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:51.136 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:51.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:51.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:51.267 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:51.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:51.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:51.399 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:51.444 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:51.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:51.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:51.574 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:51.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:51.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:51.714 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:51.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:51.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:51.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:51.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:51.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:51.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:51.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:52.074 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:52.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:51 smithi177 bash[68594]: cluster 2024-09-13T15:00:50.795577+0000 mgr.y (mgr.26252) 1041 : cluster [DBG] pgmap v1578: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 325 KiB/s wr, 1.27k op/s; 31331/300651 objects degraded (10.421%) 2024-09-13T15:00:52.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:51 smithi177 bash[68594]: cluster 2024-09-13T15:00:50.795577+0000 mgr.y (mgr.26252) 1041 : cluster [DBG] pgmap v1578: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 325 KiB/s wr, 1.27k op/s; 31331/300651 objects degraded (10.421%) 2024-09-13T15:00:52.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:51 smithi177 bash[68594]: cluster 2024-09-13T15:00:51.462935+0000 mon.a (mon.0) 2146 : cluster [WRN] Health check update: Degraded data redundancy: 31331/300651 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:52.084 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:51 smithi177 bash[68594]: cluster 2024-09-13T15:00:51.462935+0000 mon.a (mon.0) 2146 : cluster [WRN] Health check update: Degraded data redundancy: 31331/300651 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:52.096 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:52.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:52.199 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:52.211 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:51 smithi099 bash[87147]: cluster 2024-09-13T15:00:50.795577+0000 mgr.y (mgr.26252) 1041 : cluster [DBG] pgmap v1578: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 325 KiB/s wr, 1.27k op/s; 31331/300651 objects degraded (10.421%) 2024-09-13T15:00:52.211 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:51 smithi099 bash[87147]: cluster 2024-09-13T15:00:50.795577+0000 mgr.y (mgr.26252) 1041 : cluster [DBG] pgmap v1578: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 325 KiB/s wr, 1.27k op/s; 31331/300651 objects degraded (10.421%) 2024-09-13T15:00:52.211 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:51 smithi099 bash[87147]: cluster 2024-09-13T15:00:51.462935+0000 mon.a (mon.0) 2146 : cluster [WRN] Health check update: Degraded data redundancy: 31331/300651 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:52.211 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:51 smithi099 bash[87147]: cluster 2024-09-13T15:00:51.462935+0000 mon.a (mon.0) 2146 : cluster [WRN] Health check update: Degraded data redundancy: 31331/300651 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:52.211 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:51 smithi099 bash[84576]: cluster 2024-09-13T15:00:50.795577+0000 mgr.y (mgr.26252) 1041 : cluster [DBG] pgmap v1578: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 325 KiB/s wr, 1.27k op/s; 31331/300651 objects degraded (10.421%) 2024-09-13T15:00:52.212 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:51 smithi099 bash[84576]: cluster 2024-09-13T15:00:50.795577+0000 mgr.y (mgr.26252) 1041 : cluster [DBG] pgmap v1578: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 325 KiB/s wr, 1.27k op/s; 31331/300651 objects degraded (10.421%) 2024-09-13T15:00:52.212 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:51 smithi099 bash[84576]: cluster 2024-09-13T15:00:51.462935+0000 mon.a (mon.0) 2146 : cluster [WRN] Health check update: Degraded data redundancy: 31331/300651 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:52.212 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:51 smithi099 bash[84576]: cluster 2024-09-13T15:00:51.462935+0000 mon.a (mon.0) 2146 : cluster [WRN] Health check update: Degraded data redundancy: 31331/300651 objects degraded (10.421%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:52.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:52.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:52.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:52.361 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:52.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:52.465 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:52.495 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:52.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:52.598 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:52.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:52.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:52.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:52.747 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:00:52.747 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:00:52.747 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:00:52.748 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:00:52.748 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:00:52.752 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:52.856 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:52.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:52.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:53.006 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:53.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:53.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:53.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:53.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:53.235 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:53.263 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:53.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:53.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:53.394 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:53.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:53.500 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:53.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:53.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:53.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:53.667 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:53.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:53.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:53.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:53.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:53.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:53.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:53.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:54.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:00:54.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:53 smithi177 bash[68594]: cluster 2024-09-13T15:00:52.796484+0000 mgr.y (mgr.26252) 1042 : cluster [DBG] pgmap v1579: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 31054/298092 objects degraded (10.418%) 2024-09-13T15:00:54.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:53 smithi177 bash[68594]: cluster 2024-09-13T15:00:52.796484+0000 mgr.y (mgr.26252) 1042 : cluster [DBG] pgmap v1579: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 31054/298092 objects degraded (10.418%) 2024-09-13T15:00:54.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:54.120 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:53 smithi099 bash[84576]: cluster 2024-09-13T15:00:52.796484+0000 mgr.y (mgr.26252) 1042 : cluster [DBG] pgmap v1579: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 31054/298092 objects degraded (10.418%) 2024-09-13T15:00:54.120 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:53 smithi099 bash[84576]: cluster 2024-09-13T15:00:52.796484+0000 mgr.y (mgr.26252) 1042 : cluster [DBG] pgmap v1579: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 31054/298092 objects degraded (10.418%) 2024-09-13T15:00:54.120 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:53 smithi099 bash[87147]: cluster 2024-09-13T15:00:52.796484+0000 mgr.y (mgr.26252) 1042 : cluster [DBG] pgmap v1579: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 31054/298092 objects degraded (10.418%) 2024-09-13T15:00:54.121 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:53 smithi099 bash[87147]: cluster 2024-09-13T15:00:52.796484+0000 mgr.y (mgr.26252) 1042 : cluster [DBG] pgmap v1579: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 31054/298092 objects degraded (10.418%) 2024-09-13T15:00:54.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.255 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:54.361 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:54.384 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:54.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:54.491 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:54.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:54.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:54.620 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:54.644 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:54.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:54.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:54.777 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:54.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:54.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.899 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:55.002 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:55.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:55.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:55.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:55.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:55.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:55.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:55.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:55.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:55.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:55.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:55.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:55.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:55.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:55.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:55.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:55.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:55.808 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:55.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:55.913 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:55.938 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:55.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:56.041 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:56.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:55 smithi177 bash[68594]: cluster 2024-09-13T15:00:54.797266+0000 mgr.y (mgr.26252) 1043 : cluster [DBG] pgmap v1580: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:56.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:55 smithi177 bash[68594]: cluster 2024-09-13T15:00:54.797266+0000 mgr.y (mgr.26252) 1043 : cluster [DBG] pgmap v1580: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:56.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:55 smithi099 bash[84576]: cluster 2024-09-13T15:00:54.797266+0000 mgr.y (mgr.26252) 1043 : cluster [DBG] pgmap v1580: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:56.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:55 smithi099 bash[84576]: cluster 2024-09-13T15:00:54.797266+0000 mgr.y (mgr.26252) 1043 : cluster [DBG] pgmap v1580: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:56.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:55 smithi099 bash[87147]: cluster 2024-09-13T15:00:54.797266+0000 mgr.y (mgr.26252) 1043 : cluster [DBG] pgmap v1580: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:56.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:55 smithi099 bash[87147]: cluster 2024-09-13T15:00:54.797266+0000 mgr.y (mgr.26252) 1043 : cluster [DBG] pgmap v1580: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:56.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:56.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:56.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:56.206 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:56.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:56.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:56.346 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:56.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:56.454 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:56.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:56.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:56.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:56.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:56.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:56.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:56.753 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:56.859 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:56.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:56.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:56.992 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:57.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:57.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:57.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:56 smithi099 bash[84576]: cluster 2024-09-13T15:00:56.463614+0000 mon.a (mon.0) 2147 : cluster [WRN] Health check update: Degraded data redundancy: 29961/287673 objects degraded (10.415%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:57.042 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:56 smithi099 bash[84576]: cluster 2024-09-13T15:00:56.463614+0000 mon.a (mon.0) 2147 : cluster [WRN] Health check update: Degraded data redundancy: 29961/287673 objects degraded (10.415%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:57.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:56 smithi099 bash[87147]: cluster 2024-09-13T15:00:56.463614+0000 mon.a (mon.0) 2147 : cluster [WRN] Health check update: Degraded data redundancy: 29961/287673 objects degraded (10.415%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:57.043 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:56 smithi099 bash[87147]: cluster 2024-09-13T15:00:56.463614+0000 mon.a (mon.0) 2147 : cluster [WRN] Health check update: Degraded data redundancy: 29961/287673 objects degraded (10.415%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:57.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:57.130 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:56 smithi177 bash[68594]: cluster 2024-09-13T15:00:56.463614+0000 mon.a (mon.0) 2147 : cluster [WRN] Health check update: Degraded data redundancy: 29961/287673 objects degraded (10.415%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:57.130 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:56 smithi177 bash[68594]: cluster 2024-09-13T15:00:56.463614+0000 mon.a (mon.0) 2147 : cluster [WRN] Health check update: Degraded data redundancy: 29961/287673 objects degraded (10.415%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:57.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:57.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:57.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:57.278 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:57.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:57.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:57.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:57.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:57.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:57.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:57.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:57.573 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:00:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:00:57] "GET /metrics HTTP/1.1" 200 36771 "" "Prometheus/2.51.0" 2024-09-13T15:00:57.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:57.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:57.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:57.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:57.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:57.895 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:57.941 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:57.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:58.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:58.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:57 smithi099 bash[84576]: cluster 2024-09-13T15:00:56.797927+0000 mgr.y (mgr.26252) 1044 : cluster [DBG] pgmap v1581: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:58.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:57 smithi099 bash[84576]: cluster 2024-09-13T15:00:56.797927+0000 mgr.y (mgr.26252) 1044 : cluster [DBG] pgmap v1581: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:58.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:57 smithi099 bash[87147]: cluster 2024-09-13T15:00:56.797927+0000 mgr.y (mgr.26252) 1044 : cluster [DBG] pgmap v1581: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:58.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:57 smithi099 bash[87147]: cluster 2024-09-13T15:00:56.797927+0000 mgr.y (mgr.26252) 1044 : cluster [DBG] pgmap v1581: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:58.080 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:58.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:58.185 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:58.213 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:58.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:58.327 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:57 smithi177 bash[68594]: cluster 2024-09-13T15:00:56.797927+0000 mgr.y (mgr.26252) 1044 : cluster [DBG] pgmap v1581: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:58.327 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:57 smithi177 bash[68594]: cluster 2024-09-13T15:00:56.797927+0000 mgr.y (mgr.26252) 1044 : cluster [DBG] pgmap v1581: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 29961/287673 objects degraded (10.415%) 2024-09-13T15:00:58.344 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:58.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:58.448 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:58.470 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:58.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:58.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:58.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:58.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:58.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:58.736 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:58.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:58.839 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:58.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:58.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:58.923 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:00:58.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:59.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:59.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:59.119 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:59.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:59.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:59.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:59.266 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:00:59.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:00:59.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:59.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:59.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:59.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.531 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:59.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:59.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:59.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:59.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:59.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:59.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:59.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:59.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:59.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:00.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:00.025 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:59 smithi099 bash[84576]: cluster 2024-09-13T15:00:58.799077+0000 mgr.y (mgr.26252) 1045 : cluster [DBG] pgmap v1582: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 29645/284916 objects degraded (10.405%) 2024-09-13T15:01:00.025 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:00:59 smithi099 bash[84576]: cluster 2024-09-13T15:00:58.799077+0000 mgr.y (mgr.26252) 1045 : cluster [DBG] pgmap v1582: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 29645/284916 objects degraded (10.405%) 2024-09-13T15:01:00.025 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:59 smithi099 bash[87147]: cluster 2024-09-13T15:00:58.799077+0000 mgr.y (mgr.26252) 1045 : cluster [DBG] pgmap v1582: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 29645/284916 objects degraded (10.405%) 2024-09-13T15:01:00.025 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:00:59 smithi099 bash[87147]: cluster 2024-09-13T15:00:58.799077+0000 mgr.y (mgr.26252) 1045 : cluster [DBG] pgmap v1582: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 29645/284916 objects degraded (10.405%) 2024-09-13T15:01:00.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:00.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:00.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:00.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:59 smithi177 bash[68594]: cluster 2024-09-13T15:00:58.799077+0000 mgr.y (mgr.26252) 1045 : cluster [DBG] pgmap v1582: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 29645/284916 objects degraded (10.405%) 2024-09-13T15:01:00.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:00:59 smithi177 bash[68594]: cluster 2024-09-13T15:00:58.799077+0000 mgr.y (mgr.26252) 1045 : cluster [DBG] pgmap v1582: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 29645/284916 objects degraded (10.405%) 2024-09-13T15:01:00.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:00.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:00.280 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:00.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:00.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:00.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:00.437 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:00.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:00.540 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:00.565 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:00.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:00.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:01:00 smithi177 bash[61439]: ts=2024-09-13T15:01:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:00.667 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:00.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:00.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:00.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:00.836 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:00.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:00.941 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:00.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:00.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:01.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:01.113 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:01.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:01.218 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:01.242 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:01.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:01.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:01.378 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:01.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:01.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:01.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:01.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:01.547 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2024-09-13T15:01:01.547 INFO:teuthology.orchestra.run.smithi099.stderr:0 2024-09-13T15:01:01.547 INFO:teuthology.orchestra.run.smithi099.stderr:1 2024-09-13T15:01:01.547 INFO:teuthology.orchestra.run.smithi099.stderr:2 2024-09-13T15:01:01.547 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2024-09-13T15:01:01.547 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2024-09-13T15:01:01.548 INFO:teuthology.orchestra.run.smithi099.stderr:bench [] [] [] [] 2024-09-13T15:01:01.548 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs debug_inject_read_zeros 2024-09-13T15:01:01.548 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs files list 2024-09-13T15:01:01.548 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs stats 2024-09-13T15:01:01.548 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator dump block 2024-09-13T15:01:01.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error EINVAL: invalid command 2024-09-13T15:01:01.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:01.639 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:01.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:01.755 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:01.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:01.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:01.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:01.935 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:01.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:02.029 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:01:02.029 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:02.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:02.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:01 smithi177 bash[68594]: cluster 2024-09-13T15:01:00.800122+0000 mgr.y (mgr.26252) 1046 : cluster [DBG] pgmap v1583: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 28311/272349 objects degraded (10.395%) 2024-09-13T15:01:02.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:01 smithi177 bash[68594]: cluster 2024-09-13T15:01:00.800122+0000 mgr.y (mgr.26252) 1046 : cluster [DBG] pgmap v1583: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 28311/272349 objects degraded (10.395%) 2024-09-13T15:01:02.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:01 smithi177 bash[68594]: cluster 2024-09-13T15:01:01.464401+0000 mon.a (mon.0) 2148 : cluster [WRN] Health check update: Degraded data redundancy: 28311/272349 objects degraded (10.395%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:02.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:01 smithi177 bash[68594]: cluster 2024-09-13T15:01:01.464401+0000 mon.a (mon.0) 2148 : cluster [WRN] Health check update: Degraded data redundancy: 28311/272349 objects degraded (10.395%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:02.059 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:01:02.060 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:02.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:02.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:02.086 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:01:02.087 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:02.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:01:02.120 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:01:02.120 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:02.147 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:01:02.148 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:02.172 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:01:02.172 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:02.181 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:02.200 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:02.200 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:01:02.204 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:01 smithi099 bash[84576]: cluster 2024-09-13T15:01:00.800122+0000 mgr.y (mgr.26252) 1046 : cluster [DBG] pgmap v1583: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 28311/272349 objects degraded (10.395%) 2024-09-13T15:01:02.204 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:01 smithi099 bash[84576]: cluster 2024-09-13T15:01:00.800122+0000 mgr.y (mgr.26252) 1046 : cluster [DBG] pgmap v1583: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 28311/272349 objects degraded (10.395%) 2024-09-13T15:01:02.204 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:01 smithi099 bash[84576]: cluster 2024-09-13T15:01:01.464401+0000 mon.a (mon.0) 2148 : cluster [WRN] Health check update: Degraded data redundancy: 28311/272349 objects degraded (10.395%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:02.204 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:01 smithi099 bash[84576]: cluster 2024-09-13T15:01:01.464401+0000 mon.a (mon.0) 2148 : cluster [WRN] Health check update: Degraded data redundancy: 28311/272349 objects degraded (10.395%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:02.205 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:01 smithi099 bash[87147]: cluster 2024-09-13T15:01:00.800122+0000 mgr.y (mgr.26252) 1046 : cluster [DBG] pgmap v1583: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 28311/272349 objects degraded (10.395%) 2024-09-13T15:01:02.205 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:01 smithi099 bash[87147]: cluster 2024-09-13T15:01:00.800122+0000 mgr.y (mgr.26252) 1046 : cluster [DBG] pgmap v1583: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 28311/272349 objects degraded (10.395%) 2024-09-13T15:01:02.205 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:01 smithi099 bash[87147]: cluster 2024-09-13T15:01:01.464401+0000 mon.a (mon.0) 2148 : cluster [WRN] Health check update: Degraded data redundancy: 28311/272349 objects degraded (10.395%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:02.205 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:01 smithi099 bash[87147]: cluster 2024-09-13T15:01:01.464401+0000 mon.a (mon.0) 2148 : cluster [WRN] Health check update: Degraded data redundancy: 28311/272349 objects degraded (10.395%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:02.217 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:02.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:02.322 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:02.353 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:02.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:02.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:02.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:02.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:02.594 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:02.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:02.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:02.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:02.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:02.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:02.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:02.882 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:02.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:02.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:02.994 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:02 smithi177 bash[68594]: audit 2024-09-13T15:01:02.001035+0000 mon.a (mon.0) 2149 : audit [DBG] from='client.? 172.21.15.99:0/1334438512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:02.994 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:02 smithi177 bash[68594]: audit 2024-09-13T15:01:02.001035+0000 mon.a (mon.0) 2149 : audit [DBG] from='client.? 172.21.15.99:0/1334438512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:03.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:03.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:03.127 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:02 smithi099 bash[84576]: audit 2024-09-13T15:01:02.001035+0000 mon.a (mon.0) 2149 : audit [DBG] from='client.? 172.21.15.99:0/1334438512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:03.127 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:02 smithi099 bash[84576]: audit 2024-09-13T15:01:02.001035+0000 mon.a (mon.0) 2149 : audit [DBG] from='client.? 172.21.15.99:0/1334438512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:03.127 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:02 smithi099 bash[87147]: audit 2024-09-13T15:01:02.001035+0000 mon.a (mon.0) 2149 : audit [DBG] from='client.? 172.21.15.99:0/1334438512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:03.127 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:02 smithi099 bash[87147]: audit 2024-09-13T15:01:02.001035+0000 mon.a (mon.0) 2149 : audit [DBG] from='client.? 172.21.15.99:0/1334438512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:03.149 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:03.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:03.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:03.278 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:03.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:03.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:03.406 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:03.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:03.509 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:03.532 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:03.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:03.639 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:03.660 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:03.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:03.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:03.787 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:03.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:03.878 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T15:01:03.890 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:03.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:03.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:04.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:04.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:04.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:04.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:04.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:04.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:04.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:03 smithi177 bash[68594]: cluster 2024-09-13T15:01:02.800935+0000 mgr.y (mgr.26252) 1047 : cluster [DBG] pgmap v1584: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 27965/269286 objects degraded (10.385%) 2024-09-13T15:01:04.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:03 smithi177 bash[68594]: cluster 2024-09-13T15:01:02.800935+0000 mgr.y (mgr.26252) 1047 : cluster [DBG] pgmap v1584: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 27965/269286 objects degraded (10.385%) 2024-09-13T15:01:04.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:03 smithi177 bash[68594]: audit 2024-09-13T15:01:03.288906+0000 mon.a (mon.0) 2150 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:04.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:03 smithi177 bash[68594]: audit 2024-09-13T15:01:03.288906+0000 mon.a (mon.0) 2150 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:04.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:03 smithi177 bash[68594]: audit 2024-09-13T15:01:03.293366+0000 mon.a (mon.0) 2151 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:04.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:03 smithi177 bash[68594]: audit 2024-09-13T15:01:03.293366+0000 mon.a (mon.0) 2151 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:04.295 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:04.314 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:04.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:04.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:03 smithi099 bash[87147]: cluster 2024-09-13T15:01:02.800935+0000 mgr.y (mgr.26252) 1047 : cluster [DBG] pgmap v1584: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 27965/269286 objects degraded (10.385%) 2024-09-13T15:01:04.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:03 smithi099 bash[87147]: cluster 2024-09-13T15:01:02.800935+0000 mgr.y (mgr.26252) 1047 : cluster [DBG] pgmap v1584: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 27965/269286 objects degraded (10.385%) 2024-09-13T15:01:04.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:03 smithi099 bash[87147]: audit 2024-09-13T15:01:03.288906+0000 mon.a (mon.0) 2150 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:04.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:03 smithi099 bash[87147]: audit 2024-09-13T15:01:03.288906+0000 mon.a (mon.0) 2150 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:04.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:03 smithi099 bash[87147]: audit 2024-09-13T15:01:03.293366+0000 mon.a (mon.0) 2151 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:04.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:03 smithi099 bash[87147]: audit 2024-09-13T15:01:03.293366+0000 mon.a (mon.0) 2151 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:04.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:04.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:04.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:04.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:04.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:04.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:04.677 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:04.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:04.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:04.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:04.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: cluster 2024-09-13T15:01:02.800935+0000 mgr.y (mgr.26252) 1047 : cluster [DBG] pgmap v1584: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 27965/269286 objects degraded (10.385%) 2024-09-13T15:01:04.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: cluster 2024-09-13T15:01:02.800935+0000 mgr.y (mgr.26252) 1047 : cluster [DBG] pgmap v1584: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 27965/269286 objects degraded (10.385%) 2024-09-13T15:01:04.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: audit 2024-09-13T15:01:03.288906+0000 mon.a (mon.0) 2150 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:04.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: audit 2024-09-13T15:01:03.288906+0000 mon.a (mon.0) 2150 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:04.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: audit 2024-09-13T15:01:03.293366+0000 mon.a (mon.0) 2151 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:04.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: audit 2024-09-13T15:01:03.293366+0000 mon.a (mon.0) 2151 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:04.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:04.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:04.940 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:04.954 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:04 smithi177 bash[68594]: cluster 2024-09-13T15:01:03.877459+0000 mon.a (mon.0) 2152 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:04.954 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:04 smithi177 bash[68594]: cluster 2024-09-13T15:01:03.877459+0000 mon.a (mon.0) 2152 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:04.954 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:04 smithi177 bash[68594]: audit 2024-09-13T15:01:03.878900+0000 mon.a (mon.0) 2153 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:04.955 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:04 smithi177 bash[68594]: audit 2024-09-13T15:01:03.878900+0000 mon.a (mon.0) 2153 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:04.955 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:04 smithi177 bash[68594]: cluster 2024-09-13T15:01:03.885494+0000 mon.a (mon.0) 2154 : cluster [DBG] osdmap e1049: 8 total, 7 up, 8 in 2024-09-13T15:01:04.955 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:04 smithi177 bash[68594]: cluster 2024-09-13T15:01:03.885494+0000 mon.a (mon.0) 2154 : cluster [DBG] osdmap e1049: 8 total, 7 up, 8 in 2024-09-13T15:01:04.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:05.055 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T15:01:05.055 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T15:01:05.055 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-09-13T15:01:05.075 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:05.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:05.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:05.205 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:05.238 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:05.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: cluster 2024-09-13T15:01:03.877459+0000 mon.a (mon.0) 2152 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:05.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: cluster 2024-09-13T15:01:03.877459+0000 mon.a (mon.0) 2152 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:05.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: audit 2024-09-13T15:01:03.878900+0000 mon.a (mon.0) 2153 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:05.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: audit 2024-09-13T15:01:03.878900+0000 mon.a (mon.0) 2153 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:05.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: cluster 2024-09-13T15:01:03.885494+0000 mon.a (mon.0) 2154 : cluster [DBG] osdmap e1049: 8 total, 7 up, 8 in 2024-09-13T15:01:05.261 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[84576]: cluster 2024-09-13T15:01:03.885494+0000 mon.a (mon.0) 2154 : cluster [DBG] osdmap e1049: 8 total, 7 up, 8 in 2024-09-13T15:01:05.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[87147]: cluster 2024-09-13T15:01:03.877459+0000 mon.a (mon.0) 2152 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:05.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[87147]: cluster 2024-09-13T15:01:03.877459+0000 mon.a (mon.0) 2152 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:05.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[87147]: audit 2024-09-13T15:01:03.878900+0000 mon.a (mon.0) 2153 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:05.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[87147]: audit 2024-09-13T15:01:03.878900+0000 mon.a (mon.0) 2153 : audit [INF] from='client.? 172.21.15.99:0/2833387574' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:05.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[87147]: cluster 2024-09-13T15:01:03.885494+0000 mon.a (mon.0) 2154 : cluster [DBG] osdmap e1049: 8 total, 7 up, 8 in 2024-09-13T15:01:05.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:04 smithi099 bash[87147]: cluster 2024-09-13T15:01:03.885494+0000 mon.a (mon.0) 2154 : cluster [DBG] osdmap e1049: 8 total, 7 up, 8 in 2024-09-13T15:01:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:01:05.262 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:01:05.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:05.371 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:05.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:05.474 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:05.502 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:05.608 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:05.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:05.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:05.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:05.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:05.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:05.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:05.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:06.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:05 smithi099 bash[87147]: cluster 2024-09-13T15:01:04.801620+0000 mgr.y (mgr.26252) 1048 : cluster [DBG] pgmap v1586: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:06.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:05 smithi099 bash[87147]: cluster 2024-09-13T15:01:04.801620+0000 mgr.y (mgr.26252) 1048 : cluster [DBG] pgmap v1586: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:06.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:05 smithi099 bash[84576]: cluster 2024-09-13T15:01:04.801620+0000 mgr.y (mgr.26252) 1048 : cluster [DBG] pgmap v1586: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:06.005 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:05 smithi099 bash[84576]: cluster 2024-09-13T15:01:04.801620+0000 mgr.y (mgr.26252) 1048 : cluster [DBG] pgmap v1586: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:06.026 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:06.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:06.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:05 smithi177 bash[68594]: cluster 2024-09-13T15:01:04.801620+0000 mgr.y (mgr.26252) 1048 : cluster [DBG] pgmap v1586: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:06.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:05 smithi177 bash[68594]: cluster 2024-09-13T15:01:04.801620+0000 mgr.y (mgr.26252) 1048 : cluster [DBG] pgmap v1586: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:06.131 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:06.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:06.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:06.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:06.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:06.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:06.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:06.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:06.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:06.544 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:06.544 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:06.560 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:06.665 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:06.696 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:06.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:06.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:06.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:06.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:06.928 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:06.958 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:06.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:07.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:07.071 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:06 smithi177 bash[68594]: cluster 2024-09-13T15:01:06.465103+0000 mon.a (mon.0) 2155 : cluster [WRN] Health check update: Degraded data redundancy: 26651/256260 objects degraded (10.400%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:07.071 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:06 smithi177 bash[68594]: cluster 2024-09-13T15:01:06.465103+0000 mon.a (mon.0) 2155 : cluster [WRN] Health check update: Degraded data redundancy: 26651/256260 objects degraded (10.400%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:07.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:01:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:01:07] "GET /metrics HTTP/1.1" 200 36770 "" "Prometheus/2.51.0" 2024-09-13T15:01:07.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:06 smithi099 bash[84576]: cluster 2024-09-13T15:01:06.465103+0000 mon.a (mon.0) 2155 : cluster [WRN] Health check update: Degraded data redundancy: 26651/256260 objects degraded (10.400%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:07.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:06 smithi099 bash[84576]: cluster 2024-09-13T15:01:06.465103+0000 mon.a (mon.0) 2155 : cluster [WRN] Health check update: Degraded data redundancy: 26651/256260 objects degraded (10.400%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:07.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:06 smithi099 bash[87147]: cluster 2024-09-13T15:01:06.465103+0000 mon.a (mon.0) 2155 : cluster [WRN] Health check update: Degraded data redundancy: 26651/256260 objects degraded (10.400%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:07.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:06 smithi099 bash[87147]: cluster 2024-09-13T15:01:06.465103+0000 mon.a (mon.0) 2155 : cluster [WRN] Health check update: Degraded data redundancy: 26651/256260 objects degraded (10.400%), 11 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:07.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:07.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:07.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:07.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:07.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:07.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:01:07.336 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:07.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:07.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:07.469 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:07.496 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:07.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:07.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:07.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:07.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:07.737 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:07.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:07.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:07.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:07.896 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:07.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:08.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:08.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:08.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:01:08.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:08.135 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:07 smithi099 bash[84576]: cluster 2024-09-13T15:01:06.802167+0000 mgr.y (mgr.26252) 1049 : cluster [DBG] pgmap v1587: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:08.135 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:07 smithi099 bash[84576]: cluster 2024-09-13T15:01:06.802167+0000 mgr.y (mgr.26252) 1049 : cluster [DBG] pgmap v1587: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:08.135 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:07 smithi099 bash[87147]: cluster 2024-09-13T15:01:06.802167+0000 mgr.y (mgr.26252) 1049 : cluster [DBG] pgmap v1587: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:08.136 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:07 smithi099 bash[87147]: cluster 2024-09-13T15:01:06.802167+0000 mgr.y (mgr.26252) 1049 : cluster [DBG] pgmap v1587: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:08.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:08.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:08.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:01:08.254 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:08.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:08.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:07 smithi177 bash[68594]: cluster 2024-09-13T15:01:06.802167+0000 mgr.y (mgr.26252) 1049 : cluster [DBG] pgmap v1587: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:07 smithi177 bash[68594]: cluster 2024-09-13T15:01:06.802167+0000 mgr.y (mgr.26252) 1049 : cluster [DBG] pgmap v1587: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 26651/256260 objects degraded (10.400%) 2024-09-13T15:01:08.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:08.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:08.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:08.537 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:08.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:08.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:08.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:08.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:08.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:08.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:08.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:08.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:08.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:08.938 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:08.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:09.043 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:09.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:09.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:09.181 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:09.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:09.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:09.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:09.341 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:09.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:09.445 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:09.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:09.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:09.585 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:09.607 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:09.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:09.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:09.738 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:09.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:09.840 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:09.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:09.865 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:09.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:09.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:09.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:09 smithi177 bash[68594]: cluster 2024-09-13T15:01:08.802834+0000 mgr.y (mgr.26252) 1050 : cluster [DBG] pgmap v1588: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 26285/253056 objects degraded (10.387%) 2024-09-13T15:01:09.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:09 smithi177 bash[68594]: cluster 2024-09-13T15:01:08.802834+0000 mgr.y (mgr.26252) 1050 : cluster [DBG] pgmap v1588: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 26285/253056 objects degraded (10.387%) 2024-09-13T15:01:09.998 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:10.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:10.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:10.112 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:09 smithi099 bash[84576]: cluster 2024-09-13T15:01:08.802834+0000 mgr.y (mgr.26252) 1050 : cluster [DBG] pgmap v1588: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 26285/253056 objects degraded (10.387%) 2024-09-13T15:01:10.112 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:09 smithi099 bash[84576]: cluster 2024-09-13T15:01:08.802834+0000 mgr.y (mgr.26252) 1050 : cluster [DBG] pgmap v1588: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 26285/253056 objects degraded (10.387%) 2024-09-13T15:01:10.113 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:09 smithi099 bash[87147]: cluster 2024-09-13T15:01:08.802834+0000 mgr.y (mgr.26252) 1050 : cluster [DBG] pgmap v1588: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 26285/253056 objects degraded (10.387%) 2024-09-13T15:01:10.113 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:09 smithi099 bash[87147]: cluster 2024-09-13T15:01:08.802834+0000 mgr.y (mgr.26252) 1050 : cluster [DBG] pgmap v1588: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 26285/253056 objects degraded (10.387%) 2024-09-13T15:01:10.130 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:10.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:10.235 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:10.244 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:01:10 smithi177 bash[61439]: ts=2024-09-13T15:01:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:10.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:10.376 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:10.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:10.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:10.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:10.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:10.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:10.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:10.683 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:10.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:10.786 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:10.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:10.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:10.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:10.943 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:10.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:11.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:11.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:11.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:11.180 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:11.209 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:11.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:11.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:11.341 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:11.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:11.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:11.467 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:11.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:11.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:11.604 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:11.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:11.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:11.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:11.848 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:11.870 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:11.877 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:11.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:11.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:12.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:12.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:11 smithi177 bash[68594]: cluster 2024-09-13T15:01:10.803543+0000 mgr.y (mgr.26252) 1051 : cluster [DBG] pgmap v1589: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 24842/239187 objects degraded (10.386%) 2024-09-13T15:01:12.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:11 smithi177 bash[68594]: cluster 2024-09-13T15:01:10.803543+0000 mgr.y (mgr.26252) 1051 : cluster [DBG] pgmap v1589: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 24842/239187 objects degraded (10.386%) 2024-09-13T15:01:12.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:11 smithi177 bash[68594]: cluster 2024-09-13T15:01:11.465962+0000 mon.a (mon.0) 2156 : cluster [WRN] Health check update: Degraded data redundancy: 24842/239187 objects degraded (10.386%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:12.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:11 smithi177 bash[68594]: cluster 2024-09-13T15:01:11.465962+0000 mon.a (mon.0) 2156 : cluster [WRN] Health check update: Degraded data redundancy: 24842/239187 objects degraded (10.386%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:12.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:12.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:12.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:12.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:12.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:11 smithi099 bash[84576]: cluster 2024-09-13T15:01:10.803543+0000 mgr.y (mgr.26252) 1051 : cluster [DBG] pgmap v1589: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 24842/239187 objects degraded (10.386%) 2024-09-13T15:01:12.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:11 smithi099 bash[84576]: cluster 2024-09-13T15:01:10.803543+0000 mgr.y (mgr.26252) 1051 : cluster [DBG] pgmap v1589: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 24842/239187 objects degraded (10.386%) 2024-09-13T15:01:12.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:11 smithi099 bash[84576]: cluster 2024-09-13T15:01:11.465962+0000 mon.a (mon.0) 2156 : cluster [WRN] Health check update: Degraded data redundancy: 24842/239187 objects degraded (10.386%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:12.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:11 smithi099 bash[84576]: cluster 2024-09-13T15:01:11.465962+0000 mon.a (mon.0) 2156 : cluster [WRN] Health check update: Degraded data redundancy: 24842/239187 objects degraded (10.386%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:12.260 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:11 smithi099 bash[87147]: cluster 2024-09-13T15:01:10.803543+0000 mgr.y (mgr.26252) 1051 : cluster [DBG] pgmap v1589: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 24842/239187 objects degraded (10.386%) 2024-09-13T15:01:12.260 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:11 smithi099 bash[87147]: cluster 2024-09-13T15:01:10.803543+0000 mgr.y (mgr.26252) 1051 : cluster [DBG] pgmap v1589: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 24842/239187 objects degraded (10.386%) 2024-09-13T15:01:12.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:11 smithi099 bash[87147]: cluster 2024-09-13T15:01:11.465962+0000 mon.a (mon.0) 2156 : cluster [WRN] Health check update: Degraded data redundancy: 24842/239187 objects degraded (10.386%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:12.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:11 smithi099 bash[87147]: cluster 2024-09-13T15:01:11.465962+0000 mon.a (mon.0) 2156 : cluster [WRN] Health check update: Degraded data redundancy: 24842/239187 objects degraded (10.386%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:12.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:12.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:12.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:12.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:12.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:12.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:12.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:12.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:12.671 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:12.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:12.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:12.801 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:12.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:12.886 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:12.906 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:12.911 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:12.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:12.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:13.054 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:13.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:13.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:13.176 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:13.200 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:13.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:13.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:13.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:13.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:13.449 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:13.497 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:13.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:13.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:13.634 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:13.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:13.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:13.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:13.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:13.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:13.880 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:13.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:13.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:14.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:14.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:14.111 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:14.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:14.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:14.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:14.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:13 smithi099 bash[84576]: cluster 2024-09-13T15:01:12.804416+0000 mgr.y (mgr.26252) 1052 : cluster [DBG] pgmap v1590: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 24446/235806 objects degraded (10.367%) 2024-09-13T15:01:14.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:13 smithi099 bash[84576]: cluster 2024-09-13T15:01:12.804416+0000 mgr.y (mgr.26252) 1052 : cluster [DBG] pgmap v1590: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 24446/235806 objects degraded (10.367%) 2024-09-13T15:01:14.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:13 smithi099 bash[84576]: audit 2024-09-13T15:01:13.234781+0000 mon.a (mon.0) 2157 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:14.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:13 smithi099 bash[84576]: audit 2024-09-13T15:01:13.234781+0000 mon.a (mon.0) 2157 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:14.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:13 smithi099 bash[87147]: cluster 2024-09-13T15:01:12.804416+0000 mgr.y (mgr.26252) 1052 : cluster [DBG] pgmap v1590: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 24446/235806 objects degraded (10.367%) 2024-09-13T15:01:14.321 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:13 smithi099 bash[87147]: cluster 2024-09-13T15:01:12.804416+0000 mgr.y (mgr.26252) 1052 : cluster [DBG] pgmap v1590: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 24446/235806 objects degraded (10.367%) 2024-09-13T15:01:14.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:13 smithi099 bash[87147]: audit 2024-09-13T15:01:13.234781+0000 mon.a (mon.0) 2157 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:14.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:13 smithi099 bash[87147]: audit 2024-09-13T15:01:13.234781+0000 mon.a (mon.0) 2157 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:14.365 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:13 smithi177 bash[68594]: cluster 2024-09-13T15:01:12.804416+0000 mgr.y (mgr.26252) 1052 : cluster [DBG] pgmap v1590: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 24446/235806 objects degraded (10.367%) 2024-09-13T15:01:14.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:13 smithi177 bash[68594]: cluster 2024-09-13T15:01:12.804416+0000 mgr.y (mgr.26252) 1052 : cluster [DBG] pgmap v1590: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 24446/235806 objects degraded (10.367%) 2024-09-13T15:01:14.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:13 smithi177 bash[68594]: audit 2024-09-13T15:01:13.234781+0000 mon.a (mon.0) 2157 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:14.366 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:13 smithi177 bash[68594]: audit 2024-09-13T15:01:13.234781+0000 mon.a (mon.0) 2157 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:14.374 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:14.398 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:14.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:14.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:14.518 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:14.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:14.621 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:14.645 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:14.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:14.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:14.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:14.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:14.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:14.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:15.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:15.029 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:15.140 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:15.169 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:15.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:15.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:15.312 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:15.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:15.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:15.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:15.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:15.552 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:15.582 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:15.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:15.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:15.725 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:15.834 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:15.861 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:15.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:15.964 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:15.973 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:15 smithi099 bash[84576]: cluster 2024-09-13T15:01:14.805377+0000 mgr.y (mgr.26252) 1053 : cluster [DBG] pgmap v1591: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.42k op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:15.973 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:15 smithi099 bash[84576]: cluster 2024-09-13T15:01:14.805377+0000 mgr.y (mgr.26252) 1053 : cluster [DBG] pgmap v1591: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.42k op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:15.973 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:15 smithi099 bash[87147]: cluster 2024-09-13T15:01:14.805377+0000 mgr.y (mgr.26252) 1053 : cluster [DBG] pgmap v1591: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.42k op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:15.974 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:15 smithi099 bash[87147]: cluster 2024-09-13T15:01:14.805377+0000 mgr.y (mgr.26252) 1053 : cluster [DBG] pgmap v1591: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.42k op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:15.987 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:15.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:16.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:16.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:15 smithi177 bash[68594]: cluster 2024-09-13T15:01:14.805377+0000 mgr.y (mgr.26252) 1053 : cluster [DBG] pgmap v1591: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.42k op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:16.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:15 smithi177 bash[68594]: cluster 2024-09-13T15:01:14.805377+0000 mgr.y (mgr.26252) 1053 : cluster [DBG] pgmap v1591: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.42k op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:16.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:16.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:16.216 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:16.234 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:16.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:16.337 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:16.370 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:16.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:16.473 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:16.502 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:16.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:16.608 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:16.634 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:16.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:16.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:16.768 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:16.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:16.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:16.907 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:16.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:17.012 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:17.046 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:17.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:16 smithi099 bash[84576]: cluster 2024-09-13T15:01:16.466671+0000 mon.a (mon.0) 2158 : cluster [WRN] Health check update: Degraded data redundancy: 23076/222867 objects degraded (10.354%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:17.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:16 smithi099 bash[84576]: cluster 2024-09-13T15:01:16.466671+0000 mon.a (mon.0) 2158 : cluster [WRN] Health check update: Degraded data redundancy: 23076/222867 objects degraded (10.354%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:16 smithi099 bash[87147]: cluster 2024-09-13T15:01:16.466671+0000 mon.a (mon.0) 2158 : cluster [WRN] Health check update: Degraded data redundancy: 23076/222867 objects degraded (10.354%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:17.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:16 smithi099 bash[87147]: cluster 2024-09-13T15:01:16.466671+0000 mon.a (mon.0) 2158 : cluster [WRN] Health check update: Degraded data redundancy: 23076/222867 objects degraded (10.354%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:17.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:17.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:17.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:17.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:17.276 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:01:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:01:17] "GET /metrics HTTP/1.1" 200 36770 "" "Prometheus/2.51.0" 2024-09-13T15:01:17.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:16 smithi177 bash[68594]: cluster 2024-09-13T15:01:16.466671+0000 mon.a (mon.0) 2158 : cluster [WRN] Health check update: Degraded data redundancy: 23076/222867 objects degraded (10.354%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:17.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:16 smithi177 bash[68594]: cluster 2024-09-13T15:01:16.466671+0000 mon.a (mon.0) 2158 : cluster [WRN] Health check update: Degraded data redundancy: 23076/222867 objects degraded (10.354%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:17.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:17.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:17.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:17.443 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:17.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:17.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:17.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:17.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:17.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:17.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:17.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:17.836 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:17.870 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:17.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:17.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:17.997 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:17.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:18.100 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:18.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:17 smithi177 bash[68594]: cluster 2024-09-13T15:01:16.805920+0000 mgr.y (mgr.26252) 1054 : cluster [DBG] pgmap v1592: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:18.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:17 smithi177 bash[68594]: cluster 2024-09-13T15:01:16.805920+0000 mgr.y (mgr.26252) 1054 : cluster [DBG] pgmap v1592: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:18.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:18.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:18.216 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:17 smithi099 bash[84576]: cluster 2024-09-13T15:01:16.805920+0000 mgr.y (mgr.26252) 1054 : cluster [DBG] pgmap v1592: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:18.216 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:17 smithi099 bash[84576]: cluster 2024-09-13T15:01:16.805920+0000 mgr.y (mgr.26252) 1054 : cluster [DBG] pgmap v1592: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:18.216 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:17 smithi099 bash[87147]: cluster 2024-09-13T15:01:16.805920+0000 mgr.y (mgr.26252) 1054 : cluster [DBG] pgmap v1592: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:18.216 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:17 smithi099 bash[87147]: cluster 2024-09-13T15:01:16.805920+0000 mgr.y (mgr.26252) 1054 : cluster [DBG] pgmap v1592: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 927 op/s; 23076/222867 objects degraded (10.354%) 2024-09-13T15:01:18.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:18.249 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:18.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:18.352 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:18.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:18.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:18.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:18.497 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:18.600 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:18.625 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:18.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:18.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:18.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:18.863 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:18.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:18.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:18.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:19.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:19.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:19.122 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:19.144 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:19.249 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:19.314 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:19.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:19.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:19.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:19.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:19.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:19.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[84576]: audit 2024-09-13T15:01:18.297415+0000 mon.a (mon.0) 2159 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:19.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[84576]: audit 2024-09-13T15:01:18.297415+0000 mon.a (mon.0) 2159 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:19.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[84576]: audit 2024-09-13T15:01:18.298321+0000 mon.a (mon.0) 2160 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:19.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[84576]: audit 2024-09-13T15:01:18.298321+0000 mon.a (mon.0) 2160 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:19.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[84576]: cluster 2024-09-13T15:01:18.806801+0000 mgr.y (mgr.26252) 1055 : cluster [DBG] pgmap v1593: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 22767/220032 objects degraded (10.347%) 2024-09-13T15:01:19.565 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[84576]: cluster 2024-09-13T15:01:18.806801+0000 mgr.y (mgr.26252) 1055 : cluster [DBG] pgmap v1593: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 22767/220032 objects degraded (10.347%) 2024-09-13T15:01:19.565 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[87147]: audit 2024-09-13T15:01:18.297415+0000 mon.a (mon.0) 2159 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:19.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[87147]: audit 2024-09-13T15:01:18.297415+0000 mon.a (mon.0) 2159 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:19.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[87147]: audit 2024-09-13T15:01:18.298321+0000 mon.a (mon.0) 2160 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:19.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[87147]: audit 2024-09-13T15:01:18.298321+0000 mon.a (mon.0) 2160 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:19.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[87147]: cluster 2024-09-13T15:01:18.806801+0000 mgr.y (mgr.26252) 1055 : cluster [DBG] pgmap v1593: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 22767/220032 objects degraded (10.347%) 2024-09-13T15:01:19.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:19 smithi099 bash[87147]: cluster 2024-09-13T15:01:18.806801+0000 mgr.y (mgr.26252) 1055 : cluster [DBG] pgmap v1593: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 22767/220032 objects degraded (10.347%) 2024-09-13T15:01:19.575 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:19.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:19.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:19 smithi177 bash[68594]: audit 2024-09-13T15:01:18.297415+0000 mon.a (mon.0) 2159 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:19 smithi177 bash[68594]: audit 2024-09-13T15:01:18.297415+0000 mon.a (mon.0) 2159 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:19 smithi177 bash[68594]: audit 2024-09-13T15:01:18.298321+0000 mon.a (mon.0) 2160 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:19 smithi177 bash[68594]: audit 2024-09-13T15:01:18.298321+0000 mon.a (mon.0) 2160 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:19 smithi177 bash[68594]: cluster 2024-09-13T15:01:18.806801+0000 mgr.y (mgr.26252) 1055 : cluster [DBG] pgmap v1593: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 22767/220032 objects degraded (10.347%) 2024-09-13T15:01:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:19 smithi177 bash[68594]: cluster 2024-09-13T15:01:18.806801+0000 mgr.y (mgr.26252) 1055 : cluster [DBG] pgmap v1593: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 22767/220032 objects degraded (10.347%) 2024-09-13T15:01:19.678 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:19.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:19.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:19.809 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:19.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:19.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:19.950 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:19.977 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:19.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:20.010 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:20.010 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:20.085 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:20.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:20.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:20.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:20.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:20.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:20.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:20.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:01:20 smithi177 bash[61439]: ts=2024-09-13T15:01:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:20.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:20.466 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:20.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:20.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:20.589 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:20.619 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:20.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:20.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:20.760 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:20.866 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:20.874 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:20 smithi099 bash[84576]: audit 2024-09-13T15:01:19.833608+0000 mon.a (mon.0) 2161 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:20.875 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:20 smithi099 bash[84576]: audit 2024-09-13T15:01:19.833608+0000 mon.a (mon.0) 2161 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:20.875 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:20 smithi099 bash[84576]: audit 2024-09-13T15:01:19.844168+0000 mon.a (mon.0) 2162 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:20.875 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:20 smithi099 bash[84576]: audit 2024-09-13T15:01:19.844168+0000 mon.a (mon.0) 2162 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:20.875 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:20 smithi099 bash[87147]: audit 2024-09-13T15:01:19.833608+0000 mon.a (mon.0) 2161 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:20.875 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:20 smithi099 bash[87147]: audit 2024-09-13T15:01:19.833608+0000 mon.a (mon.0) 2161 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:20.875 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:20 smithi099 bash[87147]: audit 2024-09-13T15:01:19.844168+0000 mon.a (mon.0) 2162 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:20.875 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:20 smithi099 bash[87147]: audit 2024-09-13T15:01:19.844168+0000 mon.a (mon.0) 2162 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:20.896 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:21.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:21.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:21.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:21.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:20 smithi177 bash[68594]: audit 2024-09-13T15:01:19.833608+0000 mon.a (mon.0) 2161 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:21.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:20 smithi177 bash[68594]: audit 2024-09-13T15:01:19.833608+0000 mon.a (mon.0) 2161 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:21.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:20 smithi177 bash[68594]: audit 2024-09-13T15:01:19.844168+0000 mon.a (mon.0) 2162 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:21.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:20 smithi177 bash[68594]: audit 2024-09-13T15:01:19.844168+0000 mon.a (mon.0) 2162 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:21.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:21.158 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:21.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:21.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:21.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:01:21.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:21.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:21.415 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:21.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:21.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:21.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:21.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:21.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:21.670 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:21.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:21.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:21.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:21.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:21.951 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:21.960 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:21 smithi177 bash[68594]: cluster 2024-09-13T15:01:20.807685+0000 mgr.y (mgr.26252) 1056 : cluster [DBG] pgmap v1594: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s; 21711/209652 objects degraded (10.356%) 2024-09-13T15:01:21.961 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:21 smithi177 bash[68594]: cluster 2024-09-13T15:01:20.807685+0000 mgr.y (mgr.26252) 1056 : cluster [DBG] pgmap v1594: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s; 21711/209652 objects degraded (10.356%) 2024-09-13T15:01:21.961 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:21 smithi177 bash[68594]: cluster 2024-09-13T15:01:21.467464+0000 mon.a (mon.0) 2163 : cluster [WRN] Health check update: Degraded data redundancy: 21711/209652 objects degraded (10.356%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:21.961 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:21 smithi177 bash[68594]: cluster 2024-09-13T15:01:21.467464+0000 mon.a (mon.0) 2163 : cluster [WRN] Health check update: Degraded data redundancy: 21711/209652 objects degraded (10.356%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:21.982 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:21.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:22.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:22.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:21 smithi099 bash[87147]: cluster 2024-09-13T15:01:20.807685+0000 mgr.y (mgr.26252) 1056 : cluster [DBG] pgmap v1594: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s; 21711/209652 objects degraded (10.356%) 2024-09-13T15:01:22.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:21 smithi099 bash[87147]: cluster 2024-09-13T15:01:20.807685+0000 mgr.y (mgr.26252) 1056 : cluster [DBG] pgmap v1594: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s; 21711/209652 objects degraded (10.356%) 2024-09-13T15:01:22.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:21 smithi099 bash[87147]: cluster 2024-09-13T15:01:21.467464+0000 mon.a (mon.0) 2163 : cluster [WRN] Health check update: Degraded data redundancy: 21711/209652 objects degraded (10.356%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:22.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:21 smithi099 bash[87147]: cluster 2024-09-13T15:01:21.467464+0000 mon.a (mon.0) 2163 : cluster [WRN] Health check update: Degraded data redundancy: 21711/209652 objects degraded (10.356%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:22.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:21 smithi099 bash[84576]: cluster 2024-09-13T15:01:20.807685+0000 mgr.y (mgr.26252) 1056 : cluster [DBG] pgmap v1594: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s; 21711/209652 objects degraded (10.356%) 2024-09-13T15:01:22.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:21 smithi099 bash[84576]: cluster 2024-09-13T15:01:20.807685+0000 mgr.y (mgr.26252) 1056 : cluster [DBG] pgmap v1594: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s; 21711/209652 objects degraded (10.356%) 2024-09-13T15:01:22.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:21 smithi099 bash[84576]: cluster 2024-09-13T15:01:21.467464+0000 mon.a (mon.0) 2163 : cluster [WRN] Health check update: Degraded data redundancy: 21711/209652 objects degraded (10.356%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:22.097 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:21 smithi099 bash[84576]: cluster 2024-09-13T15:01:21.467464+0000 mon.a (mon.0) 2163 : cluster [WRN] Health check update: Degraded data redundancy: 21711/209652 objects degraded (10.356%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:22.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:22.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:22.431 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:22.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:22.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:22.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:22.671 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:22.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:22.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:22.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:22.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:22.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:22.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:22.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:22.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:23.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:23.114 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:23.218 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:23.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:23.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:23.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:23.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:23.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:23.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:23.551 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:23.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:23.657 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:23.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:23.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:23.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:23.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:23.931 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:23.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:23.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:24.060 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:24.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:23 smithi099 bash[84576]: cluster 2024-09-13T15:01:22.808309+0000 mgr.y (mgr.26252) 1057 : cluster [DBG] pgmap v1595: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 21441/207309 objects degraded (10.343%) 2024-09-13T15:01:24.069 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:23 smithi099 bash[84576]: cluster 2024-09-13T15:01:22.808309+0000 mgr.y (mgr.26252) 1057 : cluster [DBG] pgmap v1595: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 21441/207309 objects degraded (10.343%) 2024-09-13T15:01:24.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:23 smithi099 bash[87147]: cluster 2024-09-13T15:01:22.808309+0000 mgr.y (mgr.26252) 1057 : cluster [DBG] pgmap v1595: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 21441/207309 objects degraded (10.343%) 2024-09-13T15:01:24.069 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:23 smithi099 bash[87147]: cluster 2024-09-13T15:01:22.808309+0000 mgr.y (mgr.26252) 1057 : cluster [DBG] pgmap v1595: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 21441/207309 objects degraded (10.343%) 2024-09-13T15:01:24.085 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:24.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:24.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:23 smithi177 bash[68594]: cluster 2024-09-13T15:01:22.808309+0000 mgr.y (mgr.26252) 1057 : cluster [DBG] pgmap v1595: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 21441/207309 objects degraded (10.343%) 2024-09-13T15:01:24.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:23 smithi177 bash[68594]: cluster 2024-09-13T15:01:22.808309+0000 mgr.y (mgr.26252) 1057 : cluster [DBG] pgmap v1595: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 21441/207309 objects degraded (10.343%) 2024-09-13T15:01:24.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:24.216 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:24.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:24.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:24.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:24.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:24.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:24.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:24.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:24.596 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:24.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:24.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:24.724 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:24.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:24.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:24.849 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:24.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:24.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:24.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:25.096 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:25.125 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:25.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:25.230 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:25.258 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:25.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:25.363 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:25.396 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:25.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:25.500 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:25.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:25.641 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:25.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:25.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:25.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:25.801 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:25.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:25.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:25.933 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:25.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:26.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:26.080 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:25 smithi177 bash[68594]: cluster 2024-09-13T15:01:24.809267+0000 mgr.y (mgr.26252) 1058 : cluster [DBG] pgmap v1596: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:25 smithi177 bash[68594]: cluster 2024-09-13T15:01:24.809267+0000 mgr.y (mgr.26252) 1058 : cluster [DBG] pgmap v1596: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:26.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:26.204 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:26.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:26.311 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:25 smithi099 bash[87147]: cluster 2024-09-13T15:01:24.809267+0000 mgr.y (mgr.26252) 1058 : cluster [DBG] pgmap v1596: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:26.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:25 smithi099 bash[87147]: cluster 2024-09-13T15:01:24.809267+0000 mgr.y (mgr.26252) 1058 : cluster [DBG] pgmap v1596: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:26.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:25 smithi099 bash[84576]: cluster 2024-09-13T15:01:24.809267+0000 mgr.y (mgr.26252) 1058 : cluster [DBG] pgmap v1596: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:26.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:25 smithi099 bash[84576]: cluster 2024-09-13T15:01:24.809267+0000 mgr.y (mgr.26252) 1058 : cluster [DBG] pgmap v1596: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:26.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:26.531 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:01:26.571 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:26.592 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:26.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:26.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.719 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:26.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:26.854 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:26.957 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.983 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:27.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:27.092 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:01:27] "GET /metrics HTTP/1.1" 200 36763 "" "Prometheus/2.51.0" 2024-09-13T15:01:27.112 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:27.219 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:27.228 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: audit 2024-09-13T15:01:26.171897+0000 mon.a (mon.0) 2164 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: audit 2024-09-13T15:01:26.171897+0000 mon.a (mon.0) 2164 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: audit 2024-09-13T15:01:26.179085+0000 mon.a (mon.0) 2165 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: audit 2024-09-13T15:01:26.179085+0000 mon.a (mon.0) 2165 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: cluster 2024-09-13T15:01:26.468275+0000 mon.a (mon.0) 2166 : cluster [WRN] Health check update: Degraded data redundancy: 20338/196134 objects degraded (10.369%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: cluster 2024-09-13T15:01:26.468275+0000 mon.a (mon.0) 2166 : cluster [WRN] Health check update: Degraded data redundancy: 20338/196134 objects degraded (10.369%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: cluster 2024-09-13T15:01:26.809905+0000 mgr.y (mgr.26252) 1059 : cluster [DBG] pgmap v1597: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: cluster 2024-09-13T15:01:26.809905+0000 mgr.y (mgr.26252) 1059 : cluster [DBG] pgmap v1597: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: audit 2024-09-13T15:01:26.897645+0000 mon.a (mon.0) 2167 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: audit 2024-09-13T15:01:26.897645+0000 mon.a (mon.0) 2167 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: audit 2024-09-13T15:01:26.899345+0000 mon.a (mon.0) 2168 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: audit 2024-09-13T15:01:26.899345+0000 mon.a (mon.0) 2168 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: audit 2024-09-13T15:01:26.905202+0000 mon.a (mon.0) 2169 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.229 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:27 smithi177 bash[68594]: audit 2024-09-13T15:01:26.905202+0000 mon.a (mon.0) 2169 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.260 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:27.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:27.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: audit 2024-09-13T15:01:26.171897+0000 mon.a (mon.0) 2164 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: audit 2024-09-13T15:01:26.171897+0000 mon.a (mon.0) 2164 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: audit 2024-09-13T15:01:26.179085+0000 mon.a (mon.0) 2165 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: audit 2024-09-13T15:01:26.179085+0000 mon.a (mon.0) 2165 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: cluster 2024-09-13T15:01:26.468275+0000 mon.a (mon.0) 2166 : cluster [WRN] Health check update: Degraded data redundancy: 20338/196134 objects degraded (10.369%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:27.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: cluster 2024-09-13T15:01:26.468275+0000 mon.a (mon.0) 2166 : cluster [WRN] Health check update: Degraded data redundancy: 20338/196134 objects degraded (10.369%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:27.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: cluster 2024-09-13T15:01:26.809905+0000 mgr.y (mgr.26252) 1059 : cluster [DBG] pgmap v1597: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:27.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: cluster 2024-09-13T15:01:26.809905+0000 mgr.y (mgr.26252) 1059 : cluster [DBG] pgmap v1597: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:27.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: audit 2024-09-13T15:01:26.897645+0000 mon.a (mon.0) 2167 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:01:27.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: audit 2024-09-13T15:01:26.897645+0000 mon.a (mon.0) 2167 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:01:27.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: audit 2024-09-13T15:01:26.899345+0000 mon.a (mon.0) 2168 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:01:27.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: audit 2024-09-13T15:01:26.899345+0000 mon.a (mon.0) 2168 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:01:27.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: audit 2024-09-13T15:01:26.905202+0000 mon.a (mon.0) 2169 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[84576]: audit 2024-09-13T15:01:26.905202+0000 mon.a (mon.0) 2169 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: audit 2024-09-13T15:01:26.171897+0000 mon.a (mon.0) 2164 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: audit 2024-09-13T15:01:26.171897+0000 mon.a (mon.0) 2164 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: audit 2024-09-13T15:01:26.179085+0000 mon.a (mon.0) 2165 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: audit 2024-09-13T15:01:26.179085+0000 mon.a (mon.0) 2165 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: cluster 2024-09-13T15:01:26.468275+0000 mon.a (mon.0) 2166 : cluster [WRN] Health check update: Degraded data redundancy: 20338/196134 objects degraded (10.369%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: cluster 2024-09-13T15:01:26.468275+0000 mon.a (mon.0) 2166 : cluster [WRN] Health check update: Degraded data redundancy: 20338/196134 objects degraded (10.369%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: cluster 2024-09-13T15:01:26.809905+0000 mgr.y (mgr.26252) 1059 : cluster [DBG] pgmap v1597: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: cluster 2024-09-13T15:01:26.809905+0000 mgr.y (mgr.26252) 1059 : cluster [DBG] pgmap v1597: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 20338/196134 objects degraded (10.369%) 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: audit 2024-09-13T15:01:26.897645+0000 mon.a (mon.0) 2167 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: audit 2024-09-13T15:01:26.897645+0000 mon.a (mon.0) 2167 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: audit 2024-09-13T15:01:26.899345+0000 mon.a (mon.0) 2168 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: audit 2024-09-13T15:01:26.899345+0000 mon.a (mon.0) 2168 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: audit 2024-09-13T15:01:26.905202+0000 mon.a (mon.0) 2169 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:27 smithi099 bash[87147]: audit 2024-09-13T15:01:26.905202+0000 mon.a (mon.0) 2169 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:01:27.368 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:27.393 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:27.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:27.488 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:01:27.488 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:27.497 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:27.501 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:01:27.501 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:27.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:27.533 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:01:27.533 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:27.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:01:27.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:27.568 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:01:27.568 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:27.592 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:01:27.592 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:27.616 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:01:27.616 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:27.635 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:27.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:27.666 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:01:27.666 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:27.760 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:27.785 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:27.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:27.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:27.913 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:27.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:28.020 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.050 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:28.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:28.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:28.180 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:28.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:28.283 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:28.310 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:28.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:28.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:28.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:28.436 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:28 smithi177 bash[68594]: audit 2024-09-13T15:01:27.429372+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.99:0/3730581353' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:28.436 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:28 smithi177 bash[68594]: audit 2024-09-13T15:01:27.429372+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.99:0/3730581353' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:28.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:28.538 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:28.548 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:28 smithi099 bash[84576]: audit 2024-09-13T15:01:27.429372+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.99:0/3730581353' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:28.548 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:28 smithi099 bash[84576]: audit 2024-09-13T15:01:27.429372+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.99:0/3730581353' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:28.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:28 smithi099 bash[87147]: audit 2024-09-13T15:01:27.429372+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.99:0/3730581353' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:28.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:28 smithi099 bash[87147]: audit 2024-09-13T15:01:27.429372+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.99:0/3730581353' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:28.563 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:28.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:28.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:28.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:28.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:28.793 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:28.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:28.928 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:28.960 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:28.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:29.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:29.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:29.198 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:29.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:29.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:29.337 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:29.355 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:29.443 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:29 smithi177 bash[68594]: cluster 2024-09-13T15:01:28.810858+0000 mgr.y (mgr.26252) 1060 : cluster [DBG] pgmap v1598: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 20049/193338 objects degraded (10.370%) 2024-09-13T15:01:29.444 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:29 smithi177 bash[68594]: cluster 2024-09-13T15:01:28.810858+0000 mgr.y (mgr.26252) 1060 : cluster [DBG] pgmap v1598: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 20049/193338 objects degraded (10.370%) 2024-09-13T15:01:29.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:29.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:29.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:29.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:29 smithi099 bash[84576]: cluster 2024-09-13T15:01:28.810858+0000 mgr.y (mgr.26252) 1060 : cluster [DBG] pgmap v1598: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 20049/193338 objects degraded (10.370%) 2024-09-13T15:01:29.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:29 smithi099 bash[84576]: cluster 2024-09-13T15:01:28.810858+0000 mgr.y (mgr.26252) 1060 : cluster [DBG] pgmap v1598: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 20049/193338 objects degraded (10.370%) 2024-09-13T15:01:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:29 smithi099 bash[87147]: cluster 2024-09-13T15:01:28.810858+0000 mgr.y (mgr.26252) 1060 : cluster [DBG] pgmap v1598: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 20049/193338 objects degraded (10.370%) 2024-09-13T15:01:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:29 smithi099 bash[87147]: cluster 2024-09-13T15:01:28.810858+0000 mgr.y (mgr.26252) 1060 : cluster [DBG] pgmap v1598: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 20049/193338 objects degraded (10.370%) 2024-09-13T15:01:29.580 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:29.607 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:29.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:29.711 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:29.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:29.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:29.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:29.863 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:29.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:29.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:29.984 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:29.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:30.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:30.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:30.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:30.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:30.225 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:01:30 smithi177 bash[61439]: ts=2024-09-13T15:01:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:30.254 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:30.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:30.279 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T15:01:30.318 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:01:30.318 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T15:01:30.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-09-13T15:01:30.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:30.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:30.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:30.451 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:30 smithi099 bash[84576]: audit 2024-09-13T15:01:30.222009+0000 mon.c (mon.2) 81 : audit [INF] from='client.? 172.21.15.99:0/335568800' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.451 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:30 smithi099 bash[84576]: audit 2024-09-13T15:01:30.222009+0000 mon.c (mon.2) 81 : audit [INF] from='client.? 172.21.15.99:0/335568800' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.451 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:30 smithi099 bash[84576]: audit 2024-09-13T15:01:30.222374+0000 mon.a (mon.0) 2170 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.451 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:30 smithi099 bash[84576]: audit 2024-09-13T15:01:30.222374+0000 mon.a (mon.0) 2170 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:30 smithi099 bash[87147]: audit 2024-09-13T15:01:30.222009+0000 mon.c (mon.2) 81 : audit [INF] from='client.? 172.21.15.99:0/335568800' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:30 smithi099 bash[87147]: audit 2024-09-13T15:01:30.222009+0000 mon.c (mon.2) 81 : audit [INF] from='client.? 172.21.15.99:0/335568800' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:30 smithi099 bash[87147]: audit 2024-09-13T15:01:30.222374+0000 mon.a (mon.0) 2170 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:30 smithi099 bash[87147]: audit 2024-09-13T15:01:30.222374+0000 mon.a (mon.0) 2170 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:30.526 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:30.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:01:30.602 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:01:30.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:30 smithi177 bash[68594]: audit 2024-09-13T15:01:30.222009+0000 mon.c (mon.2) 81 : audit [INF] from='client.? 172.21.15.99:0/335568800' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:30 smithi177 bash[68594]: audit 2024-09-13T15:01:30.222009+0000 mon.c (mon.2) 81 : audit [INF] from='client.? 172.21.15.99:0/335568800' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:30 smithi177 bash[68594]: audit 2024-09-13T15:01:30.222374+0000 mon.a (mon.0) 2170 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:30 smithi177 bash[68594]: audit 2024-09-13T15:01:30.222374+0000 mon.a (mon.0) 2170 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:30.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:30.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:30.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:30.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:30.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:30.921 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:30.945 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:30.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:31.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:31.072 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:31.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:31.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:31.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:31.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:31.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:31.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:31.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:31.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:31.441 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[84576]: cluster 2024-09-13T15:01:30.272852+0000 mon.a (mon.0) 2171 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:31.441 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[84576]: cluster 2024-09-13T15:01:30.272852+0000 mon.a (mon.0) 2171 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:31.441 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[84576]: audit 2024-09-13T15:01:30.274165+0000 mon.a (mon.0) 2172 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:31.441 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[84576]: audit 2024-09-13T15:01:30.274165+0000 mon.a (mon.0) 2172 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:31.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[84576]: cluster 2024-09-13T15:01:30.280293+0000 mon.a (mon.0) 2173 : cluster [DBG] osdmap e1050: 8 total, 7 up, 8 in 2024-09-13T15:01:31.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[84576]: cluster 2024-09-13T15:01:30.280293+0000 mon.a (mon.0) 2173 : cluster [DBG] osdmap e1050: 8 total, 7 up, 8 in 2024-09-13T15:01:31.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[84576]: cluster 2024-09-13T15:01:30.811585+0000 mgr.y (mgr.26252) 1061 : cluster [DBG] pgmap v1600: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:31.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[84576]: cluster 2024-09-13T15:01:30.811585+0000 mgr.y (mgr.26252) 1061 : cluster [DBG] pgmap v1600: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:31.442 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[87147]: cluster 2024-09-13T15:01:30.272852+0000 mon.a (mon.0) 2171 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:31.442 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[87147]: cluster 2024-09-13T15:01:30.272852+0000 mon.a (mon.0) 2171 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:31.442 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[87147]: audit 2024-09-13T15:01:30.274165+0000 mon.a (mon.0) 2172 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:31.442 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[87147]: audit 2024-09-13T15:01:30.274165+0000 mon.a (mon.0) 2172 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:31.442 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[87147]: cluster 2024-09-13T15:01:30.280293+0000 mon.a (mon.0) 2173 : cluster [DBG] osdmap e1050: 8 total, 7 up, 8 in 2024-09-13T15:01:31.442 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[87147]: cluster 2024-09-13T15:01:30.280293+0000 mon.a (mon.0) 2173 : cluster [DBG] osdmap e1050: 8 total, 7 up, 8 in 2024-09-13T15:01:31.443 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[87147]: cluster 2024-09-13T15:01:30.811585+0000 mgr.y (mgr.26252) 1061 : cluster [DBG] pgmap v1600: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:31.443 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:31 smithi099 bash[87147]: cluster 2024-09-13T15:01:30.811585+0000 mgr.y (mgr.26252) 1061 : cluster [DBG] pgmap v1600: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:31.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:31.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:31.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:31.576 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:31 smithi177 bash[68594]: cluster 2024-09-13T15:01:30.272852+0000 mon.a (mon.0) 2171 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:31.577 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:31 smithi177 bash[68594]: cluster 2024-09-13T15:01:30.272852+0000 mon.a (mon.0) 2171 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:31.577 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:31 smithi177 bash[68594]: audit 2024-09-13T15:01:30.274165+0000 mon.a (mon.0) 2172 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:31.577 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:31 smithi177 bash[68594]: audit 2024-09-13T15:01:30.274165+0000 mon.a (mon.0) 2172 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:31.577 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:31 smithi177 bash[68594]: cluster 2024-09-13T15:01:30.280293+0000 mon.a (mon.0) 2173 : cluster [DBG] osdmap e1050: 8 total, 7 up, 8 in 2024-09-13T15:01:31.577 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:31 smithi177 bash[68594]: cluster 2024-09-13T15:01:30.280293+0000 mon.a (mon.0) 2173 : cluster [DBG] osdmap e1050: 8 total, 7 up, 8 in 2024-09-13T15:01:31.577 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:31 smithi177 bash[68594]: cluster 2024-09-13T15:01:30.811585+0000 mgr.y (mgr.26252) 1061 : cluster [DBG] pgmap v1600: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:31.577 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:31 smithi177 bash[68594]: cluster 2024-09-13T15:01:30.811585+0000 mgr.y (mgr.26252) 1061 : cluster [DBG] pgmap v1600: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 878 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:31.604 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:31.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:31.710 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:31.732 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:31.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:31.836 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:31.863 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:31.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:31.969 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:32.003 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:32.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:32.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:32.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:32.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:32.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:32.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:32.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:32.370 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:32.380 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:32 smithi177 bash[68594]: cluster 2024-09-13T15:01:31.468882+0000 mon.a (mon.0) 2174 : cluster [WRN] Health check update: Degraded data redundancy: 19020/183300 objects degraded (10.376%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:32.380 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:32 smithi177 bash[68594]: cluster 2024-09-13T15:01:31.468882+0000 mon.a (mon.0) 2174 : cluster [WRN] Health check update: Degraded data redundancy: 19020/183300 objects degraded (10.376%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:32.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:32.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:32.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:32.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:32.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:32.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:32 smithi099 bash[84576]: cluster 2024-09-13T15:01:31.468882+0000 mon.a (mon.0) 2174 : cluster [WRN] Health check update: Degraded data redundancy: 19020/183300 objects degraded (10.376%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:32.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:32 smithi099 bash[84576]: cluster 2024-09-13T15:01:31.468882+0000 mon.a (mon.0) 2174 : cluster [WRN] Health check update: Degraded data redundancy: 19020/183300 objects degraded (10.376%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:32.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:32 smithi099 bash[87147]: cluster 2024-09-13T15:01:31.468882+0000 mon.a (mon.0) 2174 : cluster [WRN] Health check update: Degraded data redundancy: 19020/183300 objects degraded (10.376%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:32.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:32 smithi099 bash[87147]: cluster 2024-09-13T15:01:31.468882+0000 mon.a (mon.0) 2174 : cluster [WRN] Health check update: Degraded data redundancy: 19020/183300 objects degraded (10.376%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:32.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:01:32.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:32.672 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:32.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:32.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:32.811 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:32.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:32.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:32.946 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:32.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:33.053 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:33.082 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:33.094 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:01:33.094 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:01:33.094 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:01:33.094 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:01:33.094 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:01:33.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:33.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:33.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:33.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:33 smithi099 bash[84576]: cluster 2024-09-13T15:01:32.812393+0000 mgr.y (mgr.26252) 1062 : cluster [DBG] pgmap v1601: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:33.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:33 smithi099 bash[84576]: cluster 2024-09-13T15:01:32.812393+0000 mgr.y (mgr.26252) 1062 : cluster [DBG] pgmap v1601: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:33.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:33 smithi099 bash[87147]: cluster 2024-09-13T15:01:32.812393+0000 mgr.y (mgr.26252) 1062 : cluster [DBG] pgmap v1601: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:33.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:33 smithi099 bash[87147]: cluster 2024-09-13T15:01:32.812393+0000 mgr.y (mgr.26252) 1062 : cluster [DBG] pgmap v1601: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:33.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:33.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:33.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:33 smithi177 bash[68594]: cluster 2024-09-13T15:01:32.812393+0000 mgr.y (mgr.26252) 1062 : cluster [DBG] pgmap v1601: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:33.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:33 smithi177 bash[68594]: cluster 2024-09-13T15:01:32.812393+0000 mgr.y (mgr.26252) 1062 : cluster [DBG] pgmap v1601: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 19020/183300 objects degraded (10.376%) 2024-09-13T15:01:33.439 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:33.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:33.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:33.575 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:33.592 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:01:33.598 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:33.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:33.701 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:33.730 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:33.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:33.834 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:33.858 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:33.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:33.961 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:33.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:33.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:34.094 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:34.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:34.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:34.219 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:34.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:34.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:34.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:34.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:34.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:34.495 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:34.508 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:34.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:34.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:34 smithi099 bash[84576]: audit 2024-09-13T15:01:33.293780+0000 mon.a (mon.0) 2175 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:34.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:34 smithi099 bash[84576]: audit 2024-09-13T15:01:33.293780+0000 mon.a (mon.0) 2175 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:34 smithi099 bash[87147]: audit 2024-09-13T15:01:33.293780+0000 mon.a (mon.0) 2175 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:34 smithi099 bash[87147]: audit 2024-09-13T15:01:33.293780+0000 mon.a (mon.0) 2175 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:34.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:34 smithi177 bash[68594]: audit 2024-09-13T15:01:33.293780+0000 mon.a (mon.0) 2175 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:34.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:34 smithi177 bash[68594]: audit 2024-09-13T15:01:33.293780+0000 mon.a (mon.0) 2175 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:34.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:34.654 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:34.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:34.760 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:34.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:34.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:34.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:34.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:34.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:35.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:35.072 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:35.153 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:35.177 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:35.203 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:35.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:35.334 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:35.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:35.442 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.473 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:35.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:35.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:35 smithi099 bash[84576]: cluster 2024-09-13T15:01:34.813255+0000 mgr.y (mgr.26252) 1063 : cluster [DBG] pgmap v1602: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 17856/171393 objects degraded (10.418%) 2024-09-13T15:01:35.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:35 smithi099 bash[84576]: cluster 2024-09-13T15:01:34.813255+0000 mgr.y (mgr.26252) 1063 : cluster [DBG] pgmap v1602: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 17856/171393 objects degraded (10.418%) 2024-09-13T15:01:35.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:35 smithi099 bash[87147]: cluster 2024-09-13T15:01:34.813255+0000 mgr.y (mgr.26252) 1063 : cluster [DBG] pgmap v1602: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 17856/171393 objects degraded (10.418%) 2024-09-13T15:01:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:35 smithi099 bash[87147]: cluster 2024-09-13T15:01:34.813255+0000 mgr.y (mgr.26252) 1063 : cluster [DBG] pgmap v1602: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 17856/171393 objects degraded (10.418%) 2024-09-13T15:01:35.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:35.603 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:35.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:35.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:35 smithi177 bash[68594]: cluster 2024-09-13T15:01:34.813255+0000 mgr.y (mgr.26252) 1063 : cluster [DBG] pgmap v1602: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 17856/171393 objects degraded (10.418%) 2024-09-13T15:01:35.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:35 smithi177 bash[68594]: cluster 2024-09-13T15:01:34.813255+0000 mgr.y (mgr.26252) 1063 : cluster [DBG] pgmap v1602: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 17856/171393 objects degraded (10.418%) 2024-09-13T15:01:35.706 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:35.742 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:35.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:35.849 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:35.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:35.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:35.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:36.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:36.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:36.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:36.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:36.255 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:36.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:36.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:36.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:36.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:36.510 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:36.536 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:36.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:36.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:36.649 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:36 smithi177 bash[68594]: cluster 2024-09-13T15:01:36.469722+0000 mon.a (mon.0) 2176 : cluster [WRN] Health check update: Degraded data redundancy: 17856/171393 objects degraded (10.418%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:36.649 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:36 smithi177 bash[68594]: cluster 2024-09-13T15:01:36.469722+0000 mon.a (mon.0) 2176 : cluster [WRN] Health check update: Degraded data redundancy: 17856/171393 objects degraded (10.418%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:36.667 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:36.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:36.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:36.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:36.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:36.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:36 smithi099 bash[84576]: cluster 2024-09-13T15:01:36.469722+0000 mon.a (mon.0) 2176 : cluster [WRN] Health check update: Degraded data redundancy: 17856/171393 objects degraded (10.418%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:36.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:36 smithi099 bash[84576]: cluster 2024-09-13T15:01:36.469722+0000 mon.a (mon.0) 2176 : cluster [WRN] Health check update: Degraded data redundancy: 17856/171393 objects degraded (10.418%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:36.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:36 smithi099 bash[87147]: cluster 2024-09-13T15:01:36.469722+0000 mon.a (mon.0) 2176 : cluster [WRN] Health check update: Degraded data redundancy: 17856/171393 objects degraded (10.418%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:36.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:36 smithi099 bash[87147]: cluster 2024-09-13T15:01:36.469722+0000 mon.a (mon.0) 2176 : cluster [WRN] Health check update: Degraded data redundancy: 17856/171393 objects degraded (10.418%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:36.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:36.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:37.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:37.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:37.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:37.175 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:37.179 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:37.210 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:37.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:37.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:37.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:01:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:01:37] "GET /metrics HTTP/1.1" 200 36770 "" "Prometheus/2.51.0" 2024-09-13T15:01:37.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:37.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:37.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:37.493 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:37.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:37.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:37.610 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:37 smithi099 bash[84576]: cluster 2024-09-13T15:01:36.814130+0000 mgr.y (mgr.26252) 1064 : cluster [DBG] pgmap v1603: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 17506/168396 objects degraded (10.396%) 2024-09-13T15:01:37.610 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:37 smithi099 bash[84576]: cluster 2024-09-13T15:01:36.814130+0000 mgr.y (mgr.26252) 1064 : cluster [DBG] pgmap v1603: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 17506/168396 objects degraded (10.396%) 2024-09-13T15:01:37.611 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:37 smithi099 bash[87147]: cluster 2024-09-13T15:01:36.814130+0000 mgr.y (mgr.26252) 1064 : cluster [DBG] pgmap v1603: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 17506/168396 objects degraded (10.396%) 2024-09-13T15:01:37.611 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:37 smithi099 bash[87147]: cluster 2024-09-13T15:01:36.814130+0000 mgr.y (mgr.26252) 1064 : cluster [DBG] pgmap v1603: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 17506/168396 objects degraded (10.396%) 2024-09-13T15:01:37.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:37.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:37.740 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:37.763 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:37.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:37.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:37 smithi177 bash[68594]: cluster 2024-09-13T15:01:36.814130+0000 mgr.y (mgr.26252) 1064 : cluster [DBG] pgmap v1603: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 17506/168396 objects degraded (10.396%) 2024-09-13T15:01:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:37 smithi177 bash[68594]: cluster 2024-09-13T15:01:36.814130+0000 mgr.y (mgr.26252) 1064 : cluster [DBG] pgmap v1603: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 17506/168396 objects degraded (10.396%) 2024-09-13T15:01:37.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:37.902 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:37.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:38.005 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:38.031 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:38.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:38.137 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:38.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:38.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:38.195 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:38.265 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:38.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:38.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:38.385 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:38.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:38.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:38.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:38.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:38.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:38.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:38.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:38.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:38.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:38.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:38.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:38.998 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:39.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:39.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:39.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:39.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:39.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:39.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:39.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:39.357 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:39.380 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:39.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:39.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:39.521 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:39.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:39.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:39.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:39.673 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:39.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:39.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:39.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:39.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:39.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:39.914 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:39 smithi099 bash[84576]: cluster 2024-09-13T15:01:38.814786+0000 mgr.y (mgr.26252) 1065 : cluster [DBG] pgmap v1604: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 17217/165846 objects degraded (10.381%) 2024-09-13T15:01:39.914 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:39 smithi099 bash[84576]: cluster 2024-09-13T15:01:38.814786+0000 mgr.y (mgr.26252) 1065 : cluster [DBG] pgmap v1604: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 17217/165846 objects degraded (10.381%) 2024-09-13T15:01:39.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:39 smithi099 bash[87147]: cluster 2024-09-13T15:01:38.814786+0000 mgr.y (mgr.26252) 1065 : cluster [DBG] pgmap v1604: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 17217/165846 objects degraded (10.381%) 2024-09-13T15:01:39.914 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:39 smithi099 bash[87147]: cluster 2024-09-13T15:01:38.814786+0000 mgr.y (mgr.26252) 1065 : cluster [DBG] pgmap v1604: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 17217/165846 objects degraded (10.381%) 2024-09-13T15:01:39.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:39.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:40.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:40.072 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:40.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:40.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:40.204 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:40.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:40.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:39 smithi177 bash[68594]: cluster 2024-09-13T15:01:38.814786+0000 mgr.y (mgr.26252) 1065 : cluster [DBG] pgmap v1604: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 17217/165846 objects degraded (10.381%) 2024-09-13T15:01:40.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:39 smithi177 bash[68594]: cluster 2024-09-13T15:01:38.814786+0000 mgr.y (mgr.26252) 1065 : cluster [DBG] pgmap v1604: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 17217/165846 objects degraded (10.381%) 2024-09-13T15:01:40.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:40.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:40.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:40.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:40.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:40.577 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:01:40 smithi177 bash[61439]: ts=2024-09-13T15:01:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:40.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:40.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:40.696 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:40.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:40.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:40.823 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:40.849 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:40.954 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:40.980 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:40.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:41.087 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:41.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:41.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:41.327 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:41.350 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:41.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:41.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:41.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:41.588 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:41.607 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:41.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:41.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:41.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:41.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:41.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:41.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:41.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:41.879 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:41 smithi177 bash[68594]: cluster 2024-09-13T15:01:40.815528+0000 mgr.y (mgr.26252) 1066 : cluster [DBG] pgmap v1605: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:41.879 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:41 smithi177 bash[68594]: cluster 2024-09-13T15:01:40.815528+0000 mgr.y (mgr.26252) 1066 : cluster [DBG] pgmap v1605: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:41.880 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:41 smithi177 bash[68594]: cluster 2024-09-13T15:01:41.472164+0000 mon.a (mon.0) 2177 : cluster [WRN] Health check update: Degraded data redundancy: 16093/154548 objects degraded (10.413%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:41.880 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:41 smithi177 bash[68594]: cluster 2024-09-13T15:01:41.472164+0000 mon.a (mon.0) 2177 : cluster [WRN] Health check update: Degraded data redundancy: 16093/154548 objects degraded (10.413%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:41.978 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:42.209 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:42.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:42.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:41 smithi099 bash[84576]: cluster 2024-09-13T15:01:40.815528+0000 mgr.y (mgr.26252) 1066 : cluster [DBG] pgmap v1605: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:42.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:41 smithi099 bash[84576]: cluster 2024-09-13T15:01:40.815528+0000 mgr.y (mgr.26252) 1066 : cluster [DBG] pgmap v1605: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:42.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:41 smithi099 bash[84576]: cluster 2024-09-13T15:01:41.472164+0000 mon.a (mon.0) 2177 : cluster [WRN] Health check update: Degraded data redundancy: 16093/154548 objects degraded (10.413%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:42.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:41 smithi099 bash[84576]: cluster 2024-09-13T15:01:41.472164+0000 mon.a (mon.0) 2177 : cluster [WRN] Health check update: Degraded data redundancy: 16093/154548 objects degraded (10.413%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:42.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:41 smithi099 bash[87147]: cluster 2024-09-13T15:01:40.815528+0000 mgr.y (mgr.26252) 1066 : cluster [DBG] pgmap v1605: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:42.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:41 smithi099 bash[87147]: cluster 2024-09-13T15:01:40.815528+0000 mgr.y (mgr.26252) 1066 : cluster [DBG] pgmap v1605: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:42.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:41 smithi099 bash[87147]: cluster 2024-09-13T15:01:41.472164+0000 mon.a (mon.0) 2177 : cluster [WRN] Health check update: Degraded data redundancy: 16093/154548 objects degraded (10.413%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:42.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:41 smithi099 bash[87147]: cluster 2024-09-13T15:01:41.472164+0000 mon.a (mon.0) 2177 : cluster [WRN] Health check update: Degraded data redundancy: 16093/154548 objects degraded (10.413%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:42.340 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:42.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:42.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:42.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:42.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:42.572 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:42.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:42.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:42.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:42.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:42.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:42.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:42.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:42.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:42.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:43.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:43.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:43.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:43.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:43.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:43.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:43.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:43.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:43.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:43.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:43.760 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:43.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:43.864 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:43.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:43.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:43.998 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:44.002 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:43 smithi099 bash[84576]: cluster 2024-09-13T15:01:42.816234+0000 mgr.y (mgr.26252) 1067 : cluster [DBG] pgmap v1606: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:44.002 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:43 smithi099 bash[84576]: cluster 2024-09-13T15:01:42.816234+0000 mgr.y (mgr.26252) 1067 : cluster [DBG] pgmap v1606: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:44.020 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:44.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:44.124 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:44.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:44.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:44.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:43 smithi177 bash[68594]: cluster 2024-09-13T15:01:42.816234+0000 mgr.y (mgr.26252) 1067 : cluster [DBG] pgmap v1606: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:44.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:43 smithi177 bash[68594]: cluster 2024-09-13T15:01:42.816234+0000 mgr.y (mgr.26252) 1067 : cluster [DBG] pgmap v1606: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:44.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:44.257 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:44 smithi099 bash[87147]: cluster 2024-09-13T15:01:42.816234+0000 mgr.y (mgr.26252) 1067 : cluster [DBG] pgmap v1606: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:44.257 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:44 smithi099 bash[87147]: cluster 2024-09-13T15:01:42.816234+0000 mgr.y (mgr.26252) 1067 : cluster [DBG] pgmap v1606: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 16093/154548 objects degraded (10.413%) 2024-09-13T15:01:44.269 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:44.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:44.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:44.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:44.545 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:44.655 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:44.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:44.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:44.784 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:44.812 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:44.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:44.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:44.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:44.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:45.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:45.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:45.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:45.169 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:45.193 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:45.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:45.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:45.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:45.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:45.761 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:45.784 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:45.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:45.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:45.897 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:45 smithi177 bash[68594]: cluster 2024-09-13T15:01:44.817224+0000 mgr.y (mgr.26252) 1068 : cluster [DBG] pgmap v1607: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 15191/145530 objects degraded (10.438%) 2024-09-13T15:01:45.897 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:45 smithi177 bash[68594]: cluster 2024-09-13T15:01:44.817224+0000 mgr.y (mgr.26252) 1068 : cluster [DBG] pgmap v1607: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 15191/145530 objects degraded (10.438%) 2024-09-13T15:01:45.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:45.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:46.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:46.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:46.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:46.134 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:45 smithi099 bash[87147]: cluster 2024-09-13T15:01:44.817224+0000 mgr.y (mgr.26252) 1068 : cluster [DBG] pgmap v1607: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 15191/145530 objects degraded (10.438%) 2024-09-13T15:01:46.134 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:45 smithi099 bash[87147]: cluster 2024-09-13T15:01:44.817224+0000 mgr.y (mgr.26252) 1068 : cluster [DBG] pgmap v1607: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 15191/145530 objects degraded (10.438%) 2024-09-13T15:01:46.135 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:45 smithi099 bash[84576]: cluster 2024-09-13T15:01:44.817224+0000 mgr.y (mgr.26252) 1068 : cluster [DBG] pgmap v1607: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 15191/145530 objects degraded (10.438%) 2024-09-13T15:01:46.135 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:45 smithi099 bash[84576]: cluster 2024-09-13T15:01:44.817224+0000 mgr.y (mgr.26252) 1068 : cluster [DBG] pgmap v1607: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 15191/145530 objects degraded (10.438%) 2024-09-13T15:01:46.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:46.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:46.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:46.295 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:46.528 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:46.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:46.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:46.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:46.754 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:46.777 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:46.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:46.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:46.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:46.931 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:46 smithi177 bash[68594]: cluster 2024-09-13T15:01:46.473407+0000 mon.a (mon.0) 2178 : cluster [WRN] Health check update: Degraded data redundancy: 15191/145530 objects degraded (10.438%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:46.931 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:46 smithi177 bash[68594]: cluster 2024-09-13T15:01:46.473407+0000 mon.a (mon.0) 2178 : cluster [WRN] Health check update: Degraded data redundancy: 15191/145530 objects degraded (10.438%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:47.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:47.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:47.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:47.156 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:47.167 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:01:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:01:47] "GET /metrics HTTP/1.1" 200 36770 "" "Prometheus/2.51.0" 2024-09-13T15:01:47.167 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:46 smithi099 bash[84576]: cluster 2024-09-13T15:01:46.473407+0000 mon.a (mon.0) 2178 : cluster [WRN] Health check update: Degraded data redundancy: 15191/145530 objects degraded (10.438%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:47.167 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:46 smithi099 bash[84576]: cluster 2024-09-13T15:01:46.473407+0000 mon.a (mon.0) 2178 : cluster [WRN] Health check update: Degraded data redundancy: 15191/145530 objects degraded (10.438%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:47.168 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:46 smithi099 bash[87147]: cluster 2024-09-13T15:01:46.473407+0000 mon.a (mon.0) 2178 : cluster [WRN] Health check update: Degraded data redundancy: 15191/145530 objects degraded (10.438%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:47.168 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:46 smithi099 bash[87147]: cluster 2024-09-13T15:01:46.473407+0000 mon.a (mon.0) 2178 : cluster [WRN] Health check update: Degraded data redundancy: 15191/145530 objects degraded (10.438%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:47.179 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:47.182 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:01:47.182 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T15:01:47.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T15:01:47.182 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:01:47.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.182 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:8370 42.d 42:b1dbd304:::benchmark_data_smithi099_148651_object8369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1014)", 2024-09-13T15:01:47.182 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:29.386035+0000", 2024-09-13T15:01:47.182 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 497.78827932899998, 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.80407077500000002, 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 8370 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.386035+0000", 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.386035+0000", 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.386034+0000", 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.386094+0000", 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9905999999999999e-05 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.386105+0000", 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0917999999999999e-05 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.386110+0000", 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6699999999999999e-06 2024-09-13T15:01:47.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.386148+0000", 2024-09-13T15:01:47.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7311999999999998e-05 2024-09-13T15:01:47.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.386221+0000", 2024-09-13T15:01:47.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3028000000000006e-05 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.386346+0000", 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012492800000000001 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.387169+0000", 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00082332800000000002 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.388711+0000", 2024-09-13T15:01:47.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015423279999999999 2024-09-13T15:01:47.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:30.189849+0000", 2024-09-13T15:01:47.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.80113795700000001 2024-09-13T15:01:47.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:30.189994+0000", 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000145026 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:30.190106+0000", 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000111314 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.189 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.190 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:17431 42.d 42:b91bb313:::benchmark_data_smithi099_148651_object17430:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1014)", 2024-09-13T15:01:47.190 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:34.094091+0000", 2024-09-13T15:01:47.190 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 493.08022319399998, 2024-09-13T15:01:47.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.41981202699999998, 2024-09-13T15:01:47.190 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.190 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.190 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.190 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:01:47.191 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:01:47.191 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 17431 2024-09-13T15:01:47.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.191 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.094091+0000", 2024-09-13T15:01:47.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.094091+0000", 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.094090+0000", 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.094209+0000", 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000119811 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.094218+0000", 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9979999999999992e-06 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.094223+0000", 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8060000000000002e-06 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.094263+0000", 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0086000000000002e-05 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.094366+0000", 2024-09-13T15:01:47.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102614 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.094442+0000", 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6485999999999999e-05 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.096644+0000", 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022013359999999999 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.097850+0000", 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001206068 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.513750+0000", 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.41590001399999998 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.513836+0000", 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5983999999999995e-05 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:34.513903+0000", 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7337999999999998e-05 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:32157 42.9 42:9868881c:::benchmark_data_smithi099_148651_object32156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1015)", 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:43.401386+0000", 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 483.77292861799998, 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51269745, 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:01:47.198 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 32157 2024-09-13T15:01:47.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.199 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.401386+0000", 2024-09-13T15:01:47.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.401386+0000", 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.401384+0000", 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.401419+0000", 2024-09-13T15:01:47.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4687000000000002e-05 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.401431+0000", 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2444e-05 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.401437+0000", 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1349999999999997e-06 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.401486+0000", 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8463e-05 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.401608+0000", 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012236699999999999 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.401738+0000", 2024-09-13T15:01:47.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012945400000000001 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.403058+0000", 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013198820000000001 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.754689+0000", 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35163115 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.913970+0000", 2024-09-13T15:01:47.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.15928165899999999 2024-09-13T15:01:47.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.914039+0000", 2024-09-13T15:01:47.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8703999999999998e-05 2024-09-13T15:01:47.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.914083+0000", 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4014999999999998e-05 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:32158 42.9 42:933fafde:::benchmark_data_smithi099_148651_object32157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1015)", 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:43.404286+0000", 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 483.770028642, 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.510009198, 2024-09-13T15:01:47.205 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 32158 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.404286+0000", 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.404286+0000", 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.404285+0000", 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.404383+0000", 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8086999999999998e-05 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.404392+0000", 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7079999999999992e-06 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.404396+0000", 2024-09-13T15:01:47.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9700000000000001e-06 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.404424+0000", 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7339999999999999e-05 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.404501+0000", 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7639999999999995e-05 2024-09-13T15:01:47.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.404576+0000", 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4769000000000002e-05 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.405262+0000", 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000685485 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.754774+0000", 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34951239000000001 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.914165+0000", 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.15939054899999999 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.914242+0000", 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7276999999999999e-05 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:43.914295+0000", 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2944e-05 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:67946 42.9 42:94dfe01d:::benchmark_data_smithi099_148651_object67945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1018)", 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:54:12.102635+0000", 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 455.07167968300001, 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44202493300000001, 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 67946 2024-09-13T15:01:47.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.102635+0000", 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.102635+0000", 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.102633+0000", 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.102759+0000", 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012661399999999999 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.102773+0000", 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3696999999999999e-05 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.102780+0000", 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7259999999999996e-06 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.102835+0000", 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5430999999999998e-05 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.102958+0000", 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123058 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.103097+0000", 2024-09-13T15:01:47.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013892700000000001 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.105831+0000", 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002733816 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.106119+0000", 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028793299999999998 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.544572+0000", 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43845309399999999 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.544619+0000", 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.723e-05 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.544660+0000", 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0296000000000002e-05 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:67948 42.9 42:92e2b627:::benchmark_data_smithi099_148651_object67947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1018)", 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:54:12.107121+0000", 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 455.06719346300002, 2024-09-13T15:01:47.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43790504000000002, 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 67948 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.107121+0000", 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.107121+0000", 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.107120+0000", 2024-09-13T15:01:47.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.107164+0000", 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4688000000000003e-05 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.107175+0000", 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.049e-05 2024-09-13T15:01:47.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.107179+0000", 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2560000000000004e-06 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.107212+0000", 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2669999999999997e-05 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.107305+0000", 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3435000000000002e-05 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.107397+0000", 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1723000000000003e-05 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.109517+0000", 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002120444 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.110759+0000", 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012420300000000001 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.544952+0000", 2024-09-13T15:01:47.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43419274499999999 2024-09-13T15:01:47.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.544993+0000", 2024-09-13T15:01:47.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0930000000000003e-05 2024-09-13T15:01:47.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:54:12.545026+0000", 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2673999999999999e-05 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:113466 42.3 42:c543afe6:::benchmark_data_smithi099_148651_object5626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1023)", 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:55:04.303872+0000", 2024-09-13T15:01:47.228 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 402.87044190799998, 2024-09-13T15:01:47.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43661073, 2024-09-13T15:01:47.229 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.229 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.229 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.229 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:01:47.229 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:01:47.229 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 113466 2024-09-13T15:01:47.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.229 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.303872+0000", 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.303872+0000", 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.303870+0000", 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.303881+0000", 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0858e-05 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.303894+0000", 2024-09-13T15:01:47.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3380000000000001e-05 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.303901+0000", 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3579999999999999e-06 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.303961+0000", 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0581999999999997e-05 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.304066+0000", 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000105078 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.304159+0000", 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2968999999999997e-05 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.308412+0000", 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042528779999999999 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.308677+0000", 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026425800000000002 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.740392+0000", 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43171512699999998 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.740440+0000", 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8390000000000003e-05 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:04.740483+0000", 2024-09-13T15:01:47.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2991999999999998e-05 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:189134 42.d 42:b2987e88:::benchmark_data_smithi099_148651_object81294:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1030)", 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:56:50.342693+0000", 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 296.831621402, 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53031852999999995, 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.236 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 189134 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.342693+0000", 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.342693+0000", 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.342692+0000", 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.342698+0000", 2024-09-13T15:01:47.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1290000000000001e-06 2024-09-13T15:01:47.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.342716+0000", 2024-09-13T15:01:47.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8292999999999999e-05 2024-09-13T15:01:47.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.342723+0000", 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9059999999999996e-06 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.344921+0000", 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021976080000000002 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.345024+0000", 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000103779 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.345112+0000", 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7043e-05 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.353006+0000", 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0078941020000000001 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.353053+0000", 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7401000000000002e-05 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.872920+0000", 2024-09-13T15:01:47.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51986723099999999 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.872963+0000", 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2945000000000003e-05 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.873011+0000", 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8198000000000002e-05 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.243 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.244 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.244 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:189139 42.d 42:b293c712:::benchmark_data_smithi099_148651_object81299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1030)", 2024-09-13T15:01:47.244 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:56:50.360360+0000", 2024-09-13T15:01:47.244 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 296.81395462199998, 2024-09-13T15:01:47.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51300344200000003, 2024-09-13T15:01:47.245 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.245 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.245 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.245 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:01:47.245 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:01:47.245 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 189139 2024-09-13T15:01:47.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.245 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.360360+0000", 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.360360+0000", 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.360359+0000", 2024-09-13T15:01:47.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:01:47.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.360365+0000", 2024-09-13T15:01:47.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8730000000000004e-06 2024-09-13T15:01:47.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.360375+0000", 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0609e-05 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.360379+0000", 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0509999999999998e-06 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.360416+0000", 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6738e-05 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.360503+0000", 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6977000000000004e-05 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2024-09-13T15:01:47.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.360581+0000", 2024-09-13T15:01:47.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7776999999999998e-05 2024-09-13T15:01:47.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.364168+0000", 2024-09-13T15:01:47.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035872260000000002 2024-09-13T15:01:47.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.364619+0000", 2024-09-13T15:01:47.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00045161000000000002 2024-09-13T15:01:47.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.873299+0000", 2024-09-13T15:01:47.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.5086792 2024-09-13T15:01:47.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.873327+0000", 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8294999999999999e-05 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:56:50.873363+0000", 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6093999999999997e-05 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.252 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:73200 43.0 43:085fc7fa:::benchmark_data_smithi099_171278_object73199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1044)", 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:50.353737+0000", 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 116.82057725600001, 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33342248099999999, 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.253 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.254 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.254 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 73200 2024-09-13T15:01:47.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.254 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.353737+0000", 2024-09-13T15:01:47.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.353737+0000", 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.353735+0000", 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.353777+0000", 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2240999999999998e-05 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.353807+0000", 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9643000000000001e-05 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.353813+0000", 2024-09-13T15:01:47.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3289999999999999e-06 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.353862+0000", 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8415000000000003e-05 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.353987+0000", 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012573700000000001 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.354124+0000", 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013649499999999999 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.356785+0000", 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026617329999999999 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.358348+0000", 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015625109999999999 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.686898+0000", 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32854997600000002 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.687058+0000", 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016027899999999999 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:50.687159+0000", 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000101247 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128619 43.0 43:04d6bc89:::benchmark_data_smithi099_171278_object12607:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.409527+0000", 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 61.764786901999997, 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38785059900000002, 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128619 2024-09-13T15:01:47.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409527+0000", 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409527+0000", 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409526+0000", 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409532+0000", 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.584e-06 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409546+0000", 2024-09-13T15:01:47.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4138000000000001e-05 2024-09-13T15:01:47.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409552+0000", 2024-09-13T15:01:47.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9510000000000003e-06 2024-09-13T15:01:47.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409606+0000", 2024-09-13T15:01:47.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3226000000000002e-05 2024-09-13T15:01:47.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409723+0000", 2024-09-13T15:01:47.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116926 2024-09-13T15:01:47.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409823+0000", 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100393 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.412414+0000", 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002590677 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.413247+0000", 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000833518 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.797111+0000", 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38386384200000001 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.797251+0000", 2024-09-13T15:01:47.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014034599999999999 2024-09-13T15:01:47.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.797378+0000", 2024-09-13T15:01:47.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012667000000000001 2024-09-13T15:01:47.268 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.268 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.268 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128624 43.e 43:78dd2bd4:::benchmark_data_smithi099_171278_object12612:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.414929+0000", 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 61.759385512999998, 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39337724299999999, 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128624 2024-09-13T15:01:47.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414929+0000", 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414929+0000", 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414927+0000", 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414933+0000", 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.066e-06 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414949+0000", 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6039999999999999e-05 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414956+0000", 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7120000000000003e-06 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.415007+0000", 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0877000000000001e-05 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.415121+0000", 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011437700000000001 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0", 2024-09-13T15:01:47.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.415241+0000", 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000119739 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.424992+0000", 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0097505930000000001 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.808107+0000", 2024-09-13T15:01:47.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38311545000000002 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.808180+0000", 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3110999999999998e-05 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.808306+0000", 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000125697 2024-09-13T15:01:47.275 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128626 43.e 43:727adca4:::benchmark_data_smithi099_171278_object12614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.414969+0000", 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 61.759345654000001, 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39355641800000002, 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.276 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128626 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414969+0000", 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414969+0000", 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414968+0000", 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414971+0000", 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.785e-06 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414978+0000", 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3200000000000002e-06 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414982+0000", 2024-09-13T15:01:47.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5619999999999999e-06 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.420930+0000", 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059482939999999998 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.421039+0000", 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108955 2024-09-13T15:01:47.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0", 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.421146+0000", 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010697899999999999 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425095+0000", 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039488969999999998 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.808354+0000", 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38325873599999999 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.808418+0000", 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4264999999999995e-05 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.808525+0000", 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107123 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128637 43.e 43:7d027c9b:::benchmark_data_smithi099_171278_object12625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.434266+0000", 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 61.740048725000001, 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37446178099999999, 2024-09-13T15:01:47.283 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128637 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434266+0000", 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434266+0000", 2024-09-13T15:01:47.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434264+0000", 2024-09-13T15:01:47.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:47.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434271+0000", 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2849999999999997e-06 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434283+0000", 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1633e-05 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434289+0000", 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1909999999999997e-06 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434349+0000", 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0405000000000002e-05 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434451+0000", 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010153 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0", 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434526+0000", 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4795999999999993e-05 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.438001+0000", 2024-09-13T15:01:47.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034751970000000002 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.808569+0000", 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37056846799999998 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.808628+0000", 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9036999999999997e-05 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.808727+0000", 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9105999999999996e-05 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:130661 43.b 43:d311885a:::benchmark_data_smithi099_171278_object14649:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:47.811151+0000", 2024-09-13T15:01:47.290 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 59.363163100000001, 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38861761299999997, 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 130661 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.811151+0000", 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.811151+0000", 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.811150+0000", 2024-09-13T15:01:47.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.811155+0000", 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.578e-06 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.811163+0000", 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9719999999999992e-06 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.811167+0000", 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3059999999999999e-06 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.818849+0000", 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0076817980000000001 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.818978+0000", 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012892900000000001 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2", 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.819120+0000", 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000142163 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.842175+0000", 2024-09-13T15:01:47.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023055106999999998 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.199501+0000", 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35732544799999999 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.199592+0000", 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0938000000000002e-05 2024-09-13T15:01:47.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.199769+0000", 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017717500000000001 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:130674 43.b 43:d3a5f791:::benchmark_data_smithi099_171278_object14662:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:01:47.297 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:47.819745+0000", 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 59.354568987999997, 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38030520400000001, 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 130674 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.819745+0000", 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.819745+0000", 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.299 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.819745+0000", 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.819749+0000", 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9149999999999998e-06 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.819760+0000", 2024-09-13T15:01:47.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1250000000000001e-05 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.819765+0000", 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1039999999999998e-06 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.822750+0000", 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029853119999999999 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.822867+0000", 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000117145 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2", 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.822962+0000", 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4259999999999995e-05 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.842181+0000", 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.019219327000000001 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.199836+0000", 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.357654729 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.199910+0000", 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4318000000000002e-05 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.200050+0000", 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014051600000000001 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.304 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:130683 43.b 43:df837212:::benchmark_data_smithi099_171278_object14671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:47.828966+0000", 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 59.345348624000003, 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37333645799999998, 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 130683 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.828966+0000", 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.828966+0000", 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.828965+0000", 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:01:47.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.828968+0000", 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.399e-06 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.828975+0000", 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1219999999999997e-06 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.828978+0000", 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4000000000000001e-06 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.835243+0000", 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062642330000000001 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.835321+0000", 2024-09-13T15:01:47.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8656e-05 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2", 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.835407+0000", 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5873999999999998e-05 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.842184+0000", 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067765209999999998 2024-09-13T15:01:47.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.202023+0000", 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35983883100000003 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.202125+0000", 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010229 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.202302+0000", 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017721100000000001 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:130687 43.b 43:d66389f6:::benchmark_data_smithi099_171278_object14675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:47.829978+0000", 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 59.344336726000002, 2024-09-13T15:01:47.311 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37413002499999998, 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 130687 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.829978+0000", 2024-09-13T15:01:47.312 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.829978+0000", 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.829977+0000", 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:01:47.313 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.829982+0000", 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7469999999999996e-06 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.829993+0000", 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1591e-05 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.314 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.829998+0000", 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8459999999999999e-06 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.839162+0000", 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0091638699999999993 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.839272+0000", 2024-09-13T15:01:47.315 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011061900000000001 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2", 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.839388+0000", 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000115426 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:47.842273+0000", 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028856229999999999 2024-09-13T15:01:47.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.203905+0000", 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36163152700000001 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.203976+0000", 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0926000000000005e-05 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:48.204108+0000", 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000131629 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:170457 43.b 43:df17b88d:::benchmark_data_smithi099_171278_object54445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1049)", 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:01:28.382147+0000", 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 18.792167126999999, 2024-09-13T15:01:47.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37503103500000001, 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 170457 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.319 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.382147+0000", 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.382147+0000", 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.382147+0000", 2024-09-13T15:01:47.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.382149+0000", 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7990000000000001e-06 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.382153+0000", 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8910000000000003e-06 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.382156+0000", 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.125e-06 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.385808+0000", 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036522780000000001 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.322 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.385912+0000", 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000103701 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2", 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.386018+0000", 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000106493 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.394950+0000", 2024-09-13T15:01:47.323 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0089320010000000002 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.757048+0000", 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36209756799999998 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.757096+0000", 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8161999999999997e-05 2024-09-13T15:01:47.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.757178+0000", 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2359000000000002e-05 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:170463 43.b 43:d9604ceb:::benchmark_data_smithi099_171278_object54451:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1049)", 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:01:28.389116+0000", 2024-09-13T15:01:47.325 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 18.785198529999999, 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38264109299999999, 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 170463 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.389116+0000", 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.389116+0000", 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.389115+0000", 2024-09-13T15:01:47.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.389120+0000", 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.57e-06 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.389129+0000", 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2119999999999992e-06 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.389134+0000", 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8570000000000003e-06 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.390281+0000", 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011462829999999999 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.329 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.390395+0000", 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011489699999999999 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2", 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.390968+0000", 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057259599999999998 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.395098+0000", 2024-09-13T15:01:47.330 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041298740000000004 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.771631+0000", 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37653324700000002 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.771676+0000", 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5198000000000003e-05 2024-09-13T15:01:47.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:01:47.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:01:47.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:01:47.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:28.771757+0000", 2024-09-13T15:01:47.332 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0500000000000005e-05 2024-09-13T15:01:47.332 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.332 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.332 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.332 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:01:47.332 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:01:47.332 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:01:47.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:47.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:47.518 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:47.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:47.621 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:47.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:47.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:47.759 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:47.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:47.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:47.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:47.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:47.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:48.021 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:48.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:48.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:47 smithi099 bash[84576]: cluster 2024-09-13T15:01:46.818614+0000 mgr.y (mgr.26252) 1069 : cluster [DBG] pgmap v1608: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 14939/143316 objects degraded (10.424%) 2024-09-13T15:01:48.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:47 smithi099 bash[84576]: cluster 2024-09-13T15:01:46.818614+0000 mgr.y (mgr.26252) 1069 : cluster [DBG] pgmap v1608: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 14939/143316 objects degraded (10.424%) 2024-09-13T15:01:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:47 smithi099 bash[87147]: cluster 2024-09-13T15:01:46.818614+0000 mgr.y (mgr.26252) 1069 : cluster [DBG] pgmap v1608: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 14939/143316 objects degraded (10.424%) 2024-09-13T15:01:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:47 smithi099 bash[87147]: cluster 2024-09-13T15:01:46.818614+0000 mgr.y (mgr.26252) 1069 : cluster [DBG] pgmap v1608: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 14939/143316 objects degraded (10.424%) 2024-09-13T15:01:48.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:48.175 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:48.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:48.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:48.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:48.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:47 smithi177 bash[68594]: cluster 2024-09-13T15:01:46.818614+0000 mgr.y (mgr.26252) 1069 : cluster [DBG] pgmap v1608: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 14939/143316 objects degraded (10.424%) 2024-09-13T15:01:48.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:47 smithi177 bash[68594]: cluster 2024-09-13T15:01:46.818614+0000 mgr.y (mgr.26252) 1069 : cluster [DBG] pgmap v1608: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 14939/143316 objects degraded (10.424%) 2024-09-13T15:01:48.409 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:48.412 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:01:48.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:48.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:48.538 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:48.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:48.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:48.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:48.696 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:48.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:48.787 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2024-09-13T15:01:48.787 INFO:teuthology.orchestra.run.smithi099.stderr:0 2024-09-13T15:01:48.787 INFO:teuthology.orchestra.run.smithi099.stderr:1 2024-09-13T15:01:48.787 INFO:teuthology.orchestra.run.smithi099.stderr:2 2024-09-13T15:01:48.787 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2024-09-13T15:01:48.788 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2024-09-13T15:01:48.788 INFO:teuthology.orchestra.run.smithi099.stderr:bench [] [] [] [] 2024-09-13T15:01:48.788 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs debug_inject_read_zeros 2024-09-13T15:01:48.788 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs files list 2024-09-13T15:01:48.788 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs stats 2024-09-13T15:01:48.788 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator dump block 2024-09-13T15:01:48.788 INFO:teuthology.orchestra.run.smithi099.stderr:Error EINVAL: invalid command 2024-09-13T15:01:48.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:48.823 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:48.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:48.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:48.954 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:48.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:49.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:49.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:48 smithi099 bash[87147]: audit 2024-09-13T15:01:48.297009+0000 mon.a (mon.0) 2179 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:49.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:48 smithi099 bash[87147]: audit 2024-09-13T15:01:48.297009+0000 mon.a (mon.0) 2179 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:49.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:48 smithi099 bash[84576]: audit 2024-09-13T15:01:48.297009+0000 mon.a (mon.0) 2179 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:49.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:48 smithi099 bash[84576]: audit 2024-09-13T15:01:48.297009+0000 mon.a (mon.0) 2179 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:49.090 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:49.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:49.212 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:49.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:49.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:49.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:48 smithi177 bash[68594]: audit 2024-09-13T15:01:48.297009+0000 mon.a (mon.0) 2179 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:49.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:48 smithi177 bash[68594]: audit 2024-09-13T15:01:48.297009+0000 mon.a (mon.0) 2179 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:49.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:49.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:49.447 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:49.473 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:49.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:49.580 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:49.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:49.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:49.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:49.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:49.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:49.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:49.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:50.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:50.090 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:50.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:50.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:50.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:49 smithi177 bash[68594]: cluster 2024-09-13T15:01:48.819592+0000 mgr.y (mgr.26252) 1070 : cluster [DBG] pgmap v1609: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 14691/140982 objects degraded (10.420%) 2024-09-13T15:01:50.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:49 smithi177 bash[68594]: cluster 2024-09-13T15:01:48.819592+0000 mgr.y (mgr.26252) 1070 : cluster [DBG] pgmap v1609: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 14691/140982 objects degraded (10.420%) 2024-09-13T15:01:50.219 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:50.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:50.324 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:50.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:49 smithi099 bash[84576]: cluster 2024-09-13T15:01:48.819592+0000 mgr.y (mgr.26252) 1070 : cluster [DBG] pgmap v1609: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 14691/140982 objects degraded (10.420%) 2024-09-13T15:01:50.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:49 smithi099 bash[84576]: cluster 2024-09-13T15:01:48.819592+0000 mgr.y (mgr.26252) 1070 : cluster [DBG] pgmap v1609: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 14691/140982 objects degraded (10.420%) 2024-09-13T15:01:50.329 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:49 smithi099 bash[87147]: cluster 2024-09-13T15:01:48.819592+0000 mgr.y (mgr.26252) 1070 : cluster [DBG] pgmap v1609: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 14691/140982 objects degraded (10.420%) 2024-09-13T15:01:50.329 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:49 smithi099 bash[87147]: cluster 2024-09-13T15:01:48.819592+0000 mgr.y (mgr.26252) 1070 : cluster [DBG] pgmap v1609: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 14691/140982 objects degraded (10.420%) 2024-09-13T15:01:50.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:50.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:50.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:50.471 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:01:50 smithi177 bash[61439]: ts=2024-09-13T15:01:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:01:50.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:50.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:50.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:50.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:50.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:50.755 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:50.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:50.860 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:50.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:50.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:50.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:51.009 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:51.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:51.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:51.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:51.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:51.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:51.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:51.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:51.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:51.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:51.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:51.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:51.565 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:51.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:51.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:51.692 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:51.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:51.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:51.823 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:51.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:51.925 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:51.951 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:51.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:52.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:52.066 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:51 smithi099 bash[84576]: cluster 2024-09-13T15:01:50.820701+0000 mgr.y (mgr.26252) 1071 : cluster [DBG] pgmap v1610: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:52.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:51 smithi099 bash[84576]: cluster 2024-09-13T15:01:50.820701+0000 mgr.y (mgr.26252) 1071 : cluster [DBG] pgmap v1610: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:52.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:51 smithi099 bash[84576]: cluster 2024-09-13T15:01:51.474297+0000 mon.a (mon.0) 2180 : cluster [WRN] Health check update: Degraded data redundancy: 13653/130341 objects degraded (10.475%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:52.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:51 smithi099 bash[84576]: cluster 2024-09-13T15:01:51.474297+0000 mon.a (mon.0) 2180 : cluster [WRN] Health check update: Degraded data redundancy: 13653/130341 objects degraded (10.475%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:52.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:51 smithi099 bash[87147]: cluster 2024-09-13T15:01:50.820701+0000 mgr.y (mgr.26252) 1071 : cluster [DBG] pgmap v1610: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:52.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:51 smithi099 bash[87147]: cluster 2024-09-13T15:01:50.820701+0000 mgr.y (mgr.26252) 1071 : cluster [DBG] pgmap v1610: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:52.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:51 smithi099 bash[87147]: cluster 2024-09-13T15:01:51.474297+0000 mon.a (mon.0) 2180 : cluster [WRN] Health check update: Degraded data redundancy: 13653/130341 objects degraded (10.475%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:52.067 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:51 smithi099 bash[87147]: cluster 2024-09-13T15:01:51.474297+0000 mon.a (mon.0) 2180 : cluster [WRN] Health check update: Degraded data redundancy: 13653/130341 objects degraded (10.475%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:52.084 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:52.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:52.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:51 smithi177 bash[68594]: cluster 2024-09-13T15:01:50.820701+0000 mgr.y (mgr.26252) 1071 : cluster [DBG] pgmap v1610: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:52.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:51 smithi177 bash[68594]: cluster 2024-09-13T15:01:50.820701+0000 mgr.y (mgr.26252) 1071 : cluster [DBG] pgmap v1610: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:52.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:51 smithi177 bash[68594]: cluster 2024-09-13T15:01:51.474297+0000 mon.a (mon.0) 2180 : cluster [WRN] Health check update: Degraded data redundancy: 13653/130341 objects degraded (10.475%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:52.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:51 smithi177 bash[68594]: cluster 2024-09-13T15:01:51.474297+0000 mon.a (mon.0) 2180 : cluster [WRN] Health check update: Degraded data redundancy: 13653/130341 objects degraded (10.475%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:52.142 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T15:01:52.143 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T15:01:52.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-09-13T15:01:52.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:52.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:52.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:52.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:52.350 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:52.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:52.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:52.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:52.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:52.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:52.614 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:52.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:52.721 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:52.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:52.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:52.840 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:01:52.841 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:52.859 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:52.866 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:01:52.867 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:52.882 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:52.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:52.895 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:01:52.896 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:52.902 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:01:52.936 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:52.937 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:01:52.965 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:01:52.965 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:52.986 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:52.995 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:52 smithi099 bash[84576]: audit 2024-09-13T15:01:52.789486+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.99:0/818613235' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:52.995 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:52 smithi099 bash[84576]: audit 2024-09-13T15:01:52.789486+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.99:0/818613235' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:52.996 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:52 smithi099 bash[87147]: audit 2024-09-13T15:01:52.789486+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.99:0/818613235' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:52.996 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:52 smithi099 bash[87147]: audit 2024-09-13T15:01:52.789486+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.99:0/818613235' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:52.999 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:01:52.999 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:53.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:53.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:53.031 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:01:53.031 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:53.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:53.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:52 smithi177 bash[68594]: audit 2024-09-13T15:01:52.789486+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.99:0/818613235' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:53.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:52 smithi177 bash[68594]: audit 2024-09-13T15:01:52.789486+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.99:0/818613235' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:53.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:53.257 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:53.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:53.373 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:53.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:53.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:53.539 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:53.645 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:53.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:53.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:53.785 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:53.823 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:53.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:53.925 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:01:53.928 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:53.948 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:53.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:54.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:54.061 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:53 smithi177 bash[68594]: cluster 2024-09-13T15:01:52.821952+0000 mgr.y (mgr.26252) 1072 : cluster [DBG] pgmap v1611: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:54.061 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:53 smithi177 bash[68594]: cluster 2024-09-13T15:01:52.821952+0000 mgr.y (mgr.26252) 1072 : cluster [DBG] pgmap v1611: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:54.061 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:53 smithi177 bash[68594]: audit 2024-09-13T15:01:53.225780+0000 mon.a (mon.0) 2181 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:54.062 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:53 smithi177 bash[68594]: audit 2024-09-13T15:01:53.225780+0000 mon.a (mon.0) 2181 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:54.075 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:54.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:54.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:54.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:54.308 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:54.330 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:54.348 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:53 smithi099 bash[84576]: cluster 2024-09-13T15:01:52.821952+0000 mgr.y (mgr.26252) 1072 : cluster [DBG] pgmap v1611: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:54.348 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:53 smithi099 bash[84576]: cluster 2024-09-13T15:01:52.821952+0000 mgr.y (mgr.26252) 1072 : cluster [DBG] pgmap v1611: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:54.348 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:53 smithi099 bash[84576]: audit 2024-09-13T15:01:53.225780+0000 mon.a (mon.0) 2181 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:54.348 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:53 smithi099 bash[84576]: audit 2024-09-13T15:01:53.225780+0000 mon.a (mon.0) 2181 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:54.348 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:53 smithi099 bash[87147]: cluster 2024-09-13T15:01:52.821952+0000 mgr.y (mgr.26252) 1072 : cluster [DBG] pgmap v1611: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:54.349 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:53 smithi099 bash[87147]: cluster 2024-09-13T15:01:52.821952+0000 mgr.y (mgr.26252) 1072 : cluster [DBG] pgmap v1611: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13653/130341 objects degraded (10.475%) 2024-09-13T15:01:54.349 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:53 smithi099 bash[87147]: audit 2024-09-13T15:01:53.225780+0000 mon.a (mon.0) 2181 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:54.349 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:53 smithi099 bash[87147]: audit 2024-09-13T15:01:53.225780+0000 mon.a (mon.0) 2181 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:54.433 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:54.461 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:54.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:54.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:54.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:54.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:54.711 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:54.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:54.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:54.822 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:01:54.822 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:01:54.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:54.858 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:54.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:54.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:54.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[87147]: cluster 2024-09-13T15:01:53.917149+0000 mon.a (mon.0) 2182 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:54.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[87147]: cluster 2024-09-13T15:01:53.917149+0000 mon.a (mon.0) 2182 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:54.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[87147]: audit 2024-09-13T15:01:53.924224+0000 mon.a (mon.0) 2183 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:54.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[87147]: audit 2024-09-13T15:01:53.924224+0000 mon.a (mon.0) 2183 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:54.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[87147]: cluster 2024-09-13T15:01:53.927188+0000 mon.a (mon.0) 2184 : cluster [DBG] osdmap e1051: 8 total, 7 up, 8 in 2024-09-13T15:01:54.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[87147]: cluster 2024-09-13T15:01:53.927188+0000 mon.a (mon.0) 2184 : cluster [DBG] osdmap e1051: 8 total, 7 up, 8 in 2024-09-13T15:01:54.971 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[84576]: cluster 2024-09-13T15:01:53.917149+0000 mon.a (mon.0) 2182 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:54.971 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[84576]: cluster 2024-09-13T15:01:53.917149+0000 mon.a (mon.0) 2182 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:54.971 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[84576]: audit 2024-09-13T15:01:53.924224+0000 mon.a (mon.0) 2183 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:54.972 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[84576]: audit 2024-09-13T15:01:53.924224+0000 mon.a (mon.0) 2183 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:54.972 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[84576]: cluster 2024-09-13T15:01:53.927188+0000 mon.a (mon.0) 2184 : cluster [DBG] osdmap e1051: 8 total, 7 up, 8 in 2024-09-13T15:01:54.972 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:54 smithi099 bash[84576]: cluster 2024-09-13T15:01:53.927188+0000 mon.a (mon.0) 2184 : cluster [DBG] osdmap e1051: 8 total, 7 up, 8 in 2024-09-13T15:01:54.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:54.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:55.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:55.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:55.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:55.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:55.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:54 smithi177 bash[68594]: cluster 2024-09-13T15:01:53.917149+0000 mon.a (mon.0) 2182 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:55.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:54 smithi177 bash[68594]: cluster 2024-09-13T15:01:53.917149+0000 mon.a (mon.0) 2182 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:55.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:54 smithi177 bash[68594]: audit 2024-09-13T15:01:53.924224+0000 mon.a (mon.0) 2183 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:55.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:54 smithi177 bash[68594]: audit 2024-09-13T15:01:53.924224+0000 mon.a (mon.0) 2183 : audit [INF] from='client.? 172.21.15.99:0/2693378820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:55.219 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:54 smithi177 bash[68594]: cluster 2024-09-13T15:01:53.927188+0000 mon.a (mon.0) 2184 : cluster [DBG] osdmap e1051: 8 total, 7 up, 8 in 2024-09-13T15:01:55.219 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:54 smithi177 bash[68594]: cluster 2024-09-13T15:01:53.927188+0000 mon.a (mon.0) 2184 : cluster [DBG] osdmap e1051: 8 total, 7 up, 8 in 2024-09-13T15:01:55.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:55.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:55.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:55.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:55.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:55.509 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:55.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:55.614 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:55.638 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:55.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:55.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:55.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:55.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:55.882 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:56.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:55 smithi099 bash[87147]: cluster 2024-09-13T15:01:54.822721+0000 mgr.y (mgr.26252) 1073 : cluster [DBG] pgmap v1613: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 983 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:56.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:55 smithi099 bash[87147]: cluster 2024-09-13T15:01:54.822721+0000 mgr.y (mgr.26252) 1073 : cluster [DBG] pgmap v1613: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 983 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:55 smithi099 bash[84576]: cluster 2024-09-13T15:01:54.822721+0000 mgr.y (mgr.26252) 1073 : cluster [DBG] pgmap v1613: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 983 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:55 smithi099 bash[84576]: cluster 2024-09-13T15:01:54.822721+0000 mgr.y (mgr.26252) 1073 : cluster [DBG] pgmap v1613: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 983 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:56.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:55 smithi177 bash[68594]: cluster 2024-09-13T15:01:54.822721+0000 mgr.y (mgr.26252) 1073 : cluster [DBG] pgmap v1613: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 983 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:55 smithi177 bash[68594]: cluster 2024-09-13T15:01:54.822721+0000 mgr.y (mgr.26252) 1073 : cluster [DBG] pgmap v1613: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 983 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:56.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:56.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:56.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:56.710 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:56.733 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:56.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:56.823 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:01:56.836 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:56.864 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:56.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:56.970 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:57.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:57.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:57.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:56 smithi099 bash[84576]: cluster 2024-09-13T15:01:56.475089+0000 mon.a (mon.0) 2185 : cluster [WRN] Health check update: Degraded data redundancy: 12136/116001 objects degraded (10.462%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:57.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:56 smithi099 bash[84576]: cluster 2024-09-13T15:01:56.475089+0000 mon.a (mon.0) 2185 : cluster [WRN] Health check update: Degraded data redundancy: 12136/116001 objects degraded (10.462%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:57.055 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:56 smithi099 bash[87147]: cluster 2024-09-13T15:01:56.475089+0000 mon.a (mon.0) 2185 : cluster [WRN] Health check update: Degraded data redundancy: 12136/116001 objects degraded (10.462%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:57.055 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:56 smithi099 bash[87147]: cluster 2024-09-13T15:01:56.475089+0000 mon.a (mon.0) 2185 : cluster [WRN] Health check update: Degraded data redundancy: 12136/116001 objects degraded (10.462%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:57.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:56 smithi177 bash[68594]: cluster 2024-09-13T15:01:56.475089+0000 mon.a (mon.0) 2185 : cluster [WRN] Health check update: Degraded data redundancy: 12136/116001 objects degraded (10.462%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:57.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:56 smithi177 bash[68594]: cluster 2024-09-13T15:01:56.475089+0000 mon.a (mon.0) 2185 : cluster [WRN] Health check update: Degraded data redundancy: 12136/116001 objects degraded (10.462%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:57.111 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:57.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:57.249 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:57.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:57.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:57.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:01:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:01:57] "GET /metrics HTTP/1.1" 200 36769 "" "Prometheus/2.51.0" 2024-09-13T15:01:57.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:57.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:57.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:57.508 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:57.628 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:01:57.942 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:57 smithi099 bash[87147]: cluster 2024-09-13T15:01:56.823317+0000 mgr.y (mgr.26252) 1074 : cluster [DBG] pgmap v1614: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:57.942 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:57 smithi099 bash[87147]: cluster 2024-09-13T15:01:56.823317+0000 mgr.y (mgr.26252) 1074 : cluster [DBG] pgmap v1614: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:58.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:58.242 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:58.250 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:57 smithi177 bash[68594]: cluster 2024-09-13T15:01:56.823317+0000 mgr.y (mgr.26252) 1074 : cluster [DBG] pgmap v1614: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:58.250 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:57 smithi177 bash[68594]: cluster 2024-09-13T15:01:56.823317+0000 mgr.y (mgr.26252) 1074 : cluster [DBG] pgmap v1614: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:58.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:58.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:58.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:57 smithi099 bash[84576]: cluster 2024-09-13T15:01:56.823317+0000 mgr.y (mgr.26252) 1074 : cluster [DBG] pgmap v1614: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:58.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:57 smithi099 bash[84576]: cluster 2024-09-13T15:01:56.823317+0000 mgr.y (mgr.26252) 1074 : cluster [DBG] pgmap v1614: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 12136/116001 objects degraded (10.462%) 2024-09-13T15:01:58.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:58.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:58.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:58.557 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:58.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.605 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:58.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:58.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:58.728 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:58.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:58.835 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:58.860 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:58.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:58.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:58.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:59.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:59.117 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:59.144 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:59.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:59.250 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:59.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:59.355 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:59.380 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:59.483 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:59.540 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:59.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:59.644 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:59.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:59.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:59.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:59.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:59.917 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:59.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:00.092 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:59 smithi177 bash[68594]: cluster 2024-09-13T15:01:58.823989+0000 mgr.y (mgr.26252) 1075 : cluster [DBG] pgmap v1615: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 11860/113613 objects degraded (10.439%) 2024-09-13T15:02:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:01:59 smithi177 bash[68594]: cluster 2024-09-13T15:01:58.823989+0000 mgr.y (mgr.26252) 1075 : cluster [DBG] pgmap v1615: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 11860/113613 objects degraded (10.439%) 2024-09-13T15:02:00.124 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:00.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:00.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:00.257 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:00.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:00.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:59 smithi099 bash[87147]: cluster 2024-09-13T15:01:58.823989+0000 mgr.y (mgr.26252) 1075 : cluster [DBG] pgmap v1615: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 11860/113613 objects degraded (10.439%) 2024-09-13T15:02:00.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:01:59 smithi099 bash[87147]: cluster 2024-09-13T15:01:58.823989+0000 mgr.y (mgr.26252) 1075 : cluster [DBG] pgmap v1615: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 11860/113613 objects degraded (10.439%) 2024-09-13T15:02:00.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:59 smithi099 bash[84576]: cluster 2024-09-13T15:01:58.823989+0000 mgr.y (mgr.26252) 1075 : cluster [DBG] pgmap v1615: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 11860/113613 objects degraded (10.439%) 2024-09-13T15:02:00.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:01:59 smithi099 bash[84576]: cluster 2024-09-13T15:01:58.823989+0000 mgr.y (mgr.26252) 1075 : cluster [DBG] pgmap v1615: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 11860/113613 objects degraded (10.439%) 2024-09-13T15:02:00.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:00.391 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:00.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:00.505 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:02:00 smithi177 bash[61439]: ts=2024-09-13T15:02:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:00.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:00.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:00.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:00.628 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:02:00.628 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:02:00.650 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:00.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:00.754 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:00.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:00.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:00.880 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:00.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:01.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:01.052 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:01.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:01.155 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:01.167 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:01.175 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:01.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:01.280 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:01.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:01.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:01.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:01.429 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:01.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:01.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:01.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:01.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:01.659 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:01.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:01.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:01.782 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:01.804 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:01.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:01.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:01.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:01.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:01.992 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:02:02.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:02.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:01 smithi099 bash[84576]: cluster 2024-09-13T15:02:00.824794+0000 mgr.y (mgr.26252) 1076 : cluster [DBG] pgmap v1616: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:02.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:01 smithi099 bash[84576]: cluster 2024-09-13T15:02:00.824794+0000 mgr.y (mgr.26252) 1076 : cluster [DBG] pgmap v1616: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:02.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:01 smithi099 bash[84576]: cluster 2024-09-13T15:02:01.475977+0000 mon.a (mon.0) 2186 : cluster [WRN] Health check update: Degraded data redundancy: 10691/102105 objects degraded (10.471%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:02.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:01 smithi099 bash[84576]: cluster 2024-09-13T15:02:01.475977+0000 mon.a (mon.0) 2186 : cluster [WRN] Health check update: Degraded data redundancy: 10691/102105 objects degraded (10.471%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:02.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:01 smithi099 bash[87147]: cluster 2024-09-13T15:02:00.824794+0000 mgr.y (mgr.26252) 1076 : cluster [DBG] pgmap v1616: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:02.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:01 smithi099 bash[87147]: cluster 2024-09-13T15:02:00.824794+0000 mgr.y (mgr.26252) 1076 : cluster [DBG] pgmap v1616: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:02.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:01 smithi099 bash[87147]: cluster 2024-09-13T15:02:01.475977+0000 mon.a (mon.0) 2186 : cluster [WRN] Health check update: Degraded data redundancy: 10691/102105 objects degraded (10.471%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:02.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:01 smithi099 bash[87147]: cluster 2024-09-13T15:02:01.475977+0000 mon.a (mon.0) 2186 : cluster [WRN] Health check update: Degraded data redundancy: 10691/102105 objects degraded (10.471%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:02.070 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:02.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:02.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:02.180 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:02.205 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:02.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:02.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:02.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:01 smithi177 bash[68594]: cluster 2024-09-13T15:02:00.824794+0000 mgr.y (mgr.26252) 1076 : cluster [DBG] pgmap v1616: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:02.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:01 smithi177 bash[68594]: cluster 2024-09-13T15:02:00.824794+0000 mgr.y (mgr.26252) 1076 : cluster [DBG] pgmap v1616: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:02.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:01 smithi177 bash[68594]: cluster 2024-09-13T15:02:01.475977+0000 mon.a (mon.0) 2186 : cluster [WRN] Health check update: Degraded data redundancy: 10691/102105 objects degraded (10.471%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:02.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:01 smithi177 bash[68594]: cluster 2024-09-13T15:02:01.475977+0000 mon.a (mon.0) 2186 : cluster [WRN] Health check update: Degraded data redundancy: 10691/102105 objects degraded (10.471%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:02.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:02.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:02.449 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:02.480 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:02.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:02.583 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:02.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:02.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:02.732 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:02.760 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:02.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:02.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:02.903 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:02.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:03.010 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:03.031 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:03.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:03.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:03.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:03.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:03.254 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:03.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:03.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:03.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:03.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:03.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:03.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:03.560 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:03.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:03.663 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:03.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:03.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:03.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:03.819 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:03.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:03.926 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:03.949 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:03.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:04.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:04.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:04.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:04.180 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:04.191 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:03 smithi099 bash[84576]: cluster 2024-09-13T15:02:02.825506+0000 mgr.y (mgr.26252) 1077 : cluster [DBG] pgmap v1617: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:04.191 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:03 smithi099 bash[84576]: cluster 2024-09-13T15:02:02.825506+0000 mgr.y (mgr.26252) 1077 : cluster [DBG] pgmap v1617: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:04.191 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:03 smithi099 bash[84576]: audit 2024-09-13T15:02:03.294368+0000 mon.a (mon.0) 2187 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:04.191 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:03 smithi099 bash[84576]: audit 2024-09-13T15:02:03.294368+0000 mon.a (mon.0) 2187 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:04.191 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:03 smithi099 bash[87147]: cluster 2024-09-13T15:02:02.825506+0000 mgr.y (mgr.26252) 1077 : cluster [DBG] pgmap v1617: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:04.191 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:03 smithi099 bash[87147]: cluster 2024-09-13T15:02:02.825506+0000 mgr.y (mgr.26252) 1077 : cluster [DBG] pgmap v1617: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:04.192 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:03 smithi099 bash[87147]: audit 2024-09-13T15:02:03.294368+0000 mon.a (mon.0) 2187 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:04.192 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:03 smithi099 bash[87147]: audit 2024-09-13T15:02:03.294368+0000 mon.a (mon.0) 2187 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:04.210 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:04.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:03 smithi177 bash[68594]: cluster 2024-09-13T15:02:02.825506+0000 mgr.y (mgr.26252) 1077 : cluster [DBG] pgmap v1617: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:04.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:03 smithi177 bash[68594]: cluster 2024-09-13T15:02:02.825506+0000 mgr.y (mgr.26252) 1077 : cluster [DBG] pgmap v1617: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 940 op/s; 10691/102105 objects degraded (10.471%) 2024-09-13T15:02:04.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:03 smithi177 bash[68594]: audit 2024-09-13T15:02:03.294368+0000 mon.a (mon.0) 2187 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:04.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:03 smithi177 bash[68594]: audit 2024-09-13T15:02:03.294368+0000 mon.a (mon.0) 2187 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:04.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:04.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:04.354 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:04.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:04.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:04.483 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:04.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:04.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:04.617 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:04.720 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:04.743 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:04.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:04.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:04.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:04.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:04.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:04.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:05.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:05.103 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:05.125 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:05.227 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:05.363 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:05.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:05.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:05.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:05.512 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:05.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:05.615 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:05.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:05.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:05.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:05.774 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:05.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:05.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:05.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:05.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:06.000 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:06.010 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:05 smithi177 bash[68594]: cluster 2024-09-13T15:02:04.826479+0000 mgr.y (mgr.26252) 1078 : cluster [DBG] pgmap v1618: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:06.010 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:05 smithi177 bash[68594]: cluster 2024-09-13T15:02:04.826479+0000 mgr.y (mgr.26252) 1078 : cluster [DBG] pgmap v1618: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:06.022 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:06.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:06.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:06.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:06.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:06.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:06.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:05 smithi099 bash[84576]: cluster 2024-09-13T15:02:04.826479+0000 mgr.y (mgr.26252) 1078 : cluster [DBG] pgmap v1618: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:06.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:05 smithi099 bash[84576]: cluster 2024-09-13T15:02:04.826479+0000 mgr.y (mgr.26252) 1078 : cluster [DBG] pgmap v1618: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:06.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:05 smithi099 bash[87147]: cluster 2024-09-13T15:02:04.826479+0000 mgr.y (mgr.26252) 1078 : cluster [DBG] pgmap v1618: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:06.301 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:05 smithi099 bash[87147]: cluster 2024-09-13T15:02:04.826479+0000 mgr.y (mgr.26252) 1078 : cluster [DBG] pgmap v1618: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:06.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:06.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:06.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:06.495 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:06.528 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:06.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:06.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:06.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:06.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:06.755 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:06.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:06.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:06.828 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:02:06.889 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:06.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:06.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:07.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:07.031 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:06 smithi099 bash[84576]: cluster 2024-09-13T15:02:06.476667+0000 mon.a (mon.0) 2188 : cluster [WRN] Health check update: Degraded data redundancy: 9301/88812 objects degraded (10.473%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:07.031 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:06 smithi099 bash[84576]: cluster 2024-09-13T15:02:06.476667+0000 mon.a (mon.0) 2188 : cluster [WRN] Health check update: Degraded data redundancy: 9301/88812 objects degraded (10.473%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:07.031 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:06 smithi099 bash[87147]: cluster 2024-09-13T15:02:06.476667+0000 mon.a (mon.0) 2188 : cluster [WRN] Health check update: Degraded data redundancy: 9301/88812 objects degraded (10.473%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:07.032 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:06 smithi099 bash[87147]: cluster 2024-09-13T15:02:06.476667+0000 mon.a (mon.0) 2188 : cluster [WRN] Health check update: Degraded data redundancy: 9301/88812 objects degraded (10.473%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:07.052 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:07.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:07.156 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:07.178 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:07.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:07.234 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:02:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:02:07] "GET /metrics HTTP/1.1" 200 36767 "" "Prometheus/2.51.0" 2024-09-13T15:02:07.284 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:07.293 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:06 smithi177 bash[68594]: cluster 2024-09-13T15:02:06.476667+0000 mon.a (mon.0) 2188 : cluster [WRN] Health check update: Degraded data redundancy: 9301/88812 objects degraded (10.473%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:07.293 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:06 smithi177 bash[68594]: cluster 2024-09-13T15:02:06.476667+0000 mon.a (mon.0) 2188 : cluster [WRN] Health check update: Degraded data redundancy: 9301/88812 objects degraded (10.473%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:07.308 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:07.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:07.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:07.439 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:07.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:07.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:07.570 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:07.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:07.699 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:07.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:07.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:07.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:07.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:07.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:08.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:08.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:07 smithi099 bash[84576]: cluster 2024-09-13T15:02:06.827165+0000 mgr.y (mgr.26252) 1079 : cluster [DBG] pgmap v1619: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:08.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:07 smithi099 bash[84576]: cluster 2024-09-13T15:02:06.827165+0000 mgr.y (mgr.26252) 1079 : cluster [DBG] pgmap v1619: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:08.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:07 smithi099 bash[87147]: cluster 2024-09-13T15:02:06.827165+0000 mgr.y (mgr.26252) 1079 : cluster [DBG] pgmap v1619: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:08.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:07 smithi099 bash[87147]: cluster 2024-09-13T15:02:06.827165+0000 mgr.y (mgr.26252) 1079 : cluster [DBG] pgmap v1619: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:08.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:08.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:08.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:07 smithi177 bash[68594]: cluster 2024-09-13T15:02:06.827165+0000 mgr.y (mgr.26252) 1079 : cluster [DBG] pgmap v1619: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:08.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:07 smithi177 bash[68594]: cluster 2024-09-13T15:02:06.827165+0000 mgr.y (mgr.26252) 1079 : cluster [DBG] pgmap v1619: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 9301/88812 objects degraded (10.473%) 2024-09-13T15:02:08.198 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:08.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:08.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:08.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:08.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:08.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:08.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:08.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:08.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:08.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:08.612 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:08.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:08.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:08.742 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:08.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:08.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:08.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:08.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:08.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:09.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.022 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:09.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:09.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:09.148 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:09.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:09.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:09.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:09.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:09.377 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:09.400 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:09.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:09.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:09.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:09.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:09.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:09.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:09.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:09.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:09.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:09.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:09.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:09.939 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:10.046 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.057 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:09 smithi177 bash[68594]: cluster 2024-09-13T15:02:08.828183+0000 mgr.y (mgr.26252) 1080 : cluster [DBG] pgmap v1620: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 9049/86394 objects degraded (10.474%) 2024-09-13T15:02:10.057 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:09 smithi177 bash[68594]: cluster 2024-09-13T15:02:08.828183+0000 mgr.y (mgr.26252) 1080 : cluster [DBG] pgmap v1620: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 9049/86394 objects degraded (10.474%) 2024-09-13T15:02:10.073 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:10.176 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:10.207 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:10.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:09 smithi099 bash[84576]: cluster 2024-09-13T15:02:08.828183+0000 mgr.y (mgr.26252) 1080 : cluster [DBG] pgmap v1620: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 9049/86394 objects degraded (10.474%) 2024-09-13T15:02:10.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:09 smithi099 bash[84576]: cluster 2024-09-13T15:02:08.828183+0000 mgr.y (mgr.26252) 1080 : cluster [DBG] pgmap v1620: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 9049/86394 objects degraded (10.474%) 2024-09-13T15:02:10.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:09 smithi099 bash[87147]: cluster 2024-09-13T15:02:08.828183+0000 mgr.y (mgr.26252) 1080 : cluster [DBG] pgmap v1620: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 9049/86394 objects degraded (10.474%) 2024-09-13T15:02:10.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:09 smithi099 bash[87147]: cluster 2024-09-13T15:02:08.828183+0000 mgr.y (mgr.26252) 1080 : cluster [DBG] pgmap v1620: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 9049/86394 objects degraded (10.474%) 2024-09-13T15:02:10.341 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:10.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:10.460 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:02:10 smithi177 bash[61439]: ts=2024-09-13T15:02:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:10.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:10.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:10.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:10.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:10.738 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:10.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:10.842 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:10.878 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:10.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:10.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:11.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:11.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:11.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:11.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:11.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:11.253 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:11.285 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:11.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:11.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:11.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:11.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:11.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:11.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:11.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:11.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:11.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:11.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:11.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:11.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:11.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:11.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:11.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:12.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:12.057 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:11 smithi177 bash[68594]: cluster 2024-09-13T15:02:10.829165+0000 mgr.y (mgr.26252) 1081 : cluster [DBG] pgmap v1621: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:12.057 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:11 smithi177 bash[68594]: cluster 2024-09-13T15:02:10.829165+0000 mgr.y (mgr.26252) 1081 : cluster [DBG] pgmap v1621: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:12.057 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:11 smithi177 bash[68594]: cluster 2024-09-13T15:02:11.477652+0000 mon.a (mon.0) 2189 : cluster [WRN] Health check update: Degraded data redundancy: 8009/76950 objects degraded (10.408%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:12.057 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:11 smithi177 bash[68594]: cluster 2024-09-13T15:02:11.477652+0000 mon.a (mon.0) 2189 : cluster [WRN] Health check update: Degraded data redundancy: 8009/76950 objects degraded (10.408%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:12.096 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:12.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:12.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:12.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:11 smithi099 bash[84576]: cluster 2024-09-13T15:02:10.829165+0000 mgr.y (mgr.26252) 1081 : cluster [DBG] pgmap v1621: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:12.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:11 smithi099 bash[84576]: cluster 2024-09-13T15:02:10.829165+0000 mgr.y (mgr.26252) 1081 : cluster [DBG] pgmap v1621: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:12.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:11 smithi099 bash[84576]: cluster 2024-09-13T15:02:11.477652+0000 mon.a (mon.0) 2189 : cluster [WRN] Health check update: Degraded data redundancy: 8009/76950 objects degraded (10.408%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:12.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:11 smithi099 bash[84576]: cluster 2024-09-13T15:02:11.477652+0000 mon.a (mon.0) 2189 : cluster [WRN] Health check update: Degraded data redundancy: 8009/76950 objects degraded (10.408%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:12.238 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:11 smithi099 bash[87147]: cluster 2024-09-13T15:02:10.829165+0000 mgr.y (mgr.26252) 1081 : cluster [DBG] pgmap v1621: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:12.238 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:11 smithi099 bash[87147]: cluster 2024-09-13T15:02:10.829165+0000 mgr.y (mgr.26252) 1081 : cluster [DBG] pgmap v1621: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:12.239 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:11 smithi099 bash[87147]: cluster 2024-09-13T15:02:11.477652+0000 mon.a (mon.0) 2189 : cluster [WRN] Health check update: Degraded data redundancy: 8009/76950 objects degraded (10.408%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:12.239 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:11 smithi099 bash[87147]: cluster 2024-09-13T15:02:11.477652+0000 mon.a (mon.0) 2189 : cluster [WRN] Health check update: Degraded data redundancy: 8009/76950 objects degraded (10.408%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:12.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:12.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:12.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:12.371 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2024-09-13T15:02:12.371 INFO:teuthology.orchestra.run.smithi099.stderr:0 2024-09-13T15:02:12.371 INFO:teuthology.orchestra.run.smithi099.stderr:1 2024-09-13T15:02:12.371 INFO:teuthology.orchestra.run.smithi099.stderr:2 2024-09-13T15:02:12.371 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2024-09-13T15:02:12.371 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2024-09-13T15:02:12.371 INFO:teuthology.orchestra.run.smithi099.stderr:bench [] [] [] [] 2024-09-13T15:02:12.372 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs debug_inject_read_zeros 2024-09-13T15:02:12.372 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs files list 2024-09-13T15:02:12.372 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs stats 2024-09-13T15:02:12.372 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator dump block 2024-09-13T15:02:12.372 INFO:teuthology.orchestra.run.smithi099.stderr:Error EINVAL: invalid command 2024-09-13T15:02:12.459 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.489 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:12.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:12.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:12.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:12.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:12.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:12.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:12.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:13.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:13.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:13.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:13.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:13.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:13.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:13.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:13.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:13.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:13.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:13.508 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:13.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:13.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:13.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:13.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:13.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:13.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:13.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:13.908 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:02:13.908 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:02:13.911 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:13.937 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:13.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:14.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:14.066 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:14.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:14.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:13 smithi099 bash[84576]: cluster 2024-09-13T15:02:12.830346+0000 mgr.y (mgr.26252) 1082 : cluster [DBG] pgmap v1622: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:14.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:13 smithi099 bash[84576]: cluster 2024-09-13T15:02:12.830346+0000 mgr.y (mgr.26252) 1082 : cluster [DBG] pgmap v1622: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:14.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:13 smithi099 bash[87147]: cluster 2024-09-13T15:02:12.830346+0000 mgr.y (mgr.26252) 1082 : cluster [DBG] pgmap v1622: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:14.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:13 smithi099 bash[87147]: cluster 2024-09-13T15:02:12.830346+0000 mgr.y (mgr.26252) 1082 : cluster [DBG] pgmap v1622: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:14.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:14.243 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:13 smithi177 bash[68594]: cluster 2024-09-13T15:02:12.830346+0000 mgr.y (mgr.26252) 1082 : cluster [DBG] pgmap v1622: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:14.243 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:13 smithi177 bash[68594]: cluster 2024-09-13T15:02:12.830346+0000 mgr.y (mgr.26252) 1082 : cluster [DBG] pgmap v1622: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 8009/76950 objects degraded (10.408%) 2024-09-13T15:02:14.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:14.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:14.364 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:14.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:14.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:14.494 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:14.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:14.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:14.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:14.663 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:14.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:14.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:14.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:14.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:14.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:14.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:14.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:15.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:15.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:15.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:15.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:15.216 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:15.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:15.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:15.336 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:02:15.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:15.402 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T15:02:15.402 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T15:02:15.402 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-09-13T15:02:15.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:15.504 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:15.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:15.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:15.661 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:15.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:15.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:15.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:15.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:15.937 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:15.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:16.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:16.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:16.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:16.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:15 smithi177 bash[68594]: cluster 2024-09-13T15:02:14.831454+0000 mgr.y (mgr.26252) 1083 : cluster [DBG] pgmap v1623: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:16.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:15 smithi177 bash[68594]: cluster 2024-09-13T15:02:14.831454+0000 mgr.y (mgr.26252) 1083 : cluster [DBG] pgmap v1623: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:16.176 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:16.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:16.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:16.234 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:02:16.234 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:02:16.274 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:02:16.274 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:02:16.306 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:02:16.307 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:02:16.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:02:16.309 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:16.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:15 smithi099 bash[84576]: cluster 2024-09-13T15:02:14.831454+0000 mgr.y (mgr.26252) 1083 : cluster [DBG] pgmap v1623: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:16.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:15 smithi099 bash[84576]: cluster 2024-09-13T15:02:14.831454+0000 mgr.y (mgr.26252) 1083 : cluster [DBG] pgmap v1623: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:16.318 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:15 smithi099 bash[87147]: cluster 2024-09-13T15:02:14.831454+0000 mgr.y (mgr.26252) 1083 : cluster [DBG] pgmap v1623: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:16.319 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:15 smithi099 bash[87147]: cluster 2024-09-13T15:02:14.831454+0000 mgr.y (mgr.26252) 1083 : cluster [DBG] pgmap v1623: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:16.332 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:16.352 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:02:16.352 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:02:16.376 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:02:16.376 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:02:16.412 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:02:16.413 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:02:16.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:16.460 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:02:16.461 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:02:16.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:16.572 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:16.593 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:16.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:16.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:16.723 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:16.829 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:16.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:16.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:16.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:17.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:17.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[84576]: audit 2024-09-13T15:02:16.193406+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.99:0/968721122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[84576]: audit 2024-09-13T15:02:16.193406+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.99:0/968721122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[84576]: cluster 2024-09-13T15:02:16.478394+0000 mon.a (mon.0) 2190 : cluster [WRN] Health check update: Degraded data redundancy: 6678/63897 objects degraded (10.451%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[84576]: cluster 2024-09-13T15:02:16.478394+0000 mon.a (mon.0) 2190 : cluster [WRN] Health check update: Degraded data redundancy: 6678/63897 objects degraded (10.451%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[84576]: audit 2024-09-13T15:02:16.525413+0000 mon.a (mon.0) 2191 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:17.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[84576]: audit 2024-09-13T15:02:16.525413+0000 mon.a (mon.0) 2191 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[87147]: audit 2024-09-13T15:02:16.193406+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.99:0/968721122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[87147]: audit 2024-09-13T15:02:16.193406+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.99:0/968721122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[87147]: cluster 2024-09-13T15:02:16.478394+0000 mon.a (mon.0) 2190 : cluster [WRN] Health check update: Degraded data redundancy: 6678/63897 objects degraded (10.451%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[87147]: cluster 2024-09-13T15:02:16.478394+0000 mon.a (mon.0) 2190 : cluster [WRN] Health check update: Degraded data redundancy: 6678/63897 objects degraded (10.451%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[87147]: audit 2024-09-13T15:02:16.525413+0000 mon.a (mon.0) 2191 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:17.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:16 smithi099 bash[87147]: audit 2024-09-13T15:02:16.525413+0000 mon.a (mon.0) 2191 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:17.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:17.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:17.175 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:02:17] "GET /metrics HTTP/1.1" 200 36767 "" "Prometheus/2.51.0" 2024-09-13T15:02:17.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:17.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:16 smithi177 bash[68594]: audit 2024-09-13T15:02:16.193406+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.99:0/968721122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:17.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:16 smithi177 bash[68594]: audit 2024-09-13T15:02:16.193406+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.99:0/968721122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:17.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:16 smithi177 bash[68594]: cluster 2024-09-13T15:02:16.478394+0000 mon.a (mon.0) 2190 : cluster [WRN] Health check update: Degraded data redundancy: 6678/63897 objects degraded (10.451%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:17.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:16 smithi177 bash[68594]: cluster 2024-09-13T15:02:16.478394+0000 mon.a (mon.0) 2190 : cluster [WRN] Health check update: Degraded data redundancy: 6678/63897 objects degraded (10.451%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:17.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:16 smithi177 bash[68594]: audit 2024-09-13T15:02:16.525413+0000 mon.a (mon.0) 2191 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:17.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:16 smithi177 bash[68594]: audit 2024-09-13T15:02:16.525413+0000 mon.a (mon.0) 2191 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:17.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:17.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:17.392 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:17.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:17.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:17.495 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:02:17.516 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:17.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:17.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:17.676 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:17.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:17.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:17.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:17.834 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:17.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:17.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:17.970 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:17.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:18.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:18.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:18.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:18.205 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:18.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:18.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:18.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[84576]: cluster 2024-09-13T15:02:16.832278+0000 mgr.y (mgr.26252) 1084 : cluster [DBG] pgmap v1624: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:18.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[84576]: cluster 2024-09-13T15:02:16.832278+0000 mgr.y (mgr.26252) 1084 : cluster [DBG] pgmap v1624: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[84576]: cluster 2024-09-13T15:02:17.486488+0000 mon.a (mon.0) 2192 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[84576]: cluster 2024-09-13T15:02:17.486488+0000 mon.a (mon.0) 2192 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[84576]: audit 2024-09-13T15:02:17.495592+0000 mon.a (mon.0) 2193 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[84576]: audit 2024-09-13T15:02:17.495592+0000 mon.a (mon.0) 2193 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[84576]: cluster 2024-09-13T15:02:17.502818+0000 mon.a (mon.0) 2194 : cluster [DBG] osdmap e1052: 8 total, 7 up, 8 in 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[84576]: cluster 2024-09-13T15:02:17.502818+0000 mon.a (mon.0) 2194 : cluster [DBG] osdmap e1052: 8 total, 7 up, 8 in 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[87147]: cluster 2024-09-13T15:02:16.832278+0000 mgr.y (mgr.26252) 1084 : cluster [DBG] pgmap v1624: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[87147]: cluster 2024-09-13T15:02:16.832278+0000 mgr.y (mgr.26252) 1084 : cluster [DBG] pgmap v1624: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[87147]: cluster 2024-09-13T15:02:17.486488+0000 mon.a (mon.0) 2192 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[87147]: cluster 2024-09-13T15:02:17.486488+0000 mon.a (mon.0) 2192 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:02:18.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[87147]: audit 2024-09-13T15:02:17.495592+0000 mon.a (mon.0) 2193 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:18.272 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[87147]: audit 2024-09-13T15:02:17.495592+0000 mon.a (mon.0) 2193 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:18.272 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[87147]: cluster 2024-09-13T15:02:17.502818+0000 mon.a (mon.0) 2194 : cluster [DBG] osdmap e1052: 8 total, 7 up, 8 in 2024-09-13T15:02:18.272 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:17 smithi099 bash[87147]: cluster 2024-09-13T15:02:17.502818+0000 mon.a (mon.0) 2194 : cluster [DBG] osdmap e1052: 8 total, 7 up, 8 in 2024-09-13T15:02:18.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:18.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:18.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:17 smithi177 bash[68594]: cluster 2024-09-13T15:02:16.832278+0000 mgr.y (mgr.26252) 1084 : cluster [DBG] pgmap v1624: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:18.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:17 smithi177 bash[68594]: cluster 2024-09-13T15:02:16.832278+0000 mgr.y (mgr.26252) 1084 : cluster [DBG] pgmap v1624: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 6678/63897 objects degraded (10.451%) 2024-09-13T15:02:18.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:17 smithi177 bash[68594]: cluster 2024-09-13T15:02:17.486488+0000 mon.a (mon.0) 2192 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:02:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:17 smithi177 bash[68594]: cluster 2024-09-13T15:02:17.486488+0000 mon.a (mon.0) 2192 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:02:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:17 smithi177 bash[68594]: audit 2024-09-13T15:02:17.495592+0000 mon.a (mon.0) 2193 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:17 smithi177 bash[68594]: audit 2024-09-13T15:02:17.495592+0000 mon.a (mon.0) 2193 : audit [INF] from='client.? 172.21.15.99:0/1734836539' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:17 smithi177 bash[68594]: cluster 2024-09-13T15:02:17.502818+0000 mon.a (mon.0) 2194 : cluster [DBG] osdmap e1052: 8 total, 7 up, 8 in 2024-09-13T15:02:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:17 smithi177 bash[68594]: cluster 2024-09-13T15:02:17.502818+0000 mon.a (mon.0) 2194 : cluster [DBG] osdmap e1052: 8 total, 7 up, 8 in 2024-09-13T15:02:18.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:18.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:18.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:18.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:18.661 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:18.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:18.768 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:18.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:18.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:18.856 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:02:18.857 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:02:18.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:18.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:19.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:19.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:19.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:19.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:19.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:19.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:19.281 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:19.314 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:19.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:19.418 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:19.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:19.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:19.555 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:19.564 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:19 smithi177 bash[68594]: audit 2024-09-13T15:02:18.298560+0000 mon.a (mon.0) 2195 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:19.564 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:19 smithi177 bash[68594]: audit 2024-09-13T15:02:18.298560+0000 mon.a (mon.0) 2195 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:19.564 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:19 smithi177 bash[68594]: audit 2024-09-13T15:02:18.299898+0000 mon.a (mon.0) 2196 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:19.564 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:19 smithi177 bash[68594]: audit 2024-09-13T15:02:18.299898+0000 mon.a (mon.0) 2196 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:19.564 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:19 smithi177 bash[68594]: cluster 2024-09-13T15:02:18.833525+0000 mgr.y (mgr.26252) 1085 : cluster [DBG] pgmap v1626: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 6413/61428 objects degraded (10.440%) 2024-09-13T15:02:19.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:19 smithi177 bash[68594]: cluster 2024-09-13T15:02:18.833525+0000 mgr.y (mgr.26252) 1085 : cluster [DBG] pgmap v1626: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 6413/61428 objects degraded (10.440%) 2024-09-13T15:02:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[84576]: audit 2024-09-13T15:02:18.298560+0000 mon.a (mon.0) 2195 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[84576]: audit 2024-09-13T15:02:18.298560+0000 mon.a (mon.0) 2195 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[84576]: audit 2024-09-13T15:02:18.299898+0000 mon.a (mon.0) 2196 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[84576]: audit 2024-09-13T15:02:18.299898+0000 mon.a (mon.0) 2196 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[84576]: cluster 2024-09-13T15:02:18.833525+0000 mgr.y (mgr.26252) 1085 : cluster [DBG] pgmap v1626: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 6413/61428 objects degraded (10.440%) 2024-09-13T15:02:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[84576]: cluster 2024-09-13T15:02:18.833525+0000 mgr.y (mgr.26252) 1085 : cluster [DBG] pgmap v1626: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 6413/61428 objects degraded (10.440%) 2024-09-13T15:02:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[87147]: audit 2024-09-13T15:02:18.298560+0000 mon.a (mon.0) 2195 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[87147]: audit 2024-09-13T15:02:18.298560+0000 mon.a (mon.0) 2195 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[87147]: audit 2024-09-13T15:02:18.299898+0000 mon.a (mon.0) 2196 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[87147]: audit 2024-09-13T15:02:18.299898+0000 mon.a (mon.0) 2196 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[87147]: cluster 2024-09-13T15:02:18.833525+0000 mgr.y (mgr.26252) 1085 : cluster [DBG] pgmap v1626: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 6413/61428 objects degraded (10.440%) 2024-09-13T15:02:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:19 smithi099 bash[87147]: cluster 2024-09-13T15:02:18.833525+0000 mgr.y (mgr.26252) 1085 : cluster [DBG] pgmap v1626: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 6413/61428 objects degraded (10.440%) 2024-09-13T15:02:19.581 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:19.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:19.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:19.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:19.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:19.814 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:19.850 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:19.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:19.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:19.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:19.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:20.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:20.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:20.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:20.200 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:02:20.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:20.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:20.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:20.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:20.369 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:20.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:20.610 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:02:20 smithi177 bash[61439]: ts=2024-09-13T15:02:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:20.652 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:20.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:20.755 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:20.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:20.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:20.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:02:20.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:20.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:21.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:21.060 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:21.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:21.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:21.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:21.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:21.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:21.521 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:21.547 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:21.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:21.688 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:21.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:21.814 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:21.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:21.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:02:21.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:21.939 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:21.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:22.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:22.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:21 smithi099 bash[84576]: cluster 2024-09-13T15:02:20.834598+0000 mgr.y (mgr.26252) 1086 : cluster [DBG] pgmap v1627: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:22.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:21 smithi099 bash[84576]: cluster 2024-09-13T15:02:20.834598+0000 mgr.y (mgr.26252) 1086 : cluster [DBG] pgmap v1627: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:22.054 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:21 smithi099 bash[84576]: cluster 2024-09-13T15:02:21.481805+0000 mon.a (mon.0) 2197 : cluster [WRN] Health check update: Degraded data redundancy: 5272/50250 objects degraded (10.492%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:22.054 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:21 smithi099 bash[84576]: cluster 2024-09-13T15:02:21.481805+0000 mon.a (mon.0) 2197 : cluster [WRN] Health check update: Degraded data redundancy: 5272/50250 objects degraded (10.492%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:22.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:21 smithi099 bash[87147]: cluster 2024-09-13T15:02:20.834598+0000 mgr.y (mgr.26252) 1086 : cluster [DBG] pgmap v1627: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:22.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:21 smithi099 bash[87147]: cluster 2024-09-13T15:02:20.834598+0000 mgr.y (mgr.26252) 1086 : cluster [DBG] pgmap v1627: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:22.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:21 smithi099 bash[87147]: cluster 2024-09-13T15:02:21.481805+0000 mon.a (mon.0) 2197 : cluster [WRN] Health check update: Degraded data redundancy: 5272/50250 objects degraded (10.492%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:22.054 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:21 smithi099 bash[87147]: cluster 2024-09-13T15:02:21.481805+0000 mon.a (mon.0) 2197 : cluster [WRN] Health check update: Degraded data redundancy: 5272/50250 objects degraded (10.492%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:22.084 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:22.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:22.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:21 smithi177 bash[68594]: cluster 2024-09-13T15:02:20.834598+0000 mgr.y (mgr.26252) 1086 : cluster [DBG] pgmap v1627: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:22.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:21 smithi177 bash[68594]: cluster 2024-09-13T15:02:20.834598+0000 mgr.y (mgr.26252) 1086 : cluster [DBG] pgmap v1627: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:22.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:21 smithi177 bash[68594]: cluster 2024-09-13T15:02:21.481805+0000 mon.a (mon.0) 2197 : cluster [WRN] Health check update: Degraded data redundancy: 5272/50250 objects degraded (10.492%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:22.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:21 smithi177 bash[68594]: cluster 2024-09-13T15:02:21.481805+0000 mon.a (mon.0) 2197 : cluster [WRN] Health check update: Degraded data redundancy: 5272/50250 objects degraded (10.492%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.225 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:22.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:22.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:22.352 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:22.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:22.380 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:22.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:22.488 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:22.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:22.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:22.632 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:22.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:22.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:22.770 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:22.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:22.875 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:22.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:22.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:22.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:23.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:23.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:23.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:23.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:23.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:23.274 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:23.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:23.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:23.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:23.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:23.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:23.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:23.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:24.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:24.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:24.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:23 smithi099 bash[84576]: cluster 2024-09-13T15:02:22.835315+0000 mgr.y (mgr.26252) 1087 : cluster [DBG] pgmap v1628: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:23 smithi099 bash[84576]: cluster 2024-09-13T15:02:22.835315+0000 mgr.y (mgr.26252) 1087 : cluster [DBG] pgmap v1628: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:23 smithi099 bash[87147]: cluster 2024-09-13T15:02:22.835315+0000 mgr.y (mgr.26252) 1087 : cluster [DBG] pgmap v1628: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:23 smithi099 bash[87147]: cluster 2024-09-13T15:02:22.835315+0000 mgr.y (mgr.26252) 1087 : cluster [DBG] pgmap v1628: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:24.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:23 smithi177 bash[68594]: cluster 2024-09-13T15:02:22.835315+0000 mgr.y (mgr.26252) 1087 : cluster [DBG] pgmap v1628: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:24.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:23 smithi177 bash[68594]: cluster 2024-09-13T15:02:22.835315+0000 mgr.y (mgr.26252) 1087 : cluster [DBG] pgmap v1628: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 1.0 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 5272/50250 objects degraded (10.492%) 2024-09-13T15:02:24.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:24.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:24.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:24.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:24.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:24.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:24.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:24.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:24.794 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:24.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:24.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:24.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:24.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:25.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:25.050 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:25.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:25.156 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:25.190 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:25.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:25.300 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:25.344 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:25.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:25.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:25.472 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:25.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:25.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:25.611 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:25.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:25.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:25.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:25.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:25.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:25.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:25.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:25.985 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:25 smithi099 bash[84576]: cluster 2024-09-13T15:02:24.836286+0000 mgr.y (mgr.26252) 1088 : cluster [DBG] pgmap v1629: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:25.985 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:25 smithi099 bash[84576]: cluster 2024-09-13T15:02:24.836286+0000 mgr.y (mgr.26252) 1088 : cluster [DBG] pgmap v1629: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:25.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:25 smithi099 bash[87147]: cluster 2024-09-13T15:02:24.836286+0000 mgr.y (mgr.26252) 1088 : cluster [DBG] pgmap v1629: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:25.985 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:25 smithi099 bash[87147]: cluster 2024-09-13T15:02:24.836286+0000 mgr.y (mgr.26252) 1088 : cluster [DBG] pgmap v1629: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:26.004 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:26.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:26.108 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:26.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:26.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:26.235 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:26.244 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:25 smithi177 bash[68594]: cluster 2024-09-13T15:02:24.836286+0000 mgr.y (mgr.26252) 1088 : cluster [DBG] pgmap v1629: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:26.244 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:25 smithi177 bash[68594]: cluster 2024-09-13T15:02:24.836286+0000 mgr.y (mgr.26252) 1088 : cluster [DBG] pgmap v1629: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:26.270 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:26.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:26.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:26.438 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:26.456 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:26.456 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:26.541 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:26.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:26.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:26.675 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:26.707 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:26.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:26.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:26.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:26.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:26.954 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:26 smithi177 bash[68594]: cluster 2024-09-13T15:02:26.482744+0000 mon.a (mon.0) 2198 : cluster [WRN] Health check update: Degraded data redundancy: 3840/36837 objects degraded (10.424%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:26.954 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:26 smithi177 bash[68594]: cluster 2024-09-13T15:02:26.482744+0000 mon.a (mon.0) 2198 : cluster [WRN] Health check update: Degraded data redundancy: 3840/36837 objects degraded (10.424%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:26.969 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:26.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:27.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:27.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:26 smithi099 bash[84576]: cluster 2024-09-13T15:02:26.482744+0000 mon.a (mon.0) 2198 : cluster [WRN] Health check update: Degraded data redundancy: 3840/36837 objects degraded (10.424%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:27.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:26 smithi099 bash[84576]: cluster 2024-09-13T15:02:26.482744+0000 mon.a (mon.0) 2198 : cluster [WRN] Health check update: Degraded data redundancy: 3840/36837 objects degraded (10.424%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:27.074 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:02:27] "GET /metrics HTTP/1.1" 200 36765 "" "Prometheus/2.51.0" 2024-09-13T15:02:27.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:26 smithi099 bash[87147]: cluster 2024-09-13T15:02:26.482744+0000 mon.a (mon.0) 2198 : cluster [WRN] Health check update: Degraded data redundancy: 3840/36837 objects degraded (10.424%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:27.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:26 smithi099 bash[87147]: cluster 2024-09-13T15:02:26.482744+0000 mon.a (mon.0) 2198 : cluster [WRN] Health check update: Degraded data redundancy: 3840/36837 objects degraded (10.424%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:27.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:27.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:27.207 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:27.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:27.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:27.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:27.383 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:27.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:27.457 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:02:27.457 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T15:02:27.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T15:02:27.457 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:02:27.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.457 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:8363 42.7 42:ebd0a9f7:::benchmark_data_smithi099_148651_object8362:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1014)", 2024-09-13T15:02:27.457 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:53:29.383295+0000", 2024-09-13T15:02:27.457 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 538.05828649099999, 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.70271919599999999, 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 8363 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.383295+0000", 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.383295+0000", 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.383294+0000", 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.459 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.383444+0000", 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000150452 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.383455+0000", 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0930999999999999e-05 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.383461+0000", 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3199999999999999e-06 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.383501+0000", 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0088e-05 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.383616+0000", 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000115159 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.383738+0000", 2024-09-13T15:02:27.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012240800000000001 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.385227+0000", 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001489024 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:29.385235+0000", 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5240000000000003e-06 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:30.085773+0000", 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.70053795299999999 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:30.085885+0000", 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011189299999999999 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:53:30.086014+0000", 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000129584 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.463 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45350.0:123416 42.8 42:10026f7a:::benchmark_data_smithi099_148651_object15576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1024)", 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:55:17.038083+0000", 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 430.40349895600002, 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.63023008899999999, 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45350", 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/1096746363", 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 123416 2024-09-13T15:02:27.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.038083+0000", 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.038083+0000", 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.038078+0000", 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999957 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.038087+0000", 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7150000000000004e-06 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.038097+0000", 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8300000000000008e-06 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.038102+0000", 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0799999999999996e-06 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.040184+0000", 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002082562 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.641214+0000", 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.60102940000000005 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.641489+0000", 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00027574099999999997 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.654164+0000", 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012674938 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.663321+0000", 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0091566909999999998 2024-09-13T15:02:27.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.668211+0000", 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0048901839999999997 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.668270+0000", 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9086999999999999e-05 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:55:17.668313+0000", 2024-09-13T15:02:27.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2339000000000002e-05 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:75287 43.d 43:bb233d9b:::benchmark_data_smithi099_171278_object75286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1044)", 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:52.710229+0000", 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 154.731352754, 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33457172099999999, 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.470 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75287 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.710229+0000", 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.710229+0000", 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.710227+0000", 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.710343+0000", 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011516899999999999 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.710353+0000", 2024-09-13T15:02:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0275000000000001e-05 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.710357+0000", 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6890000000000002e-06 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.710390+0000", 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3073e-05 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.710471+0000", 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1555000000000001e-05 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.710559+0000", 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7810000000000001e-05 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.712574+0000", 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020152120000000002 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:52.712876+0000", 2024-09-13T15:02:27.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00030148399999999998 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:53.044731+0000", 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33185536700000001 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:53.044771+0000", 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0275999999999999e-05 2024-09-13T15:02:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:53.044800+0000", 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.915e-05 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:76860 43.c 43:32823f6f:::benchmark_data_smithi099_171278_object76859:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1044)", 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.505531+0000", 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.93605076599999, 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.367279515, 2024-09-13T15:02:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 76860 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505531+0000", 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505531+0000", 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505530+0000", 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505576+0000", 2024-09-13T15:02:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5682000000000001e-05 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505586+0000", 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0251000000000001e-05 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505591+0000", 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6990000000000002e-06 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.506259+0000", 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00066873200000000001 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.506360+0000", 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100666 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.506472+0000", 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011189 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.509872+0000", 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003400213 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.511464+0000", 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001591914 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872451+0000", 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.360987059 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872602+0000", 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000151049 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872810+0000", 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020809100000000001 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:76861 43.8 43:17a77986:::benchmark_data_smithi099_171278_object76860:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1044)", 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.508830+0000", 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.93275160799999, 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35834657199999997, 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 76861 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.508830+0000", 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.508830+0000", 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.508828+0000", 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.508902+0000", 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3732000000000005e-05 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.508916+0000", 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3601000000000001e-05 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.508922+0000", 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0580000000000003e-06 2024-09-13T15:02:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.509181+0000", 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000258865 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.509337+0000", 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015635700000000001 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.509482+0000", 2024-09-13T15:02:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014472500000000001 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.511900+0000", 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024178419999999999 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.514333+0000", 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024330739999999999 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866936+0000", 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35260358200000003 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.867071+0000", 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013523699999999999 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.867176+0000", 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010495 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.489 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:76865 43.c 43:3c2ed577:::benchmark_data_smithi099_171278_object76864:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1044)", 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.515226+0000", 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.92635546400001, 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35820331599999999, 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 76865 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.515226+0000", 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.515226+0000", 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.515225+0000", 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.515287+0000", 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2030000000000001e-05 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.515298+0000", 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1071e-05 2024-09-13T15:02:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.515303+0000", 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.451e-06 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.515343+0000", 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9569e-05 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.515422+0000", 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8968999999999996e-05 2024-09-13T15:02:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.515503+0000", 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1018000000000003e-05 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.517977+0000", 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024739620000000001 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.517982+0000", 2024-09-13T15:02:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5520000000000004e-06 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.873338+0000", 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35535611900000003 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.873390+0000", 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1807000000000001e-05 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.873429+0000", 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9017999999999998e-05 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:76873 43.c 43:3364d0f9:::benchmark_data_smithi099_171278_object76872:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1044)", 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.519249+0000", 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.92233290300001, 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36059039300000001, 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 76873 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519249+0000", 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519249+0000", 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519247+0000", 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519347+0000", 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9692999999999996e-05 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519359+0000", 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1936999999999999e-05 2024-09-13T15:02:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519365+0000", 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6670000000000001e-06 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519394+0000", 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9090000000000001e-05 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519476+0000", 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2418999999999998e-05 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519561+0000", 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4831000000000001e-05 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872368+0000", 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35280668900000001 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.873477+0000", 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001109699 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.879711+0000", 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062338250000000001 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.879784+0000", 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3349000000000005e-05 2024-09-13T15:02:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.879839+0000", 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4462999999999997e-05 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45572.0:76871 43.5 e1044/1038 43:a98ddbd0:::benchmark_data_smithi099_171278_object76870:head v 1044'4833, mlcod=1044'4831)", 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.519398+0000", 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.922183115, 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34766186500000001, 2024-09-13T15:02:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519398+0000", 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519398+0000", 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519398+0000", 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519464+0000", 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6524000000000002e-05 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519474+0000", 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4429999999999992e-06 2024-09-13T15:02:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519481+0000", 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8299999999999998e-06 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519538+0000", 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.787e-05 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519557+0000", 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8434e-05 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866995+0000", 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.347437726 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.867060+0000", 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5678000000000004e-05 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45572.0:76874 43.f e1044/1038 43:f90ebb9d:::benchmark_data_smithi099_171278_object76873:head v 1044'4822, mlcod=1044'4820)", 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.521563+0000", 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.920018548, 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36254724199999999, 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521563+0000", 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521563+0000", 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521562+0000", 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521592+0000", 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9462999999999999e-05 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866721+0000", 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34512911400000001 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866745+0000", 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4465999999999999e-05 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866907+0000", 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016150499999999999 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866962+0000", 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4996e-05 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.884044+0000", 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017082106 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.884110+0000", 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6317999999999998e-05 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop_reply(client.45572.0:76861 43.8 e1044/1038 ondisk, result = 0)", 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.521757+0000", 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.91982478200001, 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.345430599, 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521757+0000", 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521757+0000", 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521756+0000", 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521759+0000", 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2210000000000002e-06 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866722+0000", 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.344962981 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866744+0000", 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2107e-05 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866896+0000", 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015142899999999999 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866918+0000", 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2226e-05 2024-09-13T15:02:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.867187+0000", 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026913600000000001 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop_reply(client.45572.0:76873 43.c e1044/1038 ondisk, result = 0)", 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.522912+0000", 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.91866910600001, 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34945925999999999, 2024-09-13T15:02:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.522912+0000", 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.522912+0000", 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.522912+0000", 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.522917+0000", 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0180000000000001e-06 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866670+0000", 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34375298999999998 2024-09-13T15:02:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866694+0000", 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4545e-05 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872335+0000", 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056408420000000001 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872352+0000", 2024-09-13T15:02:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6872999999999999e-05 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872372+0000", 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9698999999999999e-05 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:80519 43.d 43:b66289f9:::benchmark_data_smithi099_171278_object80518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1044)", 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:58.463055+0000", 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 148.97852675799999, 2024-09-13T15:02:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33288634099999997, 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 80519 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.463055+0000", 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.463055+0000", 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.463054+0000", 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.463111+0000", 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7185999999999998e-05 2024-09-13T15:02:27.521 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.463119+0000", 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1949999999999997e-06 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.463124+0000", 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2859999999999999e-06 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.463191+0000", 2024-09-13T15:02:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7405999999999998e-05 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.463296+0000", 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010488399999999999 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.463417+0000", 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012097 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.468846+0000", 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054291499999999998 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.472348+0000", 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035014260000000002 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.795690+0000", 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32334252299999999 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.795832+0000", 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014221599999999999 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:58.795941+0000", 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108646 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128590 43.a 43:5a3ee379:::benchmark_data_smithi099_171278_object12578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.385199+0000", 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 102.056382361, 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40636963100000001, 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128590 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385199+0000", 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385199+0000", 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385198+0000", 2024-09-13T15:02:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385202+0000", 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2430000000000001e-06 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385209+0000", 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2949999999999996e-06 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385214+0000", 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4360000000000001e-06 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.399134+0000", 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013919832 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.399203+0000", 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9133000000000004e-05 2024-09-13T15:02:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.399278+0000", 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4923000000000001e-05 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414637+0000", 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015358627 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414656+0000", 2024-09-13T15:02:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9293e-05 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.791316+0000", 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37665990399999999 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.791447+0000", 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013125699999999999 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.791569+0000", 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000121298 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128599 43.a 43:581bf91b:::benchmark_data_smithi099_171278_object12587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.394205+0000", 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 102.04737626799999, 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.41306469299999998, 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128599 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394205+0000", 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394205+0000", 2024-09-13T15:02:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394204+0000", 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394210+0000", 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1099999999999999e-06 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394219+0000", 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0680000000000003e-06 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394224+0000", 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1660000000000002e-06 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.406611+0000", 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01238702 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.406714+0000", 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102629 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.406821+0000", 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000106704 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414660+0000", 2024-09-13T15:02:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0078389289999999997 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417173+0000", 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002512977 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.807025+0000", 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38985236000000001 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.807208+0000", 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018322999999999999 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.807270+0000", 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1671e-05 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128601 43.a 43:52aeec9c:::benchmark_data_smithi099_171278_object12589:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.394499+0000", 2024-09-13T15:02:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 102.04708263800001, 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.41695673700000002, 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128601 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394499+0000", 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394499+0000", 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394498+0000", 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394502+0000", 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8909999999999997e-06 2024-09-13T15:02:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394508+0000", 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.665e-06 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394511+0000", 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.123e-06 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.410919+0000", 2024-09-13T15:02:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016407719000000001 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.411018+0000", 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9092999999999995e-05 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.411117+0000", 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9035000000000003e-05 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414735+0000", 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036183999999999999 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.423114+0000", 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0083789250000000006 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811379+0000", 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38826455900000001 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811420+0000", 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1248000000000002e-05 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811456+0000", 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5746e-05 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128615 43.8 43:16ddc61b:::benchmark_data_smithi099_171278_object12603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:02:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.405083+0000", 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 102.036498146, 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.391743862, 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128615 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405083+0000", 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405083+0000", 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405082+0000", 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405088+0000", 2024-09-13T15:02:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0610000000000001e-06 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405099+0000", 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1324e-05 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405105+0000", 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0270000000000001e-06 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405147+0000", 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1678999999999999e-05 2024-09-13T15:02:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405228+0000", 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0724999999999997e-05 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405335+0000", 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107205 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409282+0000", 2024-09-13T15:02:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039475090000000001 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414778+0000", 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054955610000000004 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.796593+0000", 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38181541299999999 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.796711+0000", 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011769699999999999 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.796827+0000", 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011608 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128628 43.a 43:516a2081:::benchmark_data_smithi099_171278_object12616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.417499+0000", 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 102.024082562, 2024-09-13T15:02:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39405699199999999, 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128628 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417499+0000", 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417499+0000", 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417497+0000", 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417503+0000", 2024-09-13T15:02:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.037e-06 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417518+0000", 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4053000000000001e-05 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417523+0000", 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9359999999999996e-06 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417562+0000", 2024-09-13T15:02:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8753e-05 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417673+0000", 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110968 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417781+0000", 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107543 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.423122+0000", 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053414420000000001 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.423152+0000", 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0076000000000001e-05 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811491+0000", 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38833923799999998 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811525+0000", 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3166000000000001e-05 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811556+0000", 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1250000000000001e-05 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128633 43.a 43:521297ac:::benchmark_data_smithi099_171278_object12621:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.425405+0000", 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 102.016176231, 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38638104099999998, 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128633 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425405+0000", 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425405+0000", 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425404+0000", 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425409+0000", 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3770000000000004e-06 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425419+0000", 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5780000000000005e-06 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425423+0000", 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1760000000000003e-06 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427662+0000", 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022386179999999999 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427781+0000", 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011887699999999999 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427896+0000", 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000114898 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.432933+0000", 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050373570000000001 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.432949+0000", 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5794000000000002e-05 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811644+0000", 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37869531000000001 2024-09-13T15:02:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811720+0000", 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5735999999999995e-05 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811786+0000", 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6519999999999993e-05 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128635 43.c 43:334e8eab:::benchmark_data_smithi099_171278_object12623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.427651+0000", 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 102.013930476, 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.369737027, 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128635 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427651+0000", 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427651+0000", 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427651+0000", 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427653+0000", 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.779e-06 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427659+0000", 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1889999999999998e-06 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427662+0000", 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1580000000000001e-06 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427695+0000", 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3723999999999998e-05 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427770+0000", 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4264999999999995e-05 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427838+0000", 2024-09-13T15:02:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7879000000000005e-05 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.431234+0000", 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033965409999999999 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434401+0000", 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003166992 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.797159+0000", 2024-09-13T15:02:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36275744100000001 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.797288+0000", 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000129534 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.797388+0000", 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9950000000000004e-05 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:177757 43.d 43:b26fdded:::benchmark_data_smithi099_171278_object61745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1050)", 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:01:36.043745+0000", 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 51.397836808000001, 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.42954842999999998, 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 177757 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043745+0000", 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043745+0000", 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043743+0000", 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043749+0000", 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4319999999999998e-06 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043760+0000", 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1226999999999999e-05 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043766+0000", 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0310000000000004e-06 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043801+0000", 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5246000000000001e-05 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043883+0000", 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1592000000000001e-05 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043956+0000", 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3192e-05 2024-09-13T15:02:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.047274+0000", 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003317445 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.048568+0000", 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001294707 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.473214+0000", 2024-09-13T15:02:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.42464559600000001 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.473258+0000", 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4279000000000001e-05 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.473293+0000", 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4854000000000002e-05 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:02:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:02:27.575 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:02:27.575 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:27.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:27.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:27.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:27.770 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:27.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:27.873 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:27.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:28.016 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:28.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:27 smithi177 bash[68594]: cluster 2024-09-13T15:02:26.836993+0000 mgr.y (mgr.26252) 1089 : cluster [DBG] pgmap v1630: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:28.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:27 smithi177 bash[68594]: cluster 2024-09-13T15:02:26.836993+0000 mgr.y (mgr.26252) 1089 : cluster [DBG] pgmap v1630: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:27 smithi177 bash[68594]: audit 2024-09-13T15:02:27.005263+0000 mon.a (mon.0) 2199 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:02:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:27 smithi177 bash[68594]: audit 2024-09-13T15:02:27.005263+0000 mon.a (mon.0) 2199 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:02:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:27 smithi177 bash[68594]: audit 2024-09-13T15:02:27.693603+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:27 smithi177 bash[68594]: audit 2024-09-13T15:02:27.693603+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:27 smithi177 bash[68594]: audit 2024-09-13T15:02:27.700892+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:27 smithi177 bash[68594]: audit 2024-09-13T15:02:27.700892+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:28.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:28.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:28.160 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[84576]: cluster 2024-09-13T15:02:26.836993+0000 mgr.y (mgr.26252) 1089 : cluster [DBG] pgmap v1630: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:28.160 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[84576]: cluster 2024-09-13T15:02:26.836993+0000 mgr.y (mgr.26252) 1089 : cluster [DBG] pgmap v1630: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:28.160 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[84576]: audit 2024-09-13T15:02:27.005263+0000 mon.a (mon.0) 2199 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:02:28.160 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[84576]: audit 2024-09-13T15:02:27.005263+0000 mon.a (mon.0) 2199 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:02:28.160 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[84576]: audit 2024-09-13T15:02:27.693603+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.160 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[84576]: audit 2024-09-13T15:02:27.693603+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.161 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[84576]: audit 2024-09-13T15:02:27.700892+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.161 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[84576]: audit 2024-09-13T15:02:27.700892+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.161 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[87147]: cluster 2024-09-13T15:02:26.836993+0000 mgr.y (mgr.26252) 1089 : cluster [DBG] pgmap v1630: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:28.161 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[87147]: cluster 2024-09-13T15:02:26.836993+0000 mgr.y (mgr.26252) 1089 : cluster [DBG] pgmap v1630: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 901 op/s; 3840/36837 objects degraded (10.424%) 2024-09-13T15:02:28.161 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[87147]: audit 2024-09-13T15:02:27.005263+0000 mon.a (mon.0) 2199 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:02:28.161 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[87147]: audit 2024-09-13T15:02:27.005263+0000 mon.a (mon.0) 2199 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:02:28.161 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[87147]: audit 2024-09-13T15:02:27.693603+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.161 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[87147]: audit 2024-09-13T15:02:27.693603+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.161 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[87147]: audit 2024-09-13T15:02:27.700892+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.162 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:27 smithi099 bash[87147]: audit 2024-09-13T15:02:27.700892+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:28.178 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:28.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:28.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:28.303 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:28.407 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:28.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:28.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:28.541 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:28.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:28.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:28.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:28.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:28.839 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:28.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:28.907 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T15:02:28.945 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:28.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:28.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:29.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:29.118 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:29.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:29.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:29.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.375 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:29.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:29.501 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:29.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:29.607 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.634 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:29.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.766 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:29.872 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:29.899 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:29.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:30.002 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:30.012 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:29 smithi099 bash[84576]: cluster 2024-09-13T15:02:28.837720+0000 mgr.y (mgr.26252) 1090 : cluster [DBG] pgmap v1631: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 717 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 3596/34461 objects degraded (10.435%) 2024-09-13T15:02:30.012 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:29 smithi099 bash[84576]: cluster 2024-09-13T15:02:28.837720+0000 mgr.y (mgr.26252) 1090 : cluster [DBG] pgmap v1631: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 717 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 3596/34461 objects degraded (10.435%) 2024-09-13T15:02:30.013 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:29 smithi099 bash[87147]: cluster 2024-09-13T15:02:28.837720+0000 mgr.y (mgr.26252) 1090 : cluster [DBG] pgmap v1631: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 717 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 3596/34461 objects degraded (10.435%) 2024-09-13T15:02:30.013 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:29 smithi099 bash[87147]: cluster 2024-09-13T15:02:28.837720+0000 mgr.y (mgr.26252) 1090 : cluster [DBG] pgmap v1631: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 717 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 3596/34461 objects degraded (10.435%) 2024-09-13T15:02:30.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:30.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:30.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:30.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:29 smithi177 bash[68594]: cluster 2024-09-13T15:02:28.837720+0000 mgr.y (mgr.26252) 1090 : cluster [DBG] pgmap v1631: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 717 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 3596/34461 objects degraded (10.435%) 2024-09-13T15:02:30.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:29 smithi177 bash[68594]: cluster 2024-09-13T15:02:28.837720+0000 mgr.y (mgr.26252) 1090 : cluster [DBG] pgmap v1631: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 717 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 3596/34461 objects degraded (10.435%) 2024-09-13T15:02:30.150 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:30.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:30.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:30.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:30.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:30.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:30.398 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:30.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:30.500 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:30.509 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:02:30 smithi177 bash[61439]: ts=2024-09-13T15:02:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:30.522 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:30.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:30.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:30.649 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:30.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:30.758 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:30.783 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:30.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:30.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:30.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:30.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:31.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:31.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:31.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:31.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:31.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:31.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:31.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:31.363 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:31.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:0 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:1 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:2 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:bench [] [] [] [] 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs debug_inject_read_zeros 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs files list 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs stats 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator dump block 2024-09-13T15:02:31.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error EINVAL: invalid command 2024-09-13T15:02:31.469 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:31.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:31.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:31.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:31.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:31.716 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:31.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:31.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:31.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:31.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:31.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:31.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:32.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:32.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:32.134 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:32.141 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:31 smithi099 bash[87147]: cluster 2024-09-13T15:02:30.838519+0000 mgr.y (mgr.26252) 1091 : cluster [DBG] pgmap v1632: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:32.142 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:31 smithi099 bash[87147]: cluster 2024-09-13T15:02:30.838519+0000 mgr.y (mgr.26252) 1091 : cluster [DBG] pgmap v1632: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:32.142 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:31 smithi099 bash[87147]: cluster 2024-09-13T15:02:31.486579+0000 mon.a (mon.0) 2202 : cluster [WRN] Health check update: Degraded data redundancy: 2553/24525 objects degraded (10.410%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:32.142 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:31 smithi099 bash[87147]: cluster 2024-09-13T15:02:31.486579+0000 mon.a (mon.0) 2202 : cluster [WRN] Health check update: Degraded data redundancy: 2553/24525 objects degraded (10.410%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:32.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:31 smithi099 bash[84576]: cluster 2024-09-13T15:02:30.838519+0000 mgr.y (mgr.26252) 1091 : cluster [DBG] pgmap v1632: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:32.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:31 smithi099 bash[84576]: cluster 2024-09-13T15:02:30.838519+0000 mgr.y (mgr.26252) 1091 : cluster [DBG] pgmap v1632: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:32.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:31 smithi099 bash[84576]: cluster 2024-09-13T15:02:31.486579+0000 mon.a (mon.0) 2202 : cluster [WRN] Health check update: Degraded data redundancy: 2553/24525 objects degraded (10.410%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:32.142 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:31 smithi099 bash[84576]: cluster 2024-09-13T15:02:31.486579+0000 mon.a (mon.0) 2202 : cluster [WRN] Health check update: Degraded data redundancy: 2553/24525 objects degraded (10.410%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:32.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:32.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:32.258 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:32.267 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:31 smithi177 bash[68594]: cluster 2024-09-13T15:02:30.838519+0000 mgr.y (mgr.26252) 1091 : cluster [DBG] pgmap v1632: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:32.267 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:31 smithi177 bash[68594]: cluster 2024-09-13T15:02:30.838519+0000 mgr.y (mgr.26252) 1091 : cluster [DBG] pgmap v1632: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:32.267 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:31 smithi177 bash[68594]: cluster 2024-09-13T15:02:31.486579+0000 mon.a (mon.0) 2202 : cluster [WRN] Health check update: Degraded data redundancy: 2553/24525 objects degraded (10.410%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:32.267 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:31 smithi177 bash[68594]: cluster 2024-09-13T15:02:31.486579+0000 mon.a (mon.0) 2202 : cluster [WRN] Health check update: Degraded data redundancy: 2553/24525 objects degraded (10.410%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:32.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:32.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:32.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:32.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:32.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:32.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:32.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:32.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:32.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:32.700 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:32.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:32.803 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:32.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:32.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:32.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:32.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:33.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:33.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:33.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:33.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:33.236 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:33.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:33.357 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:33.381 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:33.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:33.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:33.508 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:33.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:33.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:33.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:33.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:33.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:33.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:33.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:33.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:33.898 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:33.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:34.003 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:34.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:34.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:34.053 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T15:02:34.054 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T15:02:34.054 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-09-13T15:02:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:33 smithi177 bash[68594]: cluster 2024-09-13T15:02:32.839393+0000 mgr.y (mgr.26252) 1092 : cluster [DBG] pgmap v1633: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:34.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:33 smithi177 bash[68594]: cluster 2024-09-13T15:02:32.839393+0000 mgr.y (mgr.26252) 1092 : cluster [DBG] pgmap v1633: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:33 smithi177 bash[68594]: audit 2024-09-13T15:02:33.294869+0000 mon.a (mon.0) 2203 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:34.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:33 smithi177 bash[68594]: audit 2024-09-13T15:02:33.294869+0000 mon.a (mon.0) 2203 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:34.139 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:34.181 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:34.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:34.189 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:33 smithi099 bash[84576]: cluster 2024-09-13T15:02:32.839393+0000 mgr.y (mgr.26252) 1092 : cluster [DBG] pgmap v1633: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:34.190 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:33 smithi099 bash[84576]: cluster 2024-09-13T15:02:32.839393+0000 mgr.y (mgr.26252) 1092 : cluster [DBG] pgmap v1633: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:34.190 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:33 smithi099 bash[84576]: audit 2024-09-13T15:02:33.294869+0000 mon.a (mon.0) 2203 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:34.190 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:33 smithi099 bash[84576]: audit 2024-09-13T15:02:33.294869+0000 mon.a (mon.0) 2203 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:34.190 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:33 smithi099 bash[87147]: cluster 2024-09-13T15:02:32.839393+0000 mgr.y (mgr.26252) 1092 : cluster [DBG] pgmap v1633: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:34.190 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:33 smithi099 bash[87147]: cluster 2024-09-13T15:02:32.839393+0000 mgr.y (mgr.26252) 1092 : cluster [DBG] pgmap v1633: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 510 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 2553/24525 objects degraded (10.410%) 2024-09-13T15:02:34.190 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:33 smithi099 bash[87147]: audit 2024-09-13T15:02:33.294869+0000 mon.a (mon.0) 2203 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:34.190 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:33 smithi099 bash[87147]: audit 2024-09-13T15:02:33.294869+0000 mon.a (mon.0) 2203 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:34.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:34.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.321 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:34.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:34.422 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:34.448 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:34.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:34.515 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:34.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:34.579 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:34.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:34.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:34.721 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:34.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:34.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:34.866 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:34.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:34.977 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:35.008 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:35.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:35.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:35.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:35.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:35.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:35.288 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:35.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:35.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[84576]: audit 2024-09-13T15:02:34.059557+0000 mon.a (mon.0) 2204 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[84576]: audit 2024-09-13T15:02:34.059557+0000 mon.a (mon.0) 2204 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[84576]: audit 2024-09-13T15:02:34.069472+0000 mon.a (mon.0) 2205 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[84576]: audit 2024-09-13T15:02:34.069472+0000 mon.a (mon.0) 2205 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[84576]: cluster 2024-09-13T15:02:34.840388+0000 mgr.y (mgr.26252) 1093 : cluster [DBG] pgmap v1634: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:35.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[84576]: cluster 2024-09-13T15:02:34.840388+0000 mgr.y (mgr.26252) 1093 : cluster [DBG] pgmap v1634: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:35.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[87147]: audit 2024-09-13T15:02:34.059557+0000 mon.a (mon.0) 2204 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[87147]: audit 2024-09-13T15:02:34.059557+0000 mon.a (mon.0) 2204 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[87147]: audit 2024-09-13T15:02:34.069472+0000 mon.a (mon.0) 2205 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[87147]: audit 2024-09-13T15:02:34.069472+0000 mon.a (mon.0) 2205 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[87147]: cluster 2024-09-13T15:02:34.840388+0000 mgr.y (mgr.26252) 1093 : cluster [DBG] pgmap v1634: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:35.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:35 smithi099 bash[87147]: cluster 2024-09-13T15:02:34.840388+0000 mgr.y (mgr.26252) 1093 : cluster [DBG] pgmap v1634: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:35.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:35 smithi177 bash[68594]: audit 2024-09-13T15:02:34.059557+0000 mon.a (mon.0) 2204 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:35 smithi177 bash[68594]: audit 2024-09-13T15:02:34.059557+0000 mon.a (mon.0) 2204 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:35 smithi177 bash[68594]: audit 2024-09-13T15:02:34.069472+0000 mon.a (mon.0) 2205 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:35 smithi177 bash[68594]: audit 2024-09-13T15:02:34.069472+0000 mon.a (mon.0) 2205 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:35 smithi177 bash[68594]: cluster 2024-09-13T15:02:34.840388+0000 mgr.y (mgr.26252) 1093 : cluster [DBG] pgmap v1634: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:35 smithi177 bash[68594]: cluster 2024-09-13T15:02:34.840388+0000 mgr.y (mgr.26252) 1093 : cluster [DBG] pgmap v1634: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:35.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:35.422 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:35.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:35.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:35.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:35.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:35.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:35.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:35.796 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:35.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:35.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:35.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:35.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:35.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:36.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:36.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:36.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:36.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:36.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:36.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:36.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:36.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:36.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:36.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:36.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:36.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:36.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:36.610 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:36.717 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:36.765 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:36 smithi177 bash[68594]: cluster 2024-09-13T15:02:36.487462+0000 mon.a (mon.0) 2206 : cluster [WRN] Health check update: Degraded data redundancy: 1324/12681 objects degraded (10.441%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:36.765 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:36 smithi177 bash[68594]: cluster 2024-09-13T15:02:36.487462+0000 mon.a (mon.0) 2206 : cluster [WRN] Health check update: Degraded data redundancy: 1324/12681 objects degraded (10.441%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:36.785 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:36.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:36 smithi099 bash[84576]: cluster 2024-09-13T15:02:36.487462+0000 mon.a (mon.0) 2206 : cluster [WRN] Health check update: Degraded data redundancy: 1324/12681 objects degraded (10.441%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:36.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:36 smithi099 bash[84576]: cluster 2024-09-13T15:02:36.487462+0000 mon.a (mon.0) 2206 : cluster [WRN] Health check update: Degraded data redundancy: 1324/12681 objects degraded (10.441%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:36.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:36 smithi099 bash[87147]: cluster 2024-09-13T15:02:36.487462+0000 mon.a (mon.0) 2206 : cluster [WRN] Health check update: Degraded data redundancy: 1324/12681 objects degraded (10.441%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:36.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:36 smithi099 bash[87147]: cluster 2024-09-13T15:02:36.487462+0000 mon.a (mon.0) 2206 : cluster [WRN] Health check update: Degraded data redundancy: 1324/12681 objects degraded (10.441%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:36.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:36.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:36.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:37.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:37.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:37.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:37.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:37.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:37.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:37.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:37.300 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:02:37] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-13T15:02:37.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:37.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:37.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:37.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:37.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:37.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:37.590 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:37.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:37.693 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:37.726 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:37.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:37.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: audit 2024-09-13T15:02:36.593989+0000 mon.a (mon.0) 2207 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: audit 2024-09-13T15:02:36.593989+0000 mon.a (mon.0) 2207 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: audit 2024-09-13T15:02:36.605578+0000 mon.a (mon.0) 2208 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: audit 2024-09-13T15:02:36.605578+0000 mon.a (mon.0) 2208 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: cluster 2024-09-13T15:02:36.841110+0000 mgr.y (mgr.26252) 1094 : cluster [DBG] pgmap v1635: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: cluster 2024-09-13T15:02:36.841110+0000 mgr.y (mgr.26252) 1094 : cluster [DBG] pgmap v1635: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: audit 2024-09-13T15:02:37.356959+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: audit 2024-09-13T15:02:37.356959+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: audit 2024-09-13T15:02:37.357661+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: audit 2024-09-13T15:02:37.357661+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: audit 2024-09-13T15:02:37.362155+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[87147]: audit 2024-09-13T15:02:37.362155+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: audit 2024-09-13T15:02:36.593989+0000 mon.a (mon.0) 2207 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: audit 2024-09-13T15:02:36.593989+0000 mon.a (mon.0) 2207 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: audit 2024-09-13T15:02:36.605578+0000 mon.a (mon.0) 2208 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: audit 2024-09-13T15:02:36.605578+0000 mon.a (mon.0) 2208 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: cluster 2024-09-13T15:02:36.841110+0000 mgr.y (mgr.26252) 1094 : cluster [DBG] pgmap v1635: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: cluster 2024-09-13T15:02:36.841110+0000 mgr.y (mgr.26252) 1094 : cluster [DBG] pgmap v1635: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: audit 2024-09-13T15:02:37.356959+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: audit 2024-09-13T15:02:37.356959+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: audit 2024-09-13T15:02:37.357661+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: audit 2024-09-13T15:02:37.357661+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: audit 2024-09-13T15:02:37.362155+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:37 smithi099 bash[84576]: audit 2024-09-13T15:02:37.362155+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:37.851 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:37.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:37.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: audit 2024-09-13T15:02:36.593989+0000 mon.a (mon.0) 2207 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: audit 2024-09-13T15:02:36.593989+0000 mon.a (mon.0) 2207 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: audit 2024-09-13T15:02:36.605578+0000 mon.a (mon.0) 2208 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: audit 2024-09-13T15:02:36.605578+0000 mon.a (mon.0) 2208 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: cluster 2024-09-13T15:02:36.841110+0000 mgr.y (mgr.26252) 1094 : cluster [DBG] pgmap v1635: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: cluster 2024-09-13T15:02:36.841110+0000 mgr.y (mgr.26252) 1094 : cluster [DBG] pgmap v1635: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 263 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1324/12681 objects degraded (10.441%) 2024-09-13T15:02:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: audit 2024-09-13T15:02:37.356959+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: audit 2024-09-13T15:02:37.356959+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: audit 2024-09-13T15:02:37.357661+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: audit 2024-09-13T15:02:37.357661+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: audit 2024-09-13T15:02:37.362155+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:37 smithi177 bash[68594]: audit 2024-09-13T15:02:37.362155+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:37.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:37.978 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:37.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:38.083 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:38.125 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:38.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:38.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:38.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:38.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:38.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:38.396 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:38.500 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:38.521 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:38.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:38.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:38.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:38.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:38.768 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:38.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:38.871 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:38.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:38.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:39.009 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:39.028 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:39.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:39.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:39.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:39.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:39.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:39.292 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:39.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:39.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:39.426 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:39.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:39.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:39.552 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:39.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:39.557 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Removed 116009 objects 2024-09-13T15:02:39.557 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Clean up completed and total clean up time :125.073 2024-09-13T15:02:39.658 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:39.679 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:39.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:39.805 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:39.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:39.908 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:39.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:39.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:39.985 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:40.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:40.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:40.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:40.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:40.178 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:39 smithi177 bash[68594]: cluster 2024-09-13T15:02:38.842010+0000 mgr.y (mgr.26252) 1095 : cluster [DBG] pgmap v1636: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 221 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1113/10671 objects degraded (10.430%) 2024-09-13T15:02:40.178 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:39 smithi177 bash[68594]: cluster 2024-09-13T15:02:38.842010+0000 mgr.y (mgr.26252) 1095 : cluster [DBG] pgmap v1636: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 221 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1113/10671 objects degraded (10.430%) 2024-09-13T15:02:40.192 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:40.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:40.295 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:40.322 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:40.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:40.425 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:40.436 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:02:40 smithi177 bash[61439]: ts=2024-09-13T15:02:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:40.451 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:40.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:40.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:40 smithi099 bash[84576]: cluster 2024-09-13T15:02:38.842010+0000 mgr.y (mgr.26252) 1095 : cluster [DBG] pgmap v1636: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 221 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1113/10671 objects degraded (10.430%) 2024-09-13T15:02:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:40 smithi099 bash[84576]: cluster 2024-09-13T15:02:38.842010+0000 mgr.y (mgr.26252) 1095 : cluster [DBG] pgmap v1636: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 221 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1113/10671 objects degraded (10.430%) 2024-09-13T15:02:40.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:40 smithi099 bash[87147]: cluster 2024-09-13T15:02:38.842010+0000 mgr.y (mgr.26252) 1095 : cluster [DBG] pgmap v1636: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 221 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1113/10671 objects degraded (10.430%) 2024-09-13T15:02:40.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:40 smithi099 bash[87147]: cluster 2024-09-13T15:02:38.842010+0000 mgr.y (mgr.26252) 1095 : cluster [DBG] pgmap v1636: 99 pgs: 26 active+undersized, 11 active+undersized+degraded, 62 active+clean; 221 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1113/10671 objects degraded (10.430%) 2024-09-13T15:02:40.580 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:40.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:40.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:40.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:40.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:40.814 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:40.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:40.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:40.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:40.969 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:40.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:41.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:41.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:41.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:41.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:41.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:41.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:41.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:41.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:41.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:41.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:41.485 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:41.588 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:41.612 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:41.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:41.716 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:41.742 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:41.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:41.845 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:41.870 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:41.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:41.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:41 smithi177 bash[68594]: cluster 2024-09-13T15:02:40.843907+0000 mgr.y (mgr.26252) 1096 : cluster [DBG] pgmap v1637: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:41.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:41 smithi177 bash[68594]: cluster 2024-09-13T15:02:40.843907+0000 mgr.y (mgr.26252) 1096 : cluster [DBG] pgmap v1637: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:41.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:41 smithi177 bash[68594]: cluster 2024-09-13T15:02:41.488179+0000 mon.a (mon.0) 2212 : cluster [WRN] Health check update: Degraded data redundancy: 194/1317 objects degraded (14.730%), 9 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:41.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:41 smithi177 bash[68594]: cluster 2024-09-13T15:02:41.488179+0000 mon.a (mon.0) 2212 : cluster [WRN] Health check update: Degraded data redundancy: 194/1317 objects degraded (14.730%), 9 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:42.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:42.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:42.010 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:41 smithi099 bash[84576]: cluster 2024-09-13T15:02:40.843907+0000 mgr.y (mgr.26252) 1096 : cluster [DBG] pgmap v1637: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:42.104 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:42.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:42.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:42.234 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:42.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:42.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:42.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:42 smithi099 bash[84576]: cluster 2024-09-13T15:02:40.843907+0000 mgr.y (mgr.26252) 1096 : cluster [DBG] pgmap v1637: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:42.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:42 smithi099 bash[84576]: cluster 2024-09-13T15:02:41.488179+0000 mon.a (mon.0) 2212 : cluster [WRN] Health check update: Degraded data redundancy: 194/1317 objects degraded (14.730%), 9 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:42.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:42 smithi099 bash[84576]: cluster 2024-09-13T15:02:41.488179+0000 mon.a (mon.0) 2212 : cluster [WRN] Health check update: Degraded data redundancy: 194/1317 objects degraded (14.730%), 9 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:42.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:42 smithi099 bash[87147]: cluster 2024-09-13T15:02:40.843907+0000 mgr.y (mgr.26252) 1096 : cluster [DBG] pgmap v1637: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:42.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:42 smithi099 bash[87147]: cluster 2024-09-13T15:02:40.843907+0000 mgr.y (mgr.26252) 1096 : cluster [DBG] pgmap v1637: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:42.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:42 smithi099 bash[87147]: cluster 2024-09-13T15:02:41.488179+0000 mon.a (mon.0) 2212 : cluster [WRN] Health check update: Degraded data redundancy: 194/1317 objects degraded (14.730%), 9 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:42.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:42 smithi099 bash[87147]: cluster 2024-09-13T15:02:41.488179+0000 mon.a (mon.0) 2212 : cluster [WRN] Health check update: Degraded data redundancy: 194/1317 objects degraded (14.730%), 9 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:42.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:42.389 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:42.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:42.493 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:42.520 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:42.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:42.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:42.650 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:42.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:42.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:42.780 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:42.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:42.883 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:42.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:42.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:43.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:43.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:43.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:43.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:43.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:43.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:43.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:43.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:43.403 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:43.429 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:43.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:43.532 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:43.560 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:43.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:43.663 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:43.692 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:43.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:43.795 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:43.823 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:43.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:43.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:43.956 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:43.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:44.059 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:44.070 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:43 smithi177 bash[68594]: cluster 2024-09-13T15:02:42.844680+0000 mgr.y (mgr.26252) 1097 : cluster [DBG] pgmap v1638: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:44.070 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:43 smithi177 bash[68594]: cluster 2024-09-13T15:02:42.844680+0000 mgr.y (mgr.26252) 1097 : cluster [DBG] pgmap v1638: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:44.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:44.190 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:44.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:43 smithi099 bash[84576]: cluster 2024-09-13T15:02:42.844680+0000 mgr.y (mgr.26252) 1097 : cluster [DBG] pgmap v1638: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:44.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:43 smithi099 bash[84576]: cluster 2024-09-13T15:02:42.844680+0000 mgr.y (mgr.26252) 1097 : cluster [DBG] pgmap v1638: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:44.202 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:43 smithi099 bash[87147]: cluster 2024-09-13T15:02:42.844680+0000 mgr.y (mgr.26252) 1097 : cluster [DBG] pgmap v1638: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:44.202 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:43 smithi099 bash[87147]: cluster 2024-09-13T15:02:42.844680+0000 mgr.y (mgr.26252) 1097 : cluster [DBG] pgmap v1638: 99 pgs: 28 active+undersized, 9 active+undersized+degraded, 62 active+clean; 26 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 194/1317 objects degraded (14.730%) 2024-09-13T15:02:44.219 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:44.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:44.293 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-09-13T15:02:44.293 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-09-13T15:02:44.322 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:44.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:44.484 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:44.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:44.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:44.607 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:44.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:44.652 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:02:44.653 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:02:44.672 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:02:44.673 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:02:44.686 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:02:44.687 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:02:44.687 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:02:44.708 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:02:44.708 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:02:44.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:44.728 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:02:44.728 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:02:44.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:44.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:44.745 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:02:44.745 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:02:44.762 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:02:44.762 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:02:44.840 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:44.863 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:44.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:44.916 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T15:02:44.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:44.978 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:44 smithi177 bash[68594]: audit 2024-09-13T15:02:44.635806+0000 mon.a (mon.0) 2213 : audit [DBG] from='client.? 172.21.15.99:0/4000521538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:44.978 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:44 smithi177 bash[68594]: audit 2024-09-13T15:02:44.635806+0000 mon.a (mon.0) 2213 : audit [DBG] from='client.? 172.21.15.99:0/4000521538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:44.978 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:44 smithi177 bash[68594]: audit 2024-09-13T15:02:44.663393+0000 mon.c (mon.2) 84 : audit [INF] from='client.? 172.21.15.99:0/3643309246' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:44.978 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:44 smithi177 bash[68594]: audit 2024-09-13T15:02:44.663393+0000 mon.c (mon.2) 84 : audit [INF] from='client.? 172.21.15.99:0/3643309246' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:44.978 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:44 smithi177 bash[68594]: audit 2024-09-13T15:02:44.663690+0000 mon.a (mon.0) 2214 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:44.978 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:44 smithi177 bash[68594]: audit 2024-09-13T15:02:44.663690+0000 mon.a (mon.0) 2214 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:44.994 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:44.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[84576]: audit 2024-09-13T15:02:44.635806+0000 mon.a (mon.0) 2213 : audit [DBG] from='client.? 172.21.15.99:0/4000521538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[84576]: audit 2024-09-13T15:02:44.635806+0000 mon.a (mon.0) 2213 : audit [DBG] from='client.? 172.21.15.99:0/4000521538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[84576]: audit 2024-09-13T15:02:44.663393+0000 mon.c (mon.2) 84 : audit [INF] from='client.? 172.21.15.99:0/3643309246' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[84576]: audit 2024-09-13T15:02:44.663393+0000 mon.c (mon.2) 84 : audit [INF] from='client.? 172.21.15.99:0/3643309246' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[84576]: audit 2024-09-13T15:02:44.663690+0000 mon.a (mon.0) 2214 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[84576]: audit 2024-09-13T15:02:44.663690+0000 mon.a (mon.0) 2214 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:45.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[87147]: audit 2024-09-13T15:02:44.635806+0000 mon.a (mon.0) 2213 : audit [DBG] from='client.? 172.21.15.99:0/4000521538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[87147]: audit 2024-09-13T15:02:44.635806+0000 mon.a (mon.0) 2213 : audit [DBG] from='client.? 172.21.15.99:0/4000521538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[87147]: audit 2024-09-13T15:02:44.663393+0000 mon.c (mon.2) 84 : audit [INF] from='client.? 172.21.15.99:0/3643309246' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[87147]: audit 2024-09-13T15:02:44.663393+0000 mon.c (mon.2) 84 : audit [INF] from='client.? 172.21.15.99:0/3643309246' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[87147]: audit 2024-09-13T15:02:44.663690+0000 mon.a (mon.0) 2214 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:44 smithi099 bash[87147]: audit 2024-09-13T15:02:44.663690+0000 mon.a (mon.0) 2214 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:45.097 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:45.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:45.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:45.227 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:45.254 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:45.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:45.357 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:45.378 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:45.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:45.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:45.509 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:45.612 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:45.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:45.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:45.743 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:45.770 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:45.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:45.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:45.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:45.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:46.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:46.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:46.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:46.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:46.157 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:46.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[84576]: cluster 2024-09-13T15:02:44.846306+0000 mgr.y (mgr.26252) 1098 : cluster [DBG] pgmap v1639: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:46.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[84576]: cluster 2024-09-13T15:02:44.846306+0000 mgr.y (mgr.26252) 1098 : cluster [DBG] pgmap v1639: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:46.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[84576]: cluster 2024-09-13T15:02:44.905705+0000 mon.a (mon.0) 2215 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:46.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[84576]: cluster 2024-09-13T15:02:44.905705+0000 mon.a (mon.0) 2215 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:46.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[84576]: audit 2024-09-13T15:02:44.912757+0000 mon.a (mon.0) 2216 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:46.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[84576]: audit 2024-09-13T15:02:44.912757+0000 mon.a (mon.0) 2216 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:46.226 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[84576]: cluster 2024-09-13T15:02:44.920517+0000 mon.a (mon.0) 2217 : cluster [DBG] osdmap e1053: 8 total, 7 up, 8 in 2024-09-13T15:02:46.226 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[84576]: cluster 2024-09-13T15:02:44.920517+0000 mon.a (mon.0) 2217 : cluster [DBG] osdmap e1053: 8 total, 7 up, 8 in 2024-09-13T15:02:46.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[87147]: cluster 2024-09-13T15:02:44.846306+0000 mgr.y (mgr.26252) 1098 : cluster [DBG] pgmap v1639: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:46.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[87147]: cluster 2024-09-13T15:02:44.846306+0000 mgr.y (mgr.26252) 1098 : cluster [DBG] pgmap v1639: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:46.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[87147]: cluster 2024-09-13T15:02:44.905705+0000 mon.a (mon.0) 2215 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:46.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[87147]: cluster 2024-09-13T15:02:44.905705+0000 mon.a (mon.0) 2215 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:46.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[87147]: audit 2024-09-13T15:02:44.912757+0000 mon.a (mon.0) 2216 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:46.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[87147]: audit 2024-09-13T15:02:44.912757+0000 mon.a (mon.0) 2216 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:46.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[87147]: cluster 2024-09-13T15:02:44.920517+0000 mon.a (mon.0) 2217 : cluster [DBG] osdmap e1053: 8 total, 7 up, 8 in 2024-09-13T15:02:46.227 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:45 smithi099 bash[87147]: cluster 2024-09-13T15:02:44.920517+0000 mon.a (mon.0) 2217 : cluster [DBG] osdmap e1053: 8 total, 7 up, 8 in 2024-09-13T15:02:46.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:46.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:46.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:45 smithi177 bash[68594]: cluster 2024-09-13T15:02:44.846306+0000 mgr.y (mgr.26252) 1098 : cluster [DBG] pgmap v1639: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:45 smithi177 bash[68594]: cluster 2024-09-13T15:02:44.846306+0000 mgr.y (mgr.26252) 1098 : cluster [DBG] pgmap v1639: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:45 smithi177 bash[68594]: cluster 2024-09-13T15:02:44.905705+0000 mon.a (mon.0) 2215 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:45 smithi177 bash[68594]: cluster 2024-09-13T15:02:44.905705+0000 mon.a (mon.0) 2215 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:45 smithi177 bash[68594]: audit 2024-09-13T15:02:44.912757+0000 mon.a (mon.0) 2216 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:45 smithi177 bash[68594]: audit 2024-09-13T15:02:44.912757+0000 mon.a (mon.0) 2216 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:45 smithi177 bash[68594]: cluster 2024-09-13T15:02:44.920517+0000 mon.a (mon.0) 2217 : cluster [DBG] osdmap e1053: 8 total, 7 up, 8 in 2024-09-13T15:02:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:45 smithi177 bash[68594]: cluster 2024-09-13T15:02:44.920517+0000 mon.a (mon.0) 2217 : cluster [DBG] osdmap e1053: 8 total, 7 up, 8 in 2024-09-13T15:02:46.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:46.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:46.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:46.515 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:46.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:46.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:46.645 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:46.670 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:46.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:46.773 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:46.801 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:46.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:46.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:46.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:46.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:47.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:46 smithi099 bash[84576]: cluster 2024-09-13T15:02:46.489605+0000 mon.a (mon.0) 2218 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 8 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:47.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:46 smithi099 bash[84576]: cluster 2024-09-13T15:02:46.489605+0000 mon.a (mon.0) 2218 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 8 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:47.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:46 smithi099 bash[87147]: cluster 2024-09-13T15:02:46.489605+0000 mon.a (mon.0) 2218 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 8 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:47.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:46 smithi099 bash[87147]: cluster 2024-09-13T15:02:46.489605+0000 mon.a (mon.0) 2218 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 8 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:47.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:47.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:47.175 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:46 smithi177 bash[68594]: cluster 2024-09-13T15:02:46.489605+0000 mon.a (mon.0) 2218 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 8 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:47.175 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:46 smithi177 bash[68594]: cluster 2024-09-13T15:02:46.489605+0000 mon.a (mon.0) 2218 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 8 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:47.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:47.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:47.294 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:47.311 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:02:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:02:47] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-13T15:02:47.322 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:47.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:47.425 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.452 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:47.555 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.583 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:47.687 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:47.715 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:47.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:47.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:47.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:47.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:47.948 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:47.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:47 smithi177 bash[68594]: cluster 2024-09-13T15:02:46.846969+0000 mgr.y (mgr.26252) 1099 : cluster [DBG] pgmap v1641: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:47.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:47 smithi177 bash[68594]: cluster 2024-09-13T15:02:46.846969+0000 mgr.y (mgr.26252) 1099 : cluster [DBG] pgmap v1641: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:47.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:47.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:48.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:48.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.235 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:48.263 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:47 smithi099 bash[84576]: cluster 2024-09-13T15:02:46.846969+0000 mgr.y (mgr.26252) 1099 : cluster [DBG] pgmap v1641: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:48.263 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:47 smithi099 bash[84576]: cluster 2024-09-13T15:02:46.846969+0000 mgr.y (mgr.26252) 1099 : cluster [DBG] pgmap v1641: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:48.263 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:47 smithi099 bash[87147]: cluster 2024-09-13T15:02:46.846969+0000 mgr.y (mgr.26252) 1099 : cluster [DBG] pgmap v1641: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:48.263 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:47 smithi099 bash[87147]: cluster 2024-09-13T15:02:46.846969+0000 mgr.y (mgr.26252) 1099 : cluster [DBG] pgmap v1641: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:48.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:48.366 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:48.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:48.469 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:48.495 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:48.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:48.598 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:48.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:48.859 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.887 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:48.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:49.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:49.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:49.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:49.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:49.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:49.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:49.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:02:49.310 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:02:49.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:49.396 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:49 smithi177 bash[68594]: audit 2024-09-13T15:02:48.304787+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:49.396 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:49 smithi177 bash[68594]: audit 2024-09-13T15:02:48.304787+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:49.396 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:49 smithi177 bash[68594]: audit 2024-09-13T15:02:48.307034+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:49.396 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:49 smithi177 bash[68594]: audit 2024-09-13T15:02:48.307034+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:49.396 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:49 smithi177 bash[68594]: cluster 2024-09-13T15:02:48.848375+0000 mgr.y (mgr.26252) 1100 : cluster [DBG] pgmap v1642: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 352 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:49.396 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:49 smithi177 bash[68594]: cluster 2024-09-13T15:02:48.848375+0000 mgr.y (mgr.26252) 1100 : cluster [DBG] pgmap v1642: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 352 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:49.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:49.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:49.516 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:49.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:49.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[84576]: audit 2024-09-13T15:02:48.304787+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[84576]: audit 2024-09-13T15:02:48.304787+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[84576]: audit 2024-09-13T15:02:48.307034+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[84576]: audit 2024-09-13T15:02:48.307034+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[84576]: cluster 2024-09-13T15:02:48.848375+0000 mgr.y (mgr.26252) 1100 : cluster [DBG] pgmap v1642: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 352 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[84576]: cluster 2024-09-13T15:02:48.848375+0000 mgr.y (mgr.26252) 1100 : cluster [DBG] pgmap v1642: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 352 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[87147]: audit 2024-09-13T15:02:48.304787+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[87147]: audit 2024-09-13T15:02:48.304787+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:02:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[87147]: audit 2024-09-13T15:02:48.307034+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[87147]: audit 2024-09-13T15:02:48.307034+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[87147]: cluster 2024-09-13T15:02:48.848375+0000 mgr.y (mgr.26252) 1100 : cluster [DBG] pgmap v1642: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 352 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:49 smithi099 bash[87147]: cluster 2024-09-13T15:02:48.848375+0000 mgr.y (mgr.26252) 1100 : cluster [DBG] pgmap v1642: 99 pgs: 29 active+undersized, 8 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 352 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:49.619 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:02:49.620 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:02:49.647 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:49.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:49.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:49.768 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:49.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:49.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:49.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:49.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:49.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:50.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:50.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:50.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:50.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:50.182 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:50.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:50.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:50.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:50.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:50.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:50.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:50.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:50.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:50.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:50.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:02:50 smithi177 bash[61439]: ts=2024-09-13T15:02:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:02:50.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:50.702 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:50.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:50.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:50.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:50.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:50.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:50.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:51.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:02:51.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:51.096 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:51.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:51.198 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:51.224 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:51.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:51.311 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:02:51.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:51.377 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:51.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:51.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:51.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:51.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:51.625 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:51.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:51.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:51.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:51.858 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:51.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:51.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:51.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:52.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:52.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:52.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T15:02:52.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:51 smithi177 bash[68594]: cluster 2024-09-13T15:02:50.849622+0000 mgr.y (mgr.26252) 1101 : cluster [DBG] pgmap v1643: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:52.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:51 smithi177 bash[68594]: cluster 2024-09-13T15:02:50.849622+0000 mgr.y (mgr.26252) 1101 : cluster [DBG] pgmap v1643: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:52.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:51 smithi177 bash[68594]: cluster 2024-09-13T15:02:51.490784+0000 mon.a (mon.0) 2221 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 6 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:52.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:51 smithi177 bash[68594]: cluster 2024-09-13T15:02:51.490784+0000 mon.a (mon.0) 2221 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 6 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:52.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:52.144 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:52.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:52.247 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:52.259 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:51 smithi099 bash[84576]: cluster 2024-09-13T15:02:50.849622+0000 mgr.y (mgr.26252) 1101 : cluster [DBG] pgmap v1643: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:52.259 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:51 smithi099 bash[84576]: cluster 2024-09-13T15:02:50.849622+0000 mgr.y (mgr.26252) 1101 : cluster [DBG] pgmap v1643: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:52.259 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:51 smithi099 bash[84576]: cluster 2024-09-13T15:02:51.490784+0000 mon.a (mon.0) 2221 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 6 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:52.259 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:51 smithi099 bash[84576]: cluster 2024-09-13T15:02:51.490784+0000 mon.a (mon.0) 2221 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 6 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:52.259 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:51 smithi099 bash[87147]: cluster 2024-09-13T15:02:50.849622+0000 mgr.y (mgr.26252) 1101 : cluster [DBG] pgmap v1643: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:52.259 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:51 smithi099 bash[87147]: cluster 2024-09-13T15:02:50.849622+0000 mgr.y (mgr.26252) 1101 : cluster [DBG] pgmap v1643: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:52.260 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:51 smithi099 bash[87147]: cluster 2024-09-13T15:02:51.490784+0000 mon.a (mon.0) 2221 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 6 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:52.260 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:51 smithi099 bash[87147]: cluster 2024-09-13T15:02:51.490784+0000 mon.a (mon.0) 2221 : cluster [WRN] Health check update: Degraded data redundancy: 67/267 objects degraded (25.094%), 6 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-09-13T15:02:52.270 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:52.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:52.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:52.401 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:52.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:52.410 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2024-09-13T15:02:52.410 INFO:teuthology.orchestra.run.smithi099.stderr:0 2024-09-13T15:02:52.410 INFO:teuthology.orchestra.run.smithi099.stderr:1 2024-09-13T15:02:52.410 INFO:teuthology.orchestra.run.smithi099.stderr:2 2024-09-13T15:02:52.410 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2024-09-13T15:02:52.411 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2024-09-13T15:02:52.411 INFO:teuthology.orchestra.run.smithi099.stderr:bench [] [] [] [] 2024-09-13T15:02:52.411 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs debug_inject_read_zeros 2024-09-13T15:02:52.411 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs files list 2024-09-13T15:02:52.411 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs stats 2024-09-13T15:02:52.411 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator dump block 2024-09-13T15:02:52.411 INFO:teuthology.orchestra.run.smithi099.stderr:Error EINVAL: invalid command 2024-09-13T15:02:52.504 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:52.523 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:52.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:52.646 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:52.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:52.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:52.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:52.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:52.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:52.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:52.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:53.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:53.034 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:53.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:53.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:53.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:53.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:53.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:53.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:53.399 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:53.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:53.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:53.494 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T15:02:53.494 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T15:02:53.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-09-13T15:02:53.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:53.557 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:53.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:53.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:53.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:53.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:53.816 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:53.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:53.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:53.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:53.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:54.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:54.063 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:53 smithi177 bash[68594]: cluster 2024-09-13T15:02:52.851032+0000 mgr.y (mgr.26252) 1102 : cluster [DBG] pgmap v1644: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:54.063 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:53 smithi177 bash[68594]: cluster 2024-09-13T15:02:52.851032+0000 mgr.y (mgr.26252) 1102 : cluster [DBG] pgmap v1644: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:54.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.081 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:54.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:54.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:54.209 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:54.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:54.312 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:54.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:53 smithi099 bash[84576]: cluster 2024-09-13T15:02:52.851032+0000 mgr.y (mgr.26252) 1102 : cluster [DBG] pgmap v1644: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:54.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:53 smithi099 bash[84576]: cluster 2024-09-13T15:02:52.851032+0000 mgr.y (mgr.26252) 1102 : cluster [DBG] pgmap v1644: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:54.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:53 smithi099 bash[87147]: cluster 2024-09-13T15:02:52.851032+0000 mgr.y (mgr.26252) 1102 : cluster [DBG] pgmap v1644: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:54.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:53 smithi099 bash[87147]: cluster 2024-09-13T15:02:52.851032+0000 mgr.y (mgr.26252) 1102 : cluster [DBG] pgmap v1644: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 67/267 objects degraded (25.094%) 2024-09-13T15:02:54.339 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:54.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:54.442 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:54.471 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:54.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:54.575 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:54.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:54.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:54.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:54.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:54.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:54.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:54.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:54.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:54.977 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:55.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:55.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:55.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:55.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:55.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:55.264 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:55.292 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:55.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:55.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:55.423 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:55.526 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:55.554 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:55.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:55.657 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:55.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:55.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:55.787 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:55.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:55.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:55.927 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:55 smithi177 bash[68594]: cluster 2024-09-13T15:02:54.852653+0000 mgr.y (mgr.26252) 1103 : cluster [DBG] pgmap v1645: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:55.927 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:55 smithi177 bash[68594]: cluster 2024-09-13T15:02:54.852653+0000 mgr.y (mgr.26252) 1103 : cluster [DBG] pgmap v1645: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:55.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:55.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:56.046 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:56.072 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:56.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:56.175 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:56.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:56.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:56.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:55 smithi099 bash[84576]: cluster 2024-09-13T15:02:54.852653+0000 mgr.y (mgr.26252) 1103 : cluster [DBG] pgmap v1645: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:55 smithi099 bash[84576]: cluster 2024-09-13T15:02:54.852653+0000 mgr.y (mgr.26252) 1103 : cluster [DBG] pgmap v1645: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:56.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:55 smithi099 bash[87147]: cluster 2024-09-13T15:02:54.852653+0000 mgr.y (mgr.26252) 1103 : cluster [DBG] pgmap v1645: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:56.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:55 smithi099 bash[87147]: cluster 2024-09-13T15:02:54.852653+0000 mgr.y (mgr.26252) 1103 : cluster [DBG] pgmap v1645: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:56.330 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:56.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:56.434 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:56.460 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:56.564 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:56.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:56.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:56.687 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:56.715 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:56.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:56.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:56.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:56.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:56.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:56.975 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:56.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:57.008 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:57.008 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:57.009 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:57.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:02:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:02:57] "GET /metrics HTTP/1.1" 200 36713 "" "Prometheus/2.51.0" 2024-09-13T15:02:57.078 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:57.100 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:57.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:57.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:57.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:57.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:57.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:57.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:57.466 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:57.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:57.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:57.597 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:57.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:57.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:57.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:57.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:57.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:57.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:57.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:57.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:57.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:58.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:58.014 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:57 smithi099 bash[84576]: cluster 2024-09-13T15:02:56.853948+0000 mgr.y (mgr.26252) 1104 : cluster [DBG] pgmap v1646: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:58.014 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:57 smithi099 bash[84576]: cluster 2024-09-13T15:02:56.853948+0000 mgr.y (mgr.26252) 1104 : cluster [DBG] pgmap v1646: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:58.014 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:57 smithi099 bash[87147]: cluster 2024-09-13T15:02:56.853948+0000 mgr.y (mgr.26252) 1104 : cluster [DBG] pgmap v1646: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:58.015 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:57 smithi099 bash[87147]: cluster 2024-09-13T15:02:56.853948+0000 mgr.y (mgr.26252) 1104 : cluster [DBG] pgmap v1646: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:58.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:58.127 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:57 smithi177 bash[68594]: cluster 2024-09-13T15:02:56.853948+0000 mgr.y (mgr.26252) 1104 : cluster [DBG] pgmap v1646: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:58.127 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:57 smithi177 bash[68594]: cluster 2024-09-13T15:02:56.853948+0000 mgr.y (mgr.26252) 1104 : cluster [DBG] pgmap v1646: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:58.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:58.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:58.272 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:58.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:58.376 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:58.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:58.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:58.417 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:58.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:58.533 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:58.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:58.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:58.663 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:58.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:58.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:58.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:58.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:58.895 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:58.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:58.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:59.026 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:59.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:59.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:59.155 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:59.182 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:59.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:59.285 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:59.312 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:59.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:59.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:59.440 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:59.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:59.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:59.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:59.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:59.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:59.701 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:59.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:59.804 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:59.830 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:59.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:59.944 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:59 smithi099 bash[84576]: cluster 2024-09-13T15:02:58.855391+0000 mgr.y (mgr.26252) 1105 : cluster [DBG] pgmap v1647: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:59.944 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:02:59 smithi099 bash[84576]: cluster 2024-09-13T15:02:58.855391+0000 mgr.y (mgr.26252) 1105 : cluster [DBG] pgmap v1647: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:59.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:59 smithi099 bash[87147]: cluster 2024-09-13T15:02:58.855391+0000 mgr.y (mgr.26252) 1105 : cluster [DBG] pgmap v1647: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:59.944 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:02:59 smithi099 bash[87147]: cluster 2024-09-13T15:02:58.855391+0000 mgr.y (mgr.26252) 1105 : cluster [DBG] pgmap v1647: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:02:59.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:59.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:00.060 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:00.085 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:00.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:00.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:00.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:59 smithi177 bash[68594]: cluster 2024-09-13T15:02:58.855391+0000 mgr.y (mgr.26252) 1105 : cluster [DBG] pgmap v1647: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:00.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:02:59 smithi177 bash[68594]: cluster 2024-09-13T15:02:58.855391+0000 mgr.y (mgr.26252) 1105 : cluster [DBG] pgmap v1647: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:00.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:00.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:00.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:00.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:00.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:00.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:00.460 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:03:00 smithi177 bash[61439]: ts=2024-09-13T15:03:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:00.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:00.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:00.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:00.607 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:00.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:00.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:00.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:00.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:00.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:00.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:00.973 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:00.996 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:00.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:01.099 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:01.128 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:01.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:01.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:01.258 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:01.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:01.361 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:01.388 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:01.492 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:01.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:01.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:01.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:01.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:01.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:01.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:01.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:01.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:01.907 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:01.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:02.010 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:02.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:02.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:02.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:02.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:01 smithi177 bash[68594]: cluster 2024-09-13T15:03:00.856987+0000 mgr.y (mgr.26252) 1106 : cluster [DBG] pgmap v1648: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:02.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:01 smithi177 bash[68594]: cluster 2024-09-13T15:03:00.856987+0000 mgr.y (mgr.26252) 1106 : cluster [DBG] pgmap v1648: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:02.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:02.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:02.270 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:02.295 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:02.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:02.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:01 smithi099 bash[84576]: cluster 2024-09-13T15:03:00.856987+0000 mgr.y (mgr.26252) 1106 : cluster [DBG] pgmap v1648: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:02.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:01 smithi099 bash[84576]: cluster 2024-09-13T15:03:00.856987+0000 mgr.y (mgr.26252) 1106 : cluster [DBG] pgmap v1648: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:01 smithi099 bash[87147]: cluster 2024-09-13T15:03:00.856987+0000 mgr.y (mgr.26252) 1106 : cluster [DBG] pgmap v1648: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:02.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:01 smithi099 bash[87147]: cluster 2024-09-13T15:03:00.856987+0000 mgr.y (mgr.26252) 1106 : cluster [DBG] pgmap v1648: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:02.397 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:02.423 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:02.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:02.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:02.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:02.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:02.658 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:02.687 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:02.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:02.790 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:02.816 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:02.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:02.947 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_2' removed 2024-09-13T15:03:02.951 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:03.055 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:03.073 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:03.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:02 smithi177 bash[68594]: audit 2024-09-13T15:03:02.440512+0000 mon.c (mon.2) 85 : audit [INF] from='client.? 172.21.15.99:0/2379911767' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:02 smithi177 bash[68594]: audit 2024-09-13T15:03:02.440512+0000 mon.c (mon.2) 85 : audit [INF] from='client.? 172.21.15.99:0/2379911767' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:02 smithi177 bash[68594]: audit 2024-09-13T15:03:02.441171+0000 mon.a (mon.0) 2222 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:02 smithi177 bash[68594]: audit 2024-09-13T15:03:02.441171+0000 mon.a (mon.0) 2222 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.175 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:03.183 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:02 smithi099 bash[84576]: audit 2024-09-13T15:03:02.440512+0000 mon.c (mon.2) 85 : audit [INF] from='client.? 172.21.15.99:0/2379911767' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.183 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:02 smithi099 bash[84576]: audit 2024-09-13T15:03:02.440512+0000 mon.c (mon.2) 85 : audit [INF] from='client.? 172.21.15.99:0/2379911767' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.183 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:02 smithi099 bash[84576]: audit 2024-09-13T15:03:02.441171+0000 mon.a (mon.0) 2222 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.183 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:02 smithi099 bash[84576]: audit 2024-09-13T15:03:02.441171+0000 mon.a (mon.0) 2222 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:02 smithi099 bash[87147]: audit 2024-09-13T15:03:02.440512+0000 mon.c (mon.2) 85 : audit [INF] from='client.? 172.21.15.99:0/2379911767' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:02 smithi099 bash[87147]: audit 2024-09-13T15:03:02.440512+0000 mon.c (mon.2) 85 : audit [INF] from='client.? 172.21.15.99:0/2379911767' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.183 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:02 smithi099 bash[87147]: audit 2024-09-13T15:03:02.441171+0000 mon.a (mon.0) 2222 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.184 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:02 smithi099 bash[87147]: audit 2024-09-13T15:03:02.441171+0000 mon.a (mon.0) 2222 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:03:03.193 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:03.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:03.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:03.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:03.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:03.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:03.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:03.555 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:03.583 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:03.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:03.685 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:03.714 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:03.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:03.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:03.835 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:03.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:03.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:03.963 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:03.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:04.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:04.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:04.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:04.196 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:04.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[84576]: cluster 2024-09-13T15:03:02.858421+0000 mgr.y (mgr.26252) 1107 : cluster [DBG] pgmap v1649: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:04.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[84576]: cluster 2024-09-13T15:03:02.858421+0000 mgr.y (mgr.26252) 1107 : cluster [DBG] pgmap v1649: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:04.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[84576]: audit 2024-09-13T15:03:02.936974+0000 mon.a (mon.0) 2223 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:03:04.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[84576]: audit 2024-09-13T15:03:02.936974+0000 mon.a (mon.0) 2223 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:03:04.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[84576]: cluster 2024-09-13T15:03:02.941100+0000 mon.a (mon.0) 2224 : cluster [DBG] osdmap e1054: 8 total, 7 up, 8 in 2024-09-13T15:03:04.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[84576]: cluster 2024-09-13T15:03:02.941100+0000 mon.a (mon.0) 2224 : cluster [DBG] osdmap e1054: 8 total, 7 up, 8 in 2024-09-13T15:03:04.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[84576]: audit 2024-09-13T15:03:03.295636+0000 mon.a (mon.0) 2225 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:04.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[84576]: audit 2024-09-13T15:03:03.295636+0000 mon.a (mon.0) 2225 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:04.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[87147]: cluster 2024-09-13T15:03:02.858421+0000 mgr.y (mgr.26252) 1107 : cluster [DBG] pgmap v1649: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:04.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[87147]: cluster 2024-09-13T15:03:02.858421+0000 mgr.y (mgr.26252) 1107 : cluster [DBG] pgmap v1649: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:04.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[87147]: audit 2024-09-13T15:03:02.936974+0000 mon.a (mon.0) 2223 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:03:04.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[87147]: audit 2024-09-13T15:03:02.936974+0000 mon.a (mon.0) 2223 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:03:04.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[87147]: cluster 2024-09-13T15:03:02.941100+0000 mon.a (mon.0) 2224 : cluster [DBG] osdmap e1054: 8 total, 7 up, 8 in 2024-09-13T15:03:04.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[87147]: cluster 2024-09-13T15:03:02.941100+0000 mon.a (mon.0) 2224 : cluster [DBG] osdmap e1054: 8 total, 7 up, 8 in 2024-09-13T15:03:04.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[87147]: audit 2024-09-13T15:03:03.295636+0000 mon.a (mon.0) 2225 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:04.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:03 smithi099 bash[87147]: audit 2024-09-13T15:03:03.295636+0000 mon.a (mon.0) 2225 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:04.220 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:04.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:04.323 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:04.335 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:03 smithi177 bash[68594]: cluster 2024-09-13T15:03:02.858421+0000 mgr.y (mgr.26252) 1107 : cluster [DBG] pgmap v1649: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:04.335 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:03 smithi177 bash[68594]: cluster 2024-09-13T15:03:02.858421+0000 mgr.y (mgr.26252) 1107 : cluster [DBG] pgmap v1649: 99 pgs: 31 active+undersized, 6 active+undersized+degraded, 62 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 67/267 objects degraded (25.094%) 2024-09-13T15:03:04.335 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:03 smithi177 bash[68594]: audit 2024-09-13T15:03:02.936974+0000 mon.a (mon.0) 2223 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:03:04.335 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:03 smithi177 bash[68594]: audit 2024-09-13T15:03:02.936974+0000 mon.a (mon.0) 2223 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:03:04.335 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:03 smithi177 bash[68594]: cluster 2024-09-13T15:03:02.941100+0000 mon.a (mon.0) 2224 : cluster [DBG] osdmap e1054: 8 total, 7 up, 8 in 2024-09-13T15:03:04.335 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:03 smithi177 bash[68594]: cluster 2024-09-13T15:03:02.941100+0000 mon.a (mon.0) 2224 : cluster [DBG] osdmap e1054: 8 total, 7 up, 8 in 2024-09-13T15:03:04.336 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:03 smithi177 bash[68594]: audit 2024-09-13T15:03:03.295636+0000 mon.a (mon.0) 2225 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:04.336 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:03 smithi177 bash[68594]: audit 2024-09-13T15:03:03.295636+0000 mon.a (mon.0) 2225 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:04.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:04.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:04.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:04.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:04.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:04.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:04.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:04.716 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:04.745 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:04.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:04.848 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:04.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:04.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:04.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:05.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:05.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:05.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:05.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:05.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:05.250 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:04 smithi177 bash[68594]: cluster 2024-09-13T15:03:04.938389+0000 mon.a (mon.0) 2226 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:05.250 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:04 smithi177 bash[68594]: cluster 2024-09-13T15:03:04.938389+0000 mon.a (mon.0) 2226 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:05.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:05.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:05.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:04 smithi099 bash[87147]: cluster 2024-09-13T15:03:04.938389+0000 mon.a (mon.0) 2226 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:05.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:05 smithi099 bash[87147]: cluster 2024-09-13T15:03:04.938389+0000 mon.a (mon.0) 2226 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:04 smithi099 bash[84576]: cluster 2024-09-13T15:03:04.938389+0000 mon.a (mon.0) 2226 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:05.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:05 smithi099 bash[84576]: cluster 2024-09-13T15:03:04.938389+0000 mon.a (mon.0) 2226 : cluster [WRN] Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:05.370 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:05.398 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:05.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:05.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:05.528 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:05.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:05.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:05.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:05.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:05.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:05.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:05.920 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:05.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:06.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:06.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:06.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:06.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:05 smithi099 bash[84576]: cluster 2024-09-13T15:03:04.859763+0000 mgr.y (mgr.26252) 1108 : cluster [DBG] pgmap v1651: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:06.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:05 smithi099 bash[84576]: cluster 2024-09-13T15:03:04.859763+0000 mgr.y (mgr.26252) 1108 : cluster [DBG] pgmap v1651: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:06.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:05 smithi099 bash[87147]: cluster 2024-09-13T15:03:04.859763+0000 mgr.y (mgr.26252) 1108 : cluster [DBG] pgmap v1651: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:06.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:05 smithi099 bash[87147]: cluster 2024-09-13T15:03:04.859763+0000 mgr.y (mgr.26252) 1108 : cluster [DBG] pgmap v1651: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:06.153 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:06.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:06.283 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:06.294 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:05 smithi177 bash[68594]: cluster 2024-09-13T15:03:04.859763+0000 mgr.y (mgr.26252) 1108 : cluster [DBG] pgmap v1651: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:06.294 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:05 smithi177 bash[68594]: cluster 2024-09-13T15:03:04.859763+0000 mgr.y (mgr.26252) 1108 : cluster [DBG] pgmap v1651: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:06.311 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:06.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:06.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:06.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:06.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:06.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:06.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:06.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:06.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:06.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:06.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:06.832 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:06.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:06.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:06.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:06.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:07.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:07.090 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:07.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:07.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:07.221 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:07.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:07.270 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:03:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:03:07] "GET /metrics HTTP/1.1" 200 34346 "" "Prometheus/2.51.0" 2024-09-13T15:03:07.271 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T15:03:07.272 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T15:03:07.272 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-09-13T15:03:07.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-09-13T15:03:07.324 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:07.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:07.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:07.454 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:07.480 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:07.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:07.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:07.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:07.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:07.713 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:07.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:07.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:07.842 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:07.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:07.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:07.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:07.994 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:07.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:08.098 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:08.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:08.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:08.227 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:08.252 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:08.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:08.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:07 smithi099 bash[84576]: cluster 2024-09-13T15:03:06.860826+0000 mgr.y (mgr.26252) 1109 : cluster [DBG] pgmap v1652: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:08.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:07 smithi099 bash[84576]: cluster 2024-09-13T15:03:06.860826+0000 mgr.y (mgr.26252) 1109 : cluster [DBG] pgmap v1652: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:08.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:07 smithi099 bash[87147]: cluster 2024-09-13T15:03:06.860826+0000 mgr.y (mgr.26252) 1109 : cluster [DBG] pgmap v1652: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:08.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:07 smithi099 bash[87147]: cluster 2024-09-13T15:03:06.860826+0000 mgr.y (mgr.26252) 1109 : cluster [DBG] pgmap v1652: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:08.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:08.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:07 smithi177 bash[68594]: cluster 2024-09-13T15:03:06.860826+0000 mgr.y (mgr.26252) 1109 : cluster [DBG] pgmap v1652: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:07 smithi177 bash[68594]: cluster 2024-09-13T15:03:06.860826+0000 mgr.y (mgr.26252) 1109 : cluster [DBG] pgmap v1652: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:08.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:08.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:08.515 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:08.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:08.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:08.644 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:08.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:08.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:08.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:08.878 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:08.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:08.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:09.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:09.034 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:09.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:09.137 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:09.163 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:09.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:09.266 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:09.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:09.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:09.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:09.423 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:09.527 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:09.554 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:09.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:09.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:09.679 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:09.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:09.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:09.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:09.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:09.940 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:09.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:10.043 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:10.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:10 smithi099 bash[87147]: cluster 2024-09-13T15:03:08.862174+0000 mgr.y (mgr.26252) 1110 : cluster [DBG] pgmap v1653: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:10.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:10 smithi099 bash[87147]: cluster 2024-09-13T15:03:08.862174+0000 mgr.y (mgr.26252) 1110 : cluster [DBG] pgmap v1653: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:10.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:10 smithi099 bash[84576]: cluster 2024-09-13T15:03:08.862174+0000 mgr.y (mgr.26252) 1110 : cluster [DBG] pgmap v1653: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:10.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:10 smithi099 bash[84576]: cluster 2024-09-13T15:03:08.862174+0000 mgr.y (mgr.26252) 1110 : cluster [DBG] pgmap v1653: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:10.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:10.175 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:10.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:10.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:10.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:10.333 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:10.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:10 smithi177 bash[68594]: cluster 2024-09-13T15:03:08.862174+0000 mgr.y (mgr.26252) 1110 : cluster [DBG] pgmap v1653: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:10 smithi177 bash[68594]: cluster 2024-09-13T15:03:08.862174+0000 mgr.y (mgr.26252) 1110 : cluster [DBG] pgmap v1653: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:10.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:03:10 smithi177 bash[61439]: ts=2024-09-13T15:03:10.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:10.435 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:10.463 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:10.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:10.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:10.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:10.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:10.726 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:10.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:10.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:10.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:10.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:10.960 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:10.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:10.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:11.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:11.117 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:11.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:11.220 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:11.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:11.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:11.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:11.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:11.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:11.478 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:11.504 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:11.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:11.609 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:11.636 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:11.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:11.763 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:11.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:11.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:11.895 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:11.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:11.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:12.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:12.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:12.032 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:12.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:12.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:12 smithi177 bash[68594]: cluster 2024-09-13T15:03:10.863597+0000 mgr.y (mgr.26252) 1111 : cluster [DBG] pgmap v1654: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:12.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:12 smithi177 bash[68594]: cluster 2024-09-13T15:03:10.863597+0000 mgr.y (mgr.26252) 1111 : cluster [DBG] pgmap v1654: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:12.156 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:12.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:12.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:12.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:12 smithi099 bash[84576]: cluster 2024-09-13T15:03:10.863597+0000 mgr.y (mgr.26252) 1111 : cluster [DBG] pgmap v1654: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:12.271 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:12 smithi099 bash[84576]: cluster 2024-09-13T15:03:10.863597+0000 mgr.y (mgr.26252) 1111 : cluster [DBG] pgmap v1654: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:12.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:12 smithi099 bash[87147]: cluster 2024-09-13T15:03:10.863597+0000 mgr.y (mgr.26252) 1111 : cluster [DBG] pgmap v1654: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:12.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:12 smithi099 bash[87147]: cluster 2024-09-13T15:03:10.863597+0000 mgr.y (mgr.26252) 1111 : cluster [DBG] pgmap v1654: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:12.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:12.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:12.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:12.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:12.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:12.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:12.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:12.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:12.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:12.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:12.777 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:12.804 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:12.908 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:12.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:12.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:13.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:13.066 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:13.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:13.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:13.195 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:13.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:13.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:13.324 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:13.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:13.427 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:13.454 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:13.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:13.557 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:13.584 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:13.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:13.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:13.714 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:13.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:13.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:13.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:13.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:13.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:13.977 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:13.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:14.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:14.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:14.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:14.211 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:14.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:14 smithi099 bash[84576]: cluster 2024-09-13T15:03:12.864852+0000 mgr.y (mgr.26252) 1112 : cluster [DBG] pgmap v1655: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:14.238 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:14 smithi099 bash[84576]: cluster 2024-09-13T15:03:12.864852+0000 mgr.y (mgr.26252) 1112 : cluster [DBG] pgmap v1655: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:14.239 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:14 smithi099 bash[87147]: cluster 2024-09-13T15:03:12.864852+0000 mgr.y (mgr.26252) 1112 : cluster [DBG] pgmap v1655: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:14.239 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:14 smithi099 bash[87147]: cluster 2024-09-13T15:03:12.864852+0000 mgr.y (mgr.26252) 1112 : cluster [DBG] pgmap v1655: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:14.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:14.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:14.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:14.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:14 smithi177 bash[68594]: cluster 2024-09-13T15:03:12.864852+0000 mgr.y (mgr.26252) 1112 : cluster [DBG] pgmap v1655: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:14.353 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:14 smithi177 bash[68594]: cluster 2024-09-13T15:03:12.864852+0000 mgr.y (mgr.26252) 1112 : cluster [DBG] pgmap v1655: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:14.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:14.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:14.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:14.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:14.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:14.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:14.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:14.668 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:03:14.668 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:03:14.686 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:03:14.686 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:03:14.702 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:03:14.703 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:03:14.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:03:14.724 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:03:14.725 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:03:14.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:14.744 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:03:14.744 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:03:14.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:14.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:14.765 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:03:14.765 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:03:14.785 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:03:14.785 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:03:14.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:14.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:14.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:14.981 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:15.004 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:15.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:15.015 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:03:15.015 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:03:15.016 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:03:15.016 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:03:15.016 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:03:15.021 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T15:03:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[84576]: audit 2024-09-13T15:03:14.607835+0000 mon.c (mon.2) 86 : audit [INF] from='client.? 172.21.15.99:0/2153907399' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[84576]: audit 2024-09-13T15:03:14.607835+0000 mon.c (mon.2) 86 : audit [INF] from='client.? 172.21.15.99:0/2153907399' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[84576]: audit 2024-09-13T15:03:14.608323+0000 mon.a (mon.0) 2227 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[84576]: audit 2024-09-13T15:03:14.608323+0000 mon.a (mon.0) 2227 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[84576]: audit 2024-09-13T15:03:14.651723+0000 mon.a (mon.0) 2228 : audit [DBG] from='client.? 172.21.15.99:0/3420699263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:15.071 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[84576]: audit 2024-09-13T15:03:14.651723+0000 mon.a (mon.0) 2228 : audit [DBG] from='client.? 172.21.15.99:0/3420699263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:15.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[87147]: audit 2024-09-13T15:03:14.607835+0000 mon.c (mon.2) 86 : audit [INF] from='client.? 172.21.15.99:0/2153907399' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.071 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[87147]: audit 2024-09-13T15:03:14.607835+0000 mon.c (mon.2) 86 : audit [INF] from='client.? 172.21.15.99:0/2153907399' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[87147]: audit 2024-09-13T15:03:14.608323+0000 mon.a (mon.0) 2227 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[87147]: audit 2024-09-13T15:03:14.608323+0000 mon.a (mon.0) 2227 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[87147]: audit 2024-09-13T15:03:14.651723+0000 mon.a (mon.0) 2228 : audit [DBG] from='client.? 172.21.15.99:0/3420699263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:15.072 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:15 smithi099 bash[87147]: audit 2024-09-13T15:03:14.651723+0000 mon.a (mon.0) 2228 : audit [DBG] from='client.? 172.21.15.99:0/3420699263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:15.107 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:15.125 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:15.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:15.197 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2024-09-13T15:03:15.197 INFO:teuthology.orchestra.run.smithi099.stderr:0 2024-09-13T15:03:15.197 INFO:teuthology.orchestra.run.smithi099.stderr:1 2024-09-13T15:03:15.197 INFO:teuthology.orchestra.run.smithi099.stderr:2 2024-09-13T15:03:15.198 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2024-09-13T15:03:15.198 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2024-09-13T15:03:15.198 INFO:teuthology.orchestra.run.smithi099.stderr:bench [] [] [] [] 2024-09-13T15:03:15.198 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs debug_inject_read_zeros 2024-09-13T15:03:15.198 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs files list 2024-09-13T15:03:15.198 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs stats 2024-09-13T15:03:15.198 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator dump block 2024-09-13T15:03:15.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error EINVAL: invalid command 2024-09-13T15:03:15.227 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:15.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:15.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:15.359 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:15.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:15 smithi177 bash[68594]: audit 2024-09-13T15:03:14.607835+0000 mon.c (mon.2) 86 : audit [INF] from='client.? 172.21.15.99:0/2153907399' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:15 smithi177 bash[68594]: audit 2024-09-13T15:03:14.607835+0000 mon.c (mon.2) 86 : audit [INF] from='client.? 172.21.15.99:0/2153907399' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:15 smithi177 bash[68594]: audit 2024-09-13T15:03:14.608323+0000 mon.a (mon.0) 2227 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:15 smithi177 bash[68594]: audit 2024-09-13T15:03:14.608323+0000 mon.a (mon.0) 2227 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:15 smithi177 bash[68594]: audit 2024-09-13T15:03:14.651723+0000 mon.a (mon.0) 2228 : audit [DBG] from='client.? 172.21.15.99:0/3420699263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:15 smithi177 bash[68594]: audit 2024-09-13T15:03:14.651723+0000 mon.a (mon.0) 2228 : audit [DBG] from='client.? 172.21.15.99:0/3420699263' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:15.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:15.491 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:15.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:15.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:15.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:15.648 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:15.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:15.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:15.779 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:15.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:15.883 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:15.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:15.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:16.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:16.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.154 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:16 smithi177 bash[68594]: cluster 2024-09-13T15:03:14.865557+0000 mgr.y (mgr.26252) 1113 : cluster [DBG] pgmap v1656: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:16.154 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:16 smithi177 bash[68594]: cluster 2024-09-13T15:03:14.865557+0000 mgr.y (mgr.26252) 1113 : cluster [DBG] pgmap v1656: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:16.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:16 smithi177 bash[68594]: cluster 2024-09-13T15:03:15.008693+0000 mon.a (mon.0) 2229 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:16.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:16 smithi177 bash[68594]: cluster 2024-09-13T15:03:15.008693+0000 mon.a (mon.0) 2229 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:16.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:16 smithi177 bash[68594]: audit 2024-09-13T15:03:15.016518+0000 mon.a (mon.0) 2230 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:16.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:16 smithi177 bash[68594]: audit 2024-09-13T15:03:15.016518+0000 mon.a (mon.0) 2230 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:16.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:16 smithi177 bash[68594]: cluster 2024-09-13T15:03:15.021953+0000 mon.a (mon.0) 2231 : cluster [DBG] osdmap e1055: 8 total, 7 up, 8 in 2024-09-13T15:03:16.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:16 smithi177 bash[68594]: cluster 2024-09-13T15:03:15.021953+0000 mon.a (mon.0) 2231 : cluster [DBG] osdmap e1055: 8 total, 7 up, 8 in 2024-09-13T15:03:16.173 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:16.276 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.304 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[84576]: cluster 2024-09-13T15:03:14.865557+0000 mgr.y (mgr.26252) 1113 : cluster [DBG] pgmap v1656: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[84576]: cluster 2024-09-13T15:03:14.865557+0000 mgr.y (mgr.26252) 1113 : cluster [DBG] pgmap v1656: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[84576]: cluster 2024-09-13T15:03:15.008693+0000 mon.a (mon.0) 2229 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[84576]: cluster 2024-09-13T15:03:15.008693+0000 mon.a (mon.0) 2229 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[84576]: audit 2024-09-13T15:03:15.016518+0000 mon.a (mon.0) 2230 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[84576]: audit 2024-09-13T15:03:15.016518+0000 mon.a (mon.0) 2230 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[84576]: cluster 2024-09-13T15:03:15.021953+0000 mon.a (mon.0) 2231 : cluster [DBG] osdmap e1055: 8 total, 7 up, 8 in 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[84576]: cluster 2024-09-13T15:03:15.021953+0000 mon.a (mon.0) 2231 : cluster [DBG] osdmap e1055: 8 total, 7 up, 8 in 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[87147]: cluster 2024-09-13T15:03:14.865557+0000 mgr.y (mgr.26252) 1113 : cluster [DBG] pgmap v1656: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[87147]: cluster 2024-09-13T15:03:14.865557+0000 mgr.y (mgr.26252) 1113 : cluster [DBG] pgmap v1656: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:16.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[87147]: cluster 2024-09-13T15:03:15.008693+0000 mon.a (mon.0) 2229 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:16.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[87147]: cluster 2024-09-13T15:03:15.008693+0000 mon.a (mon.0) 2229 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:16.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[87147]: audit 2024-09-13T15:03:15.016518+0000 mon.a (mon.0) 2230 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:16.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[87147]: audit 2024-09-13T15:03:15.016518+0000 mon.a (mon.0) 2230 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:16.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[87147]: cluster 2024-09-13T15:03:15.021953+0000 mon.a (mon.0) 2231 : cluster [DBG] osdmap e1055: 8 total, 7 up, 8 in 2024-09-13T15:03:16.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:16 smithi099 bash[87147]: cluster 2024-09-13T15:03:15.021953+0000 mon.a (mon.0) 2231 : cluster [DBG] osdmap e1055: 8 total, 7 up, 8 in 2024-09-13T15:03:16.407 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.436 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:16.539 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:16.566 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:16.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:16.669 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:16.696 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:16.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:16.799 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:16.827 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:16.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:16.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:16.958 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:16.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:17.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:17.088 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:17.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:17.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:17.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:17.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:17.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:17.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:03:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:03:17] "GET /metrics HTTP/1.1" 200 34346 "" "Prometheus/2.51.0" 2024-09-13T15:03:17.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:17.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:17.445 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:17.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:17.575 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:17.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:17.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:17.707 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:17.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:17.838 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:17.863 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:17.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:17.965 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:17.993 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:17.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:18.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:18.125 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:18.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:18.228 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:18.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:18.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:18.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:18 smithi099 bash[87147]: cluster 2024-09-13T15:03:16.866622+0000 mgr.y (mgr.26252) 1114 : cluster [DBG] pgmap v1658: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:18.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:18 smithi099 bash[87147]: cluster 2024-09-13T15:03:16.866622+0000 mgr.y (mgr.26252) 1114 : cluster [DBG] pgmap v1658: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:18.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:18 smithi099 bash[84576]: cluster 2024-09-13T15:03:16.866622+0000 mgr.y (mgr.26252) 1114 : cluster [DBG] pgmap v1658: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:18.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:18 smithi099 bash[84576]: cluster 2024-09-13T15:03:16.866622+0000 mgr.y (mgr.26252) 1114 : cluster [DBG] pgmap v1658: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:18.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:18 smithi177 bash[68594]: cluster 2024-09-13T15:03:16.866622+0000 mgr.y (mgr.26252) 1114 : cluster [DBG] pgmap v1658: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:18.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:18 smithi177 bash[68594]: cluster 2024-09-13T15:03:16.866622+0000 mgr.y (mgr.26252) 1114 : cluster [DBG] pgmap v1658: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:18.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:18.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:18.489 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:18.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:18.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:18.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:18.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:18.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:18.749 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:18.777 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:18.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:18.875 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:03:18.876 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:03:18.880 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:18.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:18.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:19.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:19.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:19.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:19.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:19 smithi177 bash[68594]: audit 2024-09-13T15:03:18.296649+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:19.152 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:19 smithi177 bash[68594]: audit 2024-09-13T15:03:18.296649+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:19.168 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:19.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:19.271 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:19.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:19.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:19.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:19 smithi099 bash[84576]: audit 2024-09-13T15:03:18.296649+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:19 smithi099 bash[84576]: audit 2024-09-13T15:03:18.296649+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:19 smithi099 bash[87147]: audit 2024-09-13T15:03:18.296649+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:19 smithi099 bash[87147]: audit 2024-09-13T15:03:18.296649+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:19.401 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:19.426 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:19.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:19.529 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:19.539 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T15:03:19.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:19.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:19.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:19.684 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:19.787 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:19.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:19.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:19.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:19.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:19.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:20.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:20.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:20 smithi177 bash[68594]: cluster 2024-09-13T15:03:18.867532+0000 mgr.y (mgr.26252) 1115 : cluster [DBG] pgmap v1659: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:20.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:20 smithi177 bash[68594]: cluster 2024-09-13T15:03:18.867532+0000 mgr.y (mgr.26252) 1115 : cluster [DBG] pgmap v1659: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:20.075 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:20.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:20.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:20.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:20.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:20.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:20 smithi099 bash[87147]: cluster 2024-09-13T15:03:18.867532+0000 mgr.y (mgr.26252) 1115 : cluster [DBG] pgmap v1659: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:20.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:20 smithi099 bash[87147]: cluster 2024-09-13T15:03:18.867532+0000 mgr.y (mgr.26252) 1115 : cluster [DBG] pgmap v1659: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:20.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:20 smithi099 bash[84576]: cluster 2024-09-13T15:03:18.867532+0000 mgr.y (mgr.26252) 1115 : cluster [DBG] pgmap v1659: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:20.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:20 smithi099 bash[84576]: cluster 2024-09-13T15:03:18.867532+0000 mgr.y (mgr.26252) 1115 : cluster [DBG] pgmap v1659: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:20.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:20.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:03:20 smithi177 bash[61439]: ts=2024-09-13T15:03:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:20.361 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:20.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:20.390 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T15:03:20.391 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T15:03:20.464 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:20.484 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:20.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:20.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:20.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:20.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:20.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:20.742 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:20.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:20.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:20.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:20.877 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:03:20.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:20.999 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:21.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:21.103 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:21.129 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:21.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:21.232 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:21.260 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:21.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:21.363 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:21.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:21.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:21.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:21.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:21.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:03:21.626 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:21.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:21.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:21.755 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:21.773 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:21.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:21.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:21.902 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:21.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:22.006 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:22.027 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:22.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:22.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:22.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:22.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:22.260 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:22.287 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:22.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:22 smithi177 bash[68594]: cluster 2024-09-13T15:03:20.868526+0000 mgr.y (mgr.26252) 1116 : cluster [DBG] pgmap v1660: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:22.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:22 smithi177 bash[68594]: cluster 2024-09-13T15:03:20.868526+0000 mgr.y (mgr.26252) 1116 : cluster [DBG] pgmap v1660: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:22.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:22.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:22 smithi099 bash[87147]: cluster 2024-09-13T15:03:20.868526+0000 mgr.y (mgr.26252) 1116 : cluster [DBG] pgmap v1660: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:22.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:22 smithi099 bash[87147]: cluster 2024-09-13T15:03:20.868526+0000 mgr.y (mgr.26252) 1116 : cluster [DBG] pgmap v1660: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:22.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:22 smithi099 bash[84576]: cluster 2024-09-13T15:03:20.868526+0000 mgr.y (mgr.26252) 1116 : cluster [DBG] pgmap v1660: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:22 smithi099 bash[84576]: cluster 2024-09-13T15:03:20.868526+0000 mgr.y (mgr.26252) 1116 : cluster [DBG] pgmap v1660: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:22.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:22.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:22.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:22.523 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:22.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:22.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:22.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:22.678 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:22.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:22.782 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:22.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:22.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:22.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:22.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:22.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:23.044 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:23.051 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_3' created 2024-09-13T15:03:23.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:23 smithi177 bash[68594]: audit 2024-09-13T15:03:22.329706+0000 mon.c (mon.2) 87 : audit [INF] from='client.? 172.21.15.99:0/2545055380' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:23 smithi177 bash[68594]: audit 2024-09-13T15:03:22.329706+0000 mon.c (mon.2) 87 : audit [INF] from='client.? 172.21.15.99:0/2545055380' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:23 smithi177 bash[68594]: audit 2024-09-13T15:03:22.330131+0000 mon.a (mon.0) 2233 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:23 smithi177 bash[68594]: audit 2024-09-13T15:03:22.330131+0000 mon.a (mon.0) 2233 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.072 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:23.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:23.175 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:23.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:23.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:23.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:23.316 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:23 smithi099 bash[84576]: audit 2024-09-13T15:03:22.329706+0000 mon.c (mon.2) 87 : audit [INF] from='client.? 172.21.15.99:0/2545055380' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.316 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:23 smithi099 bash[84576]: audit 2024-09-13T15:03:22.329706+0000 mon.c (mon.2) 87 : audit [INF] from='client.? 172.21.15.99:0/2545055380' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.316 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:23 smithi099 bash[84576]: audit 2024-09-13T15:03:22.330131+0000 mon.a (mon.0) 2233 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:23 smithi099 bash[84576]: audit 2024-09-13T15:03:22.330131+0000 mon.a (mon.0) 2233 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:23 smithi099 bash[87147]: audit 2024-09-13T15:03:22.329706+0000 mon.c (mon.2) 87 : audit [INF] from='client.? 172.21.15.99:0/2545055380' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:23 smithi099 bash[87147]: audit 2024-09-13T15:03:22.329706+0000 mon.c (mon.2) 87 : audit [INF] from='client.? 172.21.15.99:0/2545055380' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:23 smithi099 bash[87147]: audit 2024-09-13T15:03:22.330131+0000 mon.a (mon.0) 2233 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:23 smithi099 bash[87147]: audit 2024-09-13T15:03:22.330131+0000 mon.a (mon.0) 2233 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T15:03:23.334 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:23.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:23.437 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:23.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:23.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:23.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:23.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:23.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:23.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:23.724 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:23.828 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:23.855 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:23.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:23.959 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:23.984 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:23.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:24.087 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:24.111 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:24.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:24.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:24.239 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:24.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:24.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-09-13T15:03:24.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[84576]: cluster 2024-09-13T15:03:22.869778+0000 mgr.y (mgr.26252) 1117 : cluster [DBG] pgmap v1661: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:24.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[84576]: cluster 2024-09-13T15:03:22.869778+0000 mgr.y (mgr.26252) 1117 : cluster [DBG] pgmap v1661: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[84576]: audit 2024-09-13T15:03:23.044025+0000 mon.a (mon.0) 2234 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T15:03:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[84576]: audit 2024-09-13T15:03:23.044025+0000 mon.a (mon.0) 2234 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T15:03:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[84576]: cluster 2024-09-13T15:03:23.051482+0000 mon.a (mon.0) 2235 : cluster [DBG] osdmap e1056: 8 total, 7 up, 8 in 2024-09-13T15:03:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[84576]: cluster 2024-09-13T15:03:23.051482+0000 mon.a (mon.0) 2235 : cluster [DBG] osdmap e1056: 8 total, 7 up, 8 in 2024-09-13T15:03:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[87147]: cluster 2024-09-13T15:03:22.869778+0000 mgr.y (mgr.26252) 1117 : cluster [DBG] pgmap v1661: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[87147]: cluster 2024-09-13T15:03:22.869778+0000 mgr.y (mgr.26252) 1117 : cluster [DBG] pgmap v1661: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[87147]: audit 2024-09-13T15:03:23.044025+0000 mon.a (mon.0) 2234 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T15:03:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[87147]: audit 2024-09-13T15:03:23.044025+0000 mon.a (mon.0) 2234 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T15:03:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[87147]: cluster 2024-09-13T15:03:23.051482+0000 mon.a (mon.0) 2235 : cluster [DBG] osdmap e1056: 8 total, 7 up, 8 in 2024-09-13T15:03:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:24 smithi099 bash[87147]: cluster 2024-09-13T15:03:23.051482+0000 mon.a (mon.0) 2235 : cluster [DBG] osdmap e1056: 8 total, 7 up, 8 in 2024-09-13T15:03:24.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:24 smithi177 bash[68594]: cluster 2024-09-13T15:03:22.869778+0000 mgr.y (mgr.26252) 1117 : cluster [DBG] pgmap v1661: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:24 smithi177 bash[68594]: cluster 2024-09-13T15:03:22.869778+0000 mgr.y (mgr.26252) 1117 : cluster [DBG] pgmap v1661: 73 pgs: 26 active+undersized, 3 active+undersized+degraded, 44 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:24 smithi177 bash[68594]: audit 2024-09-13T15:03:23.044025+0000 mon.a (mon.0) 2234 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T15:03:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:24 smithi177 bash[68594]: audit 2024-09-13T15:03:23.044025+0000 mon.a (mon.0) 2234 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T15:03:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:24 smithi177 bash[68594]: cluster 2024-09-13T15:03:23.051482+0000 mon.a (mon.0) 2235 : cluster [DBG] osdmap e1056: 8 total, 7 up, 8 in 2024-09-13T15:03:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:24 smithi177 bash[68594]: cluster 2024-09-13T15:03:23.051482+0000 mon.a (mon.0) 2235 : cluster [DBG] osdmap e1056: 8 total, 7 up, 8 in 2024-09-13T15:03:24.365 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:24.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:24.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:24.496 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:24.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:24.599 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:24.625 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:24.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:24.717 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:24.728 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:24.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:24.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:24.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:24.876 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:24.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:24.979 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:25.108 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:25.238 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:25.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[84576]: cluster 2024-09-13T15:03:24.054116+0000 mon.a (mon.0) 2236 : cluster [DBG] osdmap e1057: 8 total, 7 up, 8 in 2024-09-13T15:03:25.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[84576]: cluster 2024-09-13T15:03:24.054116+0000 mon.a (mon.0) 2236 : cluster [DBG] osdmap e1057: 8 total, 7 up, 8 in 2024-09-13T15:03:25.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[84576]: cluster 2024-09-13T15:03:24.870593+0000 mgr.y (mgr.26252) 1118 : cluster [DBG] pgmap v1664: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:25.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[84576]: cluster 2024-09-13T15:03:24.870593+0000 mgr.y (mgr.26252) 1118 : cluster [DBG] pgmap v1664: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:25.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[84576]: cluster 2024-09-13T15:03:25.056076+0000 mon.a (mon.0) 2237 : cluster [DBG] osdmap e1058: 8 total, 7 up, 8 in 2024-09-13T15:03:25.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[84576]: cluster 2024-09-13T15:03:25.056076+0000 mon.a (mon.0) 2237 : cluster [DBG] osdmap e1058: 8 total, 7 up, 8 in 2024-09-13T15:03:25.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[87147]: cluster 2024-09-13T15:03:24.054116+0000 mon.a (mon.0) 2236 : cluster [DBG] osdmap e1057: 8 total, 7 up, 8 in 2024-09-13T15:03:25.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[87147]: cluster 2024-09-13T15:03:24.054116+0000 mon.a (mon.0) 2236 : cluster [DBG] osdmap e1057: 8 total, 7 up, 8 in 2024-09-13T15:03:25.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[87147]: cluster 2024-09-13T15:03:24.870593+0000 mgr.y (mgr.26252) 1118 : cluster [DBG] pgmap v1664: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:25.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[87147]: cluster 2024-09-13T15:03:24.870593+0000 mgr.y (mgr.26252) 1118 : cluster [DBG] pgmap v1664: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:25.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[87147]: cluster 2024-09-13T15:03:25.056076+0000 mon.a (mon.0) 2237 : cluster [DBG] osdmap e1058: 8 total, 7 up, 8 in 2024-09-13T15:03:25.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:25 smithi099 bash[87147]: cluster 2024-09-13T15:03:25.056076+0000 mon.a (mon.0) 2237 : cluster [DBG] osdmap e1058: 8 total, 7 up, 8 in 2024-09-13T15:03:25.264 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:25.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:25.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:25 smithi177 bash[68594]: cluster 2024-09-13T15:03:24.054116+0000 mon.a (mon.0) 2236 : cluster [DBG] osdmap e1057: 8 total, 7 up, 8 in 2024-09-13T15:03:25.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:25 smithi177 bash[68594]: cluster 2024-09-13T15:03:24.054116+0000 mon.a (mon.0) 2236 : cluster [DBG] osdmap e1057: 8 total, 7 up, 8 in 2024-09-13T15:03:25.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:25 smithi177 bash[68594]: cluster 2024-09-13T15:03:24.870593+0000 mgr.y (mgr.26252) 1118 : cluster [DBG] pgmap v1664: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:25.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:25 smithi177 bash[68594]: cluster 2024-09-13T15:03:24.870593+0000 mgr.y (mgr.26252) 1118 : cluster [DBG] pgmap v1664: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:25.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:25 smithi177 bash[68594]: cluster 2024-09-13T15:03:25.056076+0000 mon.a (mon.0) 2237 : cluster [DBG] osdmap e1058: 8 total, 7 up, 8 in 2024-09-13T15:03:25.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:25 smithi177 bash[68594]: cluster 2024-09-13T15:03:25.056076+0000 mon.a (mon.0) 2237 : cluster [DBG] osdmap e1058: 8 total, 7 up, 8 in 2024-09-13T15:03:25.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.392 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T15:03:25.392 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:03:25.396 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:25.499 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:25.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.659 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:25.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:25.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:25.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:25.893 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:25.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:25.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:26.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:26.051 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:26.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:26.154 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:26.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:26.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:26.283 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:26.293 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:26 smithi177 bash[68594]: cluster 2024-09-13T15:03:25.066186+0000 mon.a (mon.0) 2238 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:03:26.293 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:26 smithi177 bash[68594]: cluster 2024-09-13T15:03:25.066186+0000 mon.a (mon.0) 2238 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:03:26.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:26.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:26.412 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:26.440 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:26.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:26.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:26 smithi099 bash[87147]: cluster 2024-09-13T15:03:25.066186+0000 mon.a (mon.0) 2238 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:03:26.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:26 smithi099 bash[87147]: cluster 2024-09-13T15:03:25.066186+0000 mon.a (mon.0) 2238 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:03:26.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:26 smithi099 bash[84576]: cluster 2024-09-13T15:03:25.066186+0000 mon.a (mon.0) 2238 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:03:26.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:26 smithi099 bash[84576]: cluster 2024-09-13T15:03:25.066186+0000 mon.a (mon.0) 2238 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:03:26.544 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:26.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:26.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:26.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:26.700 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:26.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:26.755 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:26.756 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:26.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:26.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:26.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:26.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:26.961 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:26.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:27.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:27.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:03:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:03:27] "GET /metrics HTTP/1.1" 200 34347 "" "Prometheus/2.51.0" 2024-09-13T15:03:27.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:27.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:27.196 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:27.223 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:27.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:27.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:27.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:27.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:27 smithi177 bash[68594]: cluster 2024-09-13T15:03:26.871267+0000 mgr.y (mgr.26252) 1119 : cluster [DBG] pgmap v1666: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:27.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:27 smithi177 bash[68594]: cluster 2024-09-13T15:03:26.871267+0000 mgr.y (mgr.26252) 1119 : cluster [DBG] pgmap v1666: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:27.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:27.393 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:27 smithi099 bash[84576]: cluster 2024-09-13T15:03:26.871267+0000 mgr.y (mgr.26252) 1119 : cluster [DBG] pgmap v1666: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:27.394 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:27 smithi099 bash[84576]: cluster 2024-09-13T15:03:26.871267+0000 mgr.y (mgr.26252) 1119 : cluster [DBG] pgmap v1666: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:27.394 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:27 smithi099 bash[87147]: cluster 2024-09-13T15:03:26.871267+0000 mgr.y (mgr.26252) 1119 : cluster [DBG] pgmap v1666: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:27.394 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:27 smithi099 bash[87147]: cluster 2024-09-13T15:03:26.871267+0000 mgr.y (mgr.26252) 1119 : cluster [DBG] pgmap v1666: 89 pgs: 6 creating+peering, 6 unknown, 27 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:27.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:27.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:27.585 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:27.611 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:27.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:27.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:27.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:27.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:27.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:27.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:27.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:27.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:28.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:28.107 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:28.134 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:28.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:28.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:28.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:28.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:28.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:28.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:28.499 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:28.528 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:28.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:28.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:28.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:28.759 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:28.787 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:28.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:28.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:28.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:29.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:29.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:29.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:29.152 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:29.178 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:29.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:29.281 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:29.310 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:29.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:29.401 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:29.413 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:29.435 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:29.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:29.541 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:29.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:29.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:29.670 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:29.698 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:29.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:29.801 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:29.826 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:29.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:29.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:30.016 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:29 smithi099 bash[87147]: cluster 2024-09-13T15:03:28.872550+0000 mgr.y (mgr.26252) 1120 : cluster [DBG] pgmap v1667: 89 pgs: 6 creating+peering, 28 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:30.016 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:29 smithi099 bash[87147]: cluster 2024-09-13T15:03:28.872550+0000 mgr.y (mgr.26252) 1120 : cluster [DBG] pgmap v1667: 89 pgs: 6 creating+peering, 28 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:30.017 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:29 smithi099 bash[84576]: cluster 2024-09-13T15:03:28.872550+0000 mgr.y (mgr.26252) 1120 : cluster [DBG] pgmap v1667: 89 pgs: 6 creating+peering, 28 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:30.017 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:29 smithi099 bash[84576]: cluster 2024-09-13T15:03:28.872550+0000 mgr.y (mgr.26252) 1120 : cluster [DBG] pgmap v1667: 89 pgs: 6 creating+peering, 28 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:30.060 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:30.085 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:30.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:29 smithi177 bash[68594]: cluster 2024-09-13T15:03:28.872550+0000 mgr.y (mgr.26252) 1120 : cluster [DBG] pgmap v1667: 89 pgs: 6 creating+peering, 28 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:30.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:29 smithi177 bash[68594]: cluster 2024-09-13T15:03:28.872550+0000 mgr.y (mgr.26252) 1120 : cluster [DBG] pgmap v1667: 89 pgs: 6 creating+peering, 28 active+undersized, 3 active+undersized+degraded, 52 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:30.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:30.216 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:30.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:30.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:30.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:30.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:30.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:30.459 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:03:30 smithi177 bash[61439]: ts=2024-09-13T15:03:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:30.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:30.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:30.575 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:30.603 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:30.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:30.706 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:30.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:30.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:30.834 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:30.859 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:30.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:30.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:30.987 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:30.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:31.090 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:31.117 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:31.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:31.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:31.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:31.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:31.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:31.378 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:31.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:31.508 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:31.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:31.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:31.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:31.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:31.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:31.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:31.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:31.874 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:31.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:31.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:32.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:31 smithi177 bash[68594]: cluster 2024-09-13T15:03:30.874079+0000 mgr.y (mgr.26252) 1121 : cluster [DBG] pgmap v1668: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:32.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:31 smithi177 bash[68594]: cluster 2024-09-13T15:03:30.874079+0000 mgr.y (mgr.26252) 1121 : cluster [DBG] pgmap v1668: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:32.024 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:32.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:31 smithi099 bash[87147]: cluster 2024-09-13T15:03:30.874079+0000 mgr.y (mgr.26252) 1121 : cluster [DBG] pgmap v1668: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:32.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:31 smithi099 bash[87147]: cluster 2024-09-13T15:03:30.874079+0000 mgr.y (mgr.26252) 1121 : cluster [DBG] pgmap v1668: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:32.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:31 smithi099 bash[84576]: cluster 2024-09-13T15:03:30.874079+0000 mgr.y (mgr.26252) 1121 : cluster [DBG] pgmap v1668: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:32.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:31 smithi099 bash[84576]: cluster 2024-09-13T15:03:30.874079+0000 mgr.y (mgr.26252) 1121 : cluster [DBG] pgmap v1668: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:32.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:32.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:32.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:32.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:32.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:32.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:32.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:32.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:32.516 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:32.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:32.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:32.641 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:32.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:32.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:32.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:32.800 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:32.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:32.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:32.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:32.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:33.034 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:33.062 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:33.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:33.166 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:33.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:33.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:33.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:33.322 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:33.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:33.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:33.451 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:33.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:33.555 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:33.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:33.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:33.686 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:33.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:33.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:33.814 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:33.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:33.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:33.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:33.955 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:33 smithi177 bash[68594]: cluster 2024-09-13T15:03:32.875399+0000 mgr.y (mgr.26252) 1122 : cluster [DBG] pgmap v1669: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:33.955 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:33 smithi177 bash[68594]: cluster 2024-09-13T15:03:32.875399+0000 mgr.y (mgr.26252) 1122 : cluster [DBG] pgmap v1669: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:33.955 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:33 smithi177 bash[68594]: audit 2024-09-13T15:03:33.303553+0000 mon.a (mon.0) 2239 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:33.955 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:33 smithi177 bash[68594]: audit 2024-09-13T15:03:33.303553+0000 mon.a (mon.0) 2239 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:33.956 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:33 smithi177 bash[68594]: audit 2024-09-13T15:03:33.305771+0000 mon.a (mon.0) 2240 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:33.956 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:33 smithi177 bash[68594]: audit 2024-09-13T15:03:33.305771+0000 mon.a (mon.0) 2240 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:33.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:34.074 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:34.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:34.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:34.205 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:34.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[87147]: cluster 2024-09-13T15:03:32.875399+0000 mgr.y (mgr.26252) 1122 : cluster [DBG] pgmap v1669: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:34.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[87147]: cluster 2024-09-13T15:03:32.875399+0000 mgr.y (mgr.26252) 1122 : cluster [DBG] pgmap v1669: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:34.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[87147]: audit 2024-09-13T15:03:33.303553+0000 mon.a (mon.0) 2239 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:34.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[87147]: audit 2024-09-13T15:03:33.303553+0000 mon.a (mon.0) 2239 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:34.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[87147]: audit 2024-09-13T15:03:33.305771+0000 mon.a (mon.0) 2240 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:34.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[87147]: audit 2024-09-13T15:03:33.305771+0000 mon.a (mon.0) 2240 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:34.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[84576]: cluster 2024-09-13T15:03:32.875399+0000 mgr.y (mgr.26252) 1122 : cluster [DBG] pgmap v1669: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:34.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[84576]: cluster 2024-09-13T15:03:32.875399+0000 mgr.y (mgr.26252) 1122 : cluster [DBG] pgmap v1669: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:34.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[84576]: audit 2024-09-13T15:03:33.303553+0000 mon.a (mon.0) 2239 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:34.306 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[84576]: audit 2024-09-13T15:03:33.303553+0000 mon.a (mon.0) 2239 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:34.306 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[84576]: audit 2024-09-13T15:03:33.305771+0000 mon.a (mon.0) 2240 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:34.306 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:33 smithi099 bash[84576]: audit 2024-09-13T15:03:33.305771+0000 mon.a (mon.0) 2240 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:34.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:34.363 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:34.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:34.466 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:34.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:34.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:34.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:34.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.755 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:34.858 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:34.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:35.017 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:35.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:35.122 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:35.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:35.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:35.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:35.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:35.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:35.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:35.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:35.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:35.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:35.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:35.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:35.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:35.668 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:35.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:35.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:62908 43.8 e1043/1038 43:1708afe2:::benchmark_data_smithi099_171278_object62907:head v 1043'3967, mlcod=1043'3964)", 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:41.782974+0000", 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 234.0054901, 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27706982200000002, 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.791 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.782974+0000", 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.782974+0000", 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.782974+0000", 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.39e-07 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783011+0000", 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6804999999999997e-05 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783026+0000", 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4754e-05 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783030+0000", 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9720000000000003e-06 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783067+0000", 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7697000000000002e-05 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783081+0000", 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3841e-05 2024-09-13T15:03:35.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.059969+0000", 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.276888142 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.060044+0000", 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4271999999999996e-05 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:62910 43.c e1043/1038 43:34c54c2f:::benchmark_data_smithi099_171278_object62909:head v 1043'3944, mlcod=1043'3942)", 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:41.783040+0000", 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 234.00542407399999, 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.276891832, 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783040+0000", 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783040+0000", 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783040+0000", 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5400000000000002e-07 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783072+0000", 2024-09-13T15:03:35.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1764000000000003e-05 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783076+0000", 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3610000000000003e-06 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783080+0000", 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.929e-06 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783131+0000", 2024-09-13T15:03:35.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0411000000000002e-05 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.783151+0000", 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0010999999999999e-05 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.059800+0000", 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27664949900000002 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.059932+0000", 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013150299999999999 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:62915 43.a e1043/1038 43:59232d88:::benchmark_data_smithi099_171278_object62914:head v 1043'3923, mlcod=1043'3920)", 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:41.784983+0000", 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 234.003480626, 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28444309200000001, 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.800 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.784983+0000", 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.784983+0000", 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.784984+0000", 2024-09-13T15:03:35.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3399999999999997e-07 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.785062+0000", 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7857999999999999e-05 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.785073+0000", 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0569e-05 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.785079+0000", 2024-09-13T15:03:35.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3640000000000004e-06 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.785132+0000", 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2859999999999999e-05 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.785152+0000", 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0157999999999999e-05 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.069325+0000", 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28417338800000003 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.069426+0000", 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100961 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:62914 43.e e1043/1038 43:744da3da:::benchmark_data_smithi099_171278_object62913:head v 1043'3942, mlcod=1043'3940)", 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:41.786889+0000", 2024-09-13T15:03:35.804 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 234.001575309, 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28224980500000002, 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.786889+0000", 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.786889+0000", 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.786890+0000", 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9600000000000008e-07 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.786937+0000", 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7936999999999999e-05 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.059737+0000", 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.272799556 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.059769+0000", 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1881999999999997e-05 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.060325+0000", 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00055643399999999997 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.060383+0000", 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7481000000000001e-05 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.069083+0000", 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0086998079999999998 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.069138+0000", 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5711000000000002e-05 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.808 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:62916 43.1 e1043/1038 43:81a921db:::benchmark_data_smithi099_171278_object62915:head v 1043'3809, mlcod=1043'3807)", 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:41.787612+0000", 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 234.00085199399999, 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28719391599999999, 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.787612+0000", 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.787612+0000", 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.787613+0000", 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0669999999999999e-06 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:41.787650+0000", 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6922999999999999e-05 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.059738+0000", 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27208846399999997 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.059769+0000", 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0614999999999997e-05 2024-09-13T15:03:35.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.060019+0000", 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025038799999999999 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.060078+0000", 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8695e-05 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.074543+0000", 2024-09-13T15:03:35.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01446529 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:42.074806+0000", 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026247400000000001 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:69656 43.8 e1044/1038 43:16d0437d:::benchmark_data_smithi099_171278_object69655:head v 1044'4368, mlcod=1044'4366)", 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:47.218548+0000", 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 228.56991591799999, 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.309768918, 2024-09-13T15:03:35.813 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218548+0000", 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218548+0000", 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218549+0000", 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1880000000000001e-06 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218634+0000", 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4421999999999999e-05 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218654+0000", 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9962e-05 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218714+0000", 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0857999999999999e-05 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218837+0000", 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012214499999999999 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218866+0000", 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9488999999999998e-05 2024-09-13T15:03:35.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528244+0000", 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.30937769799999998 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528317+0000", 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3156000000000002e-05 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:69662 43.8 e1044/1038 43:11b12823:::benchmark_data_smithi099_171278_object69661:head v 1044'4369, mlcod=1044'4366)", 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:47.218922+0000", 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 228.56954164499999, 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.31473454299999998, 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218922+0000", 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218922+0000", 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218923+0000", 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7199999999999998e-07 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.218989+0000", 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6174999999999996e-05 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.219002+0000", 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2536e-05 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.219010+0000", 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2430000000000005e-06 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.221909+0000", 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028993209999999998 2024-09-13T15:03:35.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.221942+0000", 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3288999999999999e-05 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.533610+0000", 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.31166793799999998 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.533657+0000", 2024-09-13T15:03:35.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6369000000000003e-05 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:69657 43.0 e1044/1038 43:0396367f:::benchmark_data_smithi099_171278_object69656:head v 1044'4422, mlcod=1044'4419)", 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:47.219115+0000", 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 228.56934891899999, 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.30924895200000002, 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.219115+0000", 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.219115+0000", 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.219116+0000", 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7000000000000004e-07 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.219171+0000", 2024-09-13T15:03:35.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5454000000000001e-05 2024-09-13T15:03:35.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.219186+0000", 2024-09-13T15:03:35.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.501e-05 2024-09-13T15:03:35.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.219196+0000", 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041e-05 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.221732+0000", 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002535542 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.221760+0000", 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7606999999999998e-05 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528330+0000", 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.30657003999999999 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528364+0000", 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4218999999999999e-05 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.826 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:69665 43.0 e1044/1038 43:076008b8:::benchmark_data_smithi099_171278_object69664:head v 1044'4423, mlcod=1044'4420)", 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:47.222989+0000", 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 228.56547502199999, 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.313249787, 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.222989+0000", 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.222989+0000", 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.222990+0000", 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.359e-06 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.223059+0000", 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8887999999999995e-05 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.223074+0000", 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.521e-05 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.223084+0000", 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.815e-06 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.224686+0000", 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001602058 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.224710+0000", 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4258000000000001e-05 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.536191+0000", 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.31148049 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.536239+0000", 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7709e-05 2024-09-13T15:03:35.830 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:69669 43.8 e1044/1038 43:15aa4d50:::benchmark_data_smithi099_171278_object69668:head v 1044'4370, mlcod=1044'4366)", 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:47.225041+0000", 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 228.56342276199999, 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.31373627599999998, 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.225041+0000", 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.225041+0000", 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.225042+0000", 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.16e-07 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.225083+0000", 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1331999999999997e-05 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.225121+0000", 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8112999999999999e-05 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.225130+0000", 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4700000000000002e-06 2024-09-13T15:03:35.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.225549+0000", 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00041859999999999998 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.225579+0000", 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0233e-05 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.538729+0000", 2024-09-13T15:03:35.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.31314978399999999 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.538777+0000", 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8828000000000001e-05 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:69671 43.1 e1044/1038 43:8853cfc2:::benchmark_data_smithi099_171278_object69670:head v 1044'4237, mlcod=1044'4235)", 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:47.226064+0000", 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 228.56239974100001, 2024-09-13T15:03:35.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.302246972, 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.226064+0000", 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.226064+0000", 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.226065+0000", 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2600000000000001e-07 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.226182+0000", 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116532 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.226195+0000", 2024-09-13T15:03:35.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3217999999999999e-05 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.226203+0000", 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2430000000000005e-06 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.227078+0000", 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00087453300000000004 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.227110+0000", 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2379000000000003e-05 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528247+0000", 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.30113750299999997 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528311+0000", 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3638000000000003e-05 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.839 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:69673 43.8 e1044/1038 43:15550e8e:::benchmark_data_smithi099_171278_object69672:head v 1044'4371, mlcod=1044'4366)", 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:47.227658+0000", 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 228.56080633400001, 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.311153342, 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.227658+0000", 2024-09-13T15:03:35.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.227658+0000", 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.227659+0000", 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.11e-06 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.227738+0000", 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9270000000000005e-05 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528218+0000", 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.30047975100000002 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528244+0000", 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5925e-05 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528348+0000", 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010431500000000001 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528375+0000", 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6886999999999999e-05 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.538785+0000", 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010410278 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.538811+0000", 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5806e-05 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:69686 43.1 e1044/1038 43:82db5163:::benchmark_data_smithi099_171278_object69685:head v 1044'4238, mlcod=1044'4235)", 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T14:59:47.238820+0000", 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 228.54964364099999, 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.29302398800000001, 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.844 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.238820+0000", 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.238820+0000", 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.238823+0000", 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3120000000000001e-06 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.238898+0000", 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5578e-05 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528219+0000", 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28932091100000001 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528243+0000", 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4462999999999999e-05 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528355+0000", 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111452 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.528385+0000", 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9924999999999999e-05 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.531782+0000", 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003397579 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T14:59:47.531844+0000", 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1767999999999995e-05 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:128608 43.16 e1048/1046 43:6a1a2edb:::benchmark_data_smithi099_171278_object12596:head v 1048'6723, mlcod=1048'6721)", 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:00:45.404381+0000", 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.38408265300001, 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.38525015299999998, 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.404381+0000", 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.404381+0000", 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.404382+0000", 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8199999999999999e-07 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.404391+0000", 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2059999999999996e-06 2024-09-13T15:03:35.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.404402+0000", 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0651e-05 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.404409+0000", 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8109999999999997e-06 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.406449+0000", 2024-09-13T15:03:35.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002040666 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.406468+0000", 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8609000000000001e-05 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.408184+0000", 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00171572 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.789631+0000", 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.381447708 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:128616 43.1 e1048/1046 43:873001f8:::benchmark_data_smithi099_171278_object12604:head v 1048'6651, mlcod=1048'6648)", 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:00:45.408370+0000", 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.38009388399999, 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.38875500299999999, 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.853 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.408370+0000", 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.408370+0000", 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.408371+0000", 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3500000000000005e-07 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.408379+0000", 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0229999999999992e-06 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.789574+0000", 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.381195161 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.789595+0000", 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1115999999999999e-05 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.791079+0000", 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001484005 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.791112+0000", 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2694999999999998e-05 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.797050+0000", 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0059384399999999997 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.797125+0000", 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4727999999999993e-05 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.857 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:128615 43.8 e1048/1046 43:16ddc61b:::benchmark_data_smithi099_171278_object12603:head v 1048'6801, mlcod=1048'6798)", 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:00:45.410842+0000", 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.37762141600001, 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.38420685500000001, 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.410842+0000", 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.410842+0000", 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.410844+0000", 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.145e-06 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.410850+0000", 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6939999999999998e-06 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.789499+0000", 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.37864828699999997 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.789522+0000", 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3621999999999998e-05 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.790695+0000", 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00117275 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.790733+0000", 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8015000000000001e-05 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.794972+0000", 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004239067 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:45.795049+0000", 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7274999999999995e-05 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:132882 43.e e1048/1038 43:73aa5aa0:::benchmark_data_smithi099_171278_object16870:head v 1048'8379, mlcod=1048'8377)", 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:00:50.332035+0000", 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 165.45642842999999, 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27602213199999998, 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.862 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332035+0000", 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332035+0000", 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332037+0000", 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.517e-06 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332045+0000", 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1529999999999997e-06 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332065+0000", 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9972000000000001e-05 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332077+0000", 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1464000000000001e-05 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332159+0000", 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2871e-05 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332194+0000", 2024-09-13T15:03:35.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4122000000000003e-05 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.607953+0000", 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27575950900000001 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.608058+0000", 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104524 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:132883 43.e e1048/1038 43:7200fb4b:::benchmark_data_smithi099_171278_object16871:head v 1048'8380, mlcod=1048'8377)", 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:00:50.332316+0000", 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 165.45614764300001, 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27583972699999998, 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332316+0000", 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332316+0000", 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332317+0000", 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.063e-06 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.332324+0000", 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6000000000000003e-06 2024-09-13T15:03:35.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.590414+0000", 2024-09-13T15:03:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.25808961699999999 2024-09-13T15:03:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.590451+0000", 2024-09-13T15:03:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7512999999999998e-05 2024-09-13T15:03:35.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.598645+0000", 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0081938029999999995 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.598712+0000", 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7605000000000001e-05 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.608063+0000", 2024-09-13T15:03:35.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0093508540000000005 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:00:50.608156+0000", 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2671999999999996e-05 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:177771 43.16 e1050/1046 43:6a23da7f:::benchmark_data_smithi099_171278_object61759:head v 1050'8332, mlcod=1050'8330)", 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:01:36.180617+0000", 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 119.607847251, 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28212134500000002, 2024-09-13T15:03:35.871 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.180617+0000", 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.180617+0000", 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.180618+0000", 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1999999999999998e-07 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.180626+0000", 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7870000000000007e-06 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.180636+0000", 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0152e-05 2024-09-13T15:03:35.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.180643+0000", 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3770000000000001e-06 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.180693+0000", 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0355999999999997e-05 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.457051+0000", 2024-09-13T15:03:35.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.276357609 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.462564+0000", 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005513469 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.462738+0000", 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017367500000000001 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45572.0:177776 43.0 e1050/1046 43:0467f27f:::benchmark_data_smithi099_171278_object61764:head v 1050'8491, mlcod=1050'8489)", 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:01:36.185051+0000", 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 119.60341309, 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27775264999999999, 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.185051+0000", 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.185051+0000", 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.185052+0000", 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.47e-07 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.185060+0000", 2024-09-13T15:03:35.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3809999999999999e-06 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.456978+0000", 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27191790100000002 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.457016+0000", 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7911999999999999e-05 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.457173+0000", 2024-09-13T15:03:35.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015747999999999999 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.457228+0000", 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4725999999999997e-05 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.462739+0000", 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055110059999999997 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:01:36.462803+0000", 2024-09-13T15:03:35.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4296999999999998e-05 2024-09-13T15:03:35.880 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.880 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.880 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.880 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:03:35.880 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:03:35.880 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:03:35.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:35.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:35.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:36.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:36.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:36.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:36.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:36.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:36.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:36.253 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:35 smithi177 bash[68594]: cluster 2024-09-13T15:03:34.876868+0000 mgr.y (mgr.26252) 1123 : cluster [DBG] pgmap v1670: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:36.254 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:35 smithi177 bash[68594]: cluster 2024-09-13T15:03:34.876868+0000 mgr.y (mgr.26252) 1123 : cluster [DBG] pgmap v1670: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:36.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:36.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:36.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:36 smithi099 bash[84576]: cluster 2024-09-13T15:03:34.876868+0000 mgr.y (mgr.26252) 1123 : cluster [DBG] pgmap v1670: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:36.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:36 smithi099 bash[84576]: cluster 2024-09-13T15:03:34.876868+0000 mgr.y (mgr.26252) 1123 : cluster [DBG] pgmap v1670: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:36.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:36 smithi099 bash[87147]: cluster 2024-09-13T15:03:34.876868+0000 mgr.y (mgr.26252) 1123 : cluster [DBG] pgmap v1670: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:36.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:36 smithi099 bash[87147]: cluster 2024-09-13T15:03:34.876868+0000 mgr.y (mgr.26252) 1123 : cluster [DBG] pgmap v1670: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:36.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:36.400 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:36.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:36.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:36.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:36.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:36.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:36.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:36.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:36.759 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:36.780 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:36.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:36.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:36.910 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:36.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:37.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:37.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:37.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:37.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:37.167 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:37.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:37.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:37.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:37.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:37.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:37.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:37.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:37.541 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:03:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:03:37] "GET /metrics HTTP/1.1" 200 36699 "" "Prometheus/2.51.0" 2024-09-13T15:03:37.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:37.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:37.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:37.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:37.792 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:37.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:37.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:37.921 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:37.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:38.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:38.070 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:38.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:38.195 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:38.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:38 smithi099 bash[87147]: cluster 2024-09-13T15:03:36.878030+0000 mgr.y (mgr.26252) 1124 : cluster [DBG] pgmap v1671: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:38.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:38 smithi099 bash[87147]: cluster 2024-09-13T15:03:36.878030+0000 mgr.y (mgr.26252) 1124 : cluster [DBG] pgmap v1671: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:38.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:38 smithi099 bash[87147]: audit 2024-09-13T15:03:37.552787+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:38.196 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:38 smithi099 bash[87147]: audit 2024-09-13T15:03:37.552787+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:38.196 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:37 smithi099 bash[84576]: cluster 2024-09-13T15:03:36.878030+0000 mgr.y (mgr.26252) 1124 : cluster [DBG] pgmap v1671: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:38.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:37 smithi099 bash[84576]: cluster 2024-09-13T15:03:36.878030+0000 mgr.y (mgr.26252) 1124 : cluster [DBG] pgmap v1671: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:38.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:37 smithi099 bash[84576]: audit 2024-09-13T15:03:37.552787+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:38.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:37 smithi099 bash[84576]: audit 2024-09-13T15:03:37.552787+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:38.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:38.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:38.308 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:37 smithi177 bash[68594]: cluster 2024-09-13T15:03:36.878030+0000 mgr.y (mgr.26252) 1124 : cluster [DBG] pgmap v1671: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:38.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:37 smithi177 bash[68594]: cluster 2024-09-13T15:03:36.878030+0000 mgr.y (mgr.26252) 1124 : cluster [DBG] pgmap v1671: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:38.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:37 smithi177 bash[68594]: audit 2024-09-13T15:03:37.552787+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:38.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:37 smithi177 bash[68594]: audit 2024-09-13T15:03:37.552787+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:38.322 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:38.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:38.425 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:38.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:38.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:38.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:38.578 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:38.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:38.702 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:38.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:38.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:38.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:38.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:38.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:38.959 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:38.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:39.062 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:39.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:39.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:39.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T15:03:39.191 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:39.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:39.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:39.325 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:39.336 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:39 smithi177 bash[68594]: audit 2024-09-13T15:03:38.137937+0000 mon.a (mon.0) 2242 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.336 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:39 smithi177 bash[68594]: audit 2024-09-13T15:03:38.137937+0000 mon.a (mon.0) 2242 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.336 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:39 smithi177 bash[68594]: audit 2024-09-13T15:03:38.149705+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.336 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:39 smithi177 bash[68594]: audit 2024-09-13T15:03:38.149705+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:39 smithi177 bash[68594]: cluster 2024-09-13T15:03:38.879734+0000 mgr.y (mgr.26252) 1125 : cluster [DBG] pgmap v1672: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:39.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:39 smithi177 bash[68594]: cluster 2024-09-13T15:03:38.879734+0000 mgr.y (mgr.26252) 1125 : cluster [DBG] pgmap v1672: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:39.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:39.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:39.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[84576]: audit 2024-09-13T15:03:38.137937+0000 mon.a (mon.0) 2242 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[84576]: audit 2024-09-13T15:03:38.137937+0000 mon.a (mon.0) 2242 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[84576]: audit 2024-09-13T15:03:38.149705+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[84576]: audit 2024-09-13T15:03:38.149705+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.357 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[84576]: cluster 2024-09-13T15:03:38.879734+0000 mgr.y (mgr.26252) 1125 : cluster [DBG] pgmap v1672: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:39.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[84576]: cluster 2024-09-13T15:03:38.879734+0000 mgr.y (mgr.26252) 1125 : cluster [DBG] pgmap v1672: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:39.358 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[87147]: audit 2024-09-13T15:03:38.137937+0000 mon.a (mon.0) 2242 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.358 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[87147]: audit 2024-09-13T15:03:38.137937+0000 mon.a (mon.0) 2242 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.358 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[87147]: audit 2024-09-13T15:03:38.149705+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.358 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[87147]: audit 2024-09-13T15:03:38.149705+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:39.358 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[87147]: cluster 2024-09-13T15:03:38.879734+0000 mgr.y (mgr.26252) 1125 : cluster [DBG] pgmap v1672: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:39.358 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:39 smithi099 bash[87147]: cluster 2024-09-13T15:03:38.879734+0000 mgr.y (mgr.26252) 1125 : cluster [DBG] pgmap v1672: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:39.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:39.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:39.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:39.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:39.607 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:39.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:39.736 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:39.839 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:39.868 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:39.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:39.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:39.998 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:40.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:40.101 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:40.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:40.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:40.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:40.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:40.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:40.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:40.386 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:40.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:40.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:40.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:40.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:03:40 smithi177 bash[61439]: ts=2024-09-13T15:03:40.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:40.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:40.766 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:40.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:40.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:40.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:40.970 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:03:40.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:41.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:41.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:41.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:40 smithi099 bash[84576]: audit 2024-09-13T15:03:40.918556+0000 mon.a (mon.0) 2244 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:41.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:40 smithi099 bash[84576]: audit 2024-09-13T15:03:40.918556+0000 mon.a (mon.0) 2244 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:41.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:40 smithi099 bash[87147]: audit 2024-09-13T15:03:40.918556+0000 mon.a (mon.0) 2244 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:41.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:40 smithi099 bash[87147]: audit 2024-09-13T15:03:40.918556+0000 mon.a (mon.0) 2244 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:41.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:41.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:41.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:41.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:41.254 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:40 smithi177 bash[68594]: audit 2024-09-13T15:03:40.918556+0000 mon.a (mon.0) 2244 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:41.254 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:40 smithi177 bash[68594]: audit 2024-09-13T15:03:40.918556+0000 mon.a (mon.0) 2244 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:41.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:41.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:41.369 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:41.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:41.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:41.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:41.524 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:41.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:41.654 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:41.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:41.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:41.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:41.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:41.901 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:41.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:41.911 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:03:41.911 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:03:41.928 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:03:41.928 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:03:41.945 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:03:41.945 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:03:41.947 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:03:41.966 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:03:41.966 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:03:41.985 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:03:41.986 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:03:42.006 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:03:42.006 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:03:42.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:42.026 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:03:42.026 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:03:42.034 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:42.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:42.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:41 smithi099 bash[84576]: cluster 2024-09-13T15:03:40.881666+0000 mgr.y (mgr.26252) 1126 : cluster [DBG] pgmap v1673: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:42.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:41 smithi099 bash[84576]: cluster 2024-09-13T15:03:40.881666+0000 mgr.y (mgr.26252) 1126 : cluster [DBG] pgmap v1673: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:42.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:41 smithi099 bash[84576]: cluster 2024-09-13T15:03:40.969116+0000 mon.a (mon.0) 2245 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:42.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:42.143 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[87147]: cluster 2024-09-13T15:03:40.881666+0000 mgr.y (mgr.26252) 1126 : cluster [DBG] pgmap v1673: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:42.143 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[87147]: cluster 2024-09-13T15:03:40.881666+0000 mgr.y (mgr.26252) 1126 : cluster [DBG] pgmap v1673: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:42.143 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[87147]: cluster 2024-09-13T15:03:40.969116+0000 mon.a (mon.0) 2245 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:42.143 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[87147]: cluster 2024-09-13T15:03:40.969116+0000 mon.a (mon.0) 2245 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:42.143 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[87147]: audit 2024-09-13T15:03:40.971067+0000 mon.a (mon.0) 2246 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:42.144 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[87147]: audit 2024-09-13T15:03:40.971067+0000 mon.a (mon.0) 2246 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:42.144 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[87147]: cluster 2024-09-13T15:03:40.975962+0000 mon.a (mon.0) 2247 : cluster [DBG] osdmap e1059: 8 total, 7 up, 8 in 2024-09-13T15:03:42.144 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[87147]: cluster 2024-09-13T15:03:40.975962+0000 mon.a (mon.0) 2247 : cluster [DBG] osdmap e1059: 8 total, 7 up, 8 in 2024-09-13T15:03:42.144 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[87147]: audit 2024-09-13T15:03:41.892769+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.99:0/1923849744' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:42.144 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[87147]: audit 2024-09-13T15:03:41.892769+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.99:0/1923849744' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:42.161 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:42.264 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:42.276 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:41 smithi177 bash[68594]: cluster 2024-09-13T15:03:40.881666+0000 mgr.y (mgr.26252) 1126 : cluster [DBG] pgmap v1673: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:42.276 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:41 smithi177 bash[68594]: cluster 2024-09-13T15:03:40.881666+0000 mgr.y (mgr.26252) 1126 : cluster [DBG] pgmap v1673: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:42.276 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:41 smithi177 bash[68594]: cluster 2024-09-13T15:03:40.969116+0000 mon.a (mon.0) 2245 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:42.276 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:41 smithi177 bash[68594]: cluster 2024-09-13T15:03:40.969116+0000 mon.a (mon.0) 2245 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:42.276 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:41 smithi177 bash[68594]: audit 2024-09-13T15:03:40.971067+0000 mon.a (mon.0) 2246 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:42.276 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:41 smithi177 bash[68594]: audit 2024-09-13T15:03:40.971067+0000 mon.a (mon.0) 2246 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:41 smithi177 bash[68594]: cluster 2024-09-13T15:03:40.975962+0000 mon.a (mon.0) 2247 : cluster [DBG] osdmap e1059: 8 total, 7 up, 8 in 2024-09-13T15:03:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:41 smithi177 bash[68594]: cluster 2024-09-13T15:03:40.975962+0000 mon.a (mon.0) 2247 : cluster [DBG] osdmap e1059: 8 total, 7 up, 8 in 2024-09-13T15:03:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:41 smithi177 bash[68594]: audit 2024-09-13T15:03:41.892769+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.99:0/1923849744' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:41 smithi177 bash[68594]: audit 2024-09-13T15:03:41.892769+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.99:0/1923849744' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:42.291 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:42.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:42.406 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[84576]: cluster 2024-09-13T15:03:40.969116+0000 mon.a (mon.0) 2245 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:42.406 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[84576]: audit 2024-09-13T15:03:40.971067+0000 mon.a (mon.0) 2246 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:42.406 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[84576]: audit 2024-09-13T15:03:40.971067+0000 mon.a (mon.0) 2246 : audit [INF] from='client.? 172.21.15.99:0/681859975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:42.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[84576]: cluster 2024-09-13T15:03:40.975962+0000 mon.a (mon.0) 2247 : cluster [DBG] osdmap e1059: 8 total, 7 up, 8 in 2024-09-13T15:03:42.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[84576]: cluster 2024-09-13T15:03:40.975962+0000 mon.a (mon.0) 2247 : cluster [DBG] osdmap e1059: 8 total, 7 up, 8 in 2024-09-13T15:03:42.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[84576]: audit 2024-09-13T15:03:41.892769+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.99:0/1923849744' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:42.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:42 smithi099 bash[84576]: audit 2024-09-13T15:03:41.892769+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.99:0/1923849744' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:42.422 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:42.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:42.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:42.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:42.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:42.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:42.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:42.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:42.787 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:42.814 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:42.917 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:42.945 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:42.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:43.048 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:43.075 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:43.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:43.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:43.204 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:43.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:43.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:43.327 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:43.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:43.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:43.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:43.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:43.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:43.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:43.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:43.676 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:43.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:43.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:43.836 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:43.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:43.939 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:43.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:43.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:44.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:44.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:43 smithi099 bash[84576]: cluster 2024-09-13T15:03:42.882437+0000 mgr.y (mgr.26252) 1127 : cluster [DBG] pgmap v1675: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:44.072 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:43 smithi099 bash[84576]: cluster 2024-09-13T15:03:42.882437+0000 mgr.y (mgr.26252) 1127 : cluster [DBG] pgmap v1675: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:44.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:43 smithi099 bash[87147]: cluster 2024-09-13T15:03:42.882437+0000 mgr.y (mgr.26252) 1127 : cluster [DBG] pgmap v1675: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:44.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:43 smithi099 bash[87147]: cluster 2024-09-13T15:03:42.882437+0000 mgr.y (mgr.26252) 1127 : cluster [DBG] pgmap v1675: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:44.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:44.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:44.191 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:44.219 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:44.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:44.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:44.331 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:43 smithi177 bash[68594]: cluster 2024-09-13T15:03:42.882437+0000 mgr.y (mgr.26252) 1127 : cluster [DBG] pgmap v1675: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:44.332 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:43 smithi177 bash[68594]: cluster 2024-09-13T15:03:42.882437+0000 mgr.y (mgr.26252) 1127 : cluster [DBG] pgmap v1675: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:44.346 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:44.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:44.449 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:44.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:44.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:44.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.645 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:44.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:44.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:44.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:44.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:44.799 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:03:44.800 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:03:44.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:44.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:44.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:45.003 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:45.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:45.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:45.159 INFO:teuthology.orchestra.run.smithi099.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-09-13T15:03:45.254 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:45.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[84576]: audit 2024-09-13T15:03:44.148090+0000 mon.a (mon.0) 2248 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[84576]: audit 2024-09-13T15:03:44.148090+0000 mon.a (mon.0) 2248 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[84576]: audit 2024-09-13T15:03:44.154072+0000 mon.a (mon.0) 2249 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[84576]: audit 2024-09-13T15:03:44.154072+0000 mon.a (mon.0) 2249 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[84576]: cluster 2024-09-13T15:03:44.883124+0000 mgr.y (mgr.26252) 1128 : cluster [DBG] pgmap v1676: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:45.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[84576]: cluster 2024-09-13T15:03:44.883124+0000 mgr.y (mgr.26252) 1128 : cluster [DBG] pgmap v1676: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:45.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[87147]: audit 2024-09-13T15:03:44.148090+0000 mon.a (mon.0) 2248 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[87147]: audit 2024-09-13T15:03:44.148090+0000 mon.a (mon.0) 2248 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[87147]: audit 2024-09-13T15:03:44.154072+0000 mon.a (mon.0) 2249 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[87147]: audit 2024-09-13T15:03:44.154072+0000 mon.a (mon.0) 2249 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[87147]: cluster 2024-09-13T15:03:44.883124+0000 mgr.y (mgr.26252) 1128 : cluster [DBG] pgmap v1676: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:45.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:45 smithi099 bash[87147]: cluster 2024-09-13T15:03:44.883124+0000 mgr.y (mgr.26252) 1128 : cluster [DBG] pgmap v1676: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:45.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:45.401 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:45.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:45.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:45.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:45 smithi177 bash[68594]: audit 2024-09-13T15:03:44.148090+0000 mon.a (mon.0) 2248 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:45 smithi177 bash[68594]: audit 2024-09-13T15:03:44.148090+0000 mon.a (mon.0) 2248 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:45 smithi177 bash[68594]: audit 2024-09-13T15:03:44.154072+0000 mon.a (mon.0) 2249 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:45 smithi177 bash[68594]: audit 2024-09-13T15:03:44.154072+0000 mon.a (mon.0) 2249 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:45.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:45 smithi177 bash[68594]: cluster 2024-09-13T15:03:44.883124+0000 mgr.y (mgr.26252) 1128 : cluster [DBG] pgmap v1676: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:45.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:45 smithi177 bash[68594]: cluster 2024-09-13T15:03:44.883124+0000 mgr.y (mgr.26252) 1128 : cluster [DBG] pgmap v1676: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:45.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:45.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:45.633 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:45.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:45.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:45.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:45.884 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:46.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:46.034 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:46.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:46.136 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:46.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:46.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:46.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[84576]: audit 2024-09-13T15:03:45.146656+0000 mon.a (mon.0) 2250 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:03:46.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[84576]: audit 2024-09-13T15:03:45.146656+0000 mon.a (mon.0) 2250 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:03:46.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[84576]: audit 2024-09-13T15:03:45.160127+0000 mon.a (mon.0) 2251 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:03:46.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[84576]: audit 2024-09-13T15:03:45.160127+0000 mon.a (mon.0) 2251 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:03:46.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[84576]: cluster 2024-09-13T15:03:45.164109+0000 mon.a (mon.0) 2252 : cluster [DBG] osdmap e1060: 8 total, 7 up, 8 in 2024-09-13T15:03:46.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[84576]: cluster 2024-09-13T15:03:45.164109+0000 mon.a (mon.0) 2252 : cluster [DBG] osdmap e1060: 8 total, 7 up, 8 in 2024-09-13T15:03:46.287 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[84576]: audit 2024-09-13T15:03:45.936767+0000 mon.a (mon.0) 2253 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.288 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[84576]: audit 2024-09-13T15:03:45.936767+0000 mon.a (mon.0) 2253 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.288 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[84576]: audit 2024-09-13T15:03:45.941603+0000 mon.a (mon.0) 2254 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.288 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[84576]: audit 2024-09-13T15:03:45.941603+0000 mon.a (mon.0) 2254 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[87147]: audit 2024-09-13T15:03:45.146656+0000 mon.a (mon.0) 2250 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:03:46.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[87147]: audit 2024-09-13T15:03:45.146656+0000 mon.a (mon.0) 2250 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:03:46.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[87147]: audit 2024-09-13T15:03:45.160127+0000 mon.a (mon.0) 2251 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:03:46.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[87147]: audit 2024-09-13T15:03:45.160127+0000 mon.a (mon.0) 2251 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:03:46.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[87147]: cluster 2024-09-13T15:03:45.164109+0000 mon.a (mon.0) 2252 : cluster [DBG] osdmap e1060: 8 total, 7 up, 8 in 2024-09-13T15:03:46.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[87147]: cluster 2024-09-13T15:03:45.164109+0000 mon.a (mon.0) 2252 : cluster [DBG] osdmap e1060: 8 total, 7 up, 8 in 2024-09-13T15:03:46.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[87147]: audit 2024-09-13T15:03:45.936767+0000 mon.a (mon.0) 2253 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.289 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[87147]: audit 2024-09-13T15:03:45.936767+0000 mon.a (mon.0) 2253 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.289 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[87147]: audit 2024-09-13T15:03:45.941603+0000 mon.a (mon.0) 2254 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.289 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:46 smithi099 bash[87147]: audit 2024-09-13T15:03:45.941603+0000 mon.a (mon.0) 2254 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.290 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:46.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:46.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:46.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:46 smithi177 bash[68594]: audit 2024-09-13T15:03:45.146656+0000 mon.a (mon.0) 2250 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:03:46.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:46 smithi177 bash[68594]: audit 2024-09-13T15:03:45.146656+0000 mon.a (mon.0) 2250 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:03:46.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:46 smithi177 bash[68594]: audit 2024-09-13T15:03:45.160127+0000 mon.a (mon.0) 2251 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:03:46.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:46 smithi177 bash[68594]: audit 2024-09-13T15:03:45.160127+0000 mon.a (mon.0) 2251 : audit [INF] from='client.? 172.21.15.99:0/203643589' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:03:46.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:46 smithi177 bash[68594]: cluster 2024-09-13T15:03:45.164109+0000 mon.a (mon.0) 2252 : cluster [DBG] osdmap e1060: 8 total, 7 up, 8 in 2024-09-13T15:03:46.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:46 smithi177 bash[68594]: cluster 2024-09-13T15:03:45.164109+0000 mon.a (mon.0) 2252 : cluster [DBG] osdmap e1060: 8 total, 7 up, 8 in 2024-09-13T15:03:46.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:46 smithi177 bash[68594]: audit 2024-09-13T15:03:45.936767+0000 mon.a (mon.0) 2253 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:46 smithi177 bash[68594]: audit 2024-09-13T15:03:45.936767+0000 mon.a (mon.0) 2253 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:46 smithi177 bash[68594]: audit 2024-09-13T15:03:45.941603+0000 mon.a (mon.0) 2254 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.403 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:46 smithi177 bash[68594]: audit 2024-09-13T15:03:45.941603+0000 mon.a (mon.0) 2254 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:46.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:46.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:46.452 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:03:46.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:46.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:46.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:46.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:46.663 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:46.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:46.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:46.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:46.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:46.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:03:46.895 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:46.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:46.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:47.026 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:47.055 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:47.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:47.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:03:47] "GET /metrics HTTP/1.1" 200 36699 "" "Prometheus/2.51.0" 2024-09-13T15:03:47.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:47.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:47.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:47.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:47.290 DEBUG:teuthology.orchestra.run.smithi099:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-09-13T15:03:47.291 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T15:03:47.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:03:47.293 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T15:03:47.293 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T15:03:47.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[87147]: audit 2024-09-13T15:03:46.551363+0000 mon.a (mon.0) 2255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:03:47.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[87147]: audit 2024-09-13T15:03:46.551363+0000 mon.a (mon.0) 2255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:03:47.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[87147]: audit 2024-09-13T15:03:46.552139+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:03:47.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[87147]: audit 2024-09-13T15:03:46.552139+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:03:47.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[87147]: audit 2024-09-13T15:03:46.557500+0000 mon.a (mon.0) 2257 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:47.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[87147]: audit 2024-09-13T15:03:46.557500+0000 mon.a (mon.0) 2257 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:47.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[87147]: cluster 2024-09-13T15:03:46.884129+0000 mgr.y (mgr.26252) 1129 : cluster [DBG] pgmap v1678: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:47.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[87147]: cluster 2024-09-13T15:03:46.884129+0000 mgr.y (mgr.26252) 1129 : cluster [DBG] pgmap v1678: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:47.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[84576]: audit 2024-09-13T15:03:46.551363+0000 mon.a (mon.0) 2255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:03:47.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[84576]: audit 2024-09-13T15:03:46.551363+0000 mon.a (mon.0) 2255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:03:47.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[84576]: audit 2024-09-13T15:03:46.552139+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:03:47.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[84576]: audit 2024-09-13T15:03:46.552139+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:03:47.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[84576]: audit 2024-09-13T15:03:46.557500+0000 mon.a (mon.0) 2257 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:47.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[84576]: audit 2024-09-13T15:03:46.557500+0000 mon.a (mon.0) 2257 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:47.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[84576]: cluster 2024-09-13T15:03:46.884129+0000 mgr.y (mgr.26252) 1129 : cluster [DBG] pgmap v1678: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:47.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:47 smithi099 bash[84576]: cluster 2024-09-13T15:03:46.884129+0000 mgr.y (mgr.26252) 1129 : cluster [DBG] pgmap v1678: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:47.311 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:47.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:47 smithi177 bash[68594]: audit 2024-09-13T15:03:46.551363+0000 mon.a (mon.0) 2255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:03:47.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:47 smithi177 bash[68594]: audit 2024-09-13T15:03:46.551363+0000 mon.a (mon.0) 2255 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:03:47.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:47 smithi177 bash[68594]: audit 2024-09-13T15:03:46.552139+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:03:47.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:47 smithi177 bash[68594]: audit 2024-09-13T15:03:46.552139+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:03:47.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:47 smithi177 bash[68594]: audit 2024-09-13T15:03:46.557500+0000 mon.a (mon.0) 2257 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:47.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:47 smithi177 bash[68594]: audit 2024-09-13T15:03:46.557500+0000 mon.a (mon.0) 2257 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:47.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:47 smithi177 bash[68594]: cluster 2024-09-13T15:03:46.884129+0000 mgr.y (mgr.26252) 1129 : cluster [DBG] pgmap v1678: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:47.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:47 smithi177 bash[68594]: cluster 2024-09-13T15:03:46.884129+0000 mgr.y (mgr.26252) 1129 : cluster [DBG] pgmap v1678: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T15:03:47.363 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:hints = 1 2024-09-13T15:03:47.363 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T15:03:47.363 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object prefix: benchmark_data_smithi099_188632 2024-09-13T15:03:47.363 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:03:47.363 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 0 0 0 0 0 0 - 0 2024-09-13T15:03:47.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:47.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:47.456 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:47.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:47.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:47.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:47.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:47.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:47.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:47.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:47.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:47.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:47.942 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:47.968 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:47.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:48.070 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:48.096 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:48.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:48.199 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:48.229 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:48 smithi099 bash[87147]: cluster 2024-09-13T15:03:47.555555+0000 mon.a (mon.0) 2258 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:03:48.229 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:48 smithi099 bash[87147]: cluster 2024-09-13T15:03:47.555555+0000 mon.a (mon.0) 2258 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:03:48.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:48.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:48.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:48.343 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:48 smithi177 bash[68594]: cluster 2024-09-13T15:03:47.555555+0000 mon.a (mon.0) 2258 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:03:48.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:48 smithi177 bash[68594]: cluster 2024-09-13T15:03:47.555555+0000 mon.a (mon.0) 2258 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:03:48.355 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:48.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:48.364 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 1 16 1899 1883 117.675 117.688 0.00167301 0.00797289 2024-09-13T15:03:48.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:48.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:48.500 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:48 smithi099 bash[84576]: cluster 2024-09-13T15:03:47.555555+0000 mon.a (mon.0) 2258 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:03:48.501 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:48 smithi099 bash[84576]: cluster 2024-09-13T15:03:47.555555+0000 mon.a (mon.0) 2258 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:03:48.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:48.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:48.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:48.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:48.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:48.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:48.866 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:48.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:48.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:48.999 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:49.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.029 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:49.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:49.130 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:49.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:49.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:49.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:49.282 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:49.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:49.364 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 2 16 5042 5026 157.046 196.438 0.00486009 0.00635628 2024-09-13T15:03:49.385 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:49.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:49.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:49.515 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:49.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:49.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:49.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[84576]: audit 2024-09-13T15:03:48.301307+0000 mon.a (mon.0) 2259 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[84576]: audit 2024-09-13T15:03:48.301307+0000 mon.a (mon.0) 2259 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[84576]: audit 2024-09-13T15:03:48.302499+0000 mon.a (mon.0) 2260 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[84576]: audit 2024-09-13T15:03:48.302499+0000 mon.a (mon.0) 2260 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[84576]: cluster 2024-09-13T15:03:48.885012+0000 mgr.y (mgr.26252) 1130 : cluster [DBG] pgmap v1679: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 15 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s wr, 27 op/s; 4/750 objects degraded (0.533%) 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[84576]: cluster 2024-09-13T15:03:48.885012+0000 mgr.y (mgr.26252) 1130 : cluster [DBG] pgmap v1679: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 15 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s wr, 27 op/s; 4/750 objects degraded (0.533%) 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[87147]: audit 2024-09-13T15:03:48.301307+0000 mon.a (mon.0) 2259 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[87147]: audit 2024-09-13T15:03:48.301307+0000 mon.a (mon.0) 2259 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[87147]: audit 2024-09-13T15:03:48.302499+0000 mon.a (mon.0) 2260 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[87147]: audit 2024-09-13T15:03:48.302499+0000 mon.a (mon.0) 2260 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:49.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[87147]: cluster 2024-09-13T15:03:48.885012+0000 mgr.y (mgr.26252) 1130 : cluster [DBG] pgmap v1679: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 15 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s wr, 27 op/s; 4/750 objects degraded (0.533%) 2024-09-13T15:03:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:49 smithi099 bash[87147]: cluster 2024-09-13T15:03:48.885012+0000 mgr.y (mgr.26252) 1130 : cluster [DBG] pgmap v1679: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 15 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s wr, 27 op/s; 4/750 objects degraded (0.533%) 2024-09-13T15:03:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:49 smithi177 bash[68594]: audit 2024-09-13T15:03:48.301307+0000 mon.a (mon.0) 2259 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:49.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:49 smithi177 bash[68594]: audit 2024-09-13T15:03:48.301307+0000 mon.a (mon.0) 2259 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:03:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:49 smithi177 bash[68594]: audit 2024-09-13T15:03:48.302499+0000 mon.a (mon.0) 2260 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:49 smithi177 bash[68594]: audit 2024-09-13T15:03:48.302499+0000 mon.a (mon.0) 2260 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:49 smithi177 bash[68594]: cluster 2024-09-13T15:03:48.885012+0000 mgr.y (mgr.26252) 1130 : cluster [DBG] pgmap v1679: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 15 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s wr, 27 op/s; 4/750 objects degraded (0.533%) 2024-09-13T15:03:49.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:49 smithi177 bash[68594]: cluster 2024-09-13T15:03:48.885012+0000 mgr.y (mgr.26252) 1130 : cluster [DBG] pgmap v1679: 89 pgs: 30 active+undersized, 3 active+undersized+degraded, 56 active+clean; 15 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s wr, 27 op/s; 4/750 objects degraded (0.533%) 2024-09-13T15:03:49.653 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:49.685 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:49.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:49.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:49.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:49.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:49.921 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:49.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:50.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:50.082 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:50.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:50.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:50.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:50.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:50.309 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:03:50 smithi177 bash[61439]: ts=2024-09-13T15:03:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:03:50.317 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:50.344 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:50.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:50.364 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 3 16 8076 8060 167.9 189.625 0.00509526 0.00594606 2024-09-13T15:03:50.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:50.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:50.479 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:50 smithi177 bash[68594]: cluster 2024-09-13T15:03:49.298459+0000 mon.a (mon.0) 2261 : cluster [WRN] Health check update: Degraded data redundancy: 4/750 objects degraded (0.533%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:50.479 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:50 smithi177 bash[68594]: cluster 2024-09-13T15:03:49.298459+0000 mon.a (mon.0) 2261 : cluster [WRN] Health check update: Degraded data redundancy: 4/750 objects degraded (0.533%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:50.511 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:50.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:50.596 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:50.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:50.650 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:50 smithi099 bash[84576]: cluster 2024-09-13T15:03:49.298459+0000 mon.a (mon.0) 2261 : cluster [WRN] Health check update: Degraded data redundancy: 4/750 objects degraded (0.533%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:50.650 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:50 smithi099 bash[84576]: cluster 2024-09-13T15:03:49.298459+0000 mon.a (mon.0) 2261 : cluster [WRN] Health check update: Degraded data redundancy: 4/750 objects degraded (0.533%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:50.650 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:50 smithi099 bash[87147]: cluster 2024-09-13T15:03:49.298459+0000 mon.a (mon.0) 2261 : cluster [WRN] Health check update: Degraded data redundancy: 4/750 objects degraded (0.533%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:50.650 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:50 smithi099 bash[87147]: cluster 2024-09-13T15:03:49.298459+0000 mon.a (mon.0) 2261 : cluster [WRN] Health check update: Degraded data redundancy: 4/750 objects degraded (0.533%), 3 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:50.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:50.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:50.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:50.823 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:50.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:50.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:50.951 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:50.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:51.080 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:51.103 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:51.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:51.207 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:51.224 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:51.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:51.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:51.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:51.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:51.364 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 4 16 10498 10482 163.766 151.375 0.00984907 0.00609852 2024-09-13T15:03:51.463 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:51.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:51.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:51.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:51 smithi099 bash[84576]: cluster 2024-09-13T15:03:50.885977+0000 mgr.y (mgr.26252) 1131 : cluster [DBG] pgmap v1680: 89 pgs: 27 active+undersized, 6 active+undersized+degraded, 56 active+clean; 201 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 323 op/s; 871/9696 objects degraded (8.983%) 2024-09-13T15:03:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:51 smithi099 bash[84576]: cluster 2024-09-13T15:03:50.885977+0000 mgr.y (mgr.26252) 1131 : cluster [DBG] pgmap v1680: 89 pgs: 27 active+undersized, 6 active+undersized+degraded, 56 active+clean; 201 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 323 op/s; 871/9696 objects degraded (8.983%) 2024-09-13T15:03:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:51 smithi099 bash[87147]: cluster 2024-09-13T15:03:50.885977+0000 mgr.y (mgr.26252) 1131 : cluster [DBG] pgmap v1680: 89 pgs: 27 active+undersized, 6 active+undersized+degraded, 56 active+clean; 201 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 323 op/s; 871/9696 objects degraded (8.983%) 2024-09-13T15:03:51.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:51 smithi099 bash[87147]: cluster 2024-09-13T15:03:50.885977+0000 mgr.y (mgr.26252) 1131 : cluster [DBG] pgmap v1680: 89 pgs: 27 active+undersized, 6 active+undersized+degraded, 56 active+clean; 201 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 323 op/s; 871/9696 objects degraded (8.983%) 2024-09-13T15:03:51.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:51 smithi177 bash[68594]: cluster 2024-09-13T15:03:50.885977+0000 mgr.y (mgr.26252) 1131 : cluster [DBG] pgmap v1680: 89 pgs: 27 active+undersized, 6 active+undersized+degraded, 56 active+clean; 201 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 323 op/s; 871/9696 objects degraded (8.983%) 2024-09-13T15:03:51.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:51 smithi177 bash[68594]: cluster 2024-09-13T15:03:50.885977+0000 mgr.y (mgr.26252) 1131 : cluster [DBG] pgmap v1680: 89 pgs: 27 active+undersized, 6 active+undersized+degraded, 56 active+clean; 201 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 323 op/s; 871/9696 objects degraded (8.983%) 2024-09-13T15:03:51.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:51.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:51.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:51.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:51.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:51.885 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:51.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:51.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:52.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:52.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:52.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:52.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:52.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:52.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:52.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:52.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:52.364 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 5 16 13243 13227 165.322 171.562 0.00396591 0.00604206 2024-09-13T15:03:52.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:52.432 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:52.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:52.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:52.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:52.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:52.659 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:52.698 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:52.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:52.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:52.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:52.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:52.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:52.963 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:53.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:53.097 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:53.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:53.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:53.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:53.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:53.330 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:53.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:53.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:53.364 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 6 16 15849 15833 164.912 162.875 0.00303074 0.00605549 2024-09-13T15:03:53.460 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:53.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:53.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:53.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:53.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:53.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:53.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:53.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:53.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:53.895 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:53.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:54.000 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:54.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:54.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:54.045 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:54.045 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:54.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:54.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:54.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:54.185 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:53 smithi177 bash[68594]: cluster 2024-09-13T15:03:52.886787+0000 mgr.y (mgr.26252) 1132 : cluster [DBG] pgmap v1681: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 449 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 1655/21579 objects degraded (7.669%) 2024-09-13T15:03:54.185 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:53 smithi177 bash[68594]: cluster 2024-09-13T15:03:52.886787+0000 mgr.y (mgr.26252) 1132 : cluster [DBG] pgmap v1681: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 449 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 1655/21579 objects degraded (7.669%) 2024-09-13T15:03:54.254 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:54.276 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:54.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:53 smithi099 bash[84576]: cluster 2024-09-13T15:03:52.886787+0000 mgr.y (mgr.26252) 1132 : cluster [DBG] pgmap v1681: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 449 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 1655/21579 objects degraded (7.669%) 2024-09-13T15:03:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:53 smithi099 bash[84576]: cluster 2024-09-13T15:03:52.886787+0000 mgr.y (mgr.26252) 1132 : cluster [DBG] pgmap v1681: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 449 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 1655/21579 objects degraded (7.669%) 2024-09-13T15:03:54.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:53 smithi099 bash[87147]: cluster 2024-09-13T15:03:52.886787+0000 mgr.y (mgr.26252) 1132 : cluster [DBG] pgmap v1681: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 449 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 1655/21579 objects degraded (7.669%) 2024-09-13T15:03:54.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:53 smithi099 bash[87147]: cluster 2024-09-13T15:03:52.886787+0000 mgr.y (mgr.26252) 1132 : cluster [DBG] pgmap v1681: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 449 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 1655/21579 objects degraded (7.669%) 2024-09-13T15:03:54.364 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 7 16 17471 17455 155.833 101.375 0.0035601 0.00627369 2024-09-13T15:03:54.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:54.406 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:54.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:54.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:54.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:54.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:54.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:54.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:54.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:54.765 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:54.789 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:54.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:54.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:54.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:54.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:55.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:55.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:54 smithi177 bash[68594]: cluster 2024-09-13T15:03:54.941741+0000 mon.a (mon.0) 2262 : cluster [WRN] Health check update: Degraded data redundancy: 4059/47928 objects degraded (8.469%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:55.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:54 smithi177 bash[68594]: cluster 2024-09-13T15:03:54.941741+0000 mon.a (mon.0) 2262 : cluster [WRN] Health check update: Degraded data redundancy: 4059/47928 objects degraded (8.469%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:55.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:55.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:55.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:54 smithi099 bash[84576]: cluster 2024-09-13T15:03:54.941741+0000 mon.a (mon.0) 2262 : cluster [WRN] Health check update: Degraded data redundancy: 4059/47928 objects degraded (8.469%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:55.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:54 smithi099 bash[84576]: cluster 2024-09-13T15:03:54.941741+0000 mon.a (mon.0) 2262 : cluster [WRN] Health check update: Degraded data redundancy: 4059/47928 objects degraded (8.469%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:55.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:54 smithi099 bash[87147]: cluster 2024-09-13T15:03:54.941741+0000 mon.a (mon.0) 2262 : cluster [WRN] Health check update: Degraded data redundancy: 4059/47928 objects degraded (8.469%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:55.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:54 smithi099 bash[87147]: cluster 2024-09-13T15:03:54.941741+0000 mon.a (mon.0) 2262 : cluster [WRN] Health check update: Degraded data redundancy: 4059/47928 objects degraded (8.469%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:03:55.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:55.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:55.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:55.304 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:55.332 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:55.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:55.364 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 8 16 18847 18831 147.102 86 0.00507444 0.00679171 2024-09-13T15:03:55.439 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:55.469 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:55.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:55.575 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:55.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:55.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:55.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:55.745 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:55.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:55.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:55.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:55.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:55.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:55.996 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:55.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:56.099 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:56.132 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:56.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:56.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:56.247 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:55 smithi177 bash[68594]: cluster 2024-09-13T15:03:54.888554+0000 mgr.y (mgr.26252) 1133 : cluster [DBG] pgmap v1682: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:56.247 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:55 smithi177 bash[68594]: cluster 2024-09-13T15:03:54.888554+0000 mgr.y (mgr.26252) 1133 : cluster [DBG] pgmap v1682: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:56.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:56 smithi099 bash[84576]: cluster 2024-09-13T15:03:54.888554+0000 mgr.y (mgr.26252) 1133 : cluster [DBG] pgmap v1682: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:56 smithi099 bash[84576]: cluster 2024-09-13T15:03:54.888554+0000 mgr.y (mgr.26252) 1133 : cluster [DBG] pgmap v1682: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:56.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:56 smithi099 bash[87147]: cluster 2024-09-13T15:03:54.888554+0000 mgr.y (mgr.26252) 1133 : cluster [DBG] pgmap v1682: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:56.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:56 smithi099 bash[87147]: cluster 2024-09-13T15:03:54.888554+0000 mgr.y (mgr.26252) 1133 : cluster [DBG] pgmap v1682: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:56.364 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 9 16 20975 20959 145.534 133 0.00510016 0.00686448 2024-09-13T15:03:56.370 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:56.389 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:56.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:56.492 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:56.518 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:56.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:56.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:56.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:56.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:56.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:56.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:56.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:56.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:56.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:56.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:57.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:57.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:57.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:57.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:57.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:57.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:57.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:57.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:57.305 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:03:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:03:57] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.51.0" 2024-09-13T15:03:57.365 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 10 16 21278 21262 132.874 18.9375 0.00998822 0.00751958 2024-09-13T15:03:57.402 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:57.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:57.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:57.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:57.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:57.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:57.678 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:57.708 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:57.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:57.812 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:57.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:57.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:57.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:57.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:57.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:58.074 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:58.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:58.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:58.205 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:58.214 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:58 smithi099 bash[84576]: cluster 2024-09-13T15:03:56.889659+0000 mgr.y (mgr.26252) 1134 : cluster [DBG] pgmap v1683: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:58.214 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:03:58 smithi099 bash[84576]: cluster 2024-09-13T15:03:56.889659+0000 mgr.y (mgr.26252) 1134 : cluster [DBG] pgmap v1683: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:58.214 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:58 smithi099 bash[87147]: cluster 2024-09-13T15:03:56.889659+0000 mgr.y (mgr.26252) 1134 : cluster [DBG] pgmap v1683: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:58.214 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:03:58 smithi099 bash[87147]: cluster 2024-09-13T15:03:56.889659+0000 mgr.y (mgr.26252) 1134 : cluster [DBG] pgmap v1683: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:58.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:58.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:58.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:58.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:57 smithi177 bash[68594]: cluster 2024-09-13T15:03:56.889659+0000 mgr.y (mgr.26252) 1134 : cluster [DBG] pgmap v1683: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:58.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:03:57 smithi177 bash[68594]: cluster 2024-09-13T15:03:56.889659+0000 mgr.y (mgr.26252) 1134 : cluster [DBG] pgmap v1683: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 4059/47928 objects degraded (8.469%) 2024-09-13T15:03:58.364 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 11 16 23575 23559 133.845 143.562 0.00347827 0.00746514 2024-09-13T15:03:58.368 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:58.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:58.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:58.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:58.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:58.591 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:58.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:58.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:58.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:58.743 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:58.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:58.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:58.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:58.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:58.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:59.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:59.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:59.137 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:59.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:59.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:59.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:59.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:59.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:59.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 12 16 24092 24076 125.383 32.3125 0.0033302 0.00791081 2024-09-13T15:03:59.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:59.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:59.497 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:59.524 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:59.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:59.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:59.654 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:59.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:59.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:59.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:59.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:59.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:59.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:00.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:00.059 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:00.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:00 smithi177 bash[68594]: cluster 2024-09-13T15:03:58.891814+0000 mgr.y (mgr.26252) 1135 : cluster [DBG] pgmap v1684: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 4568/57336 objects degraded (7.967%) 2024-09-13T15:04:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:00 smithi177 bash[68594]: cluster 2024-09-13T15:03:58.891814+0000 mgr.y (mgr.26252) 1135 : cluster [DBG] pgmap v1684: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 4568/57336 objects degraded (7.967%) 2024-09-13T15:04:00.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:00.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:00.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:00.318 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:04:00 smithi177 bash[61439]: ts=2024-09-13T15:04:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:00.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:00 smithi099 bash[84576]: cluster 2024-09-13T15:03:58.891814+0000 mgr.y (mgr.26252) 1135 : cluster [DBG] pgmap v1684: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 4568/57336 objects degraded (7.967%) 2024-09-13T15:04:00.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:00 smithi099 bash[84576]: cluster 2024-09-13T15:03:58.891814+0000 mgr.y (mgr.26252) 1135 : cluster [DBG] pgmap v1684: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 4568/57336 objects degraded (7.967%) 2024-09-13T15:04:00.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:00 smithi099 bash[87147]: cluster 2024-09-13T15:03:58.891814+0000 mgr.y (mgr.26252) 1135 : cluster [DBG] pgmap v1684: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 4568/57336 objects degraded (7.967%) 2024-09-13T15:04:00.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:00 smithi099 bash[87147]: cluster 2024-09-13T15:03:58.891814+0000 mgr.y (mgr.26252) 1135 : cluster [DBG] pgmap v1684: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 4568/57336 objects degraded (7.967%) 2024-09-13T15:04:00.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:00.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:00.365 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 13 16 25383 25367 121.945 80.6875 0.00698246 0.00819559 2024-09-13T15:04:00.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:00.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:00.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:00.577 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:00.603 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:00.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:00.706 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:00.735 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:00.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:00.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:00.864 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:00.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:00.993 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:00.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:01.096 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:01.125 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:01.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:01.175 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:01 smithi099 bash[84576]: cluster 2024-09-13T15:04:01.001759+0000 mon.a (mon.0) 2263 : cluster [WRN] Health check update: Degraded data redundancy: 5805/69726 objects degraded (8.325%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:01.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:01.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:01.359 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:01.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:01 smithi177 bash[68594]: cluster 2024-09-13T15:04:01.001759+0000 mon.a (mon.0) 2263 : cluster [WRN] Health check update: Degraded data redundancy: 5805/69726 objects degraded (8.325%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:01.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:01 smithi177 bash[68594]: cluster 2024-09-13T15:04:01.001759+0000 mon.a (mon.0) 2263 : cluster [WRN] Health check update: Degraded data redundancy: 5805/69726 objects degraded (8.325%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:01.365 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 14 16 27301 27285 121.795 119.875 0.00174325 0.00816134 2024-09-13T15:04:01.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:01.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:01.449 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:01 smithi099 bash[84576]: cluster 2024-09-13T15:04:01.001759+0000 mon.a (mon.0) 2263 : cluster [WRN] Health check update: Degraded data redundancy: 5805/69726 objects degraded (8.325%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:01.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:01 smithi099 bash[87147]: cluster 2024-09-13T15:04:01.001759+0000 mon.a (mon.0) 2263 : cluster [WRN] Health check update: Degraded data redundancy: 5805/69726 objects degraded (8.325%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:01.450 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:01 smithi099 bash[87147]: cluster 2024-09-13T15:04:01.001759+0000 mon.a (mon.0) 2263 : cluster [WRN] Health check update: Degraded data redundancy: 5805/69726 objects degraded (8.325%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:01.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:01.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:01.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:01.639 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:01.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:01.742 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:01.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:01.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:01.891 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:01.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:02.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:02.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:02.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:02.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:02.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:02.259 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:02.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:02.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:02.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:02 smithi099 bash[84576]: cluster 2024-09-13T15:04:00.892862+0000 mgr.y (mgr.26252) 1136 : cluster [DBG] pgmap v1685: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s; 5805/69726 objects degraded (8.325%) 2024-09-13T15:04:02.331 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:02 smithi099 bash[84576]: cluster 2024-09-13T15:04:00.892862+0000 mgr.y (mgr.26252) 1136 : cluster [DBG] pgmap v1685: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s; 5805/69726 objects degraded (8.325%) 2024-09-13T15:04:02.331 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:02 smithi099 bash[87147]: cluster 2024-09-13T15:04:00.892862+0000 mgr.y (mgr.26252) 1136 : cluster [DBG] pgmap v1685: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s; 5805/69726 objects degraded (8.325%) 2024-09-13T15:04:02.332 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:02 smithi099 bash[87147]: cluster 2024-09-13T15:04:00.892862+0000 mgr.y (mgr.26252) 1136 : cluster [DBG] pgmap v1685: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s; 5805/69726 objects degraded (8.325%) 2024-09-13T15:04:02.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:02 smithi177 bash[68594]: cluster 2024-09-13T15:04:00.892862+0000 mgr.y (mgr.26252) 1136 : cluster [DBG] pgmap v1685: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s; 5805/69726 objects degraded (8.325%) 2024-09-13T15:04:02.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:02 smithi177 bash[68594]: cluster 2024-09-13T15:04:00.892862+0000 mgr.y (mgr.26252) 1136 : cluster [DBG] pgmap v1685: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s; 5805/69726 objects degraded (8.325%) 2024-09-13T15:04:02.365 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 15 16 28758 28742 119.746 91.0625 0.00401044 0.00834544 2024-09-13T15:04:02.401 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:02.429 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:02.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:02.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:02.561 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:02.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:02.664 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:02.689 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:02.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:02.792 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:02.798 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:02.798 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:02.814 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:02.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:02.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:02.945 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:02.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:03.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:03.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:03.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:03.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:03.201 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:03.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:03.304 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:03.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:03.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:03.365 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 16 16 30493 30477 119.038 108.438 0.00357892 0.00832237 2024-09-13T15:04:03.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:03.458 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:03.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:03.592 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:03.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:03.720 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:03.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:03.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:03.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:03.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:03.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:03.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:04.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:04.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:04 smithi177 bash[68594]: cluster 2024-09-13T15:04:02.893653+0000 mgr.y (mgr.26252) 1137 : cluster [DBG] pgmap v1686: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 6277/76518 objects degraded (8.203%) 2024-09-13T15:04:04.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:04 smithi177 bash[68594]: cluster 2024-09-13T15:04:02.893653+0000 mgr.y (mgr.26252) 1137 : cluster [DBG] pgmap v1686: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 6277/76518 objects degraded (8.203%) 2024-09-13T15:04:04.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:04 smithi177 bash[68594]: audit 2024-09-13T15:04:03.297388+0000 mon.a (mon.0) 2264 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:04.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:04 smithi177 bash[68594]: audit 2024-09-13T15:04:03.297388+0000 mon.a (mon.0) 2264 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:04.120 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:04.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:04.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:04.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:04.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:04.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:04 smithi099 bash[84576]: cluster 2024-09-13T15:04:02.893653+0000 mgr.y (mgr.26252) 1137 : cluster [DBG] pgmap v1686: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 6277/76518 objects degraded (8.203%) 2024-09-13T15:04:04.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:04 smithi099 bash[84576]: cluster 2024-09-13T15:04:02.893653+0000 mgr.y (mgr.26252) 1137 : cluster [DBG] pgmap v1686: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 6277/76518 objects degraded (8.203%) 2024-09-13T15:04:04.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:04 smithi099 bash[84576]: audit 2024-09-13T15:04:03.297388+0000 mon.a (mon.0) 2264 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:04.332 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:04 smithi099 bash[84576]: audit 2024-09-13T15:04:03.297388+0000 mon.a (mon.0) 2264 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:04.333 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:04 smithi099 bash[87147]: cluster 2024-09-13T15:04:02.893653+0000 mgr.y (mgr.26252) 1137 : cluster [DBG] pgmap v1686: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 6277/76518 objects degraded (8.203%) 2024-09-13T15:04:04.333 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:04 smithi099 bash[87147]: cluster 2024-09-13T15:04:02.893653+0000 mgr.y (mgr.26252) 1137 : cluster [DBG] pgmap v1686: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s; 6277/76518 objects degraded (8.203%) 2024-09-13T15:04:04.333 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:04 smithi099 bash[87147]: audit 2024-09-13T15:04:03.297388+0000 mon.a (mon.0) 2264 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:04.333 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:04 smithi099 bash[87147]: audit 2024-09-13T15:04:03.297388+0000 mon.a (mon.0) 2264 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:04.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:04.365 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 17 16 32166 32150 118.186 104.562 0.00284072 0.00845527 2024-09-13T15:04:04.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:04.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:04.492 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:04.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:04.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:04.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:04.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:04.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:04.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:04.766 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:04.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:04.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:04.889 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:04.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:04.991 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:05.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:05.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:05.121 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:05.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:05.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:05.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:05.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:05.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:05.365 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 18 16 33282 33266 115.495 69.75 0.0112333 0.00865302 2024-09-13T15:04:05.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:05.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:05.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:05.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:05.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:05.653 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:05.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:05.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:05.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:05.814 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:05.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:05.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:05.945 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:05.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:06.048 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:06.075 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:06.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:06.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:06.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:06.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:06.324 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:06.334 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:06 smithi177 bash[68594]: cluster 2024-09-13T15:04:04.894347+0000 mgr.y (mgr.26252) 1138 : cluster [DBG] pgmap v1687: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:06.334 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:06 smithi177 bash[68594]: cluster 2024-09-13T15:04:04.894347+0000 mgr.y (mgr.26252) 1138 : cluster [DBG] pgmap v1687: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:06.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:06.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:06.365 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 19 16 34384 34368 113.041 68.875 0.00383153 0.00876011 2024-09-13T15:04:06.400 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:06 smithi099 bash[84576]: cluster 2024-09-13T15:04:04.894347+0000 mgr.y (mgr.26252) 1138 : cluster [DBG] pgmap v1687: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:06.400 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:06 smithi099 bash[84576]: cluster 2024-09-13T15:04:04.894347+0000 mgr.y (mgr.26252) 1138 : cluster [DBG] pgmap v1687: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:06.401 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:06 smithi099 bash[87147]: cluster 2024-09-13T15:04:04.894347+0000 mgr.y (mgr.26252) 1138 : cluster [DBG] pgmap v1687: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:06.401 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:06 smithi099 bash[87147]: cluster 2024-09-13T15:04:04.894347+0000 mgr.y (mgr.26252) 1138 : cluster [DBG] pgmap v1687: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:06.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:06.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:06.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:06.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:06.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:06.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:06.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:06.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:06.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:06.856 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:06.888 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:06.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:07.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:07.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:07.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:07.132 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:07 smithi177 bash[68594]: cluster 2024-09-13T15:04:06.506463+0000 mon.a (mon.0) 2265 : cluster [WRN] Health check update: Degraded data redundancy: 7847/93363 objects degraded (8.405%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:07.132 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:07 smithi177 bash[68594]: cluster 2024-09-13T15:04:06.506463+0000 mon.a (mon.0) 2265 : cluster [WRN] Health check update: Degraded data redundancy: 7847/93363 objects degraded (8.405%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:07.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:07.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:07.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:07.285 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:07.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:07.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:04:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:04:07] "GET /metrics HTTP/1.1" 200 36763 "" "Prometheus/2.51.0" 2024-09-13T15:04:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:07 smithi099 bash[84576]: cluster 2024-09-13T15:04:06.506463+0000 mon.a (mon.0) 2265 : cluster [WRN] Health check update: Degraded data redundancy: 7847/93363 objects degraded (8.405%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:07 smithi099 bash[84576]: cluster 2024-09-13T15:04:06.506463+0000 mon.a (mon.0) 2265 : cluster [WRN] Health check update: Degraded data redundancy: 7847/93363 objects degraded (8.405%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:07.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:07 smithi099 bash[87147]: cluster 2024-09-13T15:04:06.506463+0000 mon.a (mon.0) 2265 : cluster [WRN] Health check update: Degraded data redundancy: 7847/93363 objects degraded (8.405%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:07.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:07 smithi099 bash[87147]: cluster 2024-09-13T15:04:06.506463+0000 mon.a (mon.0) 2265 : cluster [WRN] Health check update: Degraded data redundancy: 7847/93363 objects degraded (8.405%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:07.365 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:04:07.367106+0000 min lat: 0.00126645 max lat: 0.563801 avg lat: 0.00895047 2024-09-13T15:04:07.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:04:07.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 20 16 35746 35730 111.644 85.125 0.00793918 0.00895047 2024-09-13T15:04:07.388 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:07.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:07.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:07.519 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:07.543 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:07.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:07.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:07.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:07.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:07.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:07.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:07.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:07.904 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:07.930 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:07.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:08.033 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:08.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:08.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:08.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:08.195 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:08.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:08.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T15:04:08.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:08.319 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:08.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:08.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:08 smithi177 bash[68594]: cluster 2024-09-13T15:04:06.894934+0000 mgr.y (mgr.26252) 1139 : cluster [DBG] pgmap v1688: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:08 smithi177 bash[68594]: cluster 2024-09-13T15:04:06.894934+0000 mgr.y (mgr.26252) 1139 : cluster [DBG] pgmap v1688: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:08.365 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 21 16 36750 36734 109.316 62.75 0.00290482 0.00908448 2024-09-13T15:04:08.422 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:08.448 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:08.473 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:08 smithi099 bash[84576]: cluster 2024-09-13T15:04:06.894934+0000 mgr.y (mgr.26252) 1139 : cluster [DBG] pgmap v1688: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:08.473 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:08 smithi099 bash[84576]: cluster 2024-09-13T15:04:06.894934+0000 mgr.y (mgr.26252) 1139 : cluster [DBG] pgmap v1688: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:08.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:08 smithi099 bash[87147]: cluster 2024-09-13T15:04:06.894934+0000 mgr.y (mgr.26252) 1139 : cluster [DBG] pgmap v1688: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:08.473 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:08 smithi099 bash[87147]: cluster 2024-09-13T15:04:06.894934+0000 mgr.y (mgr.26252) 1139 : cluster [DBG] pgmap v1688: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 7847/93363 objects degraded (8.405%) 2024-09-13T15:04:08.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:08.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:08.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:08.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:08.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:08.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:08.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:08.936 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:08.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:08.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:09.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:09.094 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:09.198 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:09.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:09.333 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:09.354 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:09 smithi099 bash[84576]: cluster 2024-09-13T15:04:08.895677+0000 mgr.y (mgr.26252) 1140 : cluster [DBG] pgmap v1689: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 8300/101103 objects degraded (8.209%) 2024-09-13T15:04:09.354 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:09 smithi099 bash[84576]: cluster 2024-09-13T15:04:08.895677+0000 mgr.y (mgr.26252) 1140 : cluster [DBG] pgmap v1689: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 8300/101103 objects degraded (8.209%) 2024-09-13T15:04:09.355 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:09 smithi099 bash[87147]: cluster 2024-09-13T15:04:08.895677+0000 mgr.y (mgr.26252) 1140 : cluster [DBG] pgmap v1689: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 8300/101103 objects degraded (8.209%) 2024-09-13T15:04:09.355 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:09 smithi099 bash[87147]: cluster 2024-09-13T15:04:08.895677+0000 mgr.y (mgr.26252) 1140 : cluster [DBG] pgmap v1689: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 8300/101103 objects degraded (8.209%) 2024-09-13T15:04:09.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:09 smithi177 bash[68594]: cluster 2024-09-13T15:04:08.895677+0000 mgr.y (mgr.26252) 1140 : cluster [DBG] pgmap v1689: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 8300/101103 objects degraded (8.209%) 2024-09-13T15:04:09.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:09 smithi177 bash[68594]: cluster 2024-09-13T15:04:08.895677+0000 mgr.y (mgr.26252) 1140 : cluster [DBG] pgmap v1689: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 8300/101103 objects degraded (8.209%) 2024-09-13T15:04:09.362 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:09.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 22 16 37950 37934 107.756 75 0.00555243 0.00927347 2024-09-13T15:04:09.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:09.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:09.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:09.620 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:09.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:09.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:09.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:09.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:09.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:09.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:09.904 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:09.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:10.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:10.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:10.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:10.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:10.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:10.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:10.285 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:10.295 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:04:10 smithi177 bash[61439]: ts=2024-09-13T15:04:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:10.307 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:10.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:10.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 23 16 39722 39706 107.886 110.75 0.00803272 0.00926381 2024-09-13T15:04:10.411 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:10.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:10.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:10.552 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:10.580 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:10.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:10.685 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:10.713 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:10.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:10.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:10.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:10.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:10.975 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:10.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:11.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:11.106 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:11.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:11.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:11.233 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:11.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:11.336 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:11.358 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:11.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:11.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 24 16 41156 41140 107.124 89.625 0.00158453 0.00928556 2024-09-13T15:04:11.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:11.496 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:11.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:11.601 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:11.627 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:11.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:11.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:11.759 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:11.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:11.863 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:11.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:11.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:11.994 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.023 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:12.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.137 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:11 smithi177 bash[68594]: cluster 2024-09-13T15:04:10.896604+0000 mgr.y (mgr.26252) 1141 : cluster [DBG] pgmap v1690: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 9321/111441 objects degraded (8.364%) 2024-09-13T15:04:12.137 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:11 smithi177 bash[68594]: cluster 2024-09-13T15:04:10.896604+0000 mgr.y (mgr.26252) 1141 : cluster [DBG] pgmap v1690: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 9321/111441 objects degraded (8.364%) 2024-09-13T15:04:12.137 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:11 smithi177 bash[68594]: cluster 2024-09-13T15:04:11.507313+0000 mon.a (mon.0) 2266 : cluster [WRN] Health check update: Degraded data redundancy: 9321/111441 objects degraded (8.364%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:12.137 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:11 smithi177 bash[68594]: cluster 2024-09-13T15:04:11.507313+0000 mon.a (mon.0) 2266 : cluster [WRN] Health check update: Degraded data redundancy: 9321/111441 objects degraded (8.364%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:12.156 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:12.203 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:11 smithi099 bash[84576]: cluster 2024-09-13T15:04:10.896604+0000 mgr.y (mgr.26252) 1141 : cluster [DBG] pgmap v1690: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 9321/111441 objects degraded (8.364%) 2024-09-13T15:04:12.203 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:11 smithi099 bash[84576]: cluster 2024-09-13T15:04:10.896604+0000 mgr.y (mgr.26252) 1141 : cluster [DBG] pgmap v1690: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 9321/111441 objects degraded (8.364%) 2024-09-13T15:04:12.203 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:11 smithi099 bash[84576]: cluster 2024-09-13T15:04:11.507313+0000 mon.a (mon.0) 2266 : cluster [WRN] Health check update: Degraded data redundancy: 9321/111441 objects degraded (8.364%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:12.204 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:11 smithi099 bash[84576]: cluster 2024-09-13T15:04:11.507313+0000 mon.a (mon.0) 2266 : cluster [WRN] Health check update: Degraded data redundancy: 9321/111441 objects degraded (8.364%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:12.204 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:11 smithi099 bash[87147]: cluster 2024-09-13T15:04:10.896604+0000 mgr.y (mgr.26252) 1141 : cluster [DBG] pgmap v1690: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 9321/111441 objects degraded (8.364%) 2024-09-13T15:04:12.204 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:11 smithi099 bash[87147]: cluster 2024-09-13T15:04:10.896604+0000 mgr.y (mgr.26252) 1141 : cluster [DBG] pgmap v1690: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 9321/111441 objects degraded (8.364%) 2024-09-13T15:04:12.204 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:11 smithi099 bash[87147]: cluster 2024-09-13T15:04:11.507313+0000 mon.a (mon.0) 2266 : cluster [WRN] Health check update: Degraded data redundancy: 9321/111441 objects degraded (8.364%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:12.204 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:11 smithi099 bash[87147]: cluster 2024-09-13T15:04:11.507313+0000 mon.a (mon.0) 2266 : cluster [WRN] Health check update: Degraded data redundancy: 9321/111441 objects degraded (8.364%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:12.258 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:12.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 25 16 42679 42663 106.646 95.1875 0.0103607 0.0093712 2024-09-13T15:04:12.388 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:12.411 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:12.516 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:12.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:12.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:12.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:12.767 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:12.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:12.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:12.922 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:12.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:12.997 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:13.025 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:13.053 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:13.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:13.156 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:13.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:13.276 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:13.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:13.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 26 16 43597 43581 104.751 57.375 0.0157567 0.00954095 2024-09-13T15:04:13.400 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:13.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:13.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:13.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:13.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:13.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:13.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:13.693 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:13.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:13.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:13.828 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:13.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:13.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:13.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:13.973 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:04:14.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:14.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:14.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:14.186 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:14.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:13 smithi177 bash[68594]: cluster 2024-09-13T15:04:12.897477+0000 mgr.y (mgr.26252) 1142 : cluster [DBG] pgmap v1691: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 9719/117756 objects degraded (8.254%) 2024-09-13T15:04:14.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:13 smithi177 bash[68594]: cluster 2024-09-13T15:04:12.897477+0000 mgr.y (mgr.26252) 1142 : cluster [DBG] pgmap v1691: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 9719/117756 objects degraded (8.254%) 2024-09-13T15:04:14.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:13 smithi177 bash[68594]: audit 2024-09-13T15:04:13.127305+0000 mon.c (mon.2) 89 : audit [INF] from='client.? 172.21.15.99:0/1398149737' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:13 smithi177 bash[68594]: audit 2024-09-13T15:04:13.127305+0000 mon.c (mon.2) 89 : audit [INF] from='client.? 172.21.15.99:0/1398149737' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:13 smithi177 bash[68594]: audit 2024-09-13T15:04:13.127863+0000 mon.a (mon.0) 2267 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:13 smithi177 bash[68594]: audit 2024-09-13T15:04:13.127863+0000 mon.a (mon.0) 2267 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.209 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:14.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:14.312 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:14.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[84576]: cluster 2024-09-13T15:04:12.897477+0000 mgr.y (mgr.26252) 1142 : cluster [DBG] pgmap v1691: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 9719/117756 objects degraded (8.254%) 2024-09-13T15:04:14.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[84576]: cluster 2024-09-13T15:04:12.897477+0000 mgr.y (mgr.26252) 1142 : cluster [DBG] pgmap v1691: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 9719/117756 objects degraded (8.254%) 2024-09-13T15:04:14.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[84576]: audit 2024-09-13T15:04:13.127305+0000 mon.c (mon.2) 89 : audit [INF] from='client.? 172.21.15.99:0/1398149737' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[84576]: audit 2024-09-13T15:04:13.127305+0000 mon.c (mon.2) 89 : audit [INF] from='client.? 172.21.15.99:0/1398149737' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[84576]: audit 2024-09-13T15:04:13.127863+0000 mon.a (mon.0) 2267 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[84576]: audit 2024-09-13T15:04:13.127863+0000 mon.a (mon.0) 2267 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[87147]: cluster 2024-09-13T15:04:12.897477+0000 mgr.y (mgr.26252) 1142 : cluster [DBG] pgmap v1691: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 9719/117756 objects degraded (8.254%) 2024-09-13T15:04:14.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[87147]: cluster 2024-09-13T15:04:12.897477+0000 mgr.y (mgr.26252) 1142 : cluster [DBG] pgmap v1691: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 9719/117756 objects degraded (8.254%) 2024-09-13T15:04:14.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[87147]: audit 2024-09-13T15:04:13.127305+0000 mon.c (mon.2) 89 : audit [INF] from='client.? 172.21.15.99:0/1398149737' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[87147]: audit 2024-09-13T15:04:13.127305+0000 mon.c (mon.2) 89 : audit [INF] from='client.? 172.21.15.99:0/1398149737' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[87147]: audit 2024-09-13T15:04:13.127863+0000 mon.a (mon.0) 2267 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:13 smithi099 bash[87147]: audit 2024-09-13T15:04:13.127863+0000 mon.a (mon.0) 2267 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:14.339 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:14.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:14.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 27 16 44876 44860 103.832 79.9375 0.0141132 0.00962562 2024-09-13T15:04:14.442 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:14.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:14.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:14.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:14.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:14.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:14.700 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:14.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:14.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:14.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:14.862 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:14.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:14.960 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:14 smithi177 bash[68594]: cluster 2024-09-13T15:04:13.954207+0000 mon.a (mon.0) 2268 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:14.960 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:14 smithi177 bash[68594]: cluster 2024-09-13T15:04:13.954207+0000 mon.a (mon.0) 2268 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:14.964 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:14.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:15.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:15.117 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:15.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:15.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:15.230 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[84576]: cluster 2024-09-13T15:04:13.954207+0000 mon.a (mon.0) 2268 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:15.230 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[84576]: cluster 2024-09-13T15:04:13.954207+0000 mon.a (mon.0) 2268 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:15.231 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[84576]: audit 2024-09-13T15:04:13.964969+0000 mon.a (mon.0) 2269 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:15.231 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[84576]: audit 2024-09-13T15:04:13.964969+0000 mon.a (mon.0) 2269 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:15.231 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[84576]: cluster 2024-09-13T15:04:13.968755+0000 mon.a (mon.0) 2270 : cluster [DBG] osdmap e1061: 8 total, 7 up, 8 in 2024-09-13T15:04:15.231 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[84576]: cluster 2024-09-13T15:04:13.968755+0000 mon.a (mon.0) 2270 : cluster [DBG] osdmap e1061: 8 total, 7 up, 8 in 2024-09-13T15:04:15.231 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[87147]: cluster 2024-09-13T15:04:13.954207+0000 mon.a (mon.0) 2268 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:15.231 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[87147]: cluster 2024-09-13T15:04:13.954207+0000 mon.a (mon.0) 2268 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:15.231 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[87147]: audit 2024-09-13T15:04:13.964969+0000 mon.a (mon.0) 2269 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:15.231 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[87147]: audit 2024-09-13T15:04:13.964969+0000 mon.a (mon.0) 2269 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:15.232 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[87147]: cluster 2024-09-13T15:04:13.968755+0000 mon.a (mon.0) 2270 : cluster [DBG] osdmap e1061: 8 total, 7 up, 8 in 2024-09-13T15:04:15.232 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:14 smithi099 bash[87147]: cluster 2024-09-13T15:04:13.968755+0000 mon.a (mon.0) 2270 : cluster [DBG] osdmap e1061: 8 total, 7 up, 8 in 2024-09-13T15:04:15.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:15.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:15.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:15.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:14 smithi177 bash[68594]: audit 2024-09-13T15:04:13.964969+0000 mon.a (mon.0) 2269 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:15.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:14 smithi177 bash[68594]: audit 2024-09-13T15:04:13.964969+0000 mon.a (mon.0) 2269 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:15.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:14 smithi177 bash[68594]: cluster 2024-09-13T15:04:13.968755+0000 mon.a (mon.0) 2270 : cluster [DBG] osdmap e1061: 8 total, 7 up, 8 in 2024-09-13T15:04:15.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:14 smithi177 bash[68594]: cluster 2024-09-13T15:04:13.968755+0000 mon.a (mon.0) 2270 : cluster [DBG] osdmap e1061: 8 total, 7 up, 8 in 2024-09-13T15:04:15.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 28 16 46044 46028 102.73 73 0.00979084 0.00972785 2024-09-13T15:04:15.375 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:15.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:15.482 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:15.509 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:15.615 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:15.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:15.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:15.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:15.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:15.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:15.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:15.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:16.003 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:16.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:16.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:15 smithi177 bash[68594]: cluster 2024-09-13T15:04:14.898415+0000 mgr.y (mgr.26252) 1143 : cluster [DBG] pgmap v1693: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:16.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:15 smithi177 bash[68594]: cluster 2024-09-13T15:04:14.898415+0000 mgr.y (mgr.26252) 1143 : cluster [DBG] pgmap v1693: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:16.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:16.165 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:16.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:16.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:16.277 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:15 smithi099 bash[84576]: cluster 2024-09-13T15:04:14.898415+0000 mgr.y (mgr.26252) 1143 : cluster [DBG] pgmap v1693: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:16.277 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:15 smithi099 bash[84576]: cluster 2024-09-13T15:04:14.898415+0000 mgr.y (mgr.26252) 1143 : cluster [DBG] pgmap v1693: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:16.277 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:16 smithi099 bash[87147]: cluster 2024-09-13T15:04:14.898415+0000 mgr.y (mgr.26252) 1143 : cluster [DBG] pgmap v1693: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:16.277 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:16 smithi099 bash[87147]: cluster 2024-09-13T15:04:14.898415+0000 mgr.y (mgr.26252) 1143 : cluster [DBG] pgmap v1693: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:16.287 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:16.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:16.366 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 29 16 47296 47280 101.886 78.25 0.0126305 0.00976713 2024-09-13T15:04:16.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:16.410 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:16.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:16.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:16.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:16.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:16.647 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:16.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:16.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:16.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:16.806 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:16.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:16.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.939 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:16.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:17.040 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:17.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:17.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:17.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:17.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:17.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:17.299 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:17.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:16 smithi177 bash[68594]: cluster 2024-09-13T15:04:16.508543+0000 mon.a (mon.0) 2271 : cluster [WRN] Health check update: Degraded data redundancy: 11037/132375 objects degraded (8.338%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:17.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:16 smithi177 bash[68594]: cluster 2024-09-13T15:04:16.508543+0000 mon.a (mon.0) 2271 : cluster [WRN] Health check update: Degraded data redundancy: 11037/132375 objects degraded (8.338%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:17.320 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:17.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:17.367 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 30 16 48391 48375 100.77 68.4375 0.00254097 0.00988042 2024-09-13T15:04:17.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:17.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:17.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:17.464 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:16 smithi099 bash[84576]: cluster 2024-09-13T15:04:16.508543+0000 mon.a (mon.0) 2271 : cluster [WRN] Health check update: Degraded data redundancy: 11037/132375 objects degraded (8.338%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:17.464 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:16 smithi099 bash[84576]: cluster 2024-09-13T15:04:16.508543+0000 mon.a (mon.0) 2271 : cluster [WRN] Health check update: Degraded data redundancy: 11037/132375 objects degraded (8.338%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:17.465 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:04:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:04:17] "GET /metrics HTTP/1.1" 200 36763 "" "Prometheus/2.51.0" 2024-09-13T15:04:17.466 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:16 smithi099 bash[87147]: cluster 2024-09-13T15:04:16.508543+0000 mon.a (mon.0) 2271 : cluster [WRN] Health check update: Degraded data redundancy: 11037/132375 objects degraded (8.338%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:17.466 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:16 smithi099 bash[87147]: cluster 2024-09-13T15:04:16.508543+0000 mon.a (mon.0) 2271 : cluster [WRN] Health check update: Degraded data redundancy: 11037/132375 objects degraded (8.338%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:17.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:17.592 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:17.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:17.700 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:17.725 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:17.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:17.828 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:17.855 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:17.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:17.961 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:17.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:17.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:18.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:18.117 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:18.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:18.221 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:18.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:17 smithi177 bash[68594]: cluster 2024-09-13T15:04:16.899100+0000 mgr.y (mgr.26252) 1144 : cluster [DBG] pgmap v1694: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:18.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:17 smithi177 bash[68594]: cluster 2024-09-13T15:04:16.899100+0000 mgr.y (mgr.26252) 1144 : cluster [DBG] pgmap v1694: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:18.254 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:18.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:18.355 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:17 smithi099 bash[84576]: cluster 2024-09-13T15:04:16.899100+0000 mgr.y (mgr.26252) 1144 : cluster [DBG] pgmap v1694: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:18.355 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:18 smithi099 bash[84576]: cluster 2024-09-13T15:04:16.899100+0000 mgr.y (mgr.26252) 1144 : cluster [DBG] pgmap v1694: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:18.355 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:18 smithi099 bash[87147]: cluster 2024-09-13T15:04:16.899100+0000 mgr.y (mgr.26252) 1144 : cluster [DBG] pgmap v1694: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:18.355 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:18 smithi099 bash[87147]: cluster 2024-09-13T15:04:16.899100+0000 mgr.y (mgr.26252) 1144 : cluster [DBG] pgmap v1694: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 11037/132375 objects degraded (8.338%) 2024-09-13T15:04:18.357 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:18.367 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 31 16 49482 49466 99.7191 68.1875 0.111262 0.010015 2024-09-13T15:04:18.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:18.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:18.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:18.533 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:18.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:18.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:18.672 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:18.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:18.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:18.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:18.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:18.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:18.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:18.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:18.966 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:04:18.966 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:19.007 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:19.007 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:04:19.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:19.053 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:04:19.053 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:19.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:04:19.079 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:19.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:19.113 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:04:19.113 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:19.147 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:04:19.148 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:19.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:19.189 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:19.189 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:04:19.210 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:19.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:19.214 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:04:19.214 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:19.227 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T15:04:19.228 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T15:04:19.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:19.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:19.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:19.367 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 32 16 50926 50910 99.4231 90.25 0.00768145 0.0100527 2024-09-13T15:04:19.447 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:19.478 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[84576]: audit 2024-09-13T15:04:18.300641+0000 mon.a (mon.0) 2272 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:19.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[84576]: audit 2024-09-13T15:04:18.300641+0000 mon.a (mon.0) 2272 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[84576]: audit 2024-09-13T15:04:18.301490+0000 mon.a (mon.0) 2273 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[84576]: audit 2024-09-13T15:04:18.301490+0000 mon.a (mon.0) 2273 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[84576]: cluster 2024-09-13T15:04:18.900033+0000 mgr.y (mgr.26252) 1145 : cluster [DBG] pgmap v1695: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 11386/139143 objects degraded (8.183%) 2024-09-13T15:04:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[84576]: cluster 2024-09-13T15:04:18.900033+0000 mgr.y (mgr.26252) 1145 : cluster [DBG] pgmap v1695: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 11386/139143 objects degraded (8.183%) 2024-09-13T15:04:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[84576]: audit 2024-09-13T15:04:18.937122+0000 mon.a (mon.0) 2274 : audit [DBG] from='client.? 172.21.15.99:0/2426608390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[84576]: audit 2024-09-13T15:04:18.937122+0000 mon.a (mon.0) 2274 : audit [DBG] from='client.? 172.21.15.99:0/2426608390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[87147]: audit 2024-09-13T15:04:18.300641+0000 mon.a (mon.0) 2272 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[87147]: audit 2024-09-13T15:04:18.300641+0000 mon.a (mon.0) 2272 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[87147]: audit 2024-09-13T15:04:18.301490+0000 mon.a (mon.0) 2273 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[87147]: audit 2024-09-13T15:04:18.301490+0000 mon.a (mon.0) 2273 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[87147]: cluster 2024-09-13T15:04:18.900033+0000 mgr.y (mgr.26252) 1145 : cluster [DBG] pgmap v1695: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 11386/139143 objects degraded (8.183%) 2024-09-13T15:04:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[87147]: cluster 2024-09-13T15:04:18.900033+0000 mgr.y (mgr.26252) 1145 : cluster [DBG] pgmap v1695: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 11386/139143 objects degraded (8.183%) 2024-09-13T15:04:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[87147]: audit 2024-09-13T15:04:18.937122+0000 mon.a (mon.0) 2274 : audit [DBG] from='client.? 172.21.15.99:0/2426608390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:19 smithi099 bash[87147]: audit 2024-09-13T15:04:18.937122+0000 mon.a (mon.0) 2274 : audit [DBG] from='client.? 172.21.15.99:0/2426608390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:19.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:19.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:19.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:19 smithi177 bash[68594]: audit 2024-09-13T15:04:18.300641+0000 mon.a (mon.0) 2272 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:19 smithi177 bash[68594]: audit 2024-09-13T15:04:18.300641+0000 mon.a (mon.0) 2272 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:19 smithi177 bash[68594]: audit 2024-09-13T15:04:18.301490+0000 mon.a (mon.0) 2273 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:19 smithi177 bash[68594]: audit 2024-09-13T15:04:18.301490+0000 mon.a (mon.0) 2273 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:19 smithi177 bash[68594]: cluster 2024-09-13T15:04:18.900033+0000 mgr.y (mgr.26252) 1145 : cluster [DBG] pgmap v1695: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 11386/139143 objects degraded (8.183%) 2024-09-13T15:04:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:19 smithi177 bash[68594]: cluster 2024-09-13T15:04:18.900033+0000 mgr.y (mgr.26252) 1145 : cluster [DBG] pgmap v1695: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 11386/139143 objects degraded (8.183%) 2024-09-13T15:04:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:19 smithi177 bash[68594]: audit 2024-09-13T15:04:18.937122+0000 mon.a (mon.0) 2274 : audit [DBG] from='client.? 172.21.15.99:0/2426608390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:19 smithi177 bash[68594]: audit 2024-09-13T15:04:18.937122+0000 mon.a (mon.0) 2274 : audit [DBG] from='client.? 172.21.15.99:0/2426608390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:19.711 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:19.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:19.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:19.842 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:19.871 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:19.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:19.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:20.012 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:20.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:20.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:20.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:20.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:20.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:20.277 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:20.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:20.367 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 33 16 52322 52306 99.0539 87.25 0.107968 0.0100644 2024-09-13T15:04:20.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:20.401 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:20.504 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:20.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:20.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:20 smithi099 bash[87147]: audit 2024-09-13T15:04:19.218019+0000 mgr.y (mgr.26252) 1146 : audit [DBG] from='client.45776 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:04:20.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:20 smithi099 bash[87147]: audit 2024-09-13T15:04:19.218019+0000 mgr.y (mgr.26252) 1146 : audit [DBG] from='client.45776 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:04:20.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:20 smithi099 bash[84576]: audit 2024-09-13T15:04:19.218019+0000 mgr.y (mgr.26252) 1146 : audit [DBG] from='client.45776 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:04:20.568 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:20 smithi099 bash[84576]: audit 2024-09-13T15:04:19.218019+0000 mgr.y (mgr.26252) 1146 : audit [DBG] from='client.45776 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:04:20.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:20 smithi177 bash[68594]: audit 2024-09-13T15:04:19.218019+0000 mgr.y (mgr.26252) 1146 : audit [DBG] from='client.45776 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:04:20.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:20 smithi177 bash[68594]: audit 2024-09-13T15:04:19.218019+0000 mgr.y (mgr.26252) 1146 : audit [DBG] from='client.45776 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:04:20.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:04:20 smithi177 bash[61439]: ts=2024-09-13T15:04:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:20.632 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:20.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:20.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:20.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:20.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:20.903 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:20.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:04:20.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:20.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:21.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:21.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:21.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:21.174 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:21.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:21.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:21.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:21.332 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:21.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:21.367 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 34 16 53335 53319 98.0025 63.3125 0.00464041 0.0101978 2024-09-13T15:04:21.436 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:21.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:21.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:21.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:21 smithi177 bash[68594]: cluster 2024-09-13T15:04:20.900851+0000 mgr.y (mgr.26252) 1147 : cluster [DBG] pgmap v1696: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 12450/150258 objects degraded (8.286%) 2024-09-13T15:04:21.570 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:21 smithi177 bash[68594]: cluster 2024-09-13T15:04:20.900851+0000 mgr.y (mgr.26252) 1147 : cluster [DBG] pgmap v1696: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 12450/150258 objects degraded (8.286%) 2024-09-13T15:04:21.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:21 smithi099 bash[84576]: cluster 2024-09-13T15:04:20.900851+0000 mgr.y (mgr.26252) 1147 : cluster [DBG] pgmap v1696: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 12450/150258 objects degraded (8.286%) 2024-09-13T15:04:21.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:21 smithi099 bash[84576]: cluster 2024-09-13T15:04:20.900851+0000 mgr.y (mgr.26252) 1147 : cluster [DBG] pgmap v1696: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 12450/150258 objects degraded (8.286%) 2024-09-13T15:04:21.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:21 smithi099 bash[87147]: cluster 2024-09-13T15:04:20.900851+0000 mgr.y (mgr.26252) 1147 : cluster [DBG] pgmap v1696: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 12450/150258 objects degraded (8.286%) 2024-09-13T15:04:21.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:21 smithi099 bash[87147]: cluster 2024-09-13T15:04:20.900851+0000 mgr.y (mgr.26252) 1147 : cluster [DBG] pgmap v1696: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 12450/150258 objects degraded (8.286%) 2024-09-13T15:04:21.593 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:21.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:21.724 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:21.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:21.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:21.847 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:21.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:21.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:21.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:21.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:22.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:22.106 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:22.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:22.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:22.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:22.332 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:22.343 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:22 smithi177 bash[68594]: cluster 2024-09-13T15:04:21.509599+0000 mon.a (mon.0) 2275 : cluster [WRN] Health check update: Degraded data redundancy: 12450/150258 objects degraded (8.286%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:22.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:22 smithi177 bash[68594]: cluster 2024-09-13T15:04:21.509599+0000 mon.a (mon.0) 2275 : cluster [WRN] Health check update: Degraded data redundancy: 12450/150258 objects degraded (8.286%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:22.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:22.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:22.367 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 35 16 54250 54234 96.8362 57.1875 0.0061069 0.010311 2024-09-13T15:04:22.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:22.471 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:22 smithi099 bash[84576]: cluster 2024-09-13T15:04:21.509599+0000 mon.a (mon.0) 2275 : cluster [WRN] Health check update: Degraded data redundancy: 12450/150258 objects degraded (8.286%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:22.471 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:22 smithi099 bash[84576]: cluster 2024-09-13T15:04:21.509599+0000 mon.a (mon.0) 2275 : cluster [WRN] Health check update: Degraded data redundancy: 12450/150258 objects degraded (8.286%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:22.471 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:22 smithi099 bash[87147]: cluster 2024-09-13T15:04:21.509599+0000 mon.a (mon.0) 2275 : cluster [WRN] Health check update: Degraded data redundancy: 12450/150258 objects degraded (8.286%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:22.471 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:22 smithi099 bash[87147]: cluster 2024-09-13T15:04:21.509599+0000 mon.a (mon.0) 2275 : cluster [WRN] Health check update: Degraded data redundancy: 12450/150258 objects degraded (8.286%), 7 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:22.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:22.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:22.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:22.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:22.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:22.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:22.855 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:22.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:22.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:23.006 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:23.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:23.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:23.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:23.160 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:23.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:23.264 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:23.291 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:23.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:23.367 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 36 16 55459 55443 96.2451 75.5625 0.0176405 0.0103848 2024-09-13T15:04:23.396 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:23.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:23.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:23.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:23.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:23 smithi177 bash[68594]: cluster 2024-09-13T15:04:22.901576+0000 mgr.y (mgr.26252) 1148 : cluster [DBG] pgmap v1697: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 12821/155808 objects degraded (8.229%) 2024-09-13T15:04:23.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:23 smithi177 bash[68594]: cluster 2024-09-13T15:04:22.901576+0000 mgr.y (mgr.26252) 1148 : cluster [DBG] pgmap v1697: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 12821/155808 objects degraded (8.229%) 2024-09-13T15:04:23.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:23.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:23.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:23 smithi099 bash[84576]: cluster 2024-09-13T15:04:22.901576+0000 mgr.y (mgr.26252) 1148 : cluster [DBG] pgmap v1697: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 12821/155808 objects degraded (8.229%) 2024-09-13T15:04:23.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:23 smithi099 bash[84576]: cluster 2024-09-13T15:04:22.901576+0000 mgr.y (mgr.26252) 1148 : cluster [DBG] pgmap v1697: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 12821/155808 objects degraded (8.229%) 2024-09-13T15:04:23.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:23 smithi099 bash[87147]: cluster 2024-09-13T15:04:22.901576+0000 mgr.y (mgr.26252) 1148 : cluster [DBG] pgmap v1697: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 12821/155808 objects degraded (8.229%) 2024-09-13T15:04:23.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:23 smithi099 bash[87147]: cluster 2024-09-13T15:04:22.901576+0000 mgr.y (mgr.26252) 1148 : cluster [DBG] pgmap v1697: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 12821/155808 objects degraded (8.229%) 2024-09-13T15:04:23.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:23.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:23.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:23.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:23.805 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:23.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:23.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:23.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:24.017 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1695,"stamp":"2024-09-13T15:04:18.899285+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3038517197,"num_objects":46381,"num_object_clones":0,"num_object_copies":139143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11386,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46381,"num_whiteouts":0,"num_read":344,"num_read_kb":799,"num_write":46567,"num_write_kb":2968379,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":595442,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":46519,"ondisk_log_size":46519,"up":234,"acting":234,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12244120,"kb_used_data":8212684,"kb_used_omap":353,"kb_used_meta":4031006,"kb_avail":737717096,"statfs":{"total":767960285184,"available":755422306304,"internally_reserved":0,"allocated":8409788416,"data_stored":8376580110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":361898,"internal_metadata":4127750742},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":830996480,"num_objects":12680,"num_object_clones":0,"num_object_copies":38040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3086,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12680,"num_write_kb":811520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004179"},"pg_stats":[{"pgid":"44.f","version":"1061'3055","reported_seq":3076,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:17.838984+0000","last_change":"2024-09-13T15:03:24.065298+0000","last_active":"2024-09-13T15:04:17.838984+0000","last_peered":"2024-09-13T15:04:17.838984+0000","last_clean":"2024-09-13T15:04:17.838984+0000","last_became_active":"2024-09-13T15:03:24.064467+0000","last_became_peered":"2024-09-13T15:03:24.064467+0000","last_unstale":"2024-09-13T15:04:17.838984+0000","last_undegraded":"2024-09-13T15:04:17.838984+0000","last_fullsized":"2024-09-13T15:04:17.838984+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":3055,"log_dups_size":0,"ondisk_log_size":3055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:37.327903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200212480,"num_objects":3055,"num_object_clones":0,"num_object_copies":9165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3055,"num_write_kb":195520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"44.e","version":"1061'2829","reported_seq":2849,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:14.660471+0000","last_change":"2024-09-13T15:03:24.074307+0000","last_active":"2024-09-13T15:04:14.660471+0000","last_peered":"2024-09-13T15:04:14.660471+0000","last_clean":"2024-09-13T15:04:14.660471+0000","last_became_active":"2024-09-13T15:03:24.073622+0000","last_became_peered":"2024-09-13T15:03:24.073622+0000","last_unstale":"2024-09-13T15:04:14.660471+0000","last_undegraded":"2024-09-13T15:04:14.660471+0000","last_fullsized":"2024-09-13T15:04:14.660471+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2829,"log_dups_size":0,"ondisk_log_size":2829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:21.099179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185401344,"num_objects":2829,"num_object_clones":0,"num_object_copies":8487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2829,"num_write_kb":181056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"44.d","version":"1061'2769","reported_seq":2782,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:14.057367+0000","last_change":"2024-09-13T15:03:24.074415+0000","last_active":"2024-09-13T15:04:14.057367+0000","last_peered":"2024-09-13T15:04:14.057367+0000","last_clean":"2024-09-13T15:04:14.057367+0000","last_became_active":"2024-09-13T15:03:24.074079+0000","last_became_peered":"2024-09-13T15:03:24.074079+0000","last_unstale":"2024-09-13T15:04:14.057367+0000","last_undegraded":"2024-09-13T15:04:14.057367+0000","last_fullsized":"2024-09-13T15:04:14.057367+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2769,"log_dups_size":0,"ondisk_log_size":2769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:21:31.201502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181469184,"num_objects":2769,"num_object_clones":0,"num_object_copies":8307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2769,"num_write_kb":177216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"44.c","version":"1061'3057","reported_seq":3078,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:17.144257+0000","last_change":"2024-09-13T15:03:24.074773+0000","last_active":"2024-09-13T15:04:17.144257+0000","last_peered":"2024-09-13T15:04:17.144257+0000","last_clean":"2024-09-13T15:04:17.144257+0000","last_became_active":"2024-09-13T15:03:24.073943+0000","last_became_peered":"2024-09-13T15:03:24.073943+0000","last_unstale":"2024-09-13T15:04:17.144257+0000","last_undegraded":"2024-09-13T15:04:17.144257+0000","last_fullsized":"2024-09-13T15:04:17.144257+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":3057,"log_dups_size":0,"ondisk_log_size":3057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:34:56.952158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200343552,"num_objects":3057,"num_object_clones":0,"num_object_copies":9171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3057,"num_write_kb":195648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"44.b","version":"1061'2740","reported_seq":2750,"reported_epoch":1061,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:04:14.057255+0000","last_change":"2024-09-13T15:03:47.394556+0000","last_active":"2024-09-13T15:04:14.057255+0000","last_peered":"2024-09-13T15:04:14.057255+0000","last_clean":"2024-09-13T15:03:23.031187+0000","last_became_active":"2024-09-13T15:03:24.072654+0000","last_became_peered":"2024-09-13T15:03:24.072654+0000","last_unstale":"2024-09-13T15:04:14.057255+0000","last_undegraded":"2024-09-13T15:03:47.378127+0000","last_fullsized":"2024-09-13T15:03:24.057333+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2740,"log_dups_size":0,"ondisk_log_size":2740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:28:38.432470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179568640,"num_objects":2740,"num_object_clones":0,"num_object_copies":8220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2740,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2740,"num_write_kb":175360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":2740}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"44.a","version":"1061'2874","reported_seq":2887,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:14.695783+0000","last_change":"2024-09-13T15:03:24.068238+0000","last_active":"2024-09-13T15:04:14.695783+0000","last_peered":"2024-09-13T15:04:14.695783+0000","last_clean":"2024-09-13T15:04:14.695783+0000","last_became_active":"2024-09-13T15:03:24.067501+0000","last_became_peered":"2024-09-13T15:03:24.067501+0000","last_unstale":"2024-09-13T15:04:14.695783+0000","last_undegraded":"2024-09-13T15:04:14.695783+0000","last_fullsized":"2024-09-13T15:04:14.695783+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2874,"log_dups_size":0,"ondisk_log_size":2874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:20.182063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188350464,"num_objects":2874,"num_object_clones":0,"num_object_copies":8622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2874,"num_write_kb":183936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"44.9","version":"1061'2717","reported_seq":2727,"reported_epoch":1061,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:04:14.057793+0000","last_change":"2024-09-13T15:03:47.393230+0000","last_active":"2024-09-13T15:04:14.057793+0000","last_peered":"2024-09-13T15:04:14.057793+0000","last_clean":"2024-09-13T15:03:23.031187+0000","last_became_active":"2024-09-13T15:03:24.065214+0000","last_became_peered":"2024-09-13T15:03:24.065214+0000","last_unstale":"2024-09-13T15:04:14.057793+0000","last_undegraded":"2024-09-13T15:03:47.391899+0000","last_fullsized":"2024-09-13T15:03:24.057329+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2717,"log_dups_size":0,"ondisk_log_size":2717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:03:32.533960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178061312,"num_objects":2717,"num_object_clones":0,"num_object_copies":8151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2717,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2717,"num_write_kb":173888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":2717}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"44.8","version":"1061'2720","reported_seq":2733,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:14.063014+0000","last_change":"2024-09-13T15:03:24.070447+0000","last_active":"2024-09-13T15:04:14.063014+0000","last_peered":"2024-09-13T15:04:14.063014+0000","last_clean":"2024-09-13T15:04:14.063014+0000","last_became_active":"2024-09-13T15:03:24.069765+0000","last_became_peered":"2024-09-13T15:03:24.069765+0000","last_unstale":"2024-09-13T15:04:14.063014+0000","last_undegraded":"2024-09-13T15:04:14.063014+0000","last_fullsized":"2024-09-13T15:04:14.063014+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2720,"log_dups_size":0,"ondisk_log_size":2720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:55.089422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178257920,"num_objects":2720,"num_object_clones":0,"num_object_copies":8160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2720,"num_write_kb":174080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"44.7","version":"1061'2960","reported_seq":2968,"reported_epoch":1061,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:04:14.700052+0000","last_change":"2024-09-13T15:03:47.386566+0000","last_active":"2024-09-13T15:04:14.700052+0000","last_peered":"2024-09-13T15:04:14.700052+0000","last_clean":"2024-09-13T15:03:23.031187+0000","last_became_active":"2024-09-13T15:03:24.063411+0000","last_became_peered":"2024-09-13T15:03:24.063411+0000","last_unstale":"2024-09-13T15:04:14.700052+0000","last_undegraded":"2024-09-13T15:03:47.372175+0000","last_fullsized":"2024-09-13T15:03:24.059203+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2960,"log_dups_size":0,"ondisk_log_size":2960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:07.143256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193986560,"num_objects":2960,"num_object_clones":0,"num_object_copies":8880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2960,"num_write_kb":189440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2960}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"44.6","version":"1061'2735","reported_seq":2746,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:14.027554+0000","last_change":"2024-09-13T15:03:24.070990+0000","last_active":"2024-09-13T15:04:14.027554+0000","last_peered":"2024-09-13T15:04:14.027554+0000","last_clean":"2024-09-13T15:04:14.027554+0000","last_became_active":"2024-09-13T15:03:24.070401+0000","last_became_peered":"2024-09-13T15:03:24.070401+0000","last_unstale":"2024-09-13T15:04:14.027554+0000","last_undegraded":"2024-09-13T15:04:14.027554+0000","last_fullsized":"2024-09-13T15:04:14.027554+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2735,"log_dups_size":0,"ondisk_log_size":2735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:50.863517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179240960,"num_objects":2735,"num_object_clones":0,"num_object_copies":8205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2735,"num_write_kb":175040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"44.5","version":"1061'2880","reported_seq":2893,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:14.014220+0000","last_change":"2024-09-13T15:03:24.072449+0000","last_active":"2024-09-13T15:04:14.014220+0000","last_peered":"2024-09-13T15:04:14.014220+0000","last_clean":"2024-09-13T15:04:14.014220+0000","last_became_active":"2024-09-13T15:03:24.072160+0000","last_became_peered":"2024-09-13T15:03:24.072160+0000","last_unstale":"2024-09-13T15:04:14.014220+0000","last_undegraded":"2024-09-13T15:04:14.014220+0000","last_fullsized":"2024-09-13T15:04:14.014220+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2880,"log_dups_size":0,"ondisk_log_size":2880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:38.539160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188743680,"num_objects":2880,"num_object_clones":0,"num_object_copies":8640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2880,"num_write_kb":184320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"44.4","version":"1061'3176","reported_seq":3197,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:17.851526+0000","last_change":"2024-09-13T15:03:24.068348+0000","last_active":"2024-09-13T15:04:17.851526+0000","last_peered":"2024-09-13T15:04:17.851526+0000","last_clean":"2024-09-13T15:04:17.851526+0000","last_became_active":"2024-09-13T15:03:24.067881+0000","last_became_peered":"2024-09-13T15:03:24.067881+0000","last_unstale":"2024-09-13T15:04:17.851526+0000","last_undegraded":"2024-09-13T15:04:17.851526+0000","last_fullsized":"2024-09-13T15:04:17.851526+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":3176,"log_dups_size":0,"ondisk_log_size":3176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:34:49.768741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208142336,"num_objects":3176,"num_object_clones":0,"num_object_copies":9528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3176,"num_write_kb":203264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"44.3","version":"1061'3082","reported_seq":3103,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:18.112170+0000","last_change":"2024-09-13T15:03:24.062767+0000","last_active":"2024-09-13T15:04:18.112170+0000","last_peered":"2024-09-13T15:04:18.112170+0000","last_clean":"2024-09-13T15:04:18.112170+0000","last_became_active":"2024-09-13T15:03:24.062251+0000","last_became_peered":"2024-09-13T15:03:24.062251+0000","last_unstale":"2024-09-13T15:04:18.112170+0000","last_undegraded":"2024-09-13T15:04:18.112170+0000","last_fullsized":"2024-09-13T15:04:18.112170+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":3082,"log_dups_size":0,"ondisk_log_size":3082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:02.301638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201981952,"num_objects":3082,"num_object_clones":0,"num_object_copies":9246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3082,"num_write_kb":197248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"44.2","version":"1061'2827","reported_seq":2848,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:14.658154+0000","last_change":"2024-09-13T15:03:24.067147+0000","last_active":"2024-09-13T15:04:14.658154+0000","last_peered":"2024-09-13T15:04:14.658154+0000","last_clean":"2024-09-13T15:04:14.658154+0000","last_became_active":"2024-09-13T15:03:24.066369+0000","last_became_peered":"2024-09-13T15:03:24.066369+0000","last_unstale":"2024-09-13T15:04:14.658154+0000","last_undegraded":"2024-09-13T15:04:14.658154+0000","last_fullsized":"2024-09-13T15:04:14.658154+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2827,"log_dups_size":0,"ondisk_log_size":2827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:22:12.065416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185270272,"num_objects":2827,"num_object_clones":0,"num_object_copies":8481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2827,"num_write_kb":180928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"44.1","version":"1061'2969","reported_seq":2990,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:17.164375+0000","last_change":"2024-09-13T15:03:24.065492+0000","last_active":"2024-09-13T15:04:17.164375+0000","last_peered":"2024-09-13T15:04:17.164375+0000","last_clean":"2024-09-13T15:04:17.164375+0000","last_became_active":"2024-09-13T15:03:24.065095+0000","last_became_peered":"2024-09-13T15:03:24.065095+0000","last_unstale":"2024-09-13T15:04:17.164375+0000","last_undegraded":"2024-09-13T15:04:17.164375+0000","last_fullsized":"2024-09-13T15:04:17.164375+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2969,"log_dups_size":0,"ondisk_log_size":2969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:54:06.954165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194576384,"num_objects":2969,"num_object_clones":0,"num_object_copies":8907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2969,"num_write_kb":190016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"44.0","version":"1061'2965","reported_seq":2983,"reported_epoch":1061,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:04:17.158730+0000","last_change":"2024-09-13T15:03:47.376504+0000","last_active":"2024-09-13T15:04:17.158730+0000","last_peered":"2024-09-13T15:04:17.158730+0000","last_clean":"2024-09-13T15:03:23.031187+0000","last_became_active":"2024-09-13T15:03:24.071156+0000","last_became_peered":"2024-09-13T15:03:24.071156+0000","last_unstale":"2024-09-13T15:04:17.158730+0000","last_undegraded":"2024-09-13T15:03:47.375953+0000","last_fullsized":"2024-09-13T15:03:24.050238+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":2965,"log_dups_size":0,"ondisk_log_size":2965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:56.301782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194314240,"num_objects":2965,"num_object_clones":0,"num_object_copies":8895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2965,"num_write_kb":189760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":2965}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":1517,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.979089+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T15:04:13.979089+0000","last_peered":"2024-09-13T15:04:13.979089+0000","last_clean":"2024-09-13T15:04:13.979089+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T15:04:13.979089+0000","last_undegraded":"2024-09-13T15:04:13.979089+0000","last_fullsized":"2024-09-13T15:04:13.979089+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":236,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.970048+0000","last_change":"2024-09-13T14:52:43.599071+0000","last_active":"2024-09-13T15:04:13.970048+0000","last_peered":"2024-09-13T15:04:13.970048+0000","last_clean":"2024-09-13T15:04:13.970048+0000","last_became_active":"2024-09-13T14:52:43.594847+0000","last_became_peered":"2024-09-13T14:52:43.594847+0000","last_unstale":"2024-09-13T15:04:13.970048+0000","last_undegraded":"2024-09-13T15:04:13.970048+0000","last_fullsized":"2024-09-13T15:04:13.970048+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:02.736434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":59,"reported_epoch":1060,"state":"active+undersized","last_fresh":"2024-09-13T15:03:45.165195+0000","last_change":"2024-09-13T14:57:52.527498+0000","last_active":"2024-09-13T15:03:45.165195+0000","last_peered":"2024-09-13T15:03:45.165195+0000","last_clean":"2024-09-13T14:24:16.175691+0000","last_became_active":"2024-09-13T14:57:52.527498+0000","last_became_peered":"2024-09-13T14:57:52.527498+0000","last_unstale":"2024-09-13T15:03:45.165195+0000","last_undegraded":"2024-09-13T15:03:45.165195+0000","last_fullsized":"2024-09-13T14:57:52.516910+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:20.231670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.686513+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T15:03:45.686513+0000","last_peered":"2024-09-13T15:03:45.686513+0000","last_clean":"2024-09-13T15:03:45.686513+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T15:03:45.686513+0000","last_undegraded":"2024-09-13T15:03:45.686513+0000","last_fullsized":"2024-09-13T15:03:45.686513+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1046,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.687333+0000","last_change":"2024-09-13T14:47:46.337827+0000","last_active":"2024-09-13T15:03:45.687333+0000","last_peered":"2024-09-13T15:03:45.687333+0000","last_clean":"2024-09-13T15:03:45.687333+0000","last_became_active":"2024-09-13T14:47:46.321140+0000","last_became_peered":"2024-09-13T14:47:46.321140+0000","last_unstale":"2024-09-13T15:03:45.687333+0000","last_undegraded":"2024-09-13T15:03:45.687333+0000","last_fullsized":"2024-09-13T15:03:45.687333+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:15.997382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":226,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.970207+0000","last_change":"2024-09-13T14:57:52.524068+0000","last_active":"2024-09-13T15:04:13.970207+0000","last_peered":"2024-09-13T15:04:13.970207+0000","last_clean":"2024-09-13T14:57:46.744631+0000","last_became_active":"2024-09-13T14:57:52.524068+0000","last_became_peered":"2024-09-13T14:57:52.524068+0000","last_unstale":"2024-09-13T15:04:13.970207+0000","last_undegraded":"2024-09-13T15:04:13.970207+0000","last_fullsized":"2024-09-13T14:57:52.510427+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:11:10.935170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1091,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.687156+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T15:03:45.687156+0000","last_peered":"2024-09-13T15:03:45.687156+0000","last_clean":"2024-09-13T15:03:45.687156+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T15:03:45.687156+0000","last_undegraded":"2024-09-13T15:03:45.687156+0000","last_fullsized":"2024-09-13T15:03:45.687156+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":240,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.970713+0000","last_change":"2024-09-13T14:57:52.608274+0000","last_active":"2024-09-13T15:04:13.970713+0000","last_peered":"2024-09-13T15:04:13.970713+0000","last_clean":"2024-09-13T14:57:46.744660+0000","last_became_active":"2024-09-13T14:57:52.608274+0000","last_became_peered":"2024-09-13T14:57:52.608274+0000","last_unstale":"2024-09-13T15:04:13.970713+0000","last_undegraded":"2024-09-13T15:04:13.970713+0000","last_fullsized":"2024-09-13T14:57:52.510976+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:30.239753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1524,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.975721+0000","last_change":"2024-09-13T14:47:46.330059+0000","last_active":"2024-09-13T15:04:13.975721+0000","last_peered":"2024-09-13T15:04:13.975721+0000","last_clean":"2024-09-13T15:04:13.975721+0000","last_became_active":"2024-09-13T14:47:46.329344+0000","last_became_peered":"2024-09-13T14:47:46.329344+0000","last_unstale":"2024-09-13T15:04:13.975721+0000","last_undegraded":"2024-09-13T15:04:13.975721+0000","last_fullsized":"2024-09-13T15:04:13.975721+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:58:43.470176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1432,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.973812+0000","last_change":"2024-09-13T14:57:52.531642+0000","last_active":"2024-09-13T15:04:13.973812+0000","last_peered":"2024-09-13T15:04:13.973812+0000","last_clean":"2024-09-13T14:57:42.823387+0000","last_became_active":"2024-09-13T14:57:52.531642+0000","last_became_peered":"2024-09-13T14:57:52.531642+0000","last_unstale":"2024-09-13T15:04:13.973812+0000","last_undegraded":"2024-09-13T15:04:13.973812+0000","last_fullsized":"2024-09-13T14:57:52.523032+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:46.509860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":236,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.970105+0000","last_change":"2024-09-13T14:52:43.597400+0000","last_active":"2024-09-13T15:04:13.970105+0000","last_peered":"2024-09-13T15:04:13.970105+0000","last_clean":"2024-09-13T15:04:13.970105+0000","last_became_active":"2024-09-13T14:52:43.596294+0000","last_became_peered":"2024-09-13T14:52:43.596294+0000","last_unstale":"2024-09-13T15:04:13.970105+0000","last_undegraded":"2024-09-13T15:04:13.970105+0000","last_fullsized":"2024-09-13T15:04:13.970105+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:35.218171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1353,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.978275+0000","last_change":"2024-09-13T14:57:52.537575+0000","last_active":"2024-09-13T15:04:13.978275+0000","last_peered":"2024-09-13T15:04:13.978275+0000","last_clean":"2024-09-13T14:52:40.121284+0000","last_became_active":"2024-09-13T14:57:52.537575+0000","last_became_peered":"2024-09-13T14:57:52.537575+0000","last_unstale":"2024-09-13T15:04:13.978275+0000","last_undegraded":"2024-09-13T15:04:13.978275+0000","last_fullsized":"2024-09-13T14:57:52.526034+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:01.838090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":130,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.973205+0000","last_change":"2024-09-13T14:57:52.538971+0000","last_active":"2024-09-13T15:04:13.973205+0000","last_peered":"2024-09-13T15:04:13.973205+0000","last_clean":"2024-09-13T14:57:43.384926+0000","last_became_active":"2024-09-13T14:57:52.538971+0000","last_became_peered":"2024-09-13T14:57:52.538971+0000","last_unstale":"2024-09-13T15:04:13.973205+0000","last_undegraded":"2024-09-13T15:04:13.973205+0000","last_fullsized":"2024-09-13T14:57:52.518582+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:08:15.593435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1514,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.976499+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T15:04:13.976499+0000","last_peered":"2024-09-13T15:04:13.976499+0000","last_clean":"2024-09-13T15:04:13.976499+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T15:04:13.976499+0000","last_undegraded":"2024-09-13T15:04:13.976499+0000","last_fullsized":"2024-09-13T15:04:13.976499+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1346,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.975835+0000","last_change":"2024-09-13T14:57:52.532766+0000","last_active":"2024-09-13T15:04:13.975835+0000","last_peered":"2024-09-13T15:04:13.975835+0000","last_clean":"2024-09-13T14:52:40.120556+0000","last_became_active":"2024-09-13T14:57:52.532766+0000","last_became_peered":"2024-09-13T14:57:52.532766+0000","last_unstale":"2024-09-13T15:04:13.975835+0000","last_undegraded":"2024-09-13T15:04:13.975835+0000","last_fullsized":"2024-09-13T14:57:52.515253+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:16:40.443577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1048,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.165111+0000","last_change":"2024-09-13T14:52:54.183175+0000","last_active":"2024-09-13T15:03:45.165111+0000","last_peered":"2024-09-13T15:03:45.165111+0000","last_clean":"2024-09-13T15:03:45.165111+0000","last_became_active":"2024-09-13T14:52:54.182514+0000","last_became_peered":"2024-09-13T14:52:54.182514+0000","last_unstale":"2024-09-13T15:03:45.165111+0000","last_undegraded":"2024-09-13T15:03:45.165111+0000","last_fullsized":"2024-09-13T15:03:45.165111+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:55.900845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1326,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.972832+0000","last_change":"2024-09-13T14:57:52.529977+0000","last_active":"2024-09-13T15:04:13.972832+0000","last_peered":"2024-09-13T15:04:13.972832+0000","last_clean":"2024-09-13T14:52:40.125209+0000","last_became_active":"2024-09-13T14:57:52.529977+0000","last_became_peered":"2024-09-13T14:57:52.529977+0000","last_unstale":"2024-09-13T15:04:13.972832+0000","last_undegraded":"2024-09-13T15:04:13.972832+0000","last_fullsized":"2024-09-13T14:57:52.515419+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:12:11.447938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1508,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.979129+0000","last_change":"2024-09-13T14:57:52.524642+0000","last_active":"2024-09-13T15:04:13.979129+0000","last_peered":"2024-09-13T15:04:13.979129+0000","last_clean":"2024-09-13T14:57:42.887589+0000","last_became_active":"2024-09-13T14:57:52.524642+0000","last_became_peered":"2024-09-13T14:57:52.524642+0000","last_unstale":"2024-09-13T15:04:13.979129+0000","last_undegraded":"2024-09-13T15:04:13.979129+0000","last_fullsized":"2024-09-13T14:57:52.514403+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:37.383913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":663,"reported_epoch":1060,"state":"active+undersized","last_fresh":"2024-09-13T15:03:45.686541+0000","last_change":"2024-09-13T14:57:52.526180+0000","last_active":"2024-09-13T15:03:45.686541+0000","last_peered":"2024-09-13T15:03:45.686541+0000","last_clean":"2024-09-13T14:24:17.188743+0000","last_became_active":"2024-09-13T14:57:52.526180+0000","last_became_peered":"2024-09-13T14:57:52.526180+0000","last_unstale":"2024-09-13T15:03:45.686541+0000","last_undegraded":"2024-09-13T15:03:45.686541+0000","last_fullsized":"2024-09-13T14:57:52.516434+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:47:35.364999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1035,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.686963+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T15:03:45.686963+0000","last_peered":"2024-09-13T15:03:45.686963+0000","last_clean":"2024-09-13T15:03:45.686963+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T15:03:45.686963+0000","last_undegraded":"2024-09-13T15:03:45.686963+0000","last_fullsized":"2024-09-13T15:03:45.686963+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1079,"reported_epoch":1060,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:03:45.687509+0000","last_change":"2024-09-13T14:57:52.546472+0000","last_active":"2024-09-13T15:03:45.687509+0000","last_peered":"2024-09-13T15:03:45.687509+0000","last_clean":"2024-09-13T14:57:42.894880+0000","last_became_active":"2024-09-13T14:57:52.546472+0000","last_became_peered":"2024-09-13T14:57:52.546472+0000","last_unstale":"2024-09-13T15:03:45.687509+0000","last_undegraded":"2024-09-13T14:57:52.514987+0000","last_fullsized":"2024-09-13T14:57:52.514866+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:32.888371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":126,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.978310+0000","last_change":"2024-09-13T14:52:43.586530+0000","last_active":"2024-09-13T15:04:13.978310+0000","last_peered":"2024-09-13T15:04:13.978310+0000","last_clean":"2024-09-13T15:04:13.978310+0000","last_became_active":"2024-09-13T14:52:43.586107+0000","last_became_peered":"2024-09-13T14:52:43.586107+0000","last_unstale":"2024-09-13T15:04:13.978310+0000","last_undegraded":"2024-09-13T15:04:13.978310+0000","last_fullsized":"2024-09-13T15:04:13.978310+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:18.130768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":233,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.972407+0000","last_change":"2024-09-13T14:47:46.344536+0000","last_active":"2024-09-13T15:04:13.972407+0000","last_peered":"2024-09-13T15:04:13.972407+0000","last_clean":"2024-09-13T15:04:13.972407+0000","last_became_active":"2024-09-13T14:47:46.321131+0000","last_became_peered":"2024-09-13T14:47:46.321131+0000","last_unstale":"2024-09-13T15:04:13.972407+0000","last_undegraded":"2024-09-13T15:04:13.972407+0000","last_fullsized":"2024-09-13T15:04:13.972407+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:31.114472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":237,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.970863+0000","last_change":"2024-09-13T14:52:43.600569+0000","last_active":"2024-09-13T15:04:13.970863+0000","last_peered":"2024-09-13T15:04:13.970863+0000","last_clean":"2024-09-13T15:04:13.970863+0000","last_became_active":"2024-09-13T14:52:43.599764+0000","last_became_peered":"2024-09-13T14:52:43.599764+0000","last_unstale":"2024-09-13T15:04:13.970863+0000","last_undegraded":"2024-09-13T15:04:13.970863+0000","last_fullsized":"2024-09-13T15:04:13.970863+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:53.934456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":223,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.970522+0000","last_change":"2024-09-13T14:47:46.318785+0000","last_active":"2024-09-13T15:04:13.970522+0000","last_peered":"2024-09-13T15:04:13.970522+0000","last_clean":"2024-09-13T15:04:13.970522+0000","last_became_active":"2024-09-13T14:47:46.318438+0000","last_became_peered":"2024-09-13T14:47:46.318438+0000","last_unstale":"2024-09-13T15:04:13.970522+0000","last_undegraded":"2024-09-13T15:04:13.970522+0000","last_fullsized":"2024-09-13T15:04:13.970522+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:44.745378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1031,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.165148+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T15:03:45.165148+0000","last_peered":"2024-09-13T15:03:45.165148+0000","last_clean":"2024-09-13T15:03:45.165148+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T15:03:45.165148+0000","last_undegraded":"2024-09-13T15:03:45.165148+0000","last_fullsized":"2024-09-13T15:03:45.165148+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1565,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.976578+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T15:04:13.976578+0000","last_peered":"2024-09-13T15:04:13.976578+0000","last_clean":"2024-09-13T15:04:13.976578+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T15:04:13.976578+0000","last_undegraded":"2024-09-13T15:04:13.976578+0000","last_fullsized":"2024-09-13T15:04:13.976578+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":660,"reported_epoch":1060,"state":"active+undersized","last_fresh":"2024-09-13T15:03:45.686462+0000","last_change":"2024-09-13T14:57:52.538584+0000","last_active":"2024-09-13T15:03:45.686462+0000","last_peered":"2024-09-13T15:03:45.686462+0000","last_clean":"2024-09-13T14:35:50.051972+0000","last_became_active":"2024-09-13T14:57:52.538584+0000","last_became_peered":"2024-09-13T14:57:52.538584+0000","last_unstale":"2024-09-13T15:03:45.686462+0000","last_undegraded":"2024-09-13T15:03:45.686462+0000","last_fullsized":"2024-09-13T14:57:52.515637+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:52.637609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":127,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.979929+0000","last_change":"2024-09-13T14:52:43.592967+0000","last_active":"2024-09-13T15:04:13.979929+0000","last_peered":"2024-09-13T15:04:13.979929+0000","last_clean":"2024-09-13T15:04:13.979929+0000","last_became_active":"2024-09-13T14:52:43.592431+0000","last_became_peered":"2024-09-13T14:52:43.592431+0000","last_unstale":"2024-09-13T15:04:13.979929+0000","last_undegraded":"2024-09-13T15:04:13.979929+0000","last_fullsized":"2024-09-13T15:04:13.979929+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:56.187568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":70,"reported_epoch":1060,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:03:45.687688+0000","last_change":"2024-09-13T14:57:52.608972+0000","last_active":"2024-09-13T15:03:45.687688+0000","last_peered":"2024-09-13T15:03:45.687688+0000","last_clean":"2024-09-13T14:24:20.313173+0000","last_became_active":"2024-09-13T14:57:52.608972+0000","last_became_peered":"2024-09-13T14:57:52.608972+0000","last_unstale":"2024-09-13T15:03:45.687688+0000","last_undegraded":"2024-09-13T14:57:52.519236+0000","last_fullsized":"2024-09-13T14:57:52.518945+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:56.976455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1495,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.975498+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T15:04:13.975498+0000","last_peered":"2024-09-13T15:04:13.975498+0000","last_clean":"2024-09-13T15:04:13.975498+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T15:04:13.975498+0000","last_undegraded":"2024-09-13T15:04:13.975498+0000","last_fullsized":"2024-09-13T15:04:13.975498+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1067,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.687600+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T15:03:45.687600+0000","last_peered":"2024-09-13T15:03:45.687600+0000","last_clean":"2024-09-13T15:03:45.687600+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T15:03:45.687600+0000","last_undegraded":"2024-09-13T15:03:45.687600+0000","last_fullsized":"2024-09-13T15:03:45.687600+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":127,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.972865+0000","last_change":"2024-09-13T14:52:43.583860+0000","last_active":"2024-09-13T15:04:13.972865+0000","last_peered":"2024-09-13T15:04:13.972865+0000","last_clean":"2024-09-13T15:04:13.972865+0000","last_became_active":"2024-09-13T14:52:43.583055+0000","last_became_peered":"2024-09-13T14:52:43.583055+0000","last_unstale":"2024-09-13T15:04:13.972865+0000","last_undegraded":"2024-09-13T15:04:13.972865+0000","last_fullsized":"2024-09-13T15:04:13.972865+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:55.113648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.977603+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T15:04:13.977603+0000","last_peered":"2024-09-13T15:04:13.977603+0000","last_clean":"2024-09-13T15:04:13.977603+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T15:04:13.977603+0000","last_undegraded":"2024-09-13T15:04:13.977603+0000","last_fullsized":"2024-09-13T15:04:13.977603+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":127,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.973267+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T15:04:13.973267+0000","last_peered":"2024-09-13T15:04:13.973267+0000","last_clean":"2024-09-13T15:04:13.973267+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T15:04:13.973267+0000","last_undegraded":"2024-09-13T15:04:13.973267+0000","last_fullsized":"2024-09-13T15:04:13.973267+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1039,"reported_epoch":1060,"state":"active+undersized","last_fresh":"2024-09-13T15:03:45.687431+0000","last_change":"2024-09-13T14:57:52.528372+0000","last_active":"2024-09-13T15:03:45.687431+0000","last_peered":"2024-09-13T15:03:45.687431+0000","last_clean":"2024-09-13T14:57:42.822230+0000","last_became_active":"2024-09-13T14:57:52.528372+0000","last_became_peered":"2024-09-13T14:57:52.528372+0000","last_unstale":"2024-09-13T15:03:45.687431+0000","last_undegraded":"2024-09-13T15:03:45.687431+0000","last_fullsized":"2024-09-13T14:57:52.515808+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:53:43.143322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":131,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.973098+0000","last_change":"2024-09-13T14:57:52.535638+0000","last_active":"2024-09-13T15:04:13.973098+0000","last_peered":"2024-09-13T15:04:13.973098+0000","last_clean":"2024-09-13T14:57:43.385736+0000","last_became_active":"2024-09-13T14:57:52.535638+0000","last_became_peered":"2024-09-13T14:57:52.535638+0000","last_unstale":"2024-09-13T15:04:13.973098+0000","last_undegraded":"2024-09-13T15:04:13.973098+0000","last_fullsized":"2024-09-13T14:57:52.526096+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:43.910330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":223,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.970937+0000","last_change":"2024-09-13T14:47:46.324072+0000","last_active":"2024-09-13T15:04:13.970937+0000","last_peered":"2024-09-13T15:04:13.970937+0000","last_clean":"2024-09-13T15:04:13.970937+0000","last_became_active":"2024-09-13T14:47:46.322613+0000","last_became_peered":"2024-09-13T14:47:46.322613+0000","last_unstale":"2024-09-13T15:04:13.970937+0000","last_undegraded":"2024-09-13T15:04:13.970937+0000","last_fullsized":"2024-09-13T15:04:13.970937+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:45.314706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1057,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.686486+0000","last_change":"2024-09-13T14:52:43.613106+0000","last_active":"2024-09-13T15:03:45.686486+0000","last_peered":"2024-09-13T15:03:45.686486+0000","last_clean":"2024-09-13T15:03:45.686486+0000","last_became_active":"2024-09-13T14:52:43.594081+0000","last_became_peered":"2024-09-13T14:52:43.594081+0000","last_unstale":"2024-09-13T15:03:45.686486+0000","last_undegraded":"2024-09-13T15:03:45.686486+0000","last_fullsized":"2024-09-13T15:03:45.686486+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:11.121629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.979927+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T15:04:13.979927+0000","last_peered":"2024-09-13T15:04:13.979927+0000","last_clean":"2024-09-13T15:04:13.979927+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T15:04:13.979927+0000","last_undegraded":"2024-09-13T15:04:13.979927+0000","last_fullsized":"2024-09-13T15:04:13.979927+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":127,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.972992+0000","last_change":"2024-09-13T14:52:43.582732+0000","last_active":"2024-09-13T15:04:13.972992+0000","last_peered":"2024-09-13T15:04:13.972992+0000","last_clean":"2024-09-13T15:04:13.972992+0000","last_became_active":"2024-09-13T14:52:43.582384+0000","last_became_peered":"2024-09-13T14:52:43.582384+0000","last_unstale":"2024-09-13T15:04:13.972992+0000","last_undegraded":"2024-09-13T15:04:13.972992+0000","last_fullsized":"2024-09-13T15:04:13.972992+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:06.715139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1408,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.976135+0000","last_change":"2024-09-13T14:57:52.533226+0000","last_active":"2024-09-13T15:04:13.976135+0000","last_peered":"2024-09-13T15:04:13.976135+0000","last_clean":"2024-09-13T14:57:42.822755+0000","last_became_active":"2024-09-13T14:57:52.533226+0000","last_became_peered":"2024-09-13T14:57:52.533226+0000","last_unstale":"2024-09-13T15:04:13.976135+0000","last_undegraded":"2024-09-13T15:04:13.976135+0000","last_fullsized":"2024-09-13T14:57:52.518902+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:16:10.437860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1530,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.976396+0000","last_change":"2024-09-13T14:52:43.595169+0000","last_active":"2024-09-13T15:04:13.976396+0000","last_peered":"2024-09-13T15:04:13.976396+0000","last_clean":"2024-09-13T15:04:13.976396+0000","last_became_active":"2024-09-13T14:52:43.594586+0000","last_became_peered":"2024-09-13T14:52:43.594586+0000","last_unstale":"2024-09-13T15:04:13.976396+0000","last_undegraded":"2024-09-13T15:04:13.976396+0000","last_fullsized":"2024-09-13T15:04:13.976396+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:53.946025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1535,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.975433+0000","last_change":"2024-09-13T14:57:52.526901+0000","last_active":"2024-09-13T15:04:13.975433+0000","last_peered":"2024-09-13T15:04:13.975433+0000","last_clean":"2024-09-13T14:57:42.887232+0000","last_became_active":"2024-09-13T14:57:52.526901+0000","last_became_peered":"2024-09-13T14:57:52.526901+0000","last_unstale":"2024-09-13T15:04:13.975433+0000","last_undegraded":"2024-09-13T15:04:13.975433+0000","last_fullsized":"2024-09-13T14:57:52.514561+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:47:39.792742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1067,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.686741+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T15:03:45.686741+0000","last_peered":"2024-09-13T15:03:45.686741+0000","last_clean":"2024-09-13T15:03:45.686741+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T15:03:45.686741+0000","last_undegraded":"2024-09-13T15:03:45.686741+0000","last_fullsized":"2024-09-13T15:03:45.686741+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1038,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.165139+0000","last_change":"2024-09-13T14:47:46.320830+0000","last_active":"2024-09-13T15:03:45.165139+0000","last_peered":"2024-09-13T15:03:45.165139+0000","last_clean":"2024-09-13T15:03:45.165139+0000","last_became_active":"2024-09-13T14:47:46.320443+0000","last_became_peered":"2024-09-13T14:47:46.320443+0000","last_unstale":"2024-09-13T15:03:45.165139+0000","last_undegraded":"2024-09-13T15:03:45.165139+0000","last_fullsized":"2024-09-13T15:03:45.165139+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:46.279580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":662,"reported_epoch":1060,"state":"active+undersized","last_fresh":"2024-09-13T15:03:45.686563+0000","last_change":"2024-09-13T14:57:52.522034+0000","last_active":"2024-09-13T15:03:45.686563+0000","last_peered":"2024-09-13T15:03:45.686563+0000","last_clean":"2024-09-13T14:24:16.175691+0000","last_became_active":"2024-09-13T14:57:52.522034+0000","last_became_peered":"2024-09-13T14:57:52.522034+0000","last_unstale":"2024-09-13T15:03:45.686563+0000","last_undegraded":"2024-09-13T15:03:45.686563+0000","last_fullsized":"2024-09-13T14:57:52.514676+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:15.687612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1028,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.687136+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T15:03:45.687136+0000","last_peered":"2024-09-13T15:03:45.687136+0000","last_clean":"2024-09-13T15:03:45.687136+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T15:03:45.687136+0000","last_undegraded":"2024-09-13T15:03:45.687136+0000","last_fullsized":"2024-09-13T15:03:45.687136+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1135,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.686310+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T15:03:45.686310+0000","last_peered":"2024-09-13T15:03:45.686310+0000","last_clean":"2024-09-13T15:03:45.686310+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T15:03:45.686310+0000","last_undegraded":"2024-09-13T15:03:45.686310+0000","last_fullsized":"2024-09-13T15:03:45.686310+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":131,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.979968+0000","last_change":"2024-09-13T14:57:52.537590+0000","last_active":"2024-09-13T15:04:13.979968+0000","last_peered":"2024-09-13T15:04:13.979968+0000","last_clean":"2024-09-13T14:57:43.386126+0000","last_became_active":"2024-09-13T14:57:52.537590+0000","last_became_peered":"2024-09-13T14:57:52.537590+0000","last_unstale":"2024-09-13T15:04:13.979968+0000","last_undegraded":"2024-09-13T15:04:13.979968+0000","last_fullsized":"2024-09-13T14:57:52.527356+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:03.099081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1330,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.972560+0000","last_change":"2024-09-13T14:57:52.532557+0000","last_active":"2024-09-13T15:04:13.972560+0000","last_peered":"2024-09-13T15:04:13.972560+0000","last_clean":"2024-09-13T14:52:40.121056+0000","last_became_active":"2024-09-13T14:57:52.532557+0000","last_became_peered":"2024-09-13T14:57:52.532557+0000","last_unstale":"2024-09-13T15:04:13.972560+0000","last_undegraded":"2024-09-13T15:04:13.972560+0000","last_fullsized":"2024-09-13T14:57:52.514161+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:26:26.835138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":237,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.971304+0000","last_change":"2024-09-13T14:52:43.599179+0000","last_active":"2024-09-13T15:04:13.971304+0000","last_peered":"2024-09-13T15:04:13.971304+0000","last_clean":"2024-09-13T15:04:13.971304+0000","last_became_active":"2024-09-13T14:52:43.598166+0000","last_became_peered":"2024-09-13T14:52:43.598166+0000","last_unstale":"2024-09-13T15:04:13.971304+0000","last_undegraded":"2024-09-13T15:04:13.971304+0000","last_fullsized":"2024-09-13T15:04:13.971304+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:08.647786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":223,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.970444+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T15:04:13.970444+0000","last_peered":"2024-09-13T15:04:13.970444+0000","last_clean":"2024-09-13T15:04:13.970444+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T15:04:13.970444+0000","last_undegraded":"2024-09-13T15:04:13.970444+0000","last_fullsized":"2024-09-13T15:04:13.970444+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":237,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.971281+0000","last_change":"2024-09-13T14:52:43.597595+0000","last_active":"2024-09-13T15:04:13.971281+0000","last_peered":"2024-09-13T15:04:13.971281+0000","last_clean":"2024-09-13T15:04:13.971281+0000","last_became_active":"2024-09-13T14:52:43.596592+0000","last_became_peered":"2024-09-13T14:52:43.596592+0000","last_unstale":"2024-09-13T15:04:13.971281+0000","last_undegraded":"2024-09-13T15:04:13.971281+0000","last_fullsized":"2024-09-13T15:04:13.971281+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:56.532603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1076,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.165029+0000","last_change":"2024-09-13T14:52:43.596429+0000","last_active":"2024-09-13T15:03:45.165029+0000","last_peered":"2024-09-13T15:03:45.165029+0000","last_clean":"2024-09-13T15:03:45.165029+0000","last_became_active":"2024-09-13T14:52:43.594614+0000","last_became_peered":"2024-09-13T14:52:43.594614+0000","last_unstale":"2024-09-13T15:03:45.165029+0000","last_undegraded":"2024-09-13T15:03:45.165029+0000","last_fullsized":"2024-09-13T15:03:45.165029+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:41.970225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":227,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.972229+0000","last_change":"2024-09-13T14:57:52.541208+0000","last_active":"2024-09-13T15:04:13.972229+0000","last_peered":"2024-09-13T15:04:13.972229+0000","last_clean":"2024-09-13T14:57:42.819963+0000","last_became_active":"2024-09-13T14:57:52.541208+0000","last_became_peered":"2024-09-13T14:57:52.541208+0000","last_unstale":"2024-09-13T15:04:13.972229+0000","last_undegraded":"2024-09-13T15:04:13.972229+0000","last_fullsized":"2024-09-13T14:57:52.511806+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:01.484625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":112,"reported_epoch":1061,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:04:13.975887+0000","last_change":"2024-09-13T14:57:52.608759+0000","last_active":"2024-09-13T15:04:13.975887+0000","last_peered":"2024-09-13T15:04:13.975887+0000","last_clean":"2024-09-13T14:24:20.312402+0000","last_became_active":"2024-09-13T14:57:52.608759+0000","last_became_peered":"2024-09-13T14:57:52.608759+0000","last_unstale":"2024-09-13T15:04:13.975887+0000","last_undegraded":"2024-09-13T14:57:52.520273+0000","last_fullsized":"2024-09-13T14:57:52.520078+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:53.804929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1551,"reported_epoch":1061,"state":"active+clean","last_fresh":"2024-09-13T15:04:13.975588+0000","last_change":"2024-09-13T14:52:54.185117+0000","last_active":"2024-09-13T15:04:13.975588+0000","last_peered":"2024-09-13T15:04:13.975588+0000","last_clean":"2024-09-13T15:04:13.975588+0000","last_became_active":"2024-09-13T14:52:54.184712+0000","last_became_peered":"2024-09-13T14:52:54.184712+0000","last_unstale":"2024-09-13T15:04:13.975588+0000","last_undegraded":"2024-09-13T15:04:13.975588+0000","last_fullsized":"2024-09-13T15:04:13.975588+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:32.654401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1022,"reported_epoch":1060,"state":"active+undersized","last_fresh":"2024-09-13T15:03:45.686911+0000","last_change":"2024-09-13T14:57:52.523302+0000","last_active":"2024-09-13T15:03:45.686911+0000","last_peered":"2024-09-13T15:03:45.686911+0000","last_clean":"2024-09-13T14:57:42.821860+0000","last_became_active":"2024-09-13T14:57:52.523302+0000","last_became_peered":"2024-09-13T14:57:52.523302+0000","last_unstale":"2024-09-13T15:03:45.686911+0000","last_undegraded":"2024-09-13T15:03:45.686911+0000","last_fullsized":"2024-09-13T14:57:52.514457+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.134976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1038,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.164955+0000","last_change":"2024-09-13T14:52:43.581604+0000","last_active":"2024-09-13T15:03:45.164955+0000","last_peered":"2024-09-13T15:03:45.164955+0000","last_clean":"2024-09-13T15:03:45.164955+0000","last_became_active":"2024-09-13T14:52:43.579981+0000","last_became_peered":"2024-09-13T14:52:43.579981+0000","last_unstale":"2024-09-13T15:03:45.164955+0000","last_undegraded":"2024-09-13T15:03:45.164955+0000","last_fullsized":"2024-09-13T15:03:45.164955+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:59.366292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":1003,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.686352+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T15:03:45.686352+0000","last_peered":"2024-09-13T15:03:45.686352+0000","last_clean":"2024-09-13T15:03:45.686352+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T15:03:45.686352+0000","last_undegraded":"2024-09-13T15:03:45.686352+0000","last_fullsized":"2024-09-13T15:03:45.686352+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":241,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.970594+0000","last_change":"2024-09-13T14:57:52.538489+0000","last_active":"2024-09-13T15:04:13.970594+0000","last_peered":"2024-09-13T15:04:13.970594+0000","last_clean":"2024-09-13T14:57:42.820043+0000","last_became_active":"2024-09-13T14:57:52.538489+0000","last_became_peered":"2024-09-13T14:57:52.538489+0000","last_unstale":"2024-09-13T15:04:13.970594+0000","last_undegraded":"2024-09-13T15:04:13.970594+0000","last_fullsized":"2024-09-13T14:57:52.512057+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:24.504837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1094,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.687453+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T15:03:45.687453+0000","last_peered":"2024-09-13T15:03:45.687453+0000","last_clean":"2024-09-13T15:03:45.687453+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T15:03:45.687453+0000","last_undegraded":"2024-09-13T15:03:45.687453+0000","last_fullsized":"2024-09-13T15:03:45.687453+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":997,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.686304+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T15:03:45.686304+0000","last_peered":"2024-09-13T15:03:45.686304+0000","last_clean":"2024-09-13T15:03:45.686304+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T15:03:45.686304+0000","last_undegraded":"2024-09-13T15:03:45.686304+0000","last_fullsized":"2024-09-13T15:03:45.686304+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1065,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.687516+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T15:03:45.687516+0000","last_peered":"2024-09-13T15:03:45.687516+0000","last_clean":"2024-09-13T15:03:45.687516+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T15:03:45.687516+0000","last_undegraded":"2024-09-13T15:03:45.687516+0000","last_fullsized":"2024-09-13T15:03:45.687516+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":1031,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.165281+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T15:03:45.165281+0000","last_peered":"2024-09-13T15:03:45.165281+0000","last_clean":"2024-09-13T15:03:45.165281+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T15:03:45.165281+0000","last_undegraded":"2024-09-13T15:03:45.165281+0000","last_fullsized":"2024-09-13T15:03:45.165281+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1300,"reported_epoch":1060,"state":"active+undersized","last_fresh":"2024-09-13T15:03:45.687460+0000","last_change":"2024-09-13T14:57:52.531820+0000","last_active":"2024-09-13T15:03:45.687460+0000","last_peered":"2024-09-13T15:03:45.687460+0000","last_clean":"2024-09-13T14:52:40.124168+0000","last_became_active":"2024-09-13T14:57:52.531820+0000","last_became_peered":"2024-09-13T14:57:52.531820+0000","last_unstale":"2024-09-13T15:03:45.687460+0000","last_undegraded":"2024-09-13T15:03:45.687460+0000","last_fullsized":"2024-09-13T14:57:52.517742+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:16.785361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1038,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.686742+0000","last_change":"2024-09-13T14:47:46.322428+0000","last_active":"2024-09-13T15:03:45.686742+0000","last_peered":"2024-09-13T15:03:45.686742+0000","last_clean":"2024-09-13T15:03:45.686742+0000","last_became_active":"2024-09-13T14:47:46.321955+0000","last_became_peered":"2024-09-13T14:47:46.321955+0000","last_unstale":"2024-09-13T15:03:45.686742+0000","last_undegraded":"2024-09-13T15:03:45.686742+0000","last_fullsized":"2024-09-13T15:03:45.686742+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:47.614065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1508,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.976493+0000","last_change":"2024-09-13T14:57:52.542172+0000","last_active":"2024-09-13T15:04:13.976493+0000","last_peered":"2024-09-13T15:04:13.976493+0000","last_clean":"2024-09-13T14:57:42.887703+0000","last_became_active":"2024-09-13T14:57:52.542172+0000","last_became_peered":"2024-09-13T14:57:52.542172+0000","last_unstale":"2024-09-13T15:04:13.976493+0000","last_undegraded":"2024-09-13T15:04:13.976493+0000","last_fullsized":"2024-09-13T14:57:52.522300+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:26.613968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1509,"reported_epoch":1061,"state":"active+undersized","last_fresh":"2024-09-13T15:04:13.975714+0000","last_change":"2024-09-13T14:57:52.534865+0000","last_active":"2024-09-13T15:04:13.975714+0000","last_peered":"2024-09-13T15:04:13.975714+0000","last_clean":"2024-09-13T14:57:42.887357+0000","last_became_active":"2024-09-13T14:57:52.534865+0000","last_became_peered":"2024-09-13T14:57:52.534865+0000","last_unstale":"2024-09-13T15:04:13.975714+0000","last_undegraded":"2024-09-13T15:04:13.975714+0000","last_fullsized":"2024-09-13T14:57:52.517042+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:17.508856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1021,"reported_epoch":1060,"state":"active+undersized","last_fresh":"2024-09-13T15:03:45.687292+0000","last_change":"2024-09-13T14:57:52.523699+0000","last_active":"2024-09-13T15:03:45.687292+0000","last_peered":"2024-09-13T15:03:45.687292+0000","last_clean":"2024-09-13T14:57:42.822145+0000","last_became_active":"2024-09-13T14:57:52.523699+0000","last_became_peered":"2024-09-13T14:57:52.523699+0000","last_unstale":"2024-09-13T15:03:45.687292+0000","last_undegraded":"2024-09-13T15:03:45.687292+0000","last_fullsized":"2024-09-13T14:57:52.515367+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:46:32.001179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1071,"reported_epoch":1060,"state":"active+clean","last_fresh":"2024-09-13T15:03:45.687002+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T15:03:45.687002+0000","last_peered":"2024-09-13T15:03:45.687002+0000","last_clean":"2024-09-13T15:03:45.687002+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T15:03:45.687002+0000","last_undegraded":"2024-09-13T15:03:45.687002+0000","last_fullsized":"2024-09-13T15:03:45.687002+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1311,"reported_epoch":1060,"state":"active+undersized","last_fresh":"2024-09-13T15:03:45.686754+0000","last_change":"2024-09-13T14:57:52.525160+0000","last_active":"2024-09-13T15:03:45.686754+0000","last_peered":"2024-09-13T15:03:45.686754+0000","last_clean":"2024-09-13T14:52:43.593692+0000","last_became_active":"2024-09-13T14:57:52.525160+0000","last_became_peered":"2024-09-13T14:57:52.525160+0000","last_unstale":"2024-09-13T15:03:45.686754+0000","last_undegraded":"2024-09-13T15:03:45.686754+0000","last_fullsized":"2024-09-13T14:57:52.514147+0000","mapping_epoch":1034,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:40.347221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":44,"num_pg":16,"stat_sum":{"num_bytes":3037921280,"num_objects":46355,"num_object_clones":0,"num_object_copies":139065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":46355,"num_write_kb":2966720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8417116160,"data_stored":8417116160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":46355,"ondisk_log_size":46355,"up":44,"acting":44,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":23,"acting":23,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":83,"acting":83,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":81,"acting":81,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398038,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1860128,"kb_used_data":1257752,"kb_used_omap":1,"kb_used_meta":602366,"kb_avail":91885024,"statfs":{"total":95995035648,"available":94090264576,"internally_reserved":0,"allocated":1287938048,"data_stored":1283751292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":616823128},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594273,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1662572,"kb_used_data":1068512,"kb_used_omap":1,"kb_used_meta":594046,"kb_avail":92082580,"statfs":{"total":95995035648,"available":94292561920,"internally_reserved":0,"allocated":1094156288,"data_stored":1089970102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":608303448},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790507,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1506828,"kb_used_data":904452,"kb_used_omap":1,"kb_used_meta":602366,"kb_avail":92238324,"statfs":{"total":95995035648,"available":94452043776,"internally_reserved":0,"allocated":926158848,"data_stored":921987719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":616823128},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980832036,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1814696,"kb_used_data":1347616,"kb_used_omap":76,"kb_used_meta":466995,"kb_avail":91930456,"statfs":{"total":95995035648,"available":94136786944,"internally_reserved":0,"allocated":1379958784,"data_stored":1375772123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":478203712},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":706,"seq":3032246911197,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307024,"kb_used_data":10564,"kb_used_omap":77,"kb_used_meta":296370,"kb_avail":93438128,"statfs":{"total":95995035648,"available":95680643072,"internally_reserved":0,"allocated":10817536,"data_stored":6861682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":79051,"internal_metadata":303483701},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023218,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1531928,"kb_used_data":984400,"kb_used_omap":71,"kb_used_meta":547448,"kb_avail":92213224,"statfs":{"total":95995035648,"available":94426341376,"internally_reserved":0,"allocated":1008025600,"data_stored":1003851053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":560587424},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004347,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1668632,"kb_used_data":1272396,"kb_used_omap":66,"kb_used_meta":396157,"kb_avail":92076520,"statfs":{"total":95995035648,"available":94286356480,"internally_reserved":0,"allocated":1302933504,"data_stored":1298762468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":405664938},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195540,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1892312,"kb_used_data":1366992,"kb_used_omap":56,"kb_used_meta":525255,"kb_avail":91852840,"statfs":{"total":95995035648,"available":94057308160,"internally_reserved":0,"allocated":1399799808,"data_stored":1395623671,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":537861263},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1403912192,"data_stored":1403912192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1300627456,"data_stored":1300627456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1015087104,"data_stored":1015087104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1388249088,"data_stored":1388249088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":917241856,"data_stored":917241856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1084555264,"data_stored":1084555264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1307443200,"data_stored":1307443200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:04:24.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-09-13T15:04:24.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:04:24.070 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:04:24.130 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:24.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:24.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:24.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:24.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:24.367 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 37 16 56367 56351 95.1775 56.75 0.00253912 0.0104281 2024-09-13T15:04:24.383 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:24.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:24.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:24.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:24.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:24.621 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:24.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:24.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:24.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:24.776 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:24.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:24.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:24.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:25.012 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:25.052 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:25.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:25.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:25.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:25.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:25.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:25.315 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:25.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:25.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 38 16 57711 57695 94.8831 84 0.00649262 0.0105338 2024-09-13T15:04:25.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:25.452 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:25.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:25.457 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:04:25.457 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:04:25.457 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:04:25.458 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:04:25.458 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:04:25.556 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:25.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:25.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:25.688 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:25.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:25.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:25.836 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:25.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:25.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:26.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:26.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:26.071 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:04:26.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:26.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:26.249 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:26.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:25 smithi099 bash[84576]: cluster 2024-09-13T15:04:24.902273+0000 mgr.y (mgr.26252) 1149 : cluster [DBG] pgmap v1698: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:26.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:25 smithi099 bash[84576]: cluster 2024-09-13T15:04:24.902273+0000 mgr.y (mgr.26252) 1149 : cluster [DBG] pgmap v1698: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:26.257 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:25 smithi099 bash[87147]: cluster 2024-09-13T15:04:24.902273+0000 mgr.y (mgr.26252) 1149 : cluster [DBG] pgmap v1698: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:26.257 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:25 smithi099 bash[87147]: cluster 2024-09-13T15:04:24.902273+0000 mgr.y (mgr.26252) 1149 : cluster [DBG] pgmap v1698: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:26.267 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:26.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:26.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:25 smithi177 bash[68594]: cluster 2024-09-13T15:04:24.902273+0000 mgr.y (mgr.26252) 1149 : cluster [DBG] pgmap v1698: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:26.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:25 smithi177 bash[68594]: cluster 2024-09-13T15:04:24.902273+0000 mgr.y (mgr.26252) 1149 : cluster [DBG] pgmap v1698: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:26.367 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 39 16 58996 58980 94.5093 80.3125 0.00809794 0.0105762 2024-09-13T15:04:26.370 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:26.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:26.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:26.523 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:26.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:26.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:26.649 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:26.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:26.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:26.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:26.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:26.886 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:26.912 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:26.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:27.015 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:27.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:26 smithi177 bash[68594]: cluster 2024-09-13T15:04:26.510605+0000 mon.a (mon.0) 2276 : cluster [WRN] Health check update: Degraded data redundancy: 13836/167397 objects degraded (8.265%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:27.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:26 smithi177 bash[68594]: cluster 2024-09-13T15:04:26.510605+0000 mon.a (mon.0) 2276 : cluster [WRN] Health check update: Degraded data redundancy: 13836/167397 objects degraded (8.265%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:27.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:27.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:27.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:27.168 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:27.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:27.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:27.281 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:04:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:04:27] "GET /metrics HTTP/1.1" 200 36763 "" "Prometheus/2.51.0" 2024-09-13T15:04:27.282 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:26 smithi099 bash[84576]: cluster 2024-09-13T15:04:26.510605+0000 mon.a (mon.0) 2276 : cluster [WRN] Health check update: Degraded data redundancy: 13836/167397 objects degraded (8.265%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:27.282 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:26 smithi099 bash[84576]: cluster 2024-09-13T15:04:26.510605+0000 mon.a (mon.0) 2276 : cluster [WRN] Health check update: Degraded data redundancy: 13836/167397 objects degraded (8.265%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:27.282 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:27 smithi099 bash[87147]: cluster 2024-09-13T15:04:26.510605+0000 mon.a (mon.0) 2276 : cluster [WRN] Health check update: Degraded data redundancy: 13836/167397 objects degraded (8.265%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:27.282 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:27 smithi099 bash[87147]: cluster 2024-09-13T15:04:26.510605+0000 mon.a (mon.0) 2276 : cluster [WRN] Health check update: Degraded data redundancy: 13836/167397 objects degraded (8.265%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:27.293 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:27.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:27.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:04:27.369174+0000 min lat: 0.00126645 max lat: 0.563801 avg lat: 0.0105914 2024-09-13T15:04:27.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:04:27.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 40 16 60417 60401 94.3667 88.8125 0.00834503 0.0105914 2024-09-13T15:04:27.398 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:27.432 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:27.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:27.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:27.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:27.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:27.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T15:04:27.670 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:27.701 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:27.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:27.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:27.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:27.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:27.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:27.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:28.058 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:28.063 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:28.106 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:28.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:28.211 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:28.222 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:27 smithi177 bash[68594]: cluster 2024-09-13T15:04:26.902988+0000 mgr.y (mgr.26252) 1150 : cluster [DBG] pgmap v1699: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:28.222 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:27 smithi177 bash[68594]: cluster 2024-09-13T15:04:26.902988+0000 mgr.y (mgr.26252) 1150 : cluster [DBG] pgmap v1699: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:28.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:28.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:28.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:27 smithi099 bash[84576]: cluster 2024-09-13T15:04:26.902988+0000 mgr.y (mgr.26252) 1150 : cluster [DBG] pgmap v1699: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:28.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:27 smithi099 bash[84576]: cluster 2024-09-13T15:04:26.902988+0000 mgr.y (mgr.26252) 1150 : cluster [DBG] pgmap v1699: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:28.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:27 smithi099 bash[87147]: cluster 2024-09-13T15:04:26.902988+0000 mgr.y (mgr.26252) 1150 : cluster [DBG] pgmap v1699: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:28.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:27 smithi099 bash[87147]: cluster 2024-09-13T15:04:26.902988+0000 mgr.y (mgr.26252) 1150 : cluster [DBG] pgmap v1699: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s; 13836/167397 objects degraded (8.265%) 2024-09-13T15:04:28.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:28.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 41 16 61762 61746 94.1152 84.0625 0.0165399 0.0106185 2024-09-13T15:04:28.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:28.478 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:28.503 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:28.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:28.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:28.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:28.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:28.743 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:28.769 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:28.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:28.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:28.898 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:28.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:29.002 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:29.033 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:29.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:29.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:29.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:29.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:29.270 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:29.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:29.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 42 16 63162 63146 93.9575 87.5 0.0117353 0.0106384 2024-09-13T15:04:29.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:29.423 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:29.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:29.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:29.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:29.660 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:29.688 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:29.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:29.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:29.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:29.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:29.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:29.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:30.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:30.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:30.087 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:30.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:30.196 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:29 smithi099 bash[84576]: cluster 2024-09-13T15:04:28.903958+0000 mgr.y (mgr.26252) 1151 : cluster [DBG] pgmap v1700: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 14186/174201 objects degraded (8.143%) 2024-09-13T15:04:30.196 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:29 smithi099 bash[84576]: cluster 2024-09-13T15:04:28.903958+0000 mgr.y (mgr.26252) 1151 : cluster [DBG] pgmap v1700: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 14186/174201 objects degraded (8.143%) 2024-09-13T15:04:30.197 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:29 smithi099 bash[87147]: cluster 2024-09-13T15:04:28.903958+0000 mgr.y (mgr.26252) 1151 : cluster [DBG] pgmap v1700: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 14186/174201 objects degraded (8.143%) 2024-09-13T15:04:30.197 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:29 smithi099 bash[87147]: cluster 2024-09-13T15:04:28.903958+0000 mgr.y (mgr.26252) 1151 : cluster [DBG] pgmap v1700: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 14186/174201 objects degraded (8.143%) 2024-09-13T15:04:30.212 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:30.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:30.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:30.327 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:29 smithi177 bash[68594]: cluster 2024-09-13T15:04:28.903958+0000 mgr.y (mgr.26252) 1151 : cluster [DBG] pgmap v1700: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 14186/174201 objects degraded (8.143%) 2024-09-13T15:04:30.327 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:29 smithi177 bash[68594]: cluster 2024-09-13T15:04:28.903958+0000 mgr.y (mgr.26252) 1151 : cluster [DBG] pgmap v1700: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 14186/174201 objects degraded (8.143%) 2024-09-13T15:04:30.327 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:04:30 smithi177 bash[61439]: ts=2024-09-13T15:04:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:30.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:30.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 43 16 64520 64504 93.7461 84.875 0.00867378 0.0106623 2024-09-13T15:04:30.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:30.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:30.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:30.593 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:30.638 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:30.743 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:30.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:30.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:30.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:30.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:30.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:30.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:31.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:31.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:31.104 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:31.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:31.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:31.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:31.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:31.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:31.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:31.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 44 16 65498 65482 93.0046 61.125 0.0103486 0.010747 2024-09-13T15:04:31.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:31.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:31.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:31.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:31.540 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:31.643 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:31.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:31.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:31.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:31.792 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:31.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:31.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:31.963 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:31.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:32.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:32.103 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:32.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:32.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:32.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:32.247 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:31 smithi177 bash[68594]: cluster 2024-09-13T15:04:30.904850+0000 mgr.y (mgr.26252) 1152 : cluster [DBG] pgmap v1701: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 15445/186795 objects degraded (8.268%) 2024-09-13T15:04:32.247 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:31 smithi177 bash[68594]: cluster 2024-09-13T15:04:30.904850+0000 mgr.y (mgr.26252) 1152 : cluster [DBG] pgmap v1701: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 15445/186795 objects degraded (8.268%) 2024-09-13T15:04:32.247 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:31 smithi177 bash[68594]: cluster 2024-09-13T15:04:31.511342+0000 mon.a (mon.0) 2277 : cluster [WRN] Health check update: Degraded data redundancy: 15445/186795 objects degraded (8.268%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:32.248 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:31 smithi177 bash[68594]: cluster 2024-09-13T15:04:31.511342+0000 mon.a (mon.0) 2277 : cluster [WRN] Health check update: Degraded data redundancy: 15445/186795 objects degraded (8.268%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:32.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:31 smithi099 bash[84576]: cluster 2024-09-13T15:04:30.904850+0000 mgr.y (mgr.26252) 1152 : cluster [DBG] pgmap v1701: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 15445/186795 objects degraded (8.268%) 2024-09-13T15:04:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:31 smithi099 bash[84576]: cluster 2024-09-13T15:04:30.904850+0000 mgr.y (mgr.26252) 1152 : cluster [DBG] pgmap v1701: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 15445/186795 objects degraded (8.268%) 2024-09-13T15:04:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:31 smithi099 bash[84576]: cluster 2024-09-13T15:04:31.511342+0000 mon.a (mon.0) 2277 : cluster [WRN] Health check update: Degraded data redundancy: 15445/186795 objects degraded (8.268%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:31 smithi099 bash[84576]: cluster 2024-09-13T15:04:31.511342+0000 mon.a (mon.0) 2277 : cluster [WRN] Health check update: Degraded data redundancy: 15445/186795 objects degraded (8.268%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:32.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:32 smithi099 bash[87147]: cluster 2024-09-13T15:04:30.904850+0000 mgr.y (mgr.26252) 1152 : cluster [DBG] pgmap v1701: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 15445/186795 objects degraded (8.268%) 2024-09-13T15:04:32.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:32 smithi099 bash[87147]: cluster 2024-09-13T15:04:30.904850+0000 mgr.y (mgr.26252) 1152 : cluster [DBG] pgmap v1701: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 15445/186795 objects degraded (8.268%) 2024-09-13T15:04:32.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:32 smithi099 bash[87147]: cluster 2024-09-13T15:04:31.511342+0000 mon.a (mon.0) 2277 : cluster [WRN] Health check update: Degraded data redundancy: 15445/186795 objects degraded (8.268%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:32.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:32 smithi099 bash[87147]: cluster 2024-09-13T15:04:31.511342+0000 mon.a (mon.0) 2277 : cluster [WRN] Health check update: Degraded data redundancy: 15445/186795 objects degraded (8.268%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:32.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:32.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 45 15 66457 66442 92.2711 60 0.0192039 0.0108321 2024-09-13T15:04:32.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:32.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:32.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:32.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:32.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:32.609 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:32.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:32.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:32.758 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:32.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:32.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:32.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:32.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:32.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:33.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:33.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:33.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:33.147 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:33.192 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:33.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:33.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:33.315 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:33.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:33.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 46 16 67455 67439 91.6197 62.3125 0.00973655 0.0109103 2024-09-13T15:04:33.419 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:33.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:33.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:33.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:33.582 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:33.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:33.686 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:33.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:33.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:33.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:33.851 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:33.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:33.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:33.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:33.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:34.075 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:34.097 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:34.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:34.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:34.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:34.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:33 smithi177 bash[68594]: cluster 2024-09-13T15:04:32.905506+0000 mgr.y (mgr.26252) 1153 : cluster [DBG] pgmap v1702: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 15837/192381 objects degraded (8.232%) 2024-09-13T15:04:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:33 smithi177 bash[68594]: cluster 2024-09-13T15:04:32.905506+0000 mgr.y (mgr.26252) 1153 : cluster [DBG] pgmap v1702: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 15837/192381 objects degraded (8.232%) 2024-09-13T15:04:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:33 smithi177 bash[68594]: audit 2024-09-13T15:04:33.298109+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:33 smithi177 bash[68594]: audit 2024-09-13T15:04:33.298109+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:34.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:34.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:33 smithi099 bash[84576]: cluster 2024-09-13T15:04:32.905506+0000 mgr.y (mgr.26252) 1153 : cluster [DBG] pgmap v1702: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 15837/192381 objects degraded (8.232%) 2024-09-13T15:04:34.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:34 smithi099 bash[84576]: cluster 2024-09-13T15:04:32.905506+0000 mgr.y (mgr.26252) 1153 : cluster [DBG] pgmap v1702: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 15837/192381 objects degraded (8.232%) 2024-09-13T15:04:34.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:34 smithi099 bash[84576]: audit 2024-09-13T15:04:33.298109+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:34.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:34 smithi099 bash[84576]: audit 2024-09-13T15:04:33.298109+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:34 smithi099 bash[87147]: cluster 2024-09-13T15:04:32.905506+0000 mgr.y (mgr.26252) 1153 : cluster [DBG] pgmap v1702: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 15837/192381 objects degraded (8.232%) 2024-09-13T15:04:34.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:34 smithi099 bash[87147]: cluster 2024-09-13T15:04:32.905506+0000 mgr.y (mgr.26252) 1153 : cluster [DBG] pgmap v1702: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 15837/192381 objects degraded (8.232%) 2024-09-13T15:04:34.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:34 smithi099 bash[87147]: audit 2024-09-13T15:04:33.298109+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:34.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:34 smithi099 bash[87147]: audit 2024-09-13T15:04:33.298109+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:34.332 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:34.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 47 16 68502 68486 91.0625 65.4375 0.0199377 0.0109747 2024-09-13T15:04:34.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:34.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:34.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:34.503 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:34.606 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:34.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:34.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:34.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:34.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:34.820 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:34.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:34.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:34.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:34.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:35.028 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:35.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:35.135 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:35.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:35.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:35.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:35.287 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:35.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:35.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 48 16 69592 69576 90.5845 68.125 0.00766821 0.0110339 2024-09-13T15:04:35.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:35.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:35.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:35.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:35.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:35.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:35.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:35.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:35.793 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:35.825 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:35.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:35.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:35.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:35.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:36.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:35 smithi099 bash[84576]: cluster 2024-09-13T15:04:34.906540+0000 mgr.y (mgr.26252) 1154 : cluster [DBG] pgmap v1703: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:36.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:36 smithi099 bash[84576]: cluster 2024-09-13T15:04:34.906540+0000 mgr.y (mgr.26252) 1154 : cluster [DBG] pgmap v1703: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:36.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:36 smithi099 bash[87147]: cluster 2024-09-13T15:04:34.906540+0000 mgr.y (mgr.26252) 1154 : cluster [DBG] pgmap v1703: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:36.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:36 smithi099 bash[87147]: cluster 2024-09-13T15:04:34.906540+0000 mgr.y (mgr.26252) 1154 : cluster [DBG] pgmap v1703: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:36.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:36.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:36.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:36.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:36.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:36.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:36.330 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:35 smithi177 bash[68594]: cluster 2024-09-13T15:04:34.906540+0000 mgr.y (mgr.26252) 1154 : cluster [DBG] pgmap v1703: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:36.331 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:35 smithi177 bash[68594]: cluster 2024-09-13T15:04:34.906540+0000 mgr.y (mgr.26252) 1154 : cluster [DBG] pgmap v1703: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:36.350 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:36.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:36.368 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 49 16 70690 70674 90.1363 68.625 0.00792634 0.0110893 2024-09-13T15:04:36.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:36.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:36.582 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:36.612 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:36.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:36.717 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:36.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:36.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:36.837 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:36.859 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:36.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:36.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:36.989 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:36.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:37.092 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:37.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:36 smithi177 bash[68594]: cluster 2024-09-13T15:04:36.512193+0000 mon.a (mon.0) 2279 : cluster [WRN] Health check update: Degraded data redundancy: 16813/203439 objects degraded (8.264%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:37.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:36 smithi177 bash[68594]: cluster 2024-09-13T15:04:36.512193+0000 mon.a (mon.0) 2279 : cluster [WRN] Health check update: Degraded data redundancy: 16813/203439 objects degraded (8.264%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:37.233 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:37.268 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:37.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:37 smithi099 bash[84576]: cluster 2024-09-13T15:04:36.512193+0000 mon.a (mon.0) 2279 : cluster [WRN] Health check update: Degraded data redundancy: 16813/203439 objects degraded (8.264%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:37.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:37 smithi099 bash[84576]: cluster 2024-09-13T15:04:36.512193+0000 mon.a (mon.0) 2279 : cluster [WRN] Health check update: Degraded data redundancy: 16813/203439 objects degraded (8.264%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:37.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:04:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:04:37] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-13T15:04:37.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:37 smithi099 bash[87147]: cluster 2024-09-13T15:04:36.512193+0000 mon.a (mon.0) 2279 : cluster [WRN] Health check update: Degraded data redundancy: 16813/203439 objects degraded (8.264%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:37.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:37 smithi099 bash[87147]: cluster 2024-09-13T15:04:36.512193+0000 mon.a (mon.0) 2279 : cluster [WRN] Health check update: Degraded data redundancy: 16813/203439 objects degraded (8.264%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:37.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 50 16 71354 71338 89.1635 41.5 0.0131524 0.0112104 2024-09-13T15:04:37.373 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:37.400 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:37.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:37.504 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:37.530 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:37.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:37.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:37.658 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:37.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:37.761 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:37.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:37.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:37.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:37.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:37.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:38.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:38.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:38.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:37 smithi177 bash[68594]: cluster 2024-09-13T15:04:36.907689+0000 mgr.y (mgr.26252) 1155 : cluster [DBG] pgmap v1704: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:38.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:37 smithi177 bash[68594]: cluster 2024-09-13T15:04:36.907689+0000 mgr.y (mgr.26252) 1155 : cluster [DBG] pgmap v1704: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:38.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:38.189 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:38.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:38.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:38.310 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:38.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:38.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 51 16 72411 72395 88.7103 66.0625 0.00320251 0.0112493 2024-09-13T15:04:38.381 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:38 smithi099 bash[84576]: cluster 2024-09-13T15:04:36.907689+0000 mgr.y (mgr.26252) 1155 : cluster [DBG] pgmap v1704: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:38.381 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:38 smithi099 bash[84576]: cluster 2024-09-13T15:04:36.907689+0000 mgr.y (mgr.26252) 1155 : cluster [DBG] pgmap v1704: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:38.382 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:38 smithi099 bash[87147]: cluster 2024-09-13T15:04:36.907689+0000 mgr.y (mgr.26252) 1155 : cluster [DBG] pgmap v1704: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:38.382 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:38 smithi099 bash[87147]: cluster 2024-09-13T15:04:36.907689+0000 mgr.y (mgr.26252) 1155 : cluster [DBG] pgmap v1704: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 16813/203439 objects degraded (8.264%) 2024-09-13T15:04:38.412 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:38.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:38.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:38.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:38.587 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:38.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:38.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:38.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:38.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:38.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:38.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:38.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:38.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:38.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:38.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:39.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:39.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:39.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:39.215 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:39.249 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:39.353 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:39.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 52 16 73621 73605 88.4586 75.625 0.0147074 0.0112992 2024-09-13T15:04:39.383 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:39.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:39.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:39.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:39.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:39.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:39.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:39.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:39.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:39.782 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:39.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:39.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:39.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:39.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:40.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:40.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:40.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:40.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:40 smithi099 bash[87147]: cluster 2024-09-13T15:04:38.908398+0000 mgr.y (mgr.26252) 1156 : cluster [DBG] pgmap v1705: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 17113/208977 objects degraded (8.189%) 2024-09-13T15:04:40.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:40 smithi099 bash[87147]: cluster 2024-09-13T15:04:38.908398+0000 mgr.y (mgr.26252) 1156 : cluster [DBG] pgmap v1705: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 17113/208977 objects degraded (8.189%) 2024-09-13T15:04:40.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:39 smithi099 bash[84576]: cluster 2024-09-13T15:04:38.908398+0000 mgr.y (mgr.26252) 1156 : cluster [DBG] pgmap v1705: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 17113/208977 objects degraded (8.189%) 2024-09-13T15:04:40.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:39 smithi099 bash[84576]: cluster 2024-09-13T15:04:38.908398+0000 mgr.y (mgr.26252) 1156 : cluster [DBG] pgmap v1705: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 17113/208977 objects degraded (8.189%) 2024-09-13T15:04:40.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:40.177 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:40.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:40.280 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:40.298 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:40.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:40.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:39 smithi177 bash[68594]: cluster 2024-09-13T15:04:38.908398+0000 mgr.y (mgr.26252) 1156 : cluster [DBG] pgmap v1705: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 17113/208977 objects degraded (8.189%) 2024-09-13T15:04:40.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:39 smithi177 bash[68594]: cluster 2024-09-13T15:04:38.908398+0000 mgr.y (mgr.26252) 1156 : cluster [DBG] pgmap v1705: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 17113/208977 objects degraded (8.189%) 2024-09-13T15:04:40.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:04:40 smithi177 bash[61439]: ts=2024-09-13T15:04:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:40.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 53 16 74649 74633 88.0017 64.25 0.00682266 0.0113524 2024-09-13T15:04:40.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:40.429 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:40.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:40.532 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:40.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:40.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:40.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:40.688 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:40.790 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:40.824 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:40.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:40.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:40.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:40.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:41.063 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:41.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:41.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:41.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:41.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:41.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:41.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:41.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:41.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 54 16 75669 75653 87.5525 63.75 0.0199225 0.011416 2024-09-13T15:04:41.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:41.471 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:41.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:41.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:41.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:41.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:41.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:41.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:41.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:41.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:41.855 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:41.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:41.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:41.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:42.009 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:42.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:42.111 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:42.126 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:42 smithi099 bash[84576]: cluster 2024-09-13T15:04:40.909267+0000 mgr.y (mgr.26252) 1157 : cluster [DBG] pgmap v1706: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 18079/218586 objects degraded (8.271%) 2024-09-13T15:04:42.126 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:42 smithi099 bash[84576]: cluster 2024-09-13T15:04:40.909267+0000 mgr.y (mgr.26252) 1157 : cluster [DBG] pgmap v1706: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 18079/218586 objects degraded (8.271%) 2024-09-13T15:04:42.126 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:42 smithi099 bash[84576]: cluster 2024-09-13T15:04:41.512904+0000 mon.a (mon.0) 2280 : cluster [WRN] Health check update: Degraded data redundancy: 18079/218586 objects degraded (8.271%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:42.126 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:42 smithi099 bash[84576]: cluster 2024-09-13T15:04:41.512904+0000 mon.a (mon.0) 2280 : cluster [WRN] Health check update: Degraded data redundancy: 18079/218586 objects degraded (8.271%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:42.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:42 smithi099 bash[87147]: cluster 2024-09-13T15:04:40.909267+0000 mgr.y (mgr.26252) 1157 : cluster [DBG] pgmap v1706: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 18079/218586 objects degraded (8.271%) 2024-09-13T15:04:42.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:42 smithi099 bash[87147]: cluster 2024-09-13T15:04:40.909267+0000 mgr.y (mgr.26252) 1157 : cluster [DBG] pgmap v1706: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 18079/218586 objects degraded (8.271%) 2024-09-13T15:04:42.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:42 smithi099 bash[87147]: cluster 2024-09-13T15:04:41.512904+0000 mon.a (mon.0) 2280 : cluster [WRN] Health check update: Degraded data redundancy: 18079/218586 objects degraded (8.271%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:42.127 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:42 smithi099 bash[87147]: cluster 2024-09-13T15:04:41.512904+0000 mon.a (mon.0) 2280 : cluster [WRN] Health check update: Degraded data redundancy: 18079/218586 objects degraded (8.271%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:42.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:42.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:42.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:42.278 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:42.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:42.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:42 smithi177 bash[68594]: cluster 2024-09-13T15:04:40.909267+0000 mgr.y (mgr.26252) 1157 : cluster [DBG] pgmap v1706: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 18079/218586 objects degraded (8.271%) 2024-09-13T15:04:42.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:42 smithi177 bash[68594]: cluster 2024-09-13T15:04:40.909267+0000 mgr.y (mgr.26252) 1157 : cluster [DBG] pgmap v1706: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 18079/218586 objects degraded (8.271%) 2024-09-13T15:04:42.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:42 smithi177 bash[68594]: cluster 2024-09-13T15:04:41.512904+0000 mon.a (mon.0) 2280 : cluster [WRN] Health check update: Degraded data redundancy: 18079/218586 objects degraded (8.271%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:42.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:42 smithi177 bash[68594]: cluster 2024-09-13T15:04:41.512904+0000 mon.a (mon.0) 2280 : cluster [WRN] Health check update: Degraded data redundancy: 18079/218586 objects degraded (8.271%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:42.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 55 16 76395 76379 86.7856 45.375 0.00726481 0.0115182 2024-09-13T15:04:42.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:42.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:42.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:42.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:42.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:42.643 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:42.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:42.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:42.778 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:42.806 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:42.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:42.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:42.946 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:42.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:43.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:43.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:43.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:43.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:43.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:43.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:43.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:43.343 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:43.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 56 16 77447 77431 86.4098 65.75 0.0127564 0.0115676 2024-09-13T15:04:43.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:43.480 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:43.583 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:43.615 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:43.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:43.720 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:43.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:43.848 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:43.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:43.978 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:44.012 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T15:04:44.023 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:44.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:44.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:44.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[84576]: cluster 2024-09-13T15:04:42.909926+0000 mgr.y (mgr.26252) 1158 : cluster [DBG] pgmap v1707: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 18367/223299 objects degraded (8.225%) 2024-09-13T15:04:44.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[84576]: cluster 2024-09-13T15:04:42.909926+0000 mgr.y (mgr.26252) 1158 : cluster [DBG] pgmap v1707: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 18367/223299 objects degraded (8.225%) 2024-09-13T15:04:44.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[84576]: audit 2024-09-13T15:04:43.344152+0000 mon.c (mon.2) 90 : audit [INF] from='client.? 172.21.15.99:0/1907507641' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[84576]: audit 2024-09-13T15:04:43.344152+0000 mon.c (mon.2) 90 : audit [INF] from='client.? 172.21.15.99:0/1907507641' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[84576]: audit 2024-09-13T15:04:43.344657+0000 mon.a (mon.0) 2281 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[84576]: audit 2024-09-13T15:04:43.344657+0000 mon.a (mon.0) 2281 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.153 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[87147]: cluster 2024-09-13T15:04:42.909926+0000 mgr.y (mgr.26252) 1158 : cluster [DBG] pgmap v1707: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 18367/223299 objects degraded (8.225%) 2024-09-13T15:04:44.153 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[87147]: cluster 2024-09-13T15:04:42.909926+0000 mgr.y (mgr.26252) 1158 : cluster [DBG] pgmap v1707: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 18367/223299 objects degraded (8.225%) 2024-09-13T15:04:44.153 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[87147]: audit 2024-09-13T15:04:43.344152+0000 mon.c (mon.2) 90 : audit [INF] from='client.? 172.21.15.99:0/1907507641' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.153 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[87147]: audit 2024-09-13T15:04:43.344152+0000 mon.c (mon.2) 90 : audit [INF] from='client.? 172.21.15.99:0/1907507641' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.153 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[87147]: audit 2024-09-13T15:04:43.344657+0000 mon.a (mon.0) 2281 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.153 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:44 smithi099 bash[87147]: audit 2024-09-13T15:04:43.344657+0000 mon.a (mon.0) 2281 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.163 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:44.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:44.265 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:44.293 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:44.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:44.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:44 smithi177 bash[68594]: cluster 2024-09-13T15:04:42.909926+0000 mgr.y (mgr.26252) 1158 : cluster [DBG] pgmap v1707: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 18367/223299 objects degraded (8.225%) 2024-09-13T15:04:44.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:44 smithi177 bash[68594]: cluster 2024-09-13T15:04:42.909926+0000 mgr.y (mgr.26252) 1158 : cluster [DBG] pgmap v1707: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 18367/223299 objects degraded (8.225%) 2024-09-13T15:04:44.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:44 smithi177 bash[68594]: audit 2024-09-13T15:04:43.344152+0000 mon.c (mon.2) 90 : audit [INF] from='client.? 172.21.15.99:0/1907507641' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:44 smithi177 bash[68594]: audit 2024-09-13T15:04:43.344152+0000 mon.c (mon.2) 90 : audit [INF] from='client.? 172.21.15.99:0/1907507641' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:44 smithi177 bash[68594]: audit 2024-09-13T15:04:43.344657+0000 mon.a (mon.0) 2281 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:44 smithi177 bash[68594]: audit 2024-09-13T15:04:43.344657+0000 mon.a (mon.0) 2281 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:44.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 57 16 78603 78587 86.1613 72.25 0.00780651 0.011601 2024-09-13T15:04:44.397 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:44.431 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:44.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:44.534 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:44.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:44.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:44.678 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:44.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:44.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:44.810 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:44.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:44.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:44.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:44.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:44.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:45.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:45.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:45.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:45.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:45.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:45 smithi177 bash[68594]: cluster 2024-09-13T15:04:44.001762+0000 mon.a (mon.0) 2282 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:45.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:45 smithi177 bash[68594]: cluster 2024-09-13T15:04:44.001762+0000 mon.a (mon.0) 2282 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:45.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:45 smithi177 bash[68594]: audit 2024-09-13T15:04:44.004637+0000 mon.a (mon.0) 2283 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:45.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:45 smithi177 bash[68594]: audit 2024-09-13T15:04:44.004637+0000 mon.a (mon.0) 2283 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:45.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:45 smithi177 bash[68594]: cluster 2024-09-13T15:04:44.011025+0000 mon.a (mon.0) 2284 : cluster [DBG] osdmap e1062: 8 total, 7 up, 8 in 2024-09-13T15:04:45.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:45 smithi177 bash[68594]: cluster 2024-09-13T15:04:44.011025+0000 mon.a (mon.0) 2284 : cluster [DBG] osdmap e1062: 8 total, 7 up, 8 in 2024-09-13T15:04:45.227 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:45.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[87147]: cluster 2024-09-13T15:04:44.001762+0000 mon.a (mon.0) 2282 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:45.323 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[87147]: cluster 2024-09-13T15:04:44.001762+0000 mon.a (mon.0) 2282 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:45.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[87147]: audit 2024-09-13T15:04:44.004637+0000 mon.a (mon.0) 2283 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:45.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[87147]: audit 2024-09-13T15:04:44.004637+0000 mon.a (mon.0) 2283 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:45.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[87147]: cluster 2024-09-13T15:04:44.011025+0000 mon.a (mon.0) 2284 : cluster [DBG] osdmap e1062: 8 total, 7 up, 8 in 2024-09-13T15:04:45.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[87147]: cluster 2024-09-13T15:04:44.011025+0000 mon.a (mon.0) 2284 : cluster [DBG] osdmap e1062: 8 total, 7 up, 8 in 2024-09-13T15:04:45.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[84576]: cluster 2024-09-13T15:04:44.001762+0000 mon.a (mon.0) 2282 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:45.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[84576]: cluster 2024-09-13T15:04:44.001762+0000 mon.a (mon.0) 2282 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:45.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[84576]: audit 2024-09-13T15:04:44.004637+0000 mon.a (mon.0) 2283 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:45.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[84576]: audit 2024-09-13T15:04:44.004637+0000 mon.a (mon.0) 2283 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:45.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[84576]: cluster 2024-09-13T15:04:44.011025+0000 mon.a (mon.0) 2284 : cluster [DBG] osdmap e1062: 8 total, 7 up, 8 in 2024-09-13T15:04:45.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:45 smithi099 bash[84576]: cluster 2024-09-13T15:04:44.011025+0000 mon.a (mon.0) 2284 : cluster [DBG] osdmap e1062: 8 total, 7 up, 8 in 2024-09-13T15:04:45.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:45.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:45.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:45.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 58 16 79663 79647 85.8179 66.25 0.0199959 0.011648 2024-09-13T15:04:45.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:45.497 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:45.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:45.600 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:45.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:45.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:45.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:45.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:45.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:45.903 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:46.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:46.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:46.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:46.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:46.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:46.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:46.271 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:46.279 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:46 smithi099 bash[84576]: cluster 2024-09-13T15:04:44.910812+0000 mgr.y (mgr.26252) 1159 : cluster [DBG] pgmap v1709: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:46.279 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:46 smithi099 bash[84576]: cluster 2024-09-13T15:04:44.910812+0000 mgr.y (mgr.26252) 1159 : cluster [DBG] pgmap v1709: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:46.280 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:46 smithi099 bash[87147]: cluster 2024-09-13T15:04:44.910812+0000 mgr.y (mgr.26252) 1159 : cluster [DBG] pgmap v1709: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:46.280 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:46 smithi099 bash[87147]: cluster 2024-09-13T15:04:44.910812+0000 mgr.y (mgr.26252) 1159 : cluster [DBG] pgmap v1709: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:46.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:46.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:46.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:46 smithi177 bash[68594]: cluster 2024-09-13T15:04:44.910812+0000 mgr.y (mgr.26252) 1159 : cluster [DBG] pgmap v1709: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:46.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:46 smithi177 bash[68594]: cluster 2024-09-13T15:04:44.910812+0000 mgr.y (mgr.26252) 1159 : cluster [DBG] pgmap v1709: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:46.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 59 16 80792 80776 85.5593 70.5625 0.00899574 0.0116826 2024-09-13T15:04:46.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:46.423 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:46.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:46.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:46.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:46.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:46.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:46.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:46.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:46.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:46.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:46.934 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:46.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:46.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:47.093 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:47 smithi177 bash[68594]: cluster 2024-09-13T15:04:46.513593+0000 mon.a (mon.0) 2285 : cluster [WRN] Health check update: Degraded data redundancy: 19318/233538 objects degraded (8.272%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:47 smithi177 bash[68594]: cluster 2024-09-13T15:04:46.513593+0000 mon.a (mon.0) 2285 : cluster [WRN] Health check update: Degraded data redundancy: 19318/233538 objects degraded (8.272%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:47 smithi177 bash[68594]: audit 2024-09-13T15:04:46.633720+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:04:47.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:47 smithi177 bash[68594]: audit 2024-09-13T15:04:46.633720+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:04:47.119 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:47.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:47.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:47.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:47.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:47.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:04:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:04:47] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-13T15:04:47.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:47 smithi099 bash[84576]: cluster 2024-09-13T15:04:46.513593+0000 mon.a (mon.0) 2285 : cluster [WRN] Health check update: Degraded data redundancy: 19318/233538 objects degraded (8.272%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:47.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:47 smithi099 bash[84576]: cluster 2024-09-13T15:04:46.513593+0000 mon.a (mon.0) 2285 : cluster [WRN] Health check update: Degraded data redundancy: 19318/233538 objects degraded (8.272%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:47.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:47 smithi099 bash[84576]: audit 2024-09-13T15:04:46.633720+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:04:47.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:47 smithi099 bash[84576]: audit 2024-09-13T15:04:46.633720+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:04:47.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:47 smithi099 bash[87147]: cluster 2024-09-13T15:04:46.513593+0000 mon.a (mon.0) 2285 : cluster [WRN] Health check update: Degraded data redundancy: 19318/233538 objects degraded (8.272%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:47.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:47 smithi099 bash[87147]: cluster 2024-09-13T15:04:46.513593+0000 mon.a (mon.0) 2285 : cluster [WRN] Health check update: Degraded data redundancy: 19318/233538 objects degraded (8.272%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:47.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:47 smithi099 bash[87147]: audit 2024-09-13T15:04:46.633720+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:04:47.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:47 smithi099 bash[87147]: audit 2024-09-13T15:04:46.633720+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:04:47.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:47.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:04:47.370973+0000 min lat: 0.00126645 max lat: 0.563801 avg lat: 0.011737 2024-09-13T15:04:47.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:04:47.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 60 16 81782 81766 85.1644 61.875 0.00992417 0.011737 2024-09-13T15:04:47.371 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:47.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:47.475 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:47.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:47.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:47.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:47.635 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:47.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:47.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:47.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:47.860 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:47.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:47.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:47.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:48.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:48.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:48.123 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:48.133 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:48 smithi177 bash[68594]: cluster 2024-09-13T15:04:46.911540+0000 mgr.y (mgr.26252) 1160 : cluster [DBG] pgmap v1710: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:48.133 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:48 smithi177 bash[68594]: cluster 2024-09-13T15:04:46.911540+0000 mgr.y (mgr.26252) 1160 : cluster [DBG] pgmap v1710: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:48.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:48.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:48.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:48.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:48.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:48 smithi099 bash[84576]: cluster 2024-09-13T15:04:46.911540+0000 mgr.y (mgr.26252) 1160 : cluster [DBG] pgmap v1710: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:48.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:48 smithi099 bash[84576]: cluster 2024-09-13T15:04:46.911540+0000 mgr.y (mgr.26252) 1160 : cluster [DBG] pgmap v1710: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:48.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:48 smithi099 bash[87147]: cluster 2024-09-13T15:04:46.911540+0000 mgr.y (mgr.26252) 1160 : cluster [DBG] pgmap v1710: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:48.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:48 smithi099 bash[87147]: cluster 2024-09-13T15:04:46.911540+0000 mgr.y (mgr.26252) 1160 : cluster [DBG] pgmap v1710: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 19318/233538 objects degraded (8.272%) 2024-09-13T15:04:48.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:48.369 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 61 16 82541 82525 84.5459 47.4375 0.00988343 0.0118236 2024-09-13T15:04:48.404 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:48.434 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:48.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:48.539 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:48.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:48.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:48.663 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:48.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:48.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:48.795 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:48.826 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:48.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:48.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:48.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:48.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:49.078 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:49.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:49.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:49.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:49.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:49.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:49.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:49.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 62 16 83526 83510 84.1751 61.5625 0.0145375 0.0118742 2024-09-13T15:04:49.378 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:49.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:49.483 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:49.506 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:49.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:49.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:49 smithi177 bash[68594]: audit 2024-09-13T15:04:48.302136+0000 mon.a (mon.0) 2287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:49.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:49 smithi177 bash[68594]: audit 2024-09-13T15:04:48.302136+0000 mon.a (mon.0) 2287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:49.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:49 smithi177 bash[68594]: audit 2024-09-13T15:04:48.303501+0000 mon.a (mon.0) 2288 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:49.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:49 smithi177 bash[68594]: audit 2024-09-13T15:04:48.303501+0000 mon.a (mon.0) 2288 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:49.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:49 smithi177 bash[68594]: cluster 2024-09-13T15:04:48.912418+0000 mgr.y (mgr.26252) 1161 : cluster [DBG] pgmap v1711: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 19660/239598 objects degraded (8.205%) 2024-09-13T15:04:49.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:49 smithi177 bash[68594]: cluster 2024-09-13T15:04:48.912418+0000 mgr.y (mgr.26252) 1161 : cluster [DBG] pgmap v1711: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 19660/239598 objects degraded (8.205%) 2024-09-13T15:04:49.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:04:49.609 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:49.634 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[84576]: audit 2024-09-13T15:04:48.302136+0000 mon.a (mon.0) 2287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[84576]: audit 2024-09-13T15:04:48.302136+0000 mon.a (mon.0) 2287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[84576]: audit 2024-09-13T15:04:48.303501+0000 mon.a (mon.0) 2288 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[84576]: audit 2024-09-13T15:04:48.303501+0000 mon.a (mon.0) 2288 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[84576]: cluster 2024-09-13T15:04:48.912418+0000 mgr.y (mgr.26252) 1161 : cluster [DBG] pgmap v1711: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 19660/239598 objects degraded (8.205%) 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[84576]: cluster 2024-09-13T15:04:48.912418+0000 mgr.y (mgr.26252) 1161 : cluster [DBG] pgmap v1711: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 19660/239598 objects degraded (8.205%) 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[87147]: audit 2024-09-13T15:04:48.302136+0000 mon.a (mon.0) 2287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[87147]: audit 2024-09-13T15:04:48.302136+0000 mon.a (mon.0) 2287 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[87147]: audit 2024-09-13T15:04:48.303501+0000 mon.a (mon.0) 2288 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[87147]: audit 2024-09-13T15:04:48.303501+0000 mon.a (mon.0) 2288 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[87147]: cluster 2024-09-13T15:04:48.912418+0000 mgr.y (mgr.26252) 1161 : cluster [DBG] pgmap v1711: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 19660/239598 objects degraded (8.205%) 2024-09-13T15:04:49.635 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:49 smithi099 bash[87147]: cluster 2024-09-13T15:04:48.912418+0000 mgr.y (mgr.26252) 1161 : cluster [DBG] pgmap v1711: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 19660/239598 objects degraded (8.205%) 2024-09-13T15:04:49.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:49.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:49.748 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:49.768 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:49.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:49.871 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:49.898 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:49.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:50.001 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:50.032 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:50.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:50.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:50.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:50.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:50.218 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:04:50 smithi177 bash[61439]: ts=2024-09-13T15:04:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:04:50.267 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:50.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:50.309 INFO:teuthology.orchestra.run.smithi099.stderr:set pool 1 pgp_num to 1 2024-09-13T15:04:50.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 63 16 83916 83900 83.2259 24.375 0.00936724 0.0120106 2024-09-13T15:04:50.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:50.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:50.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:50.426 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:50 smithi099 bash[84576]: audit 2024-09-13T15:04:49.736792+0000 mon.a (mon.0) 2289 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:04:50.426 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:50 smithi099 bash[84576]: audit 2024-09-13T15:04:49.736792+0000 mon.a (mon.0) 2289 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:04:50.426 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:50 smithi099 bash[87147]: audit 2024-09-13T15:04:49.736792+0000 mon.a (mon.0) 2289 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:04:50.426 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:50 smithi099 bash[87147]: audit 2024-09-13T15:04:49.736792+0000 mon.a (mon.0) 2289 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:04:50.504 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:50.504 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:04:50.518 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:50.522 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:04:50.522 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:50 smithi177 bash[68594]: audit 2024-09-13T15:04:49.736792+0000 mon.a (mon.0) 2289 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:04:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:50 smithi177 bash[68594]: audit 2024-09-13T15:04:49.736792+0000 mon.a (mon.0) 2289 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:04:50.546 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:04:50.546 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:50.547 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:04:50.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:50.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:50.578 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:04:50.579 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:50.609 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:04:50.609 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:50.653 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:04:50.653 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:50.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:50.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:50.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:50.688 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:04:50.688 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:50.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:50.819 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:50.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:50.922 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:50.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:50.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:51.048 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:51.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:51.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:51.180 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:51.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:51.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:51.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:51.328 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:51.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 64 16 84979 84963 82.9635 66.4375 0.0111506 0.0120488 2024-09-13T15:04:51.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:51.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[87147]: audit 2024-09-13T15:04:50.310075+0000 mon.a (mon.0) 2290 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:04:51.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[87147]: audit 2024-09-13T15:04:50.310075+0000 mon.a (mon.0) 2290 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:04:51.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[87147]: cluster 2024-09-13T15:04:50.313330+0000 mon.a (mon.0) 2291 : cluster [DBG] osdmap e1063: 8 total, 7 up, 8 in 2024-09-13T15:04:51.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[87147]: cluster 2024-09-13T15:04:50.313330+0000 mon.a (mon.0) 2291 : cluster [DBG] osdmap e1063: 8 total, 7 up, 8 in 2024-09-13T15:04:51.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[87147]: audit 2024-09-13T15:04:50.479970+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.99:0/2408816121' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:51.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[87147]: audit 2024-09-13T15:04:50.479970+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.99:0/2408816121' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:51.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[87147]: cluster 2024-09-13T15:04:50.912948+0000 mgr.y (mgr.26252) 1162 : cluster [DBG] pgmap v1713: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 20520/248640 objects degraded (8.253%) 2024-09-13T15:04:51.442 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[87147]: cluster 2024-09-13T15:04:50.912948+0000 mgr.y (mgr.26252) 1162 : cluster [DBG] pgmap v1713: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 20520/248640 objects degraded (8.253%) 2024-09-13T15:04:51.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[84576]: audit 2024-09-13T15:04:50.310075+0000 mon.a (mon.0) 2290 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:04:51.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[84576]: audit 2024-09-13T15:04:50.310075+0000 mon.a (mon.0) 2290 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:04:51.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[84576]: cluster 2024-09-13T15:04:50.313330+0000 mon.a (mon.0) 2291 : cluster [DBG] osdmap e1063: 8 total, 7 up, 8 in 2024-09-13T15:04:51.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[84576]: cluster 2024-09-13T15:04:50.313330+0000 mon.a (mon.0) 2291 : cluster [DBG] osdmap e1063: 8 total, 7 up, 8 in 2024-09-13T15:04:51.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[84576]: audit 2024-09-13T15:04:50.479970+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.99:0/2408816121' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:51.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[84576]: audit 2024-09-13T15:04:50.479970+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.99:0/2408816121' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:51.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[84576]: cluster 2024-09-13T15:04:50.912948+0000 mgr.y (mgr.26252) 1162 : cluster [DBG] pgmap v1713: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 20520/248640 objects degraded (8.253%) 2024-09-13T15:04:51.442 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:51 smithi099 bash[84576]: cluster 2024-09-13T15:04:50.912948+0000 mgr.y (mgr.26252) 1162 : cluster [DBG] pgmap v1713: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 20520/248640 objects degraded (8.253%) 2024-09-13T15:04:51.455 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:51.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:51.582 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:51.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:51 smithi177 bash[68594]: audit 2024-09-13T15:04:50.310075+0000 mon.a (mon.0) 2290 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:04:51.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:51 smithi177 bash[68594]: audit 2024-09-13T15:04:50.310075+0000 mon.a (mon.0) 2290 : audit [INF] from='client.? 172.21.15.99:0/609547562' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:04:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:51 smithi177 bash[68594]: cluster 2024-09-13T15:04:50.313330+0000 mon.a (mon.0) 2291 : cluster [DBG] osdmap e1063: 8 total, 7 up, 8 in 2024-09-13T15:04:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:51 smithi177 bash[68594]: cluster 2024-09-13T15:04:50.313330+0000 mon.a (mon.0) 2291 : cluster [DBG] osdmap e1063: 8 total, 7 up, 8 in 2024-09-13T15:04:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:51 smithi177 bash[68594]: audit 2024-09-13T15:04:50.479970+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.99:0/2408816121' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:51 smithi177 bash[68594]: audit 2024-09-13T15:04:50.479970+0000 mon.c (mon.2) 91 : audit [DBG] from='client.? 172.21.15.99:0/2408816121' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:51 smithi177 bash[68594]: cluster 2024-09-13T15:04:50.912948+0000 mgr.y (mgr.26252) 1162 : cluster [DBG] pgmap v1713: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 20520/248640 objects degraded (8.253%) 2024-09-13T15:04:51.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:51 smithi177 bash[68594]: cluster 2024-09-13T15:04:50.912948+0000 mgr.y (mgr.26252) 1162 : cluster [DBG] pgmap v1713: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 20520/248640 objects degraded (8.253%) 2024-09-13T15:04:51.687 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:51.705 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:51.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:51.828 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:51.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:51.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:51.956 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:51.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:52.059 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:52.078 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:52.181 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:52.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:52.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:52.318 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:52.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:52.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:52.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 65 16 85586 85570 82.2707 37.9375 0.0120771 0.0121505 2024-09-13T15:04:52.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:52.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:52 smithi177 bash[68594]: cluster 2024-09-13T15:04:51.514356+0000 mon.a (mon.0) 2292 : cluster [WRN] Health check update: Degraded data redundancy: 20520/248640 objects degraded (8.253%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:52.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:52 smithi177 bash[68594]: cluster 2024-09-13T15:04:51.514356+0000 mon.a (mon.0) 2292 : cluster [WRN] Health check update: Degraded data redundancy: 20520/248640 objects degraded (8.253%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:52.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:52.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:52.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:52.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:52.731 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:52.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:52.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:52 smithi099 bash[84576]: cluster 2024-09-13T15:04:51.514356+0000 mon.a (mon.0) 2292 : cluster [WRN] Health check update: Degraded data redundancy: 20520/248640 objects degraded (8.253%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:52.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:52 smithi099 bash[84576]: cluster 2024-09-13T15:04:51.514356+0000 mon.a (mon.0) 2292 : cluster [WRN] Health check update: Degraded data redundancy: 20520/248640 objects degraded (8.253%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:52.833 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:52.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:52.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:52.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:53.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:53.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:53.118 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:53.139 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:52 smithi099 bash[87147]: cluster 2024-09-13T15:04:51.514356+0000 mon.a (mon.0) 2292 : cluster [WRN] Health check update: Degraded data redundancy: 20520/248640 objects degraded (8.253%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:53.139 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:52 smithi099 bash[87147]: cluster 2024-09-13T15:04:51.514356+0000 mon.a (mon.0) 2292 : cluster [WRN] Health check update: Degraded data redundancy: 20520/248640 objects degraded (8.253%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:53.139 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:53.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:53.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:53.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:53.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:53.364 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:53.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 66 16 86328 86312 81.7268 46.375 0.138876 0.0122292 2024-09-13T15:04:53.389 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:53.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:53 smithi099 bash[87147]: cluster 2024-09-13T15:04:52.913693+0000 mgr.y (mgr.26252) 1163 : cluster [DBG] pgmap v1714: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 20758/252165 objects degraded (8.232%) 2024-09-13T15:04:53.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:53 smithi099 bash[87147]: cluster 2024-09-13T15:04:52.913693+0000 mgr.y (mgr.26252) 1163 : cluster [DBG] pgmap v1714: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 20758/252165 objects degraded (8.232%) 2024-09-13T15:04:53.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:53.392 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:53 smithi099 bash[84576]: cluster 2024-09-13T15:04:52.913693+0000 mgr.y (mgr.26252) 1163 : cluster [DBG] pgmap v1714: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 20758/252165 objects degraded (8.232%) 2024-09-13T15:04:53.392 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:53 smithi099 bash[84576]: cluster 2024-09-13T15:04:52.913693+0000 mgr.y (mgr.26252) 1163 : cluster [DBG] pgmap v1714: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 20758/252165 objects degraded (8.232%) 2024-09-13T15:04:53.492 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:53.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:53.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:53 smithi177 bash[68594]: cluster 2024-09-13T15:04:52.913693+0000 mgr.y (mgr.26252) 1163 : cluster [DBG] pgmap v1714: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 20758/252165 objects degraded (8.232%) 2024-09-13T15:04:53.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:53 smithi177 bash[68594]: cluster 2024-09-13T15:04:52.913693+0000 mgr.y (mgr.26252) 1163 : cluster [DBG] pgmap v1714: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 20758/252165 objects degraded (8.232%) 2024-09-13T15:04:53.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:53.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:53.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:53.755 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:53.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:53.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:53.882 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:53.917 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:53.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:53.964 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:04:53.965 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:04:54.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:54.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:54.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:54.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:54.201 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:54.224 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:54.224 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:54.309 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:54.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:54.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:54.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 67 16 87258 87242 81.3744 58.125 0.0265806 0.0122839 2024-09-13T15:04:54.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:54.506 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:54.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:54.615 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:54.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:54.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:54.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:54.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:54.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:54.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:54 smithi099 bash[87147]: audit 2024-09-13T15:04:53.756614+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:54.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:54 smithi099 bash[87147]: audit 2024-09-13T15:04:53.756614+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:54.823 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:54 smithi099 bash[87147]: audit 2024-09-13T15:04:53.763278+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:54.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:54 smithi099 bash[87147]: audit 2024-09-13T15:04:53.763278+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:54.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:54 smithi099 bash[84576]: audit 2024-09-13T15:04:53.756614+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:54.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:54 smithi099 bash[84576]: audit 2024-09-13T15:04:53.756614+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:54.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:54 smithi099 bash[84576]: audit 2024-09-13T15:04:53.763278+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:54.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:54 smithi099 bash[84576]: audit 2024-09-13T15:04:53.763278+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:54.894 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:54.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:54.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:55.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:55.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:54 smithi177 bash[68594]: audit 2024-09-13T15:04:53.756614+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:55.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:54 smithi177 bash[68594]: audit 2024-09-13T15:04:53.756614+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:55.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:54 smithi177 bash[68594]: audit 2024-09-13T15:04:53.763278+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:55.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:54 smithi177 bash[68594]: audit 2024-09-13T15:04:53.763278+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:55.043 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:55.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:55.146 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:55.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:55.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:55.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:55.307 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:55.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:55.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 68 16 87909 87893 80.776 40.6875 0.0117286 0.0123749 2024-09-13T15:04:55.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:55.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:55.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:55.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:55.579 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:55.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:55.688 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:55.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:55.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:55.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T15:04:55.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:55.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:55.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:55.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:04:55.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:56.007 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:55 smithi177 bash[68594]: cluster 2024-09-13T15:04:54.914668+0000 mgr.y (mgr.26252) 1164 : cluster [DBG] pgmap v1715: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:56.007 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:55 smithi177 bash[68594]: cluster 2024-09-13T15:04:54.914668+0000 mgr.y (mgr.26252) 1164 : cluster [DBG] pgmap v1715: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:56.007 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:55 smithi177 bash[68594]: audit 2024-09-13T15:04:55.203371+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.007 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:55 smithi177 bash[68594]: audit 2024-09-13T15:04:55.203371+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.008 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:55 smithi177 bash[68594]: audit 2024-09-13T15:04:55.209638+0000 mon.a (mon.0) 2296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.008 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:55 smithi177 bash[68594]: audit 2024-09-13T15:04:55.209638+0000 mon.a (mon.0) 2296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:56.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[84576]: cluster 2024-09-13T15:04:54.914668+0000 mgr.y (mgr.26252) 1164 : cluster [DBG] pgmap v1715: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:56.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[84576]: cluster 2024-09-13T15:04:54.914668+0000 mgr.y (mgr.26252) 1164 : cluster [DBG] pgmap v1715: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:56.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[84576]: audit 2024-09-13T15:04:55.203371+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[84576]: audit 2024-09-13T15:04:55.203371+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[84576]: audit 2024-09-13T15:04:55.209638+0000 mon.a (mon.0) 2296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[84576]: audit 2024-09-13T15:04:55.209638+0000 mon.a (mon.0) 2296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[87147]: cluster 2024-09-13T15:04:54.914668+0000 mgr.y (mgr.26252) 1164 : cluster [DBG] pgmap v1715: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:56.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[87147]: cluster 2024-09-13T15:04:54.914668+0000 mgr.y (mgr.26252) 1164 : cluster [DBG] pgmap v1715: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:56.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[87147]: audit 2024-09-13T15:04:55.203371+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[87147]: audit 2024-09-13T15:04:55.203371+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[87147]: audit 2024-09-13T15:04:55.209638+0000 mon.a (mon.0) 2296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:55 smithi099 bash[87147]: audit 2024-09-13T15:04:55.209638+0000 mon.a (mon.0) 2296 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.129 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:56.156 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:56.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:56.262 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:56.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:56.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:56.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 69 16 88931 88915 80.531 63.875 0.00703468 0.0124126 2024-09-13T15:04:56.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:56.426 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:56.475 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:56.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:56.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:56.661 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:56.686 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:56.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:56.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:56.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:56.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:56.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[84576]: audit 2024-09-13T15:04:56.134453+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:04:56.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[84576]: audit 2024-09-13T15:04:56.134453+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:04:56.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[84576]: audit 2024-09-13T15:04:56.135646+0000 mon.a (mon.0) 2298 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:04:56.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[84576]: audit 2024-09-13T15:04:56.135646+0000 mon.a (mon.0) 2298 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:04:56.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[84576]: audit 2024-09-13T15:04:56.145570+0000 mon.a (mon.0) 2299 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[84576]: audit 2024-09-13T15:04:56.145570+0000 mon.a (mon.0) 2299 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[84576]: cluster 2024-09-13T15:04:56.515120+0000 mon.a (mon.0) 2300 : cluster [WRN] Health check update: Degraded data redundancy: 21534/260046 objects degraded (8.281%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:56.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[84576]: cluster 2024-09-13T15:04:56.515120+0000 mon.a (mon.0) 2300 : cluster [WRN] Health check update: Degraded data redundancy: 21534/260046 objects degraded (8.281%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:56.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[87147]: audit 2024-09-13T15:04:56.134453+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:04:56.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[87147]: audit 2024-09-13T15:04:56.134453+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:04:56.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[87147]: audit 2024-09-13T15:04:56.135646+0000 mon.a (mon.0) 2298 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:04:56.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[87147]: audit 2024-09-13T15:04:56.135646+0000 mon.a (mon.0) 2298 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:04:56.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[87147]: audit 2024-09-13T15:04:56.145570+0000 mon.a (mon.0) 2299 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[87147]: audit 2024-09-13T15:04:56.145570+0000 mon.a (mon.0) 2299 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[87147]: cluster 2024-09-13T15:04:56.515120+0000 mon.a (mon.0) 2300 : cluster [WRN] Health check update: Degraded data redundancy: 21534/260046 objects degraded (8.281%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:56 smithi099 bash[87147]: cluster 2024-09-13T15:04:56.515120+0000 mon.a (mon.0) 2300 : cluster [WRN] Health check update: Degraded data redundancy: 21534/260046 objects degraded (8.281%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:56.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:56.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:56.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:57.054 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:57.086 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:57.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:57.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:56 smithi177 bash[68594]: audit 2024-09-13T15:04:56.134453+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:04:57.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:56 smithi177 bash[68594]: audit 2024-09-13T15:04:56.134453+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:04:57.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:56 smithi177 bash[68594]: audit 2024-09-13T15:04:56.135646+0000 mon.a (mon.0) 2298 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:04:57.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:56 smithi177 bash[68594]: audit 2024-09-13T15:04:56.135646+0000 mon.a (mon.0) 2298 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:04:57.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:56 smithi177 bash[68594]: audit 2024-09-13T15:04:56.145570+0000 mon.a (mon.0) 2299 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:57.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:56 smithi177 bash[68594]: audit 2024-09-13T15:04:56.145570+0000 mon.a (mon.0) 2299 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:04:57.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:56 smithi177 bash[68594]: cluster 2024-09-13T15:04:56.515120+0000 mon.a (mon.0) 2300 : cluster [WRN] Health check update: Degraded data redundancy: 21534/260046 objects degraded (8.281%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:57.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:56 smithi177 bash[68594]: cluster 2024-09-13T15:04:56.515120+0000 mon.a (mon.0) 2300 : cluster [WRN] Health check update: Degraded data redundancy: 21534/260046 objects degraded (8.281%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:04:57.190 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:57.214 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:57.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:57.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:57.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:04:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:04:57] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-13T15:04:57.350 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:57.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 70 16 90024 90008 80.3564 68.3125 0.0121417 0.0124391 2024-09-13T15:04:57.457 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:57.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:57.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:57.608 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:57.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:57.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:57.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:58.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:57 smithi099 bash[84576]: cluster 2024-09-13T15:04:56.915234+0000 mgr.y (mgr.26252) 1165 : cluster [DBG] pgmap v1716: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:58.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:57 smithi099 bash[84576]: cluster 2024-09-13T15:04:56.915234+0000 mgr.y (mgr.26252) 1165 : cluster [DBG] pgmap v1716: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:58.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:57 smithi099 bash[87147]: cluster 2024-09-13T15:04:56.915234+0000 mgr.y (mgr.26252) 1165 : cluster [DBG] pgmap v1716: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:58.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:57 smithi099 bash[87147]: cluster 2024-09-13T15:04:56.915234+0000 mgr.y (mgr.26252) 1165 : cluster [DBG] pgmap v1716: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:58.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:57 smithi177 bash[68594]: cluster 2024-09-13T15:04:56.915234+0000 mgr.y (mgr.26252) 1165 : cluster [DBG] pgmap v1716: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:58.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:57 smithi177 bash[68594]: cluster 2024-09-13T15:04:56.915234+0000 mgr.y (mgr.26252) 1165 : cluster [DBG] pgmap v1716: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 21534/260046 objects degraded (8.281%) 2024-09-13T15:04:58.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:58.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:58.333 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:58.362 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:58.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:58.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 71 16 91103 91087 80.1743 67.4375 0.00503526 0.0124652 2024-09-13T15:04:58.379 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-13T15:04:58.379 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:04:58.379 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2024-09-13T15:04:58.379 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2024-09-13T15:04:58.379 DEBUG:teuthology.orchestra.run.smithi099:osd.3> sudo journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T15:04:58.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl start ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.3 2024-09-13T15:04:58.443 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:04:58.465 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:58.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:58.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:58.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:58.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:58.761 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:58.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:58.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:58.895 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:58.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:58.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:59.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:59.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:59.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:59.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:59.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:59.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:59.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:59.370 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 72 16 92125 92109 79.9479 63.875 0.0109165 0.0125032 2024-09-13T15:04:59.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:59.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:59.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:59.576 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:59.598 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:59.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:59.701 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:59.728 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:59.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:59.832 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:59.862 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:59.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:59.964 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:59.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:59.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:00.095 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:59 smithi177 bash[68594]: cluster 2024-09-13T15:04:58.915903+0000 mgr.y (mgr.26252) 1166 : cluster [DBG] pgmap v1717: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 21830/265335 objects degraded (8.227%) 2024-09-13T15:05:00.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:04:59 smithi177 bash[68594]: cluster 2024-09-13T15:04:58.915903+0000 mgr.y (mgr.26252) 1166 : cluster [DBG] pgmap v1717: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 21830/265335 objects degraded (8.227%) 2024-09-13T15:05:00.118 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:00.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:00.221 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:00.229 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:05:00 smithi177 bash[61439]: ts=2024-09-13T15:05:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:00.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:00.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:00.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:59 smithi099 bash[84576]: cluster 2024-09-13T15:04:58.915903+0000 mgr.y (mgr.26252) 1166 : cluster [DBG] pgmap v1717: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 21830/265335 objects degraded (8.227%) 2024-09-13T15:05:00.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:04:59 smithi099 bash[84576]: cluster 2024-09-13T15:04:58.915903+0000 mgr.y (mgr.26252) 1166 : cluster [DBG] pgmap v1717: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 21830/265335 objects degraded (8.227%) 2024-09-13T15:05:00.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:59 smithi099 bash[87147]: cluster 2024-09-13T15:04:58.915903+0000 mgr.y (mgr.26252) 1166 : cluster [DBG] pgmap v1717: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 21830/265335 objects degraded (8.227%) 2024-09-13T15:05:00.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:04:59 smithi099 bash[87147]: cluster 2024-09-13T15:04:58.915903+0000 mgr.y (mgr.26252) 1166 : cluster [DBG] pgmap v1717: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 21830/265335 objects degraded (8.227%) 2024-09-13T15:05:00.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:00.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:00.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:00.371 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 73 16 92843 92827 79.4673 44.875 0.00425948 0.012547 2024-09-13T15:05:00.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:00.507 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:00.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:00.536 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:00.536 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:00.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:00.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:00.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:00.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:00.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.787 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:00.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:00.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:00.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:01.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:01.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:01.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:01.146 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:01.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:01.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:01.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:01.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:01.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:01.371 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 74 16 93770 93754 79.1763 57.9375 0.0136461 0.0126254 2024-09-13T15:05:01.418 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:01.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:01.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:01.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:01.562 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:01.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:01.665 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:01.697 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:01.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:01.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:01.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:01.960 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:01.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:02.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:02.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:01 smithi099 bash[87147]: cluster 2024-09-13T15:05:00.916693+0000 mgr.y (mgr.26252) 1167 : cluster [DBG] pgmap v1718: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 22719/274281 objects degraded (8.283%) 2024-09-13T15:05:02.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:01 smithi099 bash[87147]: cluster 2024-09-13T15:05:00.916693+0000 mgr.y (mgr.26252) 1167 : cluster [DBG] pgmap v1718: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 22719/274281 objects degraded (8.283%) 2024-09-13T15:05:02.073 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:01 smithi099 bash[87147]: cluster 2024-09-13T15:05:01.515931+0000 mon.a (mon.0) 2301 : cluster [WRN] Health check update: Degraded data redundancy: 22719/274281 objects degraded (8.283%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:02.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:01 smithi099 bash[87147]: cluster 2024-09-13T15:05:01.515931+0000 mon.a (mon.0) 2301 : cluster [WRN] Health check update: Degraded data redundancy: 22719/274281 objects degraded (8.283%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:02.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:01 smithi099 bash[84576]: cluster 2024-09-13T15:05:00.916693+0000 mgr.y (mgr.26252) 1167 : cluster [DBG] pgmap v1718: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 22719/274281 objects degraded (8.283%) 2024-09-13T15:05:02.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:01 smithi099 bash[84576]: cluster 2024-09-13T15:05:00.916693+0000 mgr.y (mgr.26252) 1167 : cluster [DBG] pgmap v1718: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 22719/274281 objects degraded (8.283%) 2024-09-13T15:05:02.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:01 smithi099 bash[84576]: cluster 2024-09-13T15:05:01.515931+0000 mon.a (mon.0) 2301 : cluster [WRN] Health check update: Degraded data redundancy: 22719/274281 objects degraded (8.283%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:02.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:01 smithi099 bash[84576]: cluster 2024-09-13T15:05:01.515931+0000 mon.a (mon.0) 2301 : cluster [WRN] Health check update: Degraded data redundancy: 22719/274281 objects degraded (8.283%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:02.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:02.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:02.198 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:02.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:02.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:02.337 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:02.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:01 smithi177 bash[68594]: cluster 2024-09-13T15:05:00.916693+0000 mgr.y (mgr.26252) 1167 : cluster [DBG] pgmap v1718: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 22719/274281 objects degraded (8.283%) 2024-09-13T15:05:02.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:01 smithi177 bash[68594]: cluster 2024-09-13T15:05:00.916693+0000 mgr.y (mgr.26252) 1167 : cluster [DBG] pgmap v1718: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s; 22719/274281 objects degraded (8.283%) 2024-09-13T15:05:02.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:01 smithi177 bash[68594]: cluster 2024-09-13T15:05:01.515931+0000 mon.a (mon.0) 2301 : cluster [WRN] Health check update: Degraded data redundancy: 22719/274281 objects degraded (8.283%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:02.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:01 smithi177 bash[68594]: cluster 2024-09-13T15:05:01.515931+0000 mon.a (mon.0) 2301 : cluster [WRN] Health check update: Degraded data redundancy: 22719/274281 objects degraded (8.283%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:02.366 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:02.371 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 75 16 94692 94676 78.8889 57.625 0.0072368 0.0126708 2024-09-13T15:05:02.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:02.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:02.494 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:02.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:02.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:02.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:02.722 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:02.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:02.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:02.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:02.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:02.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:02.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:03.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:03.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:03.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:03.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:03.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:03.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:03.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:03.371 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 76 16 95454 95438 78.4775 47.625 0.00893115 0.0127379 2024-09-13T15:05:03.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:03.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:03.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:03.522 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:03.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:03.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:03.653 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:03.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:03.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:03.816 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:03.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:03.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:03.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:04.054 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:04.084 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:04.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:04.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:03 smithi177 bash[68594]: cluster 2024-09-13T15:05:02.917344+0000 mgr.y (mgr.26252) 1168 : cluster [DBG] pgmap v1719: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 22995/278634 objects degraded (8.253%) 2024-09-13T15:05:04.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:03 smithi177 bash[68594]: cluster 2024-09-13T15:05:02.917344+0000 mgr.y (mgr.26252) 1168 : cluster [DBG] pgmap v1719: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 22995/278634 objects degraded (8.253%) 2024-09-13T15:05:04.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:03 smithi177 bash[68594]: audit 2024-09-13T15:05:03.298564+0000 mon.a (mon.0) 2302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:04.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:03 smithi177 bash[68594]: audit 2024-09-13T15:05:03.298564+0000 mon.a (mon.0) 2302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:04.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:04.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:04.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:04.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:04.346 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:04.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:04.362 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:03 smithi099 bash[84576]: cluster 2024-09-13T15:05:02.917344+0000 mgr.y (mgr.26252) 1168 : cluster [DBG] pgmap v1719: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 22995/278634 objects degraded (8.253%) 2024-09-13T15:05:04.362 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:03 smithi099 bash[84576]: cluster 2024-09-13T15:05:02.917344+0000 mgr.y (mgr.26252) 1168 : cluster [DBG] pgmap v1719: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 22995/278634 objects degraded (8.253%) 2024-09-13T15:05:04.363 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:03 smithi099 bash[84576]: audit 2024-09-13T15:05:03.298564+0000 mon.a (mon.0) 2302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:04.363 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:03 smithi099 bash[84576]: audit 2024-09-13T15:05:03.298564+0000 mon.a (mon.0) 2302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:04.363 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:03 smithi099 bash[87147]: cluster 2024-09-13T15:05:02.917344+0000 mgr.y (mgr.26252) 1168 : cluster [DBG] pgmap v1719: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 22995/278634 objects degraded (8.253%) 2024-09-13T15:05:04.363 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:03 smithi099 bash[87147]: cluster 2024-09-13T15:05:02.917344+0000 mgr.y (mgr.26252) 1168 : cluster [DBG] pgmap v1719: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 22995/278634 objects degraded (8.253%) 2024-09-13T15:05:04.363 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:03 smithi099 bash[87147]: audit 2024-09-13T15:05:03.298564+0000 mon.a (mon.0) 2302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:04.363 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:03 smithi099 bash[87147]: audit 2024-09-13T15:05:03.298564+0000 mon.a (mon.0) 2302 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:04.371 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 77 16 96477 96461 78.2886 63.9375 0.0125814 0.0127678 2024-09-13T15:05:04.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:04.478 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:04.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:04.560 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:04.585 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:04.616 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:04.720 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:04.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:04.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:04.840 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:04.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:04.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:04.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:05.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:05.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:05.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:05.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:05.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:05.249 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:05.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.282 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:05.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:05.371 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 78 16 97281 97265 77.9291 50.25 0.00921918 0.0128155 2024-09-13T15:05:05.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:05.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:05.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:05.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:05.540 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:05.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:05.643 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:05.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:05.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:05.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:05.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:05.906 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:05.943 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:05.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:06.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:06.056 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:05 smithi177 bash[68594]: cluster 2024-09-13T15:05:04.918117+0000 mgr.y (mgr.26252) 1169 : cluster [DBG] pgmap v1720: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:06.056 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:05 smithi177 bash[68594]: cluster 2024-09-13T15:05:04.918117+0000 mgr.y (mgr.26252) 1169 : cluster [DBG] pgmap v1720: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:06.067 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:06.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:06.169 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:06.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:06.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:06.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:06.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:06 smithi099 bash[84576]: cluster 2024-09-13T15:05:04.918117+0000 mgr.y (mgr.26252) 1169 : cluster [DBG] pgmap v1720: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:06.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:06 smithi099 bash[87147]: cluster 2024-09-13T15:05:04.918117+0000 mgr.y (mgr.26252) 1169 : cluster [DBG] pgmap v1720: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:06.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:06 smithi099 bash[87147]: cluster 2024-09-13T15:05:04.918117+0000 mgr.y (mgr.26252) 1169 : cluster [DBG] pgmap v1720: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:06.371 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 79 15 97932 97917 77.4584 40.75 0.00942548 0.012905 2024-09-13T15:05:06.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:06.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:06.557 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:06.584 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:06.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:06.643 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:06 smithi099 bash[84576]: cluster 2024-09-13T15:05:04.918117+0000 mgr.y (mgr.26252) 1169 : cluster [DBG] pgmap v1720: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:06.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:06.713 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:06.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:06.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:06.853 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:06.960 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:06.984 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:06.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:06.996 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:06 smithi099 bash[192849]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:05:06.996 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:06 smithi099 bash[192849]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:05:06.996 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:06 smithi099 bash[192849]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T15:05:06.996 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:06 smithi099 bash[192849]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:05:06.996 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:06 smithi099 bash[192849]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:05:07.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:07.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:07.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:07.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:07.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:07.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:07.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:05:07] "GET /metrics HTTP/1.1" 200 36766 "" "Prometheus/2.51.0" 2024-09-13T15:05:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[84576]: cluster 2024-09-13T15:05:06.516526+0000 mon.a (mon.0) 2303 : cluster [WRN] Health check update: Degraded data redundancy: 23823/287649 objects degraded (8.282%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[84576]: cluster 2024-09-13T15:05:06.516526+0000 mon.a (mon.0) 2303 : cluster [WRN] Health check update: Degraded data redundancy: 23823/287649 objects degraded (8.282%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[84576]: cluster 2024-09-13T15:05:06.918669+0000 mgr.y (mgr.26252) 1170 : cluster [DBG] pgmap v1721: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:07.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[84576]: cluster 2024-09-13T15:05:06.918669+0000 mgr.y (mgr.26252) 1170 : cluster [DBG] pgmap v1721: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:07.325 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:06 smithi099 bash[192849]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T15:05:07.325 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[192849]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T15:05:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[87147]: cluster 2024-09-13T15:05:06.516526+0000 mon.a (mon.0) 2303 : cluster [WRN] Health check update: Degraded data redundancy: 23823/287649 objects degraded (8.282%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[87147]: cluster 2024-09-13T15:05:06.516526+0000 mon.a (mon.0) 2303 : cluster [WRN] Health check update: Degraded data redundancy: 23823/287649 objects degraded (8.282%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[87147]: cluster 2024-09-13T15:05:06.918669+0000 mgr.y (mgr.26252) 1170 : cluster [DBG] pgmap v1721: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[87147]: cluster 2024-09-13T15:05:06.918669+0000 mgr.y (mgr.26252) 1170 : cluster [DBG] pgmap v1721: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:07.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:07 smithi177 bash[68594]: cluster 2024-09-13T15:05:06.516526+0000 mon.a (mon.0) 2303 : cluster [WRN] Health check update: Degraded data redundancy: 23823/287649 objects degraded (8.282%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:07 smithi177 bash[68594]: cluster 2024-09-13T15:05:06.516526+0000 mon.a (mon.0) 2303 : cluster [WRN] Health check update: Degraded data redundancy: 23823/287649 objects degraded (8.282%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:07 smithi177 bash[68594]: cluster 2024-09-13T15:05:06.918669+0000 mgr.y (mgr.26252) 1170 : cluster [DBG] pgmap v1721: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:07.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:07 smithi177 bash[68594]: cluster 2024-09-13T15:05:06.918669+0000 mgr.y (mgr.26252) 1170 : cluster [DBG] pgmap v1721: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 23823/287649 objects degraded (8.282%) 2024-09-13T15:05:07.371 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:05:07.372835+0000 min lat: 0.00126645 max lat: 0.563801 avg lat: 0.0129512 2024-09-13T15:05:07.371 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:05:07.371 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 80 16 98816 98800 77.1799 55.1875 0.00768144 0.0129512 2024-09-13T15:05:07.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:07.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:07.483 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:07.513 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:07.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:07.617 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:07.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:07.650 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[192849]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T15:05:07.650 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[192849]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T15:05:07.650 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[192849]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T15:05:07.650 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[192849]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T15:05:07.650 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:07 smithi099 bash[192849]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T15:05:07.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:07.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:07.787 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:07.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:07.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:07.913 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:07.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:08.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:08.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:08.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:08.146 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:08.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:08.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:08.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:08.307 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:08.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:08.372 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 81 16 99829 99813 77.0087 63.3125 0.00983295 0.0129799 2024-09-13T15:05:08.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:08.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:08.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:08.538 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:08.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.680 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:08.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:08.903 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.954 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:09.058 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:09.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:09.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:09.184 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:09.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:09.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:09.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:09.342 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:09.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:09.372 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 82 16 100620 100604 76.6724 49.4375 0.011733 0.0130378 2024-09-13T15:05:09.449 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:09.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:09.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:09.580 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:09.603 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:09.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:09.728 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:09.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:09.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:09.855 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:09.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:09.957 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:09.988 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:09.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:10.093 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:10.119 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:10.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:10.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:10.233 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:05:10 smithi177 bash[61439]: ts=2024-09-13T15:05:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:10.233 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:09 smithi177 bash[68594]: cluster 2024-09-13T15:05:08.919318+0000 mgr.y (mgr.26252) 1171 : cluster [DBG] pgmap v1722: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 24069/292434 objects degraded (8.231%) 2024-09-13T15:05:10.234 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:09 smithi177 bash[68594]: cluster 2024-09-13T15:05:08.919318+0000 mgr.y (mgr.26252) 1171 : cluster [DBG] pgmap v1722: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 24069/292434 objects degraded (8.231%) 2024-09-13T15:05:10.249 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:10.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:10.292 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:09 smithi099 bash[84576]: cluster 2024-09-13T15:05:08.919318+0000 mgr.y (mgr.26252) 1171 : cluster [DBG] pgmap v1722: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 24069/292434 objects degraded (8.231%) 2024-09-13T15:05:10.292 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:09 smithi099 bash[84576]: cluster 2024-09-13T15:05:08.919318+0000 mgr.y (mgr.26252) 1171 : cluster [DBG] pgmap v1722: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 24069/292434 objects degraded (8.231%) 2024-09-13T15:05:10.292 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:09 smithi099 bash[87147]: cluster 2024-09-13T15:05:08.919318+0000 mgr.y (mgr.26252) 1171 : cluster [DBG] pgmap v1722: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 24069/292434 objects degraded (8.231%) 2024-09-13T15:05:10.292 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:09 smithi099 bash[87147]: cluster 2024-09-13T15:05:08.919318+0000 mgr.y (mgr.26252) 1171 : cluster [DBG] pgmap v1722: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 24069/292434 objects degraded (8.231%) 2024-09-13T15:05:10.357 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:10.372 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 83 16 101259 101243 76.2298 39.9375 0.0185387 0.0131128 2024-09-13T15:05:10.399 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:10.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:10.504 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:10.526 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:10.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:10.628 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:10.660 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:10.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:10.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:10.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:10.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:10.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:11.049 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:11.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:11.184 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:11.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:11.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:11.317 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:11.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:11.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:11.372 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 84 16 102240 102224 76.0521 61.3125 0.0121367 0.0131435 2024-09-13T15:05:11.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:11.471 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:11.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:11.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:11.592 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:11.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:11.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:11.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:11.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:11.833 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:11.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:11.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:11.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:11.993 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:11.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:12.096 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:12.124 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:12.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:12.232 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:12.256 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:12.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:12.359 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:12.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:11 smithi177 bash[68594]: cluster 2024-09-13T15:05:10.919983+0000 mgr.y (mgr.26252) 1172 : cluster [DBG] pgmap v1723: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 24843/300093 objects degraded (8.278%) 2024-09-13T15:05:12.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:11 smithi177 bash[68594]: cluster 2024-09-13T15:05:10.919983+0000 mgr.y (mgr.26252) 1172 : cluster [DBG] pgmap v1723: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 24843/300093 objects degraded (8.278%) 2024-09-13T15:05:12.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:11 smithi177 bash[68594]: cluster 2024-09-13T15:05:11.517665+0000 mon.a (mon.0) 2304 : cluster [WRN] Health check update: Degraded data redundancy: 24843/300093 objects degraded (8.278%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:12.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:11 smithi177 bash[68594]: cluster 2024-09-13T15:05:11.517665+0000 mon.a (mon.0) 2304 : cluster [WRN] Health check update: Degraded data redundancy: 24843/300093 objects degraded (8.278%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:12.372 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 85 16 103105 103089 75.7934 54.0625 0.0226571 0.0131886 2024-09-13T15:05:12.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:12.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:12.406 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:12 smithi099 bash[84576]: cluster 2024-09-13T15:05:10.919983+0000 mgr.y (mgr.26252) 1172 : cluster [DBG] pgmap v1723: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 24843/300093 objects degraded (8.278%) 2024-09-13T15:05:12.406 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:12 smithi099 bash[84576]: cluster 2024-09-13T15:05:10.919983+0000 mgr.y (mgr.26252) 1172 : cluster [DBG] pgmap v1723: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 24843/300093 objects degraded (8.278%) 2024-09-13T15:05:12.406 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:12 smithi099 bash[84576]: cluster 2024-09-13T15:05:11.517665+0000 mon.a (mon.0) 2304 : cluster [WRN] Health check update: Degraded data redundancy: 24843/300093 objects degraded (8.278%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:12.406 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:12 smithi099 bash[84576]: cluster 2024-09-13T15:05:11.517665+0000 mon.a (mon.0) 2304 : cluster [WRN] Health check update: Degraded data redundancy: 24843/300093 objects degraded (8.278%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:12.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:12 smithi099 bash[87147]: cluster 2024-09-13T15:05:10.919983+0000 mgr.y (mgr.26252) 1172 : cluster [DBG] pgmap v1723: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 24843/300093 objects degraded (8.278%) 2024-09-13T15:05:12.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:12 smithi099 bash[87147]: cluster 2024-09-13T15:05:10.919983+0000 mgr.y (mgr.26252) 1172 : cluster [DBG] pgmap v1723: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 24843/300093 objects degraded (8.278%) 2024-09-13T15:05:12.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:12 smithi099 bash[87147]: cluster 2024-09-13T15:05:11.517665+0000 mon.a (mon.0) 2304 : cluster [WRN] Health check update: Degraded data redundancy: 24843/300093 objects degraded (8.278%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:12.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:12 smithi099 bash[87147]: cluster 2024-09-13T15:05:11.517665+0000 mon.a (mon.0) 2304 : cluster [WRN] Health check update: Degraded data redundancy: 24843/300093 objects degraded (8.278%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:12.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:12.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:12.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:12.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:12.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:12.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:12.765 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:12.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:12.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:12.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:12.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:12.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:13.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:13.067 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:13.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:13.171 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:13.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:13.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:13.307 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:13.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:13.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:13.372 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 86 16 103935 103919 75.5152 51.875 0.020213 0.0132377 2024-09-13T15:05:13.442 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:13.465 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:13.570 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:13.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:13.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:13.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:13.723 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:13.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:13.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:13.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:13.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:13.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:14.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:14.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:14.111 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:14.129 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:14.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:14.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:14.241 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:13 smithi099 bash[87147]: cluster 2024-09-13T15:05:12.920763+0000 mgr.y (mgr.26252) 1173 : cluster [DBG] pgmap v1724: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 25117/302502 objects degraded (8.303%) 2024-09-13T15:05:14.242 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:13 smithi099 bash[87147]: cluster 2024-09-13T15:05:12.920763+0000 mgr.y (mgr.26252) 1173 : cluster [DBG] pgmap v1724: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 25117/302502 objects degraded (8.303%) 2024-09-13T15:05:14.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:13 smithi099 bash[84576]: cluster 2024-09-13T15:05:12.920763+0000 mgr.y (mgr.26252) 1173 : cluster [DBG] pgmap v1724: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 25117/302502 objects degraded (8.303%) 2024-09-13T15:05:14.242 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:13 smithi099 bash[84576]: cluster 2024-09-13T15:05:12.920763+0000 mgr.y (mgr.26252) 1173 : cluster [DBG] pgmap v1724: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 25117/302502 objects degraded (8.303%) 2024-09-13T15:05:14.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:14.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:14.357 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:14.372 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 87 16 104585 104569 75.1141 40.625 0.0154159 0.0132801 2024-09-13T15:05:14.382 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:14.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:14.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:14.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:14.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:14.666 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:14.769 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:14.800 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:14.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:14.901 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:13 smithi177 bash[68594]: cluster 2024-09-13T15:05:12.920763+0000 mgr.y (mgr.26252) 1173 : cluster [DBG] pgmap v1724: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 25117/302502 objects degraded (8.303%) 2024-09-13T15:05:14.901 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:13 smithi177 bash[68594]: cluster 2024-09-13T15:05:12.920763+0000 mgr.y (mgr.26252) 1173 : cluster [DBG] pgmap v1724: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 25117/302502 objects degraded (8.303%) 2024-09-13T15:05:14.908 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:14.934 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:15.043 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:15.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:15.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:15.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:15.186 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:15.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:15.289 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:15.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:15.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:15.372 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 88 16 105353 105337 74.806 48 0.00984243 0.0133628 2024-09-13T15:05:15.424 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:15.448 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:15.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:15.552 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:15.584 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:15.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:15.715 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:15.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:15.819 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:15.853 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:15.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:15.957 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:15.988 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:15.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:16.091 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:16.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:16.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:16.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:15 smithi177 bash[68594]: cluster 2024-09-13T15:05:14.921693+0000 mgr.y (mgr.26252) 1174 : cluster [DBG] pgmap v1725: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:16.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:15 smithi177 bash[68594]: cluster 2024-09-13T15:05:14.921693+0000 mgr.y (mgr.26252) 1174 : cluster [DBG] pgmap v1725: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:16.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:16.237 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:15 smithi099 bash[84576]: cluster 2024-09-13T15:05:14.921693+0000 mgr.y (mgr.26252) 1174 : cluster [DBG] pgmap v1725: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:16.237 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:15 smithi099 bash[84576]: cluster 2024-09-13T15:05:14.921693+0000 mgr.y (mgr.26252) 1174 : cluster [DBG] pgmap v1725: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:16.237 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:15 smithi099 bash[87147]: cluster 2024-09-13T15:05:14.921693+0000 mgr.y (mgr.26252) 1174 : cluster [DBG] pgmap v1725: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:16.237 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:15 smithi099 bash[87147]: cluster 2024-09-13T15:05:14.921693+0000 mgr.y (mgr.26252) 1174 : cluster [DBG] pgmap v1725: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 942 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:16.249 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:16.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:16.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:16.372 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 89 16 106299 106283 74.6297 59.125 0.0159289 0.0133945 2024-09-13T15:05:16.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:16.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:16.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:16.509 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:16.615 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:16.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:16.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:16.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:16.774 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:16.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:16.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:16.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:16.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:17.004 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T15:05:17.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:17.021 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[84576]: audit 2024-09-13T15:05:15.996326+0000 mon.c (mon.2) 92 : audit [INF] from='client.? 172.21.15.99:0/1914753995' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.021 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[84576]: audit 2024-09-13T15:05:15.996326+0000 mon.c (mon.2) 92 : audit [INF] from='client.? 172.21.15.99:0/1914753995' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.021 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[84576]: audit 2024-09-13T15:05:15.996783+0000 mon.a (mon.0) 2305 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.022 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[84576]: audit 2024-09-13T15:05:15.996783+0000 mon.a (mon.0) 2305 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.022 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[84576]: cluster 2024-09-13T15:05:16.518464+0000 mon.a (mon.0) 2306 : cluster [WRN] Health check update: Degraded data redundancy: 25901/312567 objects degraded (8.287%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:17.022 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[84576]: cluster 2024-09-13T15:05:16.518464+0000 mon.a (mon.0) 2306 : cluster [WRN] Health check update: Degraded data redundancy: 25901/312567 objects degraded (8.287%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:17.022 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[87147]: audit 2024-09-13T15:05:15.996326+0000 mon.c (mon.2) 92 : audit [INF] from='client.? 172.21.15.99:0/1914753995' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.022 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[87147]: audit 2024-09-13T15:05:15.996326+0000 mon.c (mon.2) 92 : audit [INF] from='client.? 172.21.15.99:0/1914753995' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.022 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[87147]: audit 2024-09-13T15:05:15.996783+0000 mon.a (mon.0) 2305 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.022 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[87147]: audit 2024-09-13T15:05:15.996783+0000 mon.a (mon.0) 2305 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.022 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[87147]: cluster 2024-09-13T15:05:16.518464+0000 mon.a (mon.0) 2306 : cluster [WRN] Health check update: Degraded data redundancy: 25901/312567 objects degraded (8.287%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:17.022 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:16 smithi099 bash[87147]: cluster 2024-09-13T15:05:16.518464+0000 mon.a (mon.0) 2306 : cluster [WRN] Health check update: Degraded data redundancy: 25901/312567 objects degraded (8.287%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:17.041 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:17.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:17.073 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:05:17] "GET /metrics HTTP/1.1" 200 36766 "" "Prometheus/2.51.0" 2024-09-13T15:05:17.148 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:17.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:17.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:17.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:17.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:16 smithi177 bash[68594]: audit 2024-09-13T15:05:15.996326+0000 mon.c (mon.2) 92 : audit [INF] from='client.? 172.21.15.99:0/1914753995' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:16 smithi177 bash[68594]: audit 2024-09-13T15:05:15.996326+0000 mon.c (mon.2) 92 : audit [INF] from='client.? 172.21.15.99:0/1914753995' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:16 smithi177 bash[68594]: audit 2024-09-13T15:05:15.996783+0000 mon.a (mon.0) 2305 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:16 smithi177 bash[68594]: audit 2024-09-13T15:05:15.996783+0000 mon.a (mon.0) 2305 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:16 smithi177 bash[68594]: cluster 2024-09-13T15:05:16.518464+0000 mon.a (mon.0) 2306 : cluster [WRN] Health check update: Degraded data redundancy: 25901/312567 objects degraded (8.287%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:16 smithi177 bash[68594]: cluster 2024-09-13T15:05:16.518464+0000 mon.a (mon.0) 2306 : cluster [WRN] Health check update: Degraded data redundancy: 25901/312567 objects degraded (8.287%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:17.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:17.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:17.372 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 90 6 107201 107195 74.4338 57 0.0244071 0.0134301 2024-09-13T15:05:17.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:17.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:17.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:17.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:17.545 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:17.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:17.673 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:17.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:17.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:17.801 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:17.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:17.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:18.083 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:18.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:18.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:18.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:18.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:18.242 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[87147]: cluster 2024-09-13T15:05:16.922222+0000 mgr.y (mgr.26252) 1175 : cluster [DBG] pgmap v1726: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:18.242 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[87147]: cluster 2024-09-13T15:05:16.922222+0000 mgr.y (mgr.26252) 1175 : cluster [DBG] pgmap v1726: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:18.242 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[87147]: cluster 2024-09-13T15:05:16.986002+0000 mon.a (mon.0) 2307 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[87147]: cluster 2024-09-13T15:05:16.986002+0000 mon.a (mon.0) 2307 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[87147]: audit 2024-09-13T15:05:16.997748+0000 mon.a (mon.0) 2308 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[87147]: audit 2024-09-13T15:05:16.997748+0000 mon.a (mon.0) 2308 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[87147]: cluster 2024-09-13T15:05:17.003515+0000 mon.a (mon.0) 2309 : cluster [DBG] osdmap e1064: 8 total, 7 up, 8 in 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[87147]: cluster 2024-09-13T15:05:17.003515+0000 mon.a (mon.0) 2309 : cluster [DBG] osdmap e1064: 8 total, 7 up, 8 in 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[84576]: cluster 2024-09-13T15:05:16.922222+0000 mgr.y (mgr.26252) 1175 : cluster [DBG] pgmap v1726: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[84576]: cluster 2024-09-13T15:05:16.922222+0000 mgr.y (mgr.26252) 1175 : cluster [DBG] pgmap v1726: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[84576]: cluster 2024-09-13T15:05:16.986002+0000 mon.a (mon.0) 2307 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[84576]: cluster 2024-09-13T15:05:16.986002+0000 mon.a (mon.0) 2307 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[84576]: audit 2024-09-13T15:05:16.997748+0000 mon.a (mon.0) 2308 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[84576]: audit 2024-09-13T15:05:16.997748+0000 mon.a (mon.0) 2308 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:18.243 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[84576]: cluster 2024-09-13T15:05:17.003515+0000 mon.a (mon.0) 2309 : cluster [DBG] osdmap e1064: 8 total, 7 up, 8 in 2024-09-13T15:05:18.244 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:17 smithi099 bash[84576]: cluster 2024-09-13T15:05:17.003515+0000 mon.a (mon.0) 2309 : cluster [DBG] osdmap e1064: 8 total, 7 up, 8 in 2024-09-13T15:05:18.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:18.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:18.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:17 smithi177 bash[68594]: cluster 2024-09-13T15:05:16.922222+0000 mgr.y (mgr.26252) 1175 : cluster [DBG] pgmap v1726: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:18.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:17 smithi177 bash[68594]: cluster 2024-09-13T15:05:16.922222+0000 mgr.y (mgr.26252) 1175 : cluster [DBG] pgmap v1726: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 25901/312567 objects degraded (8.287%) 2024-09-13T15:05:18.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:17 smithi177 bash[68594]: cluster 2024-09-13T15:05:16.986002+0000 mon.a (mon.0) 2307 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:18.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:17 smithi177 bash[68594]: cluster 2024-09-13T15:05:16.986002+0000 mon.a (mon.0) 2307 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:18.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:17 smithi177 bash[68594]: audit 2024-09-13T15:05:16.997748+0000 mon.a (mon.0) 2308 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:18.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:17 smithi177 bash[68594]: audit 2024-09-13T15:05:16.997748+0000 mon.a (mon.0) 2308 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:18.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:17 smithi177 bash[68594]: cluster 2024-09-13T15:05:17.003515+0000 mon.a (mon.0) 2309 : cluster [DBG] osdmap e1064: 8 total, 7 up, 8 in 2024-09-13T15:05:18.357 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:17 smithi177 bash[68594]: cluster 2024-09-13T15:05:17.003515+0000 mon.a (mon.0) 2309 : cluster [DBG] osdmap e1064: 8 total, 7 up, 8 in 2024-09-13T15:05:18.372 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total time run: 90.0101 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total writes made: 107201 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Write size: 65536 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object size: 65536 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Bandwidth (MB/sec): 74.4368 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Bandwidth: 32.9119 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max bandwidth (MB/sec): 196.438 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min bandwidth (MB/sec): 18.9375 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average IOPS: 1190 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev IOPS: 526.59 2024-09-13T15:05:18.373 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max IOPS: 3143 2024-09-13T15:05:18.374 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min IOPS: 303 2024-09-13T15:05:18.374 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average Latency(s): 0.0134301 2024-09-13T15:05:18.374 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Latency(s): 0.0240103 2024-09-13T15:05:18.374 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max latency(s): 0.563801 2024-09-13T15:05:18.374 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min latency(s): 0.00126645 2024-09-13T15:05:18.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:18.383 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T15:05:18.475 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:18.524 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:18.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:18.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:18.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:18.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:18.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:18.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:18.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:18.924 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:19.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:19.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:19 smithi177 bash[68594]: audit 2024-09-13T15:05:18.298667+0000 mon.a (mon.0) 2310 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:19.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:19 smithi177 bash[68594]: audit 2024-09-13T15:05:18.298667+0000 mon.a (mon.0) 2310 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:19.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:19.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:19.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:19.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:19.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:19.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:19.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:19.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:19.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:19 smithi099 bash[84576]: audit 2024-09-13T15:05:18.298667+0000 mon.a (mon.0) 2310 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:19.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:19 smithi099 bash[84576]: audit 2024-09-13T15:05:18.298667+0000 mon.a (mon.0) 2310 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:19.334 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:19 smithi099 bash[87147]: audit 2024-09-13T15:05:18.298667+0000 mon.a (mon.0) 2310 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:19.334 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:19 smithi099 bash[87147]: audit 2024-09-13T15:05:18.298667+0000 mon.a (mon.0) 2310 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:19.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:19.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:19.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:19.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:19.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:19.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:19.686 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:19.721 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:19.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:19.828 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:19.852 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:19.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:19.980 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:19.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:20.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:20.109 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:20.216 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:20.248 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:20.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:20.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:20 smithi099 bash[84576]: cluster 2024-09-13T15:05:18.922864+0000 mgr.y (mgr.26252) 1176 : cluster [DBG] pgmap v1728: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 26144/316827 objects degraded (8.252%) 2024-09-13T15:05:20.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:20 smithi099 bash[84576]: cluster 2024-09-13T15:05:18.922864+0000 mgr.y (mgr.26252) 1176 : cluster [DBG] pgmap v1728: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 26144/316827 objects degraded (8.252%) 2024-09-13T15:05:20.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:20 smithi099 bash[87147]: cluster 2024-09-13T15:05:18.922864+0000 mgr.y (mgr.26252) 1176 : cluster [DBG] pgmap v1728: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 26144/316827 objects degraded (8.252%) 2024-09-13T15:05:20.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:20 smithi099 bash[87147]: cluster 2024-09-13T15:05:18.922864+0000 mgr.y (mgr.26252) 1176 : cluster [DBG] pgmap v1728: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 26144/316827 objects degraded (8.252%) 2024-09-13T15:05:20.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:20.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:05:20 smithi177 bash[61439]: ts=2024-09-13T15:05:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:20.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:20 smithi177 bash[68594]: cluster 2024-09-13T15:05:18.922864+0000 mgr.y (mgr.26252) 1176 : cluster [DBG] pgmap v1728: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 26144/316827 objects degraded (8.252%) 2024-09-13T15:05:20.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:20 smithi177 bash[68594]: cluster 2024-09-13T15:05:18.922864+0000 mgr.y (mgr.26252) 1176 : cluster [DBG] pgmap v1728: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 26144/316827 objects degraded (8.252%) 2024-09-13T15:05:20.370 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:20.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:20.475 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:20.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:20.602 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:20.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:20.722 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:20.743 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:20.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:20.887 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:20.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:20.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:21.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:21.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:21.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:21.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:21.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:21.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:21.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:21.392 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:21.416 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:21.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:21.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:21.557 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:21.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:21.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:21.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:21.787 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:21.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:21.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:21.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:21.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:21.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:22.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:22.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:22.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:22.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:22.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:22.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:22.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:22.315 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:22 smithi177 bash[68594]: cluster 2024-09-13T15:05:20.923705+0000 mgr.y (mgr.26252) 1177 : cluster [DBG] pgmap v1729: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 778 op/s; 26409/319818 objects degraded (8.258%) 2024-09-13T15:05:22.315 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:22 smithi177 bash[68594]: cluster 2024-09-13T15:05:20.923705+0000 mgr.y (mgr.26252) 1177 : cluster [DBG] pgmap v1729: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 778 op/s; 26409/319818 objects degraded (8.258%) 2024-09-13T15:05:22.315 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:22 smithi177 bash[68594]: cluster 2024-09-13T15:05:21.519249+0000 mon.a (mon.0) 2311 : cluster [WRN] Health check update: Degraded data redundancy: 26409/319818 objects degraded (8.258%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:22.315 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:22 smithi177 bash[68594]: cluster 2024-09-13T15:05:21.519249+0000 mon.a (mon.0) 2311 : cluster [WRN] Health check update: Degraded data redundancy: 26409/319818 objects degraded (8.258%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:22 smithi099 bash[84576]: cluster 2024-09-13T15:05:20.923705+0000 mgr.y (mgr.26252) 1177 : cluster [DBG] pgmap v1729: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 778 op/s; 26409/319818 objects degraded (8.258%) 2024-09-13T15:05:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:22 smithi099 bash[84576]: cluster 2024-09-13T15:05:20.923705+0000 mgr.y (mgr.26252) 1177 : cluster [DBG] pgmap v1729: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 778 op/s; 26409/319818 objects degraded (8.258%) 2024-09-13T15:05:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:22 smithi099 bash[84576]: cluster 2024-09-13T15:05:21.519249+0000 mon.a (mon.0) 2311 : cluster [WRN] Health check update: Degraded data redundancy: 26409/319818 objects degraded (8.258%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:22 smithi099 bash[84576]: cluster 2024-09-13T15:05:21.519249+0000 mon.a (mon.0) 2311 : cluster [WRN] Health check update: Degraded data redundancy: 26409/319818 objects degraded (8.258%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:22 smithi099 bash[87147]: cluster 2024-09-13T15:05:20.923705+0000 mgr.y (mgr.26252) 1177 : cluster [DBG] pgmap v1729: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 778 op/s; 26409/319818 objects degraded (8.258%) 2024-09-13T15:05:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:22 smithi099 bash[87147]: cluster 2024-09-13T15:05:20.923705+0000 mgr.y (mgr.26252) 1177 : cluster [DBG] pgmap v1729: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 45 MiB/s wr, 778 op/s; 26409/319818 objects degraded (8.258%) 2024-09-13T15:05:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:22 smithi099 bash[87147]: cluster 2024-09-13T15:05:21.519249+0000 mon.a (mon.0) 2311 : cluster [WRN] Health check update: Degraded data redundancy: 26409/319818 objects degraded (8.258%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:22 smithi099 bash[87147]: cluster 2024-09-13T15:05:21.519249+0000 mon.a (mon.0) 2311 : cluster [WRN] Health check update: Degraded data redundancy: 26409/319818 objects degraded (8.258%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:22.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:22.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:22.445 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:22.478 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:22.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:22.581 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:22.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:22.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:22.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:22.743 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:22.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:22.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:22.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.877 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:22.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:22.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:23.002 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:23.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:23.105 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:23.131 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:23.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:23.235 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:23.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:23.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:23.369 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:23.388 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:23.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:23.492 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:23.513 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:23.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:23.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:23.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:05:23.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:23.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:23.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:23.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:23.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:23.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:24.033 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:24.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:24.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:24.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:24 smithi099 bash[84576]: cluster 2024-09-13T15:05:22.924463+0000 mgr.y (mgr.26252) 1178 : cluster [DBG] pgmap v1730: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 757 op/s; 26224/318123 objects degraded (8.243%) 2024-09-13T15:05:24.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:24 smithi099 bash[84576]: cluster 2024-09-13T15:05:22.924463+0000 mgr.y (mgr.26252) 1178 : cluster [DBG] pgmap v1730: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 757 op/s; 26224/318123 objects degraded (8.243%) 2024-09-13T15:05:24.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:24 smithi099 bash[87147]: cluster 2024-09-13T15:05:22.924463+0000 mgr.y (mgr.26252) 1178 : cluster [DBG] pgmap v1730: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 757 op/s; 26224/318123 objects degraded (8.243%) 2024-09-13T15:05:24.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:24 smithi099 bash[87147]: cluster 2024-09-13T15:05:22.924463+0000 mgr.y (mgr.26252) 1178 : cluster [DBG] pgmap v1730: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 757 op/s; 26224/318123 objects degraded (8.243%) 2024-09-13T15:05:24.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:24.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:24.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:24.292 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:24.305 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:24 smithi177 bash[68594]: cluster 2024-09-13T15:05:22.924463+0000 mgr.y (mgr.26252) 1178 : cluster [DBG] pgmap v1730: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 757 op/s; 26224/318123 objects degraded (8.243%) 2024-09-13T15:05:24.305 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:24 smithi177 bash[68594]: cluster 2024-09-13T15:05:22.924463+0000 mgr.y (mgr.26252) 1178 : cluster [DBG] pgmap v1730: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 757 op/s; 26224/318123 objects degraded (8.243%) 2024-09-13T15:05:24.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:24.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:24.437 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:24.449 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:24 smithi099 bash[193561]: debug 2024-09-13T15:05:24.364+0000 7f58ab410640 -1 Falling back to public interface 2024-09-13T15:05:24.460 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:24.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:24.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:24.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.598 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:24.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:24.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:24.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:24.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:24.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:24.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:24.940 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:24.961 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:24.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:25.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:25.099 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:25.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:25.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:25.234 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:25.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:25.337 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:25.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:25.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:25.471 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:25.503 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:25.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:25.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:25.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:25.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:25.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:25.768 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:25.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:25.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:25.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:26.000 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:26.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:26.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:26.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:26.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:26.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:26.265 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:26.278 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:26 smithi099 bash[84576]: cluster 2024-09-13T15:05:24.925262+0000 mgr.y (mgr.26252) 1179 : cluster [DBG] pgmap v1731: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 712 op/s; 25458/309417 objects degraded (8.228%) 2024-09-13T15:05:26.278 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:26 smithi099 bash[84576]: cluster 2024-09-13T15:05:24.925262+0000 mgr.y (mgr.26252) 1179 : cluster [DBG] pgmap v1731: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 712 op/s; 25458/309417 objects degraded (8.228%) 2024-09-13T15:05:26.279 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:26 smithi099 bash[87147]: cluster 2024-09-13T15:05:24.925262+0000 mgr.y (mgr.26252) 1179 : cluster [DBG] pgmap v1731: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 712 op/s; 25458/309417 objects degraded (8.228%) 2024-09-13T15:05:26.279 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:26 smithi099 bash[87147]: cluster 2024-09-13T15:05:24.925262+0000 mgr.y (mgr.26252) 1179 : cluster [DBG] pgmap v1731: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 712 op/s; 25458/309417 objects degraded (8.228%) 2024-09-13T15:05:26.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:26.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:26.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:26 smithi177 bash[68594]: cluster 2024-09-13T15:05:24.925262+0000 mgr.y (mgr.26252) 1179 : cluster [DBG] pgmap v1731: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 712 op/s; 25458/309417 objects degraded (8.228%) 2024-09-13T15:05:26.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:26 smithi177 bash[68594]: cluster 2024-09-13T15:05:24.925262+0000 mgr.y (mgr.26252) 1179 : cluster [DBG] pgmap v1731: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 712 op/s; 25458/309417 objects degraded (8.228%) 2024-09-13T15:05:26.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:26.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:26.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:26.529 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:26.544 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:05:26.544 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:05:26.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:26.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:26.653 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:26.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:26.795 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:26.823 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:26.824 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:05:26.825 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:05:26.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:26.862 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:05:26.862 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:05:26.891 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:05:26.891 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:05:26.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-13T15:05:26.917 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:05:26.918 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:05:26.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:26.951 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:05:26.952 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:05:26.959 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:26.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:26.976 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:05:26.976 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:05:27.001 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:05:27.001 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:05:27.065 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:27.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:27.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:27.206 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:27.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:27.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:27.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:05:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:05:27] "GET /metrics HTTP/1.1" 200 36767 "" "Prometheus/2.51.0" 2024-09-13T15:05:27.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:27 smithi099 bash[84576]: cluster 2024-09-13T15:05:26.519903+0000 mon.a (mon.0) 2312 : cluster [WRN] Health check update: Degraded data redundancy: 25458/309417 objects degraded (8.228%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:27.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:27 smithi099 bash[84576]: cluster 2024-09-13T15:05:26.519903+0000 mon.a (mon.0) 2312 : cluster [WRN] Health check update: Degraded data redundancy: 25458/309417 objects degraded (8.228%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:27.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:27 smithi099 bash[84576]: audit 2024-09-13T15:05:26.798299+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.99:0/1226276587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:27.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:27 smithi099 bash[84576]: audit 2024-09-13T15:05:26.798299+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.99:0/1226276587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:27 smithi099 bash[87147]: cluster 2024-09-13T15:05:26.519903+0000 mon.a (mon.0) 2312 : cluster [WRN] Health check update: Degraded data redundancy: 25458/309417 objects degraded (8.228%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:27 smithi099 bash[87147]: cluster 2024-09-13T15:05:26.519903+0000 mon.a (mon.0) 2312 : cluster [WRN] Health check update: Degraded data redundancy: 25458/309417 objects degraded (8.228%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:27 smithi099 bash[87147]: audit 2024-09-13T15:05:26.798299+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.99:0/1226276587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:27 smithi099 bash[87147]: audit 2024-09-13T15:05:26.798299+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.99:0/1226276587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:27.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:27.359 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:27.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:27 smithi177 bash[68594]: cluster 2024-09-13T15:05:26.519903+0000 mon.a (mon.0) 2312 : cluster [WRN] Health check update: Degraded data redundancy: 25458/309417 objects degraded (8.228%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:27 smithi177 bash[68594]: cluster 2024-09-13T15:05:26.519903+0000 mon.a (mon.0) 2312 : cluster [WRN] Health check update: Degraded data redundancy: 25458/309417 objects degraded (8.228%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:27 smithi177 bash[68594]: audit 2024-09-13T15:05:26.798299+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.99:0/1226276587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:27 smithi177 bash[68594]: audit 2024-09-13T15:05:26.798299+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.99:0/1226276587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:27.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:27.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:27.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:27.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:27.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:27.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:27.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:27.753 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:27.856 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:27.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:27.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:27.996 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:28.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:28.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:28.136 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:28.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:28 smithi177 bash[68594]: cluster 2024-09-13T15:05:26.925919+0000 mgr.y (mgr.26252) 1180 : cluster [DBG] pgmap v1732: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 766 op/s; 25458/307800 objects degraded (8.271%) 2024-09-13T15:05:28.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:28 smithi177 bash[68594]: cluster 2024-09-13T15:05:26.925919+0000 mgr.y (mgr.26252) 1180 : cluster [DBG] pgmap v1732: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 766 op/s; 25458/307800 objects degraded (8.271%) 2024-09-13T15:05:28.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:28 smithi099 bash[84576]: cluster 2024-09-13T15:05:26.925919+0000 mgr.y (mgr.26252) 1180 : cluster [DBG] pgmap v1732: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 766 op/s; 25458/307800 objects degraded (8.271%) 2024-09-13T15:05:28.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:28 smithi099 bash[84576]: cluster 2024-09-13T15:05:26.925919+0000 mgr.y (mgr.26252) 1180 : cluster [DBG] pgmap v1732: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 766 op/s; 25458/307800 objects degraded (8.271%) 2024-09-13T15:05:28.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:28 smithi099 bash[87147]: cluster 2024-09-13T15:05:26.925919+0000 mgr.y (mgr.26252) 1180 : cluster [DBG] pgmap v1732: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 766 op/s; 25458/307800 objects degraded (8.271%) 2024-09-13T15:05:28.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:28 smithi099 bash[87147]: cluster 2024-09-13T15:05:26.925919+0000 mgr.y (mgr.26252) 1180 : cluster [DBG] pgmap v1732: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 766 op/s; 25458/307800 objects degraded (8.271%) 2024-09-13T15:05:28.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:28.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:28.755 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:28.792 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:28.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:28.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:28.930 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:29.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:29.058 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:29.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:29.159 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2024-09-13T15:05:29.159 INFO:teuthology.orchestra.run.smithi099.stderr:0 2024-09-13T15:05:29.159 INFO:teuthology.orchestra.run.smithi099.stderr:1 2024-09-13T15:05:29.159 INFO:teuthology.orchestra.run.smithi099.stderr:2 2024-09-13T15:05:29.159 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2024-09-13T15:05:29.159 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2024-09-13T15:05:29.160 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator dump block 2024-09-13T15:05:29.160 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator fragmentation block 2024-09-13T15:05:29.160 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator fragmentation histogram block [] [] 2024-09-13T15:05:29.160 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator score block 2024-09-13T15:05:29.160 INFO:teuthology.orchestra.run.smithi099.stderr:config diff 2024-09-13T15:05:29.160 INFO:teuthology.orchestra.run.smithi099.stderr:admin_socket: invalid command 2024-09-13T15:05:29.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:29.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.207 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:29.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:29.308 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:29.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:29.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:29.436 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:29.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:29.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:29.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:29.597 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:29.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:29.701 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:29.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:29.830 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:29.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:29.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:29.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:30.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:30.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:30.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:30.137 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:30.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:30.241 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:30.266 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:30.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:30.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:30 smithi099 bash[84576]: cluster 2024-09-13T15:05:28.926686+0000 mgr.y (mgr.26252) 1181 : cluster [DBG] pgmap v1733: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 767 op/s; 25196/303342 objects degraded (8.306%) 2024-09-13T15:05:30.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:30 smithi099 bash[84576]: cluster 2024-09-13T15:05:28.926686+0000 mgr.y (mgr.26252) 1181 : cluster [DBG] pgmap v1733: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 767 op/s; 25196/303342 objects degraded (8.306%) 2024-09-13T15:05:30.313 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:30 smithi099 bash[87147]: cluster 2024-09-13T15:05:28.926686+0000 mgr.y (mgr.26252) 1181 : cluster [DBG] pgmap v1733: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 767 op/s; 25196/303342 objects degraded (8.306%) 2024-09-13T15:05:30.313 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:30 smithi099 bash[87147]: cluster 2024-09-13T15:05:28.926686+0000 mgr.y (mgr.26252) 1181 : cluster [DBG] pgmap v1733: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 767 op/s; 25196/303342 objects degraded (8.306%) 2024-09-13T15:05:30.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T15:05:30.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:30 smithi177 bash[68594]: cluster 2024-09-13T15:05:28.926686+0000 mgr.y (mgr.26252) 1181 : cluster [DBG] pgmap v1733: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 767 op/s; 25196/303342 objects degraded (8.306%) 2024-09-13T15:05:30.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:30 smithi177 bash[68594]: cluster 2024-09-13T15:05:28.926686+0000 mgr.y (mgr.26252) 1181 : cluster [DBG] pgmap v1733: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 767 op/s; 25196/303342 objects degraded (8.306%) 2024-09-13T15:05:30.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:05:30 smithi177 bash[61439]: ts=2024-09-13T15:05:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:30.373 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:30.424 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:30.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:30.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:30.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:30.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:30.632 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:05:30.633 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:05:30.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:30.691 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:30.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:30.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:30.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:30.919 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:30.943 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:30.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:31.046 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:31.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:31.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:31.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:31.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:31.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:31.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:31.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:31.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:31.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:31.472 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T15:05:31.472 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-3, ['dump_ops_in_flight'] 2024-09-13T15:05:31.480 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:31.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:31.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:31.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:31.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:31.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:31.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:31.823 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:31 smithi099 bash[193561]: debug 2024-09-13T15:05:31.484+0000 7f58ab410640 -1 osd.3 1033 log_to_monitors true 2024-09-13T15:05:31.847 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:31.874 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:31.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:31.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:32.003 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:32.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:32.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:32.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:32.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:32.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:32.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:32.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:32.323 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[193561]: debug 2024-09-13T15:05:32.080+0000 7f589f1b2640 -1 osd.3 1033 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T15:05:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[84576]: cluster 2024-09-13T15:05:30.928128+0000 mgr.y (mgr.26252) 1182 : cluster [DBG] pgmap v1734: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 8.4 MiB/s wr, 847 op/s; 24439/296019 objects degraded (8.256%) 2024-09-13T15:05:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[84576]: cluster 2024-09-13T15:05:30.928128+0000 mgr.y (mgr.26252) 1182 : cluster [DBG] pgmap v1734: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 8.4 MiB/s wr, 847 op/s; 24439/296019 objects degraded (8.256%) 2024-09-13T15:05:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[84576]: audit 2024-09-13T15:05:31.492635+0000 mon.a (mon.0) 2313 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[84576]: audit 2024-09-13T15:05:31.492635+0000 mon.a (mon.0) 2313 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[84576]: audit 2024-09-13T15:05:31.493325+0000 mon.b (mon.1) 15 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[84576]: audit 2024-09-13T15:05:31.493325+0000 mon.b (mon.1) 15 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[84576]: cluster 2024-09-13T15:05:31.521878+0000 mon.a (mon.0) 2314 : cluster [WRN] Health check update: Degraded data redundancy: 24439/296019 objects degraded (8.256%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:32.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[84576]: cluster 2024-09-13T15:05:31.521878+0000 mon.a (mon.0) 2314 : cluster [WRN] Health check update: Degraded data redundancy: 24439/296019 objects degraded (8.256%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:32.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[87147]: cluster 2024-09-13T15:05:30.928128+0000 mgr.y (mgr.26252) 1182 : cluster [DBG] pgmap v1734: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 8.4 MiB/s wr, 847 op/s; 24439/296019 objects degraded (8.256%) 2024-09-13T15:05:32.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[87147]: cluster 2024-09-13T15:05:30.928128+0000 mgr.y (mgr.26252) 1182 : cluster [DBG] pgmap v1734: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 8.4 MiB/s wr, 847 op/s; 24439/296019 objects degraded (8.256%) 2024-09-13T15:05:32.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[87147]: audit 2024-09-13T15:05:31.492635+0000 mon.a (mon.0) 2313 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[87147]: audit 2024-09-13T15:05:31.492635+0000 mon.a (mon.0) 2313 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[87147]: audit 2024-09-13T15:05:31.493325+0000 mon.b (mon.1) 15 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[87147]: audit 2024-09-13T15:05:31.493325+0000 mon.b (mon.1) 15 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[87147]: cluster 2024-09-13T15:05:31.521878+0000 mon.a (mon.0) 2314 : cluster [WRN] Health check update: Degraded data redundancy: 24439/296019 objects degraded (8.256%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:32.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:32 smithi099 bash[87147]: cluster 2024-09-13T15:05:31.521878+0000 mon.a (mon.0) 2314 : cluster [WRN] Health check update: Degraded data redundancy: 24439/296019 objects degraded (8.256%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:32.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:32 smithi177 bash[68594]: cluster 2024-09-13T15:05:30.928128+0000 mgr.y (mgr.26252) 1182 : cluster [DBG] pgmap v1734: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 8.4 MiB/s wr, 847 op/s; 24439/296019 objects degraded (8.256%) 2024-09-13T15:05:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:32 smithi177 bash[68594]: cluster 2024-09-13T15:05:30.928128+0000 mgr.y (mgr.26252) 1182 : cluster [DBG] pgmap v1734: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 8.4 MiB/s wr, 847 op/s; 24439/296019 objects degraded (8.256%) 2024-09-13T15:05:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:32 smithi177 bash[68594]: audit 2024-09-13T15:05:31.492635+0000 mon.a (mon.0) 2313 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:32 smithi177 bash[68594]: audit 2024-09-13T15:05:31.492635+0000 mon.a (mon.0) 2313 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:32 smithi177 bash[68594]: audit 2024-09-13T15:05:31.493325+0000 mon.b (mon.1) 15 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:32 smithi177 bash[68594]: audit 2024-09-13T15:05:31.493325+0000 mon.b (mon.1) 15 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T15:05:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:32 smithi177 bash[68594]: cluster 2024-09-13T15:05:31.521878+0000 mon.a (mon.0) 2314 : cluster [WRN] Health check update: Degraded data redundancy: 24439/296019 objects degraded (8.256%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:32 smithi177 bash[68594]: cluster 2024-09-13T15:05:31.521878+0000 mon.a (mon.0) 2314 : cluster [WRN] Health check update: Degraded data redundancy: 24439/296019 objects degraded (8.256%), 7 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:32.385 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:32.416 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:32.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:32.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:32.552 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:32.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:32.634 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:05:32.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:32.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:32.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:32.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:32.822 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:32.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:32.928 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:32.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:32.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:33.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:33.097 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:33.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:33.203 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:33.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:33 smithi177 bash[68594]: audit 2024-09-13T15:05:32.031027+0000 mon.a (mon.0) 2315 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T15:05:33.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:33 smithi177 bash[68594]: audit 2024-09-13T15:05:32.031027+0000 mon.a (mon.0) 2315 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T15:05:33.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:33 smithi177 bash[68594]: cluster 2024-09-13T15:05:32.038017+0000 mon.a (mon.0) 2316 : cluster [DBG] osdmap e1065: 8 total, 7 up, 8 in 2024-09-13T15:05:33.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:33 smithi177 bash[68594]: cluster 2024-09-13T15:05:32.038017+0000 mon.a (mon.0) 2316 : cluster [DBG] osdmap e1065: 8 total, 7 up, 8 in 2024-09-13T15:05:33.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:33 smithi177 bash[68594]: audit 2024-09-13T15:05:32.041867+0000 mon.b (mon.1) 16 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:33 smithi177 bash[68594]: audit 2024-09-13T15:05:32.041867+0000 mon.b (mon.1) 16 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:33 smithi177 bash[68594]: audit 2024-09-13T15:05:32.048255+0000 mon.a (mon.0) 2317 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:33 smithi177 bash[68594]: audit 2024-09-13T15:05:32.048255+0000 mon.a (mon.0) 2317 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.223 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:33.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:33.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[84576]: audit 2024-09-13T15:05:32.031027+0000 mon.a (mon.0) 2315 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T15:05:33.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[84576]: audit 2024-09-13T15:05:32.031027+0000 mon.a (mon.0) 2315 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T15:05:33.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[84576]: cluster 2024-09-13T15:05:32.038017+0000 mon.a (mon.0) 2316 : cluster [DBG] osdmap e1065: 8 total, 7 up, 8 in 2024-09-13T15:05:33.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[84576]: cluster 2024-09-13T15:05:32.038017+0000 mon.a (mon.0) 2316 : cluster [DBG] osdmap e1065: 8 total, 7 up, 8 in 2024-09-13T15:05:33.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[84576]: audit 2024-09-13T15:05:32.041867+0000 mon.b (mon.1) 16 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[84576]: audit 2024-09-13T15:05:32.041867+0000 mon.b (mon.1) 16 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[84576]: audit 2024-09-13T15:05:32.048255+0000 mon.a (mon.0) 2317 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[84576]: audit 2024-09-13T15:05:32.048255+0000 mon.a (mon.0) 2317 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[87147]: audit 2024-09-13T15:05:32.031027+0000 mon.a (mon.0) 2315 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T15:05:33.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[87147]: audit 2024-09-13T15:05:32.031027+0000 mon.a (mon.0) 2315 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T15:05:33.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[87147]: cluster 2024-09-13T15:05:32.038017+0000 mon.a (mon.0) 2316 : cluster [DBG] osdmap e1065: 8 total, 7 up, 8 in 2024-09-13T15:05:33.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[87147]: cluster 2024-09-13T15:05:32.038017+0000 mon.a (mon.0) 2316 : cluster [DBG] osdmap e1065: 8 total, 7 up, 8 in 2024-09-13T15:05:33.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[87147]: audit 2024-09-13T15:05:32.041867+0000 mon.b (mon.1) 16 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[87147]: audit 2024-09-13T15:05:32.041867+0000 mon.b (mon.1) 16 : audit [INF] from='osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[87147]: audit 2024-09-13T15:05:32.048255+0000 mon.a (mon.0) 2317 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:33 smithi099 bash[87147]: audit 2024-09-13T15:05:32.048255+0000 mon.a (mon.0) 2317 : audit [INF] from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi099", "root=default"]} : dispatch 2024-09-13T15:05:33.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:33.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:33.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:33.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:33.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:33.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:33.577 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:33.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:33.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:33.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:33.734 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:33.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:33.837 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:33.865 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:33.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:33.968 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:33.989 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:33.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:33.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:34.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: cluster 2024-09-13T15:05:32.928902+0000 mgr.y (mgr.26252) 1183 : cluster [DBG] pgmap v1736: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 24194/293766 objects degraded (8.236%) 2024-09-13T15:05:34.068 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: cluster 2024-09-13T15:05:32.928902+0000 mgr.y (mgr.26252) 1183 : cluster [DBG] pgmap v1736: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 24194/293766 objects degraded (8.236%) 2024-09-13T15:05:34.099 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:34.120 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:34.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:34.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:34.250 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:34.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: cluster 2024-09-13T15:05:33.042232+0000 mon.a (mon.0) 2318 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T15:05:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: cluster 2024-09-13T15:05:33.042232+0000 mon.a (mon.0) 2318 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: cluster 2024-09-13T15:05:33.061247+0000 mon.a (mon.0) 2319 : cluster [INF] osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148] boot 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: cluster 2024-09-13T15:05:33.061247+0000 mon.a (mon.0) 2319 : cluster [INF] osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148] boot 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: cluster 2024-09-13T15:05:33.061292+0000 mon.a (mon.0) 2320 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: cluster 2024-09-13T15:05:33.061292+0000 mon.a (mon.0) 2320 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: audit 2024-09-13T15:05:33.064061+0000 mon.a (mon.0) 2321 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: audit 2024-09-13T15:05:33.064061+0000 mon.a (mon.0) 2321 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: audit 2024-09-13T15:05:33.305310+0000 mon.a (mon.0) 2322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: audit 2024-09-13T15:05:33.305310+0000 mon.a (mon.0) 2322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: audit 2024-09-13T15:05:33.307106+0000 mon.a (mon.0) 2323 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[84576]: audit 2024-09-13T15:05:33.307106+0000 mon.a (mon.0) 2323 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: cluster 2024-09-13T15:05:32.928902+0000 mgr.y (mgr.26252) 1183 : cluster [DBG] pgmap v1736: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 24194/293766 objects degraded (8.236%) 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: cluster 2024-09-13T15:05:32.928902+0000 mgr.y (mgr.26252) 1183 : cluster [DBG] pgmap v1736: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 24194/293766 objects degraded (8.236%) 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: cluster 2024-09-13T15:05:33.042232+0000 mon.a (mon.0) 2318 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: cluster 2024-09-13T15:05:33.042232+0000 mon.a (mon.0) 2318 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: cluster 2024-09-13T15:05:33.061247+0000 mon.a (mon.0) 2319 : cluster [INF] osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148] boot 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: cluster 2024-09-13T15:05:33.061247+0000 mon.a (mon.0) 2319 : cluster [INF] osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148] boot 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: cluster 2024-09-13T15:05:33.061292+0000 mon.a (mon.0) 2320 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: cluster 2024-09-13T15:05:33.061292+0000 mon.a (mon.0) 2320 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: audit 2024-09-13T15:05:33.064061+0000 mon.a (mon.0) 2321 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: audit 2024-09-13T15:05:33.064061+0000 mon.a (mon.0) 2321 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: audit 2024-09-13T15:05:33.305310+0000 mon.a (mon.0) 2322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: audit 2024-09-13T15:05:33.305310+0000 mon.a (mon.0) 2322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: audit 2024-09-13T15:05:33.307106+0000 mon.a (mon.0) 2323 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:34 smithi099 bash[87147]: audit 2024-09-13T15:05:33.307106+0000 mon.a (mon.0) 2323 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:34.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: cluster 2024-09-13T15:05:32.928902+0000 mgr.y (mgr.26252) 1183 : cluster [DBG] pgmap v1736: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 24194/293766 objects degraded (8.236%) 2024-09-13T15:05:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: cluster 2024-09-13T15:05:32.928902+0000 mgr.y (mgr.26252) 1183 : cluster [DBG] pgmap v1736: 89 pgs: 26 active+undersized, 7 active+undersized+degraded, 56 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 24194/293766 objects degraded (8.236%) 2024-09-13T15:05:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: cluster 2024-09-13T15:05:33.042232+0000 mon.a (mon.0) 2318 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T15:05:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: cluster 2024-09-13T15:05:33.042232+0000 mon.a (mon.0) 2318 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T15:05:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: cluster 2024-09-13T15:05:33.061247+0000 mon.a (mon.0) 2319 : cluster [INF] osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148] boot 2024-09-13T15:05:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: cluster 2024-09-13T15:05:33.061247+0000 mon.a (mon.0) 2319 : cluster [INF] osd.3 [v2:172.21.15.99:6826/2771933148,v1:172.21.15.99:6827/2771933148] boot 2024-09-13T15:05:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: cluster 2024-09-13T15:05:33.061292+0000 mon.a (mon.0) 2320 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2024-09-13T15:05:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: cluster 2024-09-13T15:05:33.061292+0000 mon.a (mon.0) 2320 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2024-09-13T15:05:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: audit 2024-09-13T15:05:33.064061+0000 mon.a (mon.0) 2321 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T15:05:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: audit 2024-09-13T15:05:33.064061+0000 mon.a (mon.0) 2321 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T15:05:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: audit 2024-09-13T15:05:33.305310+0000 mon.a (mon.0) 2322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:05:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: audit 2024-09-13T15:05:33.305310+0000 mon.a (mon.0) 2322 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:05:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: audit 2024-09-13T15:05:33.307106+0000 mon.a (mon.0) 2323 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:34 smithi177 bash[68594]: audit 2024-09-13T15:05:33.307106+0000 mon.a (mon.0) 2323 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:34.380 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:34.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:34.482 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:34.511 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:34.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:34.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:34.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:34.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:34.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:34.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:34.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:34.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:35.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:35.010 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:35.034 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:35.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:35.137 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:35.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:35.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:35.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:35.273 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:35 smithi177 bash[68594]: cluster 2024-09-13T15:05:34.057697+0000 mon.a (mon.0) 2324 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2024-09-13T15:05:35.273 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:35 smithi177 bash[68594]: cluster 2024-09-13T15:05:34.057697+0000 mon.a (mon.0) 2324 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2024-09-13T15:05:35.273 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:35 smithi177 bash[68594]: cluster 2024-09-13T15:05:35.060860+0000 mon.a (mon.0) 2325 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2024-09-13T15:05:35.274 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:35 smithi177 bash[68594]: cluster 2024-09-13T15:05:35.060860+0000 mon.a (mon.0) 2325 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2024-09-13T15:05:35.287 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:35.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:35.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:35.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:35.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:35.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:35.543 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:35.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:35.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:35 smithi099 bash[84576]: cluster 2024-09-13T15:05:34.057697+0000 mon.a (mon.0) 2324 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2024-09-13T15:05:35.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:35 smithi099 bash[84576]: cluster 2024-09-13T15:05:34.057697+0000 mon.a (mon.0) 2324 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2024-09-13T15:05:35.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:35 smithi099 bash[84576]: cluster 2024-09-13T15:05:35.060860+0000 mon.a (mon.0) 2325 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2024-09-13T15:05:35.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:35 smithi099 bash[84576]: cluster 2024-09-13T15:05:35.060860+0000 mon.a (mon.0) 2325 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2024-09-13T15:05:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:35 smithi099 bash[87147]: cluster 2024-09-13T15:05:34.057697+0000 mon.a (mon.0) 2324 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2024-09-13T15:05:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:35 smithi099 bash[87147]: cluster 2024-09-13T15:05:34.057697+0000 mon.a (mon.0) 2324 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2024-09-13T15:05:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:35 smithi099 bash[87147]: cluster 2024-09-13T15:05:35.060860+0000 mon.a (mon.0) 2325 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2024-09-13T15:05:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:35 smithi099 bash[87147]: cluster 2024-09-13T15:05:35.060860+0000 mon.a (mon.0) 2325 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2024-09-13T15:05:35.646 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:35.752 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:35.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:35.859 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:35.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:35.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:35.991 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:36.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:36.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:36.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:36.136 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:36 smithi177 bash[68594]: cluster 2024-09-13T15:05:34.930186+0000 mgr.y (mgr.26252) 1184 : cluster [DBG] pgmap v1739: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 21 active+undersized, 5 active+undersized+degraded, 59 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 18199/288102 objects degraded (6.317%) 2024-09-13T15:05:36.136 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:36 smithi177 bash[68594]: cluster 2024-09-13T15:05:34.930186+0000 mgr.y (mgr.26252) 1184 : cluster [DBG] pgmap v1739: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 21 active+undersized, 5 active+undersized+degraded, 59 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 18199/288102 objects degraded (6.317%) 2024-09-13T15:05:36.150 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:36.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:36.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:36.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:36.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:36.396 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:36 smithi099 bash[84576]: cluster 2024-09-13T15:05:34.930186+0000 mgr.y (mgr.26252) 1184 : cluster [DBG] pgmap v1739: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 21 active+undersized, 5 active+undersized+degraded, 59 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 18199/288102 objects degraded (6.317%) 2024-09-13T15:05:36.396 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:36 smithi099 bash[84576]: cluster 2024-09-13T15:05:34.930186+0000 mgr.y (mgr.26252) 1184 : cluster [DBG] pgmap v1739: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 21 active+undersized, 5 active+undersized+degraded, 59 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 18199/288102 objects degraded (6.317%) 2024-09-13T15:05:36.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:36 smithi099 bash[87147]: cluster 2024-09-13T15:05:34.930186+0000 mgr.y (mgr.26252) 1184 : cluster [DBG] pgmap v1739: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 21 active+undersized, 5 active+undersized+degraded, 59 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 18199/288102 objects degraded (6.317%) 2024-09-13T15:05:36.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:36 smithi099 bash[87147]: cluster 2024-09-13T15:05:34.930186+0000 mgr.y (mgr.26252) 1184 : cluster [DBG] pgmap v1739: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 21 active+undersized, 5 active+undersized+degraded, 59 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 18199/288102 objects degraded (6.317%) 2024-09-13T15:05:36.416 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:36.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:36.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:05:36.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:36.545 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:36.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:36.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:36.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:36.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:36.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:36.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:36.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:36.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:36.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:36.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:37.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:37.051 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:37.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:37.056 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:37.153 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:37.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:37.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:37.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:37.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:37 smithi177 bash[68594]: cluster 2024-09-13T15:05:36.525961+0000 mon.a (mon.0) 2326 : cluster [WRN] Health check update: Degraded data redundancy: 18199/288102 objects degraded (6.317%), 5 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:37.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:37 smithi177 bash[68594]: cluster 2024-09-13T15:05:36.525961+0000 mon.a (mon.0) 2326 : cluster [WRN] Health check update: Degraded data redundancy: 18199/288102 objects degraded (6.317%), 5 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:37.322 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:37.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:37.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:37 smithi099 bash[84576]: cluster 2024-09-13T15:05:36.525961+0000 mon.a (mon.0) 2326 : cluster [WRN] Health check update: Degraded data redundancy: 18199/288102 objects degraded (6.317%), 5 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:37.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:37 smithi099 bash[84576]: cluster 2024-09-13T15:05:36.525961+0000 mon.a (mon.0) 2326 : cluster [WRN] Health check update: Degraded data redundancy: 18199/288102 objects degraded (6.317%), 5 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:37.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:05:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:05:37] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.51.0" 2024-09-13T15:05:37.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:37 smithi099 bash[87147]: cluster 2024-09-13T15:05:36.525961+0000 mon.a (mon.0) 2326 : cluster [WRN] Health check update: Degraded data redundancy: 18199/288102 objects degraded (6.317%), 5 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:37.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:37 smithi099 bash[87147]: cluster 2024-09-13T15:05:36.525961+0000 mon.a (mon.0) 2326 : cluster [WRN] Health check update: Degraded data redundancy: 18199/288102 objects degraded (6.317%), 5 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2024-09-13T15:05:37.424 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:37.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:37.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:37.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:37.587 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:37.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:37.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:37.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:37.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:37.835 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:37.861 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:37.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:37.964 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:38.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:38.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:38.104 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:38.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:38 smithi177 bash[68594]: cluster 2024-09-13T15:05:36.931015+0000 mgr.y (mgr.26252) 1185 : cluster [DBG] pgmap v1741: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 18 active+undersized, 5 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 18199/287088 objects degraded (6.339%) 2024-09-13T15:05:38.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:38 smithi177 bash[68594]: cluster 2024-09-13T15:05:36.931015+0000 mgr.y (mgr.26252) 1185 : cluster [DBG] pgmap v1741: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 18 active+undersized, 5 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 18199/287088 objects degraded (6.339%) 2024-09-13T15:05:38.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:38.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:38.240 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:38.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:38 smithi099 bash[84576]: cluster 2024-09-13T15:05:36.931015+0000 mgr.y (mgr.26252) 1185 : cluster [DBG] pgmap v1741: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 18 active+undersized, 5 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 18199/287088 objects degraded (6.339%) 2024-09-13T15:05:38.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:38 smithi099 bash[84576]: cluster 2024-09-13T15:05:36.931015+0000 mgr.y (mgr.26252) 1185 : cluster [DBG] pgmap v1741: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 18 active+undersized, 5 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 18199/287088 objects degraded (6.339%) 2024-09-13T15:05:38.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:38 smithi099 bash[87147]: cluster 2024-09-13T15:05:36.931015+0000 mgr.y (mgr.26252) 1185 : cluster [DBG] pgmap v1741: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 18 active+undersized, 5 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 18199/287088 objects degraded (6.339%) 2024-09-13T15:05:38.250 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:38 smithi099 bash[87147]: cluster 2024-09-13T15:05:36.931015+0000 mgr.y (mgr.26252) 1185 : cluster [DBG] pgmap v1741: 89 pgs: 1 remapped+peering, 1 activating, 2 peering, 18 active+undersized, 5 active+undersized+degraded, 62 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 18199/287088 objects degraded (6.339%) 2024-09-13T15:05:38.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:38.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:38.386 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:38.441 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:38.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:38.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:38.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:38.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:38.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:38.700 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:38.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:38.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:38.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:38.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:38.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:38.982 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:38.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:39.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:39.108 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:39 smithi099 bash[87147]: cluster 2024-09-13T15:05:38.931677+0000 mgr.y (mgr.26252) 1186 : cluster [DBG] pgmap v1742: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 19157/283752 objects degraded (6.751%); 661 KiB/s, 10 objects/s recovering 2024-09-13T15:05:39.108 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:39 smithi099 bash[87147]: cluster 2024-09-13T15:05:38.931677+0000 mgr.y (mgr.26252) 1186 : cluster [DBG] pgmap v1742: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 19157/283752 objects degraded (6.751%); 661 KiB/s, 10 objects/s recovering 2024-09-13T15:05:39.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:39.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:39.218 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:39.241 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:39.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:39.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:39 smithi177 bash[68594]: cluster 2024-09-13T15:05:38.931677+0000 mgr.y (mgr.26252) 1186 : cluster [DBG] pgmap v1742: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 19157/283752 objects degraded (6.751%); 661 KiB/s, 10 objects/s recovering 2024-09-13T15:05:39.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:39 smithi177 bash[68594]: cluster 2024-09-13T15:05:38.931677+0000 mgr.y (mgr.26252) 1186 : cluster [DBG] pgmap v1742: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 19157/283752 objects degraded (6.751%); 661 KiB/s, 10 objects/s recovering 2024-09-13T15:05:39.372 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:39.377 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:39 smithi099 bash[84576]: cluster 2024-09-13T15:05:38.931677+0000 mgr.y (mgr.26252) 1186 : cluster [DBG] pgmap v1742: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 19157/283752 objects degraded (6.751%); 661 KiB/s, 10 objects/s recovering 2024-09-13T15:05:39.377 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:39 smithi099 bash[84576]: cluster 2024-09-13T15:05:38.931677+0000 mgr.y (mgr.26252) 1186 : cluster [DBG] pgmap v1742: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 19157/283752 objects degraded (6.751%); 661 KiB/s, 10 objects/s recovering 2024-09-13T15:05:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:39.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:39.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:39.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:39.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:39.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:39.747 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:39.773 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:39.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:39.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:39.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:40.011 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:40.043 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:40.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:40.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:40.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:40.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:40.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:40.299 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:05:40 smithi177 bash[61439]: ts=2024-09-13T15:05:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:40.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:40.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:40.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:40.447 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:40.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:40.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:40.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:40.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:40.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:40.713 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:40.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:40.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:40.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:40.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:40.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:40.996 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:41.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:41.101 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:41.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:41.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:41.130 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:41.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.268 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:41.371 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:41.401 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:41.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:41.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:41.539 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:41.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:41.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:41.774 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:41.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:41.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:41.908 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:41.938 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:41.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:42.041 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:42.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:42.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:42.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:41 smithi099 bash[84576]: cluster 2024-09-13T15:05:40.932234+0000 mgr.y (mgr.26252) 1187 : cluster [DBG] pgmap v1743: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 25711/280029 objects degraded (9.182%); 504 KiB/s, 7 objects/s recovering 2024-09-13T15:05:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:41 smithi099 bash[84576]: cluster 2024-09-13T15:05:40.932234+0000 mgr.y (mgr.26252) 1187 : cluster [DBG] pgmap v1743: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 25711/280029 objects degraded (9.182%); 504 KiB/s, 7 objects/s recovering 2024-09-13T15:05:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:41 smithi099 bash[84576]: cluster 2024-09-13T15:05:41.526815+0000 mon.a (mon.0) 2327 : cluster [WRN] Health check update: Degraded data redundancy: 25711/280029 objects degraded (9.182%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:41 smithi099 bash[84576]: cluster 2024-09-13T15:05:41.526815+0000 mon.a (mon.0) 2327 : cluster [WRN] Health check update: Degraded data redundancy: 25711/280029 objects degraded (9.182%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:42.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:41 smithi099 bash[87147]: cluster 2024-09-13T15:05:40.932234+0000 mgr.y (mgr.26252) 1187 : cluster [DBG] pgmap v1743: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 25711/280029 objects degraded (9.182%); 504 KiB/s, 7 objects/s recovering 2024-09-13T15:05:42.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:41 smithi099 bash[87147]: cluster 2024-09-13T15:05:40.932234+0000 mgr.y (mgr.26252) 1187 : cluster [DBG] pgmap v1743: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 25711/280029 objects degraded (9.182%); 504 KiB/s, 7 objects/s recovering 2024-09-13T15:05:42.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:41 smithi099 bash[87147]: cluster 2024-09-13T15:05:41.526815+0000 mon.a (mon.0) 2327 : cluster [WRN] Health check update: Degraded data redundancy: 25711/280029 objects degraded (9.182%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:42.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:41 smithi099 bash[87147]: cluster 2024-09-13T15:05:41.526815+0000 mon.a (mon.0) 2327 : cluster [WRN] Health check update: Degraded data redundancy: 25711/280029 objects degraded (9.182%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:42.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:42.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:42.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:42.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:42.323 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:41 smithi177 bash[68594]: cluster 2024-09-13T15:05:40.932234+0000 mgr.y (mgr.26252) 1187 : cluster [DBG] pgmap v1743: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 25711/280029 objects degraded (9.182%); 504 KiB/s, 7 objects/s recovering 2024-09-13T15:05:42.323 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:41 smithi177 bash[68594]: cluster 2024-09-13T15:05:40.932234+0000 mgr.y (mgr.26252) 1187 : cluster [DBG] pgmap v1743: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 25711/280029 objects degraded (9.182%); 504 KiB/s, 7 objects/s recovering 2024-09-13T15:05:42.323 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:41 smithi177 bash[68594]: cluster 2024-09-13T15:05:41.526815+0000 mon.a (mon.0) 2327 : cluster [WRN] Health check update: Degraded data redundancy: 25711/280029 objects degraded (9.182%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:42.323 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:41 smithi177 bash[68594]: cluster 2024-09-13T15:05:41.526815+0000 mon.a (mon.0) 2327 : cluster [WRN] Health check update: Degraded data redundancy: 25711/280029 objects degraded (9.182%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:42.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:42.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:42.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:42.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:42.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:42.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:42.597 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:42.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:42.705 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:42.739 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:42.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:42.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:42.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:42.964 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:42.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:42.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:43.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:43.121 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:43.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:43.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:43.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:43.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:43.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:43.384 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:43.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:43.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:43.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:43.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:43.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:43.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:43.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:43.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:43.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:43.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:43.872 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:43.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:43.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:44.005 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:44.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:44.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:44.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:44.154 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:43 smithi177 bash[68594]: cluster 2024-09-13T15:05:42.932680+0000 mgr.y (mgr.26252) 1188 : cluster [DBG] pgmap v1744: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 25711/277599 objects degraded (9.262%); 447 KiB/s, 7 objects/s recovering 2024-09-13T15:05:44.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:43 smithi177 bash[68594]: cluster 2024-09-13T15:05:42.932680+0000 mgr.y (mgr.26252) 1188 : cluster [DBG] pgmap v1744: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 25711/277599 objects degraded (9.262%); 447 KiB/s, 7 objects/s recovering 2024-09-13T15:05:44.168 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:44.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:44.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:44.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:44.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:44.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:43 smithi099 bash[84576]: cluster 2024-09-13T15:05:42.932680+0000 mgr.y (mgr.26252) 1188 : cluster [DBG] pgmap v1744: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 25711/277599 objects degraded (9.262%); 447 KiB/s, 7 objects/s recovering 2024-09-13T15:05:44.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:43 smithi099 bash[84576]: cluster 2024-09-13T15:05:42.932680+0000 mgr.y (mgr.26252) 1188 : cluster [DBG] pgmap v1744: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 25711/277599 objects degraded (9.262%); 447 KiB/s, 7 objects/s recovering 2024-09-13T15:05:44.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:43 smithi099 bash[87147]: cluster 2024-09-13T15:05:42.932680+0000 mgr.y (mgr.26252) 1188 : cluster [DBG] pgmap v1744: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 25711/277599 objects degraded (9.262%); 447 KiB/s, 7 objects/s recovering 2024-09-13T15:05:44.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:43 smithi099 bash[87147]: cluster 2024-09-13T15:05:42.932680+0000 mgr.y (mgr.26252) 1188 : cluster [DBG] pgmap v1744: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 25711/277599 objects degraded (9.262%); 447 KiB/s, 7 objects/s recovering 2024-09-13T15:05:44.400 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:44.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:44.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:44.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:44.566 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:44.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:44.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:44.708 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:44.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:44.813 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:44.852 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:44.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:44.956 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:44.982 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:44.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:45.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:45.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:45.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:45.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:45.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:45.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:45.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:45.388 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:45.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:45.491 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:45.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:45.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:45.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:45.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:45.754 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:45.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:45.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:45.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:45.919 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:45.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:46.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:46.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:45 smithi177 bash[68594]: cluster 2024-09-13T15:05:44.933709+0000 mgr.y (mgr.26252) 1189 : cluster [DBG] pgmap v1745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 25592/271383 objects degraded (9.430%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T15:05:46.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:45 smithi177 bash[68594]: cluster 2024-09-13T15:05:44.933709+0000 mgr.y (mgr.26252) 1189 : cluster [DBG] pgmap v1745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 25592/271383 objects degraded (9.430%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T15:05:46.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:46.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:46.147 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:46.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:46.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:46.275 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:46.307 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:46.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:46.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:45 smithi099 bash[84576]: cluster 2024-09-13T15:05:44.933709+0000 mgr.y (mgr.26252) 1189 : cluster [DBG] pgmap v1745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 25592/271383 objects degraded (9.430%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T15:05:46.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:45 smithi099 bash[84576]: cluster 2024-09-13T15:05:44.933709+0000 mgr.y (mgr.26252) 1189 : cluster [DBG] pgmap v1745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 25592/271383 objects degraded (9.430%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T15:05:46.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:45 smithi099 bash[87147]: cluster 2024-09-13T15:05:44.933709+0000 mgr.y (mgr.26252) 1189 : cluster [DBG] pgmap v1745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 25592/271383 objects degraded (9.430%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T15:05:46.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:45 smithi099 bash[87147]: cluster 2024-09-13T15:05:44.933709+0000 mgr.y (mgr.26252) 1189 : cluster [DBG] pgmap v1745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 25592/271383 objects degraded (9.430%); 1.1 MiB/s, 18 objects/s recovering 2024-09-13T15:05:46.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:46.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:46.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:46.557 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:46.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:46.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:46.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:46.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:46.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:46.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:46.823 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:46.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:46.926 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:46.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:46.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:47.074 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:47.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:46 smithi177 bash[68594]: cluster 2024-09-13T15:05:46.527652+0000 mon.a (mon.0) 2328 : cluster [WRN] Health check update: Degraded data redundancy: 25592/271383 objects degraded (9.430%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:47.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:46 smithi177 bash[68594]: cluster 2024-09-13T15:05:46.527652+0000 mon.a (mon.0) 2328 : cluster [WRN] Health check update: Degraded data redundancy: 25592/271383 objects degraded (9.430%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:47.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:47.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:47.198 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:47.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:47.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:47.265 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:46 smithi099 bash[84576]: cluster 2024-09-13T15:05:46.527652+0000 mon.a (mon.0) 2328 : cluster [WRN] Health check update: Degraded data redundancy: 25592/271383 objects degraded (9.430%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:47.265 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:46 smithi099 bash[84576]: cluster 2024-09-13T15:05:46.527652+0000 mon.a (mon.0) 2328 : cluster [WRN] Health check update: Degraded data redundancy: 25592/271383 objects degraded (9.430%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:47.265 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:05:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:05:47] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.51.0" 2024-09-13T15:05:47.266 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:46 smithi099 bash[87147]: cluster 2024-09-13T15:05:46.527652+0000 mon.a (mon.0) 2328 : cluster [WRN] Health check update: Degraded data redundancy: 25592/271383 objects degraded (9.430%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:47.266 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:46 smithi099 bash[87147]: cluster 2024-09-13T15:05:46.527652+0000 mon.a (mon.0) 2328 : cluster [WRN] Health check update: Degraded data redundancy: 25592/271383 objects degraded (9.430%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:47.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:47.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:47.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:47.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:47.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:47.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:47.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:47.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:47.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:47.720 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:47.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:47.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:47.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:47.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:47.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:48.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:48.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:48.125 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:48 smithi099 bash[84576]: cluster 2024-09-13T15:05:46.934788+0000 mgr.y (mgr.26252) 1190 : cluster [DBG] pgmap v1746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 25592/270213 objects degraded (9.471%); 975 KiB/s, 15 objects/s recovering 2024-09-13T15:05:48.125 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:48 smithi099 bash[84576]: cluster 2024-09-13T15:05:46.934788+0000 mgr.y (mgr.26252) 1190 : cluster [DBG] pgmap v1746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 25592/270213 objects degraded (9.471%); 975 KiB/s, 15 objects/s recovering 2024-09-13T15:05:48.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:48 smithi099 bash[87147]: cluster 2024-09-13T15:05:46.934788+0000 mgr.y (mgr.26252) 1190 : cluster [DBG] pgmap v1746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 25592/270213 objects degraded (9.471%); 975 KiB/s, 15 objects/s recovering 2024-09-13T15:05:48.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:48 smithi099 bash[87147]: cluster 2024-09-13T15:05:46.934788+0000 mgr.y (mgr.26252) 1190 : cluster [DBG] pgmap v1746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 25592/270213 objects degraded (9.471%); 975 KiB/s, 15 objects/s recovering 2024-09-13T15:05:48.140 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:48.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:48.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:48.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:48.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:48 smithi177 bash[68594]: cluster 2024-09-13T15:05:46.934788+0000 mgr.y (mgr.26252) 1190 : cluster [DBG] pgmap v1746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 25592/270213 objects degraded (9.471%); 975 KiB/s, 15 objects/s recovering 2024-09-13T15:05:48.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:48 smithi177 bash[68594]: cluster 2024-09-13T15:05:46.934788+0000 mgr.y (mgr.26252) 1190 : cluster [DBG] pgmap v1746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 25592/270213 objects degraded (9.471%); 975 KiB/s, 15 objects/s recovering 2024-09-13T15:05:48.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:48.402 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:48.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:48.504 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:48.531 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:48.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:48.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:48.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:48.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:48.773 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:48.796 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:48.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:48.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:48.935 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:48.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:49.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:49.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:49.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:49.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:49.187 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:49.289 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:49.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:49.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:49 smithi177 bash[68594]: audit 2024-09-13T15:05:48.299301+0000 mon.a (mon.0) 2329 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:49 smithi177 bash[68594]: audit 2024-09-13T15:05:48.299301+0000 mon.a (mon.0) 2329 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:49.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:49.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.465 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:49.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:49.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:49 smithi099 bash[84576]: audit 2024-09-13T15:05:48.299301+0000 mon.a (mon.0) 2329 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:49 smithi099 bash[84576]: audit 2024-09-13T15:05:48.299301+0000 mon.a (mon.0) 2329 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:49 smithi099 bash[87147]: audit 2024-09-13T15:05:48.299301+0000 mon.a (mon.0) 2329 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:49.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:49 smithi099 bash[87147]: audit 2024-09-13T15:05:48.299301+0000 mon.a (mon.0) 2329 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:49.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:49.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:49.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:49.731 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:49.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:49.840 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:49.863 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:49.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:49.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:50.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:50.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:50.106 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:50.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:50 smithi177 bash[68594]: cluster 2024-09-13T15:05:48.935420+0000 mgr.y (mgr.26252) 1191 : cluster [DBG] pgmap v1747: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 25477/265776 objects degraded (9.586%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T15:05:50.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:50 smithi177 bash[68594]: cluster 2024-09-13T15:05:48.935420+0000 mgr.y (mgr.26252) 1191 : cluster [DBG] pgmap v1747: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 25477/265776 objects degraded (9.586%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T15:05:50.123 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:50.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:50.225 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:50.250 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:50.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:50.296 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:50 smithi099 bash[84576]: cluster 2024-09-13T15:05:48.935420+0000 mgr.y (mgr.26252) 1191 : cluster [DBG] pgmap v1747: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 25477/265776 objects degraded (9.586%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T15:05:50.296 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:50 smithi099 bash[84576]: cluster 2024-09-13T15:05:48.935420+0000 mgr.y (mgr.26252) 1191 : cluster [DBG] pgmap v1747: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 25477/265776 objects degraded (9.586%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T15:05:50.296 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:50 smithi099 bash[87147]: cluster 2024-09-13T15:05:48.935420+0000 mgr.y (mgr.26252) 1191 : cluster [DBG] pgmap v1747: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 25477/265776 objects degraded (9.586%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T15:05:50.296 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:50 smithi099 bash[87147]: cluster 2024-09-13T15:05:48.935420+0000 mgr.y (mgr.26252) 1191 : cluster [DBG] pgmap v1747: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 25477/265776 objects degraded (9.586%); 1.5 MiB/s, 24 objects/s recovering 2024-09-13T15:05:50.353 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:50.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:05:50 smithi177 bash[61439]: ts=2024-09-13T15:05:50.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:05:50.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:50.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:50.502 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:50.524 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:50.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:50.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:50.655 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:50.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:50.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:50.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:50.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:50.928 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:50.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:51.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:51.058 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:51.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:51.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:51.182 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:51.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:51.314 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:51.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:51.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:51.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:51.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:51.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:51.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:51.690 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:51.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:51.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:51.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:51.851 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:51.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:51.956 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:51.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:51.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:52.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:52.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:52 smithi099 bash[87147]: cluster 2024-09-13T15:05:50.936066+0000 mgr.y (mgr.26252) 1192 : cluster [DBG] pgmap v1748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 25477/261618 objects degraded (9.738%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:52.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:52 smithi099 bash[87147]: cluster 2024-09-13T15:05:50.936066+0000 mgr.y (mgr.26252) 1192 : cluster [DBG] pgmap v1748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 25477/261618 objects degraded (9.738%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:52.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:52 smithi099 bash[87147]: cluster 2024-09-13T15:05:51.528425+0000 mon.a (mon.0) 2330 : cluster [WRN] Health check update: Degraded data redundancy: 25477/261618 objects degraded (9.738%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:52.082 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:52 smithi099 bash[87147]: cluster 2024-09-13T15:05:51.528425+0000 mon.a (mon.0) 2330 : cluster [WRN] Health check update: Degraded data redundancy: 25477/261618 objects degraded (9.738%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:52.104 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:52.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:52.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:52.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:52.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:52.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:52.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:52 smithi099 bash[84576]: cluster 2024-09-13T15:05:50.936066+0000 mgr.y (mgr.26252) 1192 : cluster [DBG] pgmap v1748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 25477/261618 objects degraded (9.738%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:52.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:52 smithi099 bash[84576]: cluster 2024-09-13T15:05:50.936066+0000 mgr.y (mgr.26252) 1192 : cluster [DBG] pgmap v1748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 25477/261618 objects degraded (9.738%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:52.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:52 smithi099 bash[84576]: cluster 2024-09-13T15:05:51.528425+0000 mon.a (mon.0) 2330 : cluster [WRN] Health check update: Degraded data redundancy: 25477/261618 objects degraded (9.738%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:52.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:52 smithi099 bash[84576]: cluster 2024-09-13T15:05:51.528425+0000 mon.a (mon.0) 2330 : cluster [WRN] Health check update: Degraded data redundancy: 25477/261618 objects degraded (9.738%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:52.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:52 smithi177 bash[68594]: cluster 2024-09-13T15:05:50.936066+0000 mgr.y (mgr.26252) 1192 : cluster [DBG] pgmap v1748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 25477/261618 objects degraded (9.738%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:52.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:52 smithi177 bash[68594]: cluster 2024-09-13T15:05:50.936066+0000 mgr.y (mgr.26252) 1192 : cluster [DBG] pgmap v1748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 25477/261618 objects degraded (9.738%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:52.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:52 smithi177 bash[68594]: cluster 2024-09-13T15:05:51.528425+0000 mon.a (mon.0) 2330 : cluster [WRN] Health check update: Degraded data redundancy: 25477/261618 objects degraded (9.738%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:52.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:52 smithi177 bash[68594]: cluster 2024-09-13T15:05:51.528425+0000 mon.a (mon.0) 2330 : cluster [WRN] Health check update: Degraded data redundancy: 25477/261618 objects degraded (9.738%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:52.364 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:52.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:52.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:52.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:52.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:52.598 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:52.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:52.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:52.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:52.774 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:52.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:52.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:52.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:53.012 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:53.041 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:53.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:53.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:53.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:53.285 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:53.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:53.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:53.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:53.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:53.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:53.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:53.590 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:53.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:53.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:53.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:53.832 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:53.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:53.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:53.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:54.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:54.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:54.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:54.132 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:54.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:54.264 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:54.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:54.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:54 smithi099 bash[84576]: cluster 2024-09-13T15:05:52.936628+0000 mgr.y (mgr.26252) 1193 : cluster [DBG] pgmap v1749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 25477/259410 objects degraded (9.821%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:54 smithi099 bash[84576]: cluster 2024-09-13T15:05:52.936628+0000 mgr.y (mgr.26252) 1193 : cluster [DBG] pgmap v1749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 25477/259410 objects degraded (9.821%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:54.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:54 smithi099 bash[87147]: cluster 2024-09-13T15:05:52.936628+0000 mgr.y (mgr.26252) 1193 : cluster [DBG] pgmap v1749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 25477/259410 objects degraded (9.821%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:54.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:54 smithi099 bash[87147]: cluster 2024-09-13T15:05:52.936628+0000 mgr.y (mgr.26252) 1193 : cluster [DBG] pgmap v1749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 25477/259410 objects degraded (9.821%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:54.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:54 smithi177 bash[68594]: cluster 2024-09-13T15:05:52.936628+0000 mgr.y (mgr.26252) 1193 : cluster [DBG] pgmap v1749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 25477/259410 objects degraded (9.821%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:54.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:54 smithi177 bash[68594]: cluster 2024-09-13T15:05:52.936628+0000 mgr.y (mgr.26252) 1193 : cluster [DBG] pgmap v1749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 25477/259410 objects degraded (9.821%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:54.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:54.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:54.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:54.504 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:54.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:54.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:54.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:54.659 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:54.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:54.792 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:54.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:54.896 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:54.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:54.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:55.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:55.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:55.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:55.210 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:55.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:55.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:55.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:55.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:55.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:55.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:55.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:55.579 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:55.611 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:55.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:55.718 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:55.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:55.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:55.861 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:55.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:55.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:55.984 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:56.006 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:56.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:56.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:56.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:56.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:56.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:56.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:56 smithi099 bash[84576]: cluster 2024-09-13T15:05:54.937281+0000 mgr.y (mgr.26252) 1194 : cluster [DBG] pgmap v1750: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25357/253806 objects degraded (9.991%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:05:56.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:56 smithi099 bash[84576]: cluster 2024-09-13T15:05:54.937281+0000 mgr.y (mgr.26252) 1194 : cluster [DBG] pgmap v1750: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25357/253806 objects degraded (9.991%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:05:56.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:56 smithi099 bash[87147]: cluster 2024-09-13T15:05:54.937281+0000 mgr.y (mgr.26252) 1194 : cluster [DBG] pgmap v1750: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25357/253806 objects degraded (9.991%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:05:56.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:56 smithi099 bash[87147]: cluster 2024-09-13T15:05:54.937281+0000 mgr.y (mgr.26252) 1194 : cluster [DBG] pgmap v1750: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25357/253806 objects degraded (9.991%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:05:56.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:56.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:56.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:56 smithi177 bash[68594]: cluster 2024-09-13T15:05:54.937281+0000 mgr.y (mgr.26252) 1194 : cluster [DBG] pgmap v1750: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25357/253806 objects degraded (9.991%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:05:56.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:56 smithi177 bash[68594]: cluster 2024-09-13T15:05:54.937281+0000 mgr.y (mgr.26252) 1194 : cluster [DBG] pgmap v1750: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 25357/253806 objects degraded (9.991%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:05:56.400 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:56.426 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:56.537 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:56.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:56.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:56.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:56.715 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:56.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:56.819 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:56.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:56.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:56.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:56.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:56.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:57.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:57.132 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:57.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:57.235 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:57.244 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:57 smithi177 bash[68594]: audit 2024-09-13T15:05:56.253186+0000 mon.a (mon.0) 2331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:05:57.244 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:57 smithi177 bash[68594]: audit 2024-09-13T15:05:56.253186+0000 mon.a (mon.0) 2331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:05:57.244 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:57 smithi177 bash[68594]: cluster 2024-09-13T15:05:56.529962+0000 mon.a (mon.0) 2332 : cluster [WRN] Health check update: Degraded data redundancy: 25357/253806 objects degraded (9.991%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:57.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:57 smithi177 bash[68594]: cluster 2024-09-13T15:05:56.529962+0000 mon.a (mon.0) 2332 : cluster [WRN] Health check update: Degraded data redundancy: 25357/253806 objects degraded (9.991%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:57.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:57 smithi099 bash[84576]: audit 2024-09-13T15:05:56.253186+0000 mon.a (mon.0) 2331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:05:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:57 smithi099 bash[84576]: audit 2024-09-13T15:05:56.253186+0000 mon.a (mon.0) 2331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:05:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:57 smithi099 bash[84576]: cluster 2024-09-13T15:05:56.529962+0000 mon.a (mon.0) 2332 : cluster [WRN] Health check update: Degraded data redundancy: 25357/253806 objects degraded (9.991%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:57 smithi099 bash[84576]: cluster 2024-09-13T15:05:56.529962+0000 mon.a (mon.0) 2332 : cluster [WRN] Health check update: Degraded data redundancy: 25357/253806 objects degraded (9.991%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:57.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:05:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:05:57] "GET /metrics HTTP/1.1" 200 36783 "" "Prometheus/2.51.0" 2024-09-13T15:05:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:57 smithi099 bash[87147]: audit 2024-09-13T15:05:56.253186+0000 mon.a (mon.0) 2331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:05:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:57 smithi099 bash[87147]: audit 2024-09-13T15:05:56.253186+0000 mon.a (mon.0) 2331 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:05:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:57 smithi099 bash[87147]: cluster 2024-09-13T15:05:56.529962+0000 mon.a (mon.0) 2332 : cluster [WRN] Health check update: Degraded data redundancy: 25357/253806 objects degraded (9.991%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:57 smithi099 bash[87147]: cluster 2024-09-13T15:05:56.529962+0000 mon.a (mon.0) 2332 : cluster [WRN] Health check update: Degraded data redundancy: 25357/253806 objects degraded (9.991%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:05:57.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:57.390 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:57.493 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:57.522 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:57.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:57.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:57.648 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:57.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:57.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:57.882 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:57.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:57.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:58.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:58.043 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:58.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:58.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:58.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:58.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:58.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:58 smithi099 bash[84576]: cluster 2024-09-13T15:05:56.937812+0000 mgr.y (mgr.26252) 1195 : cluster [DBG] pgmap v1751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 25357/252627 objects degraded (10.037%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:58.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:05:58 smithi099 bash[84576]: cluster 2024-09-13T15:05:56.937812+0000 mgr.y (mgr.26252) 1195 : cluster [DBG] pgmap v1751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 25357/252627 objects degraded (10.037%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:58.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:58 smithi099 bash[87147]: cluster 2024-09-13T15:05:56.937812+0000 mgr.y (mgr.26252) 1195 : cluster [DBG] pgmap v1751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 25357/252627 objects degraded (10.037%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:58.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:05:58 smithi099 bash[87147]: cluster 2024-09-13T15:05:56.937812+0000 mgr.y (mgr.26252) 1195 : cluster [DBG] pgmap v1751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 25357/252627 objects degraded (10.037%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:58.322 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:58.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:58.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:58 smithi177 bash[68594]: cluster 2024-09-13T15:05:56.937812+0000 mgr.y (mgr.26252) 1195 : cluster [DBG] pgmap v1751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 25357/252627 objects degraded (10.037%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:58.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:05:58 smithi177 bash[68594]: cluster 2024-09-13T15:05:56.937812+0000 mgr.y (mgr.26252) 1195 : cluster [DBG] pgmap v1751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 25357/252627 objects degraded (10.037%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:05:58.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:58.465 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:58.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:58.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:58.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:58.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:58.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:58.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:58.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:58.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:58.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:59.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:59.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:59.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:59.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:59.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:59.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:59.274 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:59.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:59.403 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:59.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:59.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:59.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:59.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:59.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:59.668 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:59.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:59.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:59.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:59.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:59.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:59.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:00.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:00.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:00.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:00.212 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:00.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:00 smithi177 bash[68594]: cluster 2024-09-13T15:05:58.938471+0000 mgr.y (mgr.26252) 1196 : cluster [DBG] pgmap v1752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 25233/247701 objects degraded (10.187%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:00.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:00 smithi177 bash[68594]: cluster 2024-09-13T15:05:58.938471+0000 mgr.y (mgr.26252) 1196 : cluster [DBG] pgmap v1752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 25233/247701 objects degraded (10.187%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:00.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:00.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:00 smithi099 bash[84576]: cluster 2024-09-13T15:05:58.938471+0000 mgr.y (mgr.26252) 1196 : cluster [DBG] pgmap v1752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 25233/247701 objects degraded (10.187%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:00.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:00 smithi099 bash[84576]: cluster 2024-09-13T15:05:58.938471+0000 mgr.y (mgr.26252) 1196 : cluster [DBG] pgmap v1752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 25233/247701 objects degraded (10.187%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:00.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:00 smithi099 bash[87147]: cluster 2024-09-13T15:05:58.938471+0000 mgr.y (mgr.26252) 1196 : cluster [DBG] pgmap v1752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 25233/247701 objects degraded (10.187%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:00.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:00 smithi099 bash[87147]: cluster 2024-09-13T15:05:58.938471+0000 mgr.y (mgr.26252) 1196 : cluster [DBG] pgmap v1752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 25233/247701 objects degraded (10.187%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:00.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:00.354 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:06:00 smithi177 bash[61439]: ts=2024-09-13T15:06:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:00.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:00.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:00.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:00.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:00.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:00.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:00.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:00.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:00.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:00.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:00.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:01.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:01.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:01.133 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:01.158 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:01.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:01.265 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:01.287 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:01.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:01.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:01.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.431 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:01.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:01.532 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:01.558 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:01.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:01.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:01.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:01.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:01.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:01.820 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:01.929 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:01.962 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:01.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:02.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:02.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:02.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:02.198 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:02.206 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:02 smithi099 bash[84576]: cluster 2024-09-13T15:06:00.939069+0000 mgr.y (mgr.26252) 1197 : cluster [DBG] pgmap v1753: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 25233/242913 objects degraded (10.388%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:02.206 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:02 smithi099 bash[84576]: cluster 2024-09-13T15:06:00.939069+0000 mgr.y (mgr.26252) 1197 : cluster [DBG] pgmap v1753: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 25233/242913 objects degraded (10.388%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:02.206 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:02 smithi099 bash[84576]: cluster 2024-09-13T15:06:01.530779+0000 mon.a (mon.0) 2333 : cluster [WRN] Health check update: Degraded data redundancy: 25233/242913 objects degraded (10.388%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:02.206 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:02 smithi099 bash[84576]: cluster 2024-09-13T15:06:01.530779+0000 mon.a (mon.0) 2333 : cluster [WRN] Health check update: Degraded data redundancy: 25233/242913 objects degraded (10.388%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:02.207 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:02 smithi099 bash[87147]: cluster 2024-09-13T15:06:00.939069+0000 mgr.y (mgr.26252) 1197 : cluster [DBG] pgmap v1753: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 25233/242913 objects degraded (10.388%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:02.207 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:02 smithi099 bash[87147]: cluster 2024-09-13T15:06:00.939069+0000 mgr.y (mgr.26252) 1197 : cluster [DBG] pgmap v1753: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 25233/242913 objects degraded (10.388%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:02.207 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:02 smithi099 bash[87147]: cluster 2024-09-13T15:06:01.530779+0000 mon.a (mon.0) 2333 : cluster [WRN] Health check update: Degraded data redundancy: 25233/242913 objects degraded (10.388%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:02.207 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:02 smithi099 bash[87147]: cluster 2024-09-13T15:06:01.530779+0000 mon.a (mon.0) 2333 : cluster [WRN] Health check update: Degraded data redundancy: 25233/242913 objects degraded (10.388%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:02.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:02.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:02.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:02.341 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:02.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:02.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:02 smithi177 bash[68594]: cluster 2024-09-13T15:06:00.939069+0000 mgr.y (mgr.26252) 1197 : cluster [DBG] pgmap v1753: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 25233/242913 objects degraded (10.388%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:02.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:02 smithi177 bash[68594]: cluster 2024-09-13T15:06:00.939069+0000 mgr.y (mgr.26252) 1197 : cluster [DBG] pgmap v1753: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 25233/242913 objects degraded (10.388%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:02.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:02 smithi177 bash[68594]: cluster 2024-09-13T15:06:01.530779+0000 mon.a (mon.0) 2333 : cluster [WRN] Health check update: Degraded data redundancy: 25233/242913 objects degraded (10.388%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:02.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:02 smithi177 bash[68594]: cluster 2024-09-13T15:06:01.530779+0000 mon.a (mon.0) 2333 : cluster [WRN] Health check update: Degraded data redundancy: 25233/242913 objects degraded (10.388%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:02.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:02.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:02.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:02.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.611 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:02.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:02.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:02.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:02.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:02.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:02.895 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:02.999 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:03.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:03.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:03.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:03.168 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:03.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:03.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:03.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:03.404 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:03.449 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:03.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:03.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:03.579 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:03.683 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:03.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:03.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:03.830 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:03.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:03.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:03.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:03.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:03 smithi177 bash[68594]: audit 2024-09-13T15:06:02.910873+0000 mon.a (mon.0) 2334 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:03.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:03 smithi177 bash[68594]: audit 2024-09-13T15:06:02.910873+0000 mon.a (mon.0) 2334 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:03.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:03 smithi177 bash[68594]: audit 2024-09-13T15:06:02.924691+0000 mon.a (mon.0) 2335 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:03.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:03 smithi177 bash[68594]: audit 2024-09-13T15:06:02.924691+0000 mon.a (mon.0) 2335 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:03.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:03 smithi177 bash[68594]: cluster 2024-09-13T15:06:02.939666+0000 mgr.y (mgr.26252) 1198 : cluster [DBG] pgmap v1754: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 25233/240780 objects degraded (10.480%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:03.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:03 smithi177 bash[68594]: cluster 2024-09-13T15:06:02.939666+0000 mgr.y (mgr.26252) 1198 : cluster [DBG] pgmap v1754: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 25233/240780 objects degraded (10.480%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:03.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:03 smithi177 bash[68594]: audit 2024-09-13T15:06:03.304080+0000 mon.a (mon.0) 2336 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:03.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:03 smithi177 bash[68594]: audit 2024-09-13T15:06:03.304080+0000 mon.a (mon.0) 2336 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:04.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:04.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:04.117 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[84576]: audit 2024-09-13T15:06:02.910873+0000 mon.a (mon.0) 2334 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:04.117 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[84576]: audit 2024-09-13T15:06:02.910873+0000 mon.a (mon.0) 2334 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:04.117 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[84576]: audit 2024-09-13T15:06:02.924691+0000 mon.a (mon.0) 2335 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:04.117 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[84576]: audit 2024-09-13T15:06:02.924691+0000 mon.a (mon.0) 2335 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:04.117 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[84576]: cluster 2024-09-13T15:06:02.939666+0000 mgr.y (mgr.26252) 1198 : cluster [DBG] pgmap v1754: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 25233/240780 objects degraded (10.480%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:04.117 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[84576]: cluster 2024-09-13T15:06:02.939666+0000 mgr.y (mgr.26252) 1198 : cluster [DBG] pgmap v1754: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 25233/240780 objects degraded (10.480%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:04.117 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[84576]: audit 2024-09-13T15:06:03.304080+0000 mon.a (mon.0) 2336 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:04.118 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[84576]: audit 2024-09-13T15:06:03.304080+0000 mon.a (mon.0) 2336 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:04.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[87147]: audit 2024-09-13T15:06:02.910873+0000 mon.a (mon.0) 2334 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:04.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[87147]: audit 2024-09-13T15:06:02.910873+0000 mon.a (mon.0) 2334 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:04.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[87147]: audit 2024-09-13T15:06:02.924691+0000 mon.a (mon.0) 2335 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:04.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[87147]: audit 2024-09-13T15:06:02.924691+0000 mon.a (mon.0) 2335 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:04.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[87147]: cluster 2024-09-13T15:06:02.939666+0000 mgr.y (mgr.26252) 1198 : cluster [DBG] pgmap v1754: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 25233/240780 objects degraded (10.480%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:04.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[87147]: cluster 2024-09-13T15:06:02.939666+0000 mgr.y (mgr.26252) 1198 : cluster [DBG] pgmap v1754: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 25233/240780 objects degraded (10.480%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:04.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[87147]: audit 2024-09-13T15:06:03.304080+0000 mon.a (mon.0) 2336 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:04.118 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:03 smithi099 bash[87147]: audit 2024-09-13T15:06:03.304080+0000 mon.a (mon.0) 2336 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:04.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:04.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:04.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:04.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:04.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:04.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:04.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:04.427 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:04.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:04.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:04.553 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:04.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:04.658 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:04.707 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:04.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:04.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:04.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:04.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:04.951 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:04.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:04.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:05.082 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:05.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:05.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:05.194 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:06:05.194 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:06:05.194 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:06:05.194 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:06:05.194 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:06:05.208 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:06:05.209 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:06:05.240 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:05.282 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:05.282 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:06:05.282 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:06:05.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:05.374 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:06:05.375 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:06:05.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:05.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:05.437 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:06:05.440 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-13T15:06:05.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:05.476 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:06:05.477 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:06:05.521 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:06:05.521 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:06:05.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:05.575 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:05.575 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:06:05.576 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:06:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:05.613 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:06:05.614 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:06:05.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:05.702 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:05.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:05.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:06.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:06.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:06.188 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:06.194 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:06:06.194 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:06:06.197 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:06:06.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:06.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:06.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: cluster 2024-09-13T15:06:04.940350+0000 mgr.y (mgr.26252) 1199 : cluster [DBG] pgmap v1755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 25079/235599 objects degraded (10.645%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T15:06:06.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: cluster 2024-09-13T15:06:04.940350+0000 mgr.y (mgr.26252) 1199 : cluster [DBG] pgmap v1755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 25079/235599 objects degraded (10.645%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T15:06:06.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: audit 2024-09-13T15:06:05.118705+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.99:0/3271447658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:06.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: audit 2024-09-13T15:06:05.118705+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.99:0/3271447658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:06.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: audit 2024-09-13T15:06:05.184549+0000 mon.a (mon.0) 2337 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: audit 2024-09-13T15:06:05.184549+0000 mon.a (mon.0) 2337 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: audit 2024-09-13T15:06:05.192238+0000 mon.a (mon.0) 2338 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: audit 2024-09-13T15:06:05.192238+0000 mon.a (mon.0) 2338 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: audit 2024-09-13T15:06:05.542478+0000 mon.c (mon.2) 94 : audit [INF] from='client.? 172.21.15.99:0/3758522868' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: audit 2024-09-13T15:06:05.542478+0000 mon.c (mon.2) 94 : audit [INF] from='client.? 172.21.15.99:0/3758522868' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: audit 2024-09-13T15:06:05.543285+0000 mon.a (mon.0) 2339 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.252 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[84576]: audit 2024-09-13T15:06:05.543285+0000 mon.a (mon.0) 2339 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: cluster 2024-09-13T15:06:04.940350+0000 mgr.y (mgr.26252) 1199 : cluster [DBG] pgmap v1755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 25079/235599 objects degraded (10.645%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T15:06:06.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: cluster 2024-09-13T15:06:04.940350+0000 mgr.y (mgr.26252) 1199 : cluster [DBG] pgmap v1755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 25079/235599 objects degraded (10.645%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T15:06:06.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: audit 2024-09-13T15:06:05.118705+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.99:0/3271447658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:06.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: audit 2024-09-13T15:06:05.118705+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.99:0/3271447658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:06.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: audit 2024-09-13T15:06:05.184549+0000 mon.a (mon.0) 2337 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: audit 2024-09-13T15:06:05.184549+0000 mon.a (mon.0) 2337 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: audit 2024-09-13T15:06:05.192238+0000 mon.a (mon.0) 2338 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.253 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: audit 2024-09-13T15:06:05.192238+0000 mon.a (mon.0) 2338 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.253 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: audit 2024-09-13T15:06:05.542478+0000 mon.c (mon.2) 94 : audit [INF] from='client.? 172.21.15.99:0/3758522868' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.253 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: audit 2024-09-13T15:06:05.542478+0000 mon.c (mon.2) 94 : audit [INF] from='client.? 172.21.15.99:0/3758522868' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.253 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: audit 2024-09-13T15:06:05.543285+0000 mon.a (mon.0) 2339 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.253 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:05 smithi099 bash[87147]: audit 2024-09-13T15:06:05.543285+0000 mon.a (mon.0) 2339 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: cluster 2024-09-13T15:06:04.940350+0000 mgr.y (mgr.26252) 1199 : cluster [DBG] pgmap v1755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 25079/235599 objects degraded (10.645%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T15:06:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: cluster 2024-09-13T15:06:04.940350+0000 mgr.y (mgr.26252) 1199 : cluster [DBG] pgmap v1755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 25079/235599 objects degraded (10.645%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T15:06:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: audit 2024-09-13T15:06:05.118705+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.99:0/3271447658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: audit 2024-09-13T15:06:05.118705+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.99:0/3271447658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: audit 2024-09-13T15:06:05.184549+0000 mon.a (mon.0) 2337 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: audit 2024-09-13T15:06:05.184549+0000 mon.a (mon.0) 2337 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: audit 2024-09-13T15:06:05.192238+0000 mon.a (mon.0) 2338 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: audit 2024-09-13T15:06:05.192238+0000 mon.a (mon.0) 2338 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: audit 2024-09-13T15:06:05.542478+0000 mon.c (mon.2) 94 : audit [INF] from='client.? 172.21.15.99:0/3758522868' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: audit 2024-09-13T15:06:05.542478+0000 mon.c (mon.2) 94 : audit [INF] from='client.? 172.21.15.99:0/3758522868' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: audit 2024-09-13T15:06:05.543285+0000 mon.a (mon.0) 2339 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:05 smithi177 bash[68594]: audit 2024-09-13T15:06:05.543285+0000 mon.a (mon.0) 2339 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:06.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:06.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:06.462 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:06.490 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:06.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:06.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:06.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:06.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:06.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:06.777 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:06.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:06.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:06.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:07.014 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:07.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:07.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:07.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:07.171 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:07.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:07.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:07.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: audit 2024-09-13T15:06:06.176060+0000 mon.a (mon.0) 2340 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:07.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: audit 2024-09-13T15:06:06.176060+0000 mon.a (mon.0) 2340 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:07.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: audit 2024-09-13T15:06:06.178751+0000 mon.a (mon.0) 2341 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:07.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: audit 2024-09-13T15:06:06.178751+0000 mon.a (mon.0) 2341 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:07.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: audit 2024-09-13T15:06:06.184898+0000 mon.a (mon.0) 2342 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:07.288 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: audit 2024-09-13T15:06:06.184898+0000 mon.a (mon.0) 2342 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:07.288 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: cluster 2024-09-13T15:06:06.188397+0000 mon.a (mon.0) 2343 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:07.288 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: cluster 2024-09-13T15:06:06.188397+0000 mon.a (mon.0) 2343 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:07.288 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: audit 2024-09-13T15:06:06.191880+0000 mon.a (mon.0) 2344 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:07.288 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: audit 2024-09-13T15:06:06.191880+0000 mon.a (mon.0) 2344 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:07.288 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: cluster 2024-09-13T15:06:06.201561+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2024-09-13T15:06:07.288 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: cluster 2024-09-13T15:06:06.201561+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2024-09-13T15:06:07.288 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: cluster 2024-09-13T15:06:06.531590+0000 mon.a (mon.0) 2346 : cluster [WRN] Health check update: Degraded data redundancy: 25079/235599 objects degraded (10.645%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:07.288 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:07 smithi177 bash[68594]: cluster 2024-09-13T15:06:06.531590+0000 mon.a (mon.0) 2346 : cluster [WRN] Health check update: Degraded data redundancy: 25079/235599 objects degraded (10.645%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:07.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:07.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: audit 2024-09-13T15:06:06.176060+0000 mon.a (mon.0) 2340 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: audit 2024-09-13T15:06:06.176060+0000 mon.a (mon.0) 2340 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: audit 2024-09-13T15:06:06.178751+0000 mon.a (mon.0) 2341 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: audit 2024-09-13T15:06:06.178751+0000 mon.a (mon.0) 2341 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: audit 2024-09-13T15:06:06.184898+0000 mon.a (mon.0) 2342 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: audit 2024-09-13T15:06:06.184898+0000 mon.a (mon.0) 2342 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: cluster 2024-09-13T15:06:06.188397+0000 mon.a (mon.0) 2343 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: cluster 2024-09-13T15:06:06.188397+0000 mon.a (mon.0) 2343 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: audit 2024-09-13T15:06:06.191880+0000 mon.a (mon.0) 2344 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: audit 2024-09-13T15:06:06.191880+0000 mon.a (mon.0) 2344 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: cluster 2024-09-13T15:06:06.201561+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: cluster 2024-09-13T15:06:06.201561+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2024-09-13T15:06:07.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: cluster 2024-09-13T15:06:06.531590+0000 mon.a (mon.0) 2346 : cluster [WRN] Health check update: Degraded data redundancy: 25079/235599 objects degraded (10.645%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:07.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[84576]: cluster 2024-09-13T15:06:06.531590+0000 mon.a (mon.0) 2346 : cluster [WRN] Health check update: Degraded data redundancy: 25079/235599 objects degraded (10.645%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:07.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:06:07] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-13T15:06:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: audit 2024-09-13T15:06:06.176060+0000 mon.a (mon.0) 2340 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: audit 2024-09-13T15:06:06.176060+0000 mon.a (mon.0) 2340 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: audit 2024-09-13T15:06:06.178751+0000 mon.a (mon.0) 2341 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: audit 2024-09-13T15:06:06.178751+0000 mon.a (mon.0) 2341 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: audit 2024-09-13T15:06:06.184898+0000 mon.a (mon.0) 2342 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:07.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: audit 2024-09-13T15:06:06.184898+0000 mon.a (mon.0) 2342 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:06:07.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: cluster 2024-09-13T15:06:06.188397+0000 mon.a (mon.0) 2343 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:07.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: cluster 2024-09-13T15:06:06.188397+0000 mon.a (mon.0) 2343 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:07.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: audit 2024-09-13T15:06:06.191880+0000 mon.a (mon.0) 2344 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:07.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: audit 2024-09-13T15:06:06.191880+0000 mon.a (mon.0) 2344 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:07.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: cluster 2024-09-13T15:06:06.201561+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2024-09-13T15:06:07.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: cluster 2024-09-13T15:06:06.201561+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2024-09-13T15:06:07.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: cluster 2024-09-13T15:06:06.531590+0000 mon.a (mon.0) 2346 : cluster [WRN] Health check update: Degraded data redundancy: 25079/235599 objects degraded (10.645%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:07.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:07 smithi099 bash[87147]: cluster 2024-09-13T15:06:06.531590+0000 mon.a (mon.0) 2346 : cluster [WRN] Health check update: Degraded data redundancy: 25079/235599 objects degraded (10.645%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:07.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:07.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:07.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:07.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:07.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:07.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:07.665 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:07.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:07.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:07.793 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:07.819 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:07.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:08.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:08.148 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:08.170 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:08.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:08.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:08.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:08 smithi099 bash[84576]: cluster 2024-09-13T15:06:06.940849+0000 mgr.y (mgr.26252) 1200 : cluster [DBG] pgmap v1757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 25079/234705 objects degraded (10.685%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:08.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:08 smithi099 bash[84576]: cluster 2024-09-13T15:06:06.940849+0000 mgr.y (mgr.26252) 1200 : cluster [DBG] pgmap v1757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 25079/234705 objects degraded (10.685%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:08.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:08 smithi099 bash[87147]: cluster 2024-09-13T15:06:06.940849+0000 mgr.y (mgr.26252) 1200 : cluster [DBG] pgmap v1757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 25079/234705 objects degraded (10.685%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:08.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:08 smithi099 bash[87147]: cluster 2024-09-13T15:06:06.940849+0000 mgr.y (mgr.26252) 1200 : cluster [DBG] pgmap v1757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 25079/234705 objects degraded (10.685%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:08.313 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:08.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:08 smithi177 bash[68594]: cluster 2024-09-13T15:06:06.940849+0000 mgr.y (mgr.26252) 1200 : cluster [DBG] pgmap v1757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 25079/234705 objects degraded (10.685%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:08 smithi177 bash[68594]: cluster 2024-09-13T15:06:06.940849+0000 mgr.y (mgr.26252) 1200 : cluster [DBG] pgmap v1757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 25079/234705 objects degraded (10.685%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:08.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:08.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:08.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:08.540 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:08.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:08.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:08.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:08.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:08.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:08.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:08.843 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:08.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:08.948 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:08.982 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:08.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:09.089 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:09.136 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:09.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:09.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:09.324 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:09.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:09.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:09.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:09.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:09.590 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:09.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:09.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:09.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:09.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:09.834 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:09.858 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:09.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:09.962 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:09.969 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T15:06:09.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:10.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:10.104 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:10.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:10 smithi177 bash[68594]: cluster 2024-09-13T15:06:08.942411+0000 mgr.y (mgr.26252) 1201 : cluster [DBG] pgmap v1758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24978/230316 objects degraded (10.845%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:10.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:10 smithi177 bash[68594]: cluster 2024-09-13T15:06:08.942411+0000 mgr.y (mgr.26252) 1201 : cluster [DBG] pgmap v1758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24978/230316 objects degraded (10.845%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:10.129 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:10.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:10.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:10.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:10.263 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:06:10 smithi177 bash[61439]: ts=2024-09-13T15:06:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:10.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:10.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:10 smithi099 bash[84576]: cluster 2024-09-13T15:06:08.942411+0000 mgr.y (mgr.26252) 1201 : cluster [DBG] pgmap v1758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24978/230316 objects degraded (10.845%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:10.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:10 smithi099 bash[84576]: cluster 2024-09-13T15:06:08.942411+0000 mgr.y (mgr.26252) 1201 : cluster [DBG] pgmap v1758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24978/230316 objects degraded (10.845%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:10.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:10 smithi099 bash[87147]: cluster 2024-09-13T15:06:08.942411+0000 mgr.y (mgr.26252) 1201 : cluster [DBG] pgmap v1758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24978/230316 objects degraded (10.845%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:10.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:10 smithi099 bash[87147]: cluster 2024-09-13T15:06:08.942411+0000 mgr.y (mgr.26252) 1201 : cluster [DBG] pgmap v1758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24978/230316 objects degraded (10.845%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:10.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:10.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:10.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:10.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.539 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:10.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:10.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:10.661 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:10.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:10.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.811 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:10.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:10.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:10.950 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:10.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:11.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:11.083 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:11.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:11.187 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:11.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:11.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:11.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:11.356 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:11.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:11.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:11.488 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:11.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:11.593 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:11.632 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:11.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:11.737 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:11.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:11.886 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:11.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:11.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:12.015 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:12.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:12.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:12.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:12.165 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:12.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:12.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:12.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:12 smithi177 bash[68594]: cluster 2024-09-13T15:06:10.943056+0000 mgr.y (mgr.26252) 1202 : cluster [DBG] pgmap v1759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 24978/226005 objects degraded (11.052%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:12.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:12 smithi177 bash[68594]: cluster 2024-09-13T15:06:10.943056+0000 mgr.y (mgr.26252) 1202 : cluster [DBG] pgmap v1759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 24978/226005 objects degraded (11.052%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:12.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:12 smithi177 bash[68594]: cluster 2024-09-13T15:06:11.532426+0000 mon.a (mon.0) 2347 : cluster [WRN] Health check update: Degraded data redundancy: 24978/226005 objects degraded (11.052%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:12.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:12 smithi177 bash[68594]: cluster 2024-09-13T15:06:11.532426+0000 mon.a (mon.0) 2347 : cluster [WRN] Health check update: Degraded data redundancy: 24978/226005 objects degraded (11.052%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:12.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:12.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:12 smithi099 bash[84576]: cluster 2024-09-13T15:06:10.943056+0000 mgr.y (mgr.26252) 1202 : cluster [DBG] pgmap v1759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 24978/226005 objects degraded (11.052%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:12.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:12 smithi099 bash[84576]: cluster 2024-09-13T15:06:10.943056+0000 mgr.y (mgr.26252) 1202 : cluster [DBG] pgmap v1759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 24978/226005 objects degraded (11.052%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:12.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:12 smithi099 bash[84576]: cluster 2024-09-13T15:06:11.532426+0000 mon.a (mon.0) 2347 : cluster [WRN] Health check update: Degraded data redundancy: 24978/226005 objects degraded (11.052%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:12.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:12 smithi099 bash[84576]: cluster 2024-09-13T15:06:11.532426+0000 mon.a (mon.0) 2347 : cluster [WRN] Health check update: Degraded data redundancy: 24978/226005 objects degraded (11.052%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:12.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:12 smithi099 bash[87147]: cluster 2024-09-13T15:06:10.943056+0000 mgr.y (mgr.26252) 1202 : cluster [DBG] pgmap v1759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 24978/226005 objects degraded (11.052%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:12.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:12 smithi099 bash[87147]: cluster 2024-09-13T15:06:10.943056+0000 mgr.y (mgr.26252) 1202 : cluster [DBG] pgmap v1759: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 24978/226005 objects degraded (11.052%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:12.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:12 smithi099 bash[87147]: cluster 2024-09-13T15:06:11.532426+0000 mon.a (mon.0) 2347 : cluster [WRN] Health check update: Degraded data redundancy: 24978/226005 objects degraded (11.052%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:12.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:12 smithi099 bash[87147]: cluster 2024-09-13T15:06:11.532426+0000 mon.a (mon.0) 2347 : cluster [WRN] Health check update: Degraded data redundancy: 24978/226005 objects degraded (11.052%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:12.403 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:12.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:12.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:12.537 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:12.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.575 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:12.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:12.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:12.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:12.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:12.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:12.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:12.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:12.948 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:12.978 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:12.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:13.085 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:13.130 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:13.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:13.235 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:13.267 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:13.371 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:13.397 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:13.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:13.502 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:13.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:13.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:13.639 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:13.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:13.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:13.774 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:13.800 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:13.907 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:13.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:13.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:14.053 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:14.065 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:14 smithi099 bash[84576]: cluster 2024-09-13T15:06:12.943701+0000 mgr.y (mgr.26252) 1203 : cluster [DBG] pgmap v1760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24978/222768 objects degraded (11.213%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:14.065 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:14 smithi099 bash[84576]: cluster 2024-09-13T15:06:12.943701+0000 mgr.y (mgr.26252) 1203 : cluster [DBG] pgmap v1760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24978/222768 objects degraded (11.213%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:14.066 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:14 smithi099 bash[87147]: cluster 2024-09-13T15:06:12.943701+0000 mgr.y (mgr.26252) 1203 : cluster [DBG] pgmap v1760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24978/222768 objects degraded (11.213%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:14.066 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:14 smithi099 bash[87147]: cluster 2024-09-13T15:06:12.943701+0000 mgr.y (mgr.26252) 1203 : cluster [DBG] pgmap v1760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24978/222768 objects degraded (11.213%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:14.078 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:14.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:14.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:14.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:14.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:14.330 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:14 smithi177 bash[68594]: cluster 2024-09-13T15:06:12.943701+0000 mgr.y (mgr.26252) 1203 : cluster [DBG] pgmap v1760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24978/222768 objects degraded (11.213%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:14.330 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:14 smithi177 bash[68594]: cluster 2024-09-13T15:06:12.943701+0000 mgr.y (mgr.26252) 1203 : cluster [DBG] pgmap v1760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24978/222768 objects degraded (11.213%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:14.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:14.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:14.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:14.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:14.585 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:14.614 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:14.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:14.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:14.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:14.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:14.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:14.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:15.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:15.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:15.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:15.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:15.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:15.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:06:15.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:15.321 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:15.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:15.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:15.472 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:15.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:15.492 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:06:15.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:15.607 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:15.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:15.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:15.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:15.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:15.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:15.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:15.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:15.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:16.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:16.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:16.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:16.128 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:16 smithi177 bash[68594]: cluster 2024-09-13T15:06:14.944331+0000 mgr.y (mgr.26252) 1204 : cluster [DBG] pgmap v1761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 24863/217995 objects degraded (11.405%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T15:06:16.128 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:16 smithi177 bash[68594]: cluster 2024-09-13T15:06:14.944331+0000 mgr.y (mgr.26252) 1204 : cluster [DBG] pgmap v1761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 24863/217995 objects degraded (11.405%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T15:06:16.156 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:16.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:16.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:16.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:16 smithi099 bash[84576]: cluster 2024-09-13T15:06:14.944331+0000 mgr.y (mgr.26252) 1204 : cluster [DBG] pgmap v1761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 24863/217995 objects degraded (11.405%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T15:06:16.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:16 smithi099 bash[84576]: cluster 2024-09-13T15:06:14.944331+0000 mgr.y (mgr.26252) 1204 : cluster [DBG] pgmap v1761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 24863/217995 objects degraded (11.405%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T15:06:16.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:16 smithi099 bash[87147]: cluster 2024-09-13T15:06:14.944331+0000 mgr.y (mgr.26252) 1204 : cluster [DBG] pgmap v1761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 24863/217995 objects degraded (11.405%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T15:06:16.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:16 smithi099 bash[87147]: cluster 2024-09-13T15:06:14.944331+0000 mgr.y (mgr.26252) 1204 : cluster [DBG] pgmap v1761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 24863/217995 objects degraded (11.405%); 1.3 MiB/s, 21 objects/s recovering 2024-09-13T15:06:16.298 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:16.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:16.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:16.429 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:16.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:16.533 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:16.543 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:16.564 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:16.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:16.667 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:16.696 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:16.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:16.802 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:16.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.831 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:16.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:16.932 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:16.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:16.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:17.062 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:17.079 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:17.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:17 smithi177 bash[68594]: cluster 2024-09-13T15:06:16.533210+0000 mon.a (mon.0) 2348 : cluster [WRN] Health check update: Degraded data redundancy: 24863/217995 objects degraded (11.405%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:17.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:17 smithi177 bash[68594]: cluster 2024-09-13T15:06:16.533210+0000 mon.a (mon.0) 2348 : cluster [WRN] Health check update: Degraded data redundancy: 24863/217995 objects degraded (11.405%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:17.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:17.206 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:17.309 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:17 smithi099 bash[84576]: cluster 2024-09-13T15:06:16.533210+0000 mon.a (mon.0) 2348 : cluster [WRN] Health check update: Degraded data redundancy: 24863/217995 objects degraded (11.405%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:17.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:06:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:06:17] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-13T15:06:17.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:17 smithi099 bash[87147]: cluster 2024-09-13T15:06:16.533210+0000 mon.a (mon.0) 2348 : cluster [WRN] Health check update: Degraded data redundancy: 24863/217995 objects degraded (11.405%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:17.335 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:17.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:17.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:17.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:17.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:17.602 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:17.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:17.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:17.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:17.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:17.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:17 smithi099 bash[84576]: cluster 2024-09-13T15:06:16.533210+0000 mon.a (mon.0) 2348 : cluster [WRN] Health check update: Degraded data redundancy: 24863/217995 objects degraded (11.405%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:17.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:17 smithi099 bash[87147]: cluster 2024-09-13T15:06:16.533210+0000 mon.a (mon.0) 2348 : cluster [WRN] Health check update: Degraded data redundancy: 24863/217995 objects degraded (11.405%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:17.836 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:17.869 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:06:17.869 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:06:17.869 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T15:06:17.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:06:17.874 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:17.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:18.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:18.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:18.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:18.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:18.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:18.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:18 smithi099 bash[84576]: cluster 2024-09-13T15:06:16.944872+0000 mgr.y (mgr.26252) 1205 : cluster [DBG] pgmap v1762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 24863/216705 objects degraded (11.473%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:18.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:18 smithi099 bash[84576]: cluster 2024-09-13T15:06:16.944872+0000 mgr.y (mgr.26252) 1205 : cluster [DBG] pgmap v1762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 24863/216705 objects degraded (11.473%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:18.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:18 smithi099 bash[87147]: cluster 2024-09-13T15:06:16.944872+0000 mgr.y (mgr.26252) 1205 : cluster [DBG] pgmap v1762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 24863/216705 objects degraded (11.473%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:18.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:18 smithi099 bash[87147]: cluster 2024-09-13T15:06:16.944872+0000 mgr.y (mgr.26252) 1205 : cluster [DBG] pgmap v1762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 24863/216705 objects degraded (11.473%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:18.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:18.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:18 smithi177 bash[68594]: cluster 2024-09-13T15:06:16.944872+0000 mgr.y (mgr.26252) 1205 : cluster [DBG] pgmap v1762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 24863/216705 objects degraded (11.473%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:18 smithi177 bash[68594]: cluster 2024-09-13T15:06:16.944872+0000 mgr.y (mgr.26252) 1205 : cluster [DBG] pgmap v1762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 24863/216705 objects degraded (11.473%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:18.380 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:18.397 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:18.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:18.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:18.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:18.627 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:18.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:18.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:18.754 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:18.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:18.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:18.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:18.926 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:19.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:19.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:19.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:19.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:19.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.240 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:19.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:19 smithi099 bash[84576]: audit 2024-09-13T15:06:18.301785+0000 mon.a (mon.0) 2349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:19 smithi099 bash[84576]: audit 2024-09-13T15:06:18.301785+0000 mon.a (mon.0) 2349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:19 smithi099 bash[87147]: audit 2024-09-13T15:06:18.301785+0000 mon.a (mon.0) 2349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:19 smithi099 bash[87147]: audit 2024-09-13T15:06:18.301785+0000 mon.a (mon.0) 2349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:19.342 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:19.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:19 smithi177 bash[68594]: audit 2024-09-13T15:06:18.301785+0000 mon.a (mon.0) 2349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:19.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:19 smithi177 bash[68594]: audit 2024-09-13T15:06:18.301785+0000 mon.a (mon.0) 2349 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:19.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.385 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:19.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:19.486 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:19.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:19.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:19.592 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:19.596 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:19.629 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:19.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:19.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:19.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:19.781 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:19.884 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:19.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:19.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:20.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:20.057 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:20.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:20.164 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:20.177 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:20 smithi177 bash[68594]: cluster 2024-09-13T15:06:18.945604+0000 mgr.y (mgr.26252) 1206 : cluster [DBG] pgmap v1763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24763/211851 objects degraded (11.689%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T15:06:20.177 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:20 smithi177 bash[68594]: cluster 2024-09-13T15:06:18.945604+0000 mgr.y (mgr.26252) 1206 : cluster [DBG] pgmap v1763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24763/211851 objects degraded (11.689%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T15:06:20.189 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:20.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:20.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:20.305 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:06:20 smithi177 bash[61439]: ts=2024-09-13T15:06:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:20.325 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:20.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:20.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:20.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:20.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:20.560 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:20.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:20 smithi099 bash[84576]: cluster 2024-09-13T15:06:18.945604+0000 mgr.y (mgr.26252) 1206 : cluster [DBG] pgmap v1763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24763/211851 objects degraded (11.689%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T15:06:20.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:20 smithi099 bash[84576]: cluster 2024-09-13T15:06:18.945604+0000 mgr.y (mgr.26252) 1206 : cluster [DBG] pgmap v1763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24763/211851 objects degraded (11.689%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T15:06:20.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:20 smithi099 bash[87147]: cluster 2024-09-13T15:06:18.945604+0000 mgr.y (mgr.26252) 1206 : cluster [DBG] pgmap v1763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24763/211851 objects degraded (11.689%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T15:06:20.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:20 smithi099 bash[87147]: cluster 2024-09-13T15:06:18.945604+0000 mgr.y (mgr.26252) 1206 : cluster [DBG] pgmap v1763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24763/211851 objects degraded (11.689%); 1.6 MiB/s, 26 objects/s recovering 2024-09-13T15:06:20.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:20.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:20.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:20.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:20.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:20.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:20.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:20.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:20.992 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:20.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:21.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:21.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:21.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:21.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:21.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:21.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:21.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:21.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:21.522 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:21.557 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:21.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:21.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:21.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:21.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:21.807 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:21.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:21.937 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:21.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:22.040 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:22.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:22.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:22.174 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:22.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:22.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:22.300 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:22.308 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:22 smithi177 bash[68594]: cluster 2024-09-13T15:06:20.946407+0000 mgr.y (mgr.26252) 1207 : cluster [DBG] pgmap v1764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24763/207519 objects degraded (11.933%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:22.308 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:22 smithi177 bash[68594]: cluster 2024-09-13T15:06:20.946407+0000 mgr.y (mgr.26252) 1207 : cluster [DBG] pgmap v1764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24763/207519 objects degraded (11.933%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:22.308 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:22 smithi177 bash[68594]: cluster 2024-09-13T15:06:21.533957+0000 mon.a (mon.0) 2350 : cluster [WRN] Health check update: Degraded data redundancy: 24763/207519 objects degraded (11.933%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:22.308 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:22 smithi177 bash[68594]: cluster 2024-09-13T15:06:21.533957+0000 mon.a (mon.0) 2350 : cluster [WRN] Health check update: Degraded data redundancy: 24763/207519 objects degraded (11.933%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:22.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:22.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:22.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:22 smithi099 bash[84576]: cluster 2024-09-13T15:06:20.946407+0000 mgr.y (mgr.26252) 1207 : cluster [DBG] pgmap v1764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24763/207519 objects degraded (11.933%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:22 smithi099 bash[84576]: cluster 2024-09-13T15:06:20.946407+0000 mgr.y (mgr.26252) 1207 : cluster [DBG] pgmap v1764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24763/207519 objects degraded (11.933%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:22 smithi099 bash[84576]: cluster 2024-09-13T15:06:21.533957+0000 mon.a (mon.0) 2350 : cluster [WRN] Health check update: Degraded data redundancy: 24763/207519 objects degraded (11.933%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:22 smithi099 bash[84576]: cluster 2024-09-13T15:06:21.533957+0000 mon.a (mon.0) 2350 : cluster [WRN] Health check update: Degraded data redundancy: 24763/207519 objects degraded (11.933%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:22 smithi099 bash[87147]: cluster 2024-09-13T15:06:20.946407+0000 mgr.y (mgr.26252) 1207 : cluster [DBG] pgmap v1764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24763/207519 objects degraded (11.933%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:22 smithi099 bash[87147]: cluster 2024-09-13T15:06:20.946407+0000 mgr.y (mgr.26252) 1207 : cluster [DBG] pgmap v1764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24763/207519 objects degraded (11.933%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:22 smithi099 bash[87147]: cluster 2024-09-13T15:06:21.533957+0000 mon.a (mon.0) 2350 : cluster [WRN] Health check update: Degraded data redundancy: 24763/207519 objects degraded (11.933%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:22 smithi099 bash[87147]: cluster 2024-09-13T15:06:21.533957+0000 mon.a (mon.0) 2350 : cluster [WRN] Health check update: Degraded data redundancy: 24763/207519 objects degraded (11.933%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:22.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:22.460 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:22.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:22.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:22.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:22.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:22.699 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:22.729 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:22.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:22.833 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:22.865 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:22.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:22.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:22.996 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:23.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:23.105 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:23.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:23.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:23.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:23.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:23.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:23.368 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:23.409 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:23.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:23.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:23.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:23.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:23.644 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:23.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:23.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:23.771 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:23.815 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:23.921 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:23.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:24.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:24.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:24.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:24.184 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:24.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:24 smithi177 bash[68594]: cluster 2024-09-13T15:06:22.946988+0000 mgr.y (mgr.26252) 1208 : cluster [DBG] pgmap v1765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24763/204363 objects degraded (12.117%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:24.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:24 smithi177 bash[68594]: cluster 2024-09-13T15:06:22.946988+0000 mgr.y (mgr.26252) 1208 : cluster [DBG] pgmap v1765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24763/204363 objects degraded (12.117%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:24.207 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:24.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:24.309 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:24 smithi099 bash[87147]: cluster 2024-09-13T15:06:22.946988+0000 mgr.y (mgr.26252) 1208 : cluster [DBG] pgmap v1765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24763/204363 objects degraded (12.117%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:24 smithi099 bash[87147]: cluster 2024-09-13T15:06:22.946988+0000 mgr.y (mgr.26252) 1208 : cluster [DBG] pgmap v1765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24763/204363 objects degraded (12.117%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:24 smithi099 bash[84576]: cluster 2024-09-13T15:06:22.946988+0000 mgr.y (mgr.26252) 1208 : cluster [DBG] pgmap v1765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24763/204363 objects degraded (12.117%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:24 smithi099 bash[84576]: cluster 2024-09-13T15:06:22.946988+0000 mgr.y (mgr.26252) 1208 : cluster [DBG] pgmap v1765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 24763/204363 objects degraded (12.117%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:24.336 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:24.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:24.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:24.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:24.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:24.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:24.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:24.690 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:24.715 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:24.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:24.819 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:24.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:24.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:24.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:24.975 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:24.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:25.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:25.115 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:25.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:25.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:25.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:25.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:25.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:25.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:25.396 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:25.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:25.524 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:25.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:25.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:25.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:25.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:25.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:25.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:25.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:25.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:25.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:25.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:26.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:26.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:26.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:26.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:26.180 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:26.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:26.283 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:26.317 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:26.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:26.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:26 smithi177 bash[68594]: cluster 2024-09-13T15:06:24.947601+0000 mgr.y (mgr.26252) 1209 : cluster [DBG] pgmap v1766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 24649/199656 objects degraded (12.346%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:26.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:26 smithi177 bash[68594]: cluster 2024-09-13T15:06:24.947601+0000 mgr.y (mgr.26252) 1209 : cluster [DBG] pgmap v1766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 24649/199656 objects degraded (12.346%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:26.420 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:26.430 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:26 smithi099 bash[84576]: cluster 2024-09-13T15:06:24.947601+0000 mgr.y (mgr.26252) 1209 : cluster [DBG] pgmap v1766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 24649/199656 objects degraded (12.346%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:26.430 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:26 smithi099 bash[84576]: cluster 2024-09-13T15:06:24.947601+0000 mgr.y (mgr.26252) 1209 : cluster [DBG] pgmap v1766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 24649/199656 objects degraded (12.346%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:26.430 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:26 smithi099 bash[87147]: cluster 2024-09-13T15:06:24.947601+0000 mgr.y (mgr.26252) 1209 : cluster [DBG] pgmap v1766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 24649/199656 objects degraded (12.346%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:26.430 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:26 smithi099 bash[87147]: cluster 2024-09-13T15:06:24.947601+0000 mgr.y (mgr.26252) 1209 : cluster [DBG] pgmap v1766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 24649/199656 objects degraded (12.346%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:06:26.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:26.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:26.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:26.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:26.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:26.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:26.729 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:26.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:26.837 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:26.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:26.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:26.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:27.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:27.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:27.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:27.153 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:27.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:27.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:27.274 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:27.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:27.287 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:06:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:06:27] "GET /metrics HTTP/1.1" 200 36786 "" "Prometheus/2.51.0" 2024-09-13T15:06:27.288 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:27 smithi099 bash[84576]: cluster 2024-09-13T15:06:26.534778+0000 mon.a (mon.0) 2351 : cluster [WRN] Health check update: Degraded data redundancy: 24649/199656 objects degraded (12.346%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:27.288 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:27 smithi099 bash[84576]: cluster 2024-09-13T15:06:26.534778+0000 mon.a (mon.0) 2351 : cluster [WRN] Health check update: Degraded data redundancy: 24649/199656 objects degraded (12.346%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:27.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:27 smithi099 bash[87147]: cluster 2024-09-13T15:06:26.534778+0000 mon.a (mon.0) 2351 : cluster [WRN] Health check update: Degraded data redundancy: 24649/199656 objects degraded (12.346%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:27.288 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:27 smithi099 bash[87147]: cluster 2024-09-13T15:06:26.534778+0000 mon.a (mon.0) 2351 : cluster [WRN] Health check update: Degraded data redundancy: 24649/199656 objects degraded (12.346%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:27 smithi177 bash[68594]: cluster 2024-09-13T15:06:26.534778+0000 mon.a (mon.0) 2351 : cluster [WRN] Health check update: Degraded data redundancy: 24649/199656 objects degraded (12.346%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:27 smithi177 bash[68594]: cluster 2024-09-13T15:06:26.534778+0000 mon.a (mon.0) 2351 : cluster [WRN] Health check update: Degraded data redundancy: 24649/199656 objects degraded (12.346%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:27.377 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:27.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:27.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:27.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:27.553 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:27.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:27.655 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:27.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:27.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:27.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:27.814 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:27.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:27.917 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:27.954 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:27.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:28.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:28.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:28.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:28.205 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:28.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:28 smithi177 bash[68594]: cluster 2024-09-13T15:06:26.948126+0000 mgr.y (mgr.26252) 1210 : cluster [DBG] pgmap v1767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24649/198471 objects degraded (12.419%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:28.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:28 smithi177 bash[68594]: cluster 2024-09-13T15:06:26.948126+0000 mgr.y (mgr.26252) 1210 : cluster [DBG] pgmap v1767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24649/198471 objects degraded (12.419%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:28.236 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:28.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:28.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:28.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:28.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:28.483 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:28.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:28.555 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:28 smithi099 bash[87147]: cluster 2024-09-13T15:06:26.948126+0000 mgr.y (mgr.26252) 1210 : cluster [DBG] pgmap v1767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24649/198471 objects degraded (12.419%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:28.555 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:28 smithi099 bash[87147]: cluster 2024-09-13T15:06:26.948126+0000 mgr.y (mgr.26252) 1210 : cluster [DBG] pgmap v1767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24649/198471 objects degraded (12.419%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:28.556 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:28 smithi099 bash[84576]: cluster 2024-09-13T15:06:26.948126+0000 mgr.y (mgr.26252) 1210 : cluster [DBG] pgmap v1767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24649/198471 objects degraded (12.419%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:28.556 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:28 smithi099 bash[84576]: cluster 2024-09-13T15:06:26.948126+0000 mgr.y (mgr.26252) 1210 : cluster [DBG] pgmap v1767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 24649/198471 objects degraded (12.419%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:28.586 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:28.623 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:28.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:28.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:28.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:28.863 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:28.886 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:28.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:29.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:29.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:29.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:29.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:29.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:29.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:29.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:29.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:29.415 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:29.447 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:29.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:29.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:29.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:29.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:29.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:29.715 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:29.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:29.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:29.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:29.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:29.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:30.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.091 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:30.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:30.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:06:30 smithi177 bash[61439]: ts=2024-09-13T15:06:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:30.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:30 smithi177 bash[68594]: cluster 2024-09-13T15:06:28.948665+0000 mgr.y (mgr.26252) 1211 : cluster [DBG] pgmap v1768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 24539/193947 objects degraded (12.652%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:06:30.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:30 smithi177 bash[68594]: cluster 2024-09-13T15:06:28.948665+0000 mgr.y (mgr.26252) 1211 : cluster [DBG] pgmap v1768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 24539/193947 objects degraded (12.652%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:06:30.366 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:30.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.494 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:30.510 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:30 smithi099 bash[84576]: cluster 2024-09-13T15:06:28.948665+0000 mgr.y (mgr.26252) 1211 : cluster [DBG] pgmap v1768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 24539/193947 objects degraded (12.652%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:06:30.510 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:30 smithi099 bash[84576]: cluster 2024-09-13T15:06:28.948665+0000 mgr.y (mgr.26252) 1211 : cluster [DBG] pgmap v1768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 24539/193947 objects degraded (12.652%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:06:30.510 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:30 smithi099 bash[87147]: cluster 2024-09-13T15:06:28.948665+0000 mgr.y (mgr.26252) 1211 : cluster [DBG] pgmap v1768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 24539/193947 objects degraded (12.652%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:06:30.510 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:30 smithi099 bash[87147]: cluster 2024-09-13T15:06:28.948665+0000 mgr.y (mgr.26252) 1211 : cluster [DBG] pgmap v1768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 24539/193947 objects degraded (12.652%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:06:30.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:30.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:30.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:30.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:30.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:30.863 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:30.999 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:31.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:31.133 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:31.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:31.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:31.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:31.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:31.431 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:31.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:31.538 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:31.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.583 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:31.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:31.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:31.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:31.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:31.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:31.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:31.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:31.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:32.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:32.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:32.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:32.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:32.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:32.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:32.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:32.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:32.311 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:32 smithi099 bash[84576]: cluster 2024-09-13T15:06:30.949290+0000 mgr.y (mgr.26252) 1212 : cluster [DBG] pgmap v1769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 24539/189864 objects degraded (12.925%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:32.311 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:32 smithi099 bash[84576]: cluster 2024-09-13T15:06:30.949290+0000 mgr.y (mgr.26252) 1212 : cluster [DBG] pgmap v1769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 24539/189864 objects degraded (12.925%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:32.312 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:32 smithi099 bash[84576]: cluster 2024-09-13T15:06:31.536517+0000 mon.a (mon.0) 2352 : cluster [WRN] Health check update: Degraded data redundancy: 24539/189864 objects degraded (12.925%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:32.312 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:32 smithi099 bash[84576]: cluster 2024-09-13T15:06:31.536517+0000 mon.a (mon.0) 2352 : cluster [WRN] Health check update: Degraded data redundancy: 24539/189864 objects degraded (12.925%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:32.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:32 smithi099 bash[87147]: cluster 2024-09-13T15:06:30.949290+0000 mgr.y (mgr.26252) 1212 : cluster [DBG] pgmap v1769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 24539/189864 objects degraded (12.925%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:32.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:32 smithi099 bash[87147]: cluster 2024-09-13T15:06:30.949290+0000 mgr.y (mgr.26252) 1212 : cluster [DBG] pgmap v1769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 24539/189864 objects degraded (12.925%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:32.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:32 smithi099 bash[87147]: cluster 2024-09-13T15:06:31.536517+0000 mon.a (mon.0) 2352 : cluster [WRN] Health check update: Degraded data redundancy: 24539/189864 objects degraded (12.925%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:32.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:32 smithi099 bash[87147]: cluster 2024-09-13T15:06:31.536517+0000 mon.a (mon.0) 2352 : cluster [WRN] Health check update: Degraded data redundancy: 24539/189864 objects degraded (12.925%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:32 smithi177 bash[68594]: cluster 2024-09-13T15:06:30.949290+0000 mgr.y (mgr.26252) 1212 : cluster [DBG] pgmap v1769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 24539/189864 objects degraded (12.925%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:32 smithi177 bash[68594]: cluster 2024-09-13T15:06:30.949290+0000 mgr.y (mgr.26252) 1212 : cluster [DBG] pgmap v1769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 24539/189864 objects degraded (12.925%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:32 smithi177 bash[68594]: cluster 2024-09-13T15:06:31.536517+0000 mon.a (mon.0) 2352 : cluster [WRN] Health check update: Degraded data redundancy: 24539/189864 objects degraded (12.925%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:32 smithi177 bash[68594]: cluster 2024-09-13T15:06:31.536517+0000 mon.a (mon.0) 2352 : cluster [WRN] Health check update: Degraded data redundancy: 24539/189864 objects degraded (12.925%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:32.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:32.410 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:32.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:32.513 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:32.536 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:32.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:32.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:32.667 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:32.771 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:32.801 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:32.908 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:32.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:32.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:33.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:33.065 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:33.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:33.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:33.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:33.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:33.295 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:33.323 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:33.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:33.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:33.461 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:33.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:33.565 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:33.667 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:33.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:33.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:33.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:33.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:33.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:33.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:34.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:34.043 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:34.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:34.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:34.177 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:34.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:34.281 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:34.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:34.301 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:34 smithi177 bash[68594]: cluster 2024-09-13T15:06:32.949942+0000 mgr.y (mgr.26252) 1213 : cluster [DBG] pgmap v1770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 24539/186561 objects degraded (13.153%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:34.301 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:34 smithi177 bash[68594]: cluster 2024-09-13T15:06:32.949942+0000 mgr.y (mgr.26252) 1213 : cluster [DBG] pgmap v1770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 24539/186561 objects degraded (13.153%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:34.301 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:34 smithi177 bash[68594]: audit 2024-09-13T15:06:33.302185+0000 mon.a (mon.0) 2353 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:34.301 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:34 smithi177 bash[68594]: audit 2024-09-13T15:06:33.302185+0000 mon.a (mon.0) 2353 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:34.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:34.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:34.439 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:34.542 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:34.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:34 smithi099 bash[84576]: cluster 2024-09-13T15:06:32.949942+0000 mgr.y (mgr.26252) 1213 : cluster [DBG] pgmap v1770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 24539/186561 objects degraded (13.153%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:34.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:34 smithi099 bash[84576]: cluster 2024-09-13T15:06:32.949942+0000 mgr.y (mgr.26252) 1213 : cluster [DBG] pgmap v1770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 24539/186561 objects degraded (13.153%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:34.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:34 smithi099 bash[84576]: audit 2024-09-13T15:06:33.302185+0000 mon.a (mon.0) 2353 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:34.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:34 smithi099 bash[84576]: audit 2024-09-13T15:06:33.302185+0000 mon.a (mon.0) 2353 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:34 smithi099 bash[87147]: cluster 2024-09-13T15:06:32.949942+0000 mgr.y (mgr.26252) 1213 : cluster [DBG] pgmap v1770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 24539/186561 objects degraded (13.153%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:34 smithi099 bash[87147]: cluster 2024-09-13T15:06:32.949942+0000 mgr.y (mgr.26252) 1213 : cluster [DBG] pgmap v1770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 24539/186561 objects degraded (13.153%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:34 smithi099 bash[87147]: audit 2024-09-13T15:06:33.302185+0000 mon.a (mon.0) 2353 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:34.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:34 smithi099 bash[87147]: audit 2024-09-13T15:06:33.302185+0000 mon.a (mon.0) 2353 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:34.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:34.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:34.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:34.793 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:34.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:35.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:35.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:35.108 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:06:35.108 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:06:35.202 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:35.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:35.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:35.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:35.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:35.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:35.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:35.515 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:35.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:35.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:35.644 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:35.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:35.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:35.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:35.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:35.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:35.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:35.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:36.010 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:36.034 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:36.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:36.142 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:36.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:36 smithi177 bash[68594]: cluster 2024-09-13T15:06:34.950535+0000 mgr.y (mgr.26252) 1214 : cluster [DBG] pgmap v1771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 24414/182283 objects degraded (13.393%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:06:36.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:36 smithi177 bash[68594]: cluster 2024-09-13T15:06:34.950535+0000 mgr.y (mgr.26252) 1214 : cluster [DBG] pgmap v1771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 24414/182283 objects degraded (13.393%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:06:36.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:36.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:36.278 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:36.295 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:36.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:36.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:36.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:36 smithi099 bash[84576]: cluster 2024-09-13T15:06:34.950535+0000 mgr.y (mgr.26252) 1214 : cluster [DBG] pgmap v1771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 24414/182283 objects degraded (13.393%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:06:36.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:36 smithi099 bash[84576]: cluster 2024-09-13T15:06:34.950535+0000 mgr.y (mgr.26252) 1214 : cluster [DBG] pgmap v1771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 24414/182283 objects degraded (13.393%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:06:36.407 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:36 smithi099 bash[87147]: cluster 2024-09-13T15:06:34.950535+0000 mgr.y (mgr.26252) 1214 : cluster [DBG] pgmap v1771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 24414/182283 objects degraded (13.393%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:06:36.407 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:36 smithi099 bash[87147]: cluster 2024-09-13T15:06:34.950535+0000 mgr.y (mgr.26252) 1214 : cluster [DBG] pgmap v1771: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 24414/182283 objects degraded (13.393%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:06:36.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:36.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:36.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:36.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:36.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:36.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:36.759 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:36.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:36.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:36.912 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:36.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:37.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:37.182 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:37.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:37.212 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:37 smithi099 bash[84576]: cluster 2024-09-13T15:06:36.537290+0000 mon.a (mon.0) 2354 : cluster [WRN] Health check update: Degraded data redundancy: 24414/182283 objects degraded (13.393%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:37.212 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:37 smithi099 bash[84576]: cluster 2024-09-13T15:06:36.537290+0000 mon.a (mon.0) 2354 : cluster [WRN] Health check update: Degraded data redundancy: 24414/182283 objects degraded (13.393%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:37.213 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:06:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:06:37] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-13T15:06:37.213 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:37 smithi099 bash[87147]: cluster 2024-09-13T15:06:36.537290+0000 mon.a (mon.0) 2354 : cluster [WRN] Health check update: Degraded data redundancy: 24414/182283 objects degraded (13.393%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:37.213 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:37 smithi099 bash[87147]: cluster 2024-09-13T15:06:36.537290+0000 mon.a (mon.0) 2354 : cluster [WRN] Health check update: Degraded data redundancy: 24414/182283 objects degraded (13.393%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:37.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:37.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:37.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:37 smithi177 bash[68594]: cluster 2024-09-13T15:06:36.537290+0000 mon.a (mon.0) 2354 : cluster [WRN] Health check update: Degraded data redundancy: 24414/182283 objects degraded (13.393%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:37.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:37 smithi177 bash[68594]: cluster 2024-09-13T15:06:36.537290+0000 mon.a (mon.0) 2354 : cluster [WRN] Health check update: Degraded data redundancy: 24414/182283 objects degraded (13.393%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:06:37.408 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:37.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:37.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:37.615 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:37.648 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:37.751 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:37.778 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:37.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:37.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:37.923 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:37.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:38.030 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:38.164 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:38.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:38.269 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:38.315 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:38.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:38.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:38 smithi099 bash[84576]: cluster 2024-09-13T15:06:36.951071+0000 mgr.y (mgr.26252) 1215 : cluster [DBG] pgmap v1772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 24414/181179 objects degraded (13.475%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:06:38.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:38 smithi099 bash[84576]: cluster 2024-09-13T15:06:36.951071+0000 mgr.y (mgr.26252) 1215 : cluster [DBG] pgmap v1772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 24414/181179 objects degraded (13.475%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:06:38.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:38 smithi099 bash[87147]: cluster 2024-09-13T15:06:36.951071+0000 mgr.y (mgr.26252) 1215 : cluster [DBG] pgmap v1772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 24414/181179 objects degraded (13.475%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:06:38.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:38 smithi099 bash[87147]: cluster 2024-09-13T15:06:36.951071+0000 mgr.y (mgr.26252) 1215 : cluster [DBG] pgmap v1772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 24414/181179 objects degraded (13.475%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:06:38.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:38 smithi177 bash[68594]: cluster 2024-09-13T15:06:36.951071+0000 mgr.y (mgr.26252) 1215 : cluster [DBG] pgmap v1772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 24414/181179 objects degraded (13.475%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:06:38.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:38 smithi177 bash[68594]: cluster 2024-09-13T15:06:36.951071+0000 mgr.y (mgr.26252) 1215 : cluster [DBG] pgmap v1772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 24414/181179 objects degraded (13.475%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:06:38.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:38.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:38.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:38.541 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:38.565 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:38.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:38.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:38.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:38.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:38.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:38.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:38.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:38.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:38.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:39.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:39.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:39.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:39.206 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:39.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:39.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:39.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:39 smithi177 bash[68594]: cluster 2024-09-13T15:06:38.951709+0000 mgr.y (mgr.26252) 1216 : cluster [DBG] pgmap v1773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 24290/177012 objects degraded (13.722%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:39.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:39 smithi177 bash[68594]: cluster 2024-09-13T15:06:38.951709+0000 mgr.y (mgr.26252) 1216 : cluster [DBG] pgmap v1773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 24290/177012 objects degraded (13.722%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:39.380 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:39.403 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:39 smithi099 bash[84576]: cluster 2024-09-13T15:06:38.951709+0000 mgr.y (mgr.26252) 1216 : cluster [DBG] pgmap v1773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 24290/177012 objects degraded (13.722%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:39.403 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:39 smithi099 bash[84576]: cluster 2024-09-13T15:06:38.951709+0000 mgr.y (mgr.26252) 1216 : cluster [DBG] pgmap v1773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 24290/177012 objects degraded (13.722%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:39.403 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:39 smithi099 bash[87147]: cluster 2024-09-13T15:06:38.951709+0000 mgr.y (mgr.26252) 1216 : cluster [DBG] pgmap v1773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 24290/177012 objects degraded (13.722%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:39.403 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:39 smithi099 bash[87147]: cluster 2024-09-13T15:06:38.951709+0000 mgr.y (mgr.26252) 1216 : cluster [DBG] pgmap v1773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 24290/177012 objects degraded (13.722%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T15:06:39.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:39.661 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:39.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:39.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:39.796 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:39.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:39.900 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:39.926 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:40.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:40.073 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:40.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:40.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.213 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:40.318 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:40.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:40.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:40.508 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:40.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:40.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:40.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:06:40 smithi177 bash[61439]: ts=2024-09-13T15:06:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:40.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:40.689 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:40.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:40.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:40.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.954 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:41.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:41.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:41.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:41.201 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:41.258 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:41.363 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:41.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:41.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:41.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:41.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:41.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:41.661 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:41.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:41.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T15:06:41.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:41.803 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:41.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:41.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:41.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:42.040 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:42.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:42.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:42.175 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:42.199 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:42.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:42.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:42.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:42 smithi099 bash[87147]: cluster 2024-09-13T15:06:40.952362+0000 mgr.y (mgr.26252) 1217 : cluster [DBG] pgmap v1774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 24290/172536 objects degraded (14.078%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:42.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:42 smithi099 bash[87147]: cluster 2024-09-13T15:06:40.952362+0000 mgr.y (mgr.26252) 1217 : cluster [DBG] pgmap v1774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 24290/172536 objects degraded (14.078%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:42.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:42 smithi099 bash[87147]: cluster 2024-09-13T15:06:41.538154+0000 mon.a (mon.0) 2355 : cluster [WRN] Health check update: Degraded data redundancy: 24290/172536 objects degraded (14.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:42.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:42 smithi099 bash[87147]: cluster 2024-09-13T15:06:41.538154+0000 mon.a (mon.0) 2355 : cluster [WRN] Health check update: Degraded data redundancy: 24290/172536 objects degraded (14.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:42.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:42 smithi099 bash[84576]: cluster 2024-09-13T15:06:40.952362+0000 mgr.y (mgr.26252) 1217 : cluster [DBG] pgmap v1774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 24290/172536 objects degraded (14.078%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:42.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:42 smithi099 bash[84576]: cluster 2024-09-13T15:06:40.952362+0000 mgr.y (mgr.26252) 1217 : cluster [DBG] pgmap v1774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 24290/172536 objects degraded (14.078%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:42.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:42 smithi099 bash[84576]: cluster 2024-09-13T15:06:41.538154+0000 mon.a (mon.0) 2355 : cluster [WRN] Health check update: Degraded data redundancy: 24290/172536 objects degraded (14.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:42.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:42 smithi099 bash[84576]: cluster 2024-09-13T15:06:41.538154+0000 mon.a (mon.0) 2355 : cluster [WRN] Health check update: Degraded data redundancy: 24290/172536 objects degraded (14.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:42.338 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:42.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:42.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:42 smithi177 bash[68594]: cluster 2024-09-13T15:06:40.952362+0000 mgr.y (mgr.26252) 1217 : cluster [DBG] pgmap v1774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 24290/172536 objects degraded (14.078%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:42.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:42 smithi177 bash[68594]: cluster 2024-09-13T15:06:40.952362+0000 mgr.y (mgr.26252) 1217 : cluster [DBG] pgmap v1774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 24290/172536 objects degraded (14.078%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:42.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:42 smithi177 bash[68594]: cluster 2024-09-13T15:06:41.538154+0000 mon.a (mon.0) 2355 : cluster [WRN] Health check update: Degraded data redundancy: 24290/172536 objects degraded (14.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:42.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:42 smithi177 bash[68594]: cluster 2024-09-13T15:06:41.538154+0000 mon.a (mon.0) 2355 : cluster [WRN] Health check update: Degraded data redundancy: 24290/172536 objects degraded (14.078%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:42.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:42.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:42.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:42.580 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:42.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:42.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:42.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:42.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:42.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:42.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:42.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:42.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:42.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:43.017 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:43.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:43.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:43.146 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:43.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:43.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:43.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:43.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:43.399 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:43.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:43.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:43.543 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:43.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:43.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:43.743 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:43.848 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:43.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:43.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:43.981 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:44.008 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:44.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:44.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:44.156 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:44.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:44.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:44.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:44 smithi099 bash[84576]: cluster 2024-09-13T15:06:42.953026+0000 mgr.y (mgr.26252) 1218 : cluster [DBG] pgmap v1775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24290/168987 objects degraded (14.374%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:44.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:44 smithi099 bash[84576]: cluster 2024-09-13T15:06:42.953026+0000 mgr.y (mgr.26252) 1218 : cluster [DBG] pgmap v1775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24290/168987 objects degraded (14.374%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:44.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:44 smithi099 bash[87147]: cluster 2024-09-13T15:06:42.953026+0000 mgr.y (mgr.26252) 1218 : cluster [DBG] pgmap v1775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24290/168987 objects degraded (14.374%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:44.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:44 smithi099 bash[87147]: cluster 2024-09-13T15:06:42.953026+0000 mgr.y (mgr.26252) 1218 : cluster [DBG] pgmap v1775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24290/168987 objects degraded (14.374%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:44.297 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:44.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:44.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:44 smithi177 bash[68594]: cluster 2024-09-13T15:06:42.953026+0000 mgr.y (mgr.26252) 1218 : cluster [DBG] pgmap v1775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24290/168987 objects degraded (14.374%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:44.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:44 smithi177 bash[68594]: cluster 2024-09-13T15:06:42.953026+0000 mgr.y (mgr.26252) 1218 : cluster [DBG] pgmap v1775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 24290/168987 objects degraded (14.374%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:06:44.403 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:44.593 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:44.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:44.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:44.726 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:44.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:44.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:44.860 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:44.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:44.968 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:44.996 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:44.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:45.100 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:45.118 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:45.221 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:45.251 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:45.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:45.354 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:45.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:45.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:45.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:45.530 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:45.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:45.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:45.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:45.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:45.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:45.806 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:45.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:45.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:45.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:45.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:46.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:46.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:46.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:46.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:46 smithi177 bash[68594]: cluster 2024-09-13T15:06:44.953700+0000 mgr.y (mgr.26252) 1219 : cluster [DBG] pgmap v1776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 24152/164034 objects degraded (14.724%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:06:46.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:46 smithi177 bash[68594]: cluster 2024-09-13T15:06:44.953700+0000 mgr.y (mgr.26252) 1219 : cluster [DBG] pgmap v1776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 24152/164034 objects degraded (14.724%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:06:46.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:46.193 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:46 smithi099 bash[84576]: cluster 2024-09-13T15:06:44.953700+0000 mgr.y (mgr.26252) 1219 : cluster [DBG] pgmap v1776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 24152/164034 objects degraded (14.724%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:06:46.193 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:46 smithi099 bash[84576]: cluster 2024-09-13T15:06:44.953700+0000 mgr.y (mgr.26252) 1219 : cluster [DBG] pgmap v1776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 24152/164034 objects degraded (14.724%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:06:46.193 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:46 smithi099 bash[87147]: cluster 2024-09-13T15:06:44.953700+0000 mgr.y (mgr.26252) 1219 : cluster [DBG] pgmap v1776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 24152/164034 objects degraded (14.724%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:06:46.193 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:46 smithi099 bash[87147]: cluster 2024-09-13T15:06:44.953700+0000 mgr.y (mgr.26252) 1219 : cluster [DBG] pgmap v1776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 24152/164034 objects degraded (14.724%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:06:46.213 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:46.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:46.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:46.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:46.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:46.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:46.590 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:46.623 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:46.705 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:06:46.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:46.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:46.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:46.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.908 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:46.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:47.009 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:47.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:47.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:47.136 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:47.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:47.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:47.206 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:47 smithi099 bash[84576]: cluster 2024-09-13T15:06:46.538781+0000 mon.a (mon.0) 2356 : cluster [WRN] Health check update: Degraded data redundancy: 24152/164034 objects degraded (14.724%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:47.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:47.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:47.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:47.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:47 smithi099 bash[84576]: cluster 2024-09-13T15:06:46.538781+0000 mon.a (mon.0) 2356 : cluster [WRN] Health check update: Degraded data redundancy: 24152/164034 objects degraded (14.724%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:47.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:06:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:06:47] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-13T15:06:47.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:47 smithi099 bash[87147]: cluster 2024-09-13T15:06:46.538781+0000 mon.a (mon.0) 2356 : cluster [WRN] Health check update: Degraded data redundancy: 24152/164034 objects degraded (14.724%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:47.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:47 smithi099 bash[87147]: cluster 2024-09-13T15:06:46.538781+0000 mon.a (mon.0) 2356 : cluster [WRN] Health check update: Degraded data redundancy: 24152/164034 objects degraded (14.724%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:47.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:47 smithi177 bash[68594]: cluster 2024-09-13T15:06:46.538781+0000 mon.a (mon.0) 2356 : cluster [WRN] Health check update: Degraded data redundancy: 24152/164034 objects degraded (14.724%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:47.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:47 smithi177 bash[68594]: cluster 2024-09-13T15:06:46.538781+0000 mon.a (mon.0) 2356 : cluster [WRN] Health check update: Degraded data redundancy: 24152/164034 objects degraded (14.724%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:47.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:47.408 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:47.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:47.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:47.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:47.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:47.685 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:47.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:47.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:47.815 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:47.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:47.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:47.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:48.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:48.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:48 smithi099 bash[84576]: cluster 2024-09-13T15:06:46.954287+0000 mgr.y (mgr.26252) 1220 : cluster [DBG] pgmap v1777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 24152/162861 objects degraded (14.830%); 1.4 MiB/s, 21 objects/s recovering 2024-09-13T15:06:48.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:48 smithi099 bash[84576]: cluster 2024-09-13T15:06:46.954287+0000 mgr.y (mgr.26252) 1220 : cluster [DBG] pgmap v1777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 24152/162861 objects degraded (14.830%); 1.4 MiB/s, 21 objects/s recovering 2024-09-13T15:06:48.061 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:48 smithi099 bash[87147]: cluster 2024-09-13T15:06:46.954287+0000 mgr.y (mgr.26252) 1220 : cluster [DBG] pgmap v1777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 24152/162861 objects degraded (14.830%); 1.4 MiB/s, 21 objects/s recovering 2024-09-13T15:06:48.061 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:48 smithi099 bash[87147]: cluster 2024-09-13T15:06:46.954287+0000 mgr.y (mgr.26252) 1220 : cluster [DBG] pgmap v1777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 24152/162861 objects degraded (14.830%); 1.4 MiB/s, 21 objects/s recovering 2024-09-13T15:06:48.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:48.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:48.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:48.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:48.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:48.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:48 smithi177 bash[68594]: cluster 2024-09-13T15:06:46.954287+0000 mgr.y (mgr.26252) 1220 : cluster [DBG] pgmap v1777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 24152/162861 objects degraded (14.830%); 1.4 MiB/s, 21 objects/s recovering 2024-09-13T15:06:48.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:48 smithi177 bash[68594]: cluster 2024-09-13T15:06:46.954287+0000 mgr.y (mgr.26252) 1220 : cluster [DBG] pgmap v1777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 24152/162861 objects degraded (14.830%); 1.4 MiB/s, 21 objects/s recovering 2024-09-13T15:06:48.377 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:48.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:48.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:48.531 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:48.539 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:06:48.539 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:06:48.595 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:06:48.595 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:06:48.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:48.643 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:06:48.643 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:06:48.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:48.685 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:06:48.685 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:06:48.717 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:06:48.717 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:06:48.767 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:48.770 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:06:48.770 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:06:48.804 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:48.808 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:06:48.809 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:06:48.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:48.853 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:06:48.854 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:06:48.874 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T15:06:48.876 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T15:06:48.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:48.939 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:48.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:49.046 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:49.077 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:06:49.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:49.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:49.138 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[84576]: audit 2024-09-13T15:06:48.302853+0000 mon.a (mon.0) 2357 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:49.138 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[84576]: audit 2024-09-13T15:06:48.302853+0000 mon.a (mon.0) 2357 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:49.138 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[84576]: audit 2024-09-13T15:06:48.486841+0000 mon.a (mon.0) 2358 : audit [DBG] from='client.? 172.21.15.99:0/29740158' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:49.138 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[84576]: audit 2024-09-13T15:06:48.486841+0000 mon.a (mon.0) 2358 : audit [DBG] from='client.? 172.21.15.99:0/29740158' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:49.138 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[84576]: audit 2024-09-13T15:06:48.886207+0000 mon.c (mon.2) 95 : audit [INF] from='client.? 172.21.15.99:0/1814377154' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.138 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[84576]: audit 2024-09-13T15:06:48.886207+0000 mon.c (mon.2) 95 : audit [INF] from='client.? 172.21.15.99:0/1814377154' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.138 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[84576]: audit 2024-09-13T15:06:48.890794+0000 mon.a (mon.0) 2359 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.138 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[84576]: audit 2024-09-13T15:06:48.890794+0000 mon.a (mon.0) 2359 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.139 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[87147]: audit 2024-09-13T15:06:48.302853+0000 mon.a (mon.0) 2357 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:49.139 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[87147]: audit 2024-09-13T15:06:48.302853+0000 mon.a (mon.0) 2357 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:49.139 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[87147]: audit 2024-09-13T15:06:48.486841+0000 mon.a (mon.0) 2358 : audit [DBG] from='client.? 172.21.15.99:0/29740158' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:49.139 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[87147]: audit 2024-09-13T15:06:48.486841+0000 mon.a (mon.0) 2358 : audit [DBG] from='client.? 172.21.15.99:0/29740158' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:49.139 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[87147]: audit 2024-09-13T15:06:48.886207+0000 mon.c (mon.2) 95 : audit [INF] from='client.? 172.21.15.99:0/1814377154' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.139 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[87147]: audit 2024-09-13T15:06:48.886207+0000 mon.c (mon.2) 95 : audit [INF] from='client.? 172.21.15.99:0/1814377154' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.139 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[87147]: audit 2024-09-13T15:06:48.890794+0000 mon.a (mon.0) 2359 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.140 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:49 smithi099 bash[87147]: audit 2024-09-13T15:06:48.890794+0000 mon.a (mon.0) 2359 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:49.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:49 smithi177 bash[68594]: audit 2024-09-13T15:06:48.302853+0000 mon.a (mon.0) 2357 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:49.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:49 smithi177 bash[68594]: audit 2024-09-13T15:06:48.302853+0000 mon.a (mon.0) 2357 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:49.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:49 smithi177 bash[68594]: audit 2024-09-13T15:06:48.486841+0000 mon.a (mon.0) 2358 : audit [DBG] from='client.? 172.21.15.99:0/29740158' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:49.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:49 smithi177 bash[68594]: audit 2024-09-13T15:06:48.486841+0000 mon.a (mon.0) 2358 : audit [DBG] from='client.? 172.21.15.99:0/29740158' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:49.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:49 smithi177 bash[68594]: audit 2024-09-13T15:06:48.886207+0000 mon.c (mon.2) 95 : audit [INF] from='client.? 172.21.15.99:0/1814377154' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:49 smithi177 bash[68594]: audit 2024-09-13T15:06:48.886207+0000 mon.c (mon.2) 95 : audit [INF] from='client.? 172.21.15.99:0/1814377154' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:49 smithi177 bash[68594]: audit 2024-09-13T15:06:48.890794+0000 mon.a (mon.0) 2359 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.199 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:49 smithi177 bash[68594]: audit 2024-09-13T15:06:48.890794+0000 mon.a (mon.0) 2359 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:49.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:49.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:49.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:49.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:49.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:49.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:49.472 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:49.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:49.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:49.605 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:49.710 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:49.744 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:49.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:49.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:49.881 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:49.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:49.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:50.012 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:50.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:50.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:50.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:50.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:50.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:50.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[84576]: audit 2024-09-13T15:06:48.863921+0000 mgr.y (mgr.26252) 1221 : audit [DBG] from='client.54958 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:50.256 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[84576]: audit 2024-09-13T15:06:48.863921+0000 mgr.y (mgr.26252) 1221 : audit [DBG] from='client.54958 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:50.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[84576]: cluster 2024-09-13T15:06:48.954958+0000 mgr.y (mgr.26252) 1222 : cluster [DBG] pgmap v1778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24031/158367 objects degraded (15.174%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:06:50.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[84576]: cluster 2024-09-13T15:06:48.954958+0000 mgr.y (mgr.26252) 1222 : cluster [DBG] pgmap v1778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24031/158367 objects degraded (15.174%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:06:50.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[84576]: cluster 2024-09-13T15:06:49.038432+0000 mon.a (mon.0) 2360 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:06:50.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[84576]: cluster 2024-09-13T15:06:49.038432+0000 mon.a (mon.0) 2360 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:06:50.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[84576]: audit 2024-09-13T15:06:49.056274+0000 mon.a (mon.0) 2361 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:50.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[84576]: audit 2024-09-13T15:06:49.056274+0000 mon.a (mon.0) 2361 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:50.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[84576]: cluster 2024-09-13T15:06:49.078583+0000 mon.a (mon.0) 2362 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2024-09-13T15:06:50.257 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[84576]: cluster 2024-09-13T15:06:49.078583+0000 mon.a (mon.0) 2362 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2024-09-13T15:06:50.257 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[87147]: audit 2024-09-13T15:06:48.863921+0000 mgr.y (mgr.26252) 1221 : audit [DBG] from='client.54958 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:50.257 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[87147]: audit 2024-09-13T15:06:48.863921+0000 mgr.y (mgr.26252) 1221 : audit [DBG] from='client.54958 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:50.258 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[87147]: cluster 2024-09-13T15:06:48.954958+0000 mgr.y (mgr.26252) 1222 : cluster [DBG] pgmap v1778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24031/158367 objects degraded (15.174%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:06:50.258 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[87147]: cluster 2024-09-13T15:06:48.954958+0000 mgr.y (mgr.26252) 1222 : cluster [DBG] pgmap v1778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24031/158367 objects degraded (15.174%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:06:50.258 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[87147]: cluster 2024-09-13T15:06:49.038432+0000 mon.a (mon.0) 2360 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:06:50.258 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[87147]: cluster 2024-09-13T15:06:49.038432+0000 mon.a (mon.0) 2360 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:06:50.258 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[87147]: audit 2024-09-13T15:06:49.056274+0000 mon.a (mon.0) 2361 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:50.258 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[87147]: audit 2024-09-13T15:06:49.056274+0000 mon.a (mon.0) 2361 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:50.258 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[87147]: cluster 2024-09-13T15:06:49.078583+0000 mon.a (mon.0) 2362 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2024-09-13T15:06:50.258 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:50 smithi099 bash[87147]: cluster 2024-09-13T15:06:49.078583+0000 mon.a (mon.0) 2362 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2024-09-13T15:06:50.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:50.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:50.360 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[61439]: ts=2024-09-13T15:06:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:06:50.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[68594]: audit 2024-09-13T15:06:48.863921+0000 mgr.y (mgr.26252) 1221 : audit [DBG] from='client.54958 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:50.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[68594]: audit 2024-09-13T15:06:48.863921+0000 mgr.y (mgr.26252) 1221 : audit [DBG] from='client.54958 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:50.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[68594]: cluster 2024-09-13T15:06:48.954958+0000 mgr.y (mgr.26252) 1222 : cluster [DBG] pgmap v1778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24031/158367 objects degraded (15.174%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:06:50.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[68594]: cluster 2024-09-13T15:06:48.954958+0000 mgr.y (mgr.26252) 1222 : cluster [DBG] pgmap v1778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 24031/158367 objects degraded (15.174%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:06:50.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[68594]: cluster 2024-09-13T15:06:49.038432+0000 mon.a (mon.0) 2360 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:06:50.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[68594]: cluster 2024-09-13T15:06:49.038432+0000 mon.a (mon.0) 2360 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:06:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[68594]: audit 2024-09-13T15:06:49.056274+0000 mon.a (mon.0) 2361 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[68594]: audit 2024-09-13T15:06:49.056274+0000 mon.a (mon.0) 2361 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[68594]: cluster 2024-09-13T15:06:49.078583+0000 mon.a (mon.0) 2362 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2024-09-13T15:06:50.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:50 smithi177 bash[68594]: cluster 2024-09-13T15:06:49.078583+0000 mon.a (mon.0) 2362 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2024-09-13T15:06:50.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:50.409 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:50.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:50.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:50.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:50.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:50.641 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:50.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:50.765 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:50.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:50.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:50.893 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:50.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:50.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:51.036 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:51.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:51.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:51.202 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:51.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:51.312 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:51.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:51.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:51.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:51.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:51.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:51.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:51.608 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:51.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:51.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:51.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:51.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:51.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:51.871 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:51.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:51.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:52.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:52.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:52.118 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:52.125 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:52 smithi099 bash[87147]: cluster 2024-09-13T15:06:50.955470+0000 mgr.y (mgr.26252) 1223 : cluster [DBG] pgmap v1780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 24031/153822 objects degraded (15.623%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:52.125 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:52 smithi099 bash[87147]: cluster 2024-09-13T15:06:50.955470+0000 mgr.y (mgr.26252) 1223 : cluster [DBG] pgmap v1780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 24031/153822 objects degraded (15.623%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:52.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:52 smithi099 bash[87147]: cluster 2024-09-13T15:06:51.539573+0000 mon.a (mon.0) 2363 : cluster [WRN] Health check update: Degraded data redundancy: 24031/153822 objects degraded (15.623%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:52.126 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:52 smithi099 bash[87147]: cluster 2024-09-13T15:06:51.539573+0000 mon.a (mon.0) 2363 : cluster [WRN] Health check update: Degraded data redundancy: 24031/153822 objects degraded (15.623%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:52.126 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:52 smithi099 bash[84576]: cluster 2024-09-13T15:06:50.955470+0000 mgr.y (mgr.26252) 1223 : cluster [DBG] pgmap v1780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 24031/153822 objects degraded (15.623%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:52.126 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:52 smithi099 bash[84576]: cluster 2024-09-13T15:06:50.955470+0000 mgr.y (mgr.26252) 1223 : cluster [DBG] pgmap v1780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 24031/153822 objects degraded (15.623%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:52.126 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:52 smithi099 bash[84576]: cluster 2024-09-13T15:06:51.539573+0000 mon.a (mon.0) 2363 : cluster [WRN] Health check update: Degraded data redundancy: 24031/153822 objects degraded (15.623%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:52.126 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:52 smithi099 bash[84576]: cluster 2024-09-13T15:06:51.539573+0000 mon.a (mon.0) 2363 : cluster [WRN] Health check update: Degraded data redundancy: 24031/153822 objects degraded (15.623%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:52.137 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:52.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:52.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:52 smithi177 bash[68594]: cluster 2024-09-13T15:06:50.955470+0000 mgr.y (mgr.26252) 1223 : cluster [DBG] pgmap v1780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 24031/153822 objects degraded (15.623%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:52 smithi177 bash[68594]: cluster 2024-09-13T15:06:50.955470+0000 mgr.y (mgr.26252) 1223 : cluster [DBG] pgmap v1780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 24031/153822 objects degraded (15.623%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:52 smithi177 bash[68594]: cluster 2024-09-13T15:06:51.539573+0000 mon.a (mon.0) 2363 : cluster [WRN] Health check update: Degraded data redundancy: 24031/153822 objects degraded (15.623%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:52 smithi177 bash[68594]: cluster 2024-09-13T15:06:51.539573+0000 mon.a (mon.0) 2363 : cluster [WRN] Health check update: Degraded data redundancy: 24031/153822 objects degraded (15.623%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:52.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:52.399 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:52.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:52.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:52.539 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:52.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:52.645 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:52.698 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:52.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:52.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:52.808 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:06:52.808 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:06:52.808 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45764.0:165401 44.4 e1070/1056 44:215f83cc:::benchmark_data_smithi099_188632_object58197:head v 1070'10512, mlcod=1070'10509)", 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:52.794102+0000", 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.013953307999999999, 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01398661, 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794102+0000", 2024-09-13T15:06:52.809 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794102+0000", 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794102+0000", 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.91e-07 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:06:52.810 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794106+0000", 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2599999999999999e-06 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794112+0000", 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8150000000000002e-06 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794117+0000", 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2290000000000002e-06 2024-09-13T15:06:52.811 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.797285+0000", 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031680969999999999 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.797304+0000", 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8822e-05 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.812 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:165402 44.c 44:36101522:::benchmark_data_smithi099_188632_object58198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1070)", 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:52.794551+0000", 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.013504251, 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013595131999999999, 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 165402 2024-09-13T15:06:52.813 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794551+0000", 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794551+0000", 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.814 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794552+0000", 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3149999999999999e-06 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794558+0000", 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6180000000000003e-06 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794576+0000", 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7946999999999999e-05 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.815 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.794585+0000", 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2420000000000006e-06 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804374+0000", 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0097892410000000006 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804483+0000", 2024-09-13T15:06:52.816 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010892599999999999 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804629+0000", 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000145666 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45764.0:165403 44.4 e1070/1056)", 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:52.801286+0000", 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0067694180000000001, 2024-09-13T15:06:52.817 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0069182949999999997, 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "queued for pg", 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.801286+0000", 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.801286+0000", 2024-09-13T15:06:52.818 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.801288+0000", 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.826e-06 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.801297+0000", 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1169999999999993e-06 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:06:52.819 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.801309+0000", 2024-09-13T15:06:52.820 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2206e-05 2024-09-13T15:06:52.820 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.820 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.820 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:06:52.820 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.801318+0000", 2024-09-13T15:06:52.820 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4059999999999994e-06 2024-09-13T15:06:52.820 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.820 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:06:52.820 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:165405 44.9 44:95b9acba:::benchmark_data_smithi099_188632_object58201:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1070)", 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:52.804227+0000", 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0038277860000000001, 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0040171920000000002, 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:06:52.821 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 165405 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804227+0000", 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804227+0000", 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.822 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804229+0000", 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.587e-06 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804235+0000", 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.562e-06 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804250+0000", 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4513e-05 2024-09-13T15:06:52.823 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804258+0000", 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5029999999999997e-06 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804321+0000", 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2739999999999999e-05 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:06:52.824 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804488+0000", 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016643300000000001 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.804636+0000", 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014860200000000001 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:165400 44.9 44:93f82326:::benchmark_data_smithi099_188632_object58196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1070)", 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:52.791118+0000", 2024-09-13T15:06:52.825 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.01693739, 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017185645999999999, 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 165400 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.826 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.791118+0000", 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.791118+0000", 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.791119+0000", 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.395e-06 2024-09-13T15:06:52.827 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.791125+0000", 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8250000000000001e-06 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.791136+0000", 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0932e-05 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.791143+0000", 2024-09-13T15:06:52.828 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8029999999999999e-06 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.797068+0000", 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059256819999999998 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.797220+0000", 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015117200000000001 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:06:52.829 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.797374+0000", 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015407100000000001 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.801414+0000", 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0040408010000000001 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45764.0:165396 44.4 e1070/1056 44:2d7c6e3d:::benchmark_data_smithi099_188632_object58192:head v 1070'10511, mlcod=1070'10509)", 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:52.789566+0000", 2024-09-13T15:06:52.830 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.018488976000000001, 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018797833999999999, 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.789566+0000", 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.789566+0000", 2024-09-13T15:06:52.831 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.789567+0000", 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6899999999999996e-07 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.789577+0000", 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0278999999999999e-05 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.832 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.793011+0000", 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034341739999999999 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.793019+0000", 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6520000000000006e-06 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.793469+0000", 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000450187 2024-09-13T15:06:52.833 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:52.793498+0000", 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8509000000000001e-05 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 6 2024-09-13T15:06:52.834 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:06:52.835 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:52.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:52.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:52.967 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:52.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:53.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:53.107 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:53.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:53.211 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:53.242 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:53.347 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:53.370 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:53.456 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":1777,"stamp":"2024-09-13T15:06:46.953913+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3556579301,"num_objects":54287,"num_object_clones":0,"num_object_copies":162861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24152,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54287,"num_whiteouts":0,"num_read":345,"num_read_kb":800,"num_write":160355,"num_write_kb":6862524,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1654,"num_bytes_recovered":106829298,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139807,"ondisk_log_size":139807,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14419136,"kb_used_data":9658560,"kb_used_omap":397,"kb_used_meta":4759794,"kb_avail":735542080,"statfs":{"total":767960285184,"available":753195089920,"internally_reserved":0,"allocated":9890365440,"data_stored":9856458609,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":406940,"internal_metadata":4874029668},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-424280064,"num_objects":-6474,"num_object_clones":0,"num_object_copies":-19422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-262,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6474,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":262,"num_bytes_recovered":17170432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003775"},"pg_stats":[{"pgid":"44.f","version":"1069'10035","reported_seq":10073,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:42.619937+0000","last_change":"2024-09-13T15:03:24.065298+0000","last_active":"2024-09-13T15:06:42.619937+0000","last_peered":"2024-09-13T15:06:42.619937+0000","last_clean":"2024-09-13T15:06:42.619937+0000","last_became_active":"2024-09-13T15:03:24.064467+0000","last_became_peered":"2024-09-13T15:03:24.064467+0000","last_unstale":"2024-09-13T15:06:42.619937+0000","last_undegraded":"2024-09-13T15:06:42.619937+0000","last_fullsized":"2024-09-13T15:06:42.619937+0000","mapping_epoch":1056,"log_start":"1060'900","ondisk_log_start":"1060'900","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":9135,"log_dups_size":0,"ondisk_log_size":9135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:37.327903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261888,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10035,"num_write_kb":432576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"44.e","version":"1069'10008","reported_seq":10046,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:45.034424+0000","last_change":"2024-09-13T15:03:24.074307+0000","last_active":"2024-09-13T15:06:45.034424+0000","last_peered":"2024-09-13T15:06:45.034424+0000","last_clean":"2024-09-13T15:06:45.034424+0000","last_became_active":"2024-09-13T15:03:24.073622+0000","last_became_peered":"2024-09-13T15:03:24.073622+0000","last_unstale":"2024-09-13T15:06:45.034424+0000","last_undegraded":"2024-09-13T15:06:45.034424+0000","last_fullsized":"2024-09-13T15:06:45.034424+0000","mapping_epoch":1056,"log_start":"1060'2300","ondisk_log_start":"1060'2300","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":7708,"log_dups_size":0,"ondisk_log_size":7708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:33:21.099179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217186304,"num_objects":3314,"num_object_clones":0,"num_object_copies":9942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10008,"num_write_kb":426304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"44.d","version":"1069'10085","reported_seq":10107,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:44.277037+0000","last_change":"2024-09-13T15:03:24.074415+0000","last_active":"2024-09-13T15:06:44.277037+0000","last_peered":"2024-09-13T15:06:44.277037+0000","last_clean":"2024-09-13T15:06:44.277037+0000","last_became_active":"2024-09-13T15:03:24.074079+0000","last_became_peered":"2024-09-13T15:03:24.074079+0000","last_unstale":"2024-09-13T15:06:44.277037+0000","last_undegraded":"2024-09-13T15:06:44.277037+0000","last_fullsized":"2024-09-13T15:06:44.277037+0000","mapping_epoch":1056,"log_start":"1060'900","ondisk_log_start":"1060'900","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":9185,"log_dups_size":0,"ondisk_log_size":9185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:21:31.201502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223084568,"num_objects":3405,"num_object_clones":0,"num_object_copies":10215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3405,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10085,"num_write_kb":431617,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"44.c","version":"1069'10074","reported_seq":10112,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:42.323515+0000","last_change":"2024-09-13T15:03:24.074773+0000","last_active":"2024-09-13T15:06:42.323515+0000","last_peered":"2024-09-13T15:06:42.323515+0000","last_clean":"2024-09-13T15:06:42.323515+0000","last_became_active":"2024-09-13T15:03:24.073943+0000","last_became_peered":"2024-09-13T15:03:24.073943+0000","last_unstale":"2024-09-13T15:06:42.323515+0000","last_undegraded":"2024-09-13T15:06:42.323515+0000","last_fullsized":"2024-09-13T15:06:42.323515+0000","mapping_epoch":1056,"log_start":"1060'2500","ondisk_log_start":"1060'2500","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":7574,"log_dups_size":0,"ondisk_log_size":7574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:34:56.952158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228196352,"num_objects":3482,"num_object_clones":0,"num_object_copies":10446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10074,"num_write_kb":433792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"44.b","version":"1069'9792","reported_seq":9820,"reported_epoch":1069,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:06:44.290696+0000","last_change":"2024-09-13T15:05:35.728995+0000","last_active":"2024-09-13T15:06:44.290696+0000","last_peered":"2024-09-13T15:06:44.290696+0000","last_clean":"2024-09-13T15:03:23.031187+0000","last_became_active":"2024-09-13T15:05:35.662800+0000","last_became_peered":"2024-09-13T15:05:35.662800+0000","last_unstale":"2024-09-13T15:06:44.290696+0000","last_undegraded":"2024-09-13T15:05:35.099004+0000","last_fullsized":"2024-09-13T15:05:35.061283+0000","mapping_epoch":1067,"log_start":"1060'700","ondisk_log_start":"1060'700","created":1056,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":9092,"log_dups_size":0,"ondisk_log_size":9092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:55.913860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217317376,"num_objects":3316,"num_object_clones":0,"num_object_copies":9948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6554,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9792,"num_write_kb":419456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,6","objects":-2482}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"44.a","version":"1069'10169","reported_seq":10189,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:44.804601+0000","last_change":"2024-09-13T15:03:24.068238+0000","last_active":"2024-09-13T15:06:44.804601+0000","last_peered":"2024-09-13T15:06:44.804601+0000","last_clean":"2024-09-13T15:06:44.804601+0000","last_became_active":"2024-09-13T15:03:24.067501+0000","last_became_peered":"2024-09-13T15:03:24.067501+0000","last_unstale":"2024-09-13T15:06:44.804601+0000","last_undegraded":"2024-09-13T15:06:44.804601+0000","last_fullsized":"2024-09-13T15:06:44.804601+0000","mapping_epoch":1056,"log_start":"1060'2000","ondisk_log_start":"1060'2000","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":8169,"log_dups_size":0,"ondisk_log_size":8169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:20.182063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218038272,"num_objects":3327,"num_object_clones":0,"num_object_copies":9981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10169,"num_write_kb":431872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"44.9","version":"1069'9875","reported_seq":9896,"reported_epoch":1069,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:06:42.348963+0000","last_change":"2024-09-13T15:05:35.734869+0000","last_active":"2024-09-13T15:06:42.348963+0000","last_peered":"2024-09-13T15:06:42.348963+0000","last_clean":"2024-09-13T15:03:23.031187+0000","last_became_active":"2024-09-13T15:05:35.690348+0000","last_became_peered":"2024-09-13T15:05:35.690348+0000","last_unstale":"2024-09-13T15:06:42.348963+0000","last_undegraded":"2024-09-13T15:05:35.108149+0000","last_fullsized":"2024-09-13T15:05:35.059294+0000","mapping_epoch":1067,"log_start":"1060'2300","ondisk_log_start":"1060'2300","created":1056,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":7575,"log_dups_size":0,"ondisk_log_size":7575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234029056,"num_objects":3571,"num_object_clones":0,"num_object_copies":10713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6723,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9875,"num_write_kb":430272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":-2467}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"44.8","version":"1069'9906","reported_seq":9927,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:44.293423+0000","last_change":"2024-09-13T15:03:24.070447+0000","last_active":"2024-09-13T15:06:44.293423+0000","last_peered":"2024-09-13T15:06:44.293423+0000","last_clean":"2024-09-13T15:06:44.293423+0000","last_became_active":"2024-09-13T15:03:24.069765+0000","last_became_peered":"2024-09-13T15:03:24.069765+0000","last_unstale":"2024-09-13T15:06:44.293423+0000","last_undegraded":"2024-09-13T15:06:44.293423+0000","last_fullsized":"2024-09-13T15:06:44.293423+0000","mapping_epoch":1056,"log_start":"1060'800","ondisk_log_start":"1060'800","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":9106,"log_dups_size":0,"ondisk_log_size":9106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:55.089422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220463104,"num_objects":3364,"num_object_clones":0,"num_object_copies":10092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9906,"num_write_kb":424640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"44.7","version":"1069'10129","reported_seq":13396,"reported_epoch":1069,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T15:06:43.736733+0000","last_change":"2024-09-13T15:05:35.736476+0000","last_active":"2024-09-13T15:06:43.736733+0000","last_peered":"2024-09-13T15:06:43.736733+0000","last_clean":"2024-09-13T15:03:23.031187+0000","last_became_active":"2024-09-13T15:05:35.693852+0000","last_became_peered":"2024-09-13T15:05:35.693852+0000","last_unstale":"2024-09-13T15:06:43.736733+0000","last_undegraded":"2024-09-13T15:05:35.092316+0000","last_fullsized":"2024-09-13T15:05:35.058296+0000","mapping_epoch":1067,"log_start":"1060'100","ondisk_log_start":"1060'100","created":1056,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":10029,"log_dups_size":0,"ondisk_log_size":10029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218169344,"num_objects":3329,"num_object_clones":0,"num_object_copies":9987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4287,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10129,"num_write_kb":430656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1621,"num_bytes_recovered":106233856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":-958}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"44.6","version":"1069'10014","reported_seq":10035,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:44.066575+0000","last_change":"2024-09-13T15:03:24.070990+0000","last_active":"2024-09-13T15:06:44.066575+0000","last_peered":"2024-09-13T15:06:44.066575+0000","last_clean":"2024-09-13T15:06:44.066575+0000","last_became_active":"2024-09-13T15:03:24.070401+0000","last_became_peered":"2024-09-13T15:03:24.070401+0000","last_unstale":"2024-09-13T15:06:44.066575+0000","last_undegraded":"2024-09-13T15:06:44.066575+0000","last_fullsized":"2024-09-13T15:06:44.066575+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":10014,"log_dups_size":0,"ondisk_log_size":10014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:23:50.863517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222560256,"num_objects":3396,"num_object_clones":0,"num_object_copies":10188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10014,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"44.5","version":"1069'10104","reported_seq":10125,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:44.105902+0000","last_change":"2024-09-13T15:03:24.072449+0000","last_active":"2024-09-13T15:06:44.105902+0000","last_peered":"2024-09-13T15:06:44.105902+0000","last_clean":"2024-09-13T15:06:44.105902+0000","last_became_active":"2024-09-13T15:03:24.072160+0000","last_became_peered":"2024-09-13T15:03:24.072160+0000","last_unstale":"2024-09-13T15:06:44.105902+0000","last_undegraded":"2024-09-13T15:06:44.105902+0000","last_fullsized":"2024-09-13T15:06:44.105902+0000","mapping_epoch":1056,"log_start":"1060'100","ondisk_log_start":"1060'100","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":10004,"log_dups_size":0,"ondisk_log_size":10004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:38.539160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218365952,"num_objects":3332,"num_object_clones":0,"num_object_copies":9996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10104,"num_write_kb":429952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"44.4","version":"1069'10166","reported_seq":10204,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:42.647445+0000","last_change":"2024-09-13T15:03:24.068348+0000","last_active":"2024-09-13T15:06:42.647445+0000","last_peered":"2024-09-13T15:06:42.647445+0000","last_clean":"2024-09-13T15:06:42.647445+0000","last_became_active":"2024-09-13T15:03:24.067881+0000","last_became_peered":"2024-09-13T15:03:24.067881+0000","last_unstale":"2024-09-13T15:06:42.647445+0000","last_undegraded":"2024-09-13T15:06:42.647445+0000","last_fullsized":"2024-09-13T15:06:42.647445+0000","mapping_epoch":1056,"log_start":"1060'1000","ondisk_log_start":"1060'1000","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":9166,"log_dups_size":0,"ondisk_log_size":9166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:34:49.768741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223477760,"num_objects":3410,"num_object_clones":0,"num_object_copies":10230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10166,"num_write_kb":434432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"44.3","version":"1069'10094","reported_seq":10132,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:43.704715+0000","last_change":"2024-09-13T15:03:24.062767+0000","last_active":"2024-09-13T15:06:43.704715+0000","last_peered":"2024-09-13T15:06:43.704715+0000","last_clean":"2024-09-13T15:06:43.704715+0000","last_became_active":"2024-09-13T15:03:24.062251+0000","last_became_peered":"2024-09-13T15:03:24.062251+0000","last_unstale":"2024-09-13T15:06:43.704715+0000","last_undegraded":"2024-09-13T15:06:43.704715+0000","last_fullsized":"2024-09-13T15:06:43.704715+0000","mapping_epoch":1056,"log_start":"0'0","ondisk_log_start":"0'0","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":10094,"log_dups_size":0,"ondisk_log_size":10094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:02.301638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225837056,"num_objects":3446,"num_object_clones":0,"num_object_copies":10338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10094,"num_write_kb":433280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"44.2","version":"1069'9997","reported_seq":10034,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:45.014066+0000","last_change":"2024-09-13T15:03:24.067147+0000","last_active":"2024-09-13T15:06:45.014066+0000","last_peered":"2024-09-13T15:06:45.014066+0000","last_clean":"2024-09-13T15:06:45.014066+0000","last_became_active":"2024-09-13T15:03:24.066369+0000","last_became_peered":"2024-09-13T15:03:24.066369+0000","last_unstale":"2024-09-13T15:06:45.014066+0000","last_undegraded":"2024-09-13T15:06:45.014066+0000","last_fullsized":"2024-09-13T15:06:45.014066+0000","mapping_epoch":1056,"log_start":"1060'2300","ondisk_log_start":"1060'2300","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":7697,"log_dups_size":0,"ondisk_log_size":7697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:22:12.065416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215154688,"num_objects":3283,"num_object_clones":0,"num_object_copies":9849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9997,"num_write_kb":424960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"44.1","version":"1069'9882","reported_seq":9920,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:42.339141+0000","last_change":"2024-09-13T15:03:24.065492+0000","last_active":"2024-09-13T15:06:42.339141+0000","last_peered":"2024-09-13T15:06:42.339141+0000","last_clean":"2024-09-13T15:06:42.339141+0000","last_became_active":"2024-09-13T15:03:24.065095+0000","last_became_peered":"2024-09-13T15:03:24.065095+0000","last_unstale":"2024-09-13T15:06:42.339141+0000","last_undegraded":"2024-09-13T15:06:42.339141+0000","last_fullsized":"2024-09-13T15:06:42.339141+0000","mapping_epoch":1056,"log_start":"1060'2300","ondisk_log_start":"1060'2300","created":1056,"last_epoch_clean":1057,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":7582,"log_dups_size":0,"ondisk_log_size":7582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:54:06.954165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225312768,"num_objects":3438,"num_object_clones":0,"num_object_copies":10314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9882,"num_write_kb":426240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"44.0","version":"1069'9813","reported_seq":9854,"reported_epoch":1069,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:06:42.313623+0000","last_change":"2024-09-13T15:05:35.701018+0000","last_active":"2024-09-13T15:06:42.313623+0000","last_peered":"2024-09-13T15:06:42.313623+0000","last_clean":"2024-09-13T15:03:23.031187+0000","last_became_active":"2024-09-13T15:05:35.676814+0000","last_became_peered":"2024-09-13T15:05:35.676814+0000","last_unstale":"2024-09-13T15:06:42.313623+0000","last_undegraded":"2024-09-13T15:05:35.181795+0000","last_fullsized":"2024-09-13T15:05:35.059296+0000","mapping_epoch":1067,"log_start":"1060'2300","ondisk_log_start":"1060'2300","created":1056,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:03:23.031187+0000","last_clean_scrub_stamp":"2024-09-13T15:03:23.031187+0000","objects_scrubbed":0,"log_size":7513,"log_dups_size":0,"ondisk_log_size":7513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220528640,"num_objects":3365,"num_object_clones":0,"num_object_copies":10095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6588,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9813,"num_write_kb":421696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":-2478}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":1534,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.237195+0000","last_change":"2024-09-13T14:38:59.343552+0000","last_active":"2024-09-13T15:06:06.237195+0000","last_peered":"2024-09-13T15:06:06.237195+0000","last_clean":"2024-09-13T15:06:06.237195+0000","last_became_active":"2024-09-13T14:38:59.343094+0000","last_became_peered":"2024-09-13T14:38:59.343094+0000","last_unstale":"2024-09-13T15:06:06.237195+0000","last_undegraded":"2024-09-13T15:06:06.237195+0000","last_fullsized":"2024-09-13T15:06:06.237195+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:56.209186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":253,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.223436+0000","last_change":"2024-09-13T14:52:43.599071+0000","last_active":"2024-09-13T15:06:06.223436+0000","last_peered":"2024-09-13T15:06:06.223436+0000","last_clean":"2024-09-13T15:06:06.223436+0000","last_became_active":"2024-09-13T14:52:43.594847+0000","last_became_peered":"2024-09-13T14:52:43.594847+0000","last_unstale":"2024-09-13T15:06:06.223436+0000","last_undegraded":"2024-09-13T15:06:06.223436+0000","last_fullsized":"2024-09-13T15:06:06.223436+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:02.736434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1247,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.208643+0000","last_change":"2024-09-13T15:05:34.085980+0000","last_active":"2024-09-13T15:06:06.208643+0000","last_peered":"2024-09-13T15:06:06.208643+0000","last_clean":"2024-09-13T15:06:06.208643+0000","last_became_active":"2024-09-13T15:05:34.085523+0000","last_became_peered":"2024-09-13T15:05:34.085523+0000","last_unstale":"2024-09-13T15:06:06.208643+0000","last_undegraded":"2024-09-13T15:06:06.208643+0000","last_fullsized":"2024-09-13T15:06:06.208643+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:20:08.715966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":100,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.378010+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T15:05:35.378010+0000","last_peered":"2024-09-13T15:05:35.378010+0000","last_clean":"2024-09-13T15:05:35.378010+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T15:05:35.378010+0000","last_undegraded":"2024-09-13T15:05:35.378010+0000","last_fullsized":"2024-09-13T15:05:35.378010+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1054,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.378463+0000","last_change":"2024-09-13T14:47:46.337827+0000","last_active":"2024-09-13T15:05:35.378463+0000","last_peered":"2024-09-13T15:05:35.378463+0000","last_clean":"2024-09-13T15:05:35.378463+0000","last_became_active":"2024-09-13T14:47:46.321140+0000","last_became_peered":"2024-09-13T14:47:46.321140+0000","last_unstale":"2024-09-13T15:05:35.378463+0000","last_undegraded":"2024-09-13T15:05:35.378463+0000","last_fullsized":"2024-09-13T15:05:35.378463+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:15.997382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":250,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.223665+0000","last_change":"2024-09-13T15:05:34.086551+0000","last_active":"2024-09-13T15:06:06.223665+0000","last_peered":"2024-09-13T15:06:06.223665+0000","last_clean":"2024-09-13T15:06:06.223665+0000","last_became_active":"2024-09-13T15:05:34.086077+0000","last_became_peered":"2024-09-13T15:05:34.086077+0000","last_unstale":"2024-09-13T15:06:06.223665+0000","last_undegraded":"2024-09-13T15:06:06.223665+0000","last_fullsized":"2024-09-13T15:06:06.223665+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:33:47.917777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1099,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.378655+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T15:05:35.378655+0000","last_peered":"2024-09-13T15:05:35.378655+0000","last_clean":"2024-09-13T15:05:35.378655+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T15:05:35.378655+0000","last_undegraded":"2024-09-13T15:05:35.378655+0000","last_fullsized":"2024-09-13T15:05:35.378655+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":264,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.223851+0000","last_change":"2024-09-13T15:05:34.075607+0000","last_active":"2024-09-13T15:06:06.223851+0000","last_peered":"2024-09-13T15:06:06.223851+0000","last_clean":"2024-09-13T15:06:06.223851+0000","last_became_active":"2024-09-13T15:05:34.075079+0000","last_became_peered":"2024-09-13T15:05:34.075079+0000","last_unstale":"2024-09-13T15:06:06.223851+0000","last_undegraded":"2024-09-13T15:06:06.223851+0000","last_fullsized":"2024-09-13T15:06:06.223851+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:05.384206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1541,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.236846+0000","last_change":"2024-09-13T14:47:46.330059+0000","last_active":"2024-09-13T15:06:06.236846+0000","last_peered":"2024-09-13T15:06:06.236846+0000","last_clean":"2024-09-13T15:06:06.236846+0000","last_became_active":"2024-09-13T14:47:46.329344+0000","last_became_peered":"2024-09-13T14:47:46.329344+0000","last_unstale":"2024-09-13T15:06:06.236846+0000","last_undegraded":"2024-09-13T15:06:06.236846+0000","last_fullsized":"2024-09-13T15:06:06.236846+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:58:43.470176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1456,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.216746+0000","last_change":"2024-09-13T15:05:34.082773+0000","last_active":"2024-09-13T15:06:06.216746+0000","last_peered":"2024-09-13T15:06:06.216746+0000","last_clean":"2024-09-13T15:06:06.216746+0000","last_became_active":"2024-09-13T15:05:34.082408+0000","last_became_peered":"2024-09-13T15:05:34.082408+0000","last_unstale":"2024-09-13T15:06:06.216746+0000","last_undegraded":"2024-09-13T15:06:06.216746+0000","last_fullsized":"2024-09-13T15:06:06.216746+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:13.447546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":253,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.223658+0000","last_change":"2024-09-13T14:52:43.597400+0000","last_active":"2024-09-13T15:06:06.223658+0000","last_peered":"2024-09-13T15:06:06.223658+0000","last_clean":"2024-09-13T15:06:06.223658+0000","last_became_active":"2024-09-13T14:52:43.596294+0000","last_became_peered":"2024-09-13T14:52:43.596294+0000","last_unstale":"2024-09-13T15:06:06.223658+0000","last_undegraded":"2024-09-13T15:06:06.223658+0000","last_fullsized":"2024-09-13T15:06:06.223658+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:35.218171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1317,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213852+0000","last_change":"2024-09-13T15:05:34.070717+0000","last_active":"2024-09-13T15:06:06.213852+0000","last_peered":"2024-09-13T15:06:06.213852+0000","last_clean":"2024-09-13T15:06:06.213852+0000","last_became_active":"2024-09-13T15:05:34.069929+0000","last_became_peered":"2024-09-13T15:05:34.069929+0000","last_unstale":"2024-09-13T15:06:06.213852+0000","last_undegraded":"2024-09-13T15:06:06.213852+0000","last_fullsized":"2024-09-13T15:06:06.213852+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:44:19.208697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":154,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213226+0000","last_change":"2024-09-13T15:05:34.071133+0000","last_active":"2024-09-13T15:06:06.213226+0000","last_peered":"2024-09-13T15:06:06.213226+0000","last_clean":"2024-09-13T15:06:06.213226+0000","last_became_active":"2024-09-13T15:05:34.070519+0000","last_became_peered":"2024-09-13T15:05:34.070519+0000","last_unstale":"2024-09-13T15:06:06.213226+0000","last_undegraded":"2024-09-13T15:06:06.213226+0000","last_fullsized":"2024-09-13T15:06:06.213226+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:02.617403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1531,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.237210+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T15:06:06.237210+0000","last_peered":"2024-09-13T15:06:06.237210+0000","last_clean":"2024-09-13T15:06:06.237210+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T15:06:06.237210+0000","last_undegraded":"2024-09-13T15:06:06.237210+0000","last_fullsized":"2024-09-13T15:06:06.237210+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1310,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.208812+0000","last_change":"2024-09-13T15:05:34.086212+0000","last_active":"2024-09-13T15:06:06.208812+0000","last_peered":"2024-09-13T15:06:06.208812+0000","last_clean":"2024-09-13T15:06:06.208812+0000","last_became_active":"2024-09-13T15:05:34.085726+0000","last_became_peered":"2024-09-13T15:05:34.085726+0000","last_unstale":"2024-09-13T15:06:06.208812+0000","last_undegraded":"2024-09-13T15:06:06.208812+0000","last_fullsized":"2024-09-13T15:06:06.208812+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:25.717683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1056,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.061056+0000","last_change":"2024-09-13T14:52:54.183175+0000","last_active":"2024-09-13T15:05:35.061056+0000","last_peered":"2024-09-13T15:05:35.061056+0000","last_clean":"2024-09-13T15:05:35.061056+0000","last_became_active":"2024-09-13T14:52:54.182514+0000","last_became_peered":"2024-09-13T14:52:54.182514+0000","last_unstale":"2024-09-13T15:05:35.061056+0000","last_undegraded":"2024-09-13T15:05:35.061056+0000","last_fullsized":"2024-09-13T15:05:35.061056+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:55.900845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1290,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.208628+0000","last_change":"2024-09-13T15:05:34.072431+0000","last_active":"2024-09-13T15:06:06.208628+0000","last_peered":"2024-09-13T15:06:06.208628+0000","last_clean":"2024-09-13T15:06:06.208628+0000","last_became_active":"2024-09-13T15:05:34.071910+0000","last_became_peered":"2024-09-13T15:05:34.071910+0000","last_unstale":"2024-09-13T15:06:06.208628+0000","last_undegraded":"2024-09-13T15:06:06.208628+0000","last_fullsized":"2024-09-13T15:06:06.208628+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:53.510703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1532,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.237295+0000","last_change":"2024-09-13T15:05:34.087376+0000","last_active":"2024-09-13T15:06:06.237295+0000","last_peered":"2024-09-13T15:06:06.237295+0000","last_clean":"2024-09-13T15:06:06.237295+0000","last_became_active":"2024-09-13T15:05:34.087046+0000","last_became_peered":"2024-09-13T15:05:34.087046+0000","last_unstale":"2024-09-13T15:06:06.237295+0000","last_undegraded":"2024-09-13T15:06:06.237295+0000","last_fullsized":"2024-09-13T15:06:06.237295+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:47.801431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1232,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.208818+0000","last_change":"2024-09-13T15:05:34.070426+0000","last_active":"2024-09-13T15:06:06.208818+0000","last_peered":"2024-09-13T15:06:06.208818+0000","last_clean":"2024-09-13T15:06:06.208818+0000","last_became_active":"2024-09-13T15:05:34.070023+0000","last_became_peered":"2024-09-13T15:05:34.070023+0000","last_unstale":"2024-09-13T15:06:06.208818+0000","last_undegraded":"2024-09-13T15:06:06.208818+0000","last_fullsized":"2024-09-13T15:06:06.208818+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:30.275530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1043,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.378632+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T15:05:35.378632+0000","last_peered":"2024-09-13T15:05:35.378632+0000","last_clean":"2024-09-13T15:05:35.378632+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T15:05:35.378632+0000","last_undegraded":"2024-09-13T15:05:35.378632+0000","last_fullsized":"2024-09-13T15:05:35.378632+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1097,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.097852+0000","last_change":"2024-09-13T15:05:34.089020+0000","last_active":"2024-09-13T15:05:35.097852+0000","last_peered":"2024-09-13T15:05:35.097852+0000","last_clean":"2024-09-13T15:05:35.097852+0000","last_became_active":"2024-09-13T15:05:34.088440+0000","last_became_peered":"2024-09-13T15:05:34.088440+0000","last_unstale":"2024-09-13T15:05:35.097852+0000","last_undegraded":"2024-09-13T15:05:35.097852+0000","last_fullsized":"2024-09-13T15:05:35.097852+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:24.826746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":144,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213575+0000","last_change":"2024-09-13T14:52:43.586530+0000","last_active":"2024-09-13T15:06:06.213575+0000","last_peered":"2024-09-13T15:06:06.213575+0000","last_clean":"2024-09-13T15:06:06.213575+0000","last_became_active":"2024-09-13T14:52:43.586107+0000","last_became_peered":"2024-09-13T14:52:43.586107+0000","last_unstale":"2024-09-13T15:06:06.213575+0000","last_undegraded":"2024-09-13T15:06:06.213575+0000","last_fullsized":"2024-09-13T15:06:06.213575+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:18.130768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":250,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.225233+0000","last_change":"2024-09-13T14:47:46.344536+0000","last_active":"2024-09-13T15:06:06.225233+0000","last_peered":"2024-09-13T15:06:06.225233+0000","last_clean":"2024-09-13T15:06:06.225233+0000","last_became_active":"2024-09-13T14:47:46.321131+0000","last_became_peered":"2024-09-13T14:47:46.321131+0000","last_unstale":"2024-09-13T15:06:06.225233+0000","last_undegraded":"2024-09-13T15:06:06.225233+0000","last_fullsized":"2024-09-13T15:06:06.225233+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:18:31.114472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":254,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.223987+0000","last_change":"2024-09-13T14:52:43.600569+0000","last_active":"2024-09-13T15:06:06.223987+0000","last_peered":"2024-09-13T15:06:06.223987+0000","last_clean":"2024-09-13T15:06:06.223987+0000","last_became_active":"2024-09-13T14:52:43.599764+0000","last_became_peered":"2024-09-13T14:52:43.599764+0000","last_unstale":"2024-09-13T15:06:06.223987+0000","last_undegraded":"2024-09-13T15:06:06.223987+0000","last_fullsized":"2024-09-13T15:06:06.223987+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:53.934456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":241,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.223849+0000","last_change":"2024-09-13T14:47:46.318785+0000","last_active":"2024-09-13T15:06:06.223849+0000","last_peered":"2024-09-13T15:06:06.223849+0000","last_clean":"2024-09-13T15:06:06.223849+0000","last_became_active":"2024-09-13T14:47:46.318438+0000","last_became_peered":"2024-09-13T14:47:46.318438+0000","last_unstale":"2024-09-13T15:06:06.223849+0000","last_undegraded":"2024-09-13T15:06:06.223849+0000","last_fullsized":"2024-09-13T15:06:06.223849+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:16:44.745378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1039,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.061142+0000","last_change":"2024-09-13T14:38:59.379362+0000","last_active":"2024-09-13T15:05:35.061142+0000","last_peered":"2024-09-13T15:05:35.061142+0000","last_clean":"2024-09-13T15:05:35.061142+0000","last_became_active":"2024-09-13T14:38:59.378725+0000","last_became_peered":"2024-09-13T14:38:59.378725+0000","last_unstale":"2024-09-13T15:05:35.061142+0000","last_undegraded":"2024-09-13T15:05:35.061142+0000","last_fullsized":"2024-09-13T15:05:35.061142+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:57.410359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1583,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.237313+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T15:06:06.237313+0000","last_peered":"2024-09-13T15:06:06.237313+0000","last_clean":"2024-09-13T15:06:06.237313+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T15:06:06.237313+0000","last_undegraded":"2024-09-13T15:06:06.237313+0000","last_fullsized":"2024-09-13T15:06:06.237313+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1230,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.208721+0000","last_change":"2024-09-13T15:05:34.097197+0000","last_active":"2024-09-13T15:06:06.208721+0000","last_peered":"2024-09-13T15:06:06.208721+0000","last_clean":"2024-09-13T15:06:06.208721+0000","last_became_active":"2024-09-13T15:05:34.096736+0000","last_became_peered":"2024-09-13T15:05:34.096736+0000","last_unstale":"2024-09-13T15:06:06.208721+0000","last_undegraded":"2024-09-13T15:06:06.208721+0000","last_fullsized":"2024-09-13T15:06:06.208721+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:20.119363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":144,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.214316+0000","last_change":"2024-09-13T14:52:43.592967+0000","last_active":"2024-09-13T15:06:06.214316+0000","last_peered":"2024-09-13T15:06:06.214316+0000","last_clean":"2024-09-13T15:06:06.214316+0000","last_became_active":"2024-09-13T14:52:43.592431+0000","last_became_peered":"2024-09-13T14:52:43.592431+0000","last_unstale":"2024-09-13T15:06:06.214316+0000","last_undegraded":"2024-09-13T15:06:06.214316+0000","last_fullsized":"2024-09-13T15:06:06.214316+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:56.187568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1233,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.209306+0000","last_change":"2024-09-13T15:05:34.085917+0000","last_active":"2024-09-13T15:06:06.209306+0000","last_peered":"2024-09-13T15:06:06.209306+0000","last_clean":"2024-09-13T15:06:06.209306+0000","last_became_active":"2024-09-13T15:05:34.085156+0000","last_became_peered":"2024-09-13T15:05:34.085156+0000","last_unstale":"2024-09-13T15:06:06.209306+0000","last_undegraded":"2024-09-13T15:06:06.209306+0000","last_fullsized":"2024-09-13T15:06:06.209306+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:33.482732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1512,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.236596+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T15:06:06.236596+0000","last_peered":"2024-09-13T15:06:06.236596+0000","last_clean":"2024-09-13T15:06:06.236596+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T15:06:06.236596+0000","last_undegraded":"2024-09-13T15:06:06.236596+0000","last_fullsized":"2024-09-13T15:06:06.236596+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1075,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.097999+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T15:05:35.097999+0000","last_peered":"2024-09-13T15:05:35.097999+0000","last_clean":"2024-09-13T15:05:35.097999+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T15:05:35.097999+0000","last_undegraded":"2024-09-13T15:05:35.097999+0000","last_fullsized":"2024-09-13T15:05:35.097999+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":144,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213441+0000","last_change":"2024-09-13T14:52:43.583860+0000","last_active":"2024-09-13T15:06:06.213441+0000","last_peered":"2024-09-13T15:06:06.213441+0000","last_clean":"2024-09-13T15:06:06.213441+0000","last_became_active":"2024-09-13T14:52:43.583055+0000","last_became_peered":"2024-09-13T14:52:43.583055+0000","last_unstale":"2024-09-13T15:06:06.213441+0000","last_undegraded":"2024-09-13T15:06:06.213441+0000","last_fullsized":"2024-09-13T15:06:06.213441+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:55.113648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":144,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213449+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T15:06:06.213449+0000","last_peered":"2024-09-13T15:06:06.213449+0000","last_clean":"2024-09-13T15:06:06.213449+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T15:06:06.213449+0000","last_undegraded":"2024-09-13T15:06:06.213449+0000","last_fullsized":"2024-09-13T15:06:06.213449+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":144,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213555+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T15:06:06.213555+0000","last_peered":"2024-09-13T15:06:06.213555+0000","last_clean":"2024-09-13T15:06:06.213555+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T15:06:06.213555+0000","last_undegraded":"2024-09-13T15:06:06.213555+0000","last_fullsized":"2024-09-13T15:06:06.213555+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1057,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.378635+0000","last_change":"2024-09-13T15:05:34.086758+0000","last_active":"2024-09-13T15:05:35.378635+0000","last_peered":"2024-09-13T15:05:35.378635+0000","last_clean":"2024-09-13T15:05:35.378635+0000","last_became_active":"2024-09-13T15:05:34.086145+0000","last_became_peered":"2024-09-13T15:05:34.086145+0000","last_unstale":"2024-09-13T15:05:35.378635+0000","last_undegraded":"2024-09-13T15:05:35.378635+0000","last_fullsized":"2024-09-13T15:05:35.378635+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:33.764698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":155,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213530+0000","last_change":"2024-09-13T15:05:34.085898+0000","last_active":"2024-09-13T15:06:06.213530+0000","last_peered":"2024-09-13T15:06:06.213530+0000","last_clean":"2024-09-13T15:06:06.213530+0000","last_became_active":"2024-09-13T15:05:34.085174+0000","last_became_peered":"2024-09-13T15:05:34.085174+0000","last_unstale":"2024-09-13T15:06:06.213530+0000","last_undegraded":"2024-09-13T15:06:06.213530+0000","last_fullsized":"2024-09-13T15:06:06.213530+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:17:39.204192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":241,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.223999+0000","last_change":"2024-09-13T14:47:46.324072+0000","last_active":"2024-09-13T15:06:06.223999+0000","last_peered":"2024-09-13T15:06:06.223999+0000","last_clean":"2024-09-13T15:06:06.223999+0000","last_became_active":"2024-09-13T14:47:46.322613+0000","last_became_peered":"2024-09-13T14:47:46.322613+0000","last_unstale":"2024-09-13T15:06:06.223999+0000","last_undegraded":"2024-09-13T15:06:06.223999+0000","last_fullsized":"2024-09-13T15:06:06.223999+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:45.314706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1065,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.377957+0000","last_change":"2024-09-13T14:52:43.613106+0000","last_active":"2024-09-13T15:05:35.377957+0000","last_peered":"2024-09-13T15:05:35.377957+0000","last_clean":"2024-09-13T15:05:35.377957+0000","last_became_active":"2024-09-13T14:52:43.594081+0000","last_became_peered":"2024-09-13T14:52:43.594081+0000","last_unstale":"2024-09-13T15:05:35.377957+0000","last_undegraded":"2024-09-13T15:05:35.377957+0000","last_fullsized":"2024-09-13T15:05:35.377957+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:11.121629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":144,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213679+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T15:06:06.213679+0000","last_peered":"2024-09-13T15:06:06.213679+0000","last_clean":"2024-09-13T15:06:06.213679+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T15:06:06.213679+0000","last_undegraded":"2024-09-13T15:06:06.213679+0000","last_fullsized":"2024-09-13T15:06:06.213679+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":144,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213119+0000","last_change":"2024-09-13T14:52:43.582732+0000","last_active":"2024-09-13T15:06:06.213119+0000","last_peered":"2024-09-13T15:06:06.213119+0000","last_clean":"2024-09-13T15:06:06.213119+0000","last_became_active":"2024-09-13T14:52:43.582384+0000","last_became_peered":"2024-09-13T14:52:43.582384+0000","last_unstale":"2024-09-13T15:06:06.213119+0000","last_undegraded":"2024-09-13T15:06:06.213119+0000","last_fullsized":"2024-09-13T15:06:06.213119+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:06.715139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1432,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.208267+0000","last_change":"2024-09-13T15:05:34.082028+0000","last_active":"2024-09-13T15:06:06.208267+0000","last_peered":"2024-09-13T15:06:06.208267+0000","last_clean":"2024-09-13T15:06:06.208267+0000","last_became_active":"2024-09-13T15:05:34.081220+0000","last_became_peered":"2024-09-13T15:05:34.081220+0000","last_unstale":"2024-09-13T15:06:06.208267+0000","last_undegraded":"2024-09-13T15:06:06.208267+0000","last_fullsized":"2024-09-13T15:06:06.208267+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:12:26.484994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1548,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.237126+0000","last_change":"2024-09-13T14:52:43.595169+0000","last_active":"2024-09-13T15:06:06.237126+0000","last_peered":"2024-09-13T15:06:06.237126+0000","last_clean":"2024-09-13T15:06:06.237126+0000","last_became_active":"2024-09-13T14:52:43.594586+0000","last_became_peered":"2024-09-13T14:52:43.594586+0000","last_unstale":"2024-09-13T15:06:06.237126+0000","last_undegraded":"2024-09-13T15:06:06.237126+0000","last_fullsized":"2024-09-13T15:06:06.237126+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:53.946025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1559,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.235915+0000","last_change":"2024-09-13T15:05:34.087817+0000","last_active":"2024-09-13T15:06:06.235915+0000","last_peered":"2024-09-13T15:06:06.235915+0000","last_clean":"2024-09-13T15:06:06.235915+0000","last_became_active":"2024-09-13T15:05:34.087450+0000","last_became_peered":"2024-09-13T15:05:34.087450+0000","last_unstale":"2024-09-13T15:06:06.235915+0000","last_undegraded":"2024-09-13T15:06:06.235915+0000","last_fullsized":"2024-09-13T15:06:06.235915+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:47.661935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1075,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.097184+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T15:05:35.097184+0000","last_peered":"2024-09-13T15:05:35.097184+0000","last_clean":"2024-09-13T15:05:35.097184+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T15:05:35.097184+0000","last_undegraded":"2024-09-13T15:05:35.097184+0000","last_fullsized":"2024-09-13T15:05:35.097184+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1046,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.061004+0000","last_change":"2024-09-13T14:47:46.320830+0000","last_active":"2024-09-13T15:05:35.061004+0000","last_peered":"2024-09-13T15:05:35.061004+0000","last_clean":"2024-09-13T15:05:35.061004+0000","last_became_active":"2024-09-13T14:47:46.320443+0000","last_became_peered":"2024-09-13T14:47:46.320443+0000","last_unstale":"2024-09-13T15:05:35.061004+0000","last_undegraded":"2024-09-13T15:05:35.061004+0000","last_fullsized":"2024-09-13T15:05:35.061004+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:46.279580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1232,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.214419+0000","last_change":"2024-09-13T15:05:34.070584+0000","last_active":"2024-09-13T15:06:06.214419+0000","last_peered":"2024-09-13T15:06:06.214419+0000","last_clean":"2024-09-13T15:06:06.214419+0000","last_became_active":"2024-09-13T15:05:34.070174+0000","last_became_peered":"2024-09-13T15:05:34.070174+0000","last_unstale":"2024-09-13T15:06:06.214419+0000","last_undegraded":"2024-09-13T15:06:06.214419+0000","last_fullsized":"2024-09-13T15:06:06.214419+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:46.411509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1036,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.378097+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T15:05:35.378097+0000","last_peered":"2024-09-13T15:05:35.378097+0000","last_clean":"2024-09-13T15:05:35.378097+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T15:05:35.378097+0000","last_undegraded":"2024-09-13T15:05:35.378097+0000","last_fullsized":"2024-09-13T15:05:35.378097+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1143,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.097270+0000","last_change":"2024-09-13T14:36:09.375395+0000","last_active":"2024-09-13T15:05:35.097270+0000","last_peered":"2024-09-13T15:05:35.097270+0000","last_clean":"2024-09-13T15:05:35.097270+0000","last_became_active":"2024-09-13T14:36:09.374796+0000","last_became_peered":"2024-09-13T14:36:09.374796+0000","last_unstale":"2024-09-13T15:05:35.097270+0000","last_undegraded":"2024-09-13T15:05:35.097270+0000","last_fullsized":"2024-09-13T15:05:35.097270+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:29.760438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":155,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213794+0000","last_change":"2024-09-13T15:05:34.078805+0000","last_active":"2024-09-13T15:06:06.213794+0000","last_peered":"2024-09-13T15:06:06.213794+0000","last_clean":"2024-09-13T15:06:06.213794+0000","last_became_active":"2024-09-13T15:05:34.078180+0000","last_became_peered":"2024-09-13T15:05:34.078180+0000","last_unstale":"2024-09-13T15:06:06.213794+0000","last_undegraded":"2024-09-13T15:06:06.213794+0000","last_fullsized":"2024-09-13T15:06:06.213794+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:47.240906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1294,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.208902+0000","last_change":"2024-09-13T15:05:34.076149+0000","last_active":"2024-09-13T15:06:06.208902+0000","last_peered":"2024-09-13T15:06:06.208902+0000","last_clean":"2024-09-13T15:06:06.208902+0000","last_became_active":"2024-09-13T15:05:34.075623+0000","last_became_peered":"2024-09-13T15:05:34.075623+0000","last_unstale":"2024-09-13T15:06:06.208902+0000","last_undegraded":"2024-09-13T15:06:06.208902+0000","last_fullsized":"2024-09-13T15:06:06.208902+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:23.608262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":254,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.229664+0000","last_change":"2024-09-13T14:52:43.599179+0000","last_active":"2024-09-13T15:06:06.229664+0000","last_peered":"2024-09-13T15:06:06.229664+0000","last_clean":"2024-09-13T15:06:06.229664+0000","last_became_active":"2024-09-13T14:52:43.598166+0000","last_became_peered":"2024-09-13T14:52:43.598166+0000","last_unstale":"2024-09-13T15:06:06.229664+0000","last_undegraded":"2024-09-13T15:06:06.229664+0000","last_fullsized":"2024-09-13T15:06:06.229664+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:08.647786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":240,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.223718+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T15:06:06.223718+0000","last_peered":"2024-09-13T15:06:06.223718+0000","last_clean":"2024-09-13T15:06:06.223718+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T15:06:06.223718+0000","last_undegraded":"2024-09-13T15:06:06.223718+0000","last_fullsized":"2024-09-13T15:06:06.223718+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":254,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.229357+0000","last_change":"2024-09-13T14:52:43.597595+0000","last_active":"2024-09-13T15:06:06.229357+0000","last_peered":"2024-09-13T15:06:06.229357+0000","last_clean":"2024-09-13T15:06:06.229357+0000","last_became_active":"2024-09-13T14:52:43.596592+0000","last_became_peered":"2024-09-13T14:52:43.596592+0000","last_unstale":"2024-09-13T15:06:06.229357+0000","last_undegraded":"2024-09-13T15:06:06.229357+0000","last_fullsized":"2024-09-13T15:06:06.229357+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:56.532603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1084,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.061675+0000","last_change":"2024-09-13T14:52:43.596429+0000","last_active":"2024-09-13T15:05:35.061675+0000","last_peered":"2024-09-13T15:05:35.061675+0000","last_clean":"2024-09-13T15:05:35.061675+0000","last_became_active":"2024-09-13T14:52:43.594614+0000","last_became_peered":"2024-09-13T14:52:43.594614+0000","last_unstale":"2024-09-13T15:05:35.061675+0000","last_undegraded":"2024-09-13T15:05:35.061675+0000","last_fullsized":"2024-09-13T15:05:35.061675+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:09:41.970225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":251,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.225160+0000","last_change":"2024-09-13T15:05:34.075658+0000","last_active":"2024-09-13T15:06:06.225160+0000","last_peered":"2024-09-13T15:06:06.225160+0000","last_clean":"2024-09-13T15:06:06.225160+0000","last_became_active":"2024-09-13T15:05:34.075141+0000","last_became_peered":"2024-09-13T15:05:34.075141+0000","last_unstale":"2024-09-13T15:06:06.225160+0000","last_undegraded":"2024-09-13T15:06:06.225160+0000","last_fullsized":"2024-09-13T15:06:06.225160+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:57.049760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1279,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.208486+0000","last_change":"2024-09-13T15:05:34.085458+0000","last_active":"2024-09-13T15:06:06.208486+0000","last_peered":"2024-09-13T15:06:06.208486+0000","last_clean":"2024-09-13T15:06:06.208486+0000","last_became_active":"2024-09-13T15:05:34.084748+0000","last_became_peered":"2024-09-13T15:05:34.084748+0000","last_unstale":"2024-09-13T15:06:06.208486+0000","last_undegraded":"2024-09-13T15:06:06.208486+0000","last_fullsized":"2024-09-13T15:06:06.208486+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:21.898959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1568,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.236697+0000","last_change":"2024-09-13T14:52:54.185117+0000","last_active":"2024-09-13T15:06:06.236697+0000","last_peered":"2024-09-13T15:06:06.236697+0000","last_clean":"2024-09-13T15:06:06.236697+0000","last_became_active":"2024-09-13T14:52:54.184712+0000","last_became_peered":"2024-09-13T14:52:54.184712+0000","last_unstale":"2024-09-13T15:06:06.236697+0000","last_undegraded":"2024-09-13T15:06:06.236697+0000","last_fullsized":"2024-09-13T15:06:06.236697+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:32.654401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1040,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.378542+0000","last_change":"2024-09-13T15:05:34.073737+0000","last_active":"2024-09-13T15:05:35.378542+0000","last_peered":"2024-09-13T15:05:35.378542+0000","last_clean":"2024-09-13T15:05:35.378542+0000","last_became_active":"2024-09-13T15:05:34.073330+0000","last_became_peered":"2024-09-13T15:05:34.073330+0000","last_unstale":"2024-09-13T15:05:35.378542+0000","last_undegraded":"2024-09-13T15:05:35.378542+0000","last_fullsized":"2024-09-13T15:05:35.378542+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:51:30.883378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1046,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.061525+0000","last_change":"2024-09-13T14:52:43.581604+0000","last_active":"2024-09-13T15:05:35.061525+0000","last_peered":"2024-09-13T15:05:35.061525+0000","last_clean":"2024-09-13T15:05:35.061525+0000","last_became_active":"2024-09-13T14:52:43.579981+0000","last_became_peered":"2024-09-13T14:52:43.579981+0000","last_unstale":"2024-09-13T15:05:35.061525+0000","last_undegraded":"2024-09-13T15:05:35.061525+0000","last_fullsized":"2024-09-13T15:05:35.061525+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:57:59.366292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":1011,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.097961+0000","last_change":"2024-09-13T14:24:17.229583+0000","last_active":"2024-09-13T15:05:35.097961+0000","last_peered":"2024-09-13T15:05:35.097961+0000","last_clean":"2024-09-13T15:05:35.097961+0000","last_became_active":"2024-09-13T14:24:17.229077+0000","last_became_peered":"2024-09-13T14:24:17.229077+0000","last_unstale":"2024-09-13T15:05:35.097961+0000","last_undegraded":"2024-09-13T15:05:35.097961+0000","last_fullsized":"2024-09-13T15:05:35.097961+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:39.850901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":265,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.223892+0000","last_change":"2024-09-13T15:05:34.072278+0000","last_active":"2024-09-13T15:06:06.223892+0000","last_peered":"2024-09-13T15:06:06.223892+0000","last_clean":"2024-09-13T15:06:06.223892+0000","last_became_active":"2024-09-13T15:05:34.071723+0000","last_became_peered":"2024-09-13T15:05:34.071723+0000","last_unstale":"2024-09-13T15:06:06.223892+0000","last_undegraded":"2024-09-13T15:06:06.223892+0000","last_fullsized":"2024-09-13T15:06:06.223892+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:02.130890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1102,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.097749+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T15:05:35.097749+0000","last_peered":"2024-09-13T15:05:35.097749+0000","last_clean":"2024-09-13T15:05:35.097749+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T15:05:35.097749+0000","last_undegraded":"2024-09-13T15:05:35.097749+0000","last_fullsized":"2024-09-13T15:05:35.097749+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1005,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.378564+0000","last_change":"2024-09-13T14:24:17.413076+0000","last_active":"2024-09-13T15:05:35.378564+0000","last_peered":"2024-09-13T15:05:35.378564+0000","last_clean":"2024-09-13T15:05:35.378564+0000","last_became_active":"2024-09-13T14:24:17.412099+0000","last_became_peered":"2024-09-13T14:24:17.412099+0000","last_unstale":"2024-09-13T15:05:35.378564+0000","last_undegraded":"2024-09-13T15:05:35.378564+0000","last_fullsized":"2024-09-13T15:05:35.378564+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:36.378818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1073,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.098330+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T15:05:35.098330+0000","last_peered":"2024-09-13T15:05:35.098330+0000","last_clean":"2024-09-13T15:05:35.098330+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T15:05:35.098330+0000","last_undegraded":"2024-09-13T15:05:35.098330+0000","last_fullsized":"2024-09-13T15:05:35.098330+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":1039,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.061316+0000","last_change":"2024-09-13T14:36:09.363949+0000","last_active":"2024-09-13T15:05:35.061316+0000","last_peered":"2024-09-13T15:05:35.061316+0000","last_clean":"2024-09-13T15:05:35.061316+0000","last_became_active":"2024-09-13T14:36:09.362429+0000","last_became_peered":"2024-09-13T14:36:09.362429+0000","last_unstale":"2024-09-13T15:05:35.061316+0000","last_undegraded":"2024-09-13T15:05:35.061316+0000","last_fullsized":"2024-09-13T15:05:35.061316+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:08:03.788019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1292,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.209174+0000","last_change":"2024-09-13T15:05:34.074069+0000","last_active":"2024-09-13T15:06:06.209174+0000","last_peered":"2024-09-13T15:06:06.209174+0000","last_clean":"2024-09-13T15:06:06.209174+0000","last_became_active":"2024-09-13T15:05:34.073282+0000","last_became_peered":"2024-09-13T15:05:34.073282+0000","last_unstale":"2024-09-13T15:06:06.209174+0000","last_undegraded":"2024-09-13T15:06:06.209174+0000","last_fullsized":"2024-09-13T15:06:06.209174+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:14.307582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1046,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.097879+0000","last_change":"2024-09-13T14:47:46.322428+0000","last_active":"2024-09-13T15:05:35.097879+0000","last_peered":"2024-09-13T15:05:35.097879+0000","last_clean":"2024-09-13T15:05:35.097879+0000","last_became_active":"2024-09-13T14:47:46.321955+0000","last_became_peered":"2024-09-13T14:47:46.321955+0000","last_unstale":"2024-09-13T15:05:35.097879+0000","last_undegraded":"2024-09-13T15:05:35.097879+0000","last_fullsized":"2024-09-13T15:05:35.097879+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:47.614065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1532,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.237226+0000","last_change":"2024-09-13T15:05:34.087636+0000","last_active":"2024-09-13T15:06:06.237226+0000","last_peered":"2024-09-13T15:06:06.237226+0000","last_clean":"2024-09-13T15:06:06.237226+0000","last_became_active":"2024-09-13T15:05:34.087255+0000","last_became_peered":"2024-09-13T15:05:34.087255+0000","last_unstale":"2024-09-13T15:06:06.237226+0000","last_undegraded":"2024-09-13T15:06:06.237226+0000","last_fullsized":"2024-09-13T15:06:06.237226+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:36.313581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1533,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.237661+0000","last_change":"2024-09-13T15:05:34.089162+0000","last_active":"2024-09-13T15:06:06.237661+0000","last_peered":"2024-09-13T15:06:06.237661+0000","last_clean":"2024-09-13T15:06:06.237661+0000","last_became_active":"2024-09-13T15:05:34.088727+0000","last_became_peered":"2024-09-13T15:05:34.088727+0000","last_unstale":"2024-09-13T15:06:06.237661+0000","last_undegraded":"2024-09-13T15:06:06.237661+0000","last_fullsized":"2024-09-13T15:06:06.237661+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:29.209082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1039,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.378375+0000","last_change":"2024-09-13T15:05:34.073363+0000","last_active":"2024-09-13T15:05:35.378375+0000","last_peered":"2024-09-13T15:05:35.378375+0000","last_clean":"2024-09-13T15:05:35.378375+0000","last_became_active":"2024-09-13T15:05:34.072906+0000","last_became_peered":"2024-09-13T15:05:34.072906+0000","last_unstale":"2024-09-13T15:05:35.378375+0000","last_undegraded":"2024-09-13T15:05:35.378375+0000","last_fullsized":"2024-09-13T15:05:35.378375+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:27.966654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1079,"reported_epoch":1068,"state":"active+clean","last_fresh":"2024-09-13T15:05:35.098234+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T15:05:35.098234+0000","last_peered":"2024-09-13T15:05:35.098234+0000","last_clean":"2024-09-13T15:05:35.098234+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T15:05:35.098234+0000","last_undegraded":"2024-09-13T15:05:35.098234+0000","last_fullsized":"2024-09-13T15:05:35.098234+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1303,"reported_epoch":1069,"state":"active+clean","last_fresh":"2024-09-13T15:06:06.213751+0000","last_change":"2024-09-13T15:05:34.075192+0000","last_active":"2024-09-13T15:06:06.213751+0000","last_peered":"2024-09-13T15:06:06.213751+0000","last_clean":"2024-09-13T15:06:06.213751+0000","last_became_active":"2024-09-13T15:05:34.073315+0000","last_became_peered":"2024-09-13T15:05:34.073315+0000","last_unstale":"2024-09-13T15:06:06.213751+0000","last_undegraded":"2024-09-13T15:06:06.213751+0000","last_fullsized":"2024-09-13T15:06:06.213751+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:38.699775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":44,"num_pg":16,"stat_sum":{"num_bytes":3555983384,"num_objects":54261,"num_object_clones":0,"num_object_copies":162783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24152,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54261,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":160143,"num_write_kb":6860865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1622,"num_bytes_recovered":106233856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9756028928,"data_stored":9756016712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139643,"ondisk_log_size":139643,"up":48,"acting":44,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":5036,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398068,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2419316,"kb_used_data":1514924,"kb_used_omap":1,"kb_used_meta":904382,"kb_avail":91325836,"statfs":{"total":95995035648,"available":93517656064,"internally_reserved":0,"allocated":1551282176,"data_stored":1547026832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":926087512},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594303,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2223980,"kb_used_data":1323684,"kb_used_omap":1,"kb_used_meta":900286,"kb_avail":91521172,"statfs":{"total":95995035648,"available":93717680128,"internally_reserved":0,"allocated":1355452416,"data_stored":1351205129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921893208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790537,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1672860,"kb_used_data":1087060,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":92072292,"statfs":{"total":95995035648,"available":94282027008,"internally_reserved":0,"allocated":1113149440,"data_stored":1108913787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980832065,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2147496,"kb_used_data":1563808,"kb_used_omap":76,"kb_used_meta":583603,"kb_avail":91597656,"statfs":{"total":95995035648,"available":93795999744,"internally_reserved":0,"allocated":1601339392,"data_stored":1597095710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":597610304},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":1066,"seq":4578435137551,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":85332,"kb_used_data":11212,"kb_used_omap":121,"kb_used_meta":73990,"kb_avail":93659820,"statfs":{"total":95995035648,"available":95907655680,"internally_reserved":0,"allocated":11481088,"data_stored":7253094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124093,"internal_metadata":75766595},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023247,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1704728,"kb_used_data":1098896,"kb_used_omap":71,"kb_used_meta":605752,"kb_avail":92040424,"statfs":{"total":95995035648,"available":94249394176,"internally_reserved":0,"allocated":1125269504,"data_stored":1121037936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":620290720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004377,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1996504,"kb_used_data":1505744,"kb_used_omap":66,"kb_used_meta":490557,"kb_avail":91748648,"statfs":{"total":95995035648,"available":93950615552,"internally_reserved":0,"allocated":1541881856,"data_stored":1537649719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":502330538},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195569,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2168920,"kb_used_data":1553232,"kb_used_omap":56,"kb_used_meta":615495,"kb_avail":91576232,"statfs":{"total":95995035648,"available":93774061568,"internally_reserved":0,"allocated":1590509568,"data_stored":1586276402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":630267023},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1576534016,"data_stored":1576534016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1525682176,"data_stored":1525678104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1107820544,"data_stored":1107820544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1583546368,"data_stored":1583546368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1101201408,"data_stored":1101201408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1327763456,"data_stored":1327759384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":44,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1533480960,"data_stored":1533476888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:06:53.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-09-13T15:06:53.476 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:53.564 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:53.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:53.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:53.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:53.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:53.813 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:53.838 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:53.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:53.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:53.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T15:06:54.018 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:54.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:54.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:54.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:54.254 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:54.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:54 smithi177 bash[68594]: cluster 2024-09-13T15:06:52.956094+0000 mgr.y (mgr.26252) 1224 : cluster [DBG] pgmap v1781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 24031/150687 objects degraded (15.948%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:54.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:54 smithi177 bash[68594]: cluster 2024-09-13T15:06:52.956094+0000 mgr.y (mgr.26252) 1224 : cluster [DBG] pgmap v1781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 24031/150687 objects degraded (15.948%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:54.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:54 smithi099 bash[84576]: cluster 2024-09-13T15:06:52.956094+0000 mgr.y (mgr.26252) 1224 : cluster [DBG] pgmap v1781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 24031/150687 objects degraded (15.948%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:54 smithi099 bash[84576]: cluster 2024-09-13T15:06:52.956094+0000 mgr.y (mgr.26252) 1224 : cluster [DBG] pgmap v1781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 24031/150687 objects degraded (15.948%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:54.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:54 smithi099 bash[87147]: cluster 2024-09-13T15:06:52.956094+0000 mgr.y (mgr.26252) 1224 : cluster [DBG] pgmap v1781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 24031/150687 objects degraded (15.948%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:54.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:54 smithi099 bash[87147]: cluster 2024-09-13T15:06:52.956094+0000 mgr.y (mgr.26252) 1224 : cluster [DBG] pgmap v1781: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 24031/150687 objects degraded (15.948%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:54.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:54.438 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:54.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:54.542 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:54.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:54.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:54.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:54.701 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:54.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:54.808 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:54.843 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:54.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:54.950 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:54.985 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:54.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:55.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:55.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:55.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:55.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:55.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:55.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:55.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:55.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:55.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:55.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:06:55.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:55.547 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:06:55.558 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:55.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:55.664 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:55.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:55.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:55.837 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:55.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:55.945 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:55.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.990 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:55.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:56.091 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:56.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:56.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:56.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:56.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:56.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:56.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:56 smithi099 bash[84576]: cluster 2024-09-13T15:06:54.956722+0000 mgr.y (mgr.26252) 1225 : cluster [DBG] pgmap v1782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 23895/146388 objects degraded (16.323%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:56.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:56 smithi099 bash[84576]: cluster 2024-09-13T15:06:54.956722+0000 mgr.y (mgr.26252) 1225 : cluster [DBG] pgmap v1782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 23895/146388 objects degraded (16.323%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:56 smithi099 bash[87147]: cluster 2024-09-13T15:06:54.956722+0000 mgr.y (mgr.26252) 1225 : cluster [DBG] pgmap v1782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 23895/146388 objects degraded (16.323%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:56 smithi099 bash[87147]: cluster 2024-09-13T15:06:54.956722+0000 mgr.y (mgr.26252) 1225 : cluster [DBG] pgmap v1782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 23895/146388 objects degraded (16.323%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:56 smithi177 bash[68594]: cluster 2024-09-13T15:06:54.956722+0000 mgr.y (mgr.26252) 1225 : cluster [DBG] pgmap v1782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 23895/146388 objects degraded (16.323%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:56 smithi177 bash[68594]: cluster 2024-09-13T15:06:54.956722+0000 mgr.y (mgr.26252) 1225 : cluster [DBG] pgmap v1782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 23895/146388 objects degraded (16.323%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:56.369 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:56.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:56.502 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:56.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:56.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:56.655 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:56.696 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:56.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:56.802 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:56.847 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:56.955 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:56.994 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:56.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:57.099 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:57.121 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:57.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:57.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:57.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:57 smithi099 bash[84576]: cluster 2024-09-13T15:06:56.540362+0000 mon.a (mon.0) 2364 : cluster [WRN] Health check update: Degraded data redundancy: 23895/146388 objects degraded (16.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:57 smithi099 bash[84576]: cluster 2024-09-13T15:06:56.540362+0000 mon.a (mon.0) 2364 : cluster [WRN] Health check update: Degraded data redundancy: 23895/146388 objects degraded (16.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:57.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:06:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:06:57] "GET /metrics HTTP/1.1" 200 36786 "" "Prometheus/2.51.0" 2024-09-13T15:06:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:57 smithi099 bash[87147]: cluster 2024-09-13T15:06:56.540362+0000 mon.a (mon.0) 2364 : cluster [WRN] Health check update: Degraded data redundancy: 23895/146388 objects degraded (16.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:57 smithi099 bash[87147]: cluster 2024-09-13T15:06:56.540362+0000 mon.a (mon.0) 2364 : cluster [WRN] Health check update: Degraded data redundancy: 23895/146388 objects degraded (16.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:57.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:57.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:57 smithi177 bash[68594]: cluster 2024-09-13T15:06:56.540362+0000 mon.a (mon.0) 2364 : cluster [WRN] Health check update: Degraded data redundancy: 23895/146388 objects degraded (16.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:57.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:57 smithi177 bash[68594]: cluster 2024-09-13T15:06:56.540362+0000 mon.a (mon.0) 2364 : cluster [WRN] Health check update: Degraded data redundancy: 23895/146388 objects degraded (16.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:06:57.384 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:57.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:57.518 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:57.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:57.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:57.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:57.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:57.805 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:57.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:57.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:57.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:57.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:58.044 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:58.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:58.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:58.206 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:58.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:58.328 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:58.356 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:58.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:58 smithi177 bash[68594]: cluster 2024-09-13T15:06:56.957282+0000 mgr.y (mgr.26252) 1226 : cluster [DBG] pgmap v1783: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 23895/145281 objects degraded (16.447%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:58.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:06:58 smithi177 bash[68594]: cluster 2024-09-13T15:06:56.957282+0000 mgr.y (mgr.26252) 1226 : cluster [DBG] pgmap v1783: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 23895/145281 objects degraded (16.447%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:58.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:58.362 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:58 smithi099 bash[87147]: cluster 2024-09-13T15:06:56.957282+0000 mgr.y (mgr.26252) 1226 : cluster [DBG] pgmap v1783: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 23895/145281 objects degraded (16.447%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:58.363 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:06:58 smithi099 bash[87147]: cluster 2024-09-13T15:06:56.957282+0000 mgr.y (mgr.26252) 1226 : cluster [DBG] pgmap v1783: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 23895/145281 objects degraded (16.447%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:58.365 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:58 smithi099 bash[84576]: cluster 2024-09-13T15:06:56.957282+0000 mgr.y (mgr.26252) 1226 : cluster [DBG] pgmap v1783: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 23895/145281 objects degraded (16.447%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:58.365 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:06:58 smithi099 bash[84576]: cluster 2024-09-13T15:06:56.957282+0000 mgr.y (mgr.26252) 1226 : cluster [DBG] pgmap v1783: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 23895/145281 objects degraded (16.447%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T15:06:58.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:58.494 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:58.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:58.601 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:58.637 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:58.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:58.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:58.778 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:58.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:58.823 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:06:58.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:58.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:58.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:59.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:59.053 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:59.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:59.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:59.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:59.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:59.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:59.335 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:59.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:59.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:59.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:59.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:59.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:59.612 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:59.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:59.717 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:59.743 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:59.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:59.849 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:59.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:59.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:59.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:00.001 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:00.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:00.109 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:00.128 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:00.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:00.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:00.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:00.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:00.355 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:00 smithi177 bash[68594]: cluster 2024-09-13T15:06:58.957984+0000 mgr.y (mgr.26252) 1227 : cluster [DBG] pgmap v1784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23763/140319 objects degraded (16.935%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:07:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:00 smithi177 bash[68594]: cluster 2024-09-13T15:06:58.957984+0000 mgr.y (mgr.26252) 1227 : cluster [DBG] pgmap v1784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23763/140319 objects degraded (16.935%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:07:00.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:07:00 smithi177 bash[61439]: ts=2024-09-13T15:07:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:00.363 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:00.365 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:00.369 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:00 smithi099 bash[84576]: cluster 2024-09-13T15:06:58.957984+0000 mgr.y (mgr.26252) 1227 : cluster [DBG] pgmap v1784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23763/140319 objects degraded (16.935%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:07:00.369 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:00 smithi099 bash[84576]: cluster 2024-09-13T15:06:58.957984+0000 mgr.y (mgr.26252) 1227 : cluster [DBG] pgmap v1784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23763/140319 objects degraded (16.935%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:07:00.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:00 smithi099 bash[87147]: cluster 2024-09-13T15:06:58.957984+0000 mgr.y (mgr.26252) 1227 : cluster [DBG] pgmap v1784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23763/140319 objects degraded (16.935%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:07:00.372 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:00 smithi099 bash[87147]: cluster 2024-09-13T15:06:58.957984+0000 mgr.y (mgr.26252) 1227 : cluster [DBG] pgmap v1784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 23763/140319 objects degraded (16.935%); 1.7 MiB/s, 26 objects/s recovering 2024-09-13T15:07:00.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:00.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:00.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:00.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:00.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:00.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:00.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:00.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:00.755 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:00.784 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:00.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:00.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:00.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:00.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:01.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:01.052 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:01.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:01.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:01.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:01.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:01.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:01.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:01.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:01.474 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:01.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:01.582 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:01.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:01.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:01.716 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:01.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:01.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:01.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:01.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:01.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:01.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:02.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:02.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:02.111 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:02.152 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:02.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:02.259 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:02.271 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:02 smithi177 bash[68594]: cluster 2024-09-13T15:07:00.958654+0000 mgr.y (mgr.26252) 1228 : cluster [DBG] pgmap v1785: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 23763/135381 objects degraded (17.553%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:02.271 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:02 smithi177 bash[68594]: cluster 2024-09-13T15:07:00.958654+0000 mgr.y (mgr.26252) 1228 : cluster [DBG] pgmap v1785: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 23763/135381 objects degraded (17.553%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:02.271 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:02 smithi177 bash[68594]: cluster 2024-09-13T15:07:01.541014+0000 mon.a (mon.0) 2365 : cluster [WRN] Health check update: Degraded data redundancy: 23763/135381 objects degraded (17.553%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:02.271 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:02 smithi177 bash[68594]: cluster 2024-09-13T15:07:01.541014+0000 mon.a (mon.0) 2365 : cluster [WRN] Health check update: Degraded data redundancy: 23763/135381 objects degraded (17.553%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:02.288 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:02.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:02.394 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:02.426 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:02.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:02.541 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:02 smithi099 bash[84576]: cluster 2024-09-13T15:07:00.958654+0000 mgr.y (mgr.26252) 1228 : cluster [DBG] pgmap v1785: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 23763/135381 objects degraded (17.553%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:02.541 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:02 smithi099 bash[84576]: cluster 2024-09-13T15:07:00.958654+0000 mgr.y (mgr.26252) 1228 : cluster [DBG] pgmap v1785: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 23763/135381 objects degraded (17.553%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:02.541 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:02 smithi099 bash[84576]: cluster 2024-09-13T15:07:01.541014+0000 mon.a (mon.0) 2365 : cluster [WRN] Health check update: Degraded data redundancy: 23763/135381 objects degraded (17.553%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:02.541 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:02 smithi099 bash[84576]: cluster 2024-09-13T15:07:01.541014+0000 mon.a (mon.0) 2365 : cluster [WRN] Health check update: Degraded data redundancy: 23763/135381 objects degraded (17.553%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:02.542 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:02 smithi099 bash[87147]: cluster 2024-09-13T15:07:00.958654+0000 mgr.y (mgr.26252) 1228 : cluster [DBG] pgmap v1785: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 23763/135381 objects degraded (17.553%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:02.542 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:02 smithi099 bash[87147]: cluster 2024-09-13T15:07:00.958654+0000 mgr.y (mgr.26252) 1228 : cluster [DBG] pgmap v1785: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 23763/135381 objects degraded (17.553%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:02.542 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:02 smithi099 bash[87147]: cluster 2024-09-13T15:07:01.541014+0000 mon.a (mon.0) 2365 : cluster [WRN] Health check update: Degraded data redundancy: 23763/135381 objects degraded (17.553%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:02.542 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:02 smithi099 bash[87147]: cluster 2024-09-13T15:07:01.541014+0000 mon.a (mon.0) 2365 : cluster [WRN] Health check update: Degraded data redundancy: 23763/135381 objects degraded (17.553%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:02.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:02.677 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:02.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:02.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:02.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:02.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:02.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:02.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:02.990 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:02.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:03.093 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:03.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:03.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:03.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:03.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:03.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:03.370 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:03.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:03.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:03.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:03.543 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:03.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:03.646 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:03.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:03.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:03.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:03.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:03.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:03.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:03.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:04.046 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:04.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:04.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:04.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:04.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:04.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:04.309 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:04.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[84576]: cluster 2024-09-13T15:07:02.959267+0000 mgr.y (mgr.26252) 1229 : cluster [DBG] pgmap v1786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23763/132435 objects degraded (17.943%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:04.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[84576]: cluster 2024-09-13T15:07:02.959267+0000 mgr.y (mgr.26252) 1229 : cluster [DBG] pgmap v1786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23763/132435 objects degraded (17.943%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:04.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[84576]: audit 2024-09-13T15:07:03.311383+0000 mon.a (mon.0) 2366 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:04.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[84576]: audit 2024-09-13T15:07:03.311383+0000 mon.a (mon.0) 2366 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:04.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[84576]: audit 2024-09-13T15:07:03.312667+0000 mon.a (mon.0) 2367 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:04.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[84576]: audit 2024-09-13T15:07:03.312667+0000 mon.a (mon.0) 2367 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:04.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[87147]: cluster 2024-09-13T15:07:02.959267+0000 mgr.y (mgr.26252) 1229 : cluster [DBG] pgmap v1786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23763/132435 objects degraded (17.943%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:04.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[87147]: cluster 2024-09-13T15:07:02.959267+0000 mgr.y (mgr.26252) 1229 : cluster [DBG] pgmap v1786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23763/132435 objects degraded (17.943%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[87147]: audit 2024-09-13T15:07:03.311383+0000 mon.a (mon.0) 2366 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[87147]: audit 2024-09-13T15:07:03.311383+0000 mon.a (mon.0) 2366 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[87147]: audit 2024-09-13T15:07:03.312667+0000 mon.a (mon.0) 2367 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:04 smithi099 bash[87147]: audit 2024-09-13T15:07:03.312667+0000 mon.a (mon.0) 2367 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:04.348 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:04.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:04 smithi177 bash[68594]: cluster 2024-09-13T15:07:02.959267+0000 mgr.y (mgr.26252) 1229 : cluster [DBG] pgmap v1786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23763/132435 objects degraded (17.943%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:04 smithi177 bash[68594]: cluster 2024-09-13T15:07:02.959267+0000 mgr.y (mgr.26252) 1229 : cluster [DBG] pgmap v1786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23763/132435 objects degraded (17.943%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:04 smithi177 bash[68594]: audit 2024-09-13T15:07:03.311383+0000 mon.a (mon.0) 2366 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:04 smithi177 bash[68594]: audit 2024-09-13T15:07:03.311383+0000 mon.a (mon.0) 2366 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:04 smithi177 bash[68594]: audit 2024-09-13T15:07:03.312667+0000 mon.a (mon.0) 2367 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:04 smithi177 bash[68594]: audit 2024-09-13T15:07:03.312667+0000 mon.a (mon.0) 2367 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:04.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:04.515 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:04.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:04.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:04.648 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:04.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:04.788 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:04.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:04.893 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:04.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:04.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:05.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:05.062 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:05.065 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:07:05.065 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:07:05.065 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:07:05.065 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:07:05.065 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:07:05.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:05.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:05.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:05.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:05.336 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:05.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:05.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:05.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:05.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:05.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:05.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:05.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:05.756 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:05.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:05.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:05.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:05.924 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:05.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:06.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:06.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:06.166 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:06.173 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:06 smithi099 bash[87147]: cluster 2024-09-13T15:07:04.962331+0000 mgr.y (mgr.26252) 1230 : cluster [DBG] pgmap v1787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 23608/128391 objects degraded (18.388%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:06.174 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:06 smithi099 bash[87147]: cluster 2024-09-13T15:07:04.962331+0000 mgr.y (mgr.26252) 1230 : cluster [DBG] pgmap v1787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 23608/128391 objects degraded (18.388%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:06.174 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:06 smithi099 bash[84576]: cluster 2024-09-13T15:07:04.962331+0000 mgr.y (mgr.26252) 1230 : cluster [DBG] pgmap v1787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 23608/128391 objects degraded (18.388%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:06.174 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:06 smithi099 bash[84576]: cluster 2024-09-13T15:07:04.962331+0000 mgr.y (mgr.26252) 1230 : cluster [DBG] pgmap v1787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 23608/128391 objects degraded (18.388%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:06.184 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:06.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:06.287 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:06.311 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:06.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:06.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:07:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:06 smithi177 bash[68594]: cluster 2024-09-13T15:07:04.962331+0000 mgr.y (mgr.26252) 1230 : cluster [DBG] pgmap v1787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 23608/128391 objects degraded (18.388%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:06.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:06 smithi177 bash[68594]: cluster 2024-09-13T15:07:04.962331+0000 mgr.y (mgr.26252) 1230 : cluster [DBG] pgmap v1787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 23608/128391 objects degraded (18.388%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:06.415 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:06.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:06.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:06.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:06.598 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:06.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:06.753 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:06.860 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:06.899 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:06.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:07.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:07.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:07.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:07.135 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:07.169 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:07.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:07 smithi099 bash[84576]: audit 2024-09-13T15:07:06.352284+0000 mon.a (mon.0) 2368 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:07 smithi099 bash[84576]: audit 2024-09-13T15:07:06.352284+0000 mon.a (mon.0) 2368 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:07 smithi099 bash[84576]: cluster 2024-09-13T15:07:06.541832+0000 mon.a (mon.0) 2369 : cluster [WRN] Health check update: Degraded data redundancy: 23608/128391 objects degraded (18.388%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:07.253 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:07 smithi099 bash[84576]: cluster 2024-09-13T15:07:06.541832+0000 mon.a (mon.0) 2369 : cluster [WRN] Health check update: Degraded data redundancy: 23608/128391 objects degraded (18.388%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:07.254 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:07:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:07:07] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-13T15:07:07.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:07 smithi099 bash[87147]: audit 2024-09-13T15:07:06.352284+0000 mon.a (mon.0) 2368 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:07.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:07 smithi099 bash[87147]: audit 2024-09-13T15:07:06.352284+0000 mon.a (mon.0) 2368 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:07.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:07 smithi099 bash[87147]: cluster 2024-09-13T15:07:06.541832+0000 mon.a (mon.0) 2369 : cluster [WRN] Health check update: Degraded data redundancy: 23608/128391 objects degraded (18.388%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:07.254 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:07 smithi099 bash[87147]: cluster 2024-09-13T15:07:06.541832+0000 mon.a (mon.0) 2369 : cluster [WRN] Health check update: Degraded data redundancy: 23608/128391 objects degraded (18.388%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:07.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:07.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:07.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:07.409 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:07.417 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:07 smithi177 bash[68594]: audit 2024-09-13T15:07:06.352284+0000 mon.a (mon.0) 2368 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:07.417 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:07 smithi177 bash[68594]: audit 2024-09-13T15:07:06.352284+0000 mon.a (mon.0) 2368 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:07.418 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:07 smithi177 bash[68594]: cluster 2024-09-13T15:07:06.541832+0000 mon.a (mon.0) 2369 : cluster [WRN] Health check update: Degraded data redundancy: 23608/128391 objects degraded (18.388%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:07.418 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:07 smithi177 bash[68594]: cluster 2024-09-13T15:07:06.541832+0000 mon.a (mon.0) 2369 : cluster [WRN] Health check update: Degraded data redundancy: 23608/128391 objects degraded (18.388%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:07.439 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:07.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:07.546 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:07.576 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:07.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:07.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:07.715 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:07.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:07.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:07.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:07.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:07.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:07.983 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:07.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:08.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:08.112 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:08.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:08.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:08.252 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:08.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:08.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:08.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:08 smithi177 bash[68594]: cluster 2024-09-13T15:07:06.962918+0000 mgr.y (mgr.26252) 1231 : cluster [DBG] pgmap v1788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 23608/127329 objects degraded (18.541%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T15:07:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:08 smithi177 bash[68594]: cluster 2024-09-13T15:07:06.962918+0000 mgr.y (mgr.26252) 1231 : cluster [DBG] pgmap v1788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 23608/127329 objects degraded (18.541%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T15:07:08.385 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:08.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:08.489 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:08.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:08.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:08.564 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:08 smithi099 bash[84576]: cluster 2024-09-13T15:07:06.962918+0000 mgr.y (mgr.26252) 1231 : cluster [DBG] pgmap v1788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 23608/127329 objects degraded (18.541%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T15:07:08.564 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:08 smithi099 bash[84576]: cluster 2024-09-13T15:07:06.962918+0000 mgr.y (mgr.26252) 1231 : cluster [DBG] pgmap v1788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 23608/127329 objects degraded (18.541%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T15:07:08.564 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:08 smithi099 bash[87147]: cluster 2024-09-13T15:07:06.962918+0000 mgr.y (mgr.26252) 1231 : cluster [DBG] pgmap v1788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 23608/127329 objects degraded (18.541%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T15:07:08.564 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:08 smithi099 bash[87147]: cluster 2024-09-13T15:07:06.962918+0000 mgr.y (mgr.26252) 1231 : cluster [DBG] pgmap v1788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 23608/127329 objects degraded (18.541%); 1.5 MiB/s, 23 objects/s recovering 2024-09-13T15:07:08.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:08.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:08.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:08.747 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:08.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:08.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:08.885 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:08.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:08.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:09.009 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:09.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:09.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:09.152 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:09.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:09.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:09.322 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:09.360 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:09.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:09.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:09.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:09 smithi177 bash[68594]: cluster 2024-09-13T15:07:08.963691+0000 mgr.y (mgr.26252) 1232 : cluster [DBG] pgmap v1789: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 23431/123003 objects degraded (19.049%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:07:09.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:09 smithi177 bash[68594]: cluster 2024-09-13T15:07:08.963691+0000 mgr.y (mgr.26252) 1232 : cluster [DBG] pgmap v1789: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 23431/123003 objects degraded (19.049%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:07:09.513 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:09.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:09.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:09.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:09.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:09.766 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:09 smithi099 bash[84576]: cluster 2024-09-13T15:07:08.963691+0000 mgr.y (mgr.26252) 1232 : cluster [DBG] pgmap v1789: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 23431/123003 objects degraded (19.049%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:07:09.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:09 smithi099 bash[84576]: cluster 2024-09-13T15:07:08.963691+0000 mgr.y (mgr.26252) 1232 : cluster [DBG] pgmap v1789: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 23431/123003 objects degraded (19.049%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:07:09.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:09 smithi099 bash[87147]: cluster 2024-09-13T15:07:08.963691+0000 mgr.y (mgr.26252) 1232 : cluster [DBG] pgmap v1789: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 23431/123003 objects degraded (19.049%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:07:09.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:09 smithi099 bash[87147]: cluster 2024-09-13T15:07:08.963691+0000 mgr.y (mgr.26252) 1232 : cluster [DBG] pgmap v1789: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 23431/123003 objects degraded (19.049%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T15:07:09.877 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:09.904 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:10.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:10.041 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:10.148 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:10.174 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:10.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:10.277 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:10.287 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:07:10 smithi177 bash[61439]: ts=2024-09-13T15:07:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:10.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:10.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:10.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:10.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:10.556 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:10.580 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:10.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:10.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:10.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:10.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:10.823 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:10.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:10.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:10.966 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:11.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:11.095 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:11.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:11.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:11.218 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:11.244 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:11.352 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:11.374 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:11.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:11.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:11.520 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:11.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:11.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:11.654 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:11.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:11.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:11.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:11.886 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:11.920 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:11.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:12.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:12.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:12.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:12.151 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:12.175 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:12.279 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:12.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:12.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:12 smithi099 bash[84576]: cluster 2024-09-13T15:07:10.964225+0000 mgr.y (mgr.26252) 1233 : cluster [DBG] pgmap v1790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 23431/119010 objects degraded (19.688%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:12 smithi099 bash[84576]: cluster 2024-09-13T15:07:10.964225+0000 mgr.y (mgr.26252) 1233 : cluster [DBG] pgmap v1790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 23431/119010 objects degraded (19.688%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:12 smithi099 bash[84576]: cluster 2024-09-13T15:07:11.542691+0000 mon.a (mon.0) 2370 : cluster [WRN] Health check update: Degraded data redundancy: 23431/119010 objects degraded (19.688%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:12 smithi099 bash[84576]: cluster 2024-09-13T15:07:11.542691+0000 mon.a (mon.0) 2370 : cluster [WRN] Health check update: Degraded data redundancy: 23431/119010 objects degraded (19.688%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:12.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:12 smithi099 bash[87147]: cluster 2024-09-13T15:07:10.964225+0000 mgr.y (mgr.26252) 1233 : cluster [DBG] pgmap v1790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 23431/119010 objects degraded (19.688%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:12 smithi099 bash[87147]: cluster 2024-09-13T15:07:10.964225+0000 mgr.y (mgr.26252) 1233 : cluster [DBG] pgmap v1790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 23431/119010 objects degraded (19.688%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:12 smithi099 bash[87147]: cluster 2024-09-13T15:07:11.542691+0000 mon.a (mon.0) 2370 : cluster [WRN] Health check update: Degraded data redundancy: 23431/119010 objects degraded (19.688%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:12 smithi099 bash[87147]: cluster 2024-09-13T15:07:11.542691+0000 mon.a (mon.0) 2370 : cluster [WRN] Health check update: Degraded data redundancy: 23431/119010 objects degraded (19.688%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:12.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:12.431 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:12.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:12.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:12 smithi177 bash[68594]: cluster 2024-09-13T15:07:10.964225+0000 mgr.y (mgr.26252) 1233 : cluster [DBG] pgmap v1790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 23431/119010 objects degraded (19.688%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:12.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:12 smithi177 bash[68594]: cluster 2024-09-13T15:07:10.964225+0000 mgr.y (mgr.26252) 1233 : cluster [DBG] pgmap v1790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 23431/119010 objects degraded (19.688%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:12.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:12 smithi177 bash[68594]: cluster 2024-09-13T15:07:11.542691+0000 mon.a (mon.0) 2370 : cluster [WRN] Health check update: Degraded data redundancy: 23431/119010 objects degraded (19.688%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:12.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:12 smithi177 bash[68594]: cluster 2024-09-13T15:07:11.542691+0000 mon.a (mon.0) 2370 : cluster [WRN] Health check update: Degraded data redundancy: 23431/119010 objects degraded (19.688%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:12.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:12.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:12.659 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:12.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:12.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:12.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:12.820 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:12.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:12.924 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:12.956 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:12.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:13.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:13.104 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:13.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:13.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:13.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:13.352 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:13.381 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:13.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:13.519 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:13.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:13.649 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:13.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:13.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:13.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.799 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:13.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:13.900 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:13.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:14.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:14.065 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:14.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:14.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:14.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:14.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:14.322 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:14.332 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:14 smithi177 bash[68594]: cluster 2024-09-13T15:07:12.964857+0000 mgr.y (mgr.26252) 1234 : cluster [DBG] pgmap v1791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 23431/115938 objects degraded (20.210%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:14.332 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:14 smithi177 bash[68594]: cluster 2024-09-13T15:07:12.964857+0000 mgr.y (mgr.26252) 1234 : cluster [DBG] pgmap v1791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 23431/115938 objects degraded (20.210%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:14 smithi099 bash[84576]: cluster 2024-09-13T15:07:12.964857+0000 mgr.y (mgr.26252) 1234 : cluster [DBG] pgmap v1791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 23431/115938 objects degraded (20.210%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:14.340 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:14 smithi099 bash[84576]: cluster 2024-09-13T15:07:12.964857+0000 mgr.y (mgr.26252) 1234 : cluster [DBG] pgmap v1791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 23431/115938 objects degraded (20.210%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:14.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:14 smithi099 bash[87147]: cluster 2024-09-13T15:07:12.964857+0000 mgr.y (mgr.26252) 1234 : cluster [DBG] pgmap v1791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 23431/115938 objects degraded (20.210%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:14.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:14 smithi099 bash[87147]: cluster 2024-09-13T15:07:12.964857+0000 mgr.y (mgr.26252) 1234 : cluster [DBG] pgmap v1791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 23431/115938 objects degraded (20.210%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T15:07:14.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:14.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:14.471 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:14.509 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:14.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:14.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:14.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:14.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:14.751 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:14.775 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:14.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:14.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:14.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:14.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:15.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:15.056 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:15.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:15.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:15.194 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:15.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:15.299 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:15.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:15 smithi177 bash[68594]: audit 2024-09-13T15:07:14.262818+0000 mon.a (mon.0) 2371 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:15 smithi177 bash[68594]: audit 2024-09-13T15:07:14.262818+0000 mon.a (mon.0) 2371 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:15 smithi177 bash[68594]: audit 2024-09-13T15:07:14.273062+0000 mon.a (mon.0) 2372 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:15 smithi177 bash[68594]: audit 2024-09-13T15:07:14.273062+0000 mon.a (mon.0) 2372 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:15 smithi177 bash[68594]: cluster 2024-09-13T15:07:14.965476+0000 mgr.y (mgr.26252) 1235 : cluster [DBG] pgmap v1792: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 23244/111693 objects degraded (20.811%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:15.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:15 smithi177 bash[68594]: cluster 2024-09-13T15:07:14.965476+0000 mgr.y (mgr.26252) 1235 : cluster [DBG] pgmap v1792: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 23244/111693 objects degraded (20.811%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:15.334 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:15.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:15.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:15.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:15.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:15.569 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:15.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[84576]: audit 2024-09-13T15:07:14.262818+0000 mon.a (mon.0) 2371 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[84576]: audit 2024-09-13T15:07:14.262818+0000 mon.a (mon.0) 2371 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[84576]: audit 2024-09-13T15:07:14.273062+0000 mon.a (mon.0) 2372 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[84576]: audit 2024-09-13T15:07:14.273062+0000 mon.a (mon.0) 2372 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[84576]: cluster 2024-09-13T15:07:14.965476+0000 mgr.y (mgr.26252) 1235 : cluster [DBG] pgmap v1792: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 23244/111693 objects degraded (20.811%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:15.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[84576]: cluster 2024-09-13T15:07:14.965476+0000 mgr.y (mgr.26252) 1235 : cluster [DBG] pgmap v1792: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 23244/111693 objects degraded (20.811%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:15.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[87147]: audit 2024-09-13T15:07:14.262818+0000 mon.a (mon.0) 2371 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[87147]: audit 2024-09-13T15:07:14.262818+0000 mon.a (mon.0) 2371 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[87147]: audit 2024-09-13T15:07:14.273062+0000 mon.a (mon.0) 2372 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[87147]: audit 2024-09-13T15:07:14.273062+0000 mon.a (mon.0) 2372 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:15.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[87147]: cluster 2024-09-13T15:07:14.965476+0000 mgr.y (mgr.26252) 1235 : cluster [DBG] pgmap v1792: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 23244/111693 objects degraded (20.811%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:15.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:15 smithi099 bash[87147]: cluster 2024-09-13T15:07:14.965476+0000 mgr.y (mgr.26252) 1235 : cluster [DBG] pgmap v1792: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 23244/111693 objects degraded (20.811%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:15.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:15.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:15.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:15.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:15.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:15.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:15.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:15.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:15.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:16.023 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:16.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:16.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:16.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:16.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:16.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:16.285 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:16.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:16.391 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:16.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:16.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:16.553 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:16.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:16.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:16.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:16 smithi099 bash[84576]: cluster 2024-09-13T15:07:16.543358+0000 mon.a (mon.0) 2373 : cluster [WRN] Health check update: Degraded data redundancy: 23244/111693 objects degraded (20.811%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:16.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:16 smithi099 bash[84576]: cluster 2024-09-13T15:07:16.543358+0000 mon.a (mon.0) 2373 : cluster [WRN] Health check update: Degraded data redundancy: 23244/111693 objects degraded (20.811%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:16.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:16 smithi099 bash[87147]: cluster 2024-09-13T15:07:16.543358+0000 mon.a (mon.0) 2373 : cluster [WRN] Health check update: Degraded data redundancy: 23244/111693 objects degraded (20.811%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:16.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:16 smithi099 bash[87147]: cluster 2024-09-13T15:07:16.543358+0000 mon.a (mon.0) 2373 : cluster [WRN] Health check update: Degraded data redundancy: 23244/111693 objects degraded (20.811%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:16.685 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:16.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:16.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:16.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:16.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:16 smithi177 bash[68594]: cluster 2024-09-13T15:07:16.543358+0000 mon.a (mon.0) 2373 : cluster [WRN] Health check update: Degraded data redundancy: 23244/111693 objects degraded (20.811%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:16.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:16 smithi177 bash[68594]: cluster 2024-09-13T15:07:16.543358+0000 mon.a (mon.0) 2373 : cluster [WRN] Health check update: Degraded data redundancy: 23244/111693 objects degraded (20.811%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:16.928 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:16.969 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:16.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:17.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:17.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:17.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:17.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:17.221 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:07:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:07:17] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-13T15:07:17.239 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:17.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:17.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:17.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:17.485 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:17.504 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:17.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:17.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:17.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:17.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:17.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:17.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:17 smithi099 bash[84576]: cluster 2024-09-13T15:07:16.966019+0000 mgr.y (mgr.26252) 1236 : cluster [DBG] pgmap v1793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 23244/110664 objects degraded (21.004%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:17.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:17 smithi099 bash[84576]: cluster 2024-09-13T15:07:16.966019+0000 mgr.y (mgr.26252) 1236 : cluster [DBG] pgmap v1793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 23244/110664 objects degraded (21.004%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:17.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:17 smithi099 bash[87147]: cluster 2024-09-13T15:07:16.966019+0000 mgr.y (mgr.26252) 1236 : cluster [DBG] pgmap v1793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 23244/110664 objects degraded (21.004%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:17.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:17 smithi099 bash[87147]: cluster 2024-09-13T15:07:16.966019+0000 mgr.y (mgr.26252) 1236 : cluster [DBG] pgmap v1793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 23244/110664 objects degraded (21.004%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:17 smithi177 bash[68594]: cluster 2024-09-13T15:07:16.966019+0000 mgr.y (mgr.26252) 1236 : cluster [DBG] pgmap v1793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 23244/110664 objects degraded (21.004%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:17.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:17 smithi177 bash[68594]: cluster 2024-09-13T15:07:16.966019+0000 mgr.y (mgr.26252) 1236 : cluster [DBG] pgmap v1793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 23244/110664 objects degraded (21.004%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:17.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:17.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:17.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:17.972 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:07:17.972 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:70992 44.c 44:3faf36cd:::benchmark_data_smithi099_188632_object70991:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1061)", 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:04:36.665354+0000", 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 161.30542952799999, 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45501346300000001, 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:17.973 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 70992 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.665354+0000", 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.665354+0000", 2024-09-13T15:07:17.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.665356+0000", 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8500000000000001e-06 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.665391+0000", 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5295e-05 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:17.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.665404+0000", 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2466999999999999e-05 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.665410+0000", 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3119999999999999e-06 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.678804+0000", 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013394071 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.678924+0000", 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011960600000000001 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.679050+0000", 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000126134 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.681556+0000", 2024-09-13T15:07:17.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025058400000000001 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.681572+0000", 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5877000000000001e-05 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:37.120181+0000", 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43860955699999998 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:17.978 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:37.120290+0000", 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108727 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:37.120368+0000", 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7726999999999996e-05 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.979 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:71017 44.c 44:3fd2d65d:::benchmark_data_smithi099_188632_object71016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1061)", 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:04:36.682120+0000", 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 161.28866389800001, 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43855387400000001, 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 71017 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.980 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.682120+0000", 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.682120+0000", 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:17.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.682121+0000", 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8700000000000004e-07 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.682184+0000", 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2769999999999997e-05 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.682194+0000", 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0729e-05 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:17.982 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.682201+0000", 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.08e-06 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.682295+0000", 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3746000000000007e-05 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.682410+0000", 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000115035 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:07:17.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.682545+0000", 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013439300000000001 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.684643+0000", 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002098534 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.687514+0000", 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028710419999999999 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:37.120489+0000", 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43297517600000002 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:37.120606+0000", 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000117081 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:37.120674+0000", 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7300999999999998e-05 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:17.985 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:71025 44.c 44:34487388:::benchmark_data_smithi099_188632_object71024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1061)", 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:04:36.696454+0000", 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 161.27432981999999, 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.42453605799999999, 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:17.986 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 71025 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.696454+0000", 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.696454+0000", 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.696459+0000", 2024-09-13T15:07:17.987 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2290000000000002e-06 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.696506+0000", 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6343e-05 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.696535+0000", 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9374999999999999e-05 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:17.988 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.696548+0000", 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3288000000000001e-05 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.696628+0000", 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9485000000000002e-05 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.696800+0000", 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017206900000000001 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.696981+0000", 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000181452 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.699613+0000", 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00263167 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:36.699730+0000", 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011735000000000001 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:37.120764+0000", 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.421033659 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:37.120920+0000", 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000155601 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:04:37.120990+0000", 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0537000000000006e-05 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:17.991 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:117314 44.0 44:0e28f848:::benchmark_data_smithi099_188632_object10110:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1064)", 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:30.644092+0000", 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 107.326691981, 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39248407699999999, 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 117314 2024-09-13T15:07:17.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:30.644092+0000", 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:30.644092+0000", 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:30.644093+0000", 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3549999999999999e-06 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:30.644098+0000", 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1699999999999996e-06 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:30.644112+0000", 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4129e-05 2024-09-13T15:07:17.994 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:30.644120+0000", 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2200000000000003e-06 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:30.644182+0000", 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1953999999999997e-05 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:17.995 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:30.644326+0000", 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000144322 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0", 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:30.644472+0000", 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014650199999999999 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:30.648322+0000", 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038494100000000002 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:31.036314+0000", 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.387992434 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:31.036412+0000", 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7415000000000002e-05 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:31.036576+0000", 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016416600000000001 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:17.997 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:119196 44.0 44:085a8861:::benchmark_data_smithi099_188632_object11992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1065)", 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:33.063566+0000", 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.90721827599999, 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6606343580000003, 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119196 2024-09-13T15:07:17.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.063566+0000", 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.063566+0000", 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:17.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.063567+0000", 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.42e-06 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.063573+0000", 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.694e-06 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.063586+0000", 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2945999999999999e-05 2024-09-13T15:07:18.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.063593+0000", 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8029999999999997e-06 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.063656+0000", 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2947000000000006e-05 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.063750+0000", 2024-09-13T15:07:18.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3220999999999994e-05 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0", 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.063819+0000", 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9701000000000002e-05 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.075517+0000", 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011698165 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.086358+0000", 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010840285999999999 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.086376+0000", 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7978e-05 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.061749+0000", 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97537344100000001 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.061772+0000", 2024-09-13T15:07:18.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2623999999999999e-05 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.700276+0000", 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6385044290000002 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.700729+0000", 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000452903 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.724083+0000", 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023354157 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.724200+0000", 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116643 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:119204 44.0 44:0c5d1c04:::benchmark_data_smithi099_188632_object12000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1066)", 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:33.081287+0000", 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.889497275, 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6465485060000002, 2024-09-13T15:07:18.005 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119204 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.081287+0000", 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.081287+0000", 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.081288+0000", 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4339999999999999e-06 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.081292+0000", 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7440000000000001e-06 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.081301+0000", 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7210000000000006e-06 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.081308+0000", 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6760000000000001e-06 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.086617+0000", 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053089310000000002 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.086624+0000", 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9949999999999999e-06 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.061815+0000", 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97519112399999996 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.061827+0000", 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1394e-05 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.704520+0000", 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.642693154 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.704768+0000", 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024841899999999999 2024-09-13T15:07:18.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.705016+0000", 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024803200000000003 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.724306+0000", 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.019289389000000001 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.724373+0000", 2024-09-13T15:07:18.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7045000000000006e-05 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.724536+0000", 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016323600000000001 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.724595+0000", 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9401000000000003e-05 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.727835+0000", 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003239811 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:119225 44.0 44:09328216:::benchmark_data_smithi099_188632_object12021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1066)", 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:33.108774+0000", 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.86200931899999, 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6739903699999998, 2024-09-13T15:07:18.013 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119225 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.108774+0000", 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.108774+0000", 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.108776+0000", 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0809999999999999e-06 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.108780+0000", 2024-09-13T15:07:18.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9960000000000004e-06 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.108788+0000", 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0059999999999992e-06 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.108794+0000", 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7429999999999996e-06 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.108852+0000", 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8035000000000003e-05 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.108875+0000", 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2308999999999999e-05 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.061862+0000", 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95298776399999996 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.061873+0000", 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0696999999999999e-05 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.727914+0000", 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6660408260000001 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.728091+0000", 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017747800000000001 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.728238+0000", 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014706100000000001 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.732803+0000", 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045643810000000002 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.756933+0000", 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024130378000000001 2024-09-13T15:07:18.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.779429+0000", 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022495969000000001 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.779470+0000", 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0785000000000003e-05 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.782765+0000", 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032948610000000001 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:119227 44.0 44:0fcf0655:::benchmark_data_smithi099_188632_object12023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1066)", 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:33.111638+0000", 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.85914569400001, 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.674265745, 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.021 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119227 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.111638+0000", 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.111638+0000", 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.111639+0000", 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4200000000000004e-07 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.111645+0000", 2024-09-13T15:07:18.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0650000000000004e-06 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.111657+0000", 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1564e-05 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.111665+0000", 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8120000000000008e-06 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.111729+0000", 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4207000000000004e-05 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.111737+0000", 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6179999999999995e-06 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.061914+0000", 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95017661099999995 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.061925+0000", 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0671e-05 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.732883+0000", 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.670958728 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.733022+0000", 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000138493 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.733150+0000", 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000128374 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.743300+0000", 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010149480000000001 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.756946+0000", 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013646096 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.782816+0000", 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025870081999999999 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.782876+0000", 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0167999999999997e-05 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.785904+0000", 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030278340000000001 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.028 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:119260 44.0 44:093fc013:::benchmark_data_smithi099_188632_object12056:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1066)", 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:33.158909+0000", 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.81187508799999, 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6299332199999998, 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.029 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119260 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.158909+0000", 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.158909+0000", 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.158910+0000", 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6270000000000001e-06 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.158917+0000", 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0729999999999999e-06 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.158935+0000", 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7674000000000001e-05 2024-09-13T15:07:18.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.158944+0000", 2024-09-13T15:07:18.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7949999999999996e-06 2024-09-13T15:07:18.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.159027+0000", 2024-09-13T15:07:18.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2663000000000005e-05 2024-09-13T15:07:18.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.159041+0000", 2024-09-13T15:07:18.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4487999999999999e-05 2024-09-13T15:07:18.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.061957+0000", 2024-09-13T15:07:18.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90291581600000004 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.061966+0000", 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1130000000000007e-06 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.737747+0000", 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6757813910000001 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.737935+0000", 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000187985 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.738117+0000", 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018173700000000001 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.743391+0000", 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005274061 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.756953+0000", 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013562171 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.785945+0000", 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028991317999999999 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.785988+0000", 2024-09-13T15:07:18.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2902999999999999e-05 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.788842+0000", 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002854405 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:119262 44.0 44:0ab5828d:::benchmark_data_smithi099_188632_object12058:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1066)", 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:33.164940+0000", 2024-09-13T15:07:18.037 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.805843674, 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6270054809999999, 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119262 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.164940+0000", 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.164940+0000", 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.164942+0000", 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.398e-06 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.164948+0000", 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5019999999999998e-06 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.164963+0000", 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5140000000000001e-05 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.164970+0000", 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9940000000000003e-06 2024-09-13T15:07:18.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.165023+0000", 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2577999999999997e-05 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:33.165031+0000", 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5699999999999993e-06 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.062000+0000", 2024-09-13T15:07:18.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89696911400000001 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.062011+0000", 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0353000000000001e-05 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.743457+0000", 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.681446043 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.743937+0000", 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00047989099999999998 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.744093+0000", 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000156241 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.756908+0000", 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012814658 2024-09-13T15:07:18.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.756960+0000", 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2714999999999999e-05 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.788916+0000", 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.031955403 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.788969+0000", 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3547000000000001e-05 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.791946+0000", 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029763340000000002 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.045 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:119207 44.9 44:902706fb:::benchmark_data_smithi099_188632_object12003:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1067)", 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:34.070844+0000", 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 103.89994003699999, 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6823545439999998, 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119207 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070844+0000", 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070844+0000", 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070846+0000", 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7659999999999999e-06 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.047 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070852+0000", 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9660000000000001e-06 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070865+0000", 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3862e-05 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070874+0000", 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3920000000000003e-06 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.048 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070947+0000", 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2941999999999994e-05 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070960+0000", 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3324e-05 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.722439+0000", 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.651478668 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.728403+0000", 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059639929999999999 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.728681+0000", 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00027857499999999998 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.746494+0000", 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017812449000000001 2024-09-13T15:07:18.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.746517+0000", 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3183000000000001e-05 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.748402+0000", 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018848840000000001 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.748509+0000", 2024-09-13T15:07:18.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107159 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.753198+0000", 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046893810000000003 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:119241 44.9 44:9af5984f:::benchmark_data_smithi099_188632_object12037:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1067)", 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:34.070894+0000", 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 103.899889708, 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.694057028, 2024-09-13T15:07:18.052 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119241 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070894+0000", 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070894+0000", 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070895+0000", 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9500000000000002e-07 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070897+0000", 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.774e-06 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070905+0000", 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9440000000000005e-06 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070910+0000", 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3409999999999996e-06 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.070994+0000", 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4042999999999995e-05 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:34.071001+0000", 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3660000000000003e-06 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.740090+0000", 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6690883410000001 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.740340+0000", 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00025000000000000001 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.740577+0000", 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000237586 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.746687+0000", 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061095500000000001 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.753417+0000", 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067304970000000002 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.753499+0000", 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1664999999999998e-05 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.753559+0000", 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9859999999999999e-05 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:35.764951+0000", 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011392466 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.058 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:119984 44.0 44:00d25faf:::benchmark_data_smithi099_188632_object12780:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1068)", 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:36.760079+0000", 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 101.210704593, 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32780662399999999, 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.059 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119984 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.760079+0000", 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.760079+0000", 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.760080+0000", 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.119e-06 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.760085+0000", 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0150000000000003e-06 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.760096+0000", 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0900000000000001e-05 2024-09-13T15:07:18.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.760104+0000", 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7999999999999999e-06 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.760156+0000", 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1446999999999997e-05 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.760267+0000", 2024-09-13T15:07:18.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000111313 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.760367+0000", 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100618 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.763448+0000", 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030806729999999999 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:36.766075+0000", 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002627131 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:37.084488+0000", 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31841257099999998 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:37.084551+0000", 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3467999999999996e-05 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:37.087886+0000", 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033345689999999999 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:125840 44.0 44:0af2ad6a:::benchmark_data_smithi099_188632_object18636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1068)", 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:45.827807+0000", 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 92.142976476000001, 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36614019399999997, 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.065 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 125840 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.827807+0000", 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.827807+0000", 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.827809+0000", 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3939999999999999e-06 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.827814+0000", 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3499999999999996e-06 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.827822+0000", 2024-09-13T15:07:18.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3299999999999999e-06 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.827829+0000", 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1410000000000002e-06 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.827911+0000", 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2162000000000004e-05 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.828044+0000", 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013348 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.828208+0000", 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000164167 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.831402+0000", 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003193549 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.841259+0000", 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0098573470000000007 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:46.191488+0000", 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.350228343 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:46.191531+0000", 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3072999999999999e-05 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:46.193948+0000", 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002416858 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:127523 44.c 44:3d5f9e71:::benchmark_data_smithi099_188632_object20319:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1068)", 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:48.987490+0000", 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 88.983294150000006, 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33235980900000001, 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.071 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 127523 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.987490+0000", 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.987490+0000", 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.987491+0000", 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.449e-06 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.987497+0000", 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9020000000000004e-06 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.987511+0000", 2024-09-13T15:07:18.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4278e-05 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.987519+0000", 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9610000000000005e-06 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.987590+0000", 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1039999999999997e-05 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.987728+0000", 2024-09-13T15:07:18.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013747400000000001 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.987857+0000", 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000129479 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.992131+0000", 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042734499999999998 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:48.992258+0000", 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0001273 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:49.319562+0000", 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32730420799999999 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:49.319719+0000", 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000156565 2024-09-13T15:07:18.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:49.319849+0000", 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000130703 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:135480 44.0 44:001408c6:::benchmark_data_smithi099_188632_object28276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1068)", 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:02.098272+0000", 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 75.872511575000004, 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.47765124599999997, 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 135480 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.098272+0000", 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.098272+0000", 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.098273+0000", 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5499999999999997e-07 2024-09-13T15:07:18.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.098277+0000", 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8850000000000001e-06 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.098287+0000", 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0210000000000001e-05 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.098294+0000", 2024-09-13T15:07:18.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7970000000000003e-06 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.098350+0000", 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6249000000000003e-05 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.098452+0000", 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102107 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.098539+0000", 2024-09-13T15:07:18.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6366000000000006e-05 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.115582+0000", 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017043309999999999 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.132928+0000", 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017346422 2024-09-13T15:07:18.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.572952+0000", 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44002335300000001 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.573019+0000", 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6964000000000005e-05 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.575923+0000", 2024-09-13T15:07:18.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029048279999999999 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:135482 44.0 44:07e3dc0a:::benchmark_data_smithi099_188632_object28278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1068)", 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:02.099027+0000", 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 75.87175723, 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.483916228, 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.084 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 135482 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.099027+0000", 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.099027+0000", 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.099028+0000", 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.102e-06 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.099032+0000", 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6859999999999997e-06 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.099043+0000", 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0582e-05 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.099051+0000", 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7419999999999997e-06 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.104361+0000", 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053107950000000001 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.104547+0000", 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018579800000000001 2024-09-13T15:07:18.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.104736+0000", 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018908400000000001 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.115586+0000", 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010849829 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.132988+0000", 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017401386000000001 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.579029+0000", 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44604154699999998 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.579107+0000", 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8217e-05 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.582943+0000", 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038354600000000002 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:135483 44.0 44:0ae93a5b:::benchmark_data_smithi099_188632_object28279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1068)", 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:02.103161+0000", 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 75.867622922999999, 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.485983106, 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.090 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 135483 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.103161+0000", 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.103161+0000", 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.103163+0000", 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0810000000000001e-06 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.103173+0000", 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0179e-05 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.103189+0000", 2024-09-13T15:07:18.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5930000000000002e-05 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.103200+0000", 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1229e-05 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.111577+0000", 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0083764879999999996 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.111756+0000", 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017920800000000001 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.111923+0000", 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016730299999999999 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.117103+0000", 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005179309 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.137070+0000", 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.019967866000000001 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.585946+0000", 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44887590999999999 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.586019+0000", 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3003000000000006e-05 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.589144+0000", 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031245999999999999 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:135494 44.0 44:0f0c01a3:::benchmark_data_smithi099_188632_object28290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1068)", 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:02.128842+0000", 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 75.841942223000004, 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.46285288299999999, 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.096 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 135494 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.128842+0000", 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.128842+0000", 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.128843+0000", 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3489999999999999e-06 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.128848+0000", 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.48e-06 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.128875+0000", 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.703e-05 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.128883+0000", 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2660000000000004e-06 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.128935+0000", 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2682000000000001e-05 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.129046+0000", 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011014399999999999 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.129145+0000", 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9810999999999997e-05 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.136936+0000", 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0077901649999999999 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.137201+0000", 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026511299999999998 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.589179+0000", 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45197842700000002 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.589217+0000", 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7693e-05 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.591694+0000", 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024777229999999998 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:170181 44.0 44:04d26303:::benchmark_data_smithi099_188632_object62977:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1070)", 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:07:00.395102+0000", 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.575681316000001, 2024-09-13T15:07:18.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45542593999999997, 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 170181 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.395102+0000", 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.395102+0000", 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.395105+0000", 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.576e-06 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.395114+0000", 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8740000000000001e-06 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.395135+0000", 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0971999999999998e-05 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.395146+0000", 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1542000000000001e-05 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.395220+0000", 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3468999999999997e-05 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.395354+0000", 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013375400000000001 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.395462+0000", 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108626 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.399197+0000", 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037343179999999999 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.403574+0000", 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.004377784 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.847703+0000", 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44412895499999999 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.847752+0000", 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8637000000000002e-05 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:07:18.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:07:18.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:07:18.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.850528+0000", 2024-09-13T15:07:18.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002776433 2024-09-13T15:07:18.108 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.108 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.108 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.108 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:07:18.108 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:07:18.108 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:07:18.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:18.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:18.258 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:18.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:18.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:18.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:18.419 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:18.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:18.526 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:18.577 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:18.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:18.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:18.694 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:18 smithi099 bash[87147]: audit 2024-09-13T15:07:18.303701+0000 mon.a (mon.0) 2374 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:18.694 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:18 smithi099 bash[87147]: audit 2024-09-13T15:07:18.303701+0000 mon.a (mon.0) 2374 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:18.703 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:18.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:18.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:18.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:18.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:18.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:18 smithi177 bash[68594]: audit 2024-09-13T15:07:18.303701+0000 mon.a (mon.0) 2374 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:18.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:18 smithi177 bash[68594]: audit 2024-09-13T15:07:18.303701+0000 mon.a (mon.0) 2374 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:18.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:18.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:18.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:19.052 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:07:19.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:18 smithi099 bash[84576]: audit 2024-09-13T15:07:18.303701+0000 mon.a (mon.0) 2374 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:19.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:18 smithi099 bash[84576]: audit 2024-09-13T15:07:18.303701+0000 mon.a (mon.0) 2374 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:19.090 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:19.123 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:19.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:19.226 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:19.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:19.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:19.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:19.388 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:19.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:19.497 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:19.519 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:19.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:19.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:19.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:19.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:19.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:19.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:19.873 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:19 smithi099 bash[84576]: cluster 2024-09-13T15:07:18.966627+0000 mgr.y (mgr.26252) 1237 : cluster [DBG] pgmap v1794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23089/105969 objects degraded (21.788%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:19.873 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:19 smithi099 bash[84576]: cluster 2024-09-13T15:07:18.966627+0000 mgr.y (mgr.26252) 1237 : cluster [DBG] pgmap v1794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23089/105969 objects degraded (21.788%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:19.873 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:19 smithi099 bash[87147]: cluster 2024-09-13T15:07:18.966627+0000 mgr.y (mgr.26252) 1237 : cluster [DBG] pgmap v1794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23089/105969 objects degraded (21.788%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:19.873 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:19 smithi099 bash[87147]: cluster 2024-09-13T15:07:18.966627+0000 mgr.y (mgr.26252) 1237 : cluster [DBG] pgmap v1794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23089/105969 objects degraded (21.788%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:19.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:19.929 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:19 smithi177 bash[68594]: cluster 2024-09-13T15:07:18.966627+0000 mgr.y (mgr.26252) 1237 : cluster [DBG] pgmap v1794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23089/105969 objects degraded (21.788%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:19.929 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:19 smithi177 bash[68594]: cluster 2024-09-13T15:07:18.966627+0000 mgr.y (mgr.26252) 1237 : cluster [DBG] pgmap v1794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 23089/105969 objects degraded (21.788%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:07:19.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:19.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:20.046 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:20.078 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:20.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:20.184 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:20.233 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:20.340 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:20.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:20.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:20.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:20.500 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:20.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:20.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:20.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:07:20 smithi177 bash[61439]: ts=2024-09-13T15:07:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:20.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:20.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:20.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:20.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:20.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:20.886 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:20.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:21.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:21.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:21.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:21.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:21.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:21.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:21.301 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:21.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:21.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:21.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:21.540 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:21.577 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:21.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:21.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:21.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:21.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:21.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:21.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:21.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:21.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:21.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:21.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:22.078 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:22 smithi177 bash[68594]: cluster 2024-09-13T15:07:20.967280+0000 mgr.y (mgr.26252) 1238 : cluster [DBG] pgmap v1795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 23089/101508 objects degraded (22.746%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:22 smithi177 bash[68594]: cluster 2024-09-13T15:07:20.967280+0000 mgr.y (mgr.26252) 1238 : cluster [DBG] pgmap v1795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 23089/101508 objects degraded (22.746%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:22 smithi177 bash[68594]: cluster 2024-09-13T15:07:21.544140+0000 mon.a (mon.0) 2375 : cluster [WRN] Health check update: Degraded data redundancy: 23089/101508 objects degraded (22.746%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:22 smithi177 bash[68594]: cluster 2024-09-13T15:07:21.544140+0000 mon.a (mon.0) 2375 : cluster [WRN] Health check update: Degraded data redundancy: 23089/101508 objects degraded (22.746%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:22 smithi177 bash[68594]: audit 2024-09-13T15:07:21.576511+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:22 smithi177 bash[68594]: audit 2024-09-13T15:07:21.576511+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:22 smithi177 bash[68594]: audit 2024-09-13T15:07:21.589103+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:22 smithi177 bash[68594]: audit 2024-09-13T15:07:21.589103+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.125 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:22.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:22.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:22.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[84576]: cluster 2024-09-13T15:07:20.967280+0000 mgr.y (mgr.26252) 1238 : cluster [DBG] pgmap v1795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 23089/101508 objects degraded (22.746%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[84576]: cluster 2024-09-13T15:07:20.967280+0000 mgr.y (mgr.26252) 1238 : cluster [DBG] pgmap v1795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 23089/101508 objects degraded (22.746%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[84576]: cluster 2024-09-13T15:07:21.544140+0000 mon.a (mon.0) 2375 : cluster [WRN] Health check update: Degraded data redundancy: 23089/101508 objects degraded (22.746%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[84576]: cluster 2024-09-13T15:07:21.544140+0000 mon.a (mon.0) 2375 : cluster [WRN] Health check update: Degraded data redundancy: 23089/101508 objects degraded (22.746%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[84576]: audit 2024-09-13T15:07:21.576511+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[84576]: audit 2024-09-13T15:07:21.576511+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[84576]: audit 2024-09-13T15:07:21.589103+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[84576]: audit 2024-09-13T15:07:21.589103+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[87147]: cluster 2024-09-13T15:07:20.967280+0000 mgr.y (mgr.26252) 1238 : cluster [DBG] pgmap v1795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 23089/101508 objects degraded (22.746%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:22.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[87147]: cluster 2024-09-13T15:07:20.967280+0000 mgr.y (mgr.26252) 1238 : cluster [DBG] pgmap v1795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 23089/101508 objects degraded (22.746%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:22.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[87147]: cluster 2024-09-13T15:07:21.544140+0000 mon.a (mon.0) 2375 : cluster [WRN] Health check update: Degraded data redundancy: 23089/101508 objects degraded (22.746%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:22.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[87147]: cluster 2024-09-13T15:07:21.544140+0000 mon.a (mon.0) 2375 : cluster [WRN] Health check update: Degraded data redundancy: 23089/101508 objects degraded (22.746%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:22.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[87147]: audit 2024-09-13T15:07:21.576511+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[87147]: audit 2024-09-13T15:07:21.576511+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[87147]: audit 2024-09-13T15:07:21.589103+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:22 smithi099 bash[87147]: audit 2024-09-13T15:07:21.589103+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:22.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:22.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:22.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:22.500 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:22.529 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:22.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:22.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:22.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:22.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:22.806 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:22.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:22.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:22.941 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:22.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:23.045 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:23.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:23.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:23.191 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:23.239 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:23.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[84576]: audit 2024-09-13T15:07:22.994912+0000 mon.a (mon.0) 2378 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:07:23.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[84576]: audit 2024-09-13T15:07:22.994912+0000 mon.a (mon.0) 2378 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:07:23.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[84576]: audit 2024-09-13T15:07:22.995982+0000 mon.a (mon.0) 2379 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:07:23.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[84576]: audit 2024-09-13T15:07:22.995982+0000 mon.a (mon.0) 2379 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:07:23.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[84576]: audit 2024-09-13T15:07:23.014043+0000 mon.a (mon.0) 2380 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:23.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[84576]: audit 2024-09-13T15:07:23.014043+0000 mon.a (mon.0) 2380 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:23.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[87147]: audit 2024-09-13T15:07:22.994912+0000 mon.a (mon.0) 2378 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:07:23.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[87147]: audit 2024-09-13T15:07:22.994912+0000 mon.a (mon.0) 2378 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:07:23.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[87147]: audit 2024-09-13T15:07:22.995982+0000 mon.a (mon.0) 2379 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:07:23.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[87147]: audit 2024-09-13T15:07:22.995982+0000 mon.a (mon.0) 2379 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:07:23.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[87147]: audit 2024-09-13T15:07:23.014043+0000 mon.a (mon.0) 2380 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:23.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:23 smithi099 bash[87147]: audit 2024-09-13T15:07:23.014043+0000 mon.a (mon.0) 2380 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:23.349 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:23.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:23 smithi177 bash[68594]: audit 2024-09-13T15:07:22.994912+0000 mon.a (mon.0) 2378 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:07:23.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:23 smithi177 bash[68594]: audit 2024-09-13T15:07:22.994912+0000 mon.a (mon.0) 2378 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:07:23.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:23 smithi177 bash[68594]: audit 2024-09-13T15:07:22.995982+0000 mon.a (mon.0) 2379 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:07:23.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:23 smithi177 bash[68594]: audit 2024-09-13T15:07:22.995982+0000 mon.a (mon.0) 2379 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:07:23.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:23 smithi177 bash[68594]: audit 2024-09-13T15:07:23.014043+0000 mon.a (mon.0) 2380 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:23.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:23 smithi177 bash[68594]: audit 2024-09-13T15:07:23.014043+0000 mon.a (mon.0) 2380 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:23.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:23.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:23.481 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:07:23.481 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:07:23.482 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:23.507 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:23.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:23.522 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:07:23.522 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:07:23.581 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:07:23.581 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:07:23.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:23.617 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:07:23.617 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:07:23.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:23.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:23.661 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:07:23.662 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:07:23.701 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:07:23.701 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:07:23.730 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:07:23.731 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:07:23.758 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:23.765 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:07:23.765 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:07:23.793 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:23.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:23.819 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:23.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:23.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:23.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:24.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:24.049 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T15:07:24.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:24.059 INFO:teuthology.orchestra.run.smithi099.stderr:set pool 1 pgp_num to 1 2024-09-13T15:07:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:24.170 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:24.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[87147]: cluster 2024-09-13T15:07:22.968004+0000 mgr.y (mgr.26252) 1239 : cluster [DBG] pgmap v1796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 23089/98214 objects degraded (23.509%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:24.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[87147]: cluster 2024-09-13T15:07:22.968004+0000 mgr.y (mgr.26252) 1239 : cluster [DBG] pgmap v1796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 23089/98214 objects degraded (23.509%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:24.180 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[87147]: audit 2024-09-13T15:07:23.246656+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.99:0/3493216002' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.181 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[87147]: audit 2024-09-13T15:07:23.246656+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.99:0/3493216002' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.181 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[87147]: audit 2024-09-13T15:07:23.247512+0000 mon.a (mon.0) 2381 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.181 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[87147]: audit 2024-09-13T15:07:23.247512+0000 mon.a (mon.0) 2381 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.181 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[87147]: audit 2024-09-13T15:07:23.424051+0000 mon.a (mon.0) 2382 : audit [DBG] from='client.? 172.21.15.99:0/23764892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:24.181 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[87147]: audit 2024-09-13T15:07:23.424051+0000 mon.a (mon.0) 2382 : audit [DBG] from='client.? 172.21.15.99:0/23764892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:24.181 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[87147]: audit 2024-09-13T15:07:23.657588+0000 mon.a (mon.0) 2383 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:07:24.181 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[87147]: audit 2024-09-13T15:07:23.657588+0000 mon.a (mon.0) 2383 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:07:24.181 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[84576]: cluster 2024-09-13T15:07:22.968004+0000 mgr.y (mgr.26252) 1239 : cluster [DBG] pgmap v1796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 23089/98214 objects degraded (23.509%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:24.181 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[84576]: cluster 2024-09-13T15:07:22.968004+0000 mgr.y (mgr.26252) 1239 : cluster [DBG] pgmap v1796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 23089/98214 objects degraded (23.509%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:24.181 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[84576]: audit 2024-09-13T15:07:23.246656+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.99:0/3493216002' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.182 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[84576]: audit 2024-09-13T15:07:23.246656+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.99:0/3493216002' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.182 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[84576]: audit 2024-09-13T15:07:23.247512+0000 mon.a (mon.0) 2381 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.182 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[84576]: audit 2024-09-13T15:07:23.247512+0000 mon.a (mon.0) 2381 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.182 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[84576]: audit 2024-09-13T15:07:23.424051+0000 mon.a (mon.0) 2382 : audit [DBG] from='client.? 172.21.15.99:0/23764892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:24.182 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[84576]: audit 2024-09-13T15:07:23.424051+0000 mon.a (mon.0) 2382 : audit [DBG] from='client.? 172.21.15.99:0/23764892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:24.182 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[84576]: audit 2024-09-13T15:07:23.657588+0000 mon.a (mon.0) 2383 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:07:24.182 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:24 smithi099 bash[84576]: audit 2024-09-13T15:07:23.657588+0000 mon.a (mon.0) 2383 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:07:24.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:24.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:24.302 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:24.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:24 smithi177 bash[68594]: cluster 2024-09-13T15:07:22.968004+0000 mgr.y (mgr.26252) 1239 : cluster [DBG] pgmap v1796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 23089/98214 objects degraded (23.509%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:24.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:24 smithi177 bash[68594]: cluster 2024-09-13T15:07:22.968004+0000 mgr.y (mgr.26252) 1239 : cluster [DBG] pgmap v1796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 23089/98214 objects degraded (23.509%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:24.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:24 smithi177 bash[68594]: audit 2024-09-13T15:07:23.246656+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.99:0/3493216002' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:24 smithi177 bash[68594]: audit 2024-09-13T15:07:23.246656+0000 mon.c (mon.2) 96 : audit [INF] from='client.? 172.21.15.99:0/3493216002' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:24 smithi177 bash[68594]: audit 2024-09-13T15:07:23.247512+0000 mon.a (mon.0) 2381 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:24 smithi177 bash[68594]: audit 2024-09-13T15:07:23.247512+0000 mon.a (mon.0) 2381 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:07:24.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:24 smithi177 bash[68594]: audit 2024-09-13T15:07:23.424051+0000 mon.a (mon.0) 2382 : audit [DBG] from='client.? 172.21.15.99:0/23764892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:24.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:24 smithi177 bash[68594]: audit 2024-09-13T15:07:23.424051+0000 mon.a (mon.0) 2382 : audit [DBG] from='client.? 172.21.15.99:0/23764892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:24.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:24 smithi177 bash[68594]: audit 2024-09-13T15:07:23.657588+0000 mon.a (mon.0) 2383 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:07:24.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:24 smithi177 bash[68594]: audit 2024-09-13T15:07:23.657588+0000 mon.a (mon.0) 2383 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:07:24.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:24.446 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:24.478 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:24.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:24.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:24.611 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:24.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:24.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:24.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:24.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:24.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:24.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:25.002 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:25.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:25.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:25.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:25.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:25.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:25.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:25 smithi177 bash[68594]: cluster 2024-09-13T15:07:24.043827+0000 mon.a (mon.0) 2384 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:25.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:25 smithi177 bash[68594]: cluster 2024-09-13T15:07:24.043827+0000 mon.a (mon.0) 2384 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:25.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:25 smithi177 bash[68594]: audit 2024-09-13T15:07:24.048719+0000 mon.a (mon.0) 2385 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:07:25.261 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:25 smithi177 bash[68594]: audit 2024-09-13T15:07:24.048719+0000 mon.a (mon.0) 2385 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:07:25.261 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:25 smithi177 bash[68594]: audit 2024-09-13T15:07:24.048954+0000 mon.a (mon.0) 2386 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:07:25.261 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:25 smithi177 bash[68594]: audit 2024-09-13T15:07:24.048954+0000 mon.a (mon.0) 2386 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:07:25.261 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:25 smithi177 bash[68594]: cluster 2024-09-13T15:07:24.054876+0000 mon.a (mon.0) 2387 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2024-09-13T15:07:25.261 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:25 smithi177 bash[68594]: cluster 2024-09-13T15:07:24.054876+0000 mon.a (mon.0) 2387 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2024-09-13T15:07:25.277 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:25.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:25.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[84576]: cluster 2024-09-13T15:07:24.043827+0000 mon.a (mon.0) 2384 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:25.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[84576]: cluster 2024-09-13T15:07:24.043827+0000 mon.a (mon.0) 2384 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:25.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[84576]: audit 2024-09-13T15:07:24.048719+0000 mon.a (mon.0) 2385 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:07:25.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[84576]: audit 2024-09-13T15:07:24.048719+0000 mon.a (mon.0) 2385 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:07:25.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[84576]: audit 2024-09-13T15:07:24.048954+0000 mon.a (mon.0) 2386 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:07:25.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[84576]: audit 2024-09-13T15:07:24.048954+0000 mon.a (mon.0) 2386 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:07:25.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[84576]: cluster 2024-09-13T15:07:24.054876+0000 mon.a (mon.0) 2387 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2024-09-13T15:07:25.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[84576]: cluster 2024-09-13T15:07:24.054876+0000 mon.a (mon.0) 2387 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2024-09-13T15:07:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[87147]: cluster 2024-09-13T15:07:24.043827+0000 mon.a (mon.0) 2384 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[87147]: cluster 2024-09-13T15:07:24.043827+0000 mon.a (mon.0) 2384 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[87147]: audit 2024-09-13T15:07:24.048719+0000 mon.a (mon.0) 2385 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:07:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[87147]: audit 2024-09-13T15:07:24.048719+0000 mon.a (mon.0) 2385 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:07:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[87147]: audit 2024-09-13T15:07:24.048954+0000 mon.a (mon.0) 2386 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:07:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[87147]: audit 2024-09-13T15:07:24.048954+0000 mon.a (mon.0) 2386 : audit [INF] from='client.? 172.21.15.99:0/2385507395' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:07:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[87147]: cluster 2024-09-13T15:07:24.054876+0000 mon.a (mon.0) 2387 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2024-09-13T15:07:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:25 smithi099 bash[87147]: cluster 2024-09-13T15:07:24.054876+0000 mon.a (mon.0) 2387 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2024-09-13T15:07:25.380 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:25.426 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:25.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:25.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:25.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:25.655 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:25.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:25.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:25.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:25.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:25.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:25.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:25.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:25.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:26.049 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:26.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:26.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:26.195 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:26.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:26.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:26 smithi099 bash[84576]: cluster 2024-09-13T15:07:24.968945+0000 mgr.y (mgr.26252) 1240 : cluster [DBG] pgmap v1798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 22916/94338 objects degraded (24.291%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:26.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:26 smithi099 bash[84576]: cluster 2024-09-13T15:07:24.968945+0000 mgr.y (mgr.26252) 1240 : cluster [DBG] pgmap v1798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 22916/94338 objects degraded (24.291%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:26.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:26 smithi099 bash[87147]: cluster 2024-09-13T15:07:24.968945+0000 mgr.y (mgr.26252) 1240 : cluster [DBG] pgmap v1798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 22916/94338 objects degraded (24.291%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:26.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:26 smithi099 bash[87147]: cluster 2024-09-13T15:07:24.968945+0000 mgr.y (mgr.26252) 1240 : cluster [DBG] pgmap v1798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 22916/94338 objects degraded (24.291%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:26.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:26.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:26 smithi177 bash[68594]: cluster 2024-09-13T15:07:24.968945+0000 mgr.y (mgr.26252) 1240 : cluster [DBG] pgmap v1798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 22916/94338 objects degraded (24.291%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:26.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:26 smithi177 bash[68594]: cluster 2024-09-13T15:07:24.968945+0000 mgr.y (mgr.26252) 1240 : cluster [DBG] pgmap v1798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 22916/94338 objects degraded (24.291%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:26.361 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:26.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:26.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:26.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:26.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:26.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:26.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:26.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:26.715 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:26.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:26.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:26.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:26.889 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:26.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:26.994 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:27.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:27.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:27.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:27.165 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:27.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:27.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:27.310 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:27.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:27.313 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:07:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:07:27] "GET /metrics HTTP/1.1" 200 36780 "" "Prometheus/2.51.0" 2024-09-13T15:07:27.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:27 smithi099 bash[84576]: cluster 2024-09-13T15:07:26.544950+0000 mon.a (mon.0) 2388 : cluster [WRN] Health check update: Degraded data redundancy: 22916/94338 objects degraded (24.291%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:27.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:27 smithi099 bash[84576]: cluster 2024-09-13T15:07:26.544950+0000 mon.a (mon.0) 2388 : cluster [WRN] Health check update: Degraded data redundancy: 22916/94338 objects degraded (24.291%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:27.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:27 smithi099 bash[87147]: cluster 2024-09-13T15:07:26.544950+0000 mon.a (mon.0) 2388 : cluster [WRN] Health check update: Degraded data redundancy: 22916/94338 objects degraded (24.291%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:27.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:27 smithi099 bash[87147]: cluster 2024-09-13T15:07:26.544950+0000 mon.a (mon.0) 2388 : cluster [WRN] Health check update: Degraded data redundancy: 22916/94338 objects degraded (24.291%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:27 smithi177 bash[68594]: cluster 2024-09-13T15:07:26.544950+0000 mon.a (mon.0) 2388 : cluster [WRN] Health check update: Degraded data redundancy: 22916/94338 objects degraded (24.291%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:27.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:27 smithi177 bash[68594]: cluster 2024-09-13T15:07:26.544950+0000 mon.a (mon.0) 2388 : cluster [WRN] Health check update: Degraded data redundancy: 22916/94338 objects degraded (24.291%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:27.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:27.431 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:27.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:27.533 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:27.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:27.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:27.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:27.685 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:27.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:27.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:27.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:27.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:27.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:27.940 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:27.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:28.046 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:28.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:28.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:28.168 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:28.178 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:28 smithi177 bash[68594]: cluster 2024-09-13T15:07:26.969567+0000 mgr.y (mgr.26252) 1241 : cluster [DBG] pgmap v1799: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 22916/92814 objects degraded (24.690%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:28.179 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:28 smithi177 bash[68594]: cluster 2024-09-13T15:07:26.969567+0000 mgr.y (mgr.26252) 1241 : cluster [DBG] pgmap v1799: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 22916/92814 objects degraded (24.690%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:28.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:28.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:28.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:28.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:28 smithi099 bash[84576]: cluster 2024-09-13T15:07:26.969567+0000 mgr.y (mgr.26252) 1241 : cluster [DBG] pgmap v1799: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 22916/92814 objects degraded (24.690%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:28.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:28 smithi099 bash[84576]: cluster 2024-09-13T15:07:26.969567+0000 mgr.y (mgr.26252) 1241 : cluster [DBG] pgmap v1799: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 22916/92814 objects degraded (24.690%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:28.309 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:28 smithi099 bash[87147]: cluster 2024-09-13T15:07:26.969567+0000 mgr.y (mgr.26252) 1241 : cluster [DBG] pgmap v1799: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 22916/92814 objects degraded (24.690%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:28.309 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:28 smithi099 bash[87147]: cluster 2024-09-13T15:07:26.969567+0000 mgr.y (mgr.26252) 1241 : cluster [DBG] pgmap v1799: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 22916/92814 objects degraded (24.690%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:07:28.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:28.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:28.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:28.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:28.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:28.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:28.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:28.707 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:28.743 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:28.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:28.880 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:28.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:29.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:29.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:29.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:29.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:29.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:29.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:29.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:29.388 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:29.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:29.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:29.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:29.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:29.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:29.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:29.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:29.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:29.793 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:29.821 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:29.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:29.828 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:29.828 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:29.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:29.956 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:29.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:30.059 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:30.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:30.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:30.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:30.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:30.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:30.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:30 smithi099 bash[84576]: cluster 2024-09-13T15:07:28.970234+0000 mgr.y (mgr.26252) 1242 : cluster [DBG] pgmap v1800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 22738/88581 objects degraded (25.669%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:30.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:30 smithi099 bash[84576]: cluster 2024-09-13T15:07:28.970234+0000 mgr.y (mgr.26252) 1242 : cluster [DBG] pgmap v1800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 22738/88581 objects degraded (25.669%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:30 smithi099 bash[87147]: cluster 2024-09-13T15:07:28.970234+0000 mgr.y (mgr.26252) 1242 : cluster [DBG] pgmap v1800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 22738/88581 objects degraded (25.669%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:30 smithi099 bash[87147]: cluster 2024-09-13T15:07:28.970234+0000 mgr.y (mgr.26252) 1242 : cluster [DBG] pgmap v1800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 22738/88581 objects degraded (25.669%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:30.338 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:30.381 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:30.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:30.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:30.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:30.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:30.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:30 smithi177 bash[68594]: cluster 2024-09-13T15:07:28.970234+0000 mgr.y (mgr.26252) 1242 : cluster [DBG] pgmap v1800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 22738/88581 objects degraded (25.669%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:30.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:30 smithi177 bash[68594]: cluster 2024-09-13T15:07:28.970234+0000 mgr.y (mgr.26252) 1242 : cluster [DBG] pgmap v1800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 22738/88581 objects degraded (25.669%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:30.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:30.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:30.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:30.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:07:30.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:07:30.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:30.777 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:30.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:30.861 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:07:30 smithi177 bash[61439]: ts=2024-09-13T15:07:30.216Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:30.885 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:30.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:30.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:31.041 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:31.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:07:31.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:31.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:31.199 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:31.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:31.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:31.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:31.361 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:31.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:31.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:31.493 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:31.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:31.599 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:31.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:31.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:31.746 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:31.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:31.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:31.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:31.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:31.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:32.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:32.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:32.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:32.153 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:32.163 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:32 smithi099 bash[87147]: cluster 2024-09-13T15:07:30.970763+0000 mgr.y (mgr.26252) 1243 : cluster [DBG] pgmap v1801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 22738/84777 objects degraded (26.821%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:32.164 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:32 smithi099 bash[87147]: cluster 2024-09-13T15:07:30.970763+0000 mgr.y (mgr.26252) 1243 : cluster [DBG] pgmap v1801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 22738/84777 objects degraded (26.821%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:32.164 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:32 smithi099 bash[87147]: cluster 2024-09-13T15:07:31.545696+0000 mon.a (mon.0) 2389 : cluster [WRN] Health check update: Degraded data redundancy: 22738/84777 objects degraded (26.821%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:32.164 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:32 smithi099 bash[87147]: cluster 2024-09-13T15:07:31.545696+0000 mon.a (mon.0) 2389 : cluster [WRN] Health check update: Degraded data redundancy: 22738/84777 objects degraded (26.821%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:32.164 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:32 smithi099 bash[84576]: cluster 2024-09-13T15:07:30.970763+0000 mgr.y (mgr.26252) 1243 : cluster [DBG] pgmap v1801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 22738/84777 objects degraded (26.821%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:32.164 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:32 smithi099 bash[84576]: cluster 2024-09-13T15:07:30.970763+0000 mgr.y (mgr.26252) 1243 : cluster [DBG] pgmap v1801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 22738/84777 objects degraded (26.821%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:32.164 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:32 smithi099 bash[84576]: cluster 2024-09-13T15:07:31.545696+0000 mon.a (mon.0) 2389 : cluster [WRN] Health check update: Degraded data redundancy: 22738/84777 objects degraded (26.821%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:32.164 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:32 smithi099 bash[84576]: cluster 2024-09-13T15:07:31.545696+0000 mon.a (mon.0) 2389 : cluster [WRN] Health check update: Degraded data redundancy: 22738/84777 objects degraded (26.821%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:32.179 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:32.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:32.283 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:32.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:32.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:32 smithi177 bash[68594]: cluster 2024-09-13T15:07:30.970763+0000 mgr.y (mgr.26252) 1243 : cluster [DBG] pgmap v1801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 22738/84777 objects degraded (26.821%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:32 smithi177 bash[68594]: cluster 2024-09-13T15:07:30.970763+0000 mgr.y (mgr.26252) 1243 : cluster [DBG] pgmap v1801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 22738/84777 objects degraded (26.821%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:32 smithi177 bash[68594]: cluster 2024-09-13T15:07:31.545696+0000 mon.a (mon.0) 2389 : cluster [WRN] Health check update: Degraded data redundancy: 22738/84777 objects degraded (26.821%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:32 smithi177 bash[68594]: cluster 2024-09-13T15:07:31.545696+0000 mon.a (mon.0) 2389 : cluster [WRN] Health check update: Degraded data redundancy: 22738/84777 objects degraded (26.821%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:32.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:32.445 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:32.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:32.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:32.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:32.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:32.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:32.722 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:32.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:32.826 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:32.859 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:32.965 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:32.990 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:32.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:33.097 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:33.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:33.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:33.226 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:33.252 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:33.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:33.356 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:33.396 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:33.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:33.503 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:33.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:33.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:33.671 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:07:33.672 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:07:33.672 INFO:tasks.thrashosds.thrasher:Reweighting osd 0 to 0.8720984153235086 2024-09-13T15:07:33.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 0 0.8720984153235086 2024-09-13T15:07:33.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:33.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:33.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:33.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:33.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:33.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:33.955 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:34.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:34.090 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:34.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:34.196 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:34.240 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:34.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[84576]: cluster 2024-09-13T15:07:32.971388+0000 mgr.y (mgr.26252) 1244 : cluster [DBG] pgmap v1802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 22738/81336 objects degraded (27.956%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[84576]: cluster 2024-09-13T15:07:32.971388+0000 mgr.y (mgr.26252) 1244 : cluster [DBG] pgmap v1802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 22738/81336 objects degraded (27.956%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[84576]: audit 2024-09-13T15:07:33.312458+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[84576]: audit 2024-09-13T15:07:33.312458+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[84576]: audit 2024-09-13T15:07:33.314752+0000 mon.a (mon.0) 2391 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[84576]: audit 2024-09-13T15:07:33.314752+0000 mon.a (mon.0) 2391 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[87147]: cluster 2024-09-13T15:07:32.971388+0000 mgr.y (mgr.26252) 1244 : cluster [DBG] pgmap v1802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 22738/81336 objects degraded (27.956%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[87147]: cluster 2024-09-13T15:07:32.971388+0000 mgr.y (mgr.26252) 1244 : cluster [DBG] pgmap v1802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 22738/81336 objects degraded (27.956%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[87147]: audit 2024-09-13T15:07:33.312458+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[87147]: audit 2024-09-13T15:07:33.312458+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[87147]: audit 2024-09-13T15:07:33.314752+0000 mon.a (mon.0) 2391 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:34 smithi099 bash[87147]: audit 2024-09-13T15:07:33.314752+0000 mon.a (mon.0) 2391 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:34.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:34 smithi177 bash[68594]: cluster 2024-09-13T15:07:32.971388+0000 mgr.y (mgr.26252) 1244 : cluster [DBG] pgmap v1802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 22738/81336 objects degraded (27.956%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:34 smithi177 bash[68594]: cluster 2024-09-13T15:07:32.971388+0000 mgr.y (mgr.26252) 1244 : cluster [DBG] pgmap v1802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 22738/81336 objects degraded (27.956%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:34 smithi177 bash[68594]: audit 2024-09-13T15:07:33.312458+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:34 smithi177 bash[68594]: audit 2024-09-13T15:07:33.312458+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:07:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:34 smithi177 bash[68594]: audit 2024-09-13T15:07:33.314752+0000 mon.a (mon.0) 2391 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:34 smithi177 bash[68594]: audit 2024-09-13T15:07:33.314752+0000 mon.a (mon.0) 2391 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:34.384 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:34.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:34.491 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:34.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:34.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:34.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:34.658 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:34.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:34.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:34.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:34.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:34.900 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:34.929 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:34.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:35.033 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:35.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:35.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:35.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:35.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:35.295 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:35.335 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:35.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:35.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:35.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:35.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:35.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:35.576 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:35.576 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:35.622 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:35.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:35.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:35.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:35.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:35.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:35.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:35.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:35.921 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:36.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:36.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:36.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:36.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:36.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:36 smithi177 bash[68594]: cluster 2024-09-13T15:07:34.972064+0000 mgr.y (mgr.26252) 1245 : cluster [DBG] pgmap v1803: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 22573/76839 objects degraded (29.377%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:07:36.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:36 smithi177 bash[68594]: cluster 2024-09-13T15:07:34.972064+0000 mgr.y (mgr.26252) 1245 : cluster [DBG] pgmap v1803: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 22573/76839 objects degraded (29.377%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:07:36.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.282 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:36.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:36.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:36 smithi099 bash[84576]: cluster 2024-09-13T15:07:34.972064+0000 mgr.y (mgr.26252) 1245 : cluster [DBG] pgmap v1803: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 22573/76839 objects degraded (29.377%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:07:36.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:36 smithi099 bash[84576]: cluster 2024-09-13T15:07:34.972064+0000 mgr.y (mgr.26252) 1245 : cluster [DBG] pgmap v1803: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 22573/76839 objects degraded (29.377%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:07:36.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:36 smithi099 bash[87147]: cluster 2024-09-13T15:07:34.972064+0000 mgr.y (mgr.26252) 1245 : cluster [DBG] pgmap v1803: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 22573/76839 objects degraded (29.377%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:07:36.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:36 smithi099 bash[87147]: cluster 2024-09-13T15:07:34.972064+0000 mgr.y (mgr.26252) 1245 : cluster [DBG] pgmap v1803: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 22573/76839 objects degraded (29.377%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T15:07:36.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:36.413 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:36.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:36.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:36.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:36.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:36.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:36.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:36.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:36.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:36.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:36.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:36.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:36.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:36.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:37.034 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:37.062 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:37.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:37.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:37.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:37.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:37.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:37.319 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:37.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:37.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:37 smithi099 bash[87147]: cluster 2024-09-13T15:07:36.546523+0000 mon.a (mon.0) 2392 : cluster [WRN] Health check update: Degraded data redundancy: 22573/76839 objects degraded (29.377%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:37.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:37 smithi099 bash[87147]: cluster 2024-09-13T15:07:36.546523+0000 mon.a (mon.0) 2392 : cluster [WRN] Health check update: Degraded data redundancy: 22573/76839 objects degraded (29.377%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:37.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:37 smithi099 bash[84576]: cluster 2024-09-13T15:07:36.546523+0000 mon.a (mon.0) 2392 : cluster [WRN] Health check update: Degraded data redundancy: 22573/76839 objects degraded (29.377%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:37.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:37 smithi099 bash[84576]: cluster 2024-09-13T15:07:36.546523+0000 mon.a (mon.0) 2392 : cluster [WRN] Health check update: Degraded data redundancy: 22573/76839 objects degraded (29.377%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:37.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:07:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:07:37] "GET /metrics HTTP/1.1" 200 36787 "" "Prometheus/2.51.0" 2024-09-13T15:07:37.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:37.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:37.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:37.550 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:37.586 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:37.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:37.611 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:37.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:37.728 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:37.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:37.839 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:37.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:37 smithi177 bash[68594]: cluster 2024-09-13T15:07:36.546523+0000 mon.a (mon.0) 2392 : cluster [WRN] Health check update: Degraded data redundancy: 22573/76839 objects degraded (29.377%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:37.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:37 smithi177 bash[68594]: cluster 2024-09-13T15:07:36.546523+0000 mon.a (mon.0) 2392 : cluster [WRN] Health check update: Degraded data redundancy: 22573/76839 objects degraded (29.377%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:37.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:37.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:37.969 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:38.004 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:38.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:38.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:38.148 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:38.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:38.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:38.277 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:38.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:38.390 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:38 smithi099 bash[84576]: cluster 2024-09-13T15:07:36.973112+0000 mgr.y (mgr.26252) 1246 : cluster [DBG] pgmap v1804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 22573/74865 objects degraded (30.152%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:38.390 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:38 smithi099 bash[84576]: cluster 2024-09-13T15:07:36.973112+0000 mgr.y (mgr.26252) 1246 : cluster [DBG] pgmap v1804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 22573/74865 objects degraded (30.152%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:38.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:38 smithi099 bash[87147]: cluster 2024-09-13T15:07:36.973112+0000 mgr.y (mgr.26252) 1246 : cluster [DBG] pgmap v1804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 22573/74865 objects degraded (30.152%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:38.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:38 smithi099 bash[87147]: cluster 2024-09-13T15:07:36.973112+0000 mgr.y (mgr.26252) 1246 : cluster [DBG] pgmap v1804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 22573/74865 objects degraded (30.152%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:38.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:38.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:38.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:38 smithi177 bash[68594]: cluster 2024-09-13T15:07:36.973112+0000 mgr.y (mgr.26252) 1246 : cluster [DBG] pgmap v1804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 22573/74865 objects degraded (30.152%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:38.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:38 smithi177 bash[68594]: cluster 2024-09-13T15:07:36.973112+0000 mgr.y (mgr.26252) 1246 : cluster [DBG] pgmap v1804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 22573/74865 objects degraded (30.152%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:07:38.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:38.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:38.644 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:38.680 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:38.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:38.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:38.814 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:38.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:38.920 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:38.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:38.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:39.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:39.111 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:39.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:39.217 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:39.253 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:39 smithi099 bash[84576]: cluster 2024-09-13T15:07:38.973843+0000 mgr.y (mgr.26252) 1247 : cluster [DBG] pgmap v1805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 22386/70425 objects degraded (31.787%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:07:39.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:39 smithi099 bash[84576]: cluster 2024-09-13T15:07:38.973843+0000 mgr.y (mgr.26252) 1247 : cluster [DBG] pgmap v1805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 22386/70425 objects degraded (31.787%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:07:39.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:39 smithi099 bash[87147]: cluster 2024-09-13T15:07:38.973843+0000 mgr.y (mgr.26252) 1247 : cluster [DBG] pgmap v1805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 22386/70425 objects degraded (31.787%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:07:39.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:39 smithi099 bash[87147]: cluster 2024-09-13T15:07:38.973843+0000 mgr.y (mgr.26252) 1247 : cluster [DBG] pgmap v1805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 22386/70425 objects degraded (31.787%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:07:39.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:39.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:39 smithi177 bash[68594]: cluster 2024-09-13T15:07:38.973843+0000 mgr.y (mgr.26252) 1247 : cluster [DBG] pgmap v1805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 22386/70425 objects degraded (31.787%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:07:39.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:39 smithi177 bash[68594]: cluster 2024-09-13T15:07:38.973843+0000 mgr.y (mgr.26252) 1247 : cluster [DBG] pgmap v1805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 22386/70425 objects degraded (31.787%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T15:07:39.389 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:39.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:39.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:39.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:39.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:39.655 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:39.764 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:39.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:39.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:39.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:39.919 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:39.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:40.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:40.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:40.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:40.152 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:40.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:40.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:40.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:40.295 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:07:40 smithi177 bash[61439]: ts=2024-09-13T15:07:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:40.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:40.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:40.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:40.441 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:40.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:40.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:40.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:40.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:40.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:40.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:40.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:40.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:40.846 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:40.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:40.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:40.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:41.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:41.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:41.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:41.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:41.255 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:41.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:41.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:41.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:41.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:41.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:41.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:41.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:41.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:41.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:41.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:41.782 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:41.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:41.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:41.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:41.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:42.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:42.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:42 smithi177 bash[68594]: cluster 2024-09-13T15:07:40.974513+0000 mgr.y (mgr.26252) 1248 : cluster [DBG] pgmap v1806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 22386/66048 objects degraded (33.894%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:42.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:42 smithi177 bash[68594]: cluster 2024-09-13T15:07:40.974513+0000 mgr.y (mgr.26252) 1248 : cluster [DBG] pgmap v1806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 22386/66048 objects degraded (33.894%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:42.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:42 smithi177 bash[68594]: cluster 2024-09-13T15:07:41.547287+0000 mon.a (mon.0) 2393 : cluster [WRN] Health check update: Degraded data redundancy: 22386/66048 objects degraded (33.894%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:42.042 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:42 smithi177 bash[68594]: cluster 2024-09-13T15:07:41.547287+0000 mon.a (mon.0) 2393 : cluster [WRN] Health check update: Degraded data redundancy: 22386/66048 objects degraded (33.894%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:42.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:42.156 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:42.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:42.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:42.216 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:42.216 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:42.295 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:42.314 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:42.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:42.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:42 smithi099 bash[84576]: cluster 2024-09-13T15:07:40.974513+0000 mgr.y (mgr.26252) 1248 : cluster [DBG] pgmap v1806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 22386/66048 objects degraded (33.894%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:42.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:42 smithi099 bash[84576]: cluster 2024-09-13T15:07:40.974513+0000 mgr.y (mgr.26252) 1248 : cluster [DBG] pgmap v1806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 22386/66048 objects degraded (33.894%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:42.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:42 smithi099 bash[84576]: cluster 2024-09-13T15:07:41.547287+0000 mon.a (mon.0) 2393 : cluster [WRN] Health check update: Degraded data redundancy: 22386/66048 objects degraded (33.894%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:42.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:42 smithi099 bash[84576]: cluster 2024-09-13T15:07:41.547287+0000 mon.a (mon.0) 2393 : cluster [WRN] Health check update: Degraded data redundancy: 22386/66048 objects degraded (33.894%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:42.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:42 smithi099 bash[87147]: cluster 2024-09-13T15:07:40.974513+0000 mgr.y (mgr.26252) 1248 : cluster [DBG] pgmap v1806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 22386/66048 objects degraded (33.894%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:42.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:42 smithi099 bash[87147]: cluster 2024-09-13T15:07:40.974513+0000 mgr.y (mgr.26252) 1248 : cluster [DBG] pgmap v1806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 22386/66048 objects degraded (33.894%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:42.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:42 smithi099 bash[87147]: cluster 2024-09-13T15:07:41.547287+0000 mon.a (mon.0) 2393 : cluster [WRN] Health check update: Degraded data redundancy: 22386/66048 objects degraded (33.894%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:42.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:42 smithi099 bash[87147]: cluster 2024-09-13T15:07:41.547287+0000 mon.a (mon.0) 2393 : cluster [WRN] Health check update: Degraded data redundancy: 22386/66048 objects degraded (33.894%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:42.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:42.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:42.552 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:42.579 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:42.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:42.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:42.700 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:42.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:42.803 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:42.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:42.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:42.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:42.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:43.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:43.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:43.194 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:43.230 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:43.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:43.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:43.377 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:43.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:43.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:07:43.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:43.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:43.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:43.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:43.653 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:43.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:43.756 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:43.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:43.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:43.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:43.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:43.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:44.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:44.072 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:44.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:44.176 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:44.184 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:44 smithi177 bash[68594]: cluster 2024-09-13T15:07:42.975506+0000 mgr.y (mgr.26252) 1249 : cluster [DBG] pgmap v1807: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 22386/62607 objects degraded (35.756%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:44.185 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:44 smithi177 bash[68594]: cluster 2024-09-13T15:07:42.975506+0000 mgr.y (mgr.26252) 1249 : cluster [DBG] pgmap v1807: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 22386/62607 objects degraded (35.756%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:44.203 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:44.307 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:44.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:44 smithi099 bash[84576]: cluster 2024-09-13T15:07:42.975506+0000 mgr.y (mgr.26252) 1249 : cluster [DBG] pgmap v1807: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 22386/62607 objects degraded (35.756%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:44.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:44 smithi099 bash[84576]: cluster 2024-09-13T15:07:42.975506+0000 mgr.y (mgr.26252) 1249 : cluster [DBG] pgmap v1807: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 22386/62607 objects degraded (35.756%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:44.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:44 smithi099 bash[87147]: cluster 2024-09-13T15:07:42.975506+0000 mgr.y (mgr.26252) 1249 : cluster [DBG] pgmap v1807: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 22386/62607 objects degraded (35.756%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:44.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:44 smithi099 bash[87147]: cluster 2024-09-13T15:07:42.975506+0000 mgr.y (mgr.26252) 1249 : cluster [DBG] pgmap v1807: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 22386/62607 objects degraded (35.756%); 1.8 MiB/s, 29 objects/s recovering 2024-09-13T15:07:44.339 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:44.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:44.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:44.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:44.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:44.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:44.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:44.744 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:44.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:44.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:44.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:44.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:45.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:45.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:45.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:45.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:45.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:45.297 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:45.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:45.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:45.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:45.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:45.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:45.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:45.678 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:45.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:45.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:45.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:45.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:45.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:45.948 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:45.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:45.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:46.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:46.127 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:46.232 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:46.256 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:46.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:46.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:46 smithi099 bash[84576]: cluster 2024-09-13T15:07:44.976151+0000 mgr.y (mgr.26252) 1250 : cluster [DBG] pgmap v1808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 22161/58857 objects degraded (37.652%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T15:07:46.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:46 smithi099 bash[84576]: cluster 2024-09-13T15:07:44.976151+0000 mgr.y (mgr.26252) 1250 : cluster [DBG] pgmap v1808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 22161/58857 objects degraded (37.652%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T15:07:46.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:46 smithi099 bash[87147]: cluster 2024-09-13T15:07:44.976151+0000 mgr.y (mgr.26252) 1250 : cluster [DBG] pgmap v1808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 22161/58857 objects degraded (37.652%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T15:07:46.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:46 smithi099 bash[87147]: cluster 2024-09-13T15:07:44.976151+0000 mgr.y (mgr.26252) 1250 : cluster [DBG] pgmap v1808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 22161/58857 objects degraded (37.652%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T15:07:46.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:46 smithi177 bash[68594]: cluster 2024-09-13T15:07:44.976151+0000 mgr.y (mgr.26252) 1250 : cluster [DBG] pgmap v1808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 22161/58857 objects degraded (37.652%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T15:07:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:46 smithi177 bash[68594]: cluster 2024-09-13T15:07:44.976151+0000 mgr.y (mgr.26252) 1250 : cluster [DBG] pgmap v1808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 22161/58857 objects degraded (37.652%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T15:07:46.389 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:46.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:46.495 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:46.531 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:46.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:46.641 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:46.675 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:46.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:46.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:46.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:46.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:46.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:46.938 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:46.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:47.044 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:47.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:47.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:47.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:47.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:47 smithi099 bash[84576]: cluster 2024-09-13T15:07:46.548085+0000 mon.a (mon.0) 2394 : cluster [WRN] Health check update: Degraded data redundancy: 22161/58857 objects degraded (37.652%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:47.178 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:47 smithi099 bash[84576]: cluster 2024-09-13T15:07:46.548085+0000 mon.a (mon.0) 2394 : cluster [WRN] Health check update: Degraded data redundancy: 22161/58857 objects degraded (37.652%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:47.178 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:07:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:07:47] "GET /metrics HTTP/1.1" 200 36787 "" "Prometheus/2.51.0" 2024-09-13T15:07:47.179 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:47 smithi099 bash[87147]: cluster 2024-09-13T15:07:46.548085+0000 mon.a (mon.0) 2394 : cluster [WRN] Health check update: Degraded data redundancy: 22161/58857 objects degraded (37.652%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:47.179 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:47 smithi099 bash[87147]: cluster 2024-09-13T15:07:46.548085+0000 mon.a (mon.0) 2394 : cluster [WRN] Health check update: Degraded data redundancy: 22161/58857 objects degraded (37.652%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:47.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.201 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:47.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:47.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:47.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:47.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:47.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:47 smithi177 bash[68594]: cluster 2024-09-13T15:07:46.548085+0000 mon.a (mon.0) 2394 : cluster [WRN] Health check update: Degraded data redundancy: 22161/58857 objects degraded (37.652%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:47.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:47 smithi177 bash[68594]: cluster 2024-09-13T15:07:46.548085+0000 mon.a (mon.0) 2394 : cluster [WRN] Health check update: Degraded data redundancy: 22161/58857 objects degraded (37.652%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:47.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:47.461 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:47.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:47.565 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:47.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:47.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:47.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:47.742 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:47.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:47.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:47.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:48.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:48.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:48.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:48.159 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:48.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:48.261 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:07:48.264 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:48.273 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:48 smithi177 bash[68594]: cluster 2024-09-13T15:07:46.976740+0000 mgr.y (mgr.26252) 1251 : cluster [DBG] pgmap v1809: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 22161/57288 objects degraded (38.683%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:07:48.273 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:48 smithi177 bash[68594]: cluster 2024-09-13T15:07:46.976740+0000 mgr.y (mgr.26252) 1251 : cluster [DBG] pgmap v1809: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 22161/57288 objects degraded (38.683%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:07:48.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:48.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:48.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:48 smithi099 bash[84576]: cluster 2024-09-13T15:07:46.976740+0000 mgr.y (mgr.26252) 1251 : cluster [DBG] pgmap v1809: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 22161/57288 objects degraded (38.683%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:07:48.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:48 smithi099 bash[84576]: cluster 2024-09-13T15:07:46.976740+0000 mgr.y (mgr.26252) 1251 : cluster [DBG] pgmap v1809: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 22161/57288 objects degraded (38.683%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:07:48.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:48 smithi099 bash[87147]: cluster 2024-09-13T15:07:46.976740+0000 mgr.y (mgr.26252) 1251 : cluster [DBG] pgmap v1809: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 22161/57288 objects degraded (38.683%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:07:48.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:48 smithi099 bash[87147]: cluster 2024-09-13T15:07:46.976740+0000 mgr.y (mgr.26252) 1251 : cluster [DBG] pgmap v1809: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 22161/57288 objects degraded (38.683%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:07:48.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:48.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:48.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:48.529 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:48.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:48.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:48.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:48.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:48.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:48.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:48.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:48.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:48.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:49.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:49.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:49.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:49.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:49.226 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:49.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:49.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:49 smithi099 bash[87147]: audit 2024-09-13T15:07:48.304510+0000 mon.a (mon.0) 2395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:49.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:49 smithi099 bash[87147]: audit 2024-09-13T15:07:48.304510+0000 mon.a (mon.0) 2395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:49.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:49 smithi099 bash[84576]: audit 2024-09-13T15:07:48.304510+0000 mon.a (mon.0) 2395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:49.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:49 smithi099 bash[84576]: audit 2024-09-13T15:07:48.304510+0000 mon.a (mon.0) 2395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:49.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:49.350 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:49.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:49 smithi177 bash[68594]: audit 2024-09-13T15:07:48.304510+0000 mon.a (mon.0) 2395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:49 smithi177 bash[68594]: audit 2024-09-13T15:07:48.304510+0000 mon.a (mon.0) 2395 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:49.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:49.478 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:49.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:49.582 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:49.612 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:49.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:49.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:49.743 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:49.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:49.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:49.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:49.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:50.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:50.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:50.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:50.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:50.190 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:50.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:50.232 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:50 smithi099 bash[84576]: cluster 2024-09-13T15:07:48.977435+0000 mgr.y (mgr.26252) 1252 : cluster [DBG] pgmap v1810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 21963/52455 objects degraded (41.870%); 3.2 MiB/s, 50 objects/s recovering 2024-09-13T15:07:50.232 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:50 smithi099 bash[84576]: cluster 2024-09-13T15:07:48.977435+0000 mgr.y (mgr.26252) 1252 : cluster [DBG] pgmap v1810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 21963/52455 objects degraded (41.870%); 3.2 MiB/s, 50 objects/s recovering 2024-09-13T15:07:50.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:50 smithi099 bash[87147]: cluster 2024-09-13T15:07:48.977435+0000 mgr.y (mgr.26252) 1252 : cluster [DBG] pgmap v1810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 21963/52455 objects degraded (41.870%); 3.2 MiB/s, 50 objects/s recovering 2024-09-13T15:07:50.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:50 smithi099 bash[87147]: cluster 2024-09-13T15:07:48.977435+0000 mgr.y (mgr.26252) 1252 : cluster [DBG] pgmap v1810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 21963/52455 objects degraded (41.870%); 3.2 MiB/s, 50 objects/s recovering 2024-09-13T15:07:50.294 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:50.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:50.323 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:50 smithi177 bash[68594]: cluster 2024-09-13T15:07:48.977435+0000 mgr.y (mgr.26252) 1252 : cluster [DBG] pgmap v1810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 21963/52455 objects degraded (41.870%); 3.2 MiB/s, 50 objects/s recovering 2024-09-13T15:07:50.323 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:50 smithi177 bash[68594]: cluster 2024-09-13T15:07:48.977435+0000 mgr.y (mgr.26252) 1252 : cluster [DBG] pgmap v1810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 21963/52455 objects degraded (41.870%); 3.2 MiB/s, 50 objects/s recovering 2024-09-13T15:07:50.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:50.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:50.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:50.540 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:50.574 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:50.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:50.614 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:07:50 smithi177 bash[61439]: ts=2024-09-13T15:07:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:07:50.680 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:50.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:50.825 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:50.859 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:50.965 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:50.993 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:50.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:51.099 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:51.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:51.235 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:51.256 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:51.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:51.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:51.383 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:51.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:51.492 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:51.520 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:51.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:51.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:51.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:51.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:51.781 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:51.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:51.886 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:51.917 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:51.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:52.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:52.046 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:52.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:52.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:52.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:52.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:52.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:52.297 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:52 smithi177 bash[68594]: cluster 2024-09-13T15:07:50.978110+0000 mgr.y (mgr.26252) 1253 : cluster [DBG] pgmap v1811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1001 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 21963/48090 objects degraded (45.671%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:52.298 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:52 smithi177 bash[68594]: cluster 2024-09-13T15:07:50.978110+0000 mgr.y (mgr.26252) 1253 : cluster [DBG] pgmap v1811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1001 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 21963/48090 objects degraded (45.671%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:52.298 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:52 smithi177 bash[68594]: cluster 2024-09-13T15:07:51.548818+0000 mon.a (mon.0) 2396 : cluster [WRN] Health check update: Degraded data redundancy: 21963/48090 objects degraded (45.671%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:52.298 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:52 smithi177 bash[68594]: cluster 2024-09-13T15:07:51.548818+0000 mon.a (mon.0) 2396 : cluster [WRN] Health check update: Degraded data redundancy: 21963/48090 objects degraded (45.671%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:52.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:52.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:52.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:52 smithi099 bash[84576]: cluster 2024-09-13T15:07:50.978110+0000 mgr.y (mgr.26252) 1253 : cluster [DBG] pgmap v1811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1001 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 21963/48090 objects degraded (45.671%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:52.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:52 smithi099 bash[84576]: cluster 2024-09-13T15:07:50.978110+0000 mgr.y (mgr.26252) 1253 : cluster [DBG] pgmap v1811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1001 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 21963/48090 objects degraded (45.671%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:52.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:52 smithi099 bash[84576]: cluster 2024-09-13T15:07:51.548818+0000 mon.a (mon.0) 2396 : cluster [WRN] Health check update: Degraded data redundancy: 21963/48090 objects degraded (45.671%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:52.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:52 smithi099 bash[84576]: cluster 2024-09-13T15:07:51.548818+0000 mon.a (mon.0) 2396 : cluster [WRN] Health check update: Degraded data redundancy: 21963/48090 objects degraded (45.671%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:52.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:52 smithi099 bash[87147]: cluster 2024-09-13T15:07:50.978110+0000 mgr.y (mgr.26252) 1253 : cluster [DBG] pgmap v1811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1001 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 21963/48090 objects degraded (45.671%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:52.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:52 smithi099 bash[87147]: cluster 2024-09-13T15:07:50.978110+0000 mgr.y (mgr.26252) 1253 : cluster [DBG] pgmap v1811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1001 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 21963/48090 objects degraded (45.671%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:52.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:52 smithi099 bash[87147]: cluster 2024-09-13T15:07:51.548818+0000 mon.a (mon.0) 2396 : cluster [WRN] Health check update: Degraded data redundancy: 21963/48090 objects degraded (45.671%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:52.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:52 smithi099 bash[87147]: cluster 2024-09-13T15:07:51.548818+0000 mon.a (mon.0) 2396 : cluster [WRN] Health check update: Degraded data redundancy: 21963/48090 objects degraded (45.671%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:52.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:52.440 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:52.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:52.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:52.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:52.680 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:52.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:52.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:52.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:52.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:52.957 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:52.984 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:52.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:53.088 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:53.125 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:53.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:53.228 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:53.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:53.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:53.375 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:53.397 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:53.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:53.503 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:53.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:53.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:53.662 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:53.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:53.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:53.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:53.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:53.929 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:54.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:54.066 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:54.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:54.194 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:54.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:54.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:54.306 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:54 smithi177 bash[68594]: cluster 2024-09-13T15:07:52.978811+0000 mgr.y (mgr.26252) 1254 : cluster [DBG] pgmap v1812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 937 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 21963/45051 objects degraded (48.751%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:54.306 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:54 smithi177 bash[68594]: cluster 2024-09-13T15:07:52.978811+0000 mgr.y (mgr.26252) 1254 : cluster [DBG] pgmap v1812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 937 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 21963/45051 objects degraded (48.751%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:54.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:54.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:54 smithi099 bash[84576]: cluster 2024-09-13T15:07:52.978811+0000 mgr.y (mgr.26252) 1254 : cluster [DBG] pgmap v1812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 937 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 21963/45051 objects degraded (48.751%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:54.330 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:54 smithi099 bash[84576]: cluster 2024-09-13T15:07:52.978811+0000 mgr.y (mgr.26252) 1254 : cluster [DBG] pgmap v1812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 937 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 21963/45051 objects degraded (48.751%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:54.331 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:54 smithi099 bash[87147]: cluster 2024-09-13T15:07:52.978811+0000 mgr.y (mgr.26252) 1254 : cluster [DBG] pgmap v1812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 937 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 21963/45051 objects degraded (48.751%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:54.331 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:54 smithi099 bash[87147]: cluster 2024-09-13T15:07:52.978811+0000 mgr.y (mgr.26252) 1254 : cluster [DBG] pgmap v1812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 937 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 21963/45051 objects degraded (48.751%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T15:07:54.373 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:07:54.373 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:07:54.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:54.432 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:07:54.432 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:07:54.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:54.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:54.492 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:07:54.492 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:07:54.523 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:54.523 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:54.543 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:07:54.543 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:07:54.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:54.577 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:07:54.577 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:07:54.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:54.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:54.604 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:07:54.605 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:07:54.633 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:07:54.634 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:07:54.670 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:07:54.670 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:07:54.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:54.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:54.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:54.844 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:54.874 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:54.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:54.978 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:55.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:55.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:55.079 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T15:07:55.079 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.0 to 0.872098 (df41) 2024-09-13T15:07:55.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:55 smithi177 bash[68594]: audit 2024-09-13T15:07:54.328594+0000 mon.a (mon.0) 2397 : audit [DBG] from='client.? 172.21.15.99:0/829585462' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:55.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:55 smithi177 bash[68594]: audit 2024-09-13T15:07:54.328594+0000 mon.a (mon.0) 2397 : audit [DBG] from='client.? 172.21.15.99:0/829585462' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:55.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:55 smithi177 bash[68594]: audit 2024-09-13T15:07:54.899690+0000 mon.c (mon.2) 97 : audit [INF] from='client.? 172.21.15.99:0/74204441' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:55 smithi177 bash[68594]: audit 2024-09-13T15:07:54.899690+0000 mon.c (mon.2) 97 : audit [INF] from='client.? 172.21.15.99:0/74204441' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:55 smithi177 bash[68594]: audit 2024-09-13T15:07:54.900073+0000 mon.a (mon.0) 2398 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:55 smithi177 bash[68594]: audit 2024-09-13T15:07:54.900073+0000 mon.a (mon.0) 2398 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:55 smithi177 bash[68594]: audit 2024-09-13T15:07:54.997859+0000 mon.c (mon.2) 98 : audit [INF] from='client.? 172.21.15.99:0/2942597987' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:55 smithi177 bash[68594]: audit 2024-09-13T15:07:54.997859+0000 mon.c (mon.2) 98 : audit [INF] from='client.? 172.21.15.99:0/2942597987' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:55 smithi177 bash[68594]: audit 2024-09-13T15:07:54.998891+0000 mon.a (mon.0) 2399 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:55 smithi177 bash[68594]: audit 2024-09-13T15:07:54.998891+0000 mon.a (mon.0) 2399 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.115 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:55.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:55.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:55.238 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:55.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:55.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:55.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:55.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:55.505 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:55.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:55.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[84576]: audit 2024-09-13T15:07:54.328594+0000 mon.a (mon.0) 2397 : audit [DBG] from='client.? 172.21.15.99:0/829585462' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[84576]: audit 2024-09-13T15:07:54.328594+0000 mon.a (mon.0) 2397 : audit [DBG] from='client.? 172.21.15.99:0/829585462' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[84576]: audit 2024-09-13T15:07:54.899690+0000 mon.c (mon.2) 97 : audit [INF] from='client.? 172.21.15.99:0/74204441' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[84576]: audit 2024-09-13T15:07:54.899690+0000 mon.c (mon.2) 97 : audit [INF] from='client.? 172.21.15.99:0/74204441' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[84576]: audit 2024-09-13T15:07:54.900073+0000 mon.a (mon.0) 2398 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[84576]: audit 2024-09-13T15:07:54.900073+0000 mon.a (mon.0) 2398 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[84576]: audit 2024-09-13T15:07:54.997859+0000 mon.c (mon.2) 98 : audit [INF] from='client.? 172.21.15.99:0/2942597987' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[84576]: audit 2024-09-13T15:07:54.997859+0000 mon.c (mon.2) 98 : audit [INF] from='client.? 172.21.15.99:0/2942597987' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[84576]: audit 2024-09-13T15:07:54.998891+0000 mon.a (mon.0) 2399 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[84576]: audit 2024-09-13T15:07:54.998891+0000 mon.a (mon.0) 2399 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[87147]: audit 2024-09-13T15:07:54.328594+0000 mon.a (mon.0) 2397 : audit [DBG] from='client.? 172.21.15.99:0/829585462' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[87147]: audit 2024-09-13T15:07:54.328594+0000 mon.a (mon.0) 2397 : audit [DBG] from='client.? 172.21.15.99:0/829585462' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[87147]: audit 2024-09-13T15:07:54.899690+0000 mon.c (mon.2) 97 : audit [INF] from='client.? 172.21.15.99:0/74204441' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[87147]: audit 2024-09-13T15:07:54.899690+0000 mon.c (mon.2) 97 : audit [INF] from='client.? 172.21.15.99:0/74204441' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[87147]: audit 2024-09-13T15:07:54.900073+0000 mon.a (mon.0) 2398 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[87147]: audit 2024-09-13T15:07:54.900073+0000 mon.a (mon.0) 2398 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086} : dispatch 2024-09-13T15:07:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[87147]: audit 2024-09-13T15:07:54.997859+0000 mon.c (mon.2) 98 : audit [INF] from='client.? 172.21.15.99:0/2942597987' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[87147]: audit 2024-09-13T15:07:54.997859+0000 mon.c (mon.2) 98 : audit [INF] from='client.? 172.21.15.99:0/2942597987' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[87147]: audit 2024-09-13T15:07:54.998891+0000 mon.a (mon.0) 2399 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:55 smithi099 bash[87147]: audit 2024-09-13T15:07:54.998891+0000 mon.a (mon.0) 2399 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:55.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:55.667 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:55.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:55.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:55.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:55.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:55.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:07:55.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:55.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:55.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:56.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:56.099 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:56.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:56.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:56.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[84576]: cluster 2024-09-13T15:07:54.979464+0000 mgr.y (mgr.26252) 1255 : cluster [DBG] pgmap v1813: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 856 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 21715/41127 objects degraded (52.800%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T15:07:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[84576]: cluster 2024-09-13T15:07:54.979464+0000 mgr.y (mgr.26252) 1255 : cluster [DBG] pgmap v1813: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 856 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 21715/41127 objects degraded (52.800%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T15:07:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[84576]: cluster 2024-09-13T15:07:55.073803+0000 mon.a (mon.0) 2400 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[84576]: cluster 2024-09-13T15:07:55.073803+0000 mon.a (mon.0) 2400 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[84576]: audit 2024-09-13T15:07:55.074967+0000 mon.a (mon.0) 2401 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086}]': finished 2024-09-13T15:07:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[84576]: audit 2024-09-13T15:07:55.074967+0000 mon.a (mon.0) 2401 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086}]': finished 2024-09-13T15:07:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[84576]: audit 2024-09-13T15:07:55.075047+0000 mon.a (mon.0) 2402 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[84576]: audit 2024-09-13T15:07:55.075047+0000 mon.a (mon.0) 2402 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[84576]: cluster 2024-09-13T15:07:55.080012+0000 mon.a (mon.0) 2403 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2024-09-13T15:07:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[84576]: cluster 2024-09-13T15:07:55.080012+0000 mon.a (mon.0) 2403 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2024-09-13T15:07:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[87147]: cluster 2024-09-13T15:07:54.979464+0000 mgr.y (mgr.26252) 1255 : cluster [DBG] pgmap v1813: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 856 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 21715/41127 objects degraded (52.800%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T15:07:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[87147]: cluster 2024-09-13T15:07:54.979464+0000 mgr.y (mgr.26252) 1255 : cluster [DBG] pgmap v1813: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 856 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 21715/41127 objects degraded (52.800%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T15:07:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[87147]: cluster 2024-09-13T15:07:55.073803+0000 mon.a (mon.0) 2400 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[87147]: cluster 2024-09-13T15:07:55.073803+0000 mon.a (mon.0) 2400 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[87147]: audit 2024-09-13T15:07:55.074967+0000 mon.a (mon.0) 2401 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086}]': finished 2024-09-13T15:07:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[87147]: audit 2024-09-13T15:07:55.074967+0000 mon.a (mon.0) 2401 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086}]': finished 2024-09-13T15:07:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[87147]: audit 2024-09-13T15:07:55.075047+0000 mon.a (mon.0) 2402 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[87147]: audit 2024-09-13T15:07:55.075047+0000 mon.a (mon.0) 2402 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[87147]: cluster 2024-09-13T15:07:55.080012+0000 mon.a (mon.0) 2403 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2024-09-13T15:07:56.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:56 smithi099 bash[87147]: cluster 2024-09-13T15:07:55.080012+0000 mon.a (mon.0) 2403 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2024-09-13T15:07:56.339 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:56 smithi177 bash[68594]: cluster 2024-09-13T15:07:54.979464+0000 mgr.y (mgr.26252) 1255 : cluster [DBG] pgmap v1813: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 856 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 21715/41127 objects degraded (52.800%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T15:07:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:56 smithi177 bash[68594]: cluster 2024-09-13T15:07:54.979464+0000 mgr.y (mgr.26252) 1255 : cluster [DBG] pgmap v1813: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 856 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 21715/41127 objects degraded (52.800%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T15:07:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:56 smithi177 bash[68594]: cluster 2024-09-13T15:07:55.073803+0000 mon.a (mon.0) 2400 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:56 smithi177 bash[68594]: cluster 2024-09-13T15:07:55.073803+0000 mon.a (mon.0) 2400 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:56 smithi177 bash[68594]: audit 2024-09-13T15:07:55.074967+0000 mon.a (mon.0) 2401 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086}]': finished 2024-09-13T15:07:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:56 smithi177 bash[68594]: audit 2024-09-13T15:07:55.074967+0000 mon.a (mon.0) 2401 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.8720984153235086}]': finished 2024-09-13T15:07:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:56 smithi177 bash[68594]: audit 2024-09-13T15:07:55.075047+0000 mon.a (mon.0) 2402 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:56.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:56 smithi177 bash[68594]: audit 2024-09-13T15:07:55.075047+0000 mon.a (mon.0) 2402 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:56.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:56 smithi177 bash[68594]: cluster 2024-09-13T15:07:55.080012+0000 mon.a (mon.0) 2403 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2024-09-13T15:07:56.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:56 smithi177 bash[68594]: cluster 2024-09-13T15:07:55.080012+0000 mon.a (mon.0) 2403 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2024-09-13T15:07:56.362 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:56.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:56.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:56.509 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:56.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:56.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:56.652 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:56.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:56.756 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:56.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:56.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:56.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:56.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:56.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:57.017 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:57.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:57.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:57.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:57.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:57.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:57.277 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:57.286 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:57 smithi177 bash[68594]: cluster 2024-09-13T15:07:56.086224+0000 mon.a (mon.0) 2404 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2024-09-13T15:07:57.286 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:57 smithi177 bash[68594]: cluster 2024-09-13T15:07:56.086224+0000 mon.a (mon.0) 2404 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2024-09-13T15:07:57.286 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:57 smithi177 bash[68594]: cluster 2024-09-13T15:07:56.549662+0000 mon.a (mon.0) 2405 : cluster [WRN] Health check update: Degraded data redundancy: 21715/41127 objects degraded (52.800%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:57.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:57 smithi177 bash[68594]: cluster 2024-09-13T15:07:56.549662+0000 mon.a (mon.0) 2405 : cluster [WRN] Health check update: Degraded data redundancy: 21715/41127 objects degraded (52.800%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:57.303 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:57.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:57.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:07:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:07:57] "GET /metrics HTTP/1.1" 200 36780 "" "Prometheus/2.51.0" 2024-09-13T15:07:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:57 smithi099 bash[84576]: cluster 2024-09-13T15:07:56.086224+0000 mon.a (mon.0) 2404 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2024-09-13T15:07:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:57 smithi099 bash[84576]: cluster 2024-09-13T15:07:56.086224+0000 mon.a (mon.0) 2404 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2024-09-13T15:07:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:57 smithi099 bash[84576]: cluster 2024-09-13T15:07:56.549662+0000 mon.a (mon.0) 2405 : cluster [WRN] Health check update: Degraded data redundancy: 21715/41127 objects degraded (52.800%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:57 smithi099 bash[84576]: cluster 2024-09-13T15:07:56.549662+0000 mon.a (mon.0) 2405 : cluster [WRN] Health check update: Degraded data redundancy: 21715/41127 objects degraded (52.800%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:57 smithi099 bash[87147]: cluster 2024-09-13T15:07:56.086224+0000 mon.a (mon.0) 2404 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2024-09-13T15:07:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:57 smithi099 bash[87147]: cluster 2024-09-13T15:07:56.086224+0000 mon.a (mon.0) 2404 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2024-09-13T15:07:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:57 smithi099 bash[87147]: cluster 2024-09-13T15:07:56.549662+0000 mon.a (mon.0) 2405 : cluster [WRN] Health check update: Degraded data redundancy: 21715/41127 objects degraded (52.800%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:57 smithi099 bash[87147]: cluster 2024-09-13T15:07:56.549662+0000 mon.a (mon.0) 2405 : cluster [WRN] Health check update: Degraded data redundancy: 21715/41127 objects degraded (52.800%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:07:57.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:57.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:57.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:57.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:57.558 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:57.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:57.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:57.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:57.810 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:57.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:57.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:57.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:57.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:58.087 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:58.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:58.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:58.161 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:58 smithi099 bash[84576]: cluster 2024-09-13T15:07:56.979983+0000 mgr.y (mgr.26252) 1256 : cluster [DBG] pgmap v1816: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 819 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 21715/39384 objects degraded (55.137%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:58.161 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:58 smithi099 bash[84576]: cluster 2024-09-13T15:07:56.979983+0000 mgr.y (mgr.26252) 1256 : cluster [DBG] pgmap v1816: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 819 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 21715/39384 objects degraded (55.137%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:58.161 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:58 smithi099 bash[84576]: cluster 2024-09-13T15:07:57.091569+0000 mon.a (mon.0) 2406 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:07:58.161 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:07:58 smithi099 bash[84576]: cluster 2024-09-13T15:07:57.091569+0000 mon.a (mon.0) 2406 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:07:58.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:58.233 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:58.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:58.336 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:58.361 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:58.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:58 smithi177 bash[68594]: cluster 2024-09-13T15:07:56.979983+0000 mgr.y (mgr.26252) 1256 : cluster [DBG] pgmap v1816: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 819 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 21715/39384 objects degraded (55.137%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:58.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:58 smithi177 bash[68594]: cluster 2024-09-13T15:07:56.979983+0000 mgr.y (mgr.26252) 1256 : cluster [DBG] pgmap v1816: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 819 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 21715/39384 objects degraded (55.137%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:58.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:58 smithi177 bash[68594]: cluster 2024-09-13T15:07:57.091569+0000 mon.a (mon.0) 2406 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:07:58.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:07:58 smithi177 bash[68594]: cluster 2024-09-13T15:07:57.091569+0000 mon.a (mon.0) 2406 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:07:58.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:58.386 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:58 smithi099 bash[87147]: cluster 2024-09-13T15:07:56.979983+0000 mgr.y (mgr.26252) 1256 : cluster [DBG] pgmap v1816: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 819 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 21715/39384 objects degraded (55.137%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:58.387 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:58 smithi099 bash[87147]: cluster 2024-09-13T15:07:56.979983+0000 mgr.y (mgr.26252) 1256 : cluster [DBG] pgmap v1816: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 819 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 21715/39384 objects degraded (55.137%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:07:58.387 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:58 smithi099 bash[87147]: cluster 2024-09-13T15:07:57.091569+0000 mon.a (mon.0) 2406 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:07:58.387 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:07:58 smithi099 bash[87147]: cluster 2024-09-13T15:07:57.091569+0000 mon.a (mon.0) 2406 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:07:58.466 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:58.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:58.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:58.588 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:58.626 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:58.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:58.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:58.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:58.889 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:58.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:58.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:59.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:59.134 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:59.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:59.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:59.307 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:59.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:59.413 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:59.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:59.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:59.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:59.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:59.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:59.670 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:59.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:59.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:59.797 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:59.827 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:59.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:59.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:59.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:59.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:00.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:00.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:00.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:00.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:00.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:00.237 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:00 smithi099 bash[84576]: cluster 2024-09-13T15:07:58.980663+0000 mgr.y (mgr.26252) 1257 : cluster [DBG] pgmap v1817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 725 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 21441/34869 objects degraded (61.490%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:00.237 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:00 smithi099 bash[84576]: cluster 2024-09-13T15:07:58.980663+0000 mgr.y (mgr.26252) 1257 : cluster [DBG] pgmap v1817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 725 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 21441/34869 objects degraded (61.490%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:00.238 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:00 smithi099 bash[87147]: cluster 2024-09-13T15:07:58.980663+0000 mgr.y (mgr.26252) 1257 : cluster [DBG] pgmap v1817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 725 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 21441/34869 objects degraded (61.490%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:00.238 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:00 smithi099 bash[87147]: cluster 2024-09-13T15:07:58.980663+0000 mgr.y (mgr.26252) 1257 : cluster [DBG] pgmap v1817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 725 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 21441/34869 objects degraded (61.490%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:00.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:00.340 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:00.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:08:00 smithi177 bash[61439]: ts=2024-09-13T15:08:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:00 smithi177 bash[68594]: cluster 2024-09-13T15:07:58.980663+0000 mgr.y (mgr.26252) 1257 : cluster [DBG] pgmap v1817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 725 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 21441/34869 objects degraded (61.490%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:00 smithi177 bash[68594]: cluster 2024-09-13T15:07:58.980663+0000 mgr.y (mgr.26252) 1257 : cluster [DBG] pgmap v1817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 725 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 21441/34869 objects degraded (61.490%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:00.365 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:00.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:00.471 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:00.501 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:00.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:00.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:00.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:00.682 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:00.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:00.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:00.877 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:00.907 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:00.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:01.011 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:01.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:01.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:01.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:01.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:01.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:01.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:08:01.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:01.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:01.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:01.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:01 smithi177 bash[68594]: cluster 2024-09-13T15:08:00.981335+0000 mgr.y (mgr.26252) 1258 : cluster [DBG] pgmap v1818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 635 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 21441/30537 objects degraded (70.213%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:01.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:01 smithi177 bash[68594]: cluster 2024-09-13T15:08:00.981335+0000 mgr.y (mgr.26252) 1258 : cluster [DBG] pgmap v1818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 635 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 21441/30537 objects degraded (70.213%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:01.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:01 smithi177 bash[68594]: cluster 2024-09-13T15:08:01.105831+0000 mon.a (mon.0) 2407 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:01.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:01 smithi177 bash[68594]: cluster 2024-09-13T15:08:01.105831+0000 mon.a (mon.0) 2407 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:01.441 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:01.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:01.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:01.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:01 smithi099 bash[84576]: cluster 2024-09-13T15:08:00.981335+0000 mgr.y (mgr.26252) 1258 : cluster [DBG] pgmap v1818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 635 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 21441/30537 objects degraded (70.213%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:01.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:01 smithi099 bash[84576]: cluster 2024-09-13T15:08:00.981335+0000 mgr.y (mgr.26252) 1258 : cluster [DBG] pgmap v1818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 635 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 21441/30537 objects degraded (70.213%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:01.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:01 smithi099 bash[84576]: cluster 2024-09-13T15:08:01.105831+0000 mon.a (mon.0) 2407 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:01.570 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:01 smithi099 bash[84576]: cluster 2024-09-13T15:08:01.105831+0000 mon.a (mon.0) 2407 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:01.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:01 smithi099 bash[87147]: cluster 2024-09-13T15:08:00.981335+0000 mgr.y (mgr.26252) 1258 : cluster [DBG] pgmap v1818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 635 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 21441/30537 objects degraded (70.213%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:01.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:01 smithi099 bash[87147]: cluster 2024-09-13T15:08:00.981335+0000 mgr.y (mgr.26252) 1258 : cluster [DBG] pgmap v1818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 635 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 21441/30537 objects degraded (70.213%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T15:08:01.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:01 smithi099 bash[87147]: cluster 2024-09-13T15:08:01.105831+0000 mon.a (mon.0) 2407 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:01.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:01 smithi099 bash[87147]: cluster 2024-09-13T15:08:01.105831+0000 mon.a (mon.0) 2407 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:01.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:01.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:01.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:01.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:01.812 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:01.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:01.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:01.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:01.969 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:01.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:02.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:02.095 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:02.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:02.198 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:02.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:02.239 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:02 smithi177 bash[68594]: cluster 2024-09-13T15:08:01.550495+0000 mon.a (mon.0) 2408 : cluster [WRN] Health check update: Degraded data redundancy: 21441/30537 objects degraded (70.213%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:02.239 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:02 smithi177 bash[68594]: cluster 2024-09-13T15:08:01.550495+0000 mon.a (mon.0) 2408 : cluster [WRN] Health check update: Degraded data redundancy: 21441/30537 objects degraded (70.213%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:02.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:08:02.340 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:02.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:02 smithi099 bash[84576]: cluster 2024-09-13T15:08:01.550495+0000 mon.a (mon.0) 2408 : cluster [WRN] Health check update: Degraded data redundancy: 21441/30537 objects degraded (70.213%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:02.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:02 smithi099 bash[84576]: cluster 2024-09-13T15:08:01.550495+0000 mon.a (mon.0) 2408 : cluster [WRN] Health check update: Degraded data redundancy: 21441/30537 objects degraded (70.213%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:02.342 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:02 smithi099 bash[87147]: cluster 2024-09-13T15:08:01.550495+0000 mon.a (mon.0) 2408 : cluster [WRN] Health check update: Degraded data redundancy: 21441/30537 objects degraded (70.213%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:02.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:02 smithi099 bash[87147]: cluster 2024-09-13T15:08:01.550495+0000 mon.a (mon.0) 2408 : cluster [WRN] Health check update: Degraded data redundancy: 21441/30537 objects degraded (70.213%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:02.370 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:02.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:02.474 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:02.494 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:02.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:02.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:02.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:02.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:02.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:02.774 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:02.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:02.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:02.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:02.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:03.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:03.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:03.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:03.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:03.186 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:03.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:03.289 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:03.298 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:03 smithi099 bash[84576]: cluster 2024-09-13T15:08:02.982024+0000 mgr.y (mgr.26252) 1259 : cluster [DBG] pgmap v1819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 572 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 21441/27528 objects degraded (77.888%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:08:03.298 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:03 smithi099 bash[84576]: cluster 2024-09-13T15:08:02.982024+0000 mgr.y (mgr.26252) 1259 : cluster [DBG] pgmap v1819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 572 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 21441/27528 objects degraded (77.888%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:08:03.299 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:03 smithi099 bash[87147]: cluster 2024-09-13T15:08:02.982024+0000 mgr.y (mgr.26252) 1259 : cluster [DBG] pgmap v1819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 572 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 21441/27528 objects degraded (77.888%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:08:03.299 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:03 smithi099 bash[87147]: cluster 2024-09-13T15:08:02.982024+0000 mgr.y (mgr.26252) 1259 : cluster [DBG] pgmap v1819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 572 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 21441/27528 objects degraded (77.888%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:08:03.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:03.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:03.436 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:03.477 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:03.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:03.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:03.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:03 smithi177 bash[68594]: cluster 2024-09-13T15:08:02.982024+0000 mgr.y (mgr.26252) 1259 : cluster [DBG] pgmap v1819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 572 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 21441/27528 objects degraded (77.888%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:08:03.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:03 smithi177 bash[68594]: cluster 2024-09-13T15:08:02.982024+0000 mgr.y (mgr.26252) 1259 : cluster [DBG] pgmap v1819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 572 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 21441/27528 objects degraded (77.888%); 2.1 MiB/s, 34 objects/s recovering 2024-09-13T15:08:03.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:03.711 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:03.734 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:03.840 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:03.875 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:03.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:03.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:04.008 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:04.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:04.111 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:04.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:04.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:04.253 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:04.266 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:04 smithi177 bash[68594]: audit 2024-09-13T15:08:03.304186+0000 mon.a (mon.0) 2409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:04.267 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:04 smithi177 bash[68594]: audit 2024-09-13T15:08:03.304186+0000 mon.a (mon.0) 2409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:04.291 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:04.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:04.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:04 smithi099 bash[84576]: audit 2024-09-13T15:08:03.304186+0000 mon.a (mon.0) 2409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:04.327 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:04 smithi099 bash[84576]: audit 2024-09-13T15:08:03.304186+0000 mon.a (mon.0) 2409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:04.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:04 smithi099 bash[87147]: audit 2024-09-13T15:08:03.304186+0000 mon.a (mon.0) 2409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:04.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:04 smithi099 bash[87147]: audit 2024-09-13T15:08:03.304186+0000 mon.a (mon.0) 2409 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:04.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:04.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:04.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:04.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.572 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:04.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:04.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:04.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:04.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:04.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:04.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:04.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:04.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:05.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:05.106 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:05.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:05.212 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:05.238 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:05.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:05.342 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:05.351 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 6, 7, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:08:05.351 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:08:05.351 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 2, 3, 5, 6, 7, 4, 1] 2024-09-13T15:08:05.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd out 6 2024-09-13T15:08:05.379 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:05.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:05.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:05.516 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:05 smithi099 bash[84576]: cluster 2024-09-13T15:08:04.982555+0000 mgr.y (mgr.26252) 1260 : cluster [DBG] pgmap v1820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21158/23946 objects degraded (88.357%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T15:08:05.517 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:05 smithi099 bash[84576]: cluster 2024-09-13T15:08:04.982555+0000 mgr.y (mgr.26252) 1260 : cluster [DBG] pgmap v1820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21158/23946 objects degraded (88.357%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T15:08:05.517 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:05 smithi099 bash[87147]: cluster 2024-09-13T15:08:04.982555+0000 mgr.y (mgr.26252) 1260 : cluster [DBG] pgmap v1820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21158/23946 objects degraded (88.357%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T15:08:05.517 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:05 smithi099 bash[87147]: cluster 2024-09-13T15:08:04.982555+0000 mgr.y (mgr.26252) 1260 : cluster [DBG] pgmap v1820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21158/23946 objects degraded (88.357%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T15:08:05.564 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:05.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:05.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:05.709 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:05.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:05.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:05.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:05 smithi177 bash[68594]: cluster 2024-09-13T15:08:04.982555+0000 mgr.y (mgr.26252) 1260 : cluster [DBG] pgmap v1820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21158/23946 objects degraded (88.357%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T15:08:05.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:05 smithi177 bash[68594]: cluster 2024-09-13T15:08:04.982555+0000 mgr.y (mgr.26252) 1260 : cluster [DBG] pgmap v1820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21158/23946 objects degraded (88.357%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T15:08:05.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:05.984 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:05.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:06.062 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:06.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:06.111 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:06.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:06.217 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:06.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.252 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:06.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:06.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:06.376 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:06.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:06.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:06.513 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:06.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:06.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:06.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:06.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:06.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:06.756 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:06 smithi099 bash[87147]: cluster 2024-09-13T15:08:06.551261+0000 mon.a (mon.0) 2410 : cluster [WRN] Health check update: Degraded data redundancy: 21158/23946 objects degraded (88.357%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:06.756 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:06 smithi099 bash[87147]: cluster 2024-09-13T15:08:06.551261+0000 mon.a (mon.0) 2410 : cluster [WRN] Health check update: Degraded data redundancy: 21158/23946 objects degraded (88.357%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:06.757 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:06 smithi099 bash[84576]: cluster 2024-09-13T15:08:06.551261+0000 mon.a (mon.0) 2410 : cluster [WRN] Health check update: Degraded data redundancy: 21158/23946 objects degraded (88.357%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:06.757 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:06 smithi099 bash[84576]: cluster 2024-09-13T15:08:06.551261+0000 mon.a (mon.0) 2410 : cluster [WRN] Health check update: Degraded data redundancy: 21158/23946 objects degraded (88.357%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:06.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:06.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:06.797 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:06 smithi177 bash[68594]: cluster 2024-09-13T15:08:06.551261+0000 mon.a (mon.0) 2410 : cluster [WRN] Health check update: Degraded data redundancy: 21158/23946 objects degraded (88.357%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:06.797 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:06 smithi177 bash[68594]: cluster 2024-09-13T15:08:06.551261+0000 mon.a (mon.0) 2410 : cluster [WRN] Health check update: Degraded data redundancy: 21158/23946 objects degraded (88.357%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:06.875 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:06.904 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:06.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:07.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:07.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:07.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:07.082 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:07.089 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:08:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:08:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45764.0:209179 44.a e1073/1056 44:52b43cb4:::benchmark_data_smithi099_188632_object101975:head v 1073'13183, mlcod=1073'13180)", 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:07.076897+0000", 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.013109176, 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01314338, 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.076897+0000", 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:08:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.076897+0000", 2024-09-13T15:08:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:08:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:08:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.076896+0000", 2024-09-13T15:08:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:08:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:08:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.076903+0000", 2024-09-13T15:08:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9569999999999996e-06 2024-09-13T15:08:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:08:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.076921+0000", 2024-09-13T15:08:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7096e-05 2024-09-13T15:08:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.076926+0000", 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.682e-06 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.080904+0000", 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003978227 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.080923+0000", 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8389999999999998e-05 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:08:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45764.0:209190 44.a e1073/1056)", 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:07.086919+0000", 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0030870160000000002, 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031780509999999999, 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "queued for pg", 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.086919+0000", 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:08:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.086919+0000", 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.086918+0000", 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.086926+0000", 2024-09-13T15:08:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1239999999999999e-06 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.086943+0000", 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7694999999999999e-05 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:07.086949+0000", 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0839999999999997e-06 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:08:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:08:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:08:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2024-09-13T15:08:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 2 2024-09-13T15:08:07.097 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:08:07.151 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:07.174 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:07.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:07.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:07.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:07.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:07.310 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:08:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:08:07] "GET /metrics HTTP/1.1" 200 36795 "" "Prometheus/2.51.0" 2024-09-13T15:08:07.411 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:07.441 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:07.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:07.546 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:07.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:07.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:07.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:07.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:07.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:07.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:07.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:07.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:07.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:07.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:07.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:08.048 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:07 smithi099 bash[84576]: cluster 2024-09-13T15:08:06.983485+0000 mgr.y (mgr.26252) 1261 : cluster [DBG] pgmap v1821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 465 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 21158/22374 objects degraded (94.565%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:08:08.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:07 smithi099 bash[84576]: cluster 2024-09-13T15:08:06.983485+0000 mgr.y (mgr.26252) 1261 : cluster [DBG] pgmap v1821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 465 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 21158/22374 objects degraded (94.565%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:08:08.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:07 smithi099 bash[87147]: cluster 2024-09-13T15:08:06.983485+0000 mgr.y (mgr.26252) 1261 : cluster [DBG] pgmap v1821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 465 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 21158/22374 objects degraded (94.565%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:08:08.049 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:07 smithi099 bash[87147]: cluster 2024-09-13T15:08:06.983485+0000 mgr.y (mgr.26252) 1261 : cluster [DBG] pgmap v1821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 465 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 21158/22374 objects degraded (94.565%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:08:08.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:08.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:08.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:08.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:08.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.258 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:08.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:08.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:07 smithi177 bash[68594]: cluster 2024-09-13T15:08:06.983485+0000 mgr.y (mgr.26252) 1261 : cluster [DBG] pgmap v1821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 465 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 21158/22374 objects degraded (94.565%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:08:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:07 smithi177 bash[68594]: cluster 2024-09-13T15:08:06.983485+0000 mgr.y (mgr.26252) 1261 : cluster [DBG] pgmap v1821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 465 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 21158/22374 objects degraded (94.565%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:08:08.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:08:08.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:08.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:08.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:08.527 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:08.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:08.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:08.670 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:08.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:08.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:08.792 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:08.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:08.895 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:08.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:08.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:09.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:09.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:09.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:09.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:09.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:09.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:09.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:09.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:09.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:09.399 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:09.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:09.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:09.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:09.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:09.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:09.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:09.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:09.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:09.803 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:09.854 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:09.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:09.962 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:09.984 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:09.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:10.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:10 smithi099 bash[84576]: cluster 2024-09-13T15:08:08.984613+0000 mgr.y (mgr.26252) 1262 : cluster [DBG] pgmap v1822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 341 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 21108/16428 objects degraded (128.488%); 3.2 MiB/s, 66 objects/s recovering 2024-09-13T15:08:10.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:10 smithi099 bash[84576]: cluster 2024-09-13T15:08:08.984613+0000 mgr.y (mgr.26252) 1262 : cluster [DBG] pgmap v1822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 341 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 21108/16428 objects degraded (128.488%); 3.2 MiB/s, 66 objects/s recovering 2024-09-13T15:08:10.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:10 smithi099 bash[87147]: cluster 2024-09-13T15:08:08.984613+0000 mgr.y (mgr.26252) 1262 : cluster [DBG] pgmap v1822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 341 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 21108/16428 objects degraded (128.488%); 3.2 MiB/s, 66 objects/s recovering 2024-09-13T15:08:10.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:10 smithi099 bash[87147]: cluster 2024-09-13T15:08:08.984613+0000 mgr.y (mgr.26252) 1262 : cluster [DBG] pgmap v1822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 341 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 21108/16428 objects degraded (128.488%); 3.2 MiB/s, 66 objects/s recovering 2024-09-13T15:08:10.093 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:10.123 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:10.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:10.136 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:10.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:10.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:10.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:10 smithi177 bash[68594]: cluster 2024-09-13T15:08:08.984613+0000 mgr.y (mgr.26252) 1262 : cluster [DBG] pgmap v1822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 341 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 21108/16428 objects degraded (128.488%); 3.2 MiB/s, 66 objects/s recovering 2024-09-13T15:08:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:10 smithi177 bash[68594]: cluster 2024-09-13T15:08:08.984613+0000 mgr.y (mgr.26252) 1262 : cluster [DBG] pgmap v1822: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 341 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 21108/16428 objects degraded (128.488%); 3.2 MiB/s, 66 objects/s recovering 2024-09-13T15:08:10.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:08:10 smithi177 bash[61439]: ts=2024-09-13T15:08:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:10.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:10.396 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:10.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:10.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:10.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:10.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:10.641 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:10.670 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:10.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:10.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:10.804 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:10.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:10.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:10.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:11.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:11.075 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:11.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:11.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:11.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:11.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:11.322 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:11.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:11.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:11.452 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:11.484 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:11.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:11.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:11.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:11.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:11.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:11.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:11.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:11.917 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:11.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:12.021 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:12.042 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:12.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:12.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:12.175 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:12.279 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:12.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:12.305 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:12 smithi177 bash[68594]: cluster 2024-09-13T15:08:10.985570+0000 mgr.y (mgr.26252) 1263 : cluster [DBG] pgmap v1823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 269 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 21108/12978 objects degraded (162.644%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:12.306 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:12 smithi177 bash[68594]: cluster 2024-09-13T15:08:10.985570+0000 mgr.y (mgr.26252) 1263 : cluster [DBG] pgmap v1823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 269 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 21108/12978 objects degraded (162.644%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:12.306 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:12 smithi177 bash[68594]: cluster 2024-09-13T15:08:11.552085+0000 mon.a (mon.0) 2411 : cluster [WRN] Health check update: Degraded data redundancy: 21108/12978 objects degraded (162.644%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:12.306 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:12 smithi177 bash[68594]: cluster 2024-09-13T15:08:11.552085+0000 mon.a (mon.0) 2411 : cluster [WRN] Health check update: Degraded data redundancy: 21108/12978 objects degraded (162.644%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:12.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:12.408 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:12.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:12.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:12.431 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:12 smithi099 bash[84576]: cluster 2024-09-13T15:08:10.985570+0000 mgr.y (mgr.26252) 1263 : cluster [DBG] pgmap v1823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 269 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 21108/12978 objects degraded (162.644%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:12.431 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:12 smithi099 bash[84576]: cluster 2024-09-13T15:08:10.985570+0000 mgr.y (mgr.26252) 1263 : cluster [DBG] pgmap v1823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 269 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 21108/12978 objects degraded (162.644%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:12.431 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:12 smithi099 bash[84576]: cluster 2024-09-13T15:08:11.552085+0000 mon.a (mon.0) 2411 : cluster [WRN] Health check update: Degraded data redundancy: 21108/12978 objects degraded (162.644%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:12.431 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:12 smithi099 bash[84576]: cluster 2024-09-13T15:08:11.552085+0000 mon.a (mon.0) 2411 : cluster [WRN] Health check update: Degraded data redundancy: 21108/12978 objects degraded (162.644%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:12.432 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:12 smithi099 bash[87147]: cluster 2024-09-13T15:08:10.985570+0000 mgr.y (mgr.26252) 1263 : cluster [DBG] pgmap v1823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 269 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 21108/12978 objects degraded (162.644%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:12.432 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:12 smithi099 bash[87147]: cluster 2024-09-13T15:08:10.985570+0000 mgr.y (mgr.26252) 1263 : cluster [DBG] pgmap v1823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 269 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 21108/12978 objects degraded (162.644%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:12.432 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:12 smithi099 bash[87147]: cluster 2024-09-13T15:08:11.552085+0000 mon.a (mon.0) 2411 : cluster [WRN] Health check update: Degraded data redundancy: 21108/12978 objects degraded (162.644%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:12.432 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:12 smithi099 bash[87147]: cluster 2024-09-13T15:08:11.552085+0000 mon.a (mon.0) 2411 : cluster [WRN] Health check update: Degraded data redundancy: 21108/12978 objects degraded (162.644%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:12.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:12.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:12.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:12.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:12.677 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:12.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:12.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:12.821 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:12.934 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:12.952 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:12.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:13.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:13.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:13.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:13.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:13.188 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:13.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:13.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:13.327 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:13.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:13.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:13.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:13.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:13.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:13.591 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:13.616 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:13.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:13.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:13.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:13.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:13.863 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:13.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:13.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:13.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:14.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:14.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:14.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:14 smithi099 bash[84576]: cluster 2024-09-13T15:08:12.986435+0000 mgr.y (mgr.26252) 1264 : cluster [DBG] pgmap v1824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 202 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 21108/9756 objects degraded (216.359%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:14.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:14 smithi099 bash[84576]: cluster 2024-09-13T15:08:12.986435+0000 mgr.y (mgr.26252) 1264 : cluster [DBG] pgmap v1824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 202 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 21108/9756 objects degraded (216.359%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:14.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:14 smithi099 bash[87147]: cluster 2024-09-13T15:08:12.986435+0000 mgr.y (mgr.26252) 1264 : cluster [DBG] pgmap v1824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 202 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 21108/9756 objects degraded (216.359%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:14.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:14 smithi099 bash[87147]: cluster 2024-09-13T15:08:12.986435+0000 mgr.y (mgr.26252) 1264 : cluster [DBG] pgmap v1824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 202 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 21108/9756 objects degraded (216.359%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:14.118 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:14.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:14.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:14.260 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:14.269 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:14 smithi177 bash[68594]: cluster 2024-09-13T15:08:12.986435+0000 mgr.y (mgr.26252) 1264 : cluster [DBG] pgmap v1824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 202 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 21108/9756 objects degraded (216.359%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:14.270 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:14 smithi177 bash[68594]: cluster 2024-09-13T15:08:12.986435+0000 mgr.y (mgr.26252) 1264 : cluster [DBG] pgmap v1824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 202 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 21108/9756 objects degraded (216.359%); 1.7 MiB/s, 43 objects/s recovering 2024-09-13T15:08:14.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.285 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:14.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:14.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:14.410 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:14.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:14.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:14.535 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:14.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:14.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:14.665 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:14.768 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:14.797 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:14.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:14.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:14.927 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:14.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:15.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:15.067 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:15.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:15.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:15.201 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:15.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:15.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:15.334 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:15.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:15.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:15.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:15.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:15.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:15.605 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:15.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:15.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:15.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:15.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:15.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:15.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:15.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:16.017 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:16.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:16.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:16.130 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:16 smithi099 bash[84576]: cluster 2024-09-13T15:08:14.987418+0000 mgr.y (mgr.26252) 1265 : cluster [DBG] pgmap v1825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 97 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 21108/4701 objects degraded (449.011%); 1.7 MiB/s, 65 objects/s recovering 2024-09-13T15:08:16.131 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:16 smithi099 bash[84576]: cluster 2024-09-13T15:08:14.987418+0000 mgr.y (mgr.26252) 1265 : cluster [DBG] pgmap v1825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 97 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 21108/4701 objects degraded (449.011%); 1.7 MiB/s, 65 objects/s recovering 2024-09-13T15:08:16.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:16 smithi099 bash[87147]: cluster 2024-09-13T15:08:14.987418+0000 mgr.y (mgr.26252) 1265 : cluster [DBG] pgmap v1825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 97 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 21108/4701 objects degraded (449.011%); 1.7 MiB/s, 65 objects/s recovering 2024-09-13T15:08:16.131 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:16 smithi099 bash[87147]: cluster 2024-09-13T15:08:14.987418+0000 mgr.y (mgr.26252) 1265 : cluster [DBG] pgmap v1825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 97 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 21108/4701 objects degraded (449.011%); 1.7 MiB/s, 65 objects/s recovering 2024-09-13T15:08:16.144 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:16.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:16.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:16.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:16.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:16.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:16 smithi177 bash[68594]: cluster 2024-09-13T15:08:14.987418+0000 mgr.y (mgr.26252) 1265 : cluster [DBG] pgmap v1825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 97 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 21108/4701 objects degraded (449.011%); 1.7 MiB/s, 65 objects/s recovering 2024-09-13T15:08:16.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:16 smithi177 bash[68594]: cluster 2024-09-13T15:08:14.987418+0000 mgr.y (mgr.26252) 1265 : cluster [DBG] pgmap v1825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 97 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 21108/4701 objects degraded (449.011%); 1.7 MiB/s, 65 objects/s recovering 2024-09-13T15:08:16.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:16.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:16.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:16.413 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Removed 107201 objects 2024-09-13T15:08:16.414 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Clean up completed and total clean up time :178.03 2024-09-13T15:08:16.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:16.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:16.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:16.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:16.658 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:16.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:16.761 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:16.780 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:16.883 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:16.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:16.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:17.012 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:17.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:17.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:17.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:17.271 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:17.284 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:17 smithi177 bash[68594]: cluster 2024-09-13T15:08:16.552749+0000 mon.a (mon.0) 2412 : cluster [WRN] Health check update: Degraded data redundancy: 21108/4701 objects degraded (449.011%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:17.285 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:17 smithi177 bash[68594]: cluster 2024-09-13T15:08:16.552749+0000 mon.a (mon.0) 2412 : cluster [WRN] Health check update: Degraded data redundancy: 21108/4701 objects degraded (449.011%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:17.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:17.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:17.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:08:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:08:17] "GET /metrics HTTP/1.1" 200 36795 "" "Prometheus/2.51.0" 2024-09-13T15:08:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:17 smithi099 bash[84576]: cluster 2024-09-13T15:08:16.552749+0000 mon.a (mon.0) 2412 : cluster [WRN] Health check update: Degraded data redundancy: 21108/4701 objects degraded (449.011%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:17.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:17 smithi099 bash[84576]: cluster 2024-09-13T15:08:16.552749+0000 mon.a (mon.0) 2412 : cluster [WRN] Health check update: Degraded data redundancy: 21108/4701 objects degraded (449.011%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:17 smithi099 bash[87147]: cluster 2024-09-13T15:08:16.552749+0000 mon.a (mon.0) 2412 : cluster [WRN] Health check update: Degraded data redundancy: 21108/4701 objects degraded (449.011%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:17 smithi099 bash[87147]: cluster 2024-09-13T15:08:16.552749+0000 mon.a (mon.0) 2412 : cluster [WRN] Health check update: Degraded data redundancy: 21108/4701 objects degraded (449.011%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:17.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:17.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:17.531 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:17.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:17.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:17.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:17.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:17.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:17.814 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:17.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:17.917 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:17.944 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:17.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:18.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:18.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:18.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:18.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:18.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:18.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:18.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:18.308 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:18 smithi099 bash[87147]: cluster 2024-09-13T15:08:16.988248+0000 mgr.y (mgr.26252) 1266 : cluster [DBG] pgmap v1826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 86 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 21108/4203 objects degraded (502.213%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:18.308 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:18 smithi099 bash[87147]: cluster 2024-09-13T15:08:16.988248+0000 mgr.y (mgr.26252) 1266 : cluster [DBG] pgmap v1826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 86 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 21108/4203 objects degraded (502.213%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:18.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:18 smithi099 bash[84576]: cluster 2024-09-13T15:08:16.988248+0000 mgr.y (mgr.26252) 1266 : cluster [DBG] pgmap v1826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 86 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 21108/4203 objects degraded (502.213%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:18.309 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:18 smithi099 bash[84576]: cluster 2024-09-13T15:08:16.988248+0000 mgr.y (mgr.26252) 1266 : cluster [DBG] pgmap v1826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 86 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 21108/4203 objects degraded (502.213%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:18.319 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:18.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:18 smithi177 bash[68594]: cluster 2024-09-13T15:08:16.988248+0000 mgr.y (mgr.26252) 1266 : cluster [DBG] pgmap v1826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 86 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 21108/4203 objects degraded (502.213%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:18 smithi177 bash[68594]: cluster 2024-09-13T15:08:16.988248+0000 mgr.y (mgr.26252) 1266 : cluster [DBG] pgmap v1826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 86 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 21108/4203 objects degraded (502.213%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:18.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:18.449 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:18.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:18.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:18.575 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:18.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:18.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:18.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:18.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:18.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:18.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:18.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:18.940 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:18.963 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:18.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:19.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:19.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:19.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:19.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:19.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:19 smithi177 bash[68594]: audit 2024-09-13T15:08:18.305197+0000 mon.a (mon.0) 2413 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:19.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:19 smithi177 bash[68594]: audit 2024-09-13T15:08:18.305197+0000 mon.a (mon.0) 2413 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:19.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:19.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:19.252 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:08:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:08:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:08:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:08:19.252 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:08:19.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:19 smithi099 bash[84576]: audit 2024-09-13T15:08:18.305197+0000 mon.a (mon.0) 2413 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:19 smithi099 bash[84576]: audit 2024-09-13T15:08:18.305197+0000 mon.a (mon.0) 2413 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:19 smithi099 bash[87147]: audit 2024-09-13T15:08:18.305197+0000 mon.a (mon.0) 2413 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:19 smithi099 bash[87147]: audit 2024-09-13T15:08:18.305197+0000 mon.a (mon.0) 2413 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:19.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:19.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:19.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:19.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:19.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:19.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:19.592 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:19.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:19.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:19.721 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:19.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:19.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:19.850 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:19.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:19.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:19.980 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:19.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:20.083 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:20.111 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:20.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:20.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:08:20.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:20.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:20.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:20 smithi177 bash[68594]: cluster 2024-09-13T15:08:18.989364+0000 mgr.y (mgr.26252) 1267 : cluster [DBG] pgmap v1827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 31 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21108/1542 objects degraded (1368.872%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:20.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:20 smithi177 bash[68594]: cluster 2024-09-13T15:08:18.989364+0000 mgr.y (mgr.26252) 1267 : cluster [DBG] pgmap v1827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 31 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21108/1542 objects degraded (1368.872%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:20.245 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:08:20 smithi177 bash[61439]: ts=2024-09-13T15:08:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:20.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:20.347 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:20.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:20.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:20.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:20.491 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:20.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:20.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:20 smithi099 bash[87147]: cluster 2024-09-13T15:08:18.989364+0000 mgr.y (mgr.26252) 1267 : cluster [DBG] pgmap v1827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 31 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21108/1542 objects degraded (1368.872%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:20.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:20 smithi099 bash[87147]: cluster 2024-09-13T15:08:18.989364+0000 mgr.y (mgr.26252) 1267 : cluster [DBG] pgmap v1827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 31 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21108/1542 objects degraded (1368.872%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:20.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:20 smithi099 bash[84576]: cluster 2024-09-13T15:08:18.989364+0000 mgr.y (mgr.26252) 1267 : cluster [DBG] pgmap v1827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 31 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21108/1542 objects degraded (1368.872%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:20.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:20 smithi099 bash[84576]: cluster 2024-09-13T15:08:18.989364+0000 mgr.y (mgr.26252) 1267 : cluster [DBG] pgmap v1827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 31 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 21108/1542 objects degraded (1368.872%); 267 KiB/s, 42 objects/s recovering 2024-09-13T15:08:20.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:20.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:20.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:20.721 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:20.746 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:20.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:20.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:20.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:20.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:20.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:21.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:21.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:21.111 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:21.138 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:21.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:21.240 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:21.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:21.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:21.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:21.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:21.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:21.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:21.519 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:21.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:21.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:21.644 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:21.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:21.747 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:21.768 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:21.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:21.856 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-09-13T15:08:21.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-09-13T15:08:21.871 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:21.897 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:21.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:22.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:22.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:22.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:22.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:22.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:22 smithi177 bash[68594]: cluster 2024-09-13T15:08:20.990481+0000 mgr.y (mgr.26252) 1268 : cluster [DBG] pgmap v1828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:22.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:22 smithi177 bash[68594]: cluster 2024-09-13T15:08:20.990481+0000 mgr.y (mgr.26252) 1268 : cluster [DBG] pgmap v1828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:22.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:22 smithi177 bash[68594]: cluster 2024-09-13T15:08:21.553335+0000 mon.a (mon.0) 2414 : cluster [WRN] Health check update: Degraded data redundancy: 20832/78 objects degraded (26707.692%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:22.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:22 smithi177 bash[68594]: cluster 2024-09-13T15:08:21.553335+0000 mon.a (mon.0) 2414 : cluster [WRN] Health check update: Degraded data redundancy: 20832/78 objects degraded (26707.692%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:22.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:22.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:22.259 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:22.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:22.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:22 smithi099 bash[87147]: cluster 2024-09-13T15:08:20.990481+0000 mgr.y (mgr.26252) 1268 : cluster [DBG] pgmap v1828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:22 smithi099 bash[87147]: cluster 2024-09-13T15:08:20.990481+0000 mgr.y (mgr.26252) 1268 : cluster [DBG] pgmap v1828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:22 smithi099 bash[87147]: cluster 2024-09-13T15:08:21.553335+0000 mon.a (mon.0) 2414 : cluster [WRN] Health check update: Degraded data redundancy: 20832/78 objects degraded (26707.692%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:22 smithi099 bash[87147]: cluster 2024-09-13T15:08:21.553335+0000 mon.a (mon.0) 2414 : cluster [WRN] Health check update: Degraded data redundancy: 20832/78 objects degraded (26707.692%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:22 smithi099 bash[84576]: cluster 2024-09-13T15:08:20.990481+0000 mgr.y (mgr.26252) 1268 : cluster [DBG] pgmap v1828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:22 smithi099 bash[84576]: cluster 2024-09-13T15:08:20.990481+0000 mgr.y (mgr.26252) 1268 : cluster [DBG] pgmap v1828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:22 smithi099 bash[84576]: cluster 2024-09-13T15:08:21.553335+0000 mon.a (mon.0) 2414 : cluster [WRN] Health check update: Degraded data redundancy: 20832/78 objects degraded (26707.692%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:22 smithi099 bash[84576]: cluster 2024-09-13T15:08:21.553335+0000 mon.a (mon.0) 2414 : cluster [WRN] Health check update: Degraded data redundancy: 20832/78 objects degraded (26707.692%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:22.330 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:08:22.330 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:08:22.349 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:08:22.350 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:08:22.371 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:08:22.371 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:08:22.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:22.392 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:08:22.392 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:08:22.417 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:08:22.417 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:08:22.420 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:22.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:22.438 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:08:22.438 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:08:22.471 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:08:22.471 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:08:22.506 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:08:22.506 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:08:22.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:22.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:22.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:22.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:22.677 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:22.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:22.807 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:22.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:22.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:22.933 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:22.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:23.036 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:23.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:23.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:23.073 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:22 smithi099 bash[84576]: debug 2024-09-13T15:08:22.760+0000 7f0688bd2640 -1 mon.a@0(leader).osd e1073 definitely_dead 0 2024-09-13T15:08:23.112 INFO:teuthology.orchestra.run.smithi099.stderr:marked out osd.6. 2024-09-13T15:08:23.119 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:08:23.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:23.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[84576]: audit 2024-09-13T15:08:22.310110+0000 mon.a (mon.0) 2415 : audit [DBG] from='client.? 172.21.15.99:0/2075455588' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:23.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[84576]: audit 2024-09-13T15:08:22.310110+0000 mon.a (mon.0) 2415 : audit [DBG] from='client.? 172.21.15.99:0/2075455588' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:23.177 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[84576]: audit 2024-09-13T15:08:22.761959+0000 mon.a (mon.0) 2416 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T15:08:23.177 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[84576]: audit 2024-09-13T15:08:22.761959+0000 mon.a (mon.0) 2416 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T15:08:23.177 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[84576]: cluster 2024-09-13T15:08:22.762123+0000 mon.a (mon.0) 2417 : cluster [INF] Client client.admin marked osd.6 out, while it was still marked up 2024-09-13T15:08:23.177 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[84576]: cluster 2024-09-13T15:08:22.762123+0000 mon.a (mon.0) 2417 : cluster [INF] Client client.admin marked osd.6 out, while it was still marked up 2024-09-13T15:08:23.177 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[84576]: audit 2024-09-13T15:08:22.775838+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.99:0/1214745572' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.177 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[84576]: audit 2024-09-13T15:08:22.775838+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.99:0/1214745572' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.177 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[84576]: audit 2024-09-13T15:08:22.776238+0000 mon.a (mon.0) 2418 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.177 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[84576]: audit 2024-09-13T15:08:22.776238+0000 mon.a (mon.0) 2418 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[87147]: audit 2024-09-13T15:08:22.310110+0000 mon.a (mon.0) 2415 : audit [DBG] from='client.? 172.21.15.99:0/2075455588' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:23.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[87147]: audit 2024-09-13T15:08:22.310110+0000 mon.a (mon.0) 2415 : audit [DBG] from='client.? 172.21.15.99:0/2075455588' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:23.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[87147]: audit 2024-09-13T15:08:22.761959+0000 mon.a (mon.0) 2416 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T15:08:23.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[87147]: audit 2024-09-13T15:08:22.761959+0000 mon.a (mon.0) 2416 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T15:08:23.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[87147]: cluster 2024-09-13T15:08:22.762123+0000 mon.a (mon.0) 2417 : cluster [INF] Client client.admin marked osd.6 out, while it was still marked up 2024-09-13T15:08:23.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[87147]: cluster 2024-09-13T15:08:22.762123+0000 mon.a (mon.0) 2417 : cluster [INF] Client client.admin marked osd.6 out, while it was still marked up 2024-09-13T15:08:23.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[87147]: audit 2024-09-13T15:08:22.775838+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.99:0/1214745572' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[87147]: audit 2024-09-13T15:08:22.775838+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.99:0/1214745572' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[87147]: audit 2024-09-13T15:08:22.776238+0000 mon.a (mon.0) 2418 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.178 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:23 smithi099 bash[87147]: audit 2024-09-13T15:08:22.776238+0000 mon.a (mon.0) 2418 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:23.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:23.290 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:23.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:23.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:23.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:23 smithi177 bash[68594]: audit 2024-09-13T15:08:22.310110+0000 mon.a (mon.0) 2415 : audit [DBG] from='client.? 172.21.15.99:0/2075455588' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:23.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:23 smithi177 bash[68594]: audit 2024-09-13T15:08:22.310110+0000 mon.a (mon.0) 2415 : audit [DBG] from='client.? 172.21.15.99:0/2075455588' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:23.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:23 smithi177 bash[68594]: audit 2024-09-13T15:08:22.761959+0000 mon.a (mon.0) 2416 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T15:08:23.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:23 smithi177 bash[68594]: audit 2024-09-13T15:08:22.761959+0000 mon.a (mon.0) 2416 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-13T15:08:23.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:23 smithi177 bash[68594]: cluster 2024-09-13T15:08:22.762123+0000 mon.a (mon.0) 2417 : cluster [INF] Client client.admin marked osd.6 out, while it was still marked up 2024-09-13T15:08:23.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:23 smithi177 bash[68594]: cluster 2024-09-13T15:08:22.762123+0000 mon.a (mon.0) 2417 : cluster [INF] Client client.admin marked osd.6 out, while it was still marked up 2024-09-13T15:08:23.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:23 smithi177 bash[68594]: audit 2024-09-13T15:08:22.775838+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.99:0/1214745572' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:23 smithi177 bash[68594]: audit 2024-09-13T15:08:22.775838+0000 mon.c (mon.2) 99 : audit [INF] from='client.? 172.21.15.99:0/1214745572' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:23 smithi177 bash[68594]: audit 2024-09-13T15:08:22.776238+0000 mon.a (mon.0) 2418 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:23 smithi177 bash[68594]: audit 2024-09-13T15:08:22.776238+0000 mon.a (mon.0) 2418 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:23.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:23.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:23.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:23.539 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:23.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:23.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:23.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:23.687 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:23.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:23.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:23.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:23.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:23.916 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:23.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:24.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:24.073 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:24.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:24.176 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:24.194 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:24.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:24.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:24.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: cluster 2024-09-13T15:08:22.991069+0000 mgr.y (mgr.26252) 1269 : cluster [DBG] pgmap v1829: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 358 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:24.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: cluster 2024-09-13T15:08:22.991069+0000 mgr.y (mgr.26252) 1269 : cluster [DBG] pgmap v1829: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 358 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:24.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: cluster 2024-09-13T15:08:23.104647+0000 mon.a (mon.0) 2419 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:24.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: cluster 2024-09-13T15:08:23.104647+0000 mon.a (mon.0) 2419 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:24.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: audit 2024-09-13T15:08:23.111852+0000 mon.a (mon.0) 2420 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-13T15:08:24.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: audit 2024-09-13T15:08:23.111852+0000 mon.a (mon.0) 2420 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-13T15:08:24.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: audit 2024-09-13T15:08:23.112014+0000 mon.a (mon.0) 2421 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:08:24.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: audit 2024-09-13T15:08:23.112014+0000 mon.a (mon.0) 2421 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: cluster 2024-09-13T15:08:23.126008+0000 mon.a (mon.0) 2422 : cluster [DBG] osdmap e1074: 8 total, 8 up, 7 in 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: cluster 2024-09-13T15:08:23.126008+0000 mon.a (mon.0) 2422 : cluster [DBG] osdmap e1074: 8 total, 8 up, 7 in 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: audit 2024-09-13T15:08:23.163502+0000 mon.a (mon.0) 2423 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[87147]: audit 2024-09-13T15:08:23.163502+0000 mon.a (mon.0) 2423 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: cluster 2024-09-13T15:08:22.991069+0000 mgr.y (mgr.26252) 1269 : cluster [DBG] pgmap v1829: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 358 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: cluster 2024-09-13T15:08:22.991069+0000 mgr.y (mgr.26252) 1269 : cluster [DBG] pgmap v1829: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 358 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: cluster 2024-09-13T15:08:23.104647+0000 mon.a (mon.0) 2419 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: cluster 2024-09-13T15:08:23.104647+0000 mon.a (mon.0) 2419 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: audit 2024-09-13T15:08:23.111852+0000 mon.a (mon.0) 2420 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: audit 2024-09-13T15:08:23.111852+0000 mon.a (mon.0) 2420 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-13T15:08:24.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: audit 2024-09-13T15:08:23.112014+0000 mon.a (mon.0) 2421 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:08:24.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: audit 2024-09-13T15:08:23.112014+0000 mon.a (mon.0) 2421 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:08:24.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: cluster 2024-09-13T15:08:23.126008+0000 mon.a (mon.0) 2422 : cluster [DBG] osdmap e1074: 8 total, 8 up, 7 in 2024-09-13T15:08:24.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: cluster 2024-09-13T15:08:23.126008+0000 mon.a (mon.0) 2422 : cluster [DBG] osdmap e1074: 8 total, 8 up, 7 in 2024-09-13T15:08:24.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: audit 2024-09-13T15:08:23.163502+0000 mon.a (mon.0) 2423 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:08:24.308 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:24 smithi099 bash[84576]: audit 2024-09-13T15:08:23.163502+0000 mon.a (mon.0) 2423 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:08:24.320 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:24.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: cluster 2024-09-13T15:08:22.991069+0000 mgr.y (mgr.26252) 1269 : cluster [DBG] pgmap v1829: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 358 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: cluster 2024-09-13T15:08:22.991069+0000 mgr.y (mgr.26252) 1269 : cluster [DBG] pgmap v1829: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 358 op/s; 20832/78 objects degraded (26707.692%); 0 B/s, 75 objects/s recovering 2024-09-13T15:08:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: cluster 2024-09-13T15:08:23.104647+0000 mon.a (mon.0) 2419 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: cluster 2024-09-13T15:08:23.104647+0000 mon.a (mon.0) 2419 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: audit 2024-09-13T15:08:23.111852+0000 mon.a (mon.0) 2420 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-13T15:08:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: audit 2024-09-13T15:08:23.111852+0000 mon.a (mon.0) 2420 : audit [INF] from='client.? 172.21.15.99:0/525018027' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-13T15:08:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: audit 2024-09-13T15:08:23.112014+0000 mon.a (mon.0) 2421 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:08:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: audit 2024-09-13T15:08:23.112014+0000 mon.a (mon.0) 2421 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:08:24.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: cluster 2024-09-13T15:08:23.126008+0000 mon.a (mon.0) 2422 : cluster [DBG] osdmap e1074: 8 total, 8 up, 7 in 2024-09-13T15:08:24.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: cluster 2024-09-13T15:08:23.126008+0000 mon.a (mon.0) 2422 : cluster [DBG] osdmap e1074: 8 total, 8 up, 7 in 2024-09-13T15:08:24.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: audit 2024-09-13T15:08:23.163502+0000 mon.a (mon.0) 2423 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:08:24.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:24 smithi177 bash[68594]: audit 2024-09-13T15:08:23.163502+0000 mon.a (mon.0) 2423 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:08:24.423 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:24.449 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:24.552 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:24.576 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:24.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:24.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:24.707 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:24.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:24.810 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:24.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:24.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:24.937 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:24.941 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:24.961 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:24.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:25.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:25.090 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:25.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:25.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:25.219 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:25.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:25.322 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:25.332 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:25 smithi177 bash[68594]: cluster 2024-09-13T15:08:24.123918+0000 mon.a (mon.0) 2424 : cluster [DBG] osdmap e1075: 8 total, 8 up, 7 in 2024-09-13T15:08:25.332 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:25 smithi177 bash[68594]: cluster 2024-09-13T15:08:24.123918+0000 mon.a (mon.0) 2424 : cluster [DBG] osdmap e1075: 8 total, 8 up, 7 in 2024-09-13T15:08:25.332 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:25 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.123594+0000 mon.a (mon.0) 2425 : cluster [DBG] osdmap e1076: 8 total, 8 up, 7 in 2024-09-13T15:08:25.332 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:25 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.123594+0000 mon.a (mon.0) 2425 : cluster [DBG] osdmap e1076: 8 total, 8 up, 7 in 2024-09-13T15:08:25.344 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:25.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:25.447 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:25.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:25.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:25.487 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:25 smithi099 bash[84576]: cluster 2024-09-13T15:08:24.123918+0000 mon.a (mon.0) 2424 : cluster [DBG] osdmap e1075: 8 total, 8 up, 7 in 2024-09-13T15:08:25.487 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:25 smithi099 bash[84576]: cluster 2024-09-13T15:08:24.123918+0000 mon.a (mon.0) 2424 : cluster [DBG] osdmap e1075: 8 total, 8 up, 7 in 2024-09-13T15:08:25.487 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:25 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.123594+0000 mon.a (mon.0) 2425 : cluster [DBG] osdmap e1076: 8 total, 8 up, 7 in 2024-09-13T15:08:25.487 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:25 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.123594+0000 mon.a (mon.0) 2425 : cluster [DBG] osdmap e1076: 8 total, 8 up, 7 in 2024-09-13T15:08:25.487 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:25 smithi099 bash[87147]: cluster 2024-09-13T15:08:24.123918+0000 mon.a (mon.0) 2424 : cluster [DBG] osdmap e1075: 8 total, 8 up, 7 in 2024-09-13T15:08:25.487 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:25 smithi099 bash[87147]: cluster 2024-09-13T15:08:24.123918+0000 mon.a (mon.0) 2424 : cluster [DBG] osdmap e1075: 8 total, 8 up, 7 in 2024-09-13T15:08:25.487 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:25 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.123594+0000 mon.a (mon.0) 2425 : cluster [DBG] osdmap e1076: 8 total, 8 up, 7 in 2024-09-13T15:08:25.487 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:25 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.123594+0000 mon.a (mon.0) 2425 : cluster [DBG] osdmap e1076: 8 total, 8 up, 7 in 2024-09-13T15:08:25.571 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:25.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:25.698 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:25.723 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:25.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:25.852 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:25.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:25.956 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:25.980 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:25.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:26.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:26.103 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:26.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:26.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:26.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:26.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:26.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:24.991838+0000 mgr.y (mgr.26252) 1270 : cluster [DBG] pgmap v1832: 89 pgs: 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 19865/78 objects degraded (25467.949%); 73 B/s, 201 objects/s recovering 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:24.991838+0000 mgr.y (mgr.26252) 1270 : cluster [DBG] pgmap v1832: 89 pgs: 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 19865/78 objects degraded (25467.949%); 73 B/s, 201 objects/s recovering 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.119839+0000 osd.7 (osd.7) 7 : cluster [DBG] 44.a starting backfill to osd.3 from (0'0,0'0] MAX to 1073'13496 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.119839+0000 osd.7 (osd.7) 7 : cluster [DBG] 44.a starting backfill to osd.3 from (0'0,0'0] MAX to 1073'13496 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.120361+0000 osd.7 (osd.7) 8 : cluster [DBG] 44.d starting backfill to osd.0 from (0'0,0'0] MAX to 1073'13490 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.120361+0000 osd.7 (osd.7) 8 : cluster [DBG] 44.d starting backfill to osd.0 from (0'0,0'0] MAX to 1073'13490 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.125825+0000 osd.3 (osd.3) 1 : cluster [DBG] 44.b starting backfill to osd.4 from (0'0,0'0] MAX to 1073'13108 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.125825+0000 osd.3 (osd.3) 1 : cluster [DBG] 44.b starting backfill to osd.4 from (0'0,0'0] MAX to 1073'13108 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.126753+0000 osd.3 (osd.3) 2 : cluster [DBG] 44.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1073'13446 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.126753+0000 osd.3 (osd.3) 2 : cluster [DBG] 44.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1073'13446 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.131396+0000 osd.0 (osd.0) 19 : cluster [DBG] 44.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13270 2024-09-13T15:08:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.131396+0000 osd.0 (osd.0) 19 : cluster [DBG] 44.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13270 2024-09-13T15:08:26.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.131622+0000 osd.1 (osd.1) 5 : cluster [DBG] 44.e starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13322 2024-09-13T15:08:26.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.131622+0000 osd.1 (osd.1) 5 : cluster [DBG] 44.e starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13322 2024-09-13T15:08:26.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.132144+0000 mon.a (mon.0) 2426 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:08:26.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[87147]: cluster 2024-09-13T15:08:25.132144+0000 mon.a (mon.0) 2426 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:08:26.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:26.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:26.436 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:24.991838+0000 mgr.y (mgr.26252) 1270 : cluster [DBG] pgmap v1832: 89 pgs: 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 19865/78 objects degraded (25467.949%); 73 B/s, 201 objects/s recovering 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:24.991838+0000 mgr.y (mgr.26252) 1270 : cluster [DBG] pgmap v1832: 89 pgs: 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 19865/78 objects degraded (25467.949%); 73 B/s, 201 objects/s recovering 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.119839+0000 osd.7 (osd.7) 7 : cluster [DBG] 44.a starting backfill to osd.3 from (0'0,0'0] MAX to 1073'13496 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.119839+0000 osd.7 (osd.7) 7 : cluster [DBG] 44.a starting backfill to osd.3 from (0'0,0'0] MAX to 1073'13496 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.120361+0000 osd.7 (osd.7) 8 : cluster [DBG] 44.d starting backfill to osd.0 from (0'0,0'0] MAX to 1073'13490 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.120361+0000 osd.7 (osd.7) 8 : cluster [DBG] 44.d starting backfill to osd.0 from (0'0,0'0] MAX to 1073'13490 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.125825+0000 osd.3 (osd.3) 1 : cluster [DBG] 44.b starting backfill to osd.4 from (0'0,0'0] MAX to 1073'13108 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.125825+0000 osd.3 (osd.3) 1 : cluster [DBG] 44.b starting backfill to osd.4 from (0'0,0'0] MAX to 1073'13108 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.126753+0000 osd.3 (osd.3) 2 : cluster [DBG] 44.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1073'13446 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.126753+0000 osd.3 (osd.3) 2 : cluster [DBG] 44.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1073'13446 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.131396+0000 osd.0 (osd.0) 19 : cluster [DBG] 44.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13270 2024-09-13T15:08:26.437 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.131396+0000 osd.0 (osd.0) 19 : cluster [DBG] 44.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13270 2024-09-13T15:08:26.438 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.131622+0000 osd.1 (osd.1) 5 : cluster [DBG] 44.e starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13322 2024-09-13T15:08:26.438 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.131622+0000 osd.1 (osd.1) 5 : cluster [DBG] 44.e starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13322 2024-09-13T15:08:26.438 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.132144+0000 mon.a (mon.0) 2426 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:08:26.438 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:26 smithi099 bash[84576]: cluster 2024-09-13T15:08:25.132144+0000 mon.a (mon.0) 2426 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:08:26.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:26.473 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:24.991838+0000 mgr.y (mgr.26252) 1270 : cluster [DBG] pgmap v1832: 89 pgs: 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 19865/78 objects degraded (25467.949%); 73 B/s, 201 objects/s recovering 2024-09-13T15:08:26.473 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:24.991838+0000 mgr.y (mgr.26252) 1270 : cluster [DBG] pgmap v1832: 89 pgs: 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 19865/78 objects degraded (25467.949%); 73 B/s, 201 objects/s recovering 2024-09-13T15:08:26.473 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.119839+0000 osd.7 (osd.7) 7 : cluster [DBG] 44.a starting backfill to osd.3 from (0'0,0'0] MAX to 1073'13496 2024-09-13T15:08:26.473 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.119839+0000 osd.7 (osd.7) 7 : cluster [DBG] 44.a starting backfill to osd.3 from (0'0,0'0] MAX to 1073'13496 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.120361+0000 osd.7 (osd.7) 8 : cluster [DBG] 44.d starting backfill to osd.0 from (0'0,0'0] MAX to 1073'13490 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.120361+0000 osd.7 (osd.7) 8 : cluster [DBG] 44.d starting backfill to osd.0 from (0'0,0'0] MAX to 1073'13490 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.125825+0000 osd.3 (osd.3) 1 : cluster [DBG] 44.b starting backfill to osd.4 from (0'0,0'0] MAX to 1073'13108 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.125825+0000 osd.3 (osd.3) 1 : cluster [DBG] 44.b starting backfill to osd.4 from (0'0,0'0] MAX to 1073'13108 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.126753+0000 osd.3 (osd.3) 2 : cluster [DBG] 44.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1073'13446 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.126753+0000 osd.3 (osd.3) 2 : cluster [DBG] 44.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1073'13446 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.131396+0000 osd.0 (osd.0) 19 : cluster [DBG] 44.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13270 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.131396+0000 osd.0 (osd.0) 19 : cluster [DBG] 44.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13270 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.131622+0000 osd.1 (osd.1) 5 : cluster [DBG] 44.e starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13322 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.131622+0000 osd.1 (osd.1) 5 : cluster [DBG] 44.e starting backfill to osd.7 from (0'0,0'0] MAX to 1073'13322 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.132144+0000 mon.a (mon.0) 2426 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:08:26.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:26 smithi177 bash[68594]: cluster 2024-09-13T15:08:25.132144+0000 mon.a (mon.0) 2426 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T15:08:26.487 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:26.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:26.590 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:26.616 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:26.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:26.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:26.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:26.847 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:26.871 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:26.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:26.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:26.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:27.100 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:27.127 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:27.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:27.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:27.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:27.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:27.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:08:27] "GET /metrics HTTP/1.1" 200 36790 "" "Prometheus/2.51.0" 2024-09-13T15:08:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[87147]: cluster 2024-09-13T15:08:26.154163+0000 mon.a (mon.0) 2427 : cluster [DBG] osdmap e1077: 8 total, 8 up, 7 in 2024-09-13T15:08:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[87147]: cluster 2024-09-13T15:08:26.154163+0000 mon.a (mon.0) 2427 : cluster [DBG] osdmap e1077: 8 total, 8 up, 7 in 2024-09-13T15:08:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[87147]: cluster 2024-09-13T15:08:26.554107+0000 mon.a (mon.0) 2428 : cluster [WRN] Health check update: Degraded data redundancy: 19865/78 objects degraded (25467.949%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[87147]: cluster 2024-09-13T15:08:26.554107+0000 mon.a (mon.0) 2428 : cluster [WRN] Health check update: Degraded data redundancy: 19865/78 objects degraded (25467.949%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[87147]: cluster 2024-09-13T15:08:26.992794+0000 mgr.y (mgr.26252) 1271 : cluster [DBG] pgmap v1835: 89 pgs: 2 remapped+peering, 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 19865/78 objects degraded (25467.949%); 144 KiB/s, 243 objects/s recovering 2024-09-13T15:08:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[87147]: cluster 2024-09-13T15:08:26.992794+0000 mgr.y (mgr.26252) 1271 : cluster [DBG] pgmap v1835: 89 pgs: 2 remapped+peering, 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 19865/78 objects degraded (25467.949%); 144 KiB/s, 243 objects/s recovering 2024-09-13T15:08:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[84576]: cluster 2024-09-13T15:08:26.154163+0000 mon.a (mon.0) 2427 : cluster [DBG] osdmap e1077: 8 total, 8 up, 7 in 2024-09-13T15:08:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[84576]: cluster 2024-09-13T15:08:26.154163+0000 mon.a (mon.0) 2427 : cluster [DBG] osdmap e1077: 8 total, 8 up, 7 in 2024-09-13T15:08:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[84576]: cluster 2024-09-13T15:08:26.554107+0000 mon.a (mon.0) 2428 : cluster [WRN] Health check update: Degraded data redundancy: 19865/78 objects degraded (25467.949%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[84576]: cluster 2024-09-13T15:08:26.554107+0000 mon.a (mon.0) 2428 : cluster [WRN] Health check update: Degraded data redundancy: 19865/78 objects degraded (25467.949%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[84576]: cluster 2024-09-13T15:08:26.992794+0000 mgr.y (mgr.26252) 1271 : cluster [DBG] pgmap v1835: 89 pgs: 2 remapped+peering, 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 19865/78 objects degraded (25467.949%); 144 KiB/s, 243 objects/s recovering 2024-09-13T15:08:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:27 smithi099 bash[84576]: cluster 2024-09-13T15:08:26.992794+0000 mgr.y (mgr.26252) 1271 : cluster [DBG] pgmap v1835: 89 pgs: 2 remapped+peering, 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 19865/78 objects degraded (25467.949%); 144 KiB/s, 243 objects/s recovering 2024-09-13T15:08:27.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:27.383 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:27.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:27.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:08:27.486 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:27.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:27.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:27.610 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:27.637 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:27.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:27 smithi177 bash[68594]: cluster 2024-09-13T15:08:26.154163+0000 mon.a (mon.0) 2427 : cluster [DBG] osdmap e1077: 8 total, 8 up, 7 in 2024-09-13T15:08:27.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:27 smithi177 bash[68594]: cluster 2024-09-13T15:08:26.154163+0000 mon.a (mon.0) 2427 : cluster [DBG] osdmap e1077: 8 total, 8 up, 7 in 2024-09-13T15:08:27.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:27 smithi177 bash[68594]: cluster 2024-09-13T15:08:26.554107+0000 mon.a (mon.0) 2428 : cluster [WRN] Health check update: Degraded data redundancy: 19865/78 objects degraded (25467.949%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:27.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:27 smithi177 bash[68594]: cluster 2024-09-13T15:08:26.554107+0000 mon.a (mon.0) 2428 : cluster [WRN] Health check update: Degraded data redundancy: 19865/78 objects degraded (25467.949%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T15:08:27.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:27 smithi177 bash[68594]: cluster 2024-09-13T15:08:26.992794+0000 mgr.y (mgr.26252) 1271 : cluster [DBG] pgmap v1835: 89 pgs: 2 remapped+peering, 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 19865/78 objects degraded (25467.949%); 144 KiB/s, 243 objects/s recovering 2024-09-13T15:08:27.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:27 smithi177 bash[68594]: cluster 2024-09-13T15:08:26.992794+0000 mgr.y (mgr.26252) 1271 : cluster [DBG] pgmap v1835: 89 pgs: 2 remapped+peering, 1 active+undersized+remapped, 5 peering, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 19865/78 objects degraded (25467.949%); 144 KiB/s, 243 objects/s recovering 2024-09-13T15:08:27.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:27.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:27.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:27.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:27.866 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:27.889 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:27.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:27.991 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:28.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:28.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:28.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:28.138 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:28.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:28.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:28.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:28 smithi177 bash[68594]: cluster 2024-09-13T15:08:27.168043+0000 mon.a (mon.0) 2429 : cluster [DBG] osdmap e1078: 8 total, 8 up, 7 in 2024-09-13T15:08:28.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:28 smithi177 bash[68594]: cluster 2024-09-13T15:08:27.168043+0000 mon.a (mon.0) 2429 : cluster [DBG] osdmap e1078: 8 total, 8 up, 7 in 2024-09-13T15:08:28.368 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:28.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:28.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:28.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:28.500 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:28 smithi099 bash[84576]: cluster 2024-09-13T15:08:27.168043+0000 mon.a (mon.0) 2429 : cluster [DBG] osdmap e1078: 8 total, 8 up, 7 in 2024-09-13T15:08:28.500 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:28 smithi099 bash[84576]: cluster 2024-09-13T15:08:27.168043+0000 mon.a (mon.0) 2429 : cluster [DBG] osdmap e1078: 8 total, 8 up, 7 in 2024-09-13T15:08:28.500 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:28 smithi099 bash[87147]: cluster 2024-09-13T15:08:27.168043+0000 mon.a (mon.0) 2429 : cluster [DBG] osdmap e1078: 8 total, 8 up, 7 in 2024-09-13T15:08:28.501 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:28 smithi099 bash[87147]: cluster 2024-09-13T15:08:27.168043+0000 mon.a (mon.0) 2429 : cluster [DBG] osdmap e1078: 8 total, 8 up, 7 in 2024-09-13T15:08:28.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:28.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:28.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:28.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:28.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:28.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:28.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:28.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:28.895 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:28.998 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:29.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:29.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:29.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:29.150 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:29.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:29.254 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:29.263 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:29 smithi099 bash[87147]: cluster 2024-09-13T15:08:28.993910+0000 mgr.y (mgr.26252) 1272 : cluster [DBG] pgmap v1837: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 2 remapped+peering, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 119 KiB/s, 52 objects/s recovering 2024-09-13T15:08:29.263 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:29 smithi099 bash[87147]: cluster 2024-09-13T15:08:28.993910+0000 mgr.y (mgr.26252) 1272 : cluster [DBG] pgmap v1837: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 2 remapped+peering, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 119 KiB/s, 52 objects/s recovering 2024-09-13T15:08:29.264 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:29 smithi099 bash[84576]: cluster 2024-09-13T15:08:28.993910+0000 mgr.y (mgr.26252) 1272 : cluster [DBG] pgmap v1837: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 2 remapped+peering, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 119 KiB/s, 52 objects/s recovering 2024-09-13T15:08:29.264 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:29 smithi099 bash[84576]: cluster 2024-09-13T15:08:28.993910+0000 mgr.y (mgr.26252) 1272 : cluster [DBG] pgmap v1837: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 2 remapped+peering, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 119 KiB/s, 52 objects/s recovering 2024-09-13T15:08:29.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:29.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:08:29.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:29.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:29.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:29.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:29.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:29.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:29.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:29 smithi177 bash[68594]: cluster 2024-09-13T15:08:28.993910+0000 mgr.y (mgr.26252) 1272 : cluster [DBG] pgmap v1837: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 2 remapped+peering, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 119 KiB/s, 52 objects/s recovering 2024-09-13T15:08:29.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:29 smithi177 bash[68594]: cluster 2024-09-13T15:08:28.993910+0000 mgr.y (mgr.26252) 1272 : cluster [DBG] pgmap v1837: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 2 remapped+peering, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 119 KiB/s, 52 objects/s recovering 2024-09-13T15:08:29.625 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:29.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:29.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:29.771 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:29.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:29.895 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:29.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:29.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:30.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:30.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:30.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:30.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:30.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:30.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:30.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:30.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:30.370 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:30.389 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:30.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[87147]: cluster 2024-09-13T15:08:29.161176+0000 mon.a (mon.0) 2430 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:30.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[87147]: cluster 2024-09-13T15:08:29.161176+0000 mon.a (mon.0) 2430 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:30.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[87147]: audit 2024-09-13T15:08:29.678717+0000 mon.a (mon.0) 2431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.391 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[87147]: audit 2024-09-13T15:08:29.678717+0000 mon.a (mon.0) 2431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.391 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[87147]: audit 2024-09-13T15:08:29.683246+0000 mon.a (mon.0) 2432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.391 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[87147]: audit 2024-09-13T15:08:29.683246+0000 mon.a (mon.0) 2432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[84576]: cluster 2024-09-13T15:08:29.161176+0000 mon.a (mon.0) 2430 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:30.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[84576]: cluster 2024-09-13T15:08:29.161176+0000 mon.a (mon.0) 2430 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:30.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[84576]: audit 2024-09-13T15:08:29.678717+0000 mon.a (mon.0) 2431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[84576]: audit 2024-09-13T15:08:29.678717+0000 mon.a (mon.0) 2431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.392 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[84576]: audit 2024-09-13T15:08:29.683246+0000 mon.a (mon.0) 2432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.392 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:30 smithi099 bash[84576]: audit 2024-09-13T15:08:29.683246+0000 mon.a (mon.0) 2432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:30.493 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:30.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:30.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:30.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:08:30 smithi177 bash[61439]: ts=2024-09-13T15:08:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:30.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:30 smithi177 bash[68594]: cluster 2024-09-13T15:08:29.161176+0000 mon.a (mon.0) 2430 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:30.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:30 smithi177 bash[68594]: cluster 2024-09-13T15:08:29.161176+0000 mon.a (mon.0) 2430 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T15:08:30.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:30 smithi177 bash[68594]: audit 2024-09-13T15:08:29.678717+0000 mon.a (mon.0) 2431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:30 smithi177 bash[68594]: audit 2024-09-13T15:08:29.678717+0000 mon.a (mon.0) 2431 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:30 smithi177 bash[68594]: audit 2024-09-13T15:08:29.683246+0000 mon.a (mon.0) 2432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:30 smithi177 bash[68594]: audit 2024-09-13T15:08:29.683246+0000 mon.a (mon.0) 2432 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:30.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:30.645 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:30.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:30.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:30.768 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:30.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:30.812 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:30.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:30.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:30.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:30.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:31.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:31.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:31.141 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:31.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:31.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:31.270 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:31.375 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:31.398 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:31.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:31.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:31.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:31 smithi177 bash[68594]: cluster 2024-09-13T15:08:30.994447+0000 mgr.y (mgr.26252) 1273 : cluster [DBG] pgmap v1838: 89 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 97 KiB/s, 42 objects/s recovering 2024-09-13T15:08:31.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:31 smithi177 bash[68594]: cluster 2024-09-13T15:08:30.994447+0000 mgr.y (mgr.26252) 1273 : cluster [DBG] pgmap v1838: 89 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 97 KiB/s, 42 objects/s recovering 2024-09-13T15:08:31.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:31.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:31.633 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:31.661 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:31.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:31.736 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:31 smithi099 bash[84576]: cluster 2024-09-13T15:08:30.994447+0000 mgr.y (mgr.26252) 1273 : cluster [DBG] pgmap v1838: 89 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 97 KiB/s, 42 objects/s recovering 2024-09-13T15:08:31.736 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:31 smithi099 bash[84576]: cluster 2024-09-13T15:08:30.994447+0000 mgr.y (mgr.26252) 1273 : cluster [DBG] pgmap v1838: 89 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 97 KiB/s, 42 objects/s recovering 2024-09-13T15:08:31.737 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:31 smithi099 bash[87147]: cluster 2024-09-13T15:08:30.994447+0000 mgr.y (mgr.26252) 1273 : cluster [DBG] pgmap v1838: 89 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 97 KiB/s, 42 objects/s recovering 2024-09-13T15:08:31.737 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:31 smithi099 bash[87147]: cluster 2024-09-13T15:08:30.994447+0000 mgr.y (mgr.26252) 1273 : cluster [DBG] pgmap v1838: 89 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 97 KiB/s, 42 objects/s recovering 2024-09-13T15:08:31.764 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:31.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:31.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:31.827 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:31.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:31.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:31.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:32.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:32.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:32.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:32.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:32.167 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:32.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:32.199 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:08:32.199 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:08:32.200 INFO:tasks.thrashosds.thrasher:Adding osd 6 2024-09-13T15:08:32.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd in 6 2024-09-13T15:08:32.271 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:32.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:32.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:32 smithi177 bash[68594]: cluster 2024-09-13T15:08:31.555747+0000 mon.a (mon.0) 2433 : cluster [WRN] Health check update: Degraded data redundancy: 5843/78 objects degraded (7491.026%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:32 smithi177 bash[68594]: cluster 2024-09-13T15:08:31.555747+0000 mon.a (mon.0) 2433 : cluster [WRN] Health check update: Degraded data redundancy: 5843/78 objects degraded (7491.026%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:32.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:32.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:32.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:32.420 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:32 smithi099 bash[84576]: cluster 2024-09-13T15:08:31.555747+0000 mon.a (mon.0) 2433 : cluster [WRN] Health check update: Degraded data redundancy: 5843/78 objects degraded (7491.026%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:32.420 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:32 smithi099 bash[84576]: cluster 2024-09-13T15:08:31.555747+0000 mon.a (mon.0) 2433 : cluster [WRN] Health check update: Degraded data redundancy: 5843/78 objects degraded (7491.026%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:32.420 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:32 smithi099 bash[87147]: cluster 2024-09-13T15:08:31.555747+0000 mon.a (mon.0) 2433 : cluster [WRN] Health check update: Degraded data redundancy: 5843/78 objects degraded (7491.026%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:32.420 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:32 smithi099 bash[87147]: cluster 2024-09-13T15:08:31.555747+0000 mon.a (mon.0) 2433 : cluster [WRN] Health check update: Degraded data redundancy: 5843/78 objects degraded (7491.026%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:32.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:32.543 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:32.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:32.646 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:32.667 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:32.771 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:32.796 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:32.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:32.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:32.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:33.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:33.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:33.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:33.156 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:33.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:33.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:33.279 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:33.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:33.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:33.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:33.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:33.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:33.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:33.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:33.600 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:08:33.600 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:08:33.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:33.664 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:32.572570+0000 mon.a (mon.0) 2434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:32.572570+0000 mon.a (mon.0) 2434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:32.579670+0000 mon.a (mon.0) 2435 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:32.579670+0000 mon.a (mon.0) 2435 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: cluster 2024-09-13T15:08:32.994928+0000 mgr.y (mgr.26252) 1274 : cluster [DBG] pgmap v1839: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 602 B/s, 69 objects/s recovering 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: cluster 2024-09-13T15:08:32.994928+0000 mgr.y (mgr.26252) 1274 : cluster [DBG] pgmap v1839: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 602 B/s, 69 objects/s recovering 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:33.258110+0000 mon.a (mon.0) 2436 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:33.258110+0000 mon.a (mon.0) 2436 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:33.258803+0000 mon.a (mon.0) 2437 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:33.258803+0000 mon.a (mon.0) 2437 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:33.263594+0000 mon.a (mon.0) 2438 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:33.263594+0000 mon.a (mon.0) 2438 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.665 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:33.304617+0000 mon.a (mon.0) 2439 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:33.666 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[84576]: audit 2024-09-13T15:08:33.304617+0000 mon.a (mon.0) 2439 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:33.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:32.572570+0000 mon.a (mon.0) 2434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:32.572570+0000 mon.a (mon.0) 2434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:32.579670+0000 mon.a (mon.0) 2435 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:32.579670+0000 mon.a (mon.0) 2435 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: cluster 2024-09-13T15:08:32.994928+0000 mgr.y (mgr.26252) 1274 : cluster [DBG] pgmap v1839: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 602 B/s, 69 objects/s recovering 2024-09-13T15:08:33.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: cluster 2024-09-13T15:08:32.994928+0000 mgr.y (mgr.26252) 1274 : cluster [DBG] pgmap v1839: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 602 B/s, 69 objects/s recovering 2024-09-13T15:08:33.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:33.258110+0000 mon.a (mon.0) 2436 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:33.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:33.258110+0000 mon.a (mon.0) 2436 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:33.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:33.258803+0000 mon.a (mon.0) 2437 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:33.667 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:33.258803+0000 mon.a (mon.0) 2437 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:33.667 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:33.263594+0000 mon.a (mon.0) 2438 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.667 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:33.263594+0000 mon.a (mon.0) 2438 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.667 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:33.304617+0000 mon.a (mon.0) 2439 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:33.667 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:33 smithi099 bash[87147]: audit 2024-09-13T15:08:33.304617+0000 mon.a (mon.0) 2439 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:33.675 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:33.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:33.778 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:33.791 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:32.572570+0000 mon.a (mon.0) 2434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.791 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:32.572570+0000 mon.a (mon.0) 2434 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.791 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:32.579670+0000 mon.a (mon.0) 2435 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.791 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:32.579670+0000 mon.a (mon.0) 2435 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.791 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: cluster 2024-09-13T15:08:32.994928+0000 mgr.y (mgr.26252) 1274 : cluster [DBG] pgmap v1839: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 602 B/s, 69 objects/s recovering 2024-09-13T15:08:33.791 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: cluster 2024-09-13T15:08:32.994928+0000 mgr.y (mgr.26252) 1274 : cluster [DBG] pgmap v1839: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 activating+remapped, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 602 B/s, 69 objects/s recovering 2024-09-13T15:08:33.791 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:33.258110+0000 mon.a (mon.0) 2436 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:33.791 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:33.258110+0000 mon.a (mon.0) 2436 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:33.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:33.258803+0000 mon.a (mon.0) 2437 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:33.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:33.258803+0000 mon.a (mon.0) 2437 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:33.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:33.263594+0000 mon.a (mon.0) 2438 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:33.263594+0000 mon.a (mon.0) 2438 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:33.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:33.304617+0000 mon.a (mon.0) 2439 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:33.792 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:33 smithi177 bash[68594]: audit 2024-09-13T15:08:33.304617+0000 mon.a (mon.0) 2439 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:33.805 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:33.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:33.856 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:33.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:33.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:33.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:34.033 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:34.060 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:34.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:34.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:34.187 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:34.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:34.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:34.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:34.421 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:34.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:34.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:34.566 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:34.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:34.669 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:34.703 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:34.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:34.806 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:34.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:34.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:34.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:34.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:34.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:35.062 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:35.083 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:35.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:35.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:35.207 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:35.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:35.311 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:35.336 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:35.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:35.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:35.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:35.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:35.566 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:35.594 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:35.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:35.698 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:35.722 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:35.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:35.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:08:35.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:35.848 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:35.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:35.951 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:35.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:35.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:36.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:36.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:36 smithi099 bash[84576]: cluster 2024-09-13T15:08:34.995862+0000 mgr.y (mgr.26252) 1275 : cluster [DBG] pgmap v1840: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 536 B/s, 135 objects/s recovering 2024-09-13T15:08:36.086 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:36 smithi099 bash[84576]: cluster 2024-09-13T15:08:34.995862+0000 mgr.y (mgr.26252) 1275 : cluster [DBG] pgmap v1840: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 536 B/s, 135 objects/s recovering 2024-09-13T15:08:36.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:36 smithi099 bash[87147]: cluster 2024-09-13T15:08:34.995862+0000 mgr.y (mgr.26252) 1275 : cluster [DBG] pgmap v1840: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 536 B/s, 135 objects/s recovering 2024-09-13T15:08:36.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:36 smithi099 bash[87147]: cluster 2024-09-13T15:08:34.995862+0000 mgr.y (mgr.26252) 1275 : cluster [DBG] pgmap v1840: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 536 B/s, 135 objects/s recovering 2024-09-13T15:08:36.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:36.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:36.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:36.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:36.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:36.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:36.342 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:36.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:36 smithi177 bash[68594]: cluster 2024-09-13T15:08:34.995862+0000 mgr.y (mgr.26252) 1275 : cluster [DBG] pgmap v1840: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 536 B/s, 135 objects/s recovering 2024-09-13T15:08:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:36 smithi177 bash[68594]: cluster 2024-09-13T15:08:34.995862+0000 mgr.y (mgr.26252) 1275 : cluster [DBG] pgmap v1840: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 536 B/s, 135 objects/s recovering 2024-09-13T15:08:36.445 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:36.473 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:36.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:36.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:36.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:36.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:36.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:36.726 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:36.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:36.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:36.853 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:36.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:36.910 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:36.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:36.978 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:36.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:37.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:37.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:37.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:37.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:37.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:37.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:37.277 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:08:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:08:37] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.51.0" 2024-09-13T15:08:37.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:37.350 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:37.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:37.480 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:37.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:37.584 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:37.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:37.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:37.720 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:37 smithi177 bash[68594]: cluster 2024-09-13T15:08:36.996350+0000 mgr.y (mgr.26252) 1276 : cluster [DBG] pgmap v1841: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 474 B/s, 119 objects/s recovering 2024-09-13T15:08:37.720 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:37 smithi177 bash[68594]: cluster 2024-09-13T15:08:36.996350+0000 mgr.y (mgr.26252) 1276 : cluster [DBG] pgmap v1841: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 474 B/s, 119 objects/s recovering 2024-09-13T15:08:37.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:37.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:37.840 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:37.863 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:37.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:37.967 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:37.997 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:37.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:38.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:37 smithi099 bash[84576]: cluster 2024-09-13T15:08:36.996350+0000 mgr.y (mgr.26252) 1276 : cluster [DBG] pgmap v1841: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 474 B/s, 119 objects/s recovering 2024-09-13T15:08:38.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:37 smithi099 bash[84576]: cluster 2024-09-13T15:08:36.996350+0000 mgr.y (mgr.26252) 1276 : cluster [DBG] pgmap v1841: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 474 B/s, 119 objects/s recovering 2024-09-13T15:08:38.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:37 smithi099 bash[87147]: cluster 2024-09-13T15:08:36.996350+0000 mgr.y (mgr.26252) 1276 : cluster [DBG] pgmap v1841: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 474 B/s, 119 objects/s recovering 2024-09-13T15:08:38.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:37 smithi099 bash[87147]: cluster 2024-09-13T15:08:36.996350+0000 mgr.y (mgr.26252) 1276 : cluster [DBG] pgmap v1841: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5843/78 objects degraded (7491.026%); 6723/78 objects misplaced (8619.231%); 474 B/s, 119 objects/s recovering 2024-09-13T15:08:38.100 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:38.122 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:38.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:38.225 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:38.251 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:38.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:38.354 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:38.381 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:38.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:38.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:38.509 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:38.612 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:38.634 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:38.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:38.737 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:38.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:38.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:38.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:38.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:38.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:38.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:39.027 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:39.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:39.131 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:39.140 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:39 smithi099 bash[87147]: cluster 2024-09-13T15:08:39.048378+0000 mon.a (mon.0) 2440 : cluster [WRN] Health check update: Degraded data redundancy: 5821/78 objects degraded (7462.821%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:39.141 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:39 smithi099 bash[87147]: cluster 2024-09-13T15:08:39.048378+0000 mon.a (mon.0) 2440 : cluster [WRN] Health check update: Degraded data redundancy: 5821/78 objects degraded (7462.821%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:39.141 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:39 smithi099 bash[84576]: cluster 2024-09-13T15:08:39.048378+0000 mon.a (mon.0) 2440 : cluster [WRN] Health check update: Degraded data redundancy: 5821/78 objects degraded (7462.821%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:39.141 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:39 smithi099 bash[84576]: cluster 2024-09-13T15:08:39.048378+0000 mon.a (mon.0) 2440 : cluster [WRN] Health check update: Degraded data redundancy: 5821/78 objects degraded (7462.821%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:39.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:39.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:39.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:39.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:39.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:39.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:39 smithi177 bash[68594]: cluster 2024-09-13T15:08:39.048378+0000 mon.a (mon.0) 2440 : cluster [WRN] Health check update: Degraded data redundancy: 5821/78 objects degraded (7462.821%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:39.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:39 smithi177 bash[68594]: cluster 2024-09-13T15:08:39.048378+0000 mon.a (mon.0) 2440 : cluster [WRN] Health check update: Degraded data redundancy: 5821/78 objects degraded (7462.821%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:39.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:39.404 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:39.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:39.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:39.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:39.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:39.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:39.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:39.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:39.765 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:39.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:39.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:39.893 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:39.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:39.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:40.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:40.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:40.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:40.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:40.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:40.164 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:40 smithi099 bash[84576]: cluster 2024-09-13T15:08:38.996956+0000 mgr.y (mgr.26252) 1277 : cluster [DBG] pgmap v1842: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 400 B/s, 179 objects/s recovering 2024-09-13T15:08:40.164 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:40 smithi099 bash[84576]: cluster 2024-09-13T15:08:38.996956+0000 mgr.y (mgr.26252) 1277 : cluster [DBG] pgmap v1842: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 400 B/s, 179 objects/s recovering 2024-09-13T15:08:40.164 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:40 smithi099 bash[87147]: cluster 2024-09-13T15:08:38.996956+0000 mgr.y (mgr.26252) 1277 : cluster [DBG] pgmap v1842: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 400 B/s, 179 objects/s recovering 2024-09-13T15:08:40.164 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:40 smithi099 bash[87147]: cluster 2024-09-13T15:08:38.996956+0000 mgr.y (mgr.26252) 1277 : cluster [DBG] pgmap v1842: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 400 B/s, 179 objects/s recovering 2024-09-13T15:08:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:40.266 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:40.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:40.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:40.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:08:40 smithi177 bash[61439]: ts=2024-09-13T15:08:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:40.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:40 smithi177 bash[68594]: cluster 2024-09-13T15:08:38.996956+0000 mgr.y (mgr.26252) 1277 : cluster [DBG] pgmap v1842: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 400 B/s, 179 objects/s recovering 2024-09-13T15:08:40.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:40 smithi177 bash[68594]: cluster 2024-09-13T15:08:38.996956+0000 mgr.y (mgr.26252) 1277 : cluster [DBG] pgmap v1842: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 400 B/s, 179 objects/s recovering 2024-09-13T15:08:40.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:40.415 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:40.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:40.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:40.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:40.547 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:40.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:40.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:40.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:40.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:40.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:40.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:40.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:40.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:40.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:41.036 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:41.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:41.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:41.166 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:41.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:41.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:41.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:41.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:41.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:41.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:41.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:41.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:41.563 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:41.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:41.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:41.692 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:41.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:41.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:41.820 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:41.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:41.923 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:41.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:41.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:42.056 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:42.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:42.187 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:42.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:42.311 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:42.322 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:42 smithi177 bash[68594]: cluster 2024-09-13T15:08:40.998021+0000 mgr.y (mgr.26252) 1278 : cluster [DBG] pgmap v1843: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:42.323 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:42 smithi177 bash[68594]: cluster 2024-09-13T15:08:40.998021+0000 mgr.y (mgr.26252) 1278 : cluster [DBG] pgmap v1843: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:42.338 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:42.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:42.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:42.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:42 smithi099 bash[84576]: cluster 2024-09-13T15:08:40.998021+0000 mgr.y (mgr.26252) 1278 : cluster [DBG] pgmap v1843: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:42.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:42 smithi099 bash[84576]: cluster 2024-09-13T15:08:40.998021+0000 mgr.y (mgr.26252) 1278 : cluster [DBG] pgmap v1843: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:42.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:42 smithi099 bash[87147]: cluster 2024-09-13T15:08:40.998021+0000 mgr.y (mgr.26252) 1278 : cluster [DBG] pgmap v1843: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:42.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:42 smithi099 bash[87147]: cluster 2024-09-13T15:08:40.998021+0000 mgr.y (mgr.26252) 1278 : cluster [DBG] pgmap v1843: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5821/78 objects degraded (7462.821%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:42.467 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:42.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:42.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:42.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:42.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:42.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:42.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:42.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:42.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:42.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:42.948 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:42.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:43.070 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:43.089 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:43.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:43.192 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:43.220 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:43.323 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:43.342 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:43.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:43.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:43.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:43.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:43.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:43.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:43.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:43.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:43.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:43.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:43.852 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:43.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:43.954 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:43.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:44.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:44.109 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:44.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:44.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:44.223 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:44 smithi099 bash[84576]: cluster 2024-09-13T15:08:42.998640+0000 mgr.y (mgr.26252) 1279 : cluster [DBG] pgmap v1844: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 179 objects/s recovering 2024-09-13T15:08:44.223 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:44 smithi099 bash[84576]: cluster 2024-09-13T15:08:42.998640+0000 mgr.y (mgr.26252) 1279 : cluster [DBG] pgmap v1844: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 179 objects/s recovering 2024-09-13T15:08:44.224 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:44 smithi099 bash[87147]: cluster 2024-09-13T15:08:42.998640+0000 mgr.y (mgr.26252) 1279 : cluster [DBG] pgmap v1844: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 179 objects/s recovering 2024-09-13T15:08:44.224 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:44 smithi099 bash[87147]: cluster 2024-09-13T15:08:42.998640+0000 mgr.y (mgr.26252) 1279 : cluster [DBG] pgmap v1844: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 179 objects/s recovering 2024-09-13T15:08:44.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:44.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:44.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:44.361 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:44.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:44 smithi177 bash[68594]: cluster 2024-09-13T15:08:42.998640+0000 mgr.y (mgr.26252) 1279 : cluster [DBG] pgmap v1844: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 179 objects/s recovering 2024-09-13T15:08:44.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:44 smithi177 bash[68594]: cluster 2024-09-13T15:08:42.998640+0000 mgr.y (mgr.26252) 1279 : cluster [DBG] pgmap v1844: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 179 objects/s recovering 2024-09-13T15:08:44.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:44.465 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:44.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:44.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:44.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:44.722 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:44.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:44.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:44.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:44.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:44.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:44.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:44.999 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:45.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:45.128 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:45.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:45.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:45.359 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.385 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:45.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.512 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:45.615 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:45.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:45.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.764 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:45.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:45.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:46.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:46.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:46.123 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:46.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:46.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:46.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:46.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:46 smithi177 bash[68594]: cluster 2024-09-13T15:08:44.999715+0000 mgr.y (mgr.26252) 1280 : cluster [DBG] pgmap v1845: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 210 objects/s recovering 2024-09-13T15:08:46.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:46 smithi177 bash[68594]: cluster 2024-09-13T15:08:44.999715+0000 mgr.y (mgr.26252) 1280 : cluster [DBG] pgmap v1845: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 210 objects/s recovering 2024-09-13T15:08:46.276 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:46.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:46 smithi099 bash[84576]: cluster 2024-09-13T15:08:44.999715+0000 mgr.y (mgr.26252) 1280 : cluster [DBG] pgmap v1845: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 210 objects/s recovering 2024-09-13T15:08:46.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:46 smithi099 bash[84576]: cluster 2024-09-13T15:08:44.999715+0000 mgr.y (mgr.26252) 1280 : cluster [DBG] pgmap v1845: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 210 objects/s recovering 2024-09-13T15:08:46.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:46 smithi099 bash[87147]: cluster 2024-09-13T15:08:44.999715+0000 mgr.y (mgr.26252) 1280 : cluster [DBG] pgmap v1845: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 210 objects/s recovering 2024-09-13T15:08:46.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:46 smithi099 bash[87147]: cluster 2024-09-13T15:08:44.999715+0000 mgr.y (mgr.26252) 1280 : cluster [DBG] pgmap v1845: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 210 objects/s recovering 2024-09-13T15:08:46.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:46.402 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:46.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:46.505 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:46.531 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:46.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:46.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:46.661 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:46.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:46.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:46.788 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:46.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:46.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:46.917 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:47.020 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:47.046 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:47.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:47.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:47.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:47.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:47.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:47.292 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:08:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:08:47] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.51.0" 2024-09-13T15:08:47.293 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:47 smithi099 bash[84576]: cluster 2024-09-13T15:08:46.559822+0000 mon.a (mon.0) 2441 : cluster [WRN] Health check update: Degraded data redundancy: 5547/78 objects degraded (7111.538%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:47.293 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:47 smithi099 bash[84576]: cluster 2024-09-13T15:08:46.559822+0000 mon.a (mon.0) 2441 : cluster [WRN] Health check update: Degraded data redundancy: 5547/78 objects degraded (7111.538%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:47.293 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:47 smithi099 bash[87147]: cluster 2024-09-13T15:08:46.559822+0000 mon.a (mon.0) 2441 : cluster [WRN] Health check update: Degraded data redundancy: 5547/78 objects degraded (7111.538%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:47.293 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:47 smithi099 bash[87147]: cluster 2024-09-13T15:08:46.559822+0000 mon.a (mon.0) 2441 : cluster [WRN] Health check update: Degraded data redundancy: 5547/78 objects degraded (7111.538%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:47.298 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:47.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:47.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:47.423 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:47.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:47.433 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:47 smithi177 bash[68594]: cluster 2024-09-13T15:08:46.559822+0000 mon.a (mon.0) 2441 : cluster [WRN] Health check update: Degraded data redundancy: 5547/78 objects degraded (7111.538%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:47.433 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:47 smithi177 bash[68594]: cluster 2024-09-13T15:08:46.559822+0000 mon.a (mon.0) 2441 : cluster [WRN] Health check update: Degraded data redundancy: 5547/78 objects degraded (7111.538%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:47.527 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:47.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:47.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:47.653 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:47.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:47.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:47.805 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:47.909 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:47.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:47.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:48.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:48.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:48.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:48.129 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:08:48.129 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:08:48.159 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:08:48.160 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:08:48.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:48.172 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:48 smithi099 bash[84576]: cluster 2024-09-13T15:08:47.000512+0000 mgr.y (mgr.26252) 1281 : cluster [DBG] pgmap v1846: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:48.172 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:48 smithi099 bash[84576]: cluster 2024-09-13T15:08:47.000512+0000 mgr.y (mgr.26252) 1281 : cluster [DBG] pgmap v1846: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:48.172 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:48 smithi099 bash[84576]: audit 2024-09-13T15:08:48.108097+0000 mon.a (mon.0) 2442 : audit [DBG] from='client.? 172.21.15.99:0/2989622544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:48.173 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:48 smithi099 bash[84576]: audit 2024-09-13T15:08:48.108097+0000 mon.a (mon.0) 2442 : audit [DBG] from='client.? 172.21.15.99:0/2989622544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:48.173 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:48 smithi099 bash[87147]: cluster 2024-09-13T15:08:47.000512+0000 mgr.y (mgr.26252) 1281 : cluster [DBG] pgmap v1846: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:48.173 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:48 smithi099 bash[87147]: cluster 2024-09-13T15:08:47.000512+0000 mgr.y (mgr.26252) 1281 : cluster [DBG] pgmap v1846: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:48.173 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:48 smithi099 bash[87147]: audit 2024-09-13T15:08:48.108097+0000 mon.a (mon.0) 2442 : audit [DBG] from='client.? 172.21.15.99:0/2989622544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:48.173 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:48 smithi099 bash[87147]: audit 2024-09-13T15:08:48.108097+0000 mon.a (mon.0) 2442 : audit [DBG] from='client.? 172.21.15.99:0/2989622544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:48.183 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:48.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:48.187 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:08:48.187 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:08:48.214 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:08:48.214 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:08:48.241 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:08:48.241 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:08:48.267 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:08:48.267 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:08:48.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:48.288 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:08:48.288 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:08:48.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:48 smithi177 bash[68594]: cluster 2024-09-13T15:08:47.000512+0000 mgr.y (mgr.26252) 1281 : cluster [DBG] pgmap v1846: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:48.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:48 smithi177 bash[68594]: cluster 2024-09-13T15:08:47.000512+0000 mgr.y (mgr.26252) 1281 : cluster [DBG] pgmap v1846: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5547/78 objects degraded (7111.538%); 6723/78 objects misplaced (8619.231%); 0 B/s, 156 objects/s recovering 2024-09-13T15:08:48.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:48 smithi177 bash[68594]: audit 2024-09-13T15:08:48.108097+0000 mon.a (mon.0) 2442 : audit [DBG] from='client.? 172.21.15.99:0/2989622544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:48.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:48 smithi177 bash[68594]: audit 2024-09-13T15:08:48.108097+0000 mon.a (mon.0) 2442 : audit [DBG] from='client.? 172.21.15.99:0/2989622544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:48.312 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:08:48.313 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:08:48.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:48.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:48.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:48.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:48.544 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:48.570 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:48.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:48.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:48.700 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:48.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:48.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:48.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:48.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:48.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:48.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:49.056 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:49.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:49.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:49.130 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_3' removed 2024-09-13T15:08:49.137 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:08:49.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:49.196 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[84576]: audit 2024-09-13T15:08:48.305038+0000 mon.a (mon.0) 2443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:49.196 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[84576]: audit 2024-09-13T15:08:48.305038+0000 mon.a (mon.0) 2443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:49.196 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[84576]: audit 2024-09-13T15:08:48.576074+0000 mon.c (mon.2) 100 : audit [INF] from='client.? 172.21.15.99:0/3218057187' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.196 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[84576]: audit 2024-09-13T15:08:48.576074+0000 mon.c (mon.2) 100 : audit [INF] from='client.? 172.21.15.99:0/3218057187' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[84576]: audit 2024-09-13T15:08:48.576391+0000 mon.a (mon.0) 2444 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[84576]: audit 2024-09-13T15:08:48.576391+0000 mon.a (mon.0) 2444 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[84576]: audit 2024-09-13T15:08:48.652449+0000 mon.a (mon.0) 2445 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:08:49.197 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[84576]: audit 2024-09-13T15:08:48.652449+0000 mon.a (mon.0) 2445 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:08:49.197 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[87147]: audit 2024-09-13T15:08:48.305038+0000 mon.a (mon.0) 2443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:49.197 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[87147]: audit 2024-09-13T15:08:48.305038+0000 mon.a (mon.0) 2443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:49.197 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[87147]: audit 2024-09-13T15:08:48.576074+0000 mon.c (mon.2) 100 : audit [INF] from='client.? 172.21.15.99:0/3218057187' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.197 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[87147]: audit 2024-09-13T15:08:48.576074+0000 mon.c (mon.2) 100 : audit [INF] from='client.? 172.21.15.99:0/3218057187' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.197 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[87147]: audit 2024-09-13T15:08:48.576391+0000 mon.a (mon.0) 2444 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.197 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[87147]: audit 2024-09-13T15:08:48.576391+0000 mon.a (mon.0) 2444 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.198 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[87147]: audit 2024-09-13T15:08:48.652449+0000 mon.a (mon.0) 2445 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:08:49.198 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:49 smithi099 bash[87147]: audit 2024-09-13T15:08:48.652449+0000 mon.a (mon.0) 2445 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:08:49.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:49.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:49.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:49.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:49.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:49.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:49.454 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:49 smithi177 bash[68594]: audit 2024-09-13T15:08:48.305038+0000 mon.a (mon.0) 2443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:49.454 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:49 smithi177 bash[68594]: audit 2024-09-13T15:08:48.305038+0000 mon.a (mon.0) 2443 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:49.454 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:49 smithi177 bash[68594]: audit 2024-09-13T15:08:48.576074+0000 mon.c (mon.2) 100 : audit [INF] from='client.? 172.21.15.99:0/3218057187' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.454 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:49 smithi177 bash[68594]: audit 2024-09-13T15:08:48.576074+0000 mon.c (mon.2) 100 : audit [INF] from='client.? 172.21.15.99:0/3218057187' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:49 smithi177 bash[68594]: audit 2024-09-13T15:08:48.576391+0000 mon.a (mon.0) 2444 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:49 smithi177 bash[68594]: audit 2024-09-13T15:08:48.576391+0000 mon.a (mon.0) 2444 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:49.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:49 smithi177 bash[68594]: audit 2024-09-13T15:08:48.652449+0000 mon.a (mon.0) 2445 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:08:49.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:49 smithi177 bash[68594]: audit 2024-09-13T15:08:48.652449+0000 mon.a (mon.0) 2445 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:08:49.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:49.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:49.569 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:49.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:49.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:49.655 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:08:49.655 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:08:49.655 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2024-09-13T15:08:49.655 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:08:49.690 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:49.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:49.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:49.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:49.850 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:49.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:49.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:49.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:49.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:50.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:50.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:50.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:50.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:50.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:50.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:50.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:50.361 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:50.389 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[87147]: cluster 2024-09-13T15:08:49.001058+0000 mgr.y (mgr.26252) 1282 : cluster [DBG] pgmap v1847: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5284/78 objects degraded (6774.359%); 6723/78 objects misplaced (8619.231%); 0 B/s, 232 objects/s recovering 2024-09-13T15:08:50.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[87147]: cluster 2024-09-13T15:08:49.001058+0000 mgr.y (mgr.26252) 1282 : cluster [DBG] pgmap v1847: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5284/78 objects degraded (6774.359%); 6723/78 objects misplaced (8619.231%); 0 B/s, 232 objects/s recovering 2024-09-13T15:08:50.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[87147]: cluster 2024-09-13T15:08:49.121804+0000 mon.a (mon.0) 2446 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:08:50.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[87147]: cluster 2024-09-13T15:08:49.121804+0000 mon.a (mon.0) 2446 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:08:50.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[87147]: audit 2024-09-13T15:08:49.130831+0000 mon.a (mon.0) 2447 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:50.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[87147]: audit 2024-09-13T15:08:49.130831+0000 mon.a (mon.0) 2447 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:50.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[87147]: audit 2024-09-13T15:08:49.130889+0000 mon.a (mon.0) 2448 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:08:50.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[87147]: audit 2024-09-13T15:08:49.130889+0000 mon.a (mon.0) 2448 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:08:50.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[87147]: cluster 2024-09-13T15:08:49.135801+0000 mon.a (mon.0) 2449 : cluster [DBG] osdmap e1079: 8 total, 8 up, 7 in 2024-09-13T15:08:50.390 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[87147]: cluster 2024-09-13T15:08:49.135801+0000 mon.a (mon.0) 2449 : cluster [DBG] osdmap e1079: 8 total, 8 up, 7 in 2024-09-13T15:08:50.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: cluster 2024-09-13T15:08:49.001058+0000 mgr.y (mgr.26252) 1282 : cluster [DBG] pgmap v1847: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5284/78 objects degraded (6774.359%); 6723/78 objects misplaced (8619.231%); 0 B/s, 232 objects/s recovering 2024-09-13T15:08:50.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: cluster 2024-09-13T15:08:49.001058+0000 mgr.y (mgr.26252) 1282 : cluster [DBG] pgmap v1847: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5284/78 objects degraded (6774.359%); 6723/78 objects misplaced (8619.231%); 0 B/s, 232 objects/s recovering 2024-09-13T15:08:50.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: cluster 2024-09-13T15:08:49.121804+0000 mon.a (mon.0) 2446 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:08:50.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: cluster 2024-09-13T15:08:49.121804+0000 mon.a (mon.0) 2446 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:08:50.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: audit 2024-09-13T15:08:49.130831+0000 mon.a (mon.0) 2447 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:50.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: audit 2024-09-13T15:08:49.130831+0000 mon.a (mon.0) 2447 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:50.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: audit 2024-09-13T15:08:49.130889+0000 mon.a (mon.0) 2448 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:08:50.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: audit 2024-09-13T15:08:49.130889+0000 mon.a (mon.0) 2448 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:08:50.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: cluster 2024-09-13T15:08:49.135801+0000 mon.a (mon.0) 2449 : cluster [DBG] osdmap e1079: 8 total, 8 up, 7 in 2024-09-13T15:08:50.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: cluster 2024-09-13T15:08:49.135801+0000 mon.a (mon.0) 2449 : cluster [DBG] osdmap e1079: 8 total, 8 up, 7 in 2024-09-13T15:08:50.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:50.491 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:50.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:50.502 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[61439]: ts=2024-09-13T15:08:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:08:50.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[68594]: cluster 2024-09-13T15:08:49.001058+0000 mgr.y (mgr.26252) 1282 : cluster [DBG] pgmap v1847: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5284/78 objects degraded (6774.359%); 6723/78 objects misplaced (8619.231%); 0 B/s, 232 objects/s recovering 2024-09-13T15:08:50.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[68594]: cluster 2024-09-13T15:08:49.001058+0000 mgr.y (mgr.26252) 1282 : cluster [DBG] pgmap v1847: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 5284/78 objects degraded (6774.359%); 6723/78 objects misplaced (8619.231%); 0 B/s, 232 objects/s recovering 2024-09-13T15:08:50.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[68594]: cluster 2024-09-13T15:08:49.121804+0000 mon.a (mon.0) 2446 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:08:50.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[68594]: cluster 2024-09-13T15:08:49.121804+0000 mon.a (mon.0) 2446 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:08:50.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[68594]: audit 2024-09-13T15:08:49.130831+0000 mon.a (mon.0) 2447 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:50.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[68594]: audit 2024-09-13T15:08:49.130831+0000 mon.a (mon.0) 2447 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:50.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[68594]: audit 2024-09-13T15:08:49.130889+0000 mon.a (mon.0) 2448 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:08:50.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[68594]: audit 2024-09-13T15:08:49.130889+0000 mon.a (mon.0) 2448 : audit [INF] from='client.? 172.21.15.99:0/2898420129' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:08:50.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[68594]: cluster 2024-09-13T15:08:49.135801+0000 mon.a (mon.0) 2449 : cluster [DBG] osdmap e1079: 8 total, 8 up, 7 in 2024-09-13T15:08:50.504 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:50 smithi177 bash[68594]: cluster 2024-09-13T15:08:49.135801+0000 mon.a (mon.0) 2449 : cluster [DBG] osdmap e1079: 8 total, 8 up, 7 in 2024-09-13T15:08:50.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:50.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:50.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:50.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:50.746 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:50.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:50.849 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:50.876 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:50.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:50.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:50.991 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:50 smithi099 bash[84576]: debug 2024-09-13T15:08:50.736+0000 7f0688bd2640 -1 mon.a@0(leader).osd e1079 definitely_dead 0 2024-09-13T15:08:50.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:08:51.007 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:51.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:51.134 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:51.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:51.142 INFO:teuthology.orchestra.run.smithi099.stderr:marked in osd.6. 2024-09-13T15:08:51.238 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:51.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:51 smithi099 bash[84576]: audit 2024-09-13T15:08:50.739399+0000 mon.a (mon.0) 2450 : audit [DBG] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T15:08:51.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:51 smithi099 bash[84576]: audit 2024-09-13T15:08:50.739399+0000 mon.a (mon.0) 2450 : audit [DBG] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T15:08:51.247 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:51 smithi099 bash[87147]: audit 2024-09-13T15:08:50.739399+0000 mon.a (mon.0) 2450 : audit [DBG] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T15:08:51.247 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:51 smithi099 bash[87147]: audit 2024-09-13T15:08:50.739399+0000 mon.a (mon.0) 2450 : audit [DBG] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T15:08:51.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:51.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:51.362 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:51.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:51.391 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:51 smithi177 bash[68594]: audit 2024-09-13T15:08:50.739399+0000 mon.a (mon.0) 2450 : audit [DBG] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T15:08:51.391 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:51 smithi177 bash[68594]: audit 2024-09-13T15:08:50.739399+0000 mon.a (mon.0) 2450 : audit [DBG] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-13T15:08:51.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:51.493 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:51.521 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:51.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:51.649 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:51.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:51.781 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:51.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:51.883 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:51.910 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:52.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:52.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:52.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:52.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:52.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:52.276 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:52.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:52.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:52.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[84576]: cluster 2024-09-13T15:08:51.001522+0000 mgr.y (mgr.26252) 1283 : cluster [DBG] pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[84576]: cluster 2024-09-13T15:08:51.001522+0000 mgr.y (mgr.26252) 1283 : cluster [DBG] pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[84576]: cluster 2024-09-13T15:08:51.132943+0000 mon.a (mon.0) 2451 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5284/78 objects degraded (6774.359%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[84576]: cluster 2024-09-13T15:08:51.132943+0000 mon.a (mon.0) 2451 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5284/78 objects degraded (6774.359%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[84576]: cluster 2024-09-13T15:08:51.132990+0000 mon.a (mon.0) 2452 : cluster [INF] Cluster is now healthy 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[84576]: cluster 2024-09-13T15:08:51.132990+0000 mon.a (mon.0) 2452 : cluster [INF] Cluster is now healthy 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[84576]: audit 2024-09-13T15:08:51.143243+0000 mon.a (mon.0) 2453 : audit [INF] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[84576]: audit 2024-09-13T15:08:51.143243+0000 mon.a (mon.0) 2453 : audit [INF] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[84576]: cluster 2024-09-13T15:08:51.154648+0000 mon.a (mon.0) 2454 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[84576]: cluster 2024-09-13T15:08:51.154648+0000 mon.a (mon.0) 2454 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[87147]: cluster 2024-09-13T15:08:51.001522+0000 mgr.y (mgr.26252) 1283 : cluster [DBG] pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:52.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[87147]: cluster 2024-09-13T15:08:51.001522+0000 mgr.y (mgr.26252) 1283 : cluster [DBG] pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:52.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[87147]: cluster 2024-09-13T15:08:51.132943+0000 mon.a (mon.0) 2451 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5284/78 objects degraded (6774.359%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:08:52.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[87147]: cluster 2024-09-13T15:08:51.132943+0000 mon.a (mon.0) 2451 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5284/78 objects degraded (6774.359%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:08:52.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[87147]: cluster 2024-09-13T15:08:51.132990+0000 mon.a (mon.0) 2452 : cluster [INF] Cluster is now healthy 2024-09-13T15:08:52.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[87147]: cluster 2024-09-13T15:08:51.132990+0000 mon.a (mon.0) 2452 : cluster [INF] Cluster is now healthy 2024-09-13T15:08:52.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[87147]: audit 2024-09-13T15:08:51.143243+0000 mon.a (mon.0) 2453 : audit [INF] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-13T15:08:52.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[87147]: audit 2024-09-13T15:08:51.143243+0000 mon.a (mon.0) 2453 : audit [INF] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-13T15:08:52.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[87147]: cluster 2024-09-13T15:08:51.154648+0000 mon.a (mon.0) 2454 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2024-09-13T15:08:52.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:52 smithi099 bash[87147]: cluster 2024-09-13T15:08:51.154648+0000 mon.a (mon.0) 2454 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2024-09-13T15:08:52.403 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:52.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:52.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:52.533 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:52.558 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:52.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:52.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:52 smithi177 bash[68594]: cluster 2024-09-13T15:08:51.001522+0000 mgr.y (mgr.26252) 1283 : cluster [DBG] pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:52 smithi177 bash[68594]: cluster 2024-09-13T15:08:51.001522+0000 mgr.y (mgr.26252) 1283 : cluster [DBG] pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:52 smithi177 bash[68594]: cluster 2024-09-13T15:08:51.132943+0000 mon.a (mon.0) 2451 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5284/78 objects degraded (6774.359%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:08:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:52 smithi177 bash[68594]: cluster 2024-09-13T15:08:51.132943+0000 mon.a (mon.0) 2451 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5284/78 objects degraded (6774.359%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:08:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:52 smithi177 bash[68594]: cluster 2024-09-13T15:08:51.132990+0000 mon.a (mon.0) 2452 : cluster [INF] Cluster is now healthy 2024-09-13T15:08:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:52 smithi177 bash[68594]: cluster 2024-09-13T15:08:51.132990+0000 mon.a (mon.0) 2452 : cluster [INF] Cluster is now healthy 2024-09-13T15:08:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:52 smithi177 bash[68594]: audit 2024-09-13T15:08:51.143243+0000 mon.a (mon.0) 2453 : audit [INF] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-13T15:08:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:52 smithi177 bash[68594]: audit 2024-09-13T15:08:51.143243+0000 mon.a (mon.0) 2453 : audit [INF] from='client.? 172.21.15.99:0/1402586935' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-13T15:08:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:52 smithi177 bash[68594]: cluster 2024-09-13T15:08:51.154648+0000 mon.a (mon.0) 2454 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2024-09-13T15:08:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:52 smithi177 bash[68594]: cluster 2024-09-13T15:08:51.154648+0000 mon.a (mon.0) 2454 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2024-09-13T15:08:52.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:52.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:52.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:52.791 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:52.817 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:52.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:52.920 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:52.947 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:53.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:53.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:53.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:53.180 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:53.209 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:53.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:53.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:53.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[84576]: cluster 2024-09-13T15:08:52.157273+0000 mon.a (mon.0) 2455 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2024-09-13T15:08:53.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[84576]: cluster 2024-09-13T15:08:52.157273+0000 mon.a (mon.0) 2455 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2024-09-13T15:08:53.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[84576]: audit 2024-09-13T15:08:52.968952+0000 mon.a (mon.0) 2456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]} : dispatch 2024-09-13T15:08:53.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[84576]: audit 2024-09-13T15:08:52.968952+0000 mon.a (mon.0) 2456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]} : dispatch 2024-09-13T15:08:53.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[84576]: cluster 2024-09-13T15:08:53.002001+0000 mgr.y (mgr.26252) 1284 : cluster [DBG] pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:53.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[84576]: cluster 2024-09-13T15:08:53.002001+0000 mgr.y (mgr.26252) 1284 : cluster [DBG] pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:53.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[87147]: cluster 2024-09-13T15:08:52.157273+0000 mon.a (mon.0) 2455 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2024-09-13T15:08:53.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[87147]: cluster 2024-09-13T15:08:52.157273+0000 mon.a (mon.0) 2455 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2024-09-13T15:08:53.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[87147]: audit 2024-09-13T15:08:52.968952+0000 mon.a (mon.0) 2456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]} : dispatch 2024-09-13T15:08:53.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[87147]: audit 2024-09-13T15:08:52.968952+0000 mon.a (mon.0) 2456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]} : dispatch 2024-09-13T15:08:53.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[87147]: cluster 2024-09-13T15:08:53.002001+0000 mgr.y (mgr.26252) 1284 : cluster [DBG] pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:53.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:53 smithi099 bash[87147]: cluster 2024-09-13T15:08:53.002001+0000 mgr.y (mgr.26252) 1284 : cluster [DBG] pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:53.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:53.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:53.373 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:08:53.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:53.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:53 smithi177 bash[68594]: cluster 2024-09-13T15:08:52.157273+0000 mon.a (mon.0) 2455 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2024-09-13T15:08:53.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:53 smithi177 bash[68594]: cluster 2024-09-13T15:08:52.157273+0000 mon.a (mon.0) 2455 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2024-09-13T15:08:53.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:53 smithi177 bash[68594]: audit 2024-09-13T15:08:52.968952+0000 mon.a (mon.0) 2456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]} : dispatch 2024-09-13T15:08:53.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:53 smithi177 bash[68594]: audit 2024-09-13T15:08:52.968952+0000 mon.a (mon.0) 2456 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]} : dispatch 2024-09-13T15:08:53.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:53 smithi177 bash[68594]: cluster 2024-09-13T15:08:53.002001+0000 mgr.y (mgr.26252) 1284 : cluster [DBG] pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:53.455 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:53 smithi177 bash[68594]: cluster 2024-09-13T15:08:53.002001+0000 mgr.y (mgr.26252) 1284 : cluster [DBG] pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-13T15:08:53.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:53.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:53.562 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T15:08:53.563 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T15:08:53.563 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-09-13T15:08:53.563 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-09-13T15:08:53.575 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:53.608 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:53.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:53.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:53.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:53.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:53.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:53.868 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:53.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:53.970 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:53.998 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:54.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:54.129 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:54.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:54.232 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:54.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:54 smithi099 bash[84576]: audit 2024-09-13T15:08:53.158751+0000 mon.a (mon.0) 2457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]': finished 2024-09-13T15:08:54.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:54 smithi099 bash[84576]: audit 2024-09-13T15:08:53.158751+0000 mon.a (mon.0) 2457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]': finished 2024-09-13T15:08:54.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:54 smithi099 bash[84576]: cluster 2024-09-13T15:08:53.174417+0000 mon.a (mon.0) 2458 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2024-09-13T15:08:54.241 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:54 smithi099 bash[84576]: cluster 2024-09-13T15:08:53.174417+0000 mon.a (mon.0) 2458 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2024-09-13T15:08:54.242 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:54 smithi099 bash[87147]: audit 2024-09-13T15:08:53.158751+0000 mon.a (mon.0) 2457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]': finished 2024-09-13T15:08:54.242 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:54 smithi099 bash[87147]: audit 2024-09-13T15:08:53.158751+0000 mon.a (mon.0) 2457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]': finished 2024-09-13T15:08:54.242 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:54 smithi099 bash[87147]: cluster 2024-09-13T15:08:53.174417+0000 mon.a (mon.0) 2458 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2024-09-13T15:08:54.242 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:54 smithi099 bash[87147]: cluster 2024-09-13T15:08:53.174417+0000 mon.a (mon.0) 2458 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2024-09-13T15:08:54.257 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:54.286 INFO:tasks.thrashosds.thrasher:Added osd 6 2024-09-13T15:08:54.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:54.385 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:54.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:54.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:54.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:54.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:54.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:54 smithi177 bash[68594]: audit 2024-09-13T15:08:53.158751+0000 mon.a (mon.0) 2457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]': finished 2024-09-13T15:08:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:54 smithi177 bash[68594]: audit 2024-09-13T15:08:53.158751+0000 mon.a (mon.0) 2457 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 0]}]': finished 2024-09-13T15:08:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:54 smithi177 bash[68594]: cluster 2024-09-13T15:08:53.174417+0000 mon.a (mon.0) 2458 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2024-09-13T15:08:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:54 smithi177 bash[68594]: cluster 2024-09-13T15:08:53.174417+0000 mon.a (mon.0) 2458 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2024-09-13T15:08:54.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:54.644 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:54.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:54.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:54.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:54.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:54.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:54.907 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:54.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:55.010 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:55.036 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:55.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:55.140 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:55.167 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:55.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:55.271 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:55.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:55.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:55.402 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:55.414 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[84576]: cluster 2024-09-13T15:08:54.167484+0000 mon.a (mon.0) 2459 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2024-09-13T15:08:55.414 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[84576]: cluster 2024-09-13T15:08:54.167484+0000 mon.a (mon.0) 2459 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2024-09-13T15:08:55.414 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[84576]: audit 2024-09-13T15:08:54.542512+0000 mon.a (mon.0) 2460 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:55.414 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[84576]: audit 2024-09-13T15:08:54.542512+0000 mon.a (mon.0) 2460 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:55.414 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[84576]: cluster 2024-09-13T15:08:55.002764+0000 mgr.y (mgr.26252) 1285 : cluster [DBG] pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.3 KiB/s, 2 objects/s recovering 2024-09-13T15:08:55.414 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[84576]: cluster 2024-09-13T15:08:55.002764+0000 mgr.y (mgr.26252) 1285 : cluster [DBG] pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.3 KiB/s, 2 objects/s recovering 2024-09-13T15:08:55.414 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[87147]: cluster 2024-09-13T15:08:54.167484+0000 mon.a (mon.0) 2459 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2024-09-13T15:08:55.414 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[87147]: cluster 2024-09-13T15:08:54.167484+0000 mon.a (mon.0) 2459 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2024-09-13T15:08:55.414 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[87147]: audit 2024-09-13T15:08:54.542512+0000 mon.a (mon.0) 2460 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:55.415 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[87147]: audit 2024-09-13T15:08:54.542512+0000 mon.a (mon.0) 2460 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:55.415 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[87147]: cluster 2024-09-13T15:08:55.002764+0000 mgr.y (mgr.26252) 1285 : cluster [DBG] pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.3 KiB/s, 2 objects/s recovering 2024-09-13T15:08:55.415 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:55 smithi099 bash[87147]: cluster 2024-09-13T15:08:55.002764+0000 mgr.y (mgr.26252) 1285 : cluster [DBG] pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.3 KiB/s, 2 objects/s recovering 2024-09-13T15:08:55.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:55.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:55.534 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:55.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:55 smithi177 bash[68594]: cluster 2024-09-13T15:08:54.167484+0000 mon.a (mon.0) 2459 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2024-09-13T15:08:55.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:55 smithi177 bash[68594]: cluster 2024-09-13T15:08:54.167484+0000 mon.a (mon.0) 2459 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2024-09-13T15:08:55.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:55 smithi177 bash[68594]: audit 2024-09-13T15:08:54.542512+0000 mon.a (mon.0) 2460 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:55.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:55 smithi177 bash[68594]: audit 2024-09-13T15:08:54.542512+0000 mon.a (mon.0) 2460 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:08:55.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:55 smithi177 bash[68594]: cluster 2024-09-13T15:08:55.002764+0000 mgr.y (mgr.26252) 1285 : cluster [DBG] pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.3 KiB/s, 2 objects/s recovering 2024-09-13T15:08:55.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:55 smithi177 bash[68594]: cluster 2024-09-13T15:08:55.002764+0000 mgr.y (mgr.26252) 1285 : cluster [DBG] pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.3 KiB/s, 2 objects/s recovering 2024-09-13T15:08:55.562 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:55.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:55.664 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:55.691 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:55.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:55.792 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:55.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:55.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:55.925 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:55.952 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:56.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:56.082 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:56.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:56.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:56.198 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:08:56.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:56.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:56.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:56.341 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:56.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:56.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:56.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:56.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:56.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:56.604 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:56.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:56.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:56.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:56.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:56.837 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:56.863 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:56.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:56.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:56.994 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:56.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:57.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:57.108 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:08:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:08:57] "GET /metrics HTTP/1.1" 200 36737 "" "Prometheus/2.51.0" 2024-09-13T15:08:57.125 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:57.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:57.228 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:57.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:57.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:57.385 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:57.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:57.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:57.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:57.617 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:57.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:57.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:57.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:57.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:57.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:57.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:57.899 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:57.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:58.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:58.030 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:58.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:58.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:58.145 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:58 smithi099 bash[84576]: cluster 2024-09-13T15:08:57.003523+0000 mgr.y (mgr.26252) 1286 : cluster [DBG] pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 99 KiB/s, 2 objects/s recovering 2024-09-13T15:08:58.145 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:08:58 smithi099 bash[84576]: cluster 2024-09-13T15:08:57.003523+0000 mgr.y (mgr.26252) 1286 : cluster [DBG] pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 99 KiB/s, 2 objects/s recovering 2024-09-13T15:08:58.145 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:58 smithi099 bash[87147]: cluster 2024-09-13T15:08:57.003523+0000 mgr.y (mgr.26252) 1286 : cluster [DBG] pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 99 KiB/s, 2 objects/s recovering 2024-09-13T15:08:58.145 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:08:58 smithi099 bash[87147]: cluster 2024-09-13T15:08:57.003523+0000 mgr.y (mgr.26252) 1286 : cluster [DBG] pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 99 KiB/s, 2 objects/s recovering 2024-09-13T15:08:58.154 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:58.154 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:58.167 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:58.270 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:58.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:58 smithi177 bash[68594]: cluster 2024-09-13T15:08:57.003523+0000 mgr.y (mgr.26252) 1286 : cluster [DBG] pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 99 KiB/s, 2 objects/s recovering 2024-09-13T15:08:58.284 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:08:58 smithi177 bash[68594]: cluster 2024-09-13T15:08:57.003523+0000 mgr.y (mgr.26252) 1286 : cluster [DBG] pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 99 KiB/s, 2 objects/s recovering 2024-09-13T15:08:58.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:58.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:58.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:58.432 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:58.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:58.535 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:58.561 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:58.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:58.664 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:58.691 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:58.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:58.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:58.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:58.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:58.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:58.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:59.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:59.086 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:59.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:59.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:59.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:59.287 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:08:59.287 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:08:59.287 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-09-13T15:08:59.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 reset_purged_snaps_last 2024-09-13T15:08:59.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:59.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:59.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:59.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:59.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:59.579 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:59.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:59.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:59.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:59.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:59.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:59.868 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:59.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:59.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:59.998 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:00.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:00.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:00.111 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:00 smithi099 bash[87147]: cluster 2024-09-13T15:08:59.004461+0000 mgr.y (mgr.26252) 1287 : cluster [DBG] pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 85 KiB/s, 2 objects/s recovering 2024-09-13T15:09:00.111 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:00 smithi099 bash[87147]: cluster 2024-09-13T15:08:59.004461+0000 mgr.y (mgr.26252) 1287 : cluster [DBG] pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 85 KiB/s, 2 objects/s recovering 2024-09-13T15:09:00.111 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:00 smithi099 bash[84576]: cluster 2024-09-13T15:08:59.004461+0000 mgr.y (mgr.26252) 1287 : cluster [DBG] pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 85 KiB/s, 2 objects/s recovering 2024-09-13T15:09:00.111 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:00 smithi099 bash[84576]: cluster 2024-09-13T15:08:59.004461+0000 mgr.y (mgr.26252) 1287 : cluster [DBG] pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 85 KiB/s, 2 objects/s recovering 2024-09-13T15:09:00.122 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:00.202 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:00.221 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:09:00.221 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:09:00.221 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:09:00.221 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:09:00.221 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:09:00.226 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:00.246 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:00.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:00.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:00 smithi177 bash[68594]: cluster 2024-09-13T15:08:59.004461+0000 mgr.y (mgr.26252) 1287 : cluster [DBG] pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 85 KiB/s, 2 objects/s recovering 2024-09-13T15:09:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:00 smithi177 bash[68594]: cluster 2024-09-13T15:08:59.004461+0000 mgr.y (mgr.26252) 1287 : cluster [DBG] pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 85 KiB/s, 2 objects/s recovering 2024-09-13T15:09:00.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:09:00 smithi177 bash[61439]: ts=2024-09-13T15:09:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:09:00.376 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:00.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:00.504 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:00.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:00.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:00.636 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:00.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:00.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:00.765 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:00.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:00.895 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:00.999 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:01.026 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:01.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:01.129 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:01.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:01.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:01.164 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:09:01.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:01.282 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:01.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:01.385 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:01.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:01.509 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:01.537 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:01.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:01.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:01.668 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:01.771 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:01.797 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:01.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:01.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:01.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:01.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:02.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:02.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:02.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:02.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:02.189 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:02.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:02.292 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:02.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:02 smithi177 bash[68594]: cluster 2024-09-13T15:09:01.005326+0000 mgr.y (mgr.26252) 1288 : cluster [DBG] pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 72 KiB/s, 1 objects/s recovering 2024-09-13T15:09:02.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:02 smithi177 bash[68594]: cluster 2024-09-13T15:09:01.005326+0000 mgr.y (mgr.26252) 1288 : cluster [DBG] pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 72 KiB/s, 1 objects/s recovering 2024-09-13T15:09:02.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:02.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:02 smithi099 bash[84576]: cluster 2024-09-13T15:09:01.005326+0000 mgr.y (mgr.26252) 1288 : cluster [DBG] pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 72 KiB/s, 1 objects/s recovering 2024-09-13T15:09:02.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:02 smithi099 bash[84576]: cluster 2024-09-13T15:09:01.005326+0000 mgr.y (mgr.26252) 1288 : cluster [DBG] pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 72 KiB/s, 1 objects/s recovering 2024-09-13T15:09:02.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:02 smithi099 bash[87147]: cluster 2024-09-13T15:09:01.005326+0000 mgr.y (mgr.26252) 1288 : cluster [DBG] pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 72 KiB/s, 1 objects/s recovering 2024-09-13T15:09:02.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:02 smithi099 bash[87147]: cluster 2024-09-13T15:09:01.005326+0000 mgr.y (mgr.26252) 1288 : cluster [DBG] pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 72 KiB/s, 1 objects/s recovering 2024-09-13T15:09:02.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:02.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.449 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:02.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:02.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:02.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:02.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:02.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:02.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:02.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:02.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:02.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:02.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:02.969 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:03.071 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:03.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:03.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:03.203 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:03.248 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:03.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:03.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:03.379 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:03.482 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:03.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:03.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:03.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:03.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:03.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:03.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:03.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:03.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:03.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:03.903 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:03.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:04.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:04.035 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:04.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:04.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:04.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:04.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:04.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:04.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:04 smithi177 bash[68594]: cluster 2024-09-13T15:09:03.005946+0000 mgr.y (mgr.26252) 1289 : cluster [DBG] pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 59 KiB/s, 1 objects/s recovering 2024-09-13T15:09:04.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:04 smithi177 bash[68594]: cluster 2024-09-13T15:09:03.005946+0000 mgr.y (mgr.26252) 1289 : cluster [DBG] pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 59 KiB/s, 1 objects/s recovering 2024-09-13T15:09:04.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:04 smithi177 bash[68594]: audit 2024-09-13T15:09:03.312252+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:04.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:04 smithi177 bash[68594]: audit 2024-09-13T15:09:03.312252+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:04.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:04 smithi177 bash[68594]: audit 2024-09-13T15:09:03.314281+0000 mon.a (mon.0) 2462 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:04.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:04 smithi177 bash[68594]: audit 2024-09-13T15:09:03.314281+0000 mon.a (mon.0) 2462 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:04.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:04.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[84576]: cluster 2024-09-13T15:09:03.005946+0000 mgr.y (mgr.26252) 1289 : cluster [DBG] pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 59 KiB/s, 1 objects/s recovering 2024-09-13T15:09:04.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[84576]: cluster 2024-09-13T15:09:03.005946+0000 mgr.y (mgr.26252) 1289 : cluster [DBG] pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 59 KiB/s, 1 objects/s recovering 2024-09-13T15:09:04.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[84576]: audit 2024-09-13T15:09:03.312252+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:04.302 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[84576]: audit 2024-09-13T15:09:03.312252+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:04.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[84576]: audit 2024-09-13T15:09:03.314281+0000 mon.a (mon.0) 2462 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:04.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[84576]: audit 2024-09-13T15:09:03.314281+0000 mon.a (mon.0) 2462 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:04.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[87147]: cluster 2024-09-13T15:09:03.005946+0000 mgr.y (mgr.26252) 1289 : cluster [DBG] pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 59 KiB/s, 1 objects/s recovering 2024-09-13T15:09:04.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[87147]: cluster 2024-09-13T15:09:03.005946+0000 mgr.y (mgr.26252) 1289 : cluster [DBG] pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 59 KiB/s, 1 objects/s recovering 2024-09-13T15:09:04.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[87147]: audit 2024-09-13T15:09:03.312252+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:04.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[87147]: audit 2024-09-13T15:09:03.312252+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:04.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[87147]: audit 2024-09-13T15:09:03.314281+0000 mon.a (mon.0) 2462 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:04.303 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:04 smithi099 bash[87147]: audit 2024-09-13T15:09:03.314281+0000 mon.a (mon.0) 2462 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:04.399 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:04.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:04.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:04.558 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:04.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:04.661 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:04.686 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:04.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:04.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:04.815 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:04.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:04.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:04.946 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:04.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:05.049 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:05.076 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:05.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:05.179 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:05.206 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:05.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:05.309 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:05.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:05.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:05.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:05.467 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:05.570 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:05.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:05.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:05.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:05.724 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:05.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:05.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:05.852 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:05.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:05.956 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:05.979 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:09:05.984 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:05.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:06.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:06.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:06.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:06.219 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:06.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:06.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:06.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:06.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:06 smithi177 bash[68594]: cluster 2024-09-13T15:09:05.006529+0000 mgr.y (mgr.26252) 1290 : cluster [DBG] pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 53 KiB/s, 1 objects/s recovering 2024-09-13T15:09:06.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:06 smithi177 bash[68594]: cluster 2024-09-13T15:09:05.006529+0000 mgr.y (mgr.26252) 1290 : cluster [DBG] pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 53 KiB/s, 1 objects/s recovering 2024-09-13T15:09:06.363 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:06 smithi099 bash[84576]: cluster 2024-09-13T15:09:05.006529+0000 mgr.y (mgr.26252) 1290 : cluster [DBG] pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 53 KiB/s, 1 objects/s recovering 2024-09-13T15:09:06.363 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:06 smithi099 bash[84576]: cluster 2024-09-13T15:09:05.006529+0000 mgr.y (mgr.26252) 1290 : cluster [DBG] pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 53 KiB/s, 1 objects/s recovering 2024-09-13T15:09:06.363 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:06 smithi099 bash[87147]: cluster 2024-09-13T15:09:05.006529+0000 mgr.y (mgr.26252) 1290 : cluster [DBG] pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 53 KiB/s, 1 objects/s recovering 2024-09-13T15:09:06.363 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:06 smithi099 bash[87147]: cluster 2024-09-13T15:09:05.006529+0000 mgr.y (mgr.26252) 1290 : cluster [DBG] pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 53 KiB/s, 1 objects/s recovering 2024-09-13T15:09:06.379 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:06.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:06.482 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:06.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:06.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:06.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:06.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:06.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:06.743 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:06.770 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:06.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:06.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:06.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:06.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:07.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:07.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:07.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:07.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:07.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:07.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:07.272 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:09:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:09:07] "GET /metrics HTTP/1.1" 200 34370 "" "Prometheus/2.51.0" 2024-09-13T15:09:07.287 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:07.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:07.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:07.419 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:07.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:07.522 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:07.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:07.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:07.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:07.682 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:07.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:07.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:07.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:07.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:07.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:07.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:08.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:08.073 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:08.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:08.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:08 smithi177 bash[68594]: cluster 2024-09-13T15:09:07.007199+0000 mgr.y (mgr.26252) 1291 : cluster [DBG] pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T15:09:08.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:08 smithi177 bash[68594]: cluster 2024-09-13T15:09:07.007199+0000 mgr.y (mgr.26252) 1291 : cluster [DBG] pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T15:09:08.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:08.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:08.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:08.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:08.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:08 smithi099 bash[84576]: cluster 2024-09-13T15:09:07.007199+0000 mgr.y (mgr.26252) 1291 : cluster [DBG] pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T15:09:08.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:08 smithi099 bash[84576]: cluster 2024-09-13T15:09:07.007199+0000 mgr.y (mgr.26252) 1291 : cluster [DBG] pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T15:09:08.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:08 smithi099 bash[87147]: cluster 2024-09-13T15:09:07.007199+0000 mgr.y (mgr.26252) 1291 : cluster [DBG] pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T15:09:08.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:08 smithi099 bash[87147]: cluster 2024-09-13T15:09:07.007199+0000 mgr.y (mgr.26252) 1291 : cluster [DBG] pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T15:09:08.334 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:08.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:08.437 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:08.464 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:08.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:08.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:08.590 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:08.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:08.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:08.721 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:08.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:08.824 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:08.850 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:08.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:08.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:08.983 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:08.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:09.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:09.113 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:09.174 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:09.216 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:09.240 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:09.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:09.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:09.370 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:09.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:09.473 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:09.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:09.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:09.604 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:09.632 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:09.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:09.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:09.763 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:09.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:09.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:09.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:09.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:10.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:10.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:10.120 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:76860 43.c 43:32823f6f:::benchmark_data_smithi099_171278_object76859:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1044)", 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.505531+0000", 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 555.61116077199995, 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.367279515, 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.121 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 76860 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505531+0000", 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505531+0000", 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505530+0000", 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505576+0000", 2024-09-13T15:09:10.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5682000000000001e-05 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505586+0000", 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0251000000000001e-05 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.505591+0000", 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6990000000000002e-06 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.506259+0000", 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00066873200000000001 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.506360+0000", 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100666 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.506472+0000", 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011189 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.509872+0000", 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003400213 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.511464+0000", 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001591914 2024-09-13T15:09:10.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872451+0000", 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.360987059 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872602+0000", 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000151049 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872810+0000", 2024-09-13T15:09:10.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020809100000000001 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:76873 43.c 43:3364d0f9:::benchmark_data_smithi099_171278_object76872:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1044)", 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.519249+0000", 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 555.59744290900005, 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36059039300000001, 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:09:10.128 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 76873 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519249+0000", 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519249+0000", 2024-09-13T15:09:10.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519247+0000", 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519347+0000", 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9692999999999996e-05 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519359+0000", 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1936999999999999e-05 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519365+0000", 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6670000000000001e-06 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519394+0000", 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9090000000000001e-05 2024-09-13T15:09:10.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519476+0000", 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2418999999999998e-05 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.519561+0000", 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4831000000000001e-05 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.872368+0000", 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35280668900000001 2024-09-13T15:09:10.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.873477+0000", 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001109699 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.879711+0000", 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062338250000000001 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.879784+0000", 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3349000000000005e-05 2024-09-13T15:09:10.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.879839+0000", 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4462999999999997e-05 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45572.0:76874 43.f e1044/1038 43:f90ebb9d:::benchmark_data_smithi099_171278_object76873:head v 1044'4822, mlcod=1044'4820)", 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T14:59:54.521563+0000", 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 555.59512855399998, 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36254724199999999, 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.134 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521563+0000", 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521563+0000", 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521562+0000", 2024-09-13T15:09:10.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.521592+0000", 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9462999999999999e-05 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866721+0000", 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34512911400000001 2024-09-13T15:09:10.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866745+0000", 2024-09-13T15:09:10.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4465999999999999e-05 2024-09-13T15:09:10.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866907+0000", 2024-09-13T15:09:10.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016150499999999999 2024-09-13T15:09:10.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.866962+0000", 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4996e-05 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.884044+0000", 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017082106 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T14:59:54.884110+0000", 2024-09-13T15:09:10.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6317999999999998e-05 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128590 43.a 43:5a3ee379:::benchmark_data_smithi099_171278_object12578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.385199+0000", 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 504.73149236699999, 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40636963100000001, 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.139 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128590 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385199+0000", 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385199+0000", 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385198+0000", 2024-09-13T15:09:10.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:09:10.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385202+0000", 2024-09-13T15:09:10.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2430000000000001e-06 2024-09-13T15:09:10.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.144 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385209+0000", 2024-09-13T15:09:10.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2949999999999996e-06 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.385214+0000", 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4360000000000001e-06 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.399134+0000", 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013919832 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.399203+0000", 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9133000000000004e-05 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.399278+0000", 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4923000000000001e-05 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414637+0000", 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015358627 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414656+0000", 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9293e-05 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.791316+0000", 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37665990399999999 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.791447+0000", 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013125699999999999 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.791569+0000", 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000121298 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128599 43.a 43:581bf91b:::benchmark_data_smithi099_171278_object12587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.394205+0000", 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 504.722486274, 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.41306469299999998, 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.148 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128599 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394205+0000", 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394205+0000", 2024-09-13T15:09:10.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394204+0000", 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394210+0000", 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1099999999999999e-06 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394219+0000", 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0680000000000003e-06 2024-09-13T15:09:10.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394224+0000", 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1660000000000002e-06 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.406611+0000", 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01238702 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.406714+0000", 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102629 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.406821+0000", 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000106704 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414660+0000", 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0078389289999999997 2024-09-13T15:09:10.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417173+0000", 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002512977 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.807025+0000", 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38985236000000001 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.807208+0000", 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018322999999999999 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.807270+0000", 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1671e-05 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128601 43.a 43:52aeec9c:::benchmark_data_smithi099_171278_object12589:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:09:10.155 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.394499+0000", 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 504.72219264400002, 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.41695673700000002, 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128601 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394499+0000", 2024-09-13T15:09:10.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394499+0000", 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394498+0000", 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394502+0000", 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8909999999999997e-06 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394508+0000", 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.665e-06 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.394511+0000", 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.123e-06 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.410919+0000", 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016407719000000001 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.411018+0000", 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9092999999999995e-05 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.411117+0000", 2024-09-13T15:09:10.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9035000000000003e-05 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414735+0000", 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036183999999999999 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.423114+0000", 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0083789250000000006 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811379+0000", 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38826455900000001 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811420+0000", 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1248000000000002e-05 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811456+0000", 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5746e-05 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.161 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128615 43.8 43:16ddc61b:::benchmark_data_smithi099_171278_object12603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.405083+0000", 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 504.711608152, 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.391743862, 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128615 2024-09-13T15:09:10.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405083+0000", 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405083+0000", 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405082+0000", 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405088+0000", 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0610000000000001e-06 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405099+0000", 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1324e-05 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405105+0000", 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0270000000000001e-06 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405147+0000", 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1678999999999999e-05 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405228+0000", 2024-09-13T15:09:10.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0724999999999997e-05 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.405335+0000", 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107205 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.409282+0000", 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039475090000000001 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.166 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.414778+0000", 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054955610000000004 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.796593+0000", 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38181541299999999 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.796711+0000", 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011769699999999999 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.796827+0000", 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011608 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128628 43.a 43:516a2081:::benchmark_data_smithi099_171278_object12616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.417499+0000", 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 504.699192568, 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39405699199999999, 2024-09-13T15:09:10.168 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128628 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417499+0000", 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417499+0000", 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417497+0000", 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.171 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417503+0000", 2024-09-13T15:09:10.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.037e-06 2024-09-13T15:09:10.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.171 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.171 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417518+0000", 2024-09-13T15:09:10.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4053000000000001e-05 2024-09-13T15:09:10.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417523+0000", 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9359999999999996e-06 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417562+0000", 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8753e-05 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417673+0000", 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110968 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.417781+0000", 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107543 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.423122+0000", 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053414420000000001 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.173 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.423152+0000", 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0076000000000001e-05 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811491+0000", 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38833923799999998 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811525+0000", 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3166000000000001e-05 2024-09-13T15:09:10.174 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811556+0000", 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1250000000000001e-05 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128633 43.a 43:521297ac:::benchmark_data_smithi099_171278_object12621:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.425405+0000", 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 504.69128623699999, 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38638104099999998, 2024-09-13T15:09:10.175 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.176 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.176 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.176 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:09:10.176 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:09:10.176 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128633 2024-09-13T15:09:10.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.176 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425405+0000", 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425405+0000", 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425404+0000", 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425409+0000", 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3770000000000004e-06 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425419+0000", 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5780000000000005e-06 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.425423+0000", 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1760000000000003e-06 2024-09-13T15:09:10.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427662+0000", 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022386179999999999 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427781+0000", 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011887699999999999 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427896+0000", 2024-09-13T15:09:10.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000114898 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.432933+0000", 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050373570000000001 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.432949+0000", 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5794000000000002e-05 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.180 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811644+0000", 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37869531000000001 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811720+0000", 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5735999999999995e-05 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.811786+0000", 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6519999999999993e-05 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.181 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:128635 43.c 43:334e8eab:::benchmark_data_smithi099_171278_object12623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1048)", 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:00:45.427651+0000", 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 504.689040482, 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.369737027, 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 128635 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427651+0000", 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427651+0000", 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427651+0000", 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427653+0000", 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.779e-06 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427659+0000", 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1889999999999998e-06 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427662+0000", 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1580000000000001e-06 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427695+0000", 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3723999999999998e-05 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427770+0000", 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4264999999999995e-05 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.427838+0000", 2024-09-13T15:09:10.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7879000000000005e-05 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.431234+0000", 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033965409999999999 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.434401+0000", 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003166992 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.797159+0000", 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36275744100000001 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.797288+0000", 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000129534 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:00:45.797388+0000", 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9950000000000004e-05 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.187 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45572.0:177757 43.d 43:b26fdded:::benchmark_data_smithi099_171278_object61745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1050)", 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:01:36.043745+0000", 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 454.07294681399998, 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.42954842999999998, 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45572", 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/768275000", 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 177757 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.188 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043745+0000", 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043745+0000", 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043743+0000", 2024-09-13T15:09:10.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043749+0000", 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4319999999999998e-06 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043760+0000", 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1226999999999999e-05 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043766+0000", 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0310000000000004e-06 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043801+0000", 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5246000000000001e-05 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043883+0000", 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1592000000000001e-05 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.043956+0000", 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3192e-05 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.047274+0000", 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003317445 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.048568+0000", 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001294707 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.473214+0000", 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.42464559600000001 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.473258+0000", 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4279000000000001e-05 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:01:36.473293+0000", 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4854000000000002e-05 2024-09-13T15:09:10.193 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:125833 44.a 44:56c508cb:::benchmark_data_smithi099_188632_object18629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1068)", 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:45.820786+0000", 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 204.29590544300001, 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38458231999999998, 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.194 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 125833 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.820786+0000", 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.820786+0000", 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.820784+0000", 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.820792+0000", 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8809999999999996e-06 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.820806+0000", 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3788000000000001e-05 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.820812+0000", 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9370000000000001e-06 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.824976+0000", 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041634929999999999 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.825098+0000", 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000122822 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.825210+0000", 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000111969 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.831738+0000", 2024-09-13T15:09:10.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065273270000000003 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.837005+0000", 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005267005 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:46.205292+0000", 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36828777600000001 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:46.205328+0000", 2024-09-13T15:09:10.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5556999999999999e-05 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:46.205368+0000", 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0302999999999997e-05 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:125849 44.a 44:51da5fc3:::benchmark_data_smithi099_188632_object18645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1068)", 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:05:45.842191+0000", 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 204.27450046300001, 2024-09-13T15:09:10.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36340076199999999, 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 125849 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.842191+0000", 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.842191+0000", 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.842190+0000", 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.842197+0000", 2024-09-13T15:09:10.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2280000000000001e-06 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.842210+0000", 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3572e-05 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.842217+0000", 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6610000000000003e-06 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.842279+0000", 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2461999999999999e-05 2024-09-13T15:09:10.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.842422+0000", 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014217400000000001 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.842557+0000", 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000135193 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.846243+0000", 2024-09-13T15:09:10.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003685799 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:45.848754+0000", 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025110060000000001 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:46.205523+0000", 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35676886899999999 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:46.205557+0000", 2024-09-13T15:09:10.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4446000000000002e-05 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:05:46.205592+0000", 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4790999999999999e-05 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:135503 44.a 44:560425ec:::benchmark_data_smithi099_188632_object28299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1068)", 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:02.143614+0000", 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 187.973077872, 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43646079700000001, 2024-09-13T15:09:10.206 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 135503 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.143614+0000", 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.143614+0000", 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.143612+0000", 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.143620+0000", 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1250000000000004e-06 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.143632+0000", 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2567e-05 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.143637+0000", 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.459e-06 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.143675+0000", 2024-09-13T15:09:10.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8047999999999999e-05 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.143767+0000", 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1929999999999996e-05 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.143844+0000", 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7424999999999998e-05 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.146987+0000", 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031426340000000001 2024-09-13T15:09:10.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.147282+0000", 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00029511199999999999 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.579823+0000", 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.432541224 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.580011+0000", 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000187873 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:02.580074+0000", 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3583000000000004e-05 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:170171 44.a 44:5e49f268:::benchmark_data_smithi099_188632_object62967:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1070)", 2024-09-13T15:09:10.212 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:07:00.376562+0000", 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 129.74012952000001, 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.46227384700000002, 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 170171 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.376562+0000", 2024-09-13T15:09:10.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.376562+0000", 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.376560+0000", 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.376569+0000", 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3600000000000002e-06 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.376591+0000", 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1364000000000001e-05 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.376599+0000", 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0430000000000008e-06 2024-09-13T15:09:10.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.376691+0000", 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1896000000000002e-05 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.376931+0000", 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024065499999999999 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.377169+0000", 2024-09-13T15:09:10.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023742699999999999 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.382464+0000", 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052955609999999998 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.382617+0000", 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015301299999999999 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.838727+0000", 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.456109761 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.838785+0000", 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8374e-05 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.838836+0000", 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0373e-05 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.218 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213425 44.a 44:5900890a:::benchmark_data_smithi099_188632_object106221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.108465+0000", 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 56.008226264000001, 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.60089703299999997, 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:09:10.219 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213425 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108465+0000", 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108465+0000", 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108464+0000", 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108471+0000", 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.204e-06 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108486+0000", 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4640999999999999e-05 2024-09-13T15:09:10.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108492+0000", 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5339999999999996e-06 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108536+0000", 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3627000000000001e-05 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108664+0000", 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012790699999999999 2024-09-13T15:09:10.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108784+0000", 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120684 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.115159+0000", 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063750070000000002 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.115169+0000", 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0076e-05 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709261+0000", 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.59409184900000001 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709318+0000", 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6325e-05 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709362+0000", 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4634e-05 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213436 44.a 44:5fef46cf:::benchmark_data_smithi099_188632_object106232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.138114+0000", 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 55.978577884000003, 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.57135144800000004, 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.225 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213436 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138114+0000", 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138114+0000", 2024-09-13T15:09:10.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138112+0000", 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138119+0000", 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3990000000000001e-06 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138131+0000", 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2343e-05 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138136+0000", 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3250000000000001e-06 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138168+0000", 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2678000000000001e-05 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138258+0000", 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9660999999999994e-05 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138333+0000", 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4777000000000006e-05 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.143497+0000", 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051641050000000004 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.148562+0000", 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050652279999999997 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709391+0000", 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.56082900400000002 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709428+0000", 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6557000000000003e-05 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709465+0000", 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7490999999999997e-05 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213437 44.a 44:5b63eef5:::benchmark_data_smithi099_188632_object106233:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.138145+0000", 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 55.978546717, 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.57140967799999998, 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.231 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213437 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138145+0000", 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138145+0000", 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138144+0000", 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138146+0000", 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7370000000000001e-06 2024-09-13T15:09:10.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138150+0000", 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6349999999999999e-06 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138152+0000", 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2189999999999998e-06 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141238+0000", 2024-09-13T15:09:10.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030859199999999998 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141369+0000", 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000131359 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141505+0000", 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013533600000000001 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.145190+0000", 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036849109999999999 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.148772+0000", 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035820600000000002 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709497+0000", 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.56072498299999995 2024-09-13T15:09:10.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709522+0000", 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5346000000000001e-05 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709554+0000", 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2471999999999997e-05 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213439 44.a 44:5b71bed8:::benchmark_data_smithi099_188632_object106235:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.141274+0000", 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 55.975417262000001, 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.56835986699999996, 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213439 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141274+0000", 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141274+0000", 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141273+0000", 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141278+0000", 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3630000000000004e-06 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141288+0000", 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0655e-05 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141294+0000", 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7309999999999998e-06 2024-09-13T15:09:10.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.143516+0000", 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002222183 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.143586+0000", 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9492000000000005e-05 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.143651+0000", 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4654999999999997e-05 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.145296+0000", 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001645758 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.153894+0000", 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0085979769999999997 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709576+0000", 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.55568201299999997 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709600+0000", 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3839e-05 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709634+0000", 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3964000000000002e-05 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.243 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213444 44.a 44:56de2f64:::benchmark_data_smithi099_188632_object106240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.159431+0000", 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 55.957260558000002, 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.55524475500000003, 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213444 2024-09-13T15:09:10.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159431+0000", 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159431+0000", 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:09:10.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159429+0000", 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159437+0000", 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.447e-06 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159452+0000", 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5247e-05 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:09:10.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159458+0000", 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0929999999999999e-06 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159494+0000", 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6439999999999997e-05 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159586+0000", 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2042999999999999e-05 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159662+0000", 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5417e-05 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.162711+0000", 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00304931 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.165941+0000", 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032297720000000001 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.714579+0000", 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.54863759599999995 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.714633+0000", 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4481999999999998e-05 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.714676+0000", 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2654000000000002e-05 2024-09-13T15:09:10.249 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.250 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.250 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.250 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:09:10.250 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:09:10.250 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:09:10.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:10.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:10 smithi099 bash[84576]: cluster 2024-09-13T15:09:09.008187+0000 mgr.y (mgr.26252) 1292 : cluster [DBG] pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:10.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:10 smithi099 bash[84576]: cluster 2024-09-13T15:09:09.008187+0000 mgr.y (mgr.26252) 1292 : cluster [DBG] pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:10.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:10 smithi099 bash[87147]: cluster 2024-09-13T15:09:09.008187+0000 mgr.y (mgr.26252) 1292 : cluster [DBG] pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:10.252 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:10 smithi099 bash[87147]: cluster 2024-09-13T15:09:09.008187+0000 mgr.y (mgr.26252) 1292 : cluster [DBG] pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:10.262 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:09:10 smithi177 bash[61439]: ts=2024-09-13T15:09:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:09:10.263 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:10 smithi177 bash[68594]: cluster 2024-09-13T15:09:09.008187+0000 mgr.y (mgr.26252) 1292 : cluster [DBG] pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:10.263 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:10 smithi177 bash[68594]: cluster 2024-09-13T15:09:09.008187+0000 mgr.y (mgr.26252) 1292 : cluster [DBG] pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:10.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:10.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:10.410 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:10.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:10.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:10.536 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:10.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:10.639 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:10.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:10.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:10.768 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:10.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:10.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:10.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:10.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:10.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:11.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:11.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:09:11.056 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:11.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:11.159 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:11.184 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:11.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:11.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:11.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:11.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:11.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:11.446 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:11.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:11.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:11.577 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:11.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:11.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:11.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:11.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:11.840 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:11.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:11.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:11.967 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:11.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:12.070 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:12.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:12.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:12.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:12.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:12.330 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:12.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:12 smithi177 bash[68594]: cluster 2024-09-13T15:09:11.009033+0000 mgr.y (mgr.26252) 1293 : cluster [DBG] pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:12.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:12 smithi177 bash[68594]: cluster 2024-09-13T15:09:11.009033+0000 mgr.y (mgr.26252) 1293 : cluster [DBG] pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:12.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:12.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:12 smithi099 bash[84576]: cluster 2024-09-13T15:09:11.009033+0000 mgr.y (mgr.26252) 1293 : cluster [DBG] pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:12.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:12 smithi099 bash[84576]: cluster 2024-09-13T15:09:11.009033+0000 mgr.y (mgr.26252) 1293 : cluster [DBG] pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:12.359 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:12 smithi099 bash[87147]: cluster 2024-09-13T15:09:11.009033+0000 mgr.y (mgr.26252) 1293 : cluster [DBG] pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:12.359 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:12 smithi099 bash[87147]: cluster 2024-09-13T15:09:11.009033+0000 mgr.y (mgr.26252) 1293 : cluster [DBG] pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:12.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:12.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:12.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:12.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:12.589 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:12.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:12.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:12.713 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:12.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:12.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:12.844 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:12.872 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:12.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:12.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:12.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:13.073 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:09:13.073 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:09:13.093 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:09:13.093 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:09:13.098 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:13.103 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_4' created 2024-09-13T15:09:13.122 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:09:13.122 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:09:13.127 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:13.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:13.138 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:09:13.139 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:09:13.156 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:09:13.156 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:09:13.172 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:09:13.172 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:09:13.188 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:09:13.188 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:09:13.206 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:09:13.206 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:09:13.230 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:13.257 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:13.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:13.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:13.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:13 smithi177 bash[68594]: audit 2024-09-13T15:09:12.685080+0000 mon.c (mon.2) 101 : audit [INF] from='client.? 172.21.15.99:0/2075596166' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:13 smithi177 bash[68594]: audit 2024-09-13T15:09:12.685080+0000 mon.c (mon.2) 101 : audit [INF] from='client.? 172.21.15.99:0/2075596166' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:13 smithi177 bash[68594]: audit 2024-09-13T15:09:12.685408+0000 mon.a (mon.0) 2463 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:13 smithi177 bash[68594]: audit 2024-09-13T15:09:12.685408+0000 mon.a (mon.0) 2463 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:13 smithi177 bash[68594]: audit 2024-09-13T15:09:13.055976+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.99:0/1075422771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:13.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:13 smithi177 bash[68594]: audit 2024-09-13T15:09:13.055976+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.99:0/1075422771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:13.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:13.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:13.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[84576]: audit 2024-09-13T15:09:12.685080+0000 mon.c (mon.2) 101 : audit [INF] from='client.? 172.21.15.99:0/2075596166' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[84576]: audit 2024-09-13T15:09:12.685080+0000 mon.c (mon.2) 101 : audit [INF] from='client.? 172.21.15.99:0/2075596166' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[84576]: audit 2024-09-13T15:09:12.685408+0000 mon.a (mon.0) 2463 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.408 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[84576]: audit 2024-09-13T15:09:12.685408+0000 mon.a (mon.0) 2463 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.408 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[84576]: audit 2024-09-13T15:09:13.055976+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.99:0/1075422771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:13.408 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[84576]: audit 2024-09-13T15:09:13.055976+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.99:0/1075422771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:13.408 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[87147]: audit 2024-09-13T15:09:12.685080+0000 mon.c (mon.2) 101 : audit [INF] from='client.? 172.21.15.99:0/2075596166' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.408 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[87147]: audit 2024-09-13T15:09:12.685080+0000 mon.c (mon.2) 101 : audit [INF] from='client.? 172.21.15.99:0/2075596166' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.408 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[87147]: audit 2024-09-13T15:09:12.685408+0000 mon.a (mon.0) 2463 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.408 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[87147]: audit 2024-09-13T15:09:12.685408+0000 mon.a (mon.0) 2463 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T15:09:13.408 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[87147]: audit 2024-09-13T15:09:13.055976+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.99:0/1075422771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:13.408 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:13 smithi099 bash[87147]: audit 2024-09-13T15:09:13.055976+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.99:0/1075422771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:13.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:13.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:13.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:13.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:13.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:13.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:13.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:13.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:13.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:13.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:13.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:13.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:14.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:14.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:14.130 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:14.140 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[84576]: cluster 2024-09-13T15:09:13.009534+0000 mgr.y (mgr.26252) 1294 : cluster [DBG] pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[84576]: cluster 2024-09-13T15:09:13.009534+0000 mgr.y (mgr.26252) 1294 : cluster [DBG] pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[84576]: audit 2024-09-13T15:09:13.095872+0000 mon.a (mon.0) 2464 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[84576]: audit 2024-09-13T15:09:13.095872+0000 mon.a (mon.0) 2464 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[84576]: cluster 2024-09-13T15:09:13.099531+0000 mon.a (mon.0) 2465 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[84576]: cluster 2024-09-13T15:09:13.099531+0000 mon.a (mon.0) 2465 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[87147]: cluster 2024-09-13T15:09:13.009534+0000 mgr.y (mgr.26252) 1294 : cluster [DBG] pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[87147]: cluster 2024-09-13T15:09:13.009534+0000 mgr.y (mgr.26252) 1294 : cluster [DBG] pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[87147]: audit 2024-09-13T15:09:13.095872+0000 mon.a (mon.0) 2464 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[87147]: audit 2024-09-13T15:09:13.095872+0000 mon.a (mon.0) 2464 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[87147]: cluster 2024-09-13T15:09:13.099531+0000 mon.a (mon.0) 2465 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2024-09-13T15:09:14.141 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:14 smithi099 bash[87147]: cluster 2024-09-13T15:09:13.099531+0000 mon.a (mon.0) 2465 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2024-09-13T15:09:14.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:14.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:14.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:14.277 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:14.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:14.380 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:14.392 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:14 smithi177 bash[68594]: cluster 2024-09-13T15:09:13.009534+0000 mgr.y (mgr.26252) 1294 : cluster [DBG] pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:14.392 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:14 smithi177 bash[68594]: cluster 2024-09-13T15:09:13.009534+0000 mgr.y (mgr.26252) 1294 : cluster [DBG] pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:14.392 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:14 smithi177 bash[68594]: audit 2024-09-13T15:09:13.095872+0000 mon.a (mon.0) 2464 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T15:09:14.392 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:14 smithi177 bash[68594]: audit 2024-09-13T15:09:13.095872+0000 mon.a (mon.0) 2464 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T15:09:14.392 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:14 smithi177 bash[68594]: cluster 2024-09-13T15:09:13.099531+0000 mon.a (mon.0) 2465 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2024-09-13T15:09:14.392 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:14 smithi177 bash[68594]: cluster 2024-09-13T15:09:13.099531+0000 mon.a (mon.0) 2465 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2024-09-13T15:09:14.409 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:14.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:14.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:14.535 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:14.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:14.638 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:14.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:14.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:14.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:14.798 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:14.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:14.900 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:14.928 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:14.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:15.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:15.060 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:15.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:15.101 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T15:09:15.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:15.183 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:15.285 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:15.297 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:15 smithi177 bash[68594]: cluster 2024-09-13T15:09:14.102537+0000 mon.a (mon.0) 2466 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2024-09-13T15:09:15.297 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:15 smithi177 bash[68594]: cluster 2024-09-13T15:09:14.102537+0000 mon.a (mon.0) 2466 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2024-09-13T15:09:15.297 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:15 smithi177 bash[68594]: audit 2024-09-13T15:09:14.534611+0000 mon.a (mon.0) 2467 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:15.297 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:15 smithi177 bash[68594]: audit 2024-09-13T15:09:14.534611+0000 mon.a (mon.0) 2467 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:15.297 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:15 smithi177 bash[68594]: cluster 2024-09-13T15:09:15.099107+0000 mon.a (mon.0) 2468 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:15.297 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:15 smithi177 bash[68594]: cluster 2024-09-13T15:09:15.099107+0000 mon.a (mon.0) 2468 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:15.297 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:15 smithi177 bash[68594]: audit 2024-09-13T15:09:15.101359+0000 mon.a (mon.0) 2469 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:09:15.297 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:15 smithi177 bash[68594]: audit 2024-09-13T15:09:15.101359+0000 mon.a (mon.0) 2469 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:09:15.298 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:15 smithi177 bash[68594]: cluster 2024-09-13T15:09:15.110507+0000 mon.a (mon.0) 2470 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2024-09-13T15:09:15.298 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:15 smithi177 bash[68594]: cluster 2024-09-13T15:09:15.110507+0000 mon.a (mon.0) 2470 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2024-09-13T15:09:15.311 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:15.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:15.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:15.441 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:15.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:15.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[87147]: cluster 2024-09-13T15:09:14.102537+0000 mon.a (mon.0) 2466 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2024-09-13T15:09:15.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[87147]: cluster 2024-09-13T15:09:14.102537+0000 mon.a (mon.0) 2466 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2024-09-13T15:09:15.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[87147]: audit 2024-09-13T15:09:14.534611+0000 mon.a (mon.0) 2467 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:15.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[87147]: audit 2024-09-13T15:09:14.534611+0000 mon.a (mon.0) 2467 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:15.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[87147]: cluster 2024-09-13T15:09:15.099107+0000 mon.a (mon.0) 2468 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:15.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[87147]: cluster 2024-09-13T15:09:15.099107+0000 mon.a (mon.0) 2468 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:15.458 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[87147]: audit 2024-09-13T15:09:15.101359+0000 mon.a (mon.0) 2469 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:09:15.459 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[87147]: audit 2024-09-13T15:09:15.101359+0000 mon.a (mon.0) 2469 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:09:15.459 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[87147]: cluster 2024-09-13T15:09:15.110507+0000 mon.a (mon.0) 2470 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2024-09-13T15:09:15.459 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[87147]: cluster 2024-09-13T15:09:15.110507+0000 mon.a (mon.0) 2470 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2024-09-13T15:09:15.459 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[84576]: cluster 2024-09-13T15:09:14.102537+0000 mon.a (mon.0) 2466 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2024-09-13T15:09:15.459 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[84576]: cluster 2024-09-13T15:09:14.102537+0000 mon.a (mon.0) 2466 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2024-09-13T15:09:15.459 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[84576]: audit 2024-09-13T15:09:14.534611+0000 mon.a (mon.0) 2467 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:15.459 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[84576]: audit 2024-09-13T15:09:14.534611+0000 mon.a (mon.0) 2467 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:15.459 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[84576]: cluster 2024-09-13T15:09:15.099107+0000 mon.a (mon.0) 2468 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:15.459 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[84576]: cluster 2024-09-13T15:09:15.099107+0000 mon.a (mon.0) 2468 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:15.459 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[84576]: audit 2024-09-13T15:09:15.101359+0000 mon.a (mon.0) 2469 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:09:15.460 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[84576]: audit 2024-09-13T15:09:15.101359+0000 mon.a (mon.0) 2469 : audit [INF] from='client.? 172.21.15.99:0/2444602357' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:09:15.460 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[84576]: cluster 2024-09-13T15:09:15.110507+0000 mon.a (mon.0) 2470 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2024-09-13T15:09:15.460 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:15 smithi099 bash[84576]: cluster 2024-09-13T15:09:15.110507+0000 mon.a (mon.0) 2470 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2024-09-13T15:09:15.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:15.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:15.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:15.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:15.704 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:15.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:15.807 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:15.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:15.940 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:15.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:15.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:16.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:16.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:16.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:16.201 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:16.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:16.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:16.332 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:16.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:16.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:16.463 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:16.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:16 smithi177 bash[68594]: cluster 2024-09-13T15:09:15.010404+0000 mgr.y (mgr.26252) 1295 : cluster [DBG] pgmap v1867: 89 pgs: 6 creating+peering, 6 unknown, 77 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:16.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:16 smithi177 bash[68594]: cluster 2024-09-13T15:09:15.010404+0000 mgr.y (mgr.26252) 1295 : cluster [DBG] pgmap v1867: 89 pgs: 6 creating+peering, 6 unknown, 77 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:16.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:16 smithi177 bash[68594]: cluster 2024-09-13T15:09:15.121237+0000 mon.a (mon.0) 2471 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:09:16.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:16 smithi177 bash[68594]: cluster 2024-09-13T15:09:15.121237+0000 mon.a (mon.0) 2471 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:09:16.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:16.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:16.585 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:16 smithi099 bash[84576]: cluster 2024-09-13T15:09:15.010404+0000 mgr.y (mgr.26252) 1295 : cluster [DBG] pgmap v1867: 89 pgs: 6 creating+peering, 6 unknown, 77 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:16.585 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:16 smithi099 bash[84576]: cluster 2024-09-13T15:09:15.010404+0000 mgr.y (mgr.26252) 1295 : cluster [DBG] pgmap v1867: 89 pgs: 6 creating+peering, 6 unknown, 77 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:16.585 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:16 smithi099 bash[84576]: cluster 2024-09-13T15:09:15.121237+0000 mon.a (mon.0) 2471 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:09:16.585 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:16 smithi099 bash[84576]: cluster 2024-09-13T15:09:15.121237+0000 mon.a (mon.0) 2471 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:09:16.585 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:16 smithi099 bash[87147]: cluster 2024-09-13T15:09:15.010404+0000 mgr.y (mgr.26252) 1295 : cluster [DBG] pgmap v1867: 89 pgs: 6 creating+peering, 6 unknown, 77 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:16.586 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:16 smithi099 bash[87147]: cluster 2024-09-13T15:09:15.010404+0000 mgr.y (mgr.26252) 1295 : cluster [DBG] pgmap v1867: 89 pgs: 6 creating+peering, 6 unknown, 77 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:16.586 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:16 smithi099 bash[87147]: cluster 2024-09-13T15:09:15.121237+0000 mon.a (mon.0) 2471 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:09:16.586 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:16 smithi099 bash[87147]: cluster 2024-09-13T15:09:15.121237+0000 mon.a (mon.0) 2471 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:09:16.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:16.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:16.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:16.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:16.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:16.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:16.855 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:16.882 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:16.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:16.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:17.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:17.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:17.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-09-13T15:09:17.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:17.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:17.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:17.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:17.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:17.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:17.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:09:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:09:17] "GET /metrics HTTP/1.1" 200 34370 "" "Prometheus/2.51.0" 2024-09-13T15:09:17.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:17.400 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:17.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:17.530 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:17.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:17.633 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:17.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:17.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:17.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:17.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:17.893 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:17.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:17.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:18.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:18.049 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:18.152 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:18.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:18.284 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:18.295 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:18 smithi099 bash[84576]: cluster 2024-09-13T15:09:17.010967+0000 mgr.y (mgr.26252) 1296 : cluster [DBG] pgmap v1869: 89 pgs: 6 creating+peering, 4 unknown, 79 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:18.296 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:18 smithi099 bash[84576]: cluster 2024-09-13T15:09:17.010967+0000 mgr.y (mgr.26252) 1296 : cluster [DBG] pgmap v1869: 89 pgs: 6 creating+peering, 4 unknown, 79 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:18.296 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:18 smithi099 bash[87147]: cluster 2024-09-13T15:09:17.010967+0000 mgr.y (mgr.26252) 1296 : cluster [DBG] pgmap v1869: 89 pgs: 6 creating+peering, 4 unknown, 79 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:18.296 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:18 smithi099 bash[87147]: cluster 2024-09-13T15:09:17.010967+0000 mgr.y (mgr.26252) 1296 : cluster [DBG] pgmap v1869: 89 pgs: 6 creating+peering, 4 unknown, 79 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:18.310 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:18.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:18.424 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:18 smithi177 bash[68594]: cluster 2024-09-13T15:09:17.010967+0000 mgr.y (mgr.26252) 1296 : cluster [DBG] pgmap v1869: 89 pgs: 6 creating+peering, 4 unknown, 79 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:18.424 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:18 smithi177 bash[68594]: cluster 2024-09-13T15:09:17.010967+0000 mgr.y (mgr.26252) 1296 : cluster [DBG] pgmap v1869: 89 pgs: 6 creating+peering, 4 unknown, 79 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:18.441 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:18.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:18.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:18.570 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:18.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:18.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:18.697 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:18.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:18.800 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:18.828 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:18.931 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:18.960 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:18.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:19.063 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:19.092 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:19.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:19.195 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:19.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:19.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:19.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:19 smithi177 bash[68594]: audit 2024-09-13T15:09:18.311161+0000 mon.a (mon.0) 2472 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:19.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:19 smithi177 bash[68594]: audit 2024-09-13T15:09:18.311161+0000 mon.a (mon.0) 2472 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:19.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:19 smithi177 bash[68594]: audit 2024-09-13T15:09:18.312388+0000 mon.a (mon.0) 2473 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:19.338 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:19 smithi177 bash[68594]: audit 2024-09-13T15:09:18.312388+0000 mon.a (mon.0) 2473 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:19.338 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:19 smithi177 bash[68594]: cluster 2024-09-13T15:09:19.011996+0000 mgr.y (mgr.26252) 1297 : cluster [DBG] pgmap v1870: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:19.338 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:19 smithi177 bash[68594]: cluster 2024-09-13T15:09:19.011996+0000 mgr.y (mgr.26252) 1297 : cluster [DBG] pgmap v1870: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:19.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:19.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:19.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:19.483 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:19.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[84576]: audit 2024-09-13T15:09:18.311161+0000 mon.a (mon.0) 2472 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[84576]: audit 2024-09-13T15:09:18.311161+0000 mon.a (mon.0) 2472 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[84576]: audit 2024-09-13T15:09:18.312388+0000 mon.a (mon.0) 2473 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[84576]: audit 2024-09-13T15:09:18.312388+0000 mon.a (mon.0) 2473 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[84576]: cluster 2024-09-13T15:09:19.011996+0000 mgr.y (mgr.26252) 1297 : cluster [DBG] pgmap v1870: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[84576]: cluster 2024-09-13T15:09:19.011996+0000 mgr.y (mgr.26252) 1297 : cluster [DBG] pgmap v1870: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[87147]: audit 2024-09-13T15:09:18.311161+0000 mon.a (mon.0) 2472 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[87147]: audit 2024-09-13T15:09:18.311161+0000 mon.a (mon.0) 2472 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[87147]: audit 2024-09-13T15:09:18.312388+0000 mon.a (mon.0) 2473 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[87147]: audit 2024-09-13T15:09:18.312388+0000 mon.a (mon.0) 2473 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[87147]: cluster 2024-09-13T15:09:19.011996+0000 mgr.y (mgr.26252) 1297 : cluster [DBG] pgmap v1870: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:19 smithi099 bash[87147]: cluster 2024-09-13T15:09:19.011996+0000 mgr.y (mgr.26252) 1297 : cluster [DBG] pgmap v1870: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:19.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:19.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:19.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:19.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:09:19.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:19.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:19.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:19.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:19.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:19.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:20.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:20.103 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:20.130 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:20.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:20.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:20.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:20.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:20.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:20.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:20.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:20.502 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:09:20 smithi177 bash[61439]: ts=2024-09-13T15:09:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:09:20.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:20.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:20.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:20.644 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:20.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:20.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:09:20.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:20.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:20.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:20.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:20.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:20.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:21.009 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:21.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:21.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:21.133 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:21.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:21.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:21.264 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:21.292 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:21.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:21.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:21.527 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:21.554 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:21.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:21.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:21.685 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:21.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:21.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:21.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:21.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:21.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:21.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:22.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:22.081 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:22.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:22.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 reset_purged_snaps_last 2024-09-13T15:09:22.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:22.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:22 smithi177 bash[68594]: cluster 2024-09-13T15:09:21.012568+0000 mgr.y (mgr.26252) 1298 : cluster [DBG] pgmap v1871: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:22.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:22 smithi177 bash[68594]: cluster 2024-09-13T15:09:21.012568+0000 mgr.y (mgr.26252) 1298 : cluster [DBG] pgmap v1871: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:22.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:22.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:22.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:22.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:22 smithi099 bash[84576]: cluster 2024-09-13T15:09:21.012568+0000 mgr.y (mgr.26252) 1298 : cluster [DBG] pgmap v1871: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:22 smithi099 bash[84576]: cluster 2024-09-13T15:09:21.012568+0000 mgr.y (mgr.26252) 1298 : cluster [DBG] pgmap v1871: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:22 smithi099 bash[87147]: cluster 2024-09-13T15:09:21.012568+0000 mgr.y (mgr.26252) 1298 : cluster [DBG] pgmap v1871: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:22 smithi099 bash[87147]: cluster 2024-09-13T15:09:21.012568+0000 mgr.y (mgr.26252) 1298 : cluster [DBG] pgmap v1871: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:22.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:22.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:22.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:22.469 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:22.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:22.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:22.581 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:22.581 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:22.600 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:22.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:22.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:22.732 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:22.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:22.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:22.862 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:22.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:22.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:22.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:22.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:23.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:23.122 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:23.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:23.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:23.254 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:23.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:23.357 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:23.385 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:23.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:23.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:23.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:23.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:23.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:23.749 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:23.776 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:23.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:23.907 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:23.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:24.010 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:24.140 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:24.265 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:24.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:24.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:24 smithi099 bash[84576]: cluster 2024-09-13T15:09:23.013259+0000 mgr.y (mgr.26252) 1299 : cluster [DBG] pgmap v1872: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:24 smithi099 bash[84576]: cluster 2024-09-13T15:09:23.013259+0000 mgr.y (mgr.26252) 1299 : cluster [DBG] pgmap v1872: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:24 smithi099 bash[87147]: cluster 2024-09-13T15:09:23.013259+0000 mgr.y (mgr.26252) 1299 : cluster [DBG] pgmap v1872: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:24 smithi099 bash[87147]: cluster 2024-09-13T15:09:23.013259+0000 mgr.y (mgr.26252) 1299 : cluster [DBG] pgmap v1872: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:24 smithi177 bash[68594]: cluster 2024-09-13T15:09:23.013259+0000 mgr.y (mgr.26252) 1299 : cluster [DBG] pgmap v1872: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:24 smithi177 bash[68594]: cluster 2024-09-13T15:09:23.013259+0000 mgr.y (mgr.26252) 1299 : cluster [DBG] pgmap v1872: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:24.397 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:24.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:24.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:24.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:24.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:24.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:24.617 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:24.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:24.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:24.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:24.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.806 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:24.909 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:25.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:25.053 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:25.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:25.156 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:25.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:25.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:25.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:25.317 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:25.420 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:25.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:25.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:25.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:25.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:25.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:25.630 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:25.675 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:25.702 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:25.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:25.810 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:25.821 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:25 smithi177 bash[68594]: audit 2024-09-13T15:09:24.562673+0000 mon.a (mon.0) 2474 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:25.821 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:25 smithi177 bash[68594]: audit 2024-09-13T15:09:24.562673+0000 mon.a (mon.0) 2474 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:25.821 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:25 smithi177 bash[68594]: cluster 2024-09-13T15:09:25.014273+0000 mgr.y (mgr.26252) 1300 : cluster [DBG] pgmap v1873: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:25.822 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:25 smithi177 bash[68594]: cluster 2024-09-13T15:09:25.014273+0000 mgr.y (mgr.26252) 1300 : cluster [DBG] pgmap v1873: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:25.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:25 smithi099 bash[84576]: audit 2024-09-13T15:09:24.562673+0000 mon.a (mon.0) 2474 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:25.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:25 smithi099 bash[84576]: audit 2024-09-13T15:09:24.562673+0000 mon.a (mon.0) 2474 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:25.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:25 smithi099 bash[84576]: cluster 2024-09-13T15:09:25.014273+0000 mgr.y (mgr.26252) 1300 : cluster [DBG] pgmap v1873: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:25.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:25 smithi099 bash[84576]: cluster 2024-09-13T15:09:25.014273+0000 mgr.y (mgr.26252) 1300 : cluster [DBG] pgmap v1873: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:25.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:25 smithi099 bash[87147]: audit 2024-09-13T15:09:24.562673+0000 mon.a (mon.0) 2474 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:25.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:25 smithi099 bash[87147]: audit 2024-09-13T15:09:24.562673+0000 mon.a (mon.0) 2474 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:25.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:25 smithi099 bash[87147]: cluster 2024-09-13T15:09:25.014273+0000 mgr.y (mgr.26252) 1300 : cluster [DBG] pgmap v1873: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:25.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:25 smithi099 bash[87147]: cluster 2024-09-13T15:09:25.014273+0000 mgr.y (mgr.26252) 1300 : cluster [DBG] pgmap v1873: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:25.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:25.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:25.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:25.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:25.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:26.071 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:26.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:26.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:26.201 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:26.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:26.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:26.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:26.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:26.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:26.489 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:26.592 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:26.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:26.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:26.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:26.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:26.856 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:26.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:26.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:26.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:27.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:27.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:27.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:27.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:27.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:27.249 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:27.260 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:09:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:09:27] "GET /metrics HTTP/1.1" 200 36706 "" "Prometheus/2.51.0" 2024-09-13T15:09:27.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:27.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:27.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:27.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:27.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:27.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:27.531 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:27.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:27.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:27.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:27.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:27.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:27.793 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:27.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:27.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:27.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:27.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:28.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:28.056 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:28.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:28.159 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:28.186 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:28.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:28.289 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:28.301 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:28 smithi177 bash[68594]: cluster 2024-09-13T15:09:27.015024+0000 mgr.y (mgr.26252) 1301 : cluster [DBG] pgmap v1874: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:28.301 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:28 smithi177 bash[68594]: cluster 2024-09-13T15:09:27.015024+0000 mgr.y (mgr.26252) 1301 : cluster [DBG] pgmap v1874: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:28.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:28.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:28 smithi099 bash[84576]: cluster 2024-09-13T15:09:27.015024+0000 mgr.y (mgr.26252) 1301 : cluster [DBG] pgmap v1874: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:28.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:28 smithi099 bash[84576]: cluster 2024-09-13T15:09:27.015024+0000 mgr.y (mgr.26252) 1301 : cluster [DBG] pgmap v1874: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:28.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:28 smithi099 bash[87147]: cluster 2024-09-13T15:09:27.015024+0000 mgr.y (mgr.26252) 1301 : cluster [DBG] pgmap v1874: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:28.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:28 smithi099 bash[87147]: cluster 2024-09-13T15:09:27.015024+0000 mgr.y (mgr.26252) 1301 : cluster [DBG] pgmap v1874: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:28.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:28.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:28.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:28.579 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:28.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:28.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:28.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:28.812 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:28.987 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:28.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:29.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:29.118 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:29.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:29.221 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:29.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:29.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:29.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:29.378 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:29.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:29.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:29.502 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:29.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:29.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:29.631 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:29.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:29.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:29.760 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:29.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:29.863 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:29.887 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:29.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:29.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:30.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:30.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:30.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:30.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:30.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:30.156 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:30 smithi099 bash[84576]: cluster 2024-09-13T15:09:29.015963+0000 mgr.y (mgr.26252) 1302 : cluster [DBG] pgmap v1875: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:30.156 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:30 smithi099 bash[84576]: cluster 2024-09-13T15:09:29.015963+0000 mgr.y (mgr.26252) 1302 : cluster [DBG] pgmap v1875: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:30.156 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:30 smithi099 bash[87147]: cluster 2024-09-13T15:09:29.015963+0000 mgr.y (mgr.26252) 1302 : cluster [DBG] pgmap v1875: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:30.156 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:30 smithi099 bash[87147]: cluster 2024-09-13T15:09:29.015963+0000 mgr.y (mgr.26252) 1302 : cluster [DBG] pgmap v1875: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:30.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:30.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:30.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:30.353 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:30.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:30 smithi177 bash[68594]: cluster 2024-09-13T15:09:29.015963+0000 mgr.y (mgr.26252) 1302 : cluster [DBG] pgmap v1875: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:30.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:30 smithi177 bash[68594]: cluster 2024-09-13T15:09:29.015963+0000 mgr.y (mgr.26252) 1302 : cluster [DBG] pgmap v1875: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:30.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:09:30 smithi177 bash[61439]: ts=2024-09-13T15:09:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:09:30.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:30.411 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:30.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:30.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:30.542 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:30.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:30.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:30.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:30.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:30.805 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:30.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:30.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:30.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:30.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:31.040 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:31.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:31.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:31.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:31.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:31.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:31.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:31.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:31.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:31.430 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:31.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:31.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:31.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:31.579 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:31.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:31.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:31.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:31.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:31.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:31.842 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:31.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:31.945 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:31.973 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:31.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:32.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:32.100 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:32.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:32.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:32.214 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:32 smithi099 bash[84576]: cluster 2024-09-13T15:09:31.016725+0000 mgr.y (mgr.26252) 1303 : cluster [DBG] pgmap v1876: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:32.214 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:32 smithi099 bash[84576]: cluster 2024-09-13T15:09:31.016725+0000 mgr.y (mgr.26252) 1303 : cluster [DBG] pgmap v1876: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:32.214 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:32 smithi099 bash[87147]: cluster 2024-09-13T15:09:31.016725+0000 mgr.y (mgr.26252) 1303 : cluster [DBG] pgmap v1876: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:32.214 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:32 smithi099 bash[87147]: cluster 2024-09-13T15:09:31.016725+0000 mgr.y (mgr.26252) 1303 : cluster [DBG] pgmap v1876: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:32.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:32.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:32.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:32.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:32.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:32.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:32 smithi177 bash[68594]: cluster 2024-09-13T15:09:31.016725+0000 mgr.y (mgr.26252) 1303 : cluster [DBG] pgmap v1876: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:32.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:32 smithi177 bash[68594]: cluster 2024-09-13T15:09:31.016725+0000 mgr.y (mgr.26252) 1303 : cluster [DBG] pgmap v1876: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:32.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:32.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:32.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:32.593 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:32.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:32.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:32.724 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:32.750 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:32.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:32.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:32.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:32.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:32.986 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:33.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:33.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:33.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:33.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:33.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:33.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:33.269 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:33.371 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:33.397 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:33.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:33.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:33.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:33.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:33.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:33.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:33.754 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:33.779 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:33.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:33.882 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:33.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:34.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:34.027 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:34.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:34.130 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:34.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:34.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:34.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:34.284 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:34.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:34 smithi177 bash[68594]: cluster 2024-09-13T15:09:33.017281+0000 mgr.y (mgr.26252) 1304 : cluster [DBG] pgmap v1877: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:34 smithi177 bash[68594]: cluster 2024-09-13T15:09:33.017281+0000 mgr.y (mgr.26252) 1304 : cluster [DBG] pgmap v1877: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:34 smithi177 bash[68594]: audit 2024-09-13T15:09:33.306022+0000 mon.a (mon.0) 2475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:34 smithi177 bash[68594]: audit 2024-09-13T15:09:33.306022+0000 mon.a (mon.0) 2475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:34 smithi177 bash[68594]: audit 2024-09-13T15:09:33.339899+0000 mon.a (mon.0) 2476 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:34 smithi177 bash[68594]: audit 2024-09-13T15:09:33.339899+0000 mon.a (mon.0) 2476 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:34.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:34.399 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[84576]: cluster 2024-09-13T15:09:33.017281+0000 mgr.y (mgr.26252) 1304 : cluster [DBG] pgmap v1877: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:34.399 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[84576]: cluster 2024-09-13T15:09:33.017281+0000 mgr.y (mgr.26252) 1304 : cluster [DBG] pgmap v1877: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:34.399 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[84576]: audit 2024-09-13T15:09:33.306022+0000 mon.a (mon.0) 2475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:34.399 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[84576]: audit 2024-09-13T15:09:33.306022+0000 mon.a (mon.0) 2475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:34.399 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[84576]: audit 2024-09-13T15:09:33.339899+0000 mon.a (mon.0) 2476 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:34.399 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[84576]: audit 2024-09-13T15:09:33.339899+0000 mon.a (mon.0) 2476 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:34.399 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[87147]: cluster 2024-09-13T15:09:33.017281+0000 mgr.y (mgr.26252) 1304 : cluster [DBG] pgmap v1877: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:34.399 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[87147]: cluster 2024-09-13T15:09:33.017281+0000 mgr.y (mgr.26252) 1304 : cluster [DBG] pgmap v1877: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:34.399 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[87147]: audit 2024-09-13T15:09:33.306022+0000 mon.a (mon.0) 2475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:34.400 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[87147]: audit 2024-09-13T15:09:33.306022+0000 mon.a (mon.0) 2475 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:34.400 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[87147]: audit 2024-09-13T15:09:33.339899+0000 mon.a (mon.0) 2476 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:34.400 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:34 smithi099 bash[87147]: audit 2024-09-13T15:09:33.339899+0000 mon.a (mon.0) 2476 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:34.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:34.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:34.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:34.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:34.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:34.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:34.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:34.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:34.804 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:34.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:34.907 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:34.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:34.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:35.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:35.065 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:35.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:35.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:35.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:35.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:35.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:35.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:35.456 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:35.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:35.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:35.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:35.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:35.691 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:35.715 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:35.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:35.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:35.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:35.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:35.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:35.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:35.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:36.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:36.103 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:36.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:36.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:36.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:36.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:36.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:36 smithi099 bash[84576]: cluster 2024-09-13T15:09:35.017996+0000 mgr.y (mgr.26252) 1305 : cluster [DBG] pgmap v1878: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:36.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:36 smithi099 bash[84576]: cluster 2024-09-13T15:09:35.017996+0000 mgr.y (mgr.26252) 1305 : cluster [DBG] pgmap v1878: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:36.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:36 smithi099 bash[87147]: cluster 2024-09-13T15:09:35.017996+0000 mgr.y (mgr.26252) 1305 : cluster [DBG] pgmap v1878: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:36.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:36 smithi099 bash[87147]: cluster 2024-09-13T15:09:35.017996+0000 mgr.y (mgr.26252) 1305 : cluster [DBG] pgmap v1878: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:36.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:36 smithi177 bash[68594]: cluster 2024-09-13T15:09:35.017996+0000 mgr.y (mgr.26252) 1305 : cluster [DBG] pgmap v1878: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:36 smithi177 bash[68594]: cluster 2024-09-13T15:09:35.017996+0000 mgr.y (mgr.26252) 1305 : cluster [DBG] pgmap v1878: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:36.362 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:36.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:36.465 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:36.491 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:36.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:36.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:36.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:36.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:36.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:36.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:36.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:36.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:36.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:36.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:36.981 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:37.009 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:37.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:37.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:37.140 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:37.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:37.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:37.255 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:09:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:09:37] "GET /metrics HTTP/1.1" 200 36710 "" "Prometheus/2.51.0" 2024-09-13T15:09:37.270 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:37.373 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:37.401 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:37.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:37.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:37.532 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:37.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:37.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:37.661 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:37.764 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:37.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:37.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:37.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:37.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:38.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:38.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:38.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:38.156 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:38.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:38.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:38.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:38 smithi099 bash[84576]: cluster 2024-09-13T15:09:37.018460+0000 mgr.y (mgr.26252) 1306 : cluster [DBG] pgmap v1879: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:38.299 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:38 smithi099 bash[84576]: cluster 2024-09-13T15:09:37.018460+0000 mgr.y (mgr.26252) 1306 : cluster [DBG] pgmap v1879: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:38.299 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:38 smithi099 bash[87147]: cluster 2024-09-13T15:09:37.018460+0000 mgr.y (mgr.26252) 1306 : cluster [DBG] pgmap v1879: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:38.299 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:38 smithi099 bash[87147]: cluster 2024-09-13T15:09:37.018460+0000 mgr.y (mgr.26252) 1306 : cluster [DBG] pgmap v1879: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:38.314 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:38.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:38.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:38 smithi177 bash[68594]: cluster 2024-09-13T15:09:37.018460+0000 mgr.y (mgr.26252) 1306 : cluster [DBG] pgmap v1879: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:38.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:38 smithi177 bash[68594]: cluster 2024-09-13T15:09:37.018460+0000 mgr.y (mgr.26252) 1306 : cluster [DBG] pgmap v1879: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:38.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:38.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:38.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:38.565 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:38.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:38.667 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:38.687 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:38.769 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:09:38.770 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:09:38.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:38.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:38.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:38.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:38.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:38.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:39.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:39.057 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:39.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:39.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:39.182 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:39.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:39.286 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:39.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:39.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:39.408 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:39.428 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:39.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:39.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:39.552 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:39.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:39.655 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:39.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:39.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:39.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:39.805 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:39.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:39.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:39.928 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:39.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:40.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:40.055 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:40.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:40.098 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: cluster 2024-09-13T15:09:39.019020+0000 mgr.y (mgr.26252) 1307 : cluster [DBG] pgmap v1880: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:40.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:40.170 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: cluster 2024-09-13T15:09:39.019020+0000 mgr.y (mgr.26252) 1307 : cluster [DBG] pgmap v1880: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:40.170 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.292354+0000 mon.a (mon.0) 2477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.170 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.292354+0000 mon.a (mon.0) 2477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.297709+0000 mon.a (mon.0) 2478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.297709+0000 mon.a (mon.0) 2478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.369580+0000 mon.a (mon.0) 2479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.369580+0000 mon.a (mon.0) 2479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.374756+0000 mon.a (mon.0) 2480 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.374756+0000 mon.a (mon.0) 2480 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.541433+0000 mon.c (mon.2) 103 : audit [INF] from='client.? 172.21.15.99:0/3168177122' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.541433+0000 mon.c (mon.2) 103 : audit [INF] from='client.? 172.21.15.99:0/3168177122' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.541995+0000 mon.a (mon.0) 2481 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[84576]: audit 2024-09-13T15:09:39.541995+0000 mon.a (mon.0) 2481 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.171 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: cluster 2024-09-13T15:09:39.019020+0000 mgr.y (mgr.26252) 1307 : cluster [DBG] pgmap v1880: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: cluster 2024-09-13T15:09:39.019020+0000 mgr.y (mgr.26252) 1307 : cluster [DBG] pgmap v1880: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.292354+0000 mon.a (mon.0) 2477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.292354+0000 mon.a (mon.0) 2477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.297709+0000 mon.a (mon.0) 2478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.297709+0000 mon.a (mon.0) 2478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.369580+0000 mon.a (mon.0) 2479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.369580+0000 mon.a (mon.0) 2479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.374756+0000 mon.a (mon.0) 2480 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.374756+0000 mon.a (mon.0) 2480 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.541433+0000 mon.c (mon.2) 103 : audit [INF] from='client.? 172.21.15.99:0/3168177122' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.541433+0000 mon.c (mon.2) 103 : audit [INF] from='client.? 172.21.15.99:0/3168177122' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.541995+0000 mon.a (mon.0) 2481 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.172 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:40 smithi099 bash[87147]: audit 2024-09-13T15:09:39.541995+0000 mon.a (mon.0) 2481 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.181 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:40.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:40.213 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: cluster 2024-09-13T15:09:39.019020+0000 mgr.y (mgr.26252) 1307 : cluster [DBG] pgmap v1880: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: cluster 2024-09-13T15:09:39.019020+0000 mgr.y (mgr.26252) 1307 : cluster [DBG] pgmap v1880: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.292354+0000 mon.a (mon.0) 2477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.292354+0000 mon.a (mon.0) 2477 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.297709+0000 mon.a (mon.0) 2478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.297709+0000 mon.a (mon.0) 2478 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.369580+0000 mon.a (mon.0) 2479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.369580+0000 mon.a (mon.0) 2479 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.374756+0000 mon.a (mon.0) 2480 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.374756+0000 mon.a (mon.0) 2480 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.541433+0000 mon.c (mon.2) 103 : audit [INF] from='client.? 172.21.15.99:0/3168177122' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.541433+0000 mon.c (mon.2) 103 : audit [INF] from='client.? 172.21.15.99:0/3168177122' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.541995+0000 mon.a (mon.0) 2481 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[68594]: audit 2024-09-13T15:09:39.541995+0000 mon.a (mon.0) 2481 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:40.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:40.310 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:40.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:40.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:09:40 smithi177 bash[61439]: ts=2024-09-13T15:09:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:09:40.378 INFO:teuthology.orchestra.run.smithi099.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-09-13T15:09:40.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:40.442 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:40.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:40.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:40.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:40.675 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:40.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:40.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:40.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:40.828 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:40.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:40.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:40.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:41.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:41.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:41.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:41.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:41.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:41.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:41.348 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:41.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:41.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:41.473 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:41.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:41.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[84576]: audit 2024-09-13T15:09:40.374757+0000 mon.a (mon.0) 2482 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:09:41.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[84576]: audit 2024-09-13T15:09:40.374757+0000 mon.a (mon.0) 2482 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:09:41.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[84576]: cluster 2024-09-13T15:09:40.378072+0000 mon.a (mon.0) 2483 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2024-09-13T15:09:41.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[84576]: cluster 2024-09-13T15:09:40.378072+0000 mon.a (mon.0) 2483 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2024-09-13T15:09:41.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[84576]: cluster 2024-09-13T15:09:41.019670+0000 mgr.y (mgr.26252) 1308 : cluster [DBG] pgmap v1882: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:41.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[84576]: cluster 2024-09-13T15:09:41.019670+0000 mgr.y (mgr.26252) 1308 : cluster [DBG] pgmap v1882: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:41.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[87147]: audit 2024-09-13T15:09:40.374757+0000 mon.a (mon.0) 2482 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:09:41.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[87147]: audit 2024-09-13T15:09:40.374757+0000 mon.a (mon.0) 2482 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:09:41.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[87147]: cluster 2024-09-13T15:09:40.378072+0000 mon.a (mon.0) 2483 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2024-09-13T15:09:41.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[87147]: cluster 2024-09-13T15:09:40.378072+0000 mon.a (mon.0) 2483 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2024-09-13T15:09:41.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[87147]: cluster 2024-09-13T15:09:41.019670+0000 mgr.y (mgr.26252) 1308 : cluster [DBG] pgmap v1882: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:41.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:41 smithi099 bash[87147]: cluster 2024-09-13T15:09:41.019670+0000 mgr.y (mgr.26252) 1308 : cluster [DBG] pgmap v1882: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:41.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:41.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:41 smithi177 bash[68594]: audit 2024-09-13T15:09:40.374757+0000 mon.a (mon.0) 2482 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:09:41.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:41 smithi177 bash[68594]: audit 2024-09-13T15:09:40.374757+0000 mon.a (mon.0) 2482 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:09:41.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:41 smithi177 bash[68594]: cluster 2024-09-13T15:09:40.378072+0000 mon.a (mon.0) 2483 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2024-09-13T15:09:41.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:41 smithi177 bash[68594]: cluster 2024-09-13T15:09:40.378072+0000 mon.a (mon.0) 2483 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2024-09-13T15:09:41.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:41 smithi177 bash[68594]: cluster 2024-09-13T15:09:41.019670+0000 mgr.y (mgr.26252) 1308 : cluster [DBG] pgmap v1882: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:41.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:41 smithi177 bash[68594]: cluster 2024-09-13T15:09:41.019670+0000 mgr.y (mgr.26252) 1308 : cluster [DBG] pgmap v1882: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:41.608 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:41.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:41.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:41.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:41.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:41.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:41.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:41.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:41.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:42.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.005 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:42.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:42.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:42.134 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:42.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:42.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:42.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:42.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:42.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:42.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:42.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:42.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:42.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:42.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:42.628 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:42.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: cluster 2024-09-13T15:09:41.374827+0000 mon.a (mon.0) 2484 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:09:42.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: cluster 2024-09-13T15:09:41.374827+0000 mon.a (mon.0) 2484 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:09:42.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: audit 2024-09-13T15:09:41.397044+0000 mon.a (mon.0) 2485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: audit 2024-09-13T15:09:41.397044+0000 mon.a (mon.0) 2485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: audit 2024-09-13T15:09:41.403515+0000 mon.a (mon.0) 2486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: audit 2024-09-13T15:09:41.403515+0000 mon.a (mon.0) 2486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: audit 2024-09-13T15:09:41.404522+0000 mon.a (mon.0) 2487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:09:42.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: audit 2024-09-13T15:09:41.404522+0000 mon.a (mon.0) 2487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:09:42.640 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: audit 2024-09-13T15:09:41.405317+0000 mon.a (mon.0) 2488 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:09:42.641 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: audit 2024-09-13T15:09:41.405317+0000 mon.a (mon.0) 2488 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:09:42.641 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: audit 2024-09-13T15:09:41.414290+0000 mon.a (mon.0) 2489 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.641 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[84576]: audit 2024-09-13T15:09:41.414290+0000 mon.a (mon.0) 2489 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.641 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: cluster 2024-09-13T15:09:41.374827+0000 mon.a (mon.0) 2484 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:09:42.641 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: cluster 2024-09-13T15:09:41.374827+0000 mon.a (mon.0) 2484 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:09:42.641 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: audit 2024-09-13T15:09:41.397044+0000 mon.a (mon.0) 2485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.641 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: audit 2024-09-13T15:09:41.397044+0000 mon.a (mon.0) 2485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.641 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: audit 2024-09-13T15:09:41.403515+0000 mon.a (mon.0) 2486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.641 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: audit 2024-09-13T15:09:41.403515+0000 mon.a (mon.0) 2486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.642 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: audit 2024-09-13T15:09:41.404522+0000 mon.a (mon.0) 2487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:09:42.642 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: audit 2024-09-13T15:09:41.404522+0000 mon.a (mon.0) 2487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:09:42.642 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: audit 2024-09-13T15:09:41.405317+0000 mon.a (mon.0) 2488 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:09:42.642 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: audit 2024-09-13T15:09:41.405317+0000 mon.a (mon.0) 2488 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:09:42.642 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: audit 2024-09-13T15:09:41.414290+0000 mon.a (mon.0) 2489 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.642 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:42 smithi099 bash[87147]: audit 2024-09-13T15:09:41.414290+0000 mon.a (mon.0) 2489 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.658 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:42.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:42.761 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:42.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:42.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: cluster 2024-09-13T15:09:41.374827+0000 mon.a (mon.0) 2484 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:09:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: cluster 2024-09-13T15:09:41.374827+0000 mon.a (mon.0) 2484 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:09:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: audit 2024-09-13T15:09:41.397044+0000 mon.a (mon.0) 2485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: audit 2024-09-13T15:09:41.397044+0000 mon.a (mon.0) 2485 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: audit 2024-09-13T15:09:41.403515+0000 mon.a (mon.0) 2486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: audit 2024-09-13T15:09:41.403515+0000 mon.a (mon.0) 2486 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: audit 2024-09-13T15:09:41.404522+0000 mon.a (mon.0) 2487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:09:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: audit 2024-09-13T15:09:41.404522+0000 mon.a (mon.0) 2487 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:09:42.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: audit 2024-09-13T15:09:41.405317+0000 mon.a (mon.0) 2488 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:09:42.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: audit 2024-09-13T15:09:41.405317+0000 mon.a (mon.0) 2488 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:09:42.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: audit 2024-09-13T15:09:41.414290+0000 mon.a (mon.0) 2489 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:42 smithi177 bash[68594]: audit 2024-09-13T15:09:41.414290+0000 mon.a (mon.0) 2489 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:42.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:42.918 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:42.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:43.021 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:43.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:43.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:43.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T15:09:43.153 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:43.181 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:43.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:43.285 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:43.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:43.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:43.443 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:43.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:43.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:43.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:43.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:43 smithi177 bash[68594]: cluster 2024-09-13T15:09:43.020663+0000 mgr.y (mgr.26252) 1309 : cluster [DBG] pgmap v1883: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:43.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:43 smithi177 bash[68594]: cluster 2024-09-13T15:09:43.020663+0000 mgr.y (mgr.26252) 1309 : cluster [DBG] pgmap v1883: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:43.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:43.687 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:43 smithi099 bash[84576]: cluster 2024-09-13T15:09:43.020663+0000 mgr.y (mgr.26252) 1309 : cluster [DBG] pgmap v1883: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:43.687 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:43 smithi099 bash[84576]: cluster 2024-09-13T15:09:43.020663+0000 mgr.y (mgr.26252) 1309 : cluster [DBG] pgmap v1883: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:43.687 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:43 smithi099 bash[87147]: cluster 2024-09-13T15:09:43.020663+0000 mgr.y (mgr.26252) 1309 : cluster [DBG] pgmap v1883: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:43.687 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:43 smithi099 bash[87147]: cluster 2024-09-13T15:09:43.020663+0000 mgr.y (mgr.26252) 1309 : cluster [DBG] pgmap v1883: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:43.701 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:43.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:43.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:43.833 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:43.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:43.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:43.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:43.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:44.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:44.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:44.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:44.197 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:44.223 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:44.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:44.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:44.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:44.481 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:44.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:44.584 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:44.612 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:44.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:44.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:44.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:44.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:44.848 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:44.876 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:44.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:44.979 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:45.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:45.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:45.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:45.103 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:45.124 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:45.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:45.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:45.227 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:45.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:45.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:45.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:45.314 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:09:45.314 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:09:45.335 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:09:45.335 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:09:45.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:45.352 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:09:45.352 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:09:45.368 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:45.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:45.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:45.370 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:09:45.370 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:09:45.391 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:09:45.391 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:09:45.411 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:09:45.411 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:09:45.431 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:09:45.431 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:09:45.451 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:09:45.451 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:09:45.470 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:45.498 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:45.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:45.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:45.625 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:45.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:45.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:45.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:45.755 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:45.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:45.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:45.859 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:45.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:45.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:45.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:45.957 DEBUG:teuthology.orchestra.run.smithi099:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-09-13T15:09:45.960 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T15:09:45.960 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T15:09:45.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:46.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:46.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:46.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:46.076 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T15:09:46.119 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:46.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:46.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:46.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:46.193 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:hints = 1 2024-09-13T15:09:46.193 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T15:09:46.194 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object prefix: benchmark_data_smithi099_210648 2024-09-13T15:09:46.194 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:09:46.194 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 0 0 0 0 0 0 - 0 2024-09-13T15:09:46.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:46.283 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:46.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:46 smithi177 bash[68594]: cluster 2024-09-13T15:09:45.021310+0000 mgr.y (mgr.26252) 1310 : cluster [DBG] pgmap v1884: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:46 smithi177 bash[68594]: cluster 2024-09-13T15:09:45.021310+0000 mgr.y (mgr.26252) 1310 : cluster [DBG] pgmap v1884: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:46 smithi177 bash[68594]: audit 2024-09-13T15:09:45.283289+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.99:0/1701635752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:46 smithi177 bash[68594]: audit 2024-09-13T15:09:45.283289+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.99:0/1701635752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:46 smithi177 bash[68594]: audit 2024-09-13T15:09:45.356386+0000 mon.a (mon.0) 2490 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:46 smithi177 bash[68594]: audit 2024-09-13T15:09:45.356386+0000 mon.a (mon.0) 2490 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:46.388 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:46.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:46.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:46.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:46.525 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:46.535 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[84576]: cluster 2024-09-13T15:09:45.021310+0000 mgr.y (mgr.26252) 1310 : cluster [DBG] pgmap v1884: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:46.535 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[84576]: cluster 2024-09-13T15:09:45.021310+0000 mgr.y (mgr.26252) 1310 : cluster [DBG] pgmap v1884: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:46.535 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[84576]: audit 2024-09-13T15:09:45.283289+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.99:0/1701635752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:46.535 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[84576]: audit 2024-09-13T15:09:45.283289+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.99:0/1701635752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:46.535 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[84576]: audit 2024-09-13T15:09:45.356386+0000 mon.a (mon.0) 2490 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:46.535 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[84576]: audit 2024-09-13T15:09:45.356386+0000 mon.a (mon.0) 2490 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:46.535 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[87147]: cluster 2024-09-13T15:09:45.021310+0000 mgr.y (mgr.26252) 1310 : cluster [DBG] pgmap v1884: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:46.535 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[87147]: cluster 2024-09-13T15:09:45.021310+0000 mgr.y (mgr.26252) 1310 : cluster [DBG] pgmap v1884: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:46.536 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[87147]: audit 2024-09-13T15:09:45.283289+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.99:0/1701635752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:46.536 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[87147]: audit 2024-09-13T15:09:45.283289+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.99:0/1701635752' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:46.536 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[87147]: audit 2024-09-13T15:09:45.356386+0000 mon.a (mon.0) 2490 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:46.536 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:46 smithi099 bash[87147]: audit 2024-09-13T15:09:45.356386+0000 mon.a (mon.0) 2490 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:46.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:46.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:46.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:46.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:46.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:46.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:46.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:46.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:46.803 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:46.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:46.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:46.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:46.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:46.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:46.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:47.033 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:47.072 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:47.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:47.176 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:47.186 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:47 smithi177 bash[68594]: cluster 2024-09-13T15:09:46.074087+0000 mon.a (mon.0) 2491 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:47.186 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:47 smithi177 bash[68594]: cluster 2024-09-13T15:09:46.074087+0000 mon.a (mon.0) 2491 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:47.186 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:47 smithi177 bash[68594]: audit 2024-09-13T15:09:46.076140+0000 mon.a (mon.0) 2492 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:09:47.186 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:47 smithi177 bash[68594]: audit 2024-09-13T15:09:46.076140+0000 mon.a (mon.0) 2492 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:09:47.186 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:47 smithi177 bash[68594]: cluster 2024-09-13T15:09:46.082148+0000 mon.a (mon.0) 2493 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2024-09-13T15:09:47.186 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:47 smithi177 bash[68594]: cluster 2024-09-13T15:09:46.082148+0000 mon.a (mon.0) 2493 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2024-09-13T15:09:47.194 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 1 16 2600 2584 161.491 161.5 0.00247279 0.00616939 2024-09-13T15:09:47.210 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:47.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:47.310 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:09:47] "GET /metrics HTTP/1.1" 200 36710 "" "Prometheus/2.51.0" 2024-09-13T15:09:47.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:47.332 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:47.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:47.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:47.435 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:47.457 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:47.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:47.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:47.560 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:47.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[84576]: cluster 2024-09-13T15:09:46.074087+0000 mon.a (mon.0) 2491 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:47.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[84576]: cluster 2024-09-13T15:09:46.074087+0000 mon.a (mon.0) 2491 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:47.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[84576]: audit 2024-09-13T15:09:46.076140+0000 mon.a (mon.0) 2492 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:09:47.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[84576]: audit 2024-09-13T15:09:46.076140+0000 mon.a (mon.0) 2492 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:09:47.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[84576]: cluster 2024-09-13T15:09:46.082148+0000 mon.a (mon.0) 2493 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2024-09-13T15:09:47.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[84576]: cluster 2024-09-13T15:09:46.082148+0000 mon.a (mon.0) 2493 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2024-09-13T15:09:47.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[87147]: cluster 2024-09-13T15:09:46.074087+0000 mon.a (mon.0) 2491 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:47.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[87147]: cluster 2024-09-13T15:09:46.074087+0000 mon.a (mon.0) 2491 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:47.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[87147]: audit 2024-09-13T15:09:46.076140+0000 mon.a (mon.0) 2492 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:09:47.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[87147]: audit 2024-09-13T15:09:46.076140+0000 mon.a (mon.0) 2492 : audit [INF] from='client.? 172.21.15.99:0/593853378' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:09:47.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[87147]: cluster 2024-09-13T15:09:46.082148+0000 mon.a (mon.0) 2493 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2024-09-13T15:09:47.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:47 smithi099 bash[87147]: cluster 2024-09-13T15:09:46.082148+0000 mon.a (mon.0) 2493 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2024-09-13T15:09:47.586 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:47.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:47.689 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:47.724 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:47.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:47.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:47.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:47.856 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:47.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:47.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:47.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:47.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:47.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:47.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:48.100 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:48.119 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:48.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:48.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:48.194 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 2 16 4165 4149 129.647 97.8125 0.00163588 0.0072623 2024-09-13T15:09:48.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:48.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:48.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:48.273 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:48 smithi099 bash[84576]: cluster 2024-09-13T15:09:47.021824+0000 mgr.y (mgr.26252) 1311 : cluster [DBG] pgmap v1886: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:48.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:48.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:48 smithi177 bash[68594]: cluster 2024-09-13T15:09:47.021824+0000 mgr.y (mgr.26252) 1311 : cluster [DBG] pgmap v1886: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:48.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:48 smithi177 bash[68594]: cluster 2024-09-13T15:09:47.021824+0000 mgr.y (mgr.26252) 1311 : cluster [DBG] pgmap v1886: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:48.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:48.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:48.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:48.440 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:48 smithi099 bash[84576]: cluster 2024-09-13T15:09:47.021824+0000 mgr.y (mgr.26252) 1311 : cluster [DBG] pgmap v1886: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:48.440 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:48 smithi099 bash[87147]: cluster 2024-09-13T15:09:47.021824+0000 mgr.y (mgr.26252) 1311 : cluster [DBG] pgmap v1886: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:48.440 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:48 smithi099 bash[87147]: cluster 2024-09-13T15:09:47.021824+0000 mgr.y (mgr.26252) 1311 : cluster [DBG] pgmap v1886: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:09:48.492 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:48.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:48.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:48.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:48.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:48.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:48.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:48.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:48.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:48.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:48.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:48.885 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:48.913 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:48.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:48.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:49.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:49.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:49.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:49.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:49.170 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:49.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:49.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:49.194 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 3 16 6230 6214 129.447 129.062 0.0314784 0.00763082 2024-09-13T15:09:49.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:49.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:49.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:49.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:49.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:49.430 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:49.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:49.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:49.533 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:49.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:49 smithi177 bash[68594]: audit 2024-09-13T15:09:48.312354+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:49.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:49 smithi177 bash[68594]: audit 2024-09-13T15:09:48.312354+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:49.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:49 smithi177 bash[68594]: audit 2024-09-13T15:09:48.313583+0000 mon.a (mon.0) 2495 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:49.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:49 smithi177 bash[68594]: audit 2024-09-13T15:09:48.313583+0000 mon.a (mon.0) 2495 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:49.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:49 smithi177 bash[68594]: cluster 2024-09-13T15:09:49.022668+0000 mgr.y (mgr.26252) 1312 : cluster [DBG] pgmap v1887: 89 pgs: 89 active+clean; 182 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 21 MiB/s wr, 336 op/s 2024-09-13T15:09:49.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:49 smithi177 bash[68594]: cluster 2024-09-13T15:09:49.022668+0000 mgr.y (mgr.26252) 1312 : cluster [DBG] pgmap v1887: 89 pgs: 89 active+clean; 182 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 21 MiB/s wr, 336 op/s 2024-09-13T15:09:49.560 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:49.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:49.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[84576]: audit 2024-09-13T15:09:48.312354+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[84576]: audit 2024-09-13T15:09:48.312354+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[84576]: audit 2024-09-13T15:09:48.313583+0000 mon.a (mon.0) 2495 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[84576]: audit 2024-09-13T15:09:48.313583+0000 mon.a (mon.0) 2495 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[84576]: cluster 2024-09-13T15:09:49.022668+0000 mgr.y (mgr.26252) 1312 : cluster [DBG] pgmap v1887: 89 pgs: 89 active+clean; 182 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 21 MiB/s wr, 336 op/s 2024-09-13T15:09:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[84576]: cluster 2024-09-13T15:09:49.022668+0000 mgr.y (mgr.26252) 1312 : cluster [DBG] pgmap v1887: 89 pgs: 89 active+clean; 182 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 21 MiB/s wr, 336 op/s 2024-09-13T15:09:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[87147]: audit 2024-09-13T15:09:48.312354+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[87147]: audit 2024-09-13T15:09:48.312354+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:09:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[87147]: audit 2024-09-13T15:09:48.313583+0000 mon.a (mon.0) 2495 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[87147]: audit 2024-09-13T15:09:48.313583+0000 mon.a (mon.0) 2495 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[87147]: cluster 2024-09-13T15:09:49.022668+0000 mgr.y (mgr.26252) 1312 : cluster [DBG] pgmap v1887: 89 pgs: 89 active+clean; 182 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 21 MiB/s wr, 336 op/s 2024-09-13T15:09:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:49 smithi099 bash[87147]: cluster 2024-09-13T15:09:49.022668+0000 mgr.y (mgr.26252) 1312 : cluster [DBG] pgmap v1887: 89 pgs: 89 active+clean; 182 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 21 MiB/s wr, 336 op/s 2024-09-13T15:09:49.664 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:49.691 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:49.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:49.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:49.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:49.830 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:49.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:49.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:49.934 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:49.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:49.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:50.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:50.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:50.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:50.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:50.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:50.194 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 4 16 7266 7250 113.271 64.75 0.00438732 0.00881812 2024-09-13T15:09:50.216 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:50.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:50.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:50.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:50.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:50.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:50.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:50.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:50.466 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:09:50 smithi177 bash[61439]: ts=2024-09-13T15:09:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:09:50.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:50.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:50.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:50.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:50.611 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:50.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:50.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:50.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:50.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:50.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:50.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:50.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:50.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:50.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:50.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:51.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:51.004 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:51.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:51.105 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:51.131 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:51.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:51.194 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 5 16 10102 10086 126.064 177.25 0.00741027 0.00792388 2024-09-13T15:09:51.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:51.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:51.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:51.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:51.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:51.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:51.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:51.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:51.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:51.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:51.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:51.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:51.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:51.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:51.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:51.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:51.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:51.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:51.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:51.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:51.885 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:51.907 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:51.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:51.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:52.010 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:52.036 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:52.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:52.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:52.167 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:52.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:52.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:52.194 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 6 16 12655 12639 131.645 159.562 0.00426285 0.00758808 2024-09-13T15:09:52.274 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:52.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:52.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:52.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:52 smithi177 bash[68594]: cluster 2024-09-13T15:09:51.023146+0000 mgr.y (mgr.26252) 1313 : cluster [DBG] pgmap v1888: 89 pgs: 89 active+clean; 333 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 33 MiB/s wr, 532 op/s 2024-09-13T15:09:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:52 smithi177 bash[68594]: cluster 2024-09-13T15:09:51.023146+0000 mgr.y (mgr.26252) 1313 : cluster [DBG] pgmap v1888: 89 pgs: 89 active+clean; 333 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 33 MiB/s wr, 532 op/s 2024-09-13T15:09:52.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:52.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:52.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:52.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:52.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:52.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:52.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:52.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:52.655 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:52.680 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:52.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:52.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:52.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:52.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:52.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:52.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:52.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:52.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:52.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:52.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:53.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:53.068 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:53.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:53.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:53.099 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:09:53.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:09:53.171 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:53.194 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 7 16 14690 14674 131.007 127.188 0.0198438 0.00762632 2024-09-13T15:09:53.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:53.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:53.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:53.304 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:53.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:53.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:53.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:53.435 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:53.446 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:53 smithi099 bash[87147]: cluster 2024-09-13T15:09:51.023146+0000 mgr.y (mgr.26252) 1313 : cluster [DBG] pgmap v1888: 89 pgs: 89 active+clean; 333 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 33 MiB/s wr, 532 op/s 2024-09-13T15:09:53.447 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:53 smithi099 bash[87147]: cluster 2024-09-13T15:09:51.023146+0000 mgr.y (mgr.26252) 1313 : cluster [DBG] pgmap v1888: 89 pgs: 89 active+clean; 333 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 33 MiB/s wr, 532 op/s 2024-09-13T15:09:53.447 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:53 smithi099 bash[84576]: cluster 2024-09-13T15:09:51.023146+0000 mgr.y (mgr.26252) 1313 : cluster [DBG] pgmap v1888: 89 pgs: 89 active+clean; 333 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 33 MiB/s wr, 532 op/s 2024-09-13T15:09:53.447 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:53 smithi099 bash[84576]: cluster 2024-09-13T15:09:51.023146+0000 mgr.y (mgr.26252) 1313 : cluster [DBG] pgmap v1888: 89 pgs: 89 active+clean; 333 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 33 MiB/s wr, 532 op/s 2024-09-13T15:09:53.461 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:53.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:53.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:53.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:53.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:53.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:53.696 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:53.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:53.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:53.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:53.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:53.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:53.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:53.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:53.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:53.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:54.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:54.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:54 smithi177 bash[68594]: cluster 2024-09-13T15:09:53.023654+0000 mgr.y (mgr.26252) 1314 : cluster [DBG] pgmap v1889: 89 pgs: 89 active+clean; 495 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-09-13T15:09:54.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:54 smithi177 bash[68594]: cluster 2024-09-13T15:09:53.023654+0000 mgr.y (mgr.26252) 1314 : cluster [DBG] pgmap v1889: 89 pgs: 89 active+clean; 495 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-09-13T15:09:54.107 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:54.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:54.195 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 8 16 17050 17034 133.066 147.5 0.00360461 0.00750569 2024-09-13T15:09:54.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:54.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:54.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:54.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:54.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:54.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:54.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:54.467 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:54.477 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:54 smithi099 bash[84576]: cluster 2024-09-13T15:09:53.023654+0000 mgr.y (mgr.26252) 1314 : cluster [DBG] pgmap v1889: 89 pgs: 89 active+clean; 495 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-09-13T15:09:54.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:54 smithi099 bash[84576]: cluster 2024-09-13T15:09:53.023654+0000 mgr.y (mgr.26252) 1314 : cluster [DBG] pgmap v1889: 89 pgs: 89 active+clean; 495 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-09-13T15:09:54.478 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:54 smithi099 bash[87147]: cluster 2024-09-13T15:09:53.023654+0000 mgr.y (mgr.26252) 1314 : cluster [DBG] pgmap v1889: 89 pgs: 89 active+clean; 495 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-09-13T15:09:54.478 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:54 smithi099 bash[87147]: cluster 2024-09-13T15:09:53.023654+0000 mgr.y (mgr.26252) 1314 : cluster [DBG] pgmap v1889: 89 pgs: 89 active+clean; 495 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s 2024-09-13T15:09:54.496 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:54.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:54.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:54.598 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:54.622 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:54.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:54.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:54.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:54.748 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:54.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:54.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:54.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:54.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:54.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:55.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:55.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:55.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:55.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:55.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:55.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:55.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:55.195 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 9 16 19517 19501 135.412 154.188 0.005573 0.00737928 2024-09-13T15:09:55.245 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:55.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:55.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:55.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:55.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:55.406 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:55.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:55.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:55.532 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:55.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:55.636 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:55.659 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:55.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:55.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:55.675 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 reset_purged_snaps_last 2024-09-13T15:09:55.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:55.786 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:55.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:55.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:55.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:55.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:55.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:56.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:56.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:56.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:56.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:56.155 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:56.177 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:56.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:56.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:56.195 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 10 16 21747 21731 135.807 139.375 0.00763622 0.00735815 2024-09-13T15:09:56.281 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:56.304 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:56.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:56.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:56 smithi099 bash[84576]: cluster 2024-09-13T15:09:55.024249+0000 mgr.y (mgr.26252) 1315 : cluster [DBG] pgmap v1890: 89 pgs: 89 active+clean; 934 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T15:09:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:56 smithi099 bash[84576]: cluster 2024-09-13T15:09:55.024249+0000 mgr.y (mgr.26252) 1315 : cluster [DBG] pgmap v1890: 89 pgs: 89 active+clean; 934 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T15:09:56.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:56 smithi099 bash[87147]: cluster 2024-09-13T15:09:55.024249+0000 mgr.y (mgr.26252) 1315 : cluster [DBG] pgmap v1890: 89 pgs: 89 active+clean; 934 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T15:09:56.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:56 smithi099 bash[87147]: cluster 2024-09-13T15:09:55.024249+0000 mgr.y (mgr.26252) 1315 : cluster [DBG] pgmap v1890: 89 pgs: 89 active+clean; 934 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T15:09:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:56 smithi177 bash[68594]: cluster 2024-09-13T15:09:55.024249+0000 mgr.y (mgr.26252) 1315 : cluster [DBG] pgmap v1890: 89 pgs: 89 active+clean; 934 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T15:09:56.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:56 smithi177 bash[68594]: cluster 2024-09-13T15:09:55.024249+0000 mgr.y (mgr.26252) 1315 : cluster [DBG] pgmap v1890: 89 pgs: 89 active+clean; 934 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-13T15:09:56.408 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:56.435 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:56.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:56.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:56.538 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:56.565 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:56.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:56.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:56.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:56.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:56.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:56.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:56.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:56.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:56.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:56.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:56.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:56.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:56.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:56.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:57.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:57.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:57.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:57.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:57.162 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:57.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:57.195 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 11 16 23969 23953 136.084 138.875 0.00434883 0.00734192 2024-09-13T15:09:57.224 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:57.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:57.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:57.258 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:09:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:09:57] "GET /metrics HTTP/1.1" 200 36709 "" "Prometheus/2.51.0" 2024-09-13T15:09:57.327 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:57.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:57.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:57.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:57.488 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:57.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:57.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:57.596 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:57.626 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:57.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:57.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:57.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:57.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:57.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:57.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:57.856 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:57.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:57.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:57.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:57.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:58.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:58.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:58.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:58.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:58.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:58.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:58.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:58.195 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 12 16 26093 26077 135.806 132.75 0.0164089 0.00735598 2024-09-13T15:09:58.243 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:58.270 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:58.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:58.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:58.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:58 smithi177 bash[68594]: cluster 2024-09-13T15:09:57.024850+0000 mgr.y (mgr.26252) 1316 : cluster [DBG] pgmap v1891: 89 pgs: 89 active+clean; 1.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:09:58.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:09:58 smithi177 bash[68594]: cluster 2024-09-13T15:09:57.024850+0000 mgr.y (mgr.26252) 1316 : cluster [DBG] pgmap v1891: 89 pgs: 89 active+clean; 1.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:09:58.375 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:58.400 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:58.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:58.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:58.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:58.514 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:58 smithi099 bash[84576]: cluster 2024-09-13T15:09:57.024850+0000 mgr.y (mgr.26252) 1316 : cluster [DBG] pgmap v1891: 89 pgs: 89 active+clean; 1.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:09:58.514 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:09:58 smithi099 bash[84576]: cluster 2024-09-13T15:09:57.024850+0000 mgr.y (mgr.26252) 1316 : cluster [DBG] pgmap v1891: 89 pgs: 89 active+clean; 1.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:09:58.514 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:58 smithi099 bash[87147]: cluster 2024-09-13T15:09:57.024850+0000 mgr.y (mgr.26252) 1316 : cluster [DBG] pgmap v1891: 89 pgs: 89 active+clean; 1.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:09:58.514 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:09:58 smithi099 bash[87147]: cluster 2024-09-13T15:09:57.024850+0000 mgr.y (mgr.26252) 1316 : cluster [DBG] pgmap v1891: 89 pgs: 89 active+clean; 1.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:09:58.530 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:58.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:58.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:58.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:58.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:58.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:58.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:58.768 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:58.796 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:58.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:58.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:58.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:58.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:58.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:59.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:59.063 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:59.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:59.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:59.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:59.185 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:59.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:59.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:59.195 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 13 16 28047 28031 134.752 122.125 0.00195186 0.00739579 2024-09-13T15:09:59.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:59.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:59.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:59.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:59.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:59.471 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:59.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:59.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:59.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:59.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:59.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:59.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:59.713 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:59.735 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:59.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:59.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:59.840 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:59.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:59.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:59.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:59.998 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:00.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:00.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:00.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:00.125 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:00.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:00.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:00.195 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 14 16 29922 29906 133.497 117.188 0.00415492 0.00748352 2024-09-13T15:10:00.215 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:00.217 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:00.219 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:00.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:00.238 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:10:00 smithi177 bash[61439]: ts=2024-09-13T15:10:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:10:00.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:00 smithi177 bash[68594]: cluster 2024-09-13T15:09:59.025492+0000 mgr.y (mgr.26252) 1317 : cluster [DBG] pgmap v1892: 89 pgs: 89 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-09-13T15:10:00.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:00 smithi177 bash[68594]: cluster 2024-09-13T15:09:59.025492+0000 mgr.y (mgr.26252) 1317 : cluster [DBG] pgmap v1892: 89 pgs: 89 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-09-13T15:10:00.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:00 smithi177 bash[68594]: cluster 2024-09-13T15:10:00.000184+0000 mon.a (mon.0) 2496 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:00 smithi177 bash[68594]: cluster 2024-09-13T15:10:00.000184+0000 mon.a (mon.0) 2496 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:00 smithi177 bash[68594]: cluster 2024-09-13T15:10:00.000271+0000 mon.a (mon.0) 2497 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:00 smithi177 bash[68594]: cluster 2024-09-13T15:10:00.000271+0000 mon.a (mon.0) 2497 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.253 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:00.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:00.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:00.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[84576]: cluster 2024-09-13T15:09:59.025492+0000 mgr.y (mgr.26252) 1317 : cluster [DBG] pgmap v1892: 89 pgs: 89 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-09-13T15:10:00.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[84576]: cluster 2024-09-13T15:09:59.025492+0000 mgr.y (mgr.26252) 1317 : cluster [DBG] pgmap v1892: 89 pgs: 89 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-09-13T15:10:00.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[84576]: cluster 2024-09-13T15:10:00.000184+0000 mon.a (mon.0) 2496 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[84576]: cluster 2024-09-13T15:10:00.000184+0000 mon.a (mon.0) 2496 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[84576]: cluster 2024-09-13T15:10:00.000271+0000 mon.a (mon.0) 2497 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[84576]: cluster 2024-09-13T15:10:00.000271+0000 mon.a (mon.0) 2497 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[87147]: cluster 2024-09-13T15:09:59.025492+0000 mgr.y (mgr.26252) 1317 : cluster [DBG] pgmap v1892: 89 pgs: 89 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-09-13T15:10:00.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[87147]: cluster 2024-09-13T15:09:59.025492+0000 mgr.y (mgr.26252) 1317 : cluster [DBG] pgmap v1892: 89 pgs: 89 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-09-13T15:10:00.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[87147]: cluster 2024-09-13T15:10:00.000184+0000 mon.a (mon.0) 2496 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[87147]: cluster 2024-09-13T15:10:00.000184+0000 mon.a (mon.0) 2496 : cluster [WRN] Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[87147]: cluster 2024-09-13T15:10:00.000271+0000 mon.a (mon.0) 2497 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:00 smithi099 bash[87147]: cluster 2024-09-13T15:10:00.000271+0000 mon.a (mon.0) 2497 : cluster [WRN] [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-13T15:10:00.359 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:00.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:00.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:00.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:00.504 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:00.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:00.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:00.607 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:00.634 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:00.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:00.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:00.738 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:00.771 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:00.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:00.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:00.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:00.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:00.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:00.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:01.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:01.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:01.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:01.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:01.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:01.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:01.184 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:01.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:01.195 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 15 15 31582 31567 131.518 103.812 0.00653061 0.00759821 2024-09-13T15:10:01.285 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:01.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:01.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:01.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:01.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:01.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:01.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:01.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:01.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:01.581 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:01.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:01.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:01.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:01.717 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:01.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:01.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:01.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:01.839 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:01.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:01.942 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:01.972 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:01.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:01.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:02.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:02.104 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:02.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:02.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:02.195 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 16 16 33213 33197 129.664 101.875 0.0209915 0.0077055 2024-09-13T15:10:02.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:02.219 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:02 smithi099 bash[87147]: cluster 2024-09-13T15:10:01.025998+0000 mgr.y (mgr.26252) 1318 : cluster [DBG] pgmap v1893: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-13T15:10:02.219 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:02 smithi099 bash[87147]: cluster 2024-09-13T15:10:01.025998+0000 mgr.y (mgr.26252) 1318 : cluster [DBG] pgmap v1893: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-13T15:10:02.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:02 smithi099 bash[84576]: cluster 2024-09-13T15:10:01.025998+0000 mgr.y (mgr.26252) 1318 : cluster [DBG] pgmap v1893: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-13T15:10:02.219 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:02 smithi099 bash[84576]: cluster 2024-09-13T15:10:01.025998+0000 mgr.y (mgr.26252) 1318 : cluster [DBG] pgmap v1893: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-13T15:10:02.252 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:02.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:02.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:02.356 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:02.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:02 smithi177 bash[68594]: cluster 2024-09-13T15:10:01.025998+0000 mgr.y (mgr.26252) 1318 : cluster [DBG] pgmap v1893: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-13T15:10:02.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:02 smithi177 bash[68594]: cluster 2024-09-13T15:10:01.025998+0000 mgr.y (mgr.26252) 1318 : cluster [DBG] pgmap v1893: 89 pgs: 89 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-13T15:10:02.386 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:02.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:02.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:02.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:02.523 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:02.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:02.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:02.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:02.660 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:02.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:02.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:02.786 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:02.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:02.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:02.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:02.909 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:02.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:02.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:03.014 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:03.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:03.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:03.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:03.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:03.161 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:03.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:03.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:03.196 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 17 16 34482 34466 126.702 79.3125 0.00259048 0.00780135 2024-09-13T15:10:03.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:03.290 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:03.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:03.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:03.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:03.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:03.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:03.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:03.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:03.551 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:03.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:03.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:03.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:03.684 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:03.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:03.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:03.791 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:03.821 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:03.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:03.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:03.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:03.942 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:03.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:03.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:04.045 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:04.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:04.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:04.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:04.175 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:04.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:04 smithi099 bash[84576]: cluster 2024-09-13T15:10:03.026447+0000 mgr.y (mgr.26252) 1319 : cluster [DBG] pgmap v1894: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T15:10:04.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:04 smithi099 bash[84576]: cluster 2024-09-13T15:10:03.026447+0000 mgr.y (mgr.26252) 1319 : cluster [DBG] pgmap v1894: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T15:10:04.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:04 smithi099 bash[84576]: audit 2024-09-13T15:10:03.306893+0000 mon.a (mon.0) 2498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:04.185 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:04 smithi099 bash[84576]: audit 2024-09-13T15:10:03.306893+0000 mon.a (mon.0) 2498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:04.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:04 smithi099 bash[87147]: cluster 2024-09-13T15:10:03.026447+0000 mgr.y (mgr.26252) 1319 : cluster [DBG] pgmap v1894: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T15:10:04.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:04 smithi099 bash[87147]: cluster 2024-09-13T15:10:03.026447+0000 mgr.y (mgr.26252) 1319 : cluster [DBG] pgmap v1894: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T15:10:04.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:04 smithi099 bash[87147]: audit 2024-09-13T15:10:03.306893+0000 mon.a (mon.0) 2498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:04.186 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:04 smithi099 bash[87147]: audit 2024-09-13T15:10:03.306893+0000 mon.a (mon.0) 2498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:04.196 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 18 16 34967 34951 121.346 30.3125 0.00227716 0.00809369 2024-09-13T15:10:04.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:04.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:04.308 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:04.320 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:04 smithi177 bash[68594]: cluster 2024-09-13T15:10:03.026447+0000 mgr.y (mgr.26252) 1319 : cluster [DBG] pgmap v1894: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T15:10:04.320 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:04 smithi177 bash[68594]: cluster 2024-09-13T15:10:03.026447+0000 mgr.y (mgr.26252) 1319 : cluster [DBG] pgmap v1894: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T15:10:04.320 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:04 smithi177 bash[68594]: audit 2024-09-13T15:10:03.306893+0000 mon.a (mon.0) 2498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:04.320 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:04 smithi177 bash[68594]: audit 2024-09-13T15:10:03.306893+0000 mon.a (mon.0) 2498 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:04.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:04.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:04.436 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:04.467 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:04.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:04.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:04.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:04.599 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:04.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:04.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:04.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:04.726 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:04.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:04.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:04.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:04.854 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:04.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:04.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:04.957 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:04.985 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:04.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:04.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:05.088 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:05.119 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:05.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:05.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:05.196 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 19 16 35849 35833 117.861 55.125 0.0131972 0.00847802 2024-09-13T15:10:05.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:05.252 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:05.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:05.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:05.356 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:05.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:05.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:05.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:05.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:05.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:05.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:05.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:05.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:05.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:05.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:05.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:05.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:05.774 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:05.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:05.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:05.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:05.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:05.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:05.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:06.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:06.030 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:06.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:06.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:06.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:06.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:06.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:06.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:06.196 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:10:06.197056+0000 min lat: 0.00117806 max lat: 0.663272 avg lat: 0.00861904 2024-09-13T15:10:06.196 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:10:06.196 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 20 16 37123 37107 115.949 79.625 0.0148018 0.00861904 2024-09-13T15:10:06.265 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:06.287 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:06.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:06.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:06.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:06 smithi177 bash[68594]: cluster 2024-09-13T15:10:05.027886+0000 mgr.y (mgr.26252) 1320 : cluster [DBG] pgmap v1895: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T15:10:06.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:06 smithi177 bash[68594]: cluster 2024-09-13T15:10:05.027886+0000 mgr.y (mgr.26252) 1320 : cluster [DBG] pgmap v1895: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T15:10:06.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:06.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:06.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:06.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:06.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:06.528 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:06 smithi099 bash[84576]: cluster 2024-09-13T15:10:05.027886+0000 mgr.y (mgr.26252) 1320 : cluster [DBG] pgmap v1895: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T15:10:06.528 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:06 smithi099 bash[84576]: cluster 2024-09-13T15:10:05.027886+0000 mgr.y (mgr.26252) 1320 : cluster [DBG] pgmap v1895: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T15:10:06.528 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:06 smithi099 bash[87147]: cluster 2024-09-13T15:10:05.027886+0000 mgr.y (mgr.26252) 1320 : cluster [DBG] pgmap v1895: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T15:10:06.528 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:06 smithi099 bash[87147]: cluster 2024-09-13T15:10:05.027886+0000 mgr.y (mgr.26252) 1320 : cluster [DBG] pgmap v1895: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T15:10:06.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:06.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:06.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:06.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:06.671 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:06.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:06.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:06.774 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:06.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:06.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:06.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:06.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:06.940 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:06.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:06.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:07.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:07.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:07.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:07.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:07.198 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:10:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:10:07] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:10:07.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 21 16 37685 37669 112.094 35.125 0.0208291 0.00891574 2024-09-13T15:10:07.205 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:07.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:07.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:07.310 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:07.338 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:07.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:07.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:07.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:07.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:07.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:07.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:07.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:07.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:07.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:07.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:07.704 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:07.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:07.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:07.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:07.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:07.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:07.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:07.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:08.002 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:08.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:08.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:08.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:08.133 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:08.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:08.197 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 22 16 39056 39040 110.893 85.6875 0.00822233 0.00901265 2024-09-13T15:10:08.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:08.248 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:08 smithi177 bash[68594]: cluster 2024-09-13T15:10:07.028557+0000 mgr.y (mgr.26252) 1321 : cluster [DBG] pgmap v1896: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T15:10:08.248 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:08 smithi177 bash[68594]: cluster 2024-09-13T15:10:07.028557+0000 mgr.y (mgr.26252) 1321 : cluster [DBG] pgmap v1896: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T15:10:08.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:08.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:08.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:08.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:08.376 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:08 smithi099 bash[84576]: cluster 2024-09-13T15:10:07.028557+0000 mgr.y (mgr.26252) 1321 : cluster [DBG] pgmap v1896: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T15:10:08.376 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:08 smithi099 bash[84576]: cluster 2024-09-13T15:10:07.028557+0000 mgr.y (mgr.26252) 1321 : cluster [DBG] pgmap v1896: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T15:10:08.376 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:08 smithi099 bash[87147]: cluster 2024-09-13T15:10:07.028557+0000 mgr.y (mgr.26252) 1321 : cluster [DBG] pgmap v1896: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T15:10:08.376 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:08 smithi099 bash[87147]: cluster 2024-09-13T15:10:07.028557+0000 mgr.y (mgr.26252) 1321 : cluster [DBG] pgmap v1896: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T15:10:08.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:08.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:08.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:08.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:08.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:08.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:08.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:08.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:08.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:08.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:08.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:08.765 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:08.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:08.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:08.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:08.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:08.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:08.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:08.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:09.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:09.042 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:09.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:09.146 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:09.167 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:09.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:09.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:09.197 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 23 16 40237 40221 109.281 73.8125 0.00250232 0.00912445 2024-09-13T15:10:09.270 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:09.295 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:09.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:09.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:09.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:09.420 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:09.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:09.523 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:09.547 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:09.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:09.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:09.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:09.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:09.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:09.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:09.779 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:09.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:09.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:09.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:09.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:10.023 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:10.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:10.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:10.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:10.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:10.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:10.197 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 24 16 40763 40747 106.097 32.875 0.00440679 0.00941898 2024-09-13T15:10:10.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:10.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:10 smithi099 bash[84576]: cluster 2024-09-13T15:10:09.029340+0000 mgr.y (mgr.26252) 1322 : cluster [DBG] pgmap v1897: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T15:10:10.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:10 smithi099 bash[84576]: cluster 2024-09-13T15:10:09.029340+0000 mgr.y (mgr.26252) 1322 : cluster [DBG] pgmap v1897: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T15:10:10.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:10 smithi099 bash[87147]: cluster 2024-09-13T15:10:09.029340+0000 mgr.y (mgr.26252) 1322 : cluster [DBG] pgmap v1897: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T15:10:10.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:10 smithi099 bash[87147]: cluster 2024-09-13T15:10:09.029340+0000 mgr.y (mgr.26252) 1322 : cluster [DBG] pgmap v1897: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T15:10:10.297 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:10.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:10.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:10 smithi177 bash[68594]: cluster 2024-09-13T15:10:09.029340+0000 mgr.y (mgr.26252) 1322 : cluster [DBG] pgmap v1897: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T15:10:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:10 smithi177 bash[68594]: cluster 2024-09-13T15:10:09.029340+0000 mgr.y (mgr.26252) 1322 : cluster [DBG] pgmap v1897: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T15:10:10.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:10:10 smithi177 bash[61439]: ts=2024-09-13T15:10:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:10:10.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:10.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:10.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:10.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:10.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:10.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:10.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:10.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:10.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:10.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:10.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:10.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:10.806 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:10.834 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:10.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:10.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:10.938 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:10.969 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:10.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:10.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:11.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:11.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:11.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:11.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:11.198 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 25 16 42338 42322 105.79 98.4375 0.0093286 0.0094478 2024-09-13T15:10:11.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:11.242 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:11.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:11.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:11.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:11.365 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:11.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:11.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:11.489 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:11.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:11.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:11.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:11.635 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:11.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:11.738 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:11.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:11.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:11.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:11.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:11.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:11.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:11.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:11.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:12.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:12.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:12.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:12.119 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:12.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:12.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:12.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:12.197 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 26 16 43101 43085 103.556 47.6875 0.00859121 0.00965151 2024-09-13T15:10:12.260 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:12.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:12.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:12.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:12.380 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:12 smithi099 bash[84576]: cluster 2024-09-13T15:10:11.029905+0000 mgr.y (mgr.26252) 1323 : cluster [DBG] pgmap v1898: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T15:10:12.380 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:12 smithi099 bash[84576]: cluster 2024-09-13T15:10:11.029905+0000 mgr.y (mgr.26252) 1323 : cluster [DBG] pgmap v1898: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T15:10:12.380 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:12 smithi099 bash[87147]: cluster 2024-09-13T15:10:11.029905+0000 mgr.y (mgr.26252) 1323 : cluster [DBG] pgmap v1898: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T15:10:12.380 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:12 smithi099 bash[87147]: cluster 2024-09-13T15:10:11.029905+0000 mgr.y (mgr.26252) 1323 : cluster [DBG] pgmap v1898: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T15:10:12.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:12.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:12.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:12.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:12.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:12.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:12.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:12.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:12 smithi177 bash[68594]: cluster 2024-09-13T15:10:11.029905+0000 mgr.y (mgr.26252) 1323 : cluster [DBG] pgmap v1898: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T15:10:12.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:12 smithi177 bash[68594]: cluster 2024-09-13T15:10:11.029905+0000 mgr.y (mgr.26252) 1323 : cluster [DBG] pgmap v1898: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T15:10:12.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:12.672 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:12.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:12.775 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:12.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:12.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:12.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:12.904 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:12.930 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:12.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:12.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:13.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:13.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:13.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:13.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:13.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:13.188 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:13.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:13.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:13.197 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 27 16 43625 43609 100.933 32.75 0.00777809 0.00990243 2024-09-13T15:10:13.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:13.320 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:13.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:13.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:13.426 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:13.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:13.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:13.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:13.550 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:13.592 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:13.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:13.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:13.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:13.721 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:13.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:13.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:13.824 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:13.852 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:13.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:13.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:13.959 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:13.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:13.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:13.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:14.094 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:14.130 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:14.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:14.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:14.198 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 28 16 44959 44943 100.306 83.375 0.0130409 0.00996308 2024-09-13T15:10:14.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:14.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:14.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:14.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:14.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:14.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:14.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:14 smithi099 bash[84576]: cluster 2024-09-13T15:10:13.030622+0000 mgr.y (mgr.26252) 1324 : cluster [DBG] pgmap v1899: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:14.391 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:14 smithi099 bash[84576]: cluster 2024-09-13T15:10:13.030622+0000 mgr.y (mgr.26252) 1324 : cluster [DBG] pgmap v1899: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:14.393 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:14 smithi099 bash[87147]: cluster 2024-09-13T15:10:13.030622+0000 mgr.y (mgr.26252) 1324 : cluster [DBG] pgmap v1899: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:14.394 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:14 smithi099 bash[87147]: cluster 2024-09-13T15:10:13.030622+0000 mgr.y (mgr.26252) 1324 : cluster [DBG] pgmap v1899: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:14.467 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:14 smithi177 bash[68594]: cluster 2024-09-13T15:10:13.030622+0000 mgr.y (mgr.26252) 1324 : cluster [DBG] pgmap v1899: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:14.467 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:14 smithi177 bash[68594]: cluster 2024-09-13T15:10:13.030622+0000 mgr.y (mgr.26252) 1324 : cluster [DBG] pgmap v1899: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:14.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:14.523 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:14.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:14.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:14.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:14.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:14.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:14.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:14.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:14.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:14.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:14.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:14.861 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:10:14 smithi177 bash[60145]: logger=cleanup t=2024-09-13T15:10:14.468201411Z level=info msg="Completed cleanup jobs" duration=22.432077ms 2024-09-13T15:10:14.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:14.913 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:14.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:15.018 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:15.036 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:15.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:15.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:15.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:15.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:15.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:15.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:15.198 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 29 16 46066 46050 99.2324 69.1875 0.057424 0.0100503 2024-09-13T15:10:15.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:15.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:15.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:15.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:15.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:15.415 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:15.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:15.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:15.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:15.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:15.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:15.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:15.648 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:15.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:15.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:15.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:15.791 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:15.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:15.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:15.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:15.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:15.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:15.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:15.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:16.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:16.083 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:16.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:16.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:16.198 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 30 16 46879 46863 97.6183 50.8125 0.0220718 0.0102382 2024-09-13T15:10:16.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:16 smithi177 bash[68594]: cluster 2024-09-13T15:10:15.031371+0000 mgr.y (mgr.26252) 1325 : cluster [DBG] pgmap v1900: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-13T15:10:16.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:16 smithi177 bash[68594]: cluster 2024-09-13T15:10:15.031371+0000 mgr.y (mgr.26252) 1325 : cluster [DBG] pgmap v1900: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-13T15:10:16.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:16.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:16.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:16.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:16.356 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:16.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:16.430 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:16 smithi099 bash[84576]: cluster 2024-09-13T15:10:15.031371+0000 mgr.y (mgr.26252) 1325 : cluster [DBG] pgmap v1900: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-13T15:10:16.430 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:16 smithi099 bash[84576]: cluster 2024-09-13T15:10:15.031371+0000 mgr.y (mgr.26252) 1325 : cluster [DBG] pgmap v1900: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-13T15:10:16.430 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:16 smithi099 bash[87147]: cluster 2024-09-13T15:10:15.031371+0000 mgr.y (mgr.26252) 1325 : cluster [DBG] pgmap v1900: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-13T15:10:16.430 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:16 smithi099 bash[87147]: cluster 2024-09-13T15:10:15.031371+0000 mgr.y (mgr.26252) 1325 : cluster [DBG] pgmap v1900: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-13T15:10:16.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:16.476 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:16.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:16.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:16.580 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:16.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:16.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:16.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:16.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:16.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:16.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:16.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:16.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:16.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:16.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:16.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:16.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:16.994 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:16.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:16.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:17.096 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:17.109 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:10:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:10:17] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:10:17.128 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:17.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:17.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:17.198 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 31 16 47740 47724 96.2051 53.8125 0.0153414 0.0103892 2024-09-13T15:10:17.232 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:17.257 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:17.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:17.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:17.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:17.391 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:17.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:17.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:17.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:17.523 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:17.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:17.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:17.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:17.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:17.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:17.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:17.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:17.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:17.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:17.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:17.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:18.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:18.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:18.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:18.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:18.173 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:18 smithi177 bash[68594]: cluster 2024-09-13T15:10:17.032160+0000 mgr.y (mgr.26252) 1326 : cluster [DBG] pgmap v1901: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T15:10:18.173 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:18 smithi177 bash[68594]: cluster 2024-09-13T15:10:17.032160+0000 mgr.y (mgr.26252) 1326 : cluster [DBG] pgmap v1901: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T15:10:18.192 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:18.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:18.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:18.198 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 32 16 48896 48880 95.4563 72.25 0.00789985 0.0104711 2024-09-13T15:10:18.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:18.322 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:18.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:18 smithi099 bash[84576]: cluster 2024-09-13T15:10:17.032160+0000 mgr.y (mgr.26252) 1326 : cluster [DBG] pgmap v1901: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T15:10:18.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:18 smithi099 bash[84576]: cluster 2024-09-13T15:10:17.032160+0000 mgr.y (mgr.26252) 1326 : cluster [DBG] pgmap v1901: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T15:10:18.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:18 smithi099 bash[87147]: cluster 2024-09-13T15:10:17.032160+0000 mgr.y (mgr.26252) 1326 : cluster [DBG] pgmap v1901: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T15:10:18.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:18 smithi099 bash[87147]: cluster 2024-09-13T15:10:17.032160+0000 mgr.y (mgr.26252) 1326 : cluster [DBG] pgmap v1901: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T15:10:18.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:18.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:18.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:18.449 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:18.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:18.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:18.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:18.578 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:18.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:18.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:18.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:18.714 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:18.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:18.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:18.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:18.842 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:18.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:18.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:18.946 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:18.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:18.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:18.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:19.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:19.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:19.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:19.198 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 33 16 50416 50400 95.4423 95 0.00227042 0.0104713 2024-09-13T15:10:19.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:19.233 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:19.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:19.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:19 smithi099 bash[87147]: audit 2024-09-13T15:10:18.307114+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:19 smithi099 bash[87147]: audit 2024-09-13T15:10:18.307114+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:19 smithi099 bash[84576]: audit 2024-09-13T15:10:18.307114+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:19 smithi099 bash[84576]: audit 2024-09-13T15:10:18.307114+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:19.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:19.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:19 smithi177 bash[68594]: audit 2024-09-13T15:10:18.307114+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:19.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:19 smithi177 bash[68594]: audit 2024-09-13T15:10:18.307114+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:19.368 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:19.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:19.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:19.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:19.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:19.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:19.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:19.602 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:19.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:19.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:19.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:19.734 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:19.752 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:19.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:19.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:19.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:19.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:19.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:19.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:19.991 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:20.020 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:20.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:20.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:20.124 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:20.141 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:20.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:20.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:20.198 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 34 16 51612 51596 94.8335 74.75 0.00991084 0.0105396 2024-09-13T15:10:20.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:20.268 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:20.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:20.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:20.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:20.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:20.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:20.504 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:20.515 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:20 smithi099 bash[84576]: cluster 2024-09-13T15:10:19.033210+0000 mgr.y (mgr.26252) 1327 : cluster [DBG] pgmap v1902: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-13T15:10:20.515 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:20 smithi099 bash[84576]: cluster 2024-09-13T15:10:19.033210+0000 mgr.y (mgr.26252) 1327 : cluster [DBG] pgmap v1902: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-13T15:10:20.515 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:20 smithi099 bash[87147]: cluster 2024-09-13T15:10:19.033210+0000 mgr.y (mgr.26252) 1327 : cluster [DBG] pgmap v1902: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-13T15:10:20.516 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:20 smithi099 bash[87147]: cluster 2024-09-13T15:10:19.033210+0000 mgr.y (mgr.26252) 1327 : cluster [DBG] pgmap v1902: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-13T15:10:20.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:20.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:20.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:20.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:10:20 smithi177 bash[61439]: ts=2024-09-13T15:10:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:10:20.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:20 smithi177 bash[68594]: cluster 2024-09-13T15:10:19.033210+0000 mgr.y (mgr.26252) 1327 : cluster [DBG] pgmap v1902: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-13T15:10:20.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:20 smithi177 bash[68594]: cluster 2024-09-13T15:10:19.033210+0000 mgr.y (mgr.26252) 1327 : cluster [DBG] pgmap v1902: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2024-09-13T15:10:20.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:20.653 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:20.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:20.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:20.756 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:20.783 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:20.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:20.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:20.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:20.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:20.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:20.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:21.021 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:21.046 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:21.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:21.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:21.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:21.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:21.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:21.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:21.198 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 35 16 52950 52934 94.5131 83.625 0.00670647 0.0105641 2024-09-13T15:10:21.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:21.285 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:21 smithi177 bash[68594]: cluster 2024-09-13T15:10:21.033927+0000 mgr.y (mgr.26252) 1328 : cluster [DBG] pgmap v1903: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:10:21.285 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:21 smithi177 bash[68594]: cluster 2024-09-13T15:10:21.033927+0000 mgr.y (mgr.26252) 1328 : cluster [DBG] pgmap v1903: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:10:21.305 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:21.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:21.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:21.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:21.420 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:21 smithi099 bash[84576]: cluster 2024-09-13T15:10:21.033927+0000 mgr.y (mgr.26252) 1328 : cluster [DBG] pgmap v1903: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:10:21.420 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:21 smithi099 bash[84576]: cluster 2024-09-13T15:10:21.033927+0000 mgr.y (mgr.26252) 1328 : cluster [DBG] pgmap v1903: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:10:21.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:21 smithi099 bash[87147]: cluster 2024-09-13T15:10:21.033927+0000 mgr.y (mgr.26252) 1328 : cluster [DBG] pgmap v1903: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:10:21.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:21 smithi099 bash[87147]: cluster 2024-09-13T15:10:21.033927+0000 mgr.y (mgr.26252) 1328 : cluster [DBG] pgmap v1903: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:10:21.444 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:21.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:21.548 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:21.571 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:21.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:21.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:21.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:21.696 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:21.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:21.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:21.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:21.832 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:21.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:21.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:21.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:21.963 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:21.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:21.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:22.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:22.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:22.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:22.198 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 36 16 54167 54151 94.0004 76.0625 0.0152466 0.0106324 2024-09-13T15:10:22.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:22.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:22.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:22.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:22.333 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:22.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:22.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:22.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:22.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:22.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:22.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:22.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:22.580 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:22.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:22.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:22.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:22.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:22.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:22.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:22.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:22.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:22.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:22.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:22.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:22.970 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:22.997 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:22.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:23.099 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:23.127 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:23.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:23.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:23.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 37 16 55265 55249 93.3145 68.625 0.0106412 0.0107107 2024-09-13T15:10:23.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:23.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:23.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:23.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:23.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:23.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:23.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:23.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:23.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:23.513 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:23.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:23.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:23.615 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:23.649 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:23.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:23.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:23.753 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:23.780 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:23.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:23.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:23.885 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:23.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:23.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:23.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:24.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:24.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:24.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:24.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:24.133 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:24.159 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:24.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:24.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:24.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 38 16 56291 56275 92.5462 64.125 0.00229916 0.0107341 2024-09-13T15:10:24.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:24.288 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:24.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:24.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:24 smithi177 bash[68594]: cluster 2024-09-13T15:10:23.034473+0000 mgr.y (mgr.26252) 1329 : cluster [DBG] pgmap v1904: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-09-13T15:10:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:24 smithi177 bash[68594]: cluster 2024-09-13T15:10:23.034473+0000 mgr.y (mgr.26252) 1329 : cluster [DBG] pgmap v1904: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-09-13T15:10:24.375 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:24 smithi099 bash[84576]: cluster 2024-09-13T15:10:23.034473+0000 mgr.y (mgr.26252) 1329 : cluster [DBG] pgmap v1904: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-09-13T15:10:24.376 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:24 smithi099 bash[84576]: cluster 2024-09-13T15:10:23.034473+0000 mgr.y (mgr.26252) 1329 : cluster [DBG] pgmap v1904: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-09-13T15:10:24.376 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:24 smithi099 bash[87147]: cluster 2024-09-13T15:10:23.034473+0000 mgr.y (mgr.26252) 1329 : cluster [DBG] pgmap v1904: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-09-13T15:10:24.376 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:24 smithi099 bash[87147]: cluster 2024-09-13T15:10:23.034473+0000 mgr.y (mgr.26252) 1329 : cluster [DBG] pgmap v1904: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-09-13T15:10:24.391 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:24.428 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:24.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:24.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:24.533 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:24.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:24.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:24.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:24.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:24.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:24.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:24.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:24.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:24.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:24.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:24.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:24.917 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:24.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:24.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:24.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:25.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:25.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:25.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:25.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:25.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:25.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 39 16 57121 57105 91.5031 51.875 0.00779676 0.010862 2024-09-13T15:10:25.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:25.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:25.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:25.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:25.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:25.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:25.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:25.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:25.464 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:25.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:25.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:25.573 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:25.598 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:25.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:25.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:25.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:25.728 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:25.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:25.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:25.834 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:25.872 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:25.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:25.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:25.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:26.007 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:26.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:26.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:26.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:26 smithi177 bash[68594]: cluster 2024-09-13T15:10:25.035459+0000 mgr.y (mgr.26252) 1330 : cluster [DBG] pgmap v1905: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T15:10:26.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:26 smithi177 bash[68594]: cluster 2024-09-13T15:10:25.035459+0000 mgr.y (mgr.26252) 1330 : cluster [DBG] pgmap v1905: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T15:10:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:26.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:26.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:26.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:10:26.200148+0000 min lat: 0.00117806 max lat: 0.663272 avg lat: 0.0109606 2024-09-13T15:10:26.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:10:26.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 40 16 58382 58366 91.1856 78.8125 0.0233228 0.0109606 2024-09-13T15:10:26.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:26.272 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:26.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:26.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:26.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:26 smithi099 bash[84576]: cluster 2024-09-13T15:10:25.035459+0000 mgr.y (mgr.26252) 1330 : cluster [DBG] pgmap v1905: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T15:10:26.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:26 smithi099 bash[84576]: cluster 2024-09-13T15:10:25.035459+0000 mgr.y (mgr.26252) 1330 : cluster [DBG] pgmap v1905: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T15:10:26.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:26 smithi099 bash[87147]: cluster 2024-09-13T15:10:25.035459+0000 mgr.y (mgr.26252) 1330 : cluster [DBG] pgmap v1905: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T15:10:26.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:26 smithi099 bash[87147]: cluster 2024-09-13T15:10:25.035459+0000 mgr.y (mgr.26252) 1330 : cluster [DBG] pgmap v1905: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-09-13T15:10:26.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:26.401 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:26.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:26.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:26.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:26.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:26.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:26.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:26.657 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:26.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:26.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:26.765 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:26.783 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:26.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:26.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:26.886 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:26.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:26.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:26.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:27.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:27.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:27.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:27.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:27.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:27.165 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:27.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:27.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 41 16 59080 59064 90.0255 43.625 0.0119662 0.0110894 2024-09-13T15:10:27.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:27.293 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:27.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:27.373 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:10:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:10:27] "GET /metrics HTTP/1.1" 200 36771 "" "Prometheus/2.51.0" 2024-09-13T15:10:27.396 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:27.436 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:27.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:27.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:27.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:27.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:27.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:27.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:27.658 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:27 smithi099 bash[87147]: cluster 2024-09-13T15:10:27.036011+0000 mgr.y (mgr.26252) 1331 : cluster [DBG] pgmap v1906: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:27.659 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:27 smithi099 bash[87147]: cluster 2024-09-13T15:10:27.036011+0000 mgr.y (mgr.26252) 1331 : cluster [DBG] pgmap v1906: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:27.676 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:27.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:27.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:27.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:27.814 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:27.826 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:27 smithi177 bash[68594]: cluster 2024-09-13T15:10:27.036011+0000 mgr.y (mgr.26252) 1331 : cluster [DBG] pgmap v1906: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:27.826 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:27 smithi177 bash[68594]: cluster 2024-09-13T15:10:27.036011+0000 mgr.y (mgr.26252) 1331 : cluster [DBG] pgmap v1906: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:27.843 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:27.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:27.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:27.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:27.978 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:27.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:27.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:27.989 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:10:27.989 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:10:27.989 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:10:27.989 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:10:27.989 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:10:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:27 smithi099 bash[84576]: cluster 2024-09-13T15:10:27.036011+0000 mgr.y (mgr.26252) 1331 : cluster [DBG] pgmap v1906: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:27 smithi099 bash[84576]: cluster 2024-09-13T15:10:27.036011+0000 mgr.y (mgr.26252) 1331 : cluster [DBG] pgmap v1906: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:28.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:28.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:28.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:28.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:28.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 42 16 60157 60141 89.4846 67.3125 0.00614447 0.0111694 2024-09-13T15:10:28.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:28.238 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:28.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:28.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:28.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:28.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:28.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:28.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:28.471 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:28.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:28.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:28.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:28.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:28.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:28.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:28.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:28.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:28.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:28.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:28.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:28.891 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:28.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:28.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:28.994 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:29.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:29.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:29.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:29.140 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:29.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:29.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:29.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 43 16 61127 61111 88.8134 60.625 0.00294828 0.0112251 2024-09-13T15:10:29.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:29.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:29.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:29.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:29.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:29.403 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:29.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:29.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:29.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:29.531 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:29.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:29.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:29.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:29.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:29.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:29.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:29.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:29.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:29.906 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:29.928 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:29.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:29.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:30.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:30.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:30.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:30.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:30.184 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:30.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:30.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 44 16 62284 62268 88.4382 72.3125 0.00989986 0.0112758 2024-09-13T15:10:30.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:30 smithi099 bash[84576]: cluster 2024-09-13T15:10:29.036729+0000 mgr.y (mgr.26252) 1332 : cluster [DBG] pgmap v1907: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:10:30.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:30 smithi099 bash[87147]: cluster 2024-09-13T15:10:29.036729+0000 mgr.y (mgr.26252) 1332 : cluster [DBG] pgmap v1907: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:10:30.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:30.305 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:30 smithi177 bash[68594]: cluster 2024-09-13T15:10:29.036729+0000 mgr.y (mgr.26252) 1332 : cluster [DBG] pgmap v1907: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:10:30.305 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:30 smithi177 bash[68594]: cluster 2024-09-13T15:10:29.036729+0000 mgr.y (mgr.26252) 1332 : cluster [DBG] pgmap v1907: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:10:30.305 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:10:30 smithi177 bash[61439]: ts=2024-09-13T15:10:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:10:30.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:30.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:30.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:30 smithi099 bash[84576]: cluster 2024-09-13T15:10:29.036729+0000 mgr.y (mgr.26252) 1332 : cluster [DBG] pgmap v1907: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:10:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:30 smithi099 bash[87147]: cluster 2024-09-13T15:10:29.036729+0000 mgr.y (mgr.26252) 1332 : cluster [DBG] pgmap v1907: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:10:30.423 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:30.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:30.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:30.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:30.593 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:30.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:30.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:30.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:30.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:30.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:30.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:30.830 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:30.853 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:30.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:30.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:30.956 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:30.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:30.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:30.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:31.083 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:31.108 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:31.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:31.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:31.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 45 16 62824 62808 87.2229 33.75 0.015032 0.0114356 2024-09-13T15:10:31.212 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:31.240 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:31.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:31.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:31.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:31.365 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:31.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:31.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:31.497 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:31.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:31.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:31.602 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:31.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:31.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:31.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:31.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:31.768 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:31.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:31.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:31.871 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:31.898 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:31.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:31.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:32.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:32.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T15:10:32.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:32.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:32.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:32.135 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:32 smithi099 bash[84576]: cluster 2024-09-13T15:10:31.037699+0000 mgr.y (mgr.26252) 1333 : cluster [DBG] pgmap v1908: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T15:10:32.135 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:32 smithi099 bash[84576]: cluster 2024-09-13T15:10:31.037699+0000 mgr.y (mgr.26252) 1333 : cluster [DBG] pgmap v1908: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T15:10:32.135 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:32 smithi099 bash[87147]: cluster 2024-09-13T15:10:31.037699+0000 mgr.y (mgr.26252) 1333 : cluster [DBG] pgmap v1908: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T15:10:32.135 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:32 smithi099 bash[87147]: cluster 2024-09-13T15:10:31.037699+0000 mgr.y (mgr.26252) 1333 : cluster [DBG] pgmap v1908: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T15:10:32.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:32.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:32.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:32.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:32.199 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 46 16 63717 63701 86.5399 55.8125 0.00410907 0.0115474 2024-09-13T15:10:32.244 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:10:32.245 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:10:32.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:32.314 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:32.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:32.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:32.329 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:10:32.329 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:10:32.361 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:10:32.361 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:10:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:32 smithi177 bash[68594]: cluster 2024-09-13T15:10:31.037699+0000 mgr.y (mgr.26252) 1333 : cluster [DBG] pgmap v1908: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T15:10:32.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:32 smithi177 bash[68594]: cluster 2024-09-13T15:10:31.037699+0000 mgr.y (mgr.26252) 1333 : cluster [DBG] pgmap v1908: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T15:10:32.396 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:10:32.396 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:10:32.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:32.431 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:10:32.432 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:10:32.447 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:32.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:32.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:32.463 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:10:32.464 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:10:32.492 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:10:32.492 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:10:32.532 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:10:32.533 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:10:32.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:32.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:32.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:32.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:32.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:32.715 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:32.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:32.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:32.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:32.850 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:32.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:32.955 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:32.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:32.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:33.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:33.109 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:33.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:33.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:33.114 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:10:33.200 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 47 16 64805 64789 86.1453 68 0.0179882 0.0116033 2024-09-13T15:10:33.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:33.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:33.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:33.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:33.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:33.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:33 smithi177 bash[68594]: audit 2024-09-13T15:10:32.212973+0000 mon.a (mon.0) 2500 : audit [DBG] from='client.? 172.21.15.99:0/2000141778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:10:33.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:33 smithi177 bash[68594]: audit 2024-09-13T15:10:32.212973+0000 mon.a (mon.0) 2500 : audit [DBG] from='client.? 172.21.15.99:0/2000141778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:10:33.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:33 smithi177 bash[68594]: audit 2024-09-13T15:10:32.672284+0000 mon.a (mon.0) 2501 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:10:33.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:33 smithi177 bash[68594]: audit 2024-09-13T15:10:32.672284+0000 mon.a (mon.0) 2501 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:10:33.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:33.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:33.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:33.394 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:33 smithi099 bash[84576]: audit 2024-09-13T15:10:32.212973+0000 mon.a (mon.0) 2500 : audit [DBG] from='client.? 172.21.15.99:0/2000141778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:10:33.394 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:33 smithi099 bash[84576]: audit 2024-09-13T15:10:32.212973+0000 mon.a (mon.0) 2500 : audit [DBG] from='client.? 172.21.15.99:0/2000141778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:10:33.394 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:33 smithi099 bash[84576]: audit 2024-09-13T15:10:32.672284+0000 mon.a (mon.0) 2501 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:10:33.394 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:33 smithi099 bash[84576]: audit 2024-09-13T15:10:32.672284+0000 mon.a (mon.0) 2501 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:10:33.395 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:33 smithi099 bash[87147]: audit 2024-09-13T15:10:32.212973+0000 mon.a (mon.0) 2500 : audit [DBG] from='client.? 172.21.15.99:0/2000141778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:10:33.395 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:33 smithi099 bash[87147]: audit 2024-09-13T15:10:32.212973+0000 mon.a (mon.0) 2500 : audit [DBG] from='client.? 172.21.15.99:0/2000141778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:10:33.395 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:33 smithi099 bash[87147]: audit 2024-09-13T15:10:32.672284+0000 mon.a (mon.0) 2501 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:10:33.395 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:33 smithi099 bash[87147]: audit 2024-09-13T15:10:32.672284+0000 mon.a (mon.0) 2501 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:10:33.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:33.506 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:33.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:33.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:33.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:33.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:33.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:33.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:33.746 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:33.770 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:33.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:33.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:33.912 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:33.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:33.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:34.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:34.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:34.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:34.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:34.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:34.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:34.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:34.200 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 48 16 66094 66078 86.0289 80.5625 0.00917511 0.0116182 2024-09-13T15:10:34.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:34.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:34.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:34.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[84576]: cluster 2024-09-13T15:10:33.038268+0000 mgr.y (mgr.26252) 1334 : cluster [DBG] pgmap v1909: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T15:10:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[84576]: cluster 2024-09-13T15:10:33.038268+0000 mgr.y (mgr.26252) 1334 : cluster [DBG] pgmap v1909: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T15:10:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[84576]: cluster 2024-09-13T15:10:33.096843+0000 mon.a (mon.0) 2502 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:10:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[84576]: cluster 2024-09-13T15:10:33.096843+0000 mon.a (mon.0) 2502 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:10:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[84576]: audit 2024-09-13T15:10:33.106775+0000 mon.a (mon.0) 2503 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:10:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[84576]: audit 2024-09-13T15:10:33.106775+0000 mon.a (mon.0) 2503 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:10:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[84576]: cluster 2024-09-13T15:10:33.111807+0000 mon.a (mon.0) 2504 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2024-09-13T15:10:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[84576]: cluster 2024-09-13T15:10:33.111807+0000 mon.a (mon.0) 2504 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2024-09-13T15:10:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[84576]: audit 2024-09-13T15:10:33.307399+0000 mon.a (mon.0) 2505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[84576]: audit 2024-09-13T15:10:33.307399+0000 mon.a (mon.0) 2505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[87147]: cluster 2024-09-13T15:10:33.038268+0000 mgr.y (mgr.26252) 1334 : cluster [DBG] pgmap v1909: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T15:10:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[87147]: cluster 2024-09-13T15:10:33.038268+0000 mgr.y (mgr.26252) 1334 : cluster [DBG] pgmap v1909: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T15:10:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[87147]: cluster 2024-09-13T15:10:33.096843+0000 mon.a (mon.0) 2502 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:10:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[87147]: cluster 2024-09-13T15:10:33.096843+0000 mon.a (mon.0) 2502 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:10:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[87147]: audit 2024-09-13T15:10:33.106775+0000 mon.a (mon.0) 2503 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:10:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[87147]: audit 2024-09-13T15:10:33.106775+0000 mon.a (mon.0) 2503 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:10:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[87147]: cluster 2024-09-13T15:10:33.111807+0000 mon.a (mon.0) 2504 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2024-09-13T15:10:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[87147]: cluster 2024-09-13T15:10:33.111807+0000 mon.a (mon.0) 2504 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2024-09-13T15:10:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[87147]: audit 2024-09-13T15:10:33.307399+0000 mon.a (mon.0) 2505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:34.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:34 smithi099 bash[87147]: audit 2024-09-13T15:10:33.307399+0000 mon.a (mon.0) 2505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:34 smithi177 bash[68594]: cluster 2024-09-13T15:10:33.038268+0000 mgr.y (mgr.26252) 1334 : cluster [DBG] pgmap v1909: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T15:10:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:34 smithi177 bash[68594]: cluster 2024-09-13T15:10:33.038268+0000 mgr.y (mgr.26252) 1334 : cluster [DBG] pgmap v1909: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T15:10:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:34 smithi177 bash[68594]: cluster 2024-09-13T15:10:33.096843+0000 mon.a (mon.0) 2502 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:10:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:34 smithi177 bash[68594]: cluster 2024-09-13T15:10:33.096843+0000 mon.a (mon.0) 2502 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:10:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:34 smithi177 bash[68594]: audit 2024-09-13T15:10:33.106775+0000 mon.a (mon.0) 2503 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:10:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:34 smithi177 bash[68594]: audit 2024-09-13T15:10:33.106775+0000 mon.a (mon.0) 2503 : audit [INF] from='client.? 172.21.15.99:0/1277665760' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:10:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:34 smithi177 bash[68594]: cluster 2024-09-13T15:10:33.111807+0000 mon.a (mon.0) 2504 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2024-09-13T15:10:34.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:34 smithi177 bash[68594]: cluster 2024-09-13T15:10:33.111807+0000 mon.a (mon.0) 2504 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2024-09-13T15:10:34.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:34 smithi177 bash[68594]: audit 2024-09-13T15:10:33.307399+0000 mon.a (mon.0) 2505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:34.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:34 smithi177 bash[68594]: audit 2024-09-13T15:10:33.307399+0000 mon.a (mon.0) 2505 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:34.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:34.432 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:34.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:34.542 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:34.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:34.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:34.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:34.698 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:34.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:34.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:34.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:34.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:34.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:34.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:34.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:34.962 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:34.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:34.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:35.065 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:35.097 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:35.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:35.200 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 49 16 67147 67131 85.6161 65.8125 0.00269172 0.0116659 2024-09-13T15:10:35.203 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:35.223 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:35.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:35.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:35.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:35.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:35.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:35.455 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:35.480 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:35.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:35.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:35.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:35.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:35.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:35.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:35.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:35.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:35.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:35.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:35.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:35.977 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:36.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:36.009 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:36.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:36.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:36.138 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:36.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:36.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:36.200 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 50 16 68170 68154 85.1824 63.9375 0.00811402 0.0117338 2024-09-13T15:10:36.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:36.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:36.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:36.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:36 smithi099 bash[84576]: cluster 2024-09-13T15:10:35.038905+0000 mgr.y (mgr.26252) 1335 : cluster [DBG] pgmap v1911: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-13T15:10:36.321 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:36 smithi099 bash[84576]: cluster 2024-09-13T15:10:35.038905+0000 mgr.y (mgr.26252) 1335 : cluster [DBG] pgmap v1911: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-13T15:10:36.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:36 smithi099 bash[87147]: cluster 2024-09-13T15:10:35.038905+0000 mgr.y (mgr.26252) 1335 : cluster [DBG] pgmap v1911: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-13T15:10:36.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:36 smithi099 bash[87147]: cluster 2024-09-13T15:10:35.038905+0000 mgr.y (mgr.26252) 1335 : cluster [DBG] pgmap v1911: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-13T15:10:36.376 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:36.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:36.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:36.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:36.536 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:36.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:36.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:36.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:36 smithi177 bash[68594]: cluster 2024-09-13T15:10:35.038905+0000 mgr.y (mgr.26252) 1335 : cluster [DBG] pgmap v1911: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-13T15:10:36.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:36 smithi177 bash[68594]: cluster 2024-09-13T15:10:35.038905+0000 mgr.y (mgr.26252) 1335 : cluster [DBG] pgmap v1911: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-13T15:10:36.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:36.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:36.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:36.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:36.760 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:36.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:36.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:36.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:36.887 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:36.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:36.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:36.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:37.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:37.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:37.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:37.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:37.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:37.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:37.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:37.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:37.200 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 51 16 69380 69364 84.9949 75.625 0.015197 0.0117605 2024-09-13T15:10:37.284 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:37.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:37.311 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:10:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:10:37] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:10:37.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:37.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:37.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 reset_purged_snaps_last 2024-09-13T15:10:37.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:37.440 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:37.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:37.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:37.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:37.582 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:37.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:37.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:37.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:37.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:37.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:37.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:37.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:37.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:37 smithi099 bash[84576]: cluster 2024-09-13T15:10:37.039505+0000 mgr.y (mgr.26252) 1336 : cluster [DBG] pgmap v1912: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2024-09-13T15:10:37.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:37 smithi099 bash[84576]: cluster 2024-09-13T15:10:37.039505+0000 mgr.y (mgr.26252) 1336 : cluster [DBG] pgmap v1912: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2024-09-13T15:10:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:37 smithi099 bash[87147]: cluster 2024-09-13T15:10:37.039505+0000 mgr.y (mgr.26252) 1336 : cluster [DBG] pgmap v1912: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2024-09-13T15:10:37.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:37 smithi099 bash[87147]: cluster 2024-09-13T15:10:37.039505+0000 mgr.y (mgr.26252) 1336 : cluster [DBG] pgmap v1912: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2024-09-13T15:10:37.824 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:37 smithi177 bash[68594]: cluster 2024-09-13T15:10:37.039505+0000 mgr.y (mgr.26252) 1336 : cluster [DBG] pgmap v1912: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2024-09-13T15:10:37.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:37 smithi177 bash[68594]: cluster 2024-09-13T15:10:37.039505+0000 mgr.y (mgr.26252) 1336 : cluster [DBG] pgmap v1912: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2024-09-13T15:10:37.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:37.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:37.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:37.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:37.978 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:37.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:37.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:38.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:38.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:38.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:38.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:38.200 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 52 15 70552 70537 84.7701 73.3125 0.0113831 0.0117915 2024-09-13T15:10:38.215 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:38.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:38.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:38.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:38.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:38.379 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:38.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:38.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:38.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:38.518 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:38.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:38.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:38.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:38.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:38.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:38.756 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:38.789 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:38.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:38.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:38.896 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:38.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:38.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:38.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:39.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:39.055 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:39.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:39.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:39.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:39.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:39.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:39.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:39.200 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 53 16 71771 71755 84.6069 76.125 0.00752351 0.0118141 2024-09-13T15:10:39.294 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:39.314 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:39.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:39.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:10:39.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:10:39.418 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:39.494 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:39.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:39.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:39.599 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:39.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:39.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:39.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:39.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:39.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:39.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:39.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:39.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:39.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:39.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:39.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:39.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:40.030 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:40.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:40.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:40.134 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:40.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:40.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:40.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:40.200 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 54 16 72872 72856 84.3143 68.8125 0.0108987 0.0118538 2024-09-13T15:10:40.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:40 smithi177 bash[68594]: cluster 2024-09-13T15:10:39.040118+0000 mgr.y (mgr.26252) 1337 : cluster [DBG] pgmap v1913: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T15:10:40.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:40 smithi177 bash[68594]: cluster 2024-09-13T15:10:39.040118+0000 mgr.y (mgr.26252) 1337 : cluster [DBG] pgmap v1913: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T15:10:40.270 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:40.295 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:40.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:40.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:40.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:10:40 smithi177 bash[61439]: ts=2024-09-13T15:10:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:10:40.399 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:40.412 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:40 smithi099 bash[84576]: cluster 2024-09-13T15:10:39.040118+0000 mgr.y (mgr.26252) 1337 : cluster [DBG] pgmap v1913: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T15:10:40.412 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:40 smithi099 bash[84576]: cluster 2024-09-13T15:10:39.040118+0000 mgr.y (mgr.26252) 1337 : cluster [DBG] pgmap v1913: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T15:10:40.413 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:40 smithi099 bash[87147]: cluster 2024-09-13T15:10:39.040118+0000 mgr.y (mgr.26252) 1337 : cluster [DBG] pgmap v1913: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T15:10:40.413 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:40 smithi099 bash[87147]: cluster 2024-09-13T15:10:39.040118+0000 mgr.y (mgr.26252) 1337 : cluster [DBG] pgmap v1913: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T15:10:40.429 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:40.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:40.532 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:40.557 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:40.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:40.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:40.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:40.687 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:40.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:40.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:40.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:40.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:40.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:40.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:40.934 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:40.937 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:40.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:40.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:41.040 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:41.065 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:41.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:41.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:41.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:41.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 55 16 74050 74034 84.1198 73.625 0.022129 0.0118823 2024-09-13T15:10:41.201 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:41.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:41.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:41.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:41.332 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:41.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:41.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:41.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:41.474 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:41.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:41.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:41.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:41.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:41.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:41.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:41.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:41.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:41.760 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:41.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:41.861 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:41.888 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:41.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:41.957 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:41.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:42.029 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:42.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:42.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:42.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:42.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:42.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:42.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:42.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 56 16 75054 75038 83.7378 62.75 0.020677 0.0119361 2024-09-13T15:10:42.279 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:42.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:42 smithi177 bash[68594]: cluster 2024-09-13T15:10:41.040742+0000 mgr.y (mgr.26252) 1338 : cluster [DBG] pgmap v1914: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:10:42.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:42 smithi177 bash[68594]: cluster 2024-09-13T15:10:41.040742+0000 mgr.y (mgr.26252) 1338 : cluster [DBG] pgmap v1914: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:10:42.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:42 smithi177 bash[68594]: audit 2024-09-13T15:10:41.504642+0000 mon.a (mon.0) 2506 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:10:42.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:42 smithi177 bash[68594]: audit 2024-09-13T15:10:41.504642+0000 mon.a (mon.0) 2506 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:10:42.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:42.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:42.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:42.407 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:42.418 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:42 smithi099 bash[84576]: cluster 2024-09-13T15:10:41.040742+0000 mgr.y (mgr.26252) 1338 : cluster [DBG] pgmap v1914: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:10:42.418 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:42 smithi099 bash[84576]: cluster 2024-09-13T15:10:41.040742+0000 mgr.y (mgr.26252) 1338 : cluster [DBG] pgmap v1914: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:10:42.418 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:42 smithi099 bash[84576]: audit 2024-09-13T15:10:41.504642+0000 mon.a (mon.0) 2506 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:10:42.419 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:42 smithi099 bash[84576]: audit 2024-09-13T15:10:41.504642+0000 mon.a (mon.0) 2506 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:10:42.419 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:42 smithi099 bash[87147]: cluster 2024-09-13T15:10:41.040742+0000 mgr.y (mgr.26252) 1338 : cluster [DBG] pgmap v1914: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:10:42.419 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:42 smithi099 bash[87147]: cluster 2024-09-13T15:10:41.040742+0000 mgr.y (mgr.26252) 1338 : cluster [DBG] pgmap v1914: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:10:42.419 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:42 smithi099 bash[87147]: audit 2024-09-13T15:10:41.504642+0000 mon.a (mon.0) 2506 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:10:42.419 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:42 smithi099 bash[87147]: audit 2024-09-13T15:10:41.504642+0000 mon.a (mon.0) 2506 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:10:42.435 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:42.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:42.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:42.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:42.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:42.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:42.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:42.691 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:42.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:42.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:42.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:42.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:42.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:42.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:42.931 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:42.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:42.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:42.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:43.051 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:43.084 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:43.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:43.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:43.187 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:43.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 57 16 76021 76005 83.3289 60.4375 0.00611877 0.0119919 2024-09-13T15:10:43.204 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:43.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:43.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:43.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:43.341 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:43.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:43.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:43.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:43.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:43.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:43.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:43.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:43.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:43.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:43.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:43.707 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:43.731 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:43.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:43.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:43.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:43.852 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:43.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:43.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:43.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:43.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:43.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:43.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:43.998 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:43.998 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:44.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:44.104 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:44.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:44.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:44.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 58 16 76936 76920 82.8781 57.1875 0.0122223 0.0120609 2024-09-13T15:10:44.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:44.233 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:44.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:44.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:44.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:44.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:44 smithi177 bash[68594]: cluster 2024-09-13T15:10:43.041483+0000 mgr.y (mgr.26252) 1339 : cluster [DBG] pgmap v1915: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T15:10:44.345 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:44 smithi177 bash[68594]: cluster 2024-09-13T15:10:43.041483+0000 mgr.y (mgr.26252) 1339 : cluster [DBG] pgmap v1915: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T15:10:44.366 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:44.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:44.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:44.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:44 smithi099 bash[84576]: cluster 2024-09-13T15:10:43.041483+0000 mgr.y (mgr.26252) 1339 : cluster [DBG] pgmap v1915: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T15:10:44.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:44 smithi099 bash[84576]: cluster 2024-09-13T15:10:43.041483+0000 mgr.y (mgr.26252) 1339 : cluster [DBG] pgmap v1915: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T15:10:44.481 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:44 smithi099 bash[87147]: cluster 2024-09-13T15:10:43.041483+0000 mgr.y (mgr.26252) 1339 : cluster [DBG] pgmap v1915: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T15:10:44.481 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:44 smithi099 bash[87147]: cluster 2024-09-13T15:10:43.041483+0000 mgr.y (mgr.26252) 1339 : cluster [DBG] pgmap v1915: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-13T15:10:44.506 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:44.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:44.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:44.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:44.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:44.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:44.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:44.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:44.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:44.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:44.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:44.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:44.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:44.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:44.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:44.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:45.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:45.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:45.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:45.131 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:45.159 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:45.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:45.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:45.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 59 16 78027 78011 82.629 68.1875 0.0140442 0.0120974 2024-09-13T15:10:45.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:45.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:45.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:45.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:45.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:45.442 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:45.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:45.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:45.564 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:45.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:45.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:45.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:45.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:45.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:45.791 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:45.816 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:45.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:45.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:45.919 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:45.945 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:45.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:45.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:46.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:46.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:46.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:46.171 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:46.187 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:46 smithi177 bash[68594]: cluster 2024-09-13T15:10:45.042103+0000 mgr.y (mgr.26252) 1340 : cluster [DBG] pgmap v1916: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T15:10:46.187 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:46 smithi177 bash[68594]: cluster 2024-09-13T15:10:45.042103+0000 mgr.y (mgr.26252) 1340 : cluster [DBG] pgmap v1916: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T15:10:46.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:10:46.202251+0000 min lat: 0.00117806 max lat: 0.663272 avg lat: 0.0122152 2024-09-13T15:10:46.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:10:46.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 60 16 78580 78564 81.8279 34.5625 0.0197924 0.0122152 2024-09-13T15:10:46.207 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:46.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:46.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:46.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:46.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:46.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:46.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:46.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:46.470 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:46.470 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:46 smithi099 bash[87147]: cluster 2024-09-13T15:10:45.042103+0000 mgr.y (mgr.26252) 1340 : cluster [DBG] pgmap v1916: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T15:10:46.470 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:46 smithi099 bash[87147]: cluster 2024-09-13T15:10:45.042103+0000 mgr.y (mgr.26252) 1340 : cluster [DBG] pgmap v1916: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T15:10:46.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:46 smithi099 bash[84576]: cluster 2024-09-13T15:10:45.042103+0000 mgr.y (mgr.26252) 1340 : cluster [DBG] pgmap v1916: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T15:10:46.470 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:46 smithi099 bash[84576]: cluster 2024-09-13T15:10:45.042103+0000 mgr.y (mgr.26252) 1340 : cluster [DBG] pgmap v1916: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T15:10:46.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:46.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:46.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:46.604 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:46.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:46.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:46.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:46.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:46.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:46.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:46.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:46.863 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:46.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:46.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:46.970 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:47.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:47.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:47.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:47.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:47.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:47.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:47.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 61 16 79604 79588 81.5355 64 0.0217857 0.0122593 2024-09-13T15:10:47.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:47.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:47.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:47.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:47.323 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:10:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:10:47] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:10:47.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:47.420 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:47.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:47.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:47.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:47.548 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:47.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:47.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:47.658 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:47.690 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:47.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:47.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:47.792 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:47.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:47.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:47.913 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:47.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:47.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:48.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:48.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:48.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:48.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:48.165 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:48.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 62 16 80358 80342 80.9805 47.125 0.0240839 0.0123425 2024-09-13T15:10:48.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:48.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:48.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:48.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:48.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:48 smithi177 bash[68594]: cluster 2024-09-13T15:10:47.044390+0000 mgr.y (mgr.26252) 1341 : cluster [DBG] pgmap v1917: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-09-13T15:10:48.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:48 smithi177 bash[68594]: cluster 2024-09-13T15:10:47.044390+0000 mgr.y (mgr.26252) 1341 : cluster [DBG] pgmap v1917: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-09-13T15:10:48.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:48 smithi177 bash[68594]: audit 2024-09-13T15:10:47.710754+0000 mon.a (mon.0) 2507 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:48 smithi177 bash[68594]: audit 2024-09-13T15:10:47.710754+0000 mon.a (mon.0) 2507 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:48 smithi177 bash[68594]: audit 2024-09-13T15:10:47.718019+0000 mon.a (mon.0) 2508 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:48 smithi177 bash[68594]: audit 2024-09-13T15:10:47.718019+0000 mon.a (mon.0) 2508 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:48 smithi177 bash[68594]: audit 2024-09-13T15:10:48.032846+0000 mon.a (mon.0) 2509 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:48 smithi177 bash[68594]: audit 2024-09-13T15:10:48.032846+0000 mon.a (mon.0) 2509 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:48 smithi177 bash[68594]: audit 2024-09-13T15:10:48.039754+0000 mon.a (mon.0) 2510 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.316 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:48 smithi177 bash[68594]: audit 2024-09-13T15:10:48.039754+0000 mon.a (mon.0) 2510 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:48.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:48.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:48.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[84576]: cluster 2024-09-13T15:10:47.044390+0000 mgr.y (mgr.26252) 1341 : cluster [DBG] pgmap v1917: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-09-13T15:10:48.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[84576]: cluster 2024-09-13T15:10:47.044390+0000 mgr.y (mgr.26252) 1341 : cluster [DBG] pgmap v1917: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-09-13T15:10:48.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[84576]: audit 2024-09-13T15:10:47.710754+0000 mon.a (mon.0) 2507 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.405 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[84576]: audit 2024-09-13T15:10:47.710754+0000 mon.a (mon.0) 2507 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.405 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[84576]: audit 2024-09-13T15:10:47.718019+0000 mon.a (mon.0) 2508 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.405 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[84576]: audit 2024-09-13T15:10:47.718019+0000 mon.a (mon.0) 2508 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.405 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[84576]: audit 2024-09-13T15:10:48.032846+0000 mon.a (mon.0) 2509 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.405 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[84576]: audit 2024-09-13T15:10:48.032846+0000 mon.a (mon.0) 2509 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.405 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[84576]: audit 2024-09-13T15:10:48.039754+0000 mon.a (mon.0) 2510 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.405 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[84576]: audit 2024-09-13T15:10:48.039754+0000 mon.a (mon.0) 2510 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[87147]: cluster 2024-09-13T15:10:47.044390+0000 mgr.y (mgr.26252) 1341 : cluster [DBG] pgmap v1917: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-09-13T15:10:48.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[87147]: cluster 2024-09-13T15:10:47.044390+0000 mgr.y (mgr.26252) 1341 : cluster [DBG] pgmap v1917: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-09-13T15:10:48.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[87147]: audit 2024-09-13T15:10:47.710754+0000 mon.a (mon.0) 2507 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[87147]: audit 2024-09-13T15:10:47.710754+0000 mon.a (mon.0) 2507 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[87147]: audit 2024-09-13T15:10:47.718019+0000 mon.a (mon.0) 2508 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[87147]: audit 2024-09-13T15:10:47.718019+0000 mon.a (mon.0) 2508 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[87147]: audit 2024-09-13T15:10:48.032846+0000 mon.a (mon.0) 2509 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[87147]: audit 2024-09-13T15:10:48.032846+0000 mon.a (mon.0) 2509 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[87147]: audit 2024-09-13T15:10:48.039754+0000 mon.a (mon.0) 2510 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.406 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:48 smithi099 bash[87147]: audit 2024-09-13T15:10:48.039754+0000 mon.a (mon.0) 2510 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:48.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:48.470 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:48.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:48.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:48.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:48.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:48.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:48.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:48.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:48.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:48.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:48.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:48.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:48.872 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:48.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:48.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:48.975 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:48.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:48.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:48.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:49.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:49.130 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:49.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:49.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:49.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 63 16 80780 80764 80.1137 26.375 0.00721057 0.0124771 2024-09-13T15:10:49.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:49.266 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:49.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:49.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:49.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:49 smithi177 bash[68594]: audit 2024-09-13T15:10:48.307704+0000 mon.a (mon.0) 2511 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:49.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:49 smithi177 bash[68594]: audit 2024-09-13T15:10:48.307704+0000 mon.a (mon.0) 2511 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:49.369 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:49.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:49.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:49.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:49.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:49.528 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:49 smithi099 bash[84576]: audit 2024-09-13T15:10:48.307704+0000 mon.a (mon.0) 2511 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:49.528 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:49 smithi099 bash[84576]: audit 2024-09-13T15:10:48.307704+0000 mon.a (mon.0) 2511 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:49.529 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:49 smithi099 bash[87147]: audit 2024-09-13T15:10:48.307704+0000 mon.a (mon.0) 2511 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:49.529 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:49 smithi099 bash[87147]: audit 2024-09-13T15:10:48.307704+0000 mon.a (mon.0) 2511 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:49.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:49.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:49.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:49.647 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:49.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:49.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:49.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:49.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:49.822 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:49.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:49.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:49.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:49.955 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:49.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:50.065 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:50.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:50.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:50.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:50.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:50.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 64 16 81839 81823 79.896 66.1875 0.00810547 0.0125113 2024-09-13T15:10:50.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[84576]: cluster 2024-09-13T15:10:49.044851+0000 mgr.y (mgr.26252) 1342 : cluster [DBG] pgmap v1918: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T15:10:50.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[84576]: cluster 2024-09-13T15:10:49.044851+0000 mgr.y (mgr.26252) 1342 : cluster [DBG] pgmap v1918: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T15:10:50.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[84576]: audit 2024-09-13T15:10:49.416543+0000 mon.a (mon.0) 2512 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[84576]: audit 2024-09-13T15:10:49.416543+0000 mon.a (mon.0) 2512 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[84576]: audit 2024-09-13T15:10:49.423510+0000 mon.a (mon.0) 2513 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[84576]: audit 2024-09-13T15:10:49.423510+0000 mon.a (mon.0) 2513 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[87147]: cluster 2024-09-13T15:10:49.044851+0000 mgr.y (mgr.26252) 1342 : cluster [DBG] pgmap v1918: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T15:10:50.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[87147]: cluster 2024-09-13T15:10:49.044851+0000 mgr.y (mgr.26252) 1342 : cluster [DBG] pgmap v1918: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T15:10:50.209 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[87147]: audit 2024-09-13T15:10:49.416543+0000 mon.a (mon.0) 2512 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.209 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[87147]: audit 2024-09-13T15:10:49.416543+0000 mon.a (mon.0) 2512 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.209 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[87147]: audit 2024-09-13T15:10:49.423510+0000 mon.a (mon.0) 2513 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.209 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:50 smithi099 bash[87147]: audit 2024-09-13T15:10:49.423510+0000 mon.a (mon.0) 2513 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:50.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:50.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:50.323 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:50.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:50.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:50.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:50.463 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:50 smithi177 bash[68594]: cluster 2024-09-13T15:10:49.044851+0000 mgr.y (mgr.26252) 1342 : cluster [DBG] pgmap v1918: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T15:10:50.463 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:50 smithi177 bash[68594]: cluster 2024-09-13T15:10:49.044851+0000 mgr.y (mgr.26252) 1342 : cluster [DBG] pgmap v1918: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-13T15:10:50.463 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:50 smithi177 bash[68594]: audit 2024-09-13T15:10:49.416543+0000 mon.a (mon.0) 2512 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.463 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:50 smithi177 bash[68594]: audit 2024-09-13T15:10:49.416543+0000 mon.a (mon.0) 2512 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.463 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:50 smithi177 bash[68594]: audit 2024-09-13T15:10:49.423510+0000 mon.a (mon.0) 2513 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.463 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:50 smithi177 bash[68594]: audit 2024-09-13T15:10:49.423510+0000 mon.a (mon.0) 2513 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:10:50.463 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:10:50 smithi177 bash[61439]: ts=2024-09-13T15:10:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:10:50.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:50.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:50.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:50.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:50.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:50.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:50.731 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:50.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:50.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:50.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:50.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:50.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:50.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:51.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:51.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:51.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:51.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:51.139 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:51.164 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:51.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:51.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:51.201 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 65 16 82905 82889 79.6918 66.625 0.0102726 0.0125433 2024-09-13T15:10:51.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:51.291 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:51.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:51.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:51.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:51.423 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:51.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:51.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:51.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:51.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:51.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:51.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:51.665 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:51.691 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:51.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:51.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:51.797 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:51.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:51.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:51.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:51.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:51.962 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:51.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:51.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:52.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:52.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:52.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:52.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:52.202 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 66 16 84009 83993 79.5297 69 0.0189866 0.0125674 2024-09-13T15:10:52.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:52.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:52 smithi177 bash[68594]: cluster 2024-09-13T15:10:51.045511+0000 mgr.y (mgr.26252) 1343 : cluster [DBG] pgmap v1919: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:52.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:52 smithi177 bash[68594]: cluster 2024-09-13T15:10:51.045511+0000 mgr.y (mgr.26252) 1343 : cluster [DBG] pgmap v1919: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:52.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:52.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:52.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:52.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:52 smithi099 bash[84576]: cluster 2024-09-13T15:10:51.045511+0000 mgr.y (mgr.26252) 1343 : cluster [DBG] pgmap v1919: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:52.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:52 smithi099 bash[84576]: cluster 2024-09-13T15:10:51.045511+0000 mgr.y (mgr.26252) 1343 : cluster [DBG] pgmap v1919: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:52.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:52 smithi099 bash[87147]: cluster 2024-09-13T15:10:51.045511+0000 mgr.y (mgr.26252) 1343 : cluster [DBG] pgmap v1919: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:52.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:52 smithi099 bash[87147]: cluster 2024-09-13T15:10:51.045511+0000 mgr.y (mgr.26252) 1343 : cluster [DBG] pgmap v1919: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:10:52.340 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:52.372 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:52.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:52.476 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:52.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:52.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:52.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:52.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:52.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:52.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:52.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:52.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:52.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:52.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:52.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:52.865 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:52.899 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:52.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:53.005 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:53.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:53.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:53.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:53.134 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:53.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:53.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:53.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:53.202 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 67 16 84795 84779 79.0758 49.125 0.0113974 0.0126411 2024-09-13T15:10:53.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:53.288 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:53.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:53.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:53.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:53.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:53.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:53.516 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:53.536 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:53.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:53.639 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:53.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:53.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:53.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:53.822 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:53.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:53.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:53.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:53.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:53.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:53.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:54.051 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:54.085 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:54.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:54.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:54.192 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:54.202 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 68 16 85326 85310 78.401 33.1875 0.00822689 0.0127496 2024-09-13T15:10:54.220 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:54.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:54.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:54.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:54 smithi099 bash[87147]: cluster 2024-09-13T15:10:53.046084+0000 mgr.y (mgr.26252) 1344 : cluster [DBG] pgmap v1920: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2024-09-13T15:10:54.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:54 smithi099 bash[87147]: cluster 2024-09-13T15:10:53.046084+0000 mgr.y (mgr.26252) 1344 : cluster [DBG] pgmap v1920: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2024-09-13T15:10:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:54 smithi099 bash[84576]: cluster 2024-09-13T15:10:53.046084+0000 mgr.y (mgr.26252) 1344 : cluster [DBG] pgmap v1920: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2024-09-13T15:10:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:54 smithi099 bash[84576]: cluster 2024-09-13T15:10:53.046084+0000 mgr.y (mgr.26252) 1344 : cluster [DBG] pgmap v1920: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2024-09-13T15:10:54.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:54.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:54.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:54.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:54.455 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:54.476 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:54.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:54.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:54.579 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:54.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:54 smithi177 bash[68594]: cluster 2024-09-13T15:10:53.046084+0000 mgr.y (mgr.26252) 1344 : cluster [DBG] pgmap v1920: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2024-09-13T15:10:54.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:54 smithi177 bash[68594]: cluster 2024-09-13T15:10:53.046084+0000 mgr.y (mgr.26252) 1344 : cluster [DBG] pgmap v1920: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2024-09-13T15:10:54.600 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:54.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:54.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:54.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:54.724 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:54.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:54.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:54.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:54.855 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:54.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:54.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:54.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:54.994 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:54.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:55.099 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:55.125 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:55.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:55.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:55.202 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 69 16 85961 85945 77.8399 39.6875 0.0044103 0.0128411 2024-09-13T15:10:55.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:55.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:55.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:55.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:55.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:55.390 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:55.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:55.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:55.493 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:55.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:55.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:55.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:55.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:55.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:55.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:55.756 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:55.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:55.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:55.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:55.890 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:55.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:55.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:55.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:56.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:56.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:56.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:56.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:56.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:56.170 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:56 smithi099 bash[84576]: cluster 2024-09-13T15:10:55.047469+0000 mgr.y (mgr.26252) 1345 : cluster [DBG] pgmap v1921: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:56.170 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:56 smithi099 bash[84576]: cluster 2024-09-13T15:10:55.047469+0000 mgr.y (mgr.26252) 1345 : cluster [DBG] pgmap v1921: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:56.170 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:56 smithi099 bash[87147]: cluster 2024-09-13T15:10:55.047469+0000 mgr.y (mgr.26252) 1345 : cluster [DBG] pgmap v1921: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:56.170 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:56 smithi099 bash[87147]: cluster 2024-09-13T15:10:55.047469+0000 mgr.y (mgr.26252) 1345 : cluster [DBG] pgmap v1921: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:56.180 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:56.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:56.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:56.202 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 70 16 86834 86818 77.5072 54.5625 0.00398478 0.0128877 2024-09-13T15:10:56.283 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:56.304 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:56.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:56 smithi177 bash[68594]: cluster 2024-09-13T15:10:55.047469+0000 mgr.y (mgr.26252) 1345 : cluster [DBG] pgmap v1921: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:56 smithi177 bash[68594]: cluster 2024-09-13T15:10:55.047469+0000 mgr.y (mgr.26252) 1345 : cluster [DBG] pgmap v1921: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T15:10:56.407 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:56.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:56.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:56.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:56.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:56.575 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:56.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:56.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:56.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:56.703 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:56.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:56.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:56.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:56.826 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:56.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:56.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:56.929 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:56.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:56.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:56.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:57.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:57.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:57.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:57.194 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:57.202 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 71 16 87471 87455 76.9763 39.8125 0.00716436 0.0129844 2024-09-13T15:10:57.212 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:10:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:10:57] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-09-13T15:10:57.225 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:57.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:57.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:57.330 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:57.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:57.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:57.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:57.460 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:57.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:57.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:57.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:57.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:57.610 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:10:57.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:57.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:10:57.713 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:57.738 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:57.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:57.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:57.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:57.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:10:57.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:57.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:10:57.970 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:58.005 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:58.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:58.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:58.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:58.143 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:10:58.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:58.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:10:58.202 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 72 16 88041 88025 76.4019 35.625 0.0121527 0.0130827 2024-09-13T15:10:58.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:58.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:10:58.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:58.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:10:58.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:58.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:58.415 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:58 smithi177 bash[68594]: cluster 2024-09-13T15:10:57.047901+0000 mgr.y (mgr.26252) 1346 : cluster [DBG] pgmap v1922: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:10:58.415 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:10:58 smithi177 bash[68594]: cluster 2024-09-13T15:10:57.047901+0000 mgr.y (mgr.26252) 1346 : cluster [DBG] pgmap v1922: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:10:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:58.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:58.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:58.526 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:58 smithi099 bash[84576]: cluster 2024-09-13T15:10:57.047901+0000 mgr.y (mgr.26252) 1346 : cluster [DBG] pgmap v1922: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:10:58.526 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:10:58 smithi099 bash[84576]: cluster 2024-09-13T15:10:57.047901+0000 mgr.y (mgr.26252) 1346 : cluster [DBG] pgmap v1922: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:10:58.527 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:58 smithi099 bash[87147]: cluster 2024-09-13T15:10:57.047901+0000 mgr.y (mgr.26252) 1346 : cluster [DBG] pgmap v1922: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:10:58.527 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:10:58 smithi099 bash[87147]: cluster 2024-09-13T15:10:57.047901+0000 mgr.y (mgr.26252) 1346 : cluster [DBG] pgmap v1922: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:10:58.540 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:58.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:58.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:58.643 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:58.670 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:10:58.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:58.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:10:58.774 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:58.800 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:58.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:58.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:10:58.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:58.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:59.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:59.059 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:59.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:59.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:59.166 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:59.202 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 73 16 89078 89062 76.2431 64.8125 0.00765339 0.0131111 2024-09-13T15:10:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:59.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:59.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:59.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:59.332 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:59.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:59.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:59.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:59.460 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:59.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:59.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:59.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:59.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:59.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:59.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:59.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:59.714 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:10:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:59.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:10:59.819 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:59.850 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:10:59.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:59.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:10:59.959 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:59.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:10:59.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:10:59.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:00.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:00.109 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:00.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:00.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:00.202 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 74 16 89764 89748 75.7921 42.875 0.0116066 0.013189 2024-09-13T15:11:00.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:00.238 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:00.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:00.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:00.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:00.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:00 smithi177 bash[68594]: cluster 2024-09-13T15:10:59.049693+0000 mgr.y (mgr.26252) 1347 : cluster [DBG] pgmap v1923: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2024-09-13T15:11:00.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:00 smithi177 bash[68594]: cluster 2024-09-13T15:10:59.049693+0000 mgr.y (mgr.26252) 1347 : cluster [DBG] pgmap v1923: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2024-09-13T15:11:00.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:11:00 smithi177 bash[61439]: ts=2024-09-13T15:11:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:11:00.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:00.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:00.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:00.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:00.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:00.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:00.511 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:00 smithi099 bash[87147]: cluster 2024-09-13T15:10:59.049693+0000 mgr.y (mgr.26252) 1347 : cluster [DBG] pgmap v1923: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2024-09-13T15:11:00.511 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:00 smithi099 bash[87147]: cluster 2024-09-13T15:10:59.049693+0000 mgr.y (mgr.26252) 1347 : cluster [DBG] pgmap v1923: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2024-09-13T15:11:00.511 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:00 smithi099 bash[84576]: cluster 2024-09-13T15:10:59.049693+0000 mgr.y (mgr.26252) 1347 : cluster [DBG] pgmap v1923: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2024-09-13T15:11:00.511 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:00 smithi099 bash[84576]: cluster 2024-09-13T15:10:59.049693+0000 mgr.y (mgr.26252) 1347 : cluster [DBG] pgmap v1923: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2024-09-13T15:11:00.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:00.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:00.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:00.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:00.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:00.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:00.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:00.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:00.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:00.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:00.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:01.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:01.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:01.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:01.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:01.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:01.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:01.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:01.204 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 75 16 90337 90321 75.2572 35.8125 0.0251312 0.0132828 2024-09-13T15:11:01.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:01.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:01.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:01.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:01.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:01.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:01.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:01.532 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:01.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:01.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:01.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:01.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:01.701 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:01.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:01.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:01.807 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:01.835 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:01.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:01.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:01.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:01.966 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:01.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:01.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:02.070 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:02.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:02.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:02.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:02.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:02.204 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 76 16 91286 91270 75.0473 59.3125 0.0112196 0.0133198 2024-09-13T15:11:02.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:02.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:02.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:02.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:02.326 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:02 smithi177 bash[68594]: cluster 2024-09-13T15:11:01.050343+0000 mgr.y (mgr.26252) 1348 : cluster [DBG] pgmap v1924: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-09-13T15:11:02.327 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:02 smithi177 bash[68594]: cluster 2024-09-13T15:11:01.050343+0000 mgr.y (mgr.26252) 1348 : cluster [DBG] pgmap v1924: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-09-13T15:11:02.341 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:02.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:02.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:02.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:02.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:02.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:02.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:02.514 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:02 smithi099 bash[84576]: cluster 2024-09-13T15:11:01.050343+0000 mgr.y (mgr.26252) 1348 : cluster [DBG] pgmap v1924: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-09-13T15:11:02.514 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:02 smithi099 bash[84576]: cluster 2024-09-13T15:11:01.050343+0000 mgr.y (mgr.26252) 1348 : cluster [DBG] pgmap v1924: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-09-13T15:11:02.514 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:02 smithi099 bash[87147]: cluster 2024-09-13T15:11:01.050343+0000 mgr.y (mgr.26252) 1348 : cluster [DBG] pgmap v1924: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-09-13T15:11:02.514 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:02 smithi099 bash[87147]: cluster 2024-09-13T15:11:01.050343+0000 mgr.y (mgr.26252) 1348 : cluster [DBG] pgmap v1924: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2024-09-13T15:11:02.582 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:02.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:02.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:02.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:02.725 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:02.750 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:02.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:02.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:02.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:02.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:02.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:02.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:02.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:02.997 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:02.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:03.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:03.100 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:03.120 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:03.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:03.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:03.205 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 77 16 92022 92006 74.6701 46 0.0179351 0.013387 2024-09-13T15:11:03.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:03.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:03.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:03.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:03.356 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:03.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:03.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:03.482 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:03.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:03.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:03.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:03.610 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:03.636 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:03.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:03.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:03.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:03.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:03.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:03.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:03.883 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:03.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:03.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:03.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:04.016 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:04.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:04.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:04.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:04.175 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:04.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:04.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 78 16 93001 92985 74.4961 61.1875 0.0101681 0.0134175 2024-09-13T15:11:04.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:04.301 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:04.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:04.407 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:04.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:04 smithi099 bash[84576]: cluster 2024-09-13T15:11:03.051146+0000 mgr.y (mgr.26252) 1349 : cluster [DBG] pgmap v1925: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s 2024-09-13T15:11:04.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:04 smithi099 bash[84576]: cluster 2024-09-13T15:11:03.051146+0000 mgr.y (mgr.26252) 1349 : cluster [DBG] pgmap v1925: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s 2024-09-13T15:11:04.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:04 smithi099 bash[84576]: audit 2024-09-13T15:11:03.308291+0000 mon.a (mon.0) 2514 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:04.418 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:04 smithi099 bash[84576]: audit 2024-09-13T15:11:03.308291+0000 mon.a (mon.0) 2514 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:04.418 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:04 smithi099 bash[87147]: cluster 2024-09-13T15:11:03.051146+0000 mgr.y (mgr.26252) 1349 : cluster [DBG] pgmap v1925: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s 2024-09-13T15:11:04.418 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:04 smithi099 bash[87147]: cluster 2024-09-13T15:11:03.051146+0000 mgr.y (mgr.26252) 1349 : cluster [DBG] pgmap v1925: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s 2024-09-13T15:11:04.418 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:04 smithi099 bash[87147]: audit 2024-09-13T15:11:03.308291+0000 mon.a (mon.0) 2514 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:04.418 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:04 smithi099 bash[87147]: audit 2024-09-13T15:11:03.308291+0000 mon.a (mon.0) 2514 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:04.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:04.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:04.555 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:04.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:04 smithi177 bash[68594]: cluster 2024-09-13T15:11:03.051146+0000 mgr.y (mgr.26252) 1349 : cluster [DBG] pgmap v1925: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s 2024-09-13T15:11:04.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:04 smithi177 bash[68594]: cluster 2024-09-13T15:11:03.051146+0000 mgr.y (mgr.26252) 1349 : cluster [DBG] pgmap v1925: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s 2024-09-13T15:11:04.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:04 smithi177 bash[68594]: audit 2024-09-13T15:11:03.308291+0000 mon.a (mon.0) 2514 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:04.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:04 smithi177 bash[68594]: audit 2024-09-13T15:11:03.308291+0000 mon.a (mon.0) 2514 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:04.585 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:04.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:04.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:04.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:04.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:04.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:04.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:04.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:04.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:04.948 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:04.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:04.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:04.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:05.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:05.094 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:05.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:05.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:05.196 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:05.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 79 16 93854 93838 74.2279 53.3125 0.0157377 0.0134667 2024-09-13T15:11:05.220 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:05.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:05.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:05.323 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:05.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:05.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:05.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:05.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:05.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:05.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:05.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:05.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:05.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:05.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:05.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:05.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:05.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:05.861 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:05.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:05.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:05.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:05.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:06.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:06.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:06.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:06.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:06.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:06.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:06.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:06.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:11:06.207060+0000 min lat: 0.00117806 max lat: 0.663272 avg lat: 0.0134893 2024-09-13T15:11:06.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:11:06.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 80 16 94704 94688 73.964 53.125 0.00544393 0.0134893 2024-09-13T15:11:06.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:06.268 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[84576]: cluster 2024-09-13T15:11:05.051825+0000 mgr.y (mgr.26252) 1350 : cluster [DBG] pgmap v1926: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-09-13T15:11:06.268 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[84576]: cluster 2024-09-13T15:11:05.051825+0000 mgr.y (mgr.26252) 1350 : cluster [DBG] pgmap v1926: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-09-13T15:11:06.268 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[84576]: audit 2024-09-13T15:11:06.015336+0000 mon.a (mon.0) 2515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.268 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[84576]: audit 2024-09-13T15:11:06.015336+0000 mon.a (mon.0) 2515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.268 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[84576]: audit 2024-09-13T15:11:06.026105+0000 mon.a (mon.0) 2516 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.268 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[84576]: audit 2024-09-13T15:11:06.026105+0000 mon.a (mon.0) 2516 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.268 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[87147]: cluster 2024-09-13T15:11:05.051825+0000 mgr.y (mgr.26252) 1350 : cluster [DBG] pgmap v1926: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-09-13T15:11:06.268 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[87147]: cluster 2024-09-13T15:11:05.051825+0000 mgr.y (mgr.26252) 1350 : cluster [DBG] pgmap v1926: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-09-13T15:11:06.268 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[87147]: audit 2024-09-13T15:11:06.015336+0000 mon.a (mon.0) 2515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[87147]: audit 2024-09-13T15:11:06.015336+0000 mon.a (mon.0) 2515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[87147]: audit 2024-09-13T15:11:06.026105+0000 mon.a (mon.0) 2516 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:06 smithi099 bash[87147]: audit 2024-09-13T15:11:06.026105+0000 mon.a (mon.0) 2516 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:06.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:06.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:06.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:06.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:06.425 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:06 smithi177 bash[68594]: cluster 2024-09-13T15:11:05.051825+0000 mgr.y (mgr.26252) 1350 : cluster [DBG] pgmap v1926: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-09-13T15:11:06.425 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:06 smithi177 bash[68594]: cluster 2024-09-13T15:11:05.051825+0000 mgr.y (mgr.26252) 1350 : cluster [DBG] pgmap v1926: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-09-13T15:11:06.425 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:06 smithi177 bash[68594]: audit 2024-09-13T15:11:06.015336+0000 mon.a (mon.0) 2515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.425 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:06 smithi177 bash[68594]: audit 2024-09-13T15:11:06.015336+0000 mon.a (mon.0) 2515 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.425 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:06 smithi177 bash[68594]: audit 2024-09-13T15:11:06.026105+0000 mon.a (mon.0) 2516 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.425 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:06 smithi177 bash[68594]: audit 2024-09-13T15:11:06.026105+0000 mon.a (mon.0) 2516 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:06.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:06.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:06.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:06.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:06.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:06.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:06.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:06.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:06.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:06.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:06.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:06.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:06.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:06.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:06.937 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:06.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:06.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:07.042 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:07.073 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:07.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:07.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:07.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:07.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:07.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:07.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 81 16 95504 95488 73.6681 50 0.00853414 0.0135695 2024-09-13T15:11:07.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:07.321 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:11:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:11:07] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-09-13T15:11:07.329 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:07.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:07.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:07.436 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:07.458 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:07.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:07.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:07.564 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:07.593 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:07.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:07.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:07.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:07.715 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:07.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:07.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:07.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:07.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:07.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:07.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:07.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:07.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:07.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:07.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:08.100 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:08.125 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:08.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:08.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:08.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 82 16 96235 96219 73.3269 45.6875 0.0326185 0.0136318 2024-09-13T15:11:08.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:08.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:08.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:08.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:08.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:08.390 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:08.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:08.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:08.493 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:08.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:08.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:08.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:08.570 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:08 smithi099 bash[87147]: cluster 2024-09-13T15:11:07.052356+0000 mgr.y (mgr.26252) 1351 : cluster [DBG] pgmap v1927: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:08.571 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:08 smithi099 bash[87147]: cluster 2024-09-13T15:11:07.052356+0000 mgr.y (mgr.26252) 1351 : cluster [DBG] pgmap v1927: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:08 smithi099 bash[84576]: cluster 2024-09-13T15:11:07.052356+0000 mgr.y (mgr.26252) 1351 : cluster [DBG] pgmap v1927: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:08.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:08 smithi099 bash[84576]: cluster 2024-09-13T15:11:07.052356+0000 mgr.y (mgr.26252) 1351 : cluster [DBG] pgmap v1927: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:08.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:08 smithi177 bash[68594]: cluster 2024-09-13T15:11:07.052356+0000 mgr.y (mgr.26252) 1351 : cluster [DBG] pgmap v1927: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:08.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:08 smithi177 bash[68594]: cluster 2024-09-13T15:11:07.052356+0000 mgr.y (mgr.26252) 1351 : cluster [DBG] pgmap v1927: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:08.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:08.651 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:08.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:08.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:08.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:08.778 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:08.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:08.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:08.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:08.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:08.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:09.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:09.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:09.155 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:09.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:09.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:09.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:09.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 83 16 96922 96906 72.9607 42.9375 0.0113537 0.0137004 2024-09-13T15:11:09.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:09.315 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:09.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:09.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:09.418 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:09.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:09.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:09.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:09.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:09.558 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:09 smithi099 bash[84576]: cluster 2024-09-13T15:11:09.052836+0000 mgr.y (mgr.26252) 1352 : cluster [DBG] pgmap v1928: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2024-09-13T15:11:09.558 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:09 smithi099 bash[84576]: cluster 2024-09-13T15:11:09.052836+0000 mgr.y (mgr.26252) 1352 : cluster [DBG] pgmap v1928: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2024-09-13T15:11:09.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:09 smithi099 bash[87147]: cluster 2024-09-13T15:11:09.052836+0000 mgr.y (mgr.26252) 1352 : cluster [DBG] pgmap v1928: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2024-09-13T15:11:09.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:09 smithi099 bash[87147]: cluster 2024-09-13T15:11:09.052836+0000 mgr.y (mgr.26252) 1352 : cluster [DBG] pgmap v1928: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2024-09-13T15:11:09.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:09.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:09.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:09.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:09 smithi177 bash[68594]: cluster 2024-09-13T15:11:09.052836+0000 mgr.y (mgr.26252) 1352 : cluster [DBG] pgmap v1928: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2024-09-13T15:11:09.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:09 smithi177 bash[68594]: cluster 2024-09-13T15:11:09.052836+0000 mgr.y (mgr.26252) 1352 : cluster [DBG] pgmap v1928: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2024-09-13T15:11:09.686 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:09.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:09.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:09.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:09.838 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:09.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:09.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:09.868 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:11:09.868 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:11:09.941 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:09.977 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:09.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:10.083 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:10.113 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:10.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:10.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:10.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 84 16 97711 97695 72.6792 49.3125 0.0106025 0.0137538 2024-09-13T15:11:10.217 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:10.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:10.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:10.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:10.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:11:10 smithi177 bash[61439]: ts=2024-09-13T15:11:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:11:10.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:10.394 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:10.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:10.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:10.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:10.526 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:10.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:10.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:10.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:10.672 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:10.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:10.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:10.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:10.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:10.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:10.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:10.900 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:10.922 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:10.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:10.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:11.025 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:11.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:11.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:11.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:11.151 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:11.186 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:11.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:11.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:11.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 85 16 98354 98338 72.2969 40.1875 0.0261023 0.0138263 2024-09-13T15:11:11.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:11.320 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:11.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:11.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:11.424 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:11.451 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:11.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:11.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:11.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:11.587 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:11.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:11.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:11.690 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:11.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:11.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:11.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:11.823 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:11.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:11.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:11.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:11.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:11.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:11.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:11.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:12.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:12.113 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:12.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:12.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:12.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 86 16 99234 99218 72.0957 55 0.0285302 0.0138579 2024-09-13T15:11:12.216 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:12.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:12.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:12.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:12.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:12.360 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:12 smithi177 bash[68594]: cluster 2024-09-13T15:11:11.053995+0000 mgr.y (mgr.26252) 1353 : cluster [DBG] pgmap v1929: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T15:11:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:12 smithi177 bash[68594]: cluster 2024-09-13T15:11:11.053995+0000 mgr.y (mgr.26252) 1353 : cluster [DBG] pgmap v1929: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T15:11:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:12.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:12.463 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:12.465 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:12 smithi099 bash[84576]: cluster 2024-09-13T15:11:11.053995+0000 mgr.y (mgr.26252) 1353 : cluster [DBG] pgmap v1929: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T15:11:12.465 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:12 smithi099 bash[84576]: cluster 2024-09-13T15:11:11.053995+0000 mgr.y (mgr.26252) 1353 : cluster [DBG] pgmap v1929: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T15:11:12.465 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:12 smithi099 bash[87147]: cluster 2024-09-13T15:11:11.053995+0000 mgr.y (mgr.26252) 1353 : cluster [DBG] pgmap v1929: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T15:11:12.465 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:12 smithi099 bash[87147]: cluster 2024-09-13T15:11:11.053995+0000 mgr.y (mgr.26252) 1353 : cluster [DBG] pgmap v1929: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T15:11:12.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:12.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:12.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:12.512 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:11:12.512 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:11:12.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 reset_purged_snaps_last 2024-09-13T15:11:12.537 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:11:12.537 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:11:12.577 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:11:12.578 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:11:12.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:12.628 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:11:12.628 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:11:12.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:12.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:12.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:12.665 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:11:12.665 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:11:12.696 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:11:12.696 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:11:12.727 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:11:12.727 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:11:12.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:12.756 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:11:12.757 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:11:12.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:12.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:12.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:12.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:12.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:12.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:12.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:13.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:13.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:13.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:13.135 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:11:13.151 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[84576]: audit 2024-09-13T15:11:12.219938+0000 mon.a (mon.0) 2517 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.151 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[84576]: audit 2024-09-13T15:11:12.219938+0000 mon.a (mon.0) 2517 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.151 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[84576]: audit 2024-09-13T15:11:12.220053+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.99:0/1477488887' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[84576]: audit 2024-09-13T15:11:12.220053+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.99:0/1477488887' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[84576]: audit 2024-09-13T15:11:12.224712+0000 mon.a (mon.0) 2518 : audit [DBG] from='client.? 172.21.15.99:0/295025892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:13.152 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[84576]: audit 2024-09-13T15:11:12.224712+0000 mon.a (mon.0) 2518 : audit [DBG] from='client.? 172.21.15.99:0/295025892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:13.152 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[87147]: audit 2024-09-13T15:11:12.219938+0000 mon.a (mon.0) 2517 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.152 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[87147]: audit 2024-09-13T15:11:12.219938+0000 mon.a (mon.0) 2517 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.152 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[87147]: audit 2024-09-13T15:11:12.220053+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.99:0/1477488887' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.152 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[87147]: audit 2024-09-13T15:11:12.220053+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.99:0/1477488887' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.152 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[87147]: audit 2024-09-13T15:11:12.224712+0000 mon.a (mon.0) 2518 : audit [DBG] from='client.? 172.21.15.99:0/295025892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:13.152 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:13 smithi099 bash[87147]: audit 2024-09-13T15:11:12.224712+0000 mon.a (mon.0) 2518 : audit [DBG] from='client.? 172.21.15.99:0/295025892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:13.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:13.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:13.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:13.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:13.206 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 87 16 99838 99822 71.7009 37.75 0.00784891 0.0139416 2024-09-13T15:11:13.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:13.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:13.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:13.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:13.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:13.446 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:13.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:13.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:13.572 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:13.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:13.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:13 smithi177 bash[68594]: audit 2024-09-13T15:11:12.219938+0000 mon.a (mon.0) 2517 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:13 smithi177 bash[68594]: audit 2024-09-13T15:11:12.219938+0000 mon.a (mon.0) 2517 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:13 smithi177 bash[68594]: audit 2024-09-13T15:11:12.220053+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.99:0/1477488887' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:13 smithi177 bash[68594]: audit 2024-09-13T15:11:12.220053+0000 mon.b (mon.1) 18 : audit [INF] from='client.? 172.21.15.99:0/1477488887' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:11:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:13 smithi177 bash[68594]: audit 2024-09-13T15:11:12.224712+0000 mon.a (mon.0) 2518 : audit [DBG] from='client.? 172.21.15.99:0/295025892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:13 smithi177 bash[68594]: audit 2024-09-13T15:11:12.224712+0000 mon.a (mon.0) 2518 : audit [DBG] from='client.? 172.21.15.99:0/295025892' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:13.675 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:13.700 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:13.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:13.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:13.804 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:13.837 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:13.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:13.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:13.942 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:13.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:13.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:13.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:14.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:14.112 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:14.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:14.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:14.207 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 88 16 100409 100393 71.2917 35.6875 0.0214222 0.0140215 2024-09-13T15:11:14.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:14.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[84576]: cluster 2024-09-13T15:11:13.054850+0000 mgr.y (mgr.26252) 1354 : cluster [DBG] pgmap v1930: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2024-09-13T15:11:14.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[84576]: cluster 2024-09-13T15:11:13.054850+0000 mgr.y (mgr.26252) 1354 : cluster [DBG] pgmap v1930: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2024-09-13T15:11:14.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[84576]: cluster 2024-09-13T15:11:13.112770+0000 mon.a (mon.0) 2519 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:11:14.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[84576]: cluster 2024-09-13T15:11:13.112770+0000 mon.a (mon.0) 2519 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:11:14.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[84576]: cluster 2024-09-13T15:11:13.112802+0000 mon.a (mon.0) 2520 : cluster [INF] Cluster is now healthy 2024-09-13T15:11:14.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[84576]: cluster 2024-09-13T15:11:13.112802+0000 mon.a (mon.0) 2520 : cluster [INF] Cluster is now healthy 2024-09-13T15:11:14.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[84576]: audit 2024-09-13T15:11:13.125468+0000 mon.a (mon.0) 2521 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:11:14.225 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[84576]: audit 2024-09-13T15:11:13.125468+0000 mon.a (mon.0) 2521 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[84576]: cluster 2024-09-13T15:11:13.131840+0000 mon.a (mon.0) 2522 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[84576]: cluster 2024-09-13T15:11:13.131840+0000 mon.a (mon.0) 2522 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[87147]: cluster 2024-09-13T15:11:13.054850+0000 mgr.y (mgr.26252) 1354 : cluster [DBG] pgmap v1930: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[87147]: cluster 2024-09-13T15:11:13.054850+0000 mgr.y (mgr.26252) 1354 : cluster [DBG] pgmap v1930: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[87147]: cluster 2024-09-13T15:11:13.112770+0000 mon.a (mon.0) 2519 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[87147]: cluster 2024-09-13T15:11:13.112770+0000 mon.a (mon.0) 2519 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[87147]: cluster 2024-09-13T15:11:13.112802+0000 mon.a (mon.0) 2520 : cluster [INF] Cluster is now healthy 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[87147]: cluster 2024-09-13T15:11:13.112802+0000 mon.a (mon.0) 2520 : cluster [INF] Cluster is now healthy 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[87147]: audit 2024-09-13T15:11:13.125468+0000 mon.a (mon.0) 2521 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[87147]: audit 2024-09-13T15:11:13.125468+0000 mon.a (mon.0) 2521 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:11:14.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[87147]: cluster 2024-09-13T15:11:13.131840+0000 mon.a (mon.0) 2522 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2024-09-13T15:11:14.227 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:14 smithi099 bash[87147]: cluster 2024-09-13T15:11:13.131840+0000 mon.a (mon.0) 2522 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2024-09-13T15:11:14.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:14.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:14.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:14.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:14.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:14.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:14.479 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:14.511 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:14.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:14.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:14.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:14 smithi177 bash[68594]: cluster 2024-09-13T15:11:13.054850+0000 mgr.y (mgr.26252) 1354 : cluster [DBG] pgmap v1930: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2024-09-13T15:11:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:14 smithi177 bash[68594]: cluster 2024-09-13T15:11:13.054850+0000 mgr.y (mgr.26252) 1354 : cluster [DBG] pgmap v1930: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2024-09-13T15:11:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:14 smithi177 bash[68594]: cluster 2024-09-13T15:11:13.112770+0000 mon.a (mon.0) 2519 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:11:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:14 smithi177 bash[68594]: cluster 2024-09-13T15:11:13.112770+0000 mon.a (mon.0) 2519 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:11:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:14 smithi177 bash[68594]: cluster 2024-09-13T15:11:13.112802+0000 mon.a (mon.0) 2520 : cluster [INF] Cluster is now healthy 2024-09-13T15:11:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:14 smithi177 bash[68594]: cluster 2024-09-13T15:11:13.112802+0000 mon.a (mon.0) 2520 : cluster [INF] Cluster is now healthy 2024-09-13T15:11:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:14 smithi177 bash[68594]: audit 2024-09-13T15:11:13.125468+0000 mon.a (mon.0) 2521 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:11:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:14 smithi177 bash[68594]: audit 2024-09-13T15:11:13.125468+0000 mon.a (mon.0) 2521 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:11:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:14 smithi177 bash[68594]: cluster 2024-09-13T15:11:13.131840+0000 mon.a (mon.0) 2522 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2024-09-13T15:11:14.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:14 smithi177 bash[68594]: cluster 2024-09-13T15:11:13.131840+0000 mon.a (mon.0) 2522 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2024-09-13T15:11:14.615 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:14.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:14.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:14.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:14.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:14.766 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:14.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:14.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:14.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:14.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:14.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:14.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:14.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:15.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:15.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:15.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:15.131 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:15.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:15.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:15.207 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 89 16 101065 101049 70.9513 41 0.00947031 0.0140869 2024-09-13T15:11:15.254 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:15.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:15.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:15.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:15.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:15.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:15.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:15.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:15.514 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:15.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:15.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:15.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:15.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:15.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:15.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:15.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:15.779 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:15.823 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:15.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:15.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:15.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T15:11:15.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:15.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:15.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:15.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:16.051 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:16.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:16.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:16.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:16.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:16.197 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:16.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:16.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:16.207 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 90 16 101730 101714 70.6247 41.5625 0.00422081 0.0141058 2024-09-13T15:11:16.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:16.311 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:16 smithi099 bash[87147]: cluster 2024-09-13T15:11:15.055485+0000 mgr.y (mgr.26252) 1355 : cluster [DBG] pgmap v1932: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s 2024-09-13T15:11:16.311 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:16 smithi099 bash[87147]: cluster 2024-09-13T15:11:15.055485+0000 mgr.y (mgr.26252) 1355 : cluster [DBG] pgmap v1932: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s 2024-09-13T15:11:16.311 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:16 smithi099 bash[84576]: cluster 2024-09-13T15:11:15.055485+0000 mgr.y (mgr.26252) 1355 : cluster [DBG] pgmap v1932: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s 2024-09-13T15:11:16.311 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:16 smithi099 bash[84576]: cluster 2024-09-13T15:11:15.055485+0000 mgr.y (mgr.26252) 1355 : cluster [DBG] pgmap v1932: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s 2024-09-13T15:11:16.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:16.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:16.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:16.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:16 smithi177 bash[68594]: cluster 2024-09-13T15:11:15.055485+0000 mgr.y (mgr.26252) 1355 : cluster [DBG] pgmap v1932: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s 2024-09-13T15:11:16.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:16 smithi177 bash[68594]: cluster 2024-09-13T15:11:15.055485+0000 mgr.y (mgr.26252) 1355 : cluster [DBG] pgmap v1932: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s 2024-09-13T15:11:16.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:16.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:16.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:16.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:16.584 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:16.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:16.688 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:16.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:16.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:16.819 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:16.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:16.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:16.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:16.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:16.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:16.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:16.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:17.080 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:17.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:17.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:17.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:17.118 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:11:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:11:17] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-09-13T15:11:17.207 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total time run: 90.2831 2024-09-13T15:11:17.207 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total writes made: 101730 2024-09-13T15:11:17.207 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Write size: 65536 2024-09-13T15:11:17.207 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object size: 65536 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Bandwidth (MB/sec): 70.4243 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Bandwidth: 33.6592 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max bandwidth (MB/sec): 177.25 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min bandwidth (MB/sec): 26.375 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average IOPS: 1126 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev IOPS: 538.547 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max IOPS: 2836 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min IOPS: 422 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average Latency(s): 0.0141952 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Latency(s): 0.0306693 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max latency(s): 0.663272 2024-09-13T15:11:17.208 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min latency(s): 0.00117806 2024-09-13T15:11:17.212 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:17.218 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T15:11:17.248 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:17.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:17.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:17.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:17.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:17.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:17.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:17.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:17.507 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:17.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:17.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:17.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:17.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:17.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:17.739 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:17.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:17.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:17.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:17.860 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:17.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:17.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:17.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:17.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:18.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:18.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:18.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:18.118 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:18.158 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:18.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:18.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:18.264 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:18.273 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:18 smithi177 bash[68594]: cluster 2024-09-13T15:11:17.056398+0000 mgr.y (mgr.26252) 1356 : cluster [DBG] pgmap v1933: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2024-09-13T15:11:18.273 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:18 smithi177 bash[68594]: cluster 2024-09-13T15:11:17.056398+0000 mgr.y (mgr.26252) 1356 : cluster [DBG] pgmap v1933: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2024-09-13T15:11:18.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:18.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:18.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:18.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:18.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:18.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:18.498 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:18 smithi099 bash[84576]: cluster 2024-09-13T15:11:17.056398+0000 mgr.y (mgr.26252) 1356 : cluster [DBG] pgmap v1933: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2024-09-13T15:11:18.498 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:18 smithi099 bash[84576]: cluster 2024-09-13T15:11:17.056398+0000 mgr.y (mgr.26252) 1356 : cluster [DBG] pgmap v1933: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2024-09-13T15:11:18.498 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:18 smithi099 bash[87147]: cluster 2024-09-13T15:11:17.056398+0000 mgr.y (mgr.26252) 1356 : cluster [DBG] pgmap v1933: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2024-09-13T15:11:18.499 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:18 smithi099 bash[87147]: cluster 2024-09-13T15:11:17.056398+0000 mgr.y (mgr.26252) 1356 : cluster [DBG] pgmap v1933: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2024-09-13T15:11:18.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:18.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:18.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:18.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:18.665 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:18.693 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:18.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:18.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:18.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:18.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:18.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:18.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:18.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:18.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:18.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:19.053 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:19.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:19.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:19.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:19.180 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:19.206 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:19.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:19.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:19.309 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:19.313 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[87147]: audit 2024-09-13T15:11:18.313053+0000 mon.a (mon.0) 2523 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:19.348 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:19.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:19.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:19.452 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:19.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:19.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[84576]: audit 2024-09-13T15:11:18.313053+0000 mon.a (mon.0) 2523 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[84576]: audit 2024-09-13T15:11:18.313053+0000 mon.a (mon.0) 2523 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[84576]: audit 2024-09-13T15:11:18.318381+0000 mon.a (mon.0) 2524 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[84576]: audit 2024-09-13T15:11:18.318381+0000 mon.a (mon.0) 2524 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[84576]: cluster 2024-09-13T15:11:19.057155+0000 mgr.y (mgr.26252) 1357 : cluster [DBG] pgmap v1934: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[84576]: cluster 2024-09-13T15:11:19.057155+0000 mgr.y (mgr.26252) 1357 : cluster [DBG] pgmap v1934: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[87147]: audit 2024-09-13T15:11:18.313053+0000 mon.a (mon.0) 2523 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[87147]: audit 2024-09-13T15:11:18.318381+0000 mon.a (mon.0) 2524 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[87147]: audit 2024-09-13T15:11:18.318381+0000 mon.a (mon.0) 2524 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[87147]: cluster 2024-09-13T15:11:19.057155+0000 mgr.y (mgr.26252) 1357 : cluster [DBG] pgmap v1934: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:19 smithi099 bash[87147]: cluster 2024-09-13T15:11:19.057155+0000 mgr.y (mgr.26252) 1357 : cluster [DBG] pgmap v1934: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:19.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:19.599 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:19 smithi177 bash[68594]: audit 2024-09-13T15:11:18.313053+0000 mon.a (mon.0) 2523 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:19.599 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:19 smithi177 bash[68594]: audit 2024-09-13T15:11:18.313053+0000 mon.a (mon.0) 2523 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:19.599 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:19 smithi177 bash[68594]: audit 2024-09-13T15:11:18.318381+0000 mon.a (mon.0) 2524 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:19.600 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:19 smithi177 bash[68594]: audit 2024-09-13T15:11:18.318381+0000 mon.a (mon.0) 2524 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:19.600 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:19 smithi177 bash[68594]: cluster 2024-09-13T15:11:19.057155+0000 mgr.y (mgr.26252) 1357 : cluster [DBG] pgmap v1934: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:19.600 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:19 smithi177 bash[68594]: cluster 2024-09-13T15:11:19.057155+0000 mgr.y (mgr.26252) 1357 : cluster [DBG] pgmap v1934: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:11:19.615 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:19.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:19.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:19.718 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:19.738 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:19.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:19.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:19.864 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:19.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:19.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:19.969 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:19.993 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:19.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:20.100 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:20.130 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:20.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:20.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:20.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:20.269 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:20.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:20.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:20.373 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:20.413 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:20.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:20.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:20.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:11:20.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:20.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:11:20.598 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:20.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:20.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:11:20 smithi177 bash[61439]: ts=2024-09-13T15:11:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:11:20.707 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:20.745 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:20.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:20.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:20.853 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:20.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:20.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:21.003 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:21.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:21.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:21.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:21.168 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:21.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:21.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:21.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:21.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:21.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:21.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:21.405 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:21.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:21.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:21.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:21.556 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:21.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:21.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:21.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:21.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:21.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:21.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:21.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:21.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:21.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:21.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:21.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:21.990 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:21.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:22.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:22.115 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:22.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:22.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:22.221 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:22.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:22.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:22 smithi099 bash[84576]: cluster 2024-09-13T15:11:21.057978+0000 mgr.y (mgr.26252) 1358 : cluster [DBG] pgmap v1935: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 619 op/s 2024-09-13T15:11:22.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:22 smithi099 bash[84576]: cluster 2024-09-13T15:11:21.057978+0000 mgr.y (mgr.26252) 1358 : cluster [DBG] pgmap v1935: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 619 op/s 2024-09-13T15:11:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:22 smithi099 bash[87147]: cluster 2024-09-13T15:11:21.057978+0000 mgr.y (mgr.26252) 1358 : cluster [DBG] pgmap v1935: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 619 op/s 2024-09-13T15:11:22.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:22 smithi099 bash[87147]: cluster 2024-09-13T15:11:21.057978+0000 mgr.y (mgr.26252) 1358 : cluster [DBG] pgmap v1935: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 619 op/s 2024-09-13T15:11:22.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:22.358 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:22 smithi177 bash[68594]: cluster 2024-09-13T15:11:21.057978+0000 mgr.y (mgr.26252) 1358 : cluster [DBG] pgmap v1935: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 619 op/s 2024-09-13T15:11:22.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:22 smithi177 bash[68594]: cluster 2024-09-13T15:11:21.057978+0000 mgr.y (mgr.26252) 1358 : cluster [DBG] pgmap v1935: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 619 op/s 2024-09-13T15:11:22.372 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:22.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:22.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:22.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:22.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:22.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:22.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:22.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:22.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:22.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:22.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:22.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:22.792 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:22.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:22.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:22.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:22.933 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:22.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:22.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:23.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:23.075 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:23.077 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:23.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:23.100 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:23.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:23.213 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:23.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:23.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:23.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:23.346 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:23.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:23.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:23.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:23.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:23.584 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:23.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:23.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:23.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:23.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:23.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:23.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:23.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:23.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:23.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:23.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:23.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:23.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:24.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:24.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:24.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:24.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:24.153 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:24.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:24.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:24.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:24.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:24.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:24 smithi099 bash[84576]: cluster 2024-09-13T15:11:23.058535+0000 mgr.y (mgr.26252) 1359 : cluster [DBG] pgmap v1936: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 620 op/s 2024-09-13T15:11:24.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:24 smithi099 bash[84576]: cluster 2024-09-13T15:11:23.058535+0000 mgr.y (mgr.26252) 1359 : cluster [DBG] pgmap v1936: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 620 op/s 2024-09-13T15:11:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:24 smithi099 bash[87147]: cluster 2024-09-13T15:11:23.058535+0000 mgr.y (mgr.26252) 1359 : cluster [DBG] pgmap v1936: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 620 op/s 2024-09-13T15:11:24.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:24 smithi099 bash[87147]: cluster 2024-09-13T15:11:23.058535+0000 mgr.y (mgr.26252) 1359 : cluster [DBG] pgmap v1936: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 620 op/s 2024-09-13T15:11:24.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:24.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:24 smithi177 bash[68594]: cluster 2024-09-13T15:11:23.058535+0000 mgr.y (mgr.26252) 1359 : cluster [DBG] pgmap v1936: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 620 op/s 2024-09-13T15:11:24.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:24 smithi177 bash[68594]: cluster 2024-09-13T15:11:23.058535+0000 mgr.y (mgr.26252) 1359 : cluster [DBG] pgmap v1936: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 620 op/s 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:24.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:24.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:24.505 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:24.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:24.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:24.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:24.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:24.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:24.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:24.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:24.810 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:24.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:24.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:24.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:24.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:24.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:24.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:25.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:25.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:25.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:25.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:25.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:25.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:25.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:25.315 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:25.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:25.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:25.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:25.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:25.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:25.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:25.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:25.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:25.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:25.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:25.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:25.711 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:25.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:25.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:25.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:25.856 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:25.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:25.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:25.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:25.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:26.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:26.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:26.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:26.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:26.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:26.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:26.247 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:26.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:26.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:26.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:26.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:26.423 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:26.427 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:26 smithi177 bash[68594]: cluster 2024-09-13T15:11:25.059319+0000 mgr.y (mgr.26252) 1360 : cluster [DBG] pgmap v1937: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 792 op/s 2024-09-13T15:11:26.427 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:26 smithi177 bash[68594]: cluster 2024-09-13T15:11:25.059319+0000 mgr.y (mgr.26252) 1360 : cluster [DBG] pgmap v1937: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 792 op/s 2024-09-13T15:11:26.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:26.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:26.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:26.539 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:26 smithi099 bash[84576]: cluster 2024-09-13T15:11:25.059319+0000 mgr.y (mgr.26252) 1360 : cluster [DBG] pgmap v1937: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 792 op/s 2024-09-13T15:11:26.539 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:26 smithi099 bash[84576]: cluster 2024-09-13T15:11:25.059319+0000 mgr.y (mgr.26252) 1360 : cluster [DBG] pgmap v1937: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 792 op/s 2024-09-13T15:11:26.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:26 smithi099 bash[87147]: cluster 2024-09-13T15:11:25.059319+0000 mgr.y (mgr.26252) 1360 : cluster [DBG] pgmap v1937: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 792 op/s 2024-09-13T15:11:26.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:26 smithi099 bash[87147]: cluster 2024-09-13T15:11:25.059319+0000 mgr.y (mgr.26252) 1360 : cluster [DBG] pgmap v1937: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 792 op/s 2024-09-13T15:11:26.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:26.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:26.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:26.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:26.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:26.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:26.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:26.799 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:26.822 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:26.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:26.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:26.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:26.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:26.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:26.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:27.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:27.086 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:27.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:27.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:27.130 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:27.131 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:27.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:27.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:27.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:27.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:27.316 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:27.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:11:27] "GET /metrics HTTP/1.1" 200 36775 "" "Prometheus/2.51.0" 2024-09-13T15:11:27.339 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:27.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:27.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:27.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:27.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:27.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:27.586 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:27.612 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:27.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:27.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:27.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:27.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: audit 2024-09-13T15:11:26.694314+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:27.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: audit 2024-09-13T15:11:26.694314+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: audit 2024-09-13T15:11:26.703612+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: audit 2024-09-13T15:11:26.703612+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: audit 2024-09-13T15:11:26.706998+0000 mon.a (mon.0) 2527 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:11:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: audit 2024-09-13T15:11:26.706998+0000 mon.a (mon.0) 2527 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:11:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: audit 2024-09-13T15:11:26.708606+0000 mon.a (mon.0) 2528 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:11:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: audit 2024-09-13T15:11:26.708606+0000 mon.a (mon.0) 2528 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:11:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: audit 2024-09-13T15:11:26.722269+0000 mon.a (mon.0) 2529 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: audit 2024-09-13T15:11:26.722269+0000 mon.a (mon.0) 2529 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: cluster 2024-09-13T15:11:27.059860+0000 mgr.y (mgr.26252) 1361 : cluster [DBG] pgmap v1938: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 634 op/s 2024-09-13T15:11:27.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:27 smithi177 bash[68594]: cluster 2024-09-13T15:11:27.059860+0000 mgr.y (mgr.26252) 1361 : cluster [DBG] pgmap v1938: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 634 op/s 2024-09-13T15:11:27.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:27.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:27.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:27.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:27.885 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:27.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:27.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: audit 2024-09-13T15:11:26.694314+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: audit 2024-09-13T15:11:26.694314+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: audit 2024-09-13T15:11:26.703612+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: audit 2024-09-13T15:11:26.703612+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: audit 2024-09-13T15:11:26.706998+0000 mon.a (mon.0) 2527 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: audit 2024-09-13T15:11:26.706998+0000 mon.a (mon.0) 2527 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: audit 2024-09-13T15:11:26.708606+0000 mon.a (mon.0) 2528 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: audit 2024-09-13T15:11:26.708606+0000 mon.a (mon.0) 2528 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: audit 2024-09-13T15:11:26.722269+0000 mon.a (mon.0) 2529 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: audit 2024-09-13T15:11:26.722269+0000 mon.a (mon.0) 2529 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.003 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: cluster 2024-09-13T15:11:27.059860+0000 mgr.y (mgr.26252) 1361 : cluster [DBG] pgmap v1938: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 634 op/s 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[84576]: cluster 2024-09-13T15:11:27.059860+0000 mgr.y (mgr.26252) 1361 : cluster [DBG] pgmap v1938: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 634 op/s 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: audit 2024-09-13T15:11:26.694314+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: audit 2024-09-13T15:11:26.694314+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: audit 2024-09-13T15:11:26.703612+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: audit 2024-09-13T15:11:26.703612+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: audit 2024-09-13T15:11:26.706998+0000 mon.a (mon.0) 2527 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: audit 2024-09-13T15:11:26.706998+0000 mon.a (mon.0) 2527 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: audit 2024-09-13T15:11:26.708606+0000 mon.a (mon.0) 2528 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: audit 2024-09-13T15:11:26.708606+0000 mon.a (mon.0) 2528 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: audit 2024-09-13T15:11:26.722269+0000 mon.a (mon.0) 2529 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.004 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: audit 2024-09-13T15:11:26.722269+0000 mon.a (mon.0) 2529 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:11:28.005 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: cluster 2024-09-13T15:11:27.059860+0000 mgr.y (mgr.26252) 1361 : cluster [DBG] pgmap v1938: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 634 op/s 2024-09-13T15:11:28.005 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:27 smithi099 bash[87147]: cluster 2024-09-13T15:11:27.059860+0000 mgr.y (mgr.26252) 1361 : cluster [DBG] pgmap v1938: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 634 op/s 2024-09-13T15:11:28.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:28.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:28.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:28.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:28.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:28.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:28.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:28.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:28.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:28.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:28.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:28.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:28.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:28.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:28.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:28.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:28.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:28.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:28.664 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:28.691 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:28.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:28.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:28.797 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:28.821 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:28.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:28.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:28.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:28.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:28.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:28.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:29.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:29.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:29.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:29.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:29.180 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:29.211 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:29.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:29.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:29.316 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:29.338 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:29.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:29.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:29.442 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:29.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:29.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:29.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:29.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:29.601 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:29.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:29.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:29.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:29.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:29.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:29.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:29.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:29.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:29.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:29.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:29.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:30.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:30.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:30.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:30.124 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:30.154 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:30 smithi099 bash[84576]: cluster 2024-09-13T15:11:29.060298+0000 mgr.y (mgr.26252) 1362 : cluster [DBG] pgmap v1939: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.9 MiB/s wr, 638 op/s 2024-09-13T15:11:30.154 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:30 smithi099 bash[84576]: cluster 2024-09-13T15:11:29.060298+0000 mgr.y (mgr.26252) 1362 : cluster [DBG] pgmap v1939: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.9 MiB/s wr, 638 op/s 2024-09-13T15:11:30.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:30.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:30.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:30.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:30.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:30.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:30.399 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:30.410 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:30 smithi099 bash[87147]: cluster 2024-09-13T15:11:29.060298+0000 mgr.y (mgr.26252) 1362 : cluster [DBG] pgmap v1939: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.9 MiB/s wr, 638 op/s 2024-09-13T15:11:30.410 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:30 smithi099 bash[87147]: cluster 2024-09-13T15:11:29.060298+0000 mgr.y (mgr.26252) 1362 : cluster [DBG] pgmap v1939: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.9 MiB/s wr, 638 op/s 2024-09-13T15:11:30.424 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:30.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:30.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:30.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:30.557 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:30.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:30.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:30.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:11:30 smithi177 bash[61439]: ts=2024-09-13T15:11:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:11:30.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:30 smithi177 bash[68594]: cluster 2024-09-13T15:11:29.060298+0000 mgr.y (mgr.26252) 1362 : cluster [DBG] pgmap v1939: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.9 MiB/s wr, 638 op/s 2024-09-13T15:11:30.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:30 smithi177 bash[68594]: cluster 2024-09-13T15:11:29.060298+0000 mgr.y (mgr.26252) 1362 : cluster [DBG] pgmap v1939: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.9 MiB/s wr, 638 op/s 2024-09-13T15:11:30.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:30.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:30.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:30.800 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:30.831 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:30.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:30.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:30.935 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:30.961 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:30.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:30.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:31.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:31.086 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:31.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:31.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:31.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:31.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:31.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:31.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:31.371 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:31.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:31.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:31.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:31.504 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:31.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:31.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:31.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:31.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:31.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:31.734 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:31.762 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:31.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:31.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:31.865 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:31.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:31.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:31.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:32.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:32.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:32.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:32.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:32.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:32.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:32.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:32.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:32.267 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:32 smithi099 bash[84576]: cluster 2024-09-13T15:11:31.060843+0000 mgr.y (mgr.26252) 1363 : cluster [DBG] pgmap v1940: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 863 op/s 2024-09-13T15:11:32.267 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:32 smithi099 bash[84576]: cluster 2024-09-13T15:11:31.060843+0000 mgr.y (mgr.26252) 1363 : cluster [DBG] pgmap v1940: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 863 op/s 2024-09-13T15:11:32.267 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:32 smithi099 bash[87147]: cluster 2024-09-13T15:11:31.060843+0000 mgr.y (mgr.26252) 1363 : cluster [DBG] pgmap v1940: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 863 op/s 2024-09-13T15:11:32.268 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:32 smithi099 bash[87147]: cluster 2024-09-13T15:11:31.060843+0000 mgr.y (mgr.26252) 1363 : cluster [DBG] pgmap v1940: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 863 op/s 2024-09-13T15:11:32.283 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:32.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:32.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:32.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:32.413 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:32.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:32.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:32.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:32.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:32 smithi177 bash[68594]: cluster 2024-09-13T15:11:31.060843+0000 mgr.y (mgr.26252) 1363 : cluster [DBG] pgmap v1940: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 863 op/s 2024-09-13T15:11:32.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:32 smithi177 bash[68594]: cluster 2024-09-13T15:11:31.060843+0000 mgr.y (mgr.26252) 1363 : cluster [DBG] pgmap v1940: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 863 op/s 2024-09-13T15:11:32.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:32.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:32.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:32.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:32.685 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:32.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:32.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:32.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:32.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:32.814 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:32.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:32.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:32.945 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:32.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:32.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:33.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:33.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:33.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:33.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:33.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:33.210 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:33.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:33.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:33.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:33.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:33.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:33.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:33.449 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:33.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:33.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:33.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:33.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:33.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:33.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:33.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:33.720 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:33.749 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:33.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:33.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:33.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:33.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:33.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:33.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:34.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:34.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:34.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:34.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:34.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:34.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:34.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:34.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:34.272 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:34.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:34.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:34 smithi099 bash[87147]: cluster 2024-09-13T15:11:33.061300+0000 mgr.y (mgr.26252) 1364 : cluster [DBG] pgmap v1941: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T15:11:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:34 smithi099 bash[87147]: cluster 2024-09-13T15:11:33.061300+0000 mgr.y (mgr.26252) 1364 : cluster [DBG] pgmap v1941: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T15:11:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:34 smithi099 bash[87147]: audit 2024-09-13T15:11:33.308681+0000 mon.a (mon.0) 2530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:34 smithi099 bash[87147]: audit 2024-09-13T15:11:33.308681+0000 mon.a (mon.0) 2530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:34 smithi099 bash[84576]: cluster 2024-09-13T15:11:33.061300+0000 mgr.y (mgr.26252) 1364 : cluster [DBG] pgmap v1941: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T15:11:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:34 smithi099 bash[84576]: cluster 2024-09-13T15:11:33.061300+0000 mgr.y (mgr.26252) 1364 : cluster [DBG] pgmap v1941: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T15:11:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:34 smithi099 bash[84576]: audit 2024-09-13T15:11:33.308681+0000 mon.a (mon.0) 2530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:34 smithi099 bash[84576]: audit 2024-09-13T15:11:33.308681+0000 mon.a (mon.0) 2530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:34.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:34.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:34 smithi177 bash[68594]: cluster 2024-09-13T15:11:33.061300+0000 mgr.y (mgr.26252) 1364 : cluster [DBG] pgmap v1941: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T15:11:34.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:34 smithi177 bash[68594]: cluster 2024-09-13T15:11:33.061300+0000 mgr.y (mgr.26252) 1364 : cluster [DBG] pgmap v1941: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T15:11:34.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:34 smithi177 bash[68594]: audit 2024-09-13T15:11:33.308681+0000 mon.a (mon.0) 2530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:34.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:34 smithi177 bash[68594]: audit 2024-09-13T15:11:33.308681+0000 mon.a (mon.0) 2530 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:34.401 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:34.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:34.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:34.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:34.539 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:34.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:34.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:34.673 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:34.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:34.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:34.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:34.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:34.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:34.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:34.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:34.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:34.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:34.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:35.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:35.064 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:35.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:35.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:35.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:35.189 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:35.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:35.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:35.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:35.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:35.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:35.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:35.447 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:35.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:35.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:35.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:35.580 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:35.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:35.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:35.686 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:35.718 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:35.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:35.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:35.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:35.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:35.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:35.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:35.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:35.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:36.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:36.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:36.100 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:36.122 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:36.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:36.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:36.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:36.258 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:36.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:36 smithi177 bash[68594]: cluster 2024-09-13T15:11:35.061812+0000 mgr.y (mgr.26252) 1365 : cluster [DBG] pgmap v1942: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T15:11:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:36 smithi177 bash[68594]: cluster 2024-09-13T15:11:35.061812+0000 mgr.y (mgr.26252) 1365 : cluster [DBG] pgmap v1942: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T15:11:36.363 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:36.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:36.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:36.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:36.456 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:36 smithi099 bash[87147]: cluster 2024-09-13T15:11:35.061812+0000 mgr.y (mgr.26252) 1365 : cluster [DBG] pgmap v1942: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T15:11:36.457 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:36 smithi099 bash[87147]: cluster 2024-09-13T15:11:35.061812+0000 mgr.y (mgr.26252) 1365 : cluster [DBG] pgmap v1942: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T15:11:36.457 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:36 smithi099 bash[84576]: cluster 2024-09-13T15:11:35.061812+0000 mgr.y (mgr.26252) 1365 : cluster [DBG] pgmap v1942: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T15:11:36.457 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:36 smithi099 bash[84576]: cluster 2024-09-13T15:11:35.061812+0000 mgr.y (mgr.26252) 1365 : cluster [DBG] pgmap v1942: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T15:11:36.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:36.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:36.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:36.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:36.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:36.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:36.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:36.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:36.765 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:36.805 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:36.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:36.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:36.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:36.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:36.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:36.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:37.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:37.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:37.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:37.180 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:37.205 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:37.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:37.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:37.312 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:37.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:37.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:37.405 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:11:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:11:37] "GET /metrics HTTP/1.1" 200 36779 "" "Prometheus/2.51.0" 2024-09-13T15:11:37.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:37.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:37.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:37.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:37.585 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:37.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:37.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:37.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:37.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:37.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:37.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:37.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:37.880 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:37.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:37.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:37.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:38.007 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:38.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:38.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:38.109 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:38.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:38.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:38.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:38.234 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:38.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:38.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:38.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:38 smithi099 bash[84576]: cluster 2024-09-13T15:11:37.062335+0000 mgr.y (mgr.26252) 1366 : cluster [DBG] pgmap v1943: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-09-13T15:11:38.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:38 smithi099 bash[84576]: cluster 2024-09-13T15:11:37.062335+0000 mgr.y (mgr.26252) 1366 : cluster [DBG] pgmap v1943: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-09-13T15:11:38.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:38 smithi099 bash[87147]: cluster 2024-09-13T15:11:37.062335+0000 mgr.y (mgr.26252) 1366 : cluster [DBG] pgmap v1943: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-09-13T15:11:38.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:38 smithi099 bash[87147]: cluster 2024-09-13T15:11:37.062335+0000 mgr.y (mgr.26252) 1366 : cluster [DBG] pgmap v1943: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-09-13T15:11:38.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:38.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:38.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:38.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:38.503 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:38.511 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:38 smithi177 bash[68594]: cluster 2024-09-13T15:11:37.062335+0000 mgr.y (mgr.26252) 1366 : cluster [DBG] pgmap v1943: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-09-13T15:11:38.512 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:38 smithi177 bash[68594]: cluster 2024-09-13T15:11:37.062335+0000 mgr.y (mgr.26252) 1366 : cluster [DBG] pgmap v1943: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2024-09-13T15:11:38.532 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:38.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:38.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:38.637 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:38.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:38.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:38.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:38.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:38.792 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:38.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:38.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:38.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:38.929 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:38.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:38.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:39.034 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:39.065 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:39.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:39.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:39.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:39.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:39.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:39.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:39.304 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:39.330 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:39.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:39.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:39.433 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:39.452 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:39.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:39.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:39.598 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:39.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:39.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:39.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:39.738 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:39.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:39.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:39.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:39.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:39.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:39.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:39.992 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:39.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:39.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:40.095 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:40.136 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:40.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:40.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:40.242 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:40.251 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:40 smithi177 bash[68594]: cluster 2024-09-13T15:11:39.062839+0000 mgr.y (mgr.26252) 1367 : cluster [DBG] pgmap v1944: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T15:11:40.251 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:40 smithi177 bash[68594]: cluster 2024-09-13T15:11:39.062839+0000 mgr.y (mgr.26252) 1367 : cluster [DBG] pgmap v1944: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T15:11:40.252 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:11:40 smithi177 bash[61439]: ts=2024-09-13T15:11:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:11:40.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:40.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:40.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:40.399 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:40.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:40.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:40.517 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:40 smithi099 bash[84576]: cluster 2024-09-13T15:11:39.062839+0000 mgr.y (mgr.26252) 1367 : cluster [DBG] pgmap v1944: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T15:11:40.518 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:40 smithi099 bash[84576]: cluster 2024-09-13T15:11:39.062839+0000 mgr.y (mgr.26252) 1367 : cluster [DBG] pgmap v1944: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T15:11:40.518 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:40 smithi099 bash[87147]: cluster 2024-09-13T15:11:39.062839+0000 mgr.y (mgr.26252) 1367 : cluster [DBG] pgmap v1944: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T15:11:40.518 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:40 smithi099 bash[87147]: cluster 2024-09-13T15:11:39.062839+0000 mgr.y (mgr.26252) 1367 : cluster [DBG] pgmap v1944: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2024-09-13T15:11:40.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:40.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:40.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:40.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:40.678 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:40.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:40.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:40.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:40.806 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:40.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:40.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:40.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:40.929 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:40.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:40.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:41.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:41.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:41.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:41.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:41.179 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:41.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:41.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:41.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:41.312 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:41.339 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:41.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:41.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:41.443 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:41.473 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:41.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:41.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:41.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:41.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:41.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:41.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:41.738 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:41.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:41.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:41.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:41.871 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:41.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:41.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:41.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:41.998 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:42.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:42.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:42.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:42.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:42.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:42.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:42.241 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:42.268 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:42 smithi099 bash[87147]: cluster 2024-09-13T15:11:41.063533+0000 mgr.y (mgr.26252) 1368 : cluster [DBG] pgmap v1945: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:11:42.268 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:42 smithi099 bash[87147]: cluster 2024-09-13T15:11:41.063533+0000 mgr.y (mgr.26252) 1368 : cluster [DBG] pgmap v1945: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:11:42.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:42.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:42.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:42.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:42.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:42.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:42.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:42.521 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:42.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:42 smithi177 bash[68594]: cluster 2024-09-13T15:11:41.063533+0000 mgr.y (mgr.26252) 1368 : cluster [DBG] pgmap v1945: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:11:42.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:42 smithi177 bash[68594]: cluster 2024-09-13T15:11:41.063533+0000 mgr.y (mgr.26252) 1368 : cluster [DBG] pgmap v1945: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:11:42.543 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:42.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:42.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:42.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:42.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:42.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:42.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:42.748 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:42 smithi099 bash[84576]: cluster 2024-09-13T15:11:41.063533+0000 mgr.y (mgr.26252) 1368 : cluster [DBG] pgmap v1945: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:11:42.748 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:42 smithi099 bash[84576]: cluster 2024-09-13T15:11:41.063533+0000 mgr.y (mgr.26252) 1368 : cluster [DBG] pgmap v1945: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:11:42.782 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:42.805 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:42.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:42.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:42.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:42.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:42.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:43.046 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:43.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:43.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:43.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:43.179 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:43.207 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:43.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:43.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:43.317 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:43.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:43.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:43.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:43.439 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:43.463 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:43.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:43.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:43.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:43.591 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:43.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:43.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:43.701 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:43.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:43.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:43.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:43.839 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:43.871 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:43.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:43.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:43.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:44.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:44.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:44.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:44.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:44.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:44.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:44.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:44.259 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:44.290 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:44.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:44.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:44.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:44.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:44 smithi177 bash[68594]: cluster 2024-09-13T15:11:43.063992+0000 mgr.y (mgr.26252) 1369 : cluster [DBG] pgmap v1946: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T15:11:44.403 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:44 smithi177 bash[68594]: cluster 2024-09-13T15:11:43.063992+0000 mgr.y (mgr.26252) 1369 : cluster [DBG] pgmap v1946: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T15:11:44.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:44.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:44.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:44.523 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:44.533 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:44 smithi099 bash[84576]: cluster 2024-09-13T15:11:43.063992+0000 mgr.y (mgr.26252) 1369 : cluster [DBG] pgmap v1946: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T15:11:44.533 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:44 smithi099 bash[84576]: cluster 2024-09-13T15:11:43.063992+0000 mgr.y (mgr.26252) 1369 : cluster [DBG] pgmap v1946: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T15:11:44.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:44 smithi099 bash[87147]: cluster 2024-09-13T15:11:43.063992+0000 mgr.y (mgr.26252) 1369 : cluster [DBG] pgmap v1946: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T15:11:44.534 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:44 smithi099 bash[87147]: cluster 2024-09-13T15:11:43.063992+0000 mgr.y (mgr.26252) 1369 : cluster [DBG] pgmap v1946: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-09-13T15:11:44.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:44.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:44.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:44.653 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:44.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:44.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:44.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:44.787 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:44.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:44.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:44.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:44.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:44.938 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:44.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:45.041 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:45.065 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:45.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:45.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:45.168 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:45.186 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:45.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:45.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:45.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:45.307 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:45.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:45.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:45.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:45.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:45.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:45.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:45.537 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:45.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:45.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:45.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:45.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:45.698 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:45.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:45.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:45.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:45.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:45.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:45.935 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:45.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:45.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:45.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:46.055 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:46.082 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:46.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:46.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:46.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:46 smithi177 bash[68594]: cluster 2024-09-13T15:11:45.064583+0000 mgr.y (mgr.26252) 1370 : cluster [DBG] pgmap v1947: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T15:11:46.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:46 smithi177 bash[68594]: cluster 2024-09-13T15:11:45.064583+0000 mgr.y (mgr.26252) 1370 : cluster [DBG] pgmap v1947: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T15:11:46.210 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:46.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:46.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:46.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:46.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:46 smithi099 bash[84576]: cluster 2024-09-13T15:11:45.064583+0000 mgr.y (mgr.26252) 1370 : cluster [DBG] pgmap v1947: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T15:11:46.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:46 smithi099 bash[84576]: cluster 2024-09-13T15:11:45.064583+0000 mgr.y (mgr.26252) 1370 : cluster [DBG] pgmap v1947: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T15:11:46.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:46 smithi099 bash[87147]: cluster 2024-09-13T15:11:45.064583+0000 mgr.y (mgr.26252) 1370 : cluster [DBG] pgmap v1947: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T15:11:46.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:46 smithi099 bash[87147]: cluster 2024-09-13T15:11:45.064583+0000 mgr.y (mgr.26252) 1370 : cluster [DBG] pgmap v1947: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T15:11:46.338 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:46.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:46.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:46.443 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:46.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:46.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:46.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:46.583 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:46.623 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:46.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:46.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:46.728 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:46.750 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:46.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:46.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:46.880 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:46.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:46.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:46.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:47.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:47.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:47.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:47.115 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:47.148 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:47.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:47.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:47.250 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:11:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:11:47] "GET /metrics HTTP/1.1" 200 36779 "" "Prometheus/2.51.0" 2024-09-13T15:11:47.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:47.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:47.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:47.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:47.388 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:47.410 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:47.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:47.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:47.450 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:11:47.450 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:11:47.450 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2024-09-13T15:11:47.451 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:11:47.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:47.552 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:47.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:47.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:47.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:47.682 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:47.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:47.786 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:47.808 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:47.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:47.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:47.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:47.943 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:47.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:48.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:48.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:48.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:48.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:48.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:48.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:48.313 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:48.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:48 smithi099 bash[84576]: cluster 2024-09-13T15:11:47.065094+0000 mgr.y (mgr.26252) 1371 : cluster [DBG] pgmap v1948: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:11:48.322 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:48 smithi099 bash[84576]: cluster 2024-09-13T15:11:47.065094+0000 mgr.y (mgr.26252) 1371 : cluster [DBG] pgmap v1948: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:11:48.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:48 smithi099 bash[87147]: cluster 2024-09-13T15:11:47.065094+0000 mgr.y (mgr.26252) 1371 : cluster [DBG] pgmap v1948: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:11:48.322 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:48 smithi099 bash[87147]: cluster 2024-09-13T15:11:47.065094+0000 mgr.y (mgr.26252) 1371 : cluster [DBG] pgmap v1948: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:11:48.335 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:48.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:48.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:48.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:48 smithi177 bash[68594]: cluster 2024-09-13T15:11:47.065094+0000 mgr.y (mgr.26252) 1371 : cluster [DBG] pgmap v1948: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:11:48.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:48 smithi177 bash[68594]: cluster 2024-09-13T15:11:47.065094+0000 mgr.y (mgr.26252) 1371 : cluster [DBG] pgmap v1948: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:11:48.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:48.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:48.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:48.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:48.571 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:48.593 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:48.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:48.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:48.757 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:48.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:48.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:48.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:48.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:48.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:48.910 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:11:48.910 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:11:48.959 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:11:48.959 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:11:48.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:49.006 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:11:49.006 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:11:49.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:49.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:49.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:49.056 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:11:49.056 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:11:49.082 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:11:49.083 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:11:49.110 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:11:49.110 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:11:49.129 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:49.140 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:11:49.140 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:11:49.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:49.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:49.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:49.158 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:11:49.158 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:11:49.173 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T15:11:49.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:49.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:49.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:49.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:49.385 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:49.395 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[84576]: audit 2024-09-13T15:11:48.309198+0000 mon.a (mon.0) 2531 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:49.395 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[84576]: audit 2024-09-13T15:11:48.309198+0000 mon.a (mon.0) 2531 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:49.395 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[84576]: audit 2024-09-13T15:11:48.877126+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.99:0/701195812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:49.395 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[84576]: audit 2024-09-13T15:11:48.877126+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.99:0/701195812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:49.395 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[84576]: audit 2024-09-13T15:11:49.019163+0000 mon.a (mon.0) 2532 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:11:49.395 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[84576]: audit 2024-09-13T15:11:49.019163+0000 mon.a (mon.0) 2532 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:11:49.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[87147]: audit 2024-09-13T15:11:48.309198+0000 mon.a (mon.0) 2531 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:49.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[87147]: audit 2024-09-13T15:11:48.309198+0000 mon.a (mon.0) 2531 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:49.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[87147]: audit 2024-09-13T15:11:48.877126+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.99:0/701195812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:49.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[87147]: audit 2024-09-13T15:11:48.877126+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.99:0/701195812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:49.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[87147]: audit 2024-09-13T15:11:49.019163+0000 mon.a (mon.0) 2532 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:11:49.396 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:49 smithi099 bash[87147]: audit 2024-09-13T15:11:49.019163+0000 mon.a (mon.0) 2532 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:11:49.406 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:49.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:49.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:49.512 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:49.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:49 smithi177 bash[68594]: audit 2024-09-13T15:11:48.309198+0000 mon.a (mon.0) 2531 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:49.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:49 smithi177 bash[68594]: audit 2024-09-13T15:11:48.309198+0000 mon.a (mon.0) 2531 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:49.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:49 smithi177 bash[68594]: audit 2024-09-13T15:11:48.877126+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.99:0/701195812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:49.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:49 smithi177 bash[68594]: audit 2024-09-13T15:11:48.877126+0000 mon.b (mon.1) 19 : audit [DBG] from='client.? 172.21.15.99:0/701195812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:11:49.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:49 smithi177 bash[68594]: audit 2024-09-13T15:11:49.019163+0000 mon.a (mon.0) 2532 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:11:49.521 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:49 smithi177 bash[68594]: audit 2024-09-13T15:11:49.019163+0000 mon.a (mon.0) 2532 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:11:49.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:49.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:49.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:49.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:49.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:49.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:49.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:49.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:49.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:49.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:49.894 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:49.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:49.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:49.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:50.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:50.066 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:50.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:50.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:50.170 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:50.193 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:50.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:50.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:50.302 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:50.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:50 smithi177 bash[68594]: cluster 2024-09-13T15:11:49.065567+0000 mgr.y (mgr.26252) 1372 : cluster [DBG] pgmap v1949: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-09-13T15:11:50.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:50 smithi177 bash[68594]: cluster 2024-09-13T15:11:49.065567+0000 mgr.y (mgr.26252) 1372 : cluster [DBG] pgmap v1949: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-09-13T15:11:50.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:50 smithi177 bash[68594]: cluster 2024-09-13T15:11:49.167818+0000 mon.a (mon.0) 2533 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:11:50.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:50 smithi177 bash[68594]: cluster 2024-09-13T15:11:49.167818+0000 mon.a (mon.0) 2533 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:11:50.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:50 smithi177 bash[68594]: audit 2024-09-13T15:11:49.173416+0000 mon.a (mon.0) 2534 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:11:50.312 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:50 smithi177 bash[68594]: audit 2024-09-13T15:11:49.173416+0000 mon.a (mon.0) 2534 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:11:50.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:50 smithi177 bash[68594]: cluster 2024-09-13T15:11:49.180146+0000 mon.a (mon.0) 2535 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2024-09-13T15:11:50.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:50 smithi177 bash[68594]: cluster 2024-09-13T15:11:49.180146+0000 mon.a (mon.0) 2535 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2024-09-13T15:11:50.313 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:11:50 smithi177 bash[61439]: ts=2024-09-13T15:11:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:11:50.335 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:50.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:50.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:50.451 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[84576]: cluster 2024-09-13T15:11:49.065567+0000 mgr.y (mgr.26252) 1372 : cluster [DBG] pgmap v1949: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-09-13T15:11:50.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[84576]: cluster 2024-09-13T15:11:49.065567+0000 mgr.y (mgr.26252) 1372 : cluster [DBG] pgmap v1949: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-09-13T15:11:50.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[84576]: cluster 2024-09-13T15:11:49.167818+0000 mon.a (mon.0) 2533 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:11:50.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[84576]: cluster 2024-09-13T15:11:49.167818+0000 mon.a (mon.0) 2533 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:11:50.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[84576]: audit 2024-09-13T15:11:49.173416+0000 mon.a (mon.0) 2534 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:11:50.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[84576]: audit 2024-09-13T15:11:49.173416+0000 mon.a (mon.0) 2534 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:11:50.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[84576]: cluster 2024-09-13T15:11:49.180146+0000 mon.a (mon.0) 2535 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2024-09-13T15:11:50.452 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[84576]: cluster 2024-09-13T15:11:49.180146+0000 mon.a (mon.0) 2535 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2024-09-13T15:11:50.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[87147]: cluster 2024-09-13T15:11:49.065567+0000 mgr.y (mgr.26252) 1372 : cluster [DBG] pgmap v1949: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-09-13T15:11:50.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[87147]: cluster 2024-09-13T15:11:49.065567+0000 mgr.y (mgr.26252) 1372 : cluster [DBG] pgmap v1949: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-09-13T15:11:50.453 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[87147]: cluster 2024-09-13T15:11:49.167818+0000 mon.a (mon.0) 2533 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:11:50.453 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[87147]: cluster 2024-09-13T15:11:49.167818+0000 mon.a (mon.0) 2533 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:11:50.453 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[87147]: audit 2024-09-13T15:11:49.173416+0000 mon.a (mon.0) 2534 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:11:50.453 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[87147]: audit 2024-09-13T15:11:49.173416+0000 mon.a (mon.0) 2534 : audit [INF] from='client.? 172.21.15.99:0/1200910032' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:11:50.453 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[87147]: cluster 2024-09-13T15:11:49.180146+0000 mon.a (mon.0) 2535 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2024-09-13T15:11:50.453 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:50 smithi099 bash[87147]: cluster 2024-09-13T15:11:49.180146+0000 mon.a (mon.0) 2535 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2024-09-13T15:11:50.468 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:50.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:50.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:50.575 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:50.593 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:50.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:50.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:50.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:50.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:50.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:50.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:50.854 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:50.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:50.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:50.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:50.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:50.990 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:50.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:51.091 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:51.125 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:51.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:51.230 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:51.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:51.254 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:51.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:51.355 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:51.385 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:51.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:51.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:51.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:51.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:51.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:51.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:51.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:51.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:51.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:51.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:51.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:51.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:51.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:51.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:51.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:51.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:51.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:52.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:52.032 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:52.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:52.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:52.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:52.204 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:52.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:52.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:52.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:52.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:52 smithi099 bash[84576]: cluster 2024-09-13T15:11:51.066203+0000 mgr.y (mgr.26252) 1373 : cluster [DBG] pgmap v1951: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-09-13T15:11:52.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:52 smithi099 bash[84576]: cluster 2024-09-13T15:11:51.066203+0000 mgr.y (mgr.26252) 1373 : cluster [DBG] pgmap v1951: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-09-13T15:11:52.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:52 smithi099 bash[87147]: cluster 2024-09-13T15:11:51.066203+0000 mgr.y (mgr.26252) 1373 : cluster [DBG] pgmap v1951: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-09-13T15:11:52.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:52 smithi099 bash[87147]: cluster 2024-09-13T15:11:51.066203+0000 mgr.y (mgr.26252) 1373 : cluster [DBG] pgmap v1951: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-09-13T15:11:52.334 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:52.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:52.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:52.450 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:52 smithi177 bash[68594]: cluster 2024-09-13T15:11:51.066203+0000 mgr.y (mgr.26252) 1373 : cluster [DBG] pgmap v1951: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-09-13T15:11:52.450 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:52 smithi177 bash[68594]: cluster 2024-09-13T15:11:51.066203+0000 mgr.y (mgr.26252) 1373 : cluster [DBG] pgmap v1951: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-09-13T15:11:52.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:52.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:52.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T15:11:52.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 reset_purged_snaps_last 2024-09-13T15:11:52.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:52.602 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:52.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:52.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:52.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:52.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:52.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:52.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:52.840 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:52.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:52.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:52.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:52.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:53.008 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:53.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:53.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:53.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:53.140 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:53.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:53.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:53.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:53.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:53.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:53.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:53.377 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:53.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:53.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:53.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:53.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:53.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:53.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:53.638 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:53.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:53.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:53.770 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:53.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:53.819 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:53.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:53.920 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:53.945 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:53.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:53.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:54.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:54.092 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:54.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:54.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:54.181 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:54 smithi099 bash[87147]: cluster 2024-09-13T15:11:53.066687+0000 mgr.y (mgr.26252) 1374 : cluster [DBG] pgmap v1952: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T15:11:54.181 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:54 smithi099 bash[87147]: cluster 2024-09-13T15:11:53.066687+0000 mgr.y (mgr.26252) 1374 : cluster [DBG] pgmap v1952: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T15:11:54.195 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:54.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:54.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:54.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:54.329 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:54.338 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:54 smithi177 bash[68594]: cluster 2024-09-13T15:11:53.066687+0000 mgr.y (mgr.26252) 1374 : cluster [DBG] pgmap v1952: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T15:11:54.338 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:54 smithi177 bash[68594]: cluster 2024-09-13T15:11:53.066687+0000 mgr.y (mgr.26252) 1374 : cluster [DBG] pgmap v1952: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T15:11:54.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:54.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:54.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:54.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:54.480 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:54.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:54.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:54.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:54 smithi099 bash[84576]: cluster 2024-09-13T15:11:53.066687+0000 mgr.y (mgr.26252) 1374 : cluster [DBG] pgmap v1952: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T15:11:54.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:54 smithi099 bash[84576]: cluster 2024-09-13T15:11:53.066687+0000 mgr.y (mgr.26252) 1374 : cluster [DBG] pgmap v1952: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2024-09-13T15:11:54.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:54.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:54.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:54.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:54.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:54.759 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:54.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:54.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:54.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:54.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:54.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:54.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:54.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:55.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:55.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:55.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:55.119 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:55.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:55.154 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:11:55.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:55.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:55.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:11:55.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:55.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:55.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:55.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:55.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:55.452 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:55.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:55.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:55.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:55.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:55.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:55.706 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:55.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:55.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:55.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:55.833 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:55.859 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:55.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:55.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:55.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:55.989 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:55.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:55.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:56.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:56.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:56.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:56.233 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:56.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:56.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:56.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:56 smithi099 bash[84576]: cluster 2024-09-13T15:11:55.067320+0000 mgr.y (mgr.26252) 1375 : cluster [DBG] pgmap v1953: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:56.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:56 smithi099 bash[84576]: cluster 2024-09-13T15:11:55.067320+0000 mgr.y (mgr.26252) 1375 : cluster [DBG] pgmap v1953: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:56.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:56 smithi099 bash[87147]: cluster 2024-09-13T15:11:55.067320+0000 mgr.y (mgr.26252) 1375 : cluster [DBG] pgmap v1953: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:56.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:56 smithi099 bash[87147]: cluster 2024-09-13T15:11:55.067320+0000 mgr.y (mgr.26252) 1375 : cluster [DBG] pgmap v1953: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:56 smithi177 bash[68594]: cluster 2024-09-13T15:11:55.067320+0000 mgr.y (mgr.26252) 1375 : cluster [DBG] pgmap v1953: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:56 smithi177 bash[68594]: cluster 2024-09-13T15:11:55.067320+0000 mgr.y (mgr.26252) 1375 : cluster [DBG] pgmap v1953: 89 pgs: 89 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:56.363 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:56.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:56.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:56.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:56.497 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:56.524 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:56.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:56.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:56.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:56.652 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:56.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:56.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:56.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:56.786 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:56.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:56.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:56.889 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:56.923 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:56.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:56.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:57.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:57.062 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:11:57.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:57.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:11:57.165 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:57.194 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:57.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:57.239 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:11:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:11:57] "GET /metrics HTTP/1.1" 200 36778 "" "Prometheus/2.51.0" 2024-09-13T15:11:57.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:57.309 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:57 smithi099 bash[84576]: cluster 2024-09-13T15:11:57.067874+0000 mgr.y (mgr.26252) 1376 : cluster [DBG] pgmap v1954: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:57.309 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:11:57 smithi099 bash[84576]: cluster 2024-09-13T15:11:57.067874+0000 mgr.y (mgr.26252) 1376 : cluster [DBG] pgmap v1954: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:57.309 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:57 smithi099 bash[87147]: cluster 2024-09-13T15:11:57.067874+0000 mgr.y (mgr.26252) 1376 : cluster [DBG] pgmap v1954: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:57.309 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:11:57 smithi099 bash[87147]: cluster 2024-09-13T15:11:57.067874+0000 mgr.y (mgr.26252) 1376 : cluster [DBG] pgmap v1954: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:57.322 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:57.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:57.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:57.407 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:57.407 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:57.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:57.439 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:57 smithi177 bash[68594]: cluster 2024-09-13T15:11:57.067874+0000 mgr.y (mgr.26252) 1376 : cluster [DBG] pgmap v1954: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:57.439 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:11:57 smithi177 bash[68594]: cluster 2024-09-13T15:11:57.067874+0000 mgr.y (mgr.26252) 1376 : cluster [DBG] pgmap v1954: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-13T15:11:57.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:57.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:57.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:57.561 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:57.581 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:11:57.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:57.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:11:57.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:57.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:57.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:57.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:57.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:57.844 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:57.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:57.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:57.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:57.983 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:57.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:58.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:58.116 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:58.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:58.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:58.221 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:58.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:58.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:58.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:58.362 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:58.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:58.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:58.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:58.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:58.532 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:58.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:58.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:58.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:58.670 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:11:58.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:58.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:11:58.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:58.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:11:58.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:58.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:11:58.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:58.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:11:58.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:58.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:11:59.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:59.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:11:59.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:59.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:11:59.176 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:59.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:59.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:59.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:59.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:59.338 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:59.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:59.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:59.441 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:59.446 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:59.446 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:59.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:59.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:59.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:59.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:59.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:59.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:59.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:59.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:11:59.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:59.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:11:59.863 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:59.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:11:59.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:11:59.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:11:59.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:00.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:00.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:00.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:00.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:00.140 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:00.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:00.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:00.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:00.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:00 smithi177 bash[68594]: cluster 2024-09-13T15:11:59.069776+0000 mgr.y (mgr.26252) 1377 : cluster [DBG] pgmap v1955: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:00 smithi177 bash[68594]: cluster 2024-09-13T15:11:59.069776+0000 mgr.y (mgr.26252) 1377 : cluster [DBG] pgmap v1955: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:00.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:12:00 smithi177 bash[61439]: ts=2024-09-13T15:12:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:12:00.388 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:00.397 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:00 smithi099 bash[84576]: cluster 2024-09-13T15:11:59.069776+0000 mgr.y (mgr.26252) 1377 : cluster [DBG] pgmap v1955: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:00.398 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:00 smithi099 bash[84576]: cluster 2024-09-13T15:11:59.069776+0000 mgr.y (mgr.26252) 1377 : cluster [DBG] pgmap v1955: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:00.398 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:00 smithi099 bash[87147]: cluster 2024-09-13T15:11:59.069776+0000 mgr.y (mgr.26252) 1377 : cluster [DBG] pgmap v1955: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:00.398 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:00 smithi099 bash[87147]: cluster 2024-09-13T15:11:59.069776+0000 mgr.y (mgr.26252) 1377 : cluster [DBG] pgmap v1955: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:00.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:00.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:00.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:00.540 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:00.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:00.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:00.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:00.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:00.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:00.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:00.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:00.814 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:00.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:00.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:00.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:00.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:00.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:01.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:01.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:01.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:01.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:01.188 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:01.214 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:01.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:01.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:01.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:01.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:01.454 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:01.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:01.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:01.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:01.585 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:01.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:01.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:01.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:01.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:01.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:01.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:01.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:01.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:01.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:01.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:01.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:02.005 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:02.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:02.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:02.137 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:02.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:02.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:02.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:02.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:02.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:02.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:02.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:02.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:02 smithi177 bash[68594]: cluster 2024-09-13T15:12:01.070384+0000 mgr.y (mgr.26252) 1378 : cluster [DBG] pgmap v1956: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T15:12:02.390 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:02 smithi177 bash[68594]: cluster 2024-09-13T15:12:01.070384+0000 mgr.y (mgr.26252) 1378 : cluster [DBG] pgmap v1956: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T15:12:02.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:02.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:02.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:02.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:02.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:02 smithi099 bash[84576]: cluster 2024-09-13T15:12:01.070384+0000 mgr.y (mgr.26252) 1378 : cluster [DBG] pgmap v1956: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T15:12:02.524 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:02 smithi099 bash[84576]: cluster 2024-09-13T15:12:01.070384+0000 mgr.y (mgr.26252) 1378 : cluster [DBG] pgmap v1956: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T15:12:02.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:02 smithi099 bash[87147]: cluster 2024-09-13T15:12:01.070384+0000 mgr.y (mgr.26252) 1378 : cluster [DBG] pgmap v1956: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T15:12:02.525 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:02 smithi099 bash[87147]: cluster 2024-09-13T15:12:01.070384+0000 mgr.y (mgr.26252) 1378 : cluster [DBG] pgmap v1956: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T15:12:02.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:02.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:02.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:02.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:02.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:02.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:02.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:02.786 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:02.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:02.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:02.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:02.966 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:02.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:03.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:03.090 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:03.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:03.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:03.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:03.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:03.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:03.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:03.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:03.343 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:03.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:03.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:03.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:03.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:03.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:03.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:03.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:03.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:03.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:03.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:03.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:03.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:03.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:03.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:03.845 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:03.876 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:03.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:03.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:03.979 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:04.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:04.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:04.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:04.122 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:04.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:04.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:04.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:04.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:04.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[84576]: cluster 2024-09-13T15:12:03.070772+0000 mgr.y (mgr.26252) 1379 : cluster [DBG] pgmap v1957: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-13T15:12:04.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[84576]: cluster 2024-09-13T15:12:03.070772+0000 mgr.y (mgr.26252) 1379 : cluster [DBG] pgmap v1957: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-13T15:12:04.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[84576]: audit 2024-09-13T15:12:03.319008+0000 mon.a (mon.0) 2536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:04.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[84576]: audit 2024-09-13T15:12:03.319008+0000 mon.a (mon.0) 2536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:04.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[84576]: audit 2024-09-13T15:12:03.324976+0000 mon.a (mon.0) 2537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:04.260 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[84576]: audit 2024-09-13T15:12:03.324976+0000 mon.a (mon.0) 2537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:04.260 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[87147]: cluster 2024-09-13T15:12:03.070772+0000 mgr.y (mgr.26252) 1379 : cluster [DBG] pgmap v1957: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-13T15:12:04.260 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[87147]: cluster 2024-09-13T15:12:03.070772+0000 mgr.y (mgr.26252) 1379 : cluster [DBG] pgmap v1957: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-13T15:12:04.260 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[87147]: audit 2024-09-13T15:12:03.319008+0000 mon.a (mon.0) 2536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:04.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[87147]: audit 2024-09-13T15:12:03.319008+0000 mon.a (mon.0) 2536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:04.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[87147]: audit 2024-09-13T15:12:03.324976+0000 mon.a (mon.0) 2537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:04.261 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:04 smithi099 bash[87147]: audit 2024-09-13T15:12:03.324976+0000 mon.a (mon.0) 2537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:04.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:04.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:04.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:04.385 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:04.419 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:04.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:04.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:04.523 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:04.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:04.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:04.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:04 smithi177 bash[68594]: cluster 2024-09-13T15:12:03.070772+0000 mgr.y (mgr.26252) 1379 : cluster [DBG] pgmap v1957: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-13T15:12:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:04 smithi177 bash[68594]: cluster 2024-09-13T15:12:03.070772+0000 mgr.y (mgr.26252) 1379 : cluster [DBG] pgmap v1957: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-13T15:12:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:04 smithi177 bash[68594]: audit 2024-09-13T15:12:03.319008+0000 mon.a (mon.0) 2536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:04 smithi177 bash[68594]: audit 2024-09-13T15:12:03.319008+0000 mon.a (mon.0) 2536 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:04 smithi177 bash[68594]: audit 2024-09-13T15:12:03.324976+0000 mon.a (mon.0) 2537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:04.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:04 smithi177 bash[68594]: audit 2024-09-13T15:12:03.324976+0000 mon.a (mon.0) 2537 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:04.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:04.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:04.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:04.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:04.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:04.817 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:04.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:04.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:04.922 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:04.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:04.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:04.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:05.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:05.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:05.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:05.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:05.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:05.207 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:05.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:05.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:05.311 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:05.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:05.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:05.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:05.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:05.456 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:05.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:05.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:05.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:05.583 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:05.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:05.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:05.686 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:05.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:05.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:05.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:05.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:05.847 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:05.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:05.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:05.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:05.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:05.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:05.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:06.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:06.107 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:06.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:06.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:06.212 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:06.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:06.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:06.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:06.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:06.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:06 smithi177 bash[68594]: cluster 2024-09-13T15:12:05.071359+0000 mgr.y (mgr.26252) 1380 : cluster [DBG] pgmap v1958: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-13T15:12:06.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:06 smithi177 bash[68594]: cluster 2024-09-13T15:12:05.071359+0000 mgr.y (mgr.26252) 1380 : cluster [DBG] pgmap v1958: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-13T15:12:06.382 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:06.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:06.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:06.485 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:06.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:06.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:06.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:06 smithi099 bash[84576]: cluster 2024-09-13T15:12:05.071359+0000 mgr.y (mgr.26252) 1380 : cluster [DBG] pgmap v1958: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-13T15:12:06.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:06 smithi099 bash[84576]: cluster 2024-09-13T15:12:05.071359+0000 mgr.y (mgr.26252) 1380 : cluster [DBG] pgmap v1958: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-13T15:12:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:06 smithi099 bash[87147]: cluster 2024-09-13T15:12:05.071359+0000 mgr.y (mgr.26252) 1380 : cluster [DBG] pgmap v1958: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-13T15:12:06.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:06 smithi099 bash[87147]: cluster 2024-09-13T15:12:05.071359+0000 mgr.y (mgr.26252) 1380 : cluster [DBG] pgmap v1958: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-13T15:12:06.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:06.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:06.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:06.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:06.739 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:06.777 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:06.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:06.883 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:06.901 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:06.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:06.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:07.005 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:07.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:07.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:07.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:07.131 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:07.174 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:07.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:07.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:07.177 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:12:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:12:07] "GET /metrics HTTP/1.1" 200 36780 "" "Prometheus/2.51.0" 2024-09-13T15:12:07.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:07.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:07.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:07.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:07.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:07.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:07.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:07.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:07.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:07.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:07.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:07.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:07.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:07.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:07.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:07.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:07.822 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:07.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:07.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:07.929 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:07.950 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:07.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:07.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:08.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:08.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:08.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:08.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:08.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:08.207 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:08.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:08.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:08.315 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:08.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:08.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:08.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:08 smithi177 bash[68594]: cluster 2024-09-13T15:12:07.071887+0000 mgr.y (mgr.26252) 1381 : cluster [DBG] pgmap v1959: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-13T15:12:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:08 smithi177 bash[68594]: cluster 2024-09-13T15:12:07.071887+0000 mgr.y (mgr.26252) 1381 : cluster [DBG] pgmap v1959: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-13T15:12:08.452 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:08.462 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:08 smithi099 bash[84576]: cluster 2024-09-13T15:12:07.071887+0000 mgr.y (mgr.26252) 1381 : cluster [DBG] pgmap v1959: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-13T15:12:08.462 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:08 smithi099 bash[84576]: cluster 2024-09-13T15:12:07.071887+0000 mgr.y (mgr.26252) 1381 : cluster [DBG] pgmap v1959: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-13T15:12:08.462 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:08 smithi099 bash[87147]: cluster 2024-09-13T15:12:07.071887+0000 mgr.y (mgr.26252) 1381 : cluster [DBG] pgmap v1959: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-13T15:12:08.463 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:08 smithi099 bash[87147]: cluster 2024-09-13T15:12:07.071887+0000 mgr.y (mgr.26252) 1381 : cluster [DBG] pgmap v1959: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-13T15:12:08.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:08.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:08.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:08.592 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:08.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:08.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:08.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:08.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:08.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:08.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:08.862 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:08.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:08.964 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:09.002 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:09.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:09.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:09.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:09.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:09.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:09.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:09.242 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:09.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:09.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:09.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:09.368 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:09.423 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:09.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:09.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:09.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:09.553 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:09.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:09.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:09.660 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:09.689 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:09.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:09.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:09.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:09.827 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:09.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:09.934 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:09.965 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:09.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:09.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:10.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:10.108 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:10.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:10.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:10.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:10 smithi177 bash[68594]: cluster 2024-09-13T15:12:09.072358+0000 mgr.y (mgr.26252) 1382 : cluster [DBG] pgmap v1960: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-09-13T15:12:10.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:10 smithi177 bash[68594]: cluster 2024-09-13T15:12:09.072358+0000 mgr.y (mgr.26252) 1382 : cluster [DBG] pgmap v1960: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-09-13T15:12:10.236 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:10.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:10.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:10.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:10 smithi099 bash[84576]: cluster 2024-09-13T15:12:09.072358+0000 mgr.y (mgr.26252) 1382 : cluster [DBG] pgmap v1960: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-09-13T15:12:10.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:10 smithi099 bash[84576]: cluster 2024-09-13T15:12:09.072358+0000 mgr.y (mgr.26252) 1382 : cluster [DBG] pgmap v1960: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-09-13T15:12:10.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:10 smithi099 bash[87147]: cluster 2024-09-13T15:12:09.072358+0000 mgr.y (mgr.26252) 1382 : cluster [DBG] pgmap v1960: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-09-13T15:12:10.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:10 smithi099 bash[87147]: cluster 2024-09-13T15:12:09.072358+0000 mgr.y (mgr.26252) 1382 : cluster [DBG] pgmap v1960: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-09-13T15:12:10.339 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:10.369 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:10.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:10.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:10.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:10.503 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:10.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:10.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:10.605 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:10.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:12:10 smithi177 bash[61439]: ts=2024-09-13T15:12:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:12:10.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:10.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:10.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:10.755 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:10.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:10.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:10.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:10.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:10.899 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:10.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:10.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:11.002 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:11.029 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:11.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:11.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:11.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:11.155 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:11.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:11.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:11.266 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:11.301 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:11.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:11.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:11.407 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:11.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:11.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:11.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:11.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:11.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:11.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:11.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:11.701 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:11.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:11.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:11.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:11.833 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:11.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:11.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:11.963 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:11.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:11.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:12.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:12.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:12.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:12.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:12.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:12.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:12.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:12.354 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:12.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:12.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:12 smithi177 bash[68594]: cluster 2024-09-13T15:12:11.072998+0000 mgr.y (mgr.26252) 1383 : cluster [DBG] pgmap v1961: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T15:12:12.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:12 smithi177 bash[68594]: cluster 2024-09-13T15:12:11.072998+0000 mgr.y (mgr.26252) 1383 : cluster [DBG] pgmap v1961: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T15:12:12.416 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:12 smithi099 bash[84576]: cluster 2024-09-13T15:12:11.072998+0000 mgr.y (mgr.26252) 1383 : cluster [DBG] pgmap v1961: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T15:12:12.416 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:12 smithi099 bash[84576]: cluster 2024-09-13T15:12:11.072998+0000 mgr.y (mgr.26252) 1383 : cluster [DBG] pgmap v1961: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T15:12:12.417 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:12 smithi099 bash[87147]: cluster 2024-09-13T15:12:11.072998+0000 mgr.y (mgr.26252) 1383 : cluster [DBG] pgmap v1961: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T15:12:12.417 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:12 smithi099 bash[87147]: cluster 2024-09-13T15:12:11.072998+0000 mgr.y (mgr.26252) 1383 : cluster [DBG] pgmap v1961: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-09-13T15:12:12.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:12.490 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:12.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:12.594 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:12.622 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:12.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:12.726 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:12.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:12.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:12.865 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:12.889 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:12.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:12.996 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:13.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:13.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:13.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:13.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:13.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:13.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:13.259 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:13.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:13.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:13.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:13.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:13.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:13.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:13.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:13.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:13.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:13.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:13.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:13.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:13.681 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:13.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:13.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:13.784 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:13.814 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:13.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:13.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:13.921 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:13.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:13.951 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:13.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:14.052 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:14.074 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:14.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:14.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:14.180 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:14.188 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:14 smithi177 bash[68594]: cluster 2024-09-13T15:12:13.075445+0000 mgr.y (mgr.26252) 1384 : cluster [DBG] pgmap v1962: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:12:14.188 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:14 smithi177 bash[68594]: cluster 2024-09-13T15:12:13.075445+0000 mgr.y (mgr.26252) 1384 : cluster [DBG] pgmap v1962: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:12:14.201 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:14.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:14.304 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:14.327 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:14.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:14.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:14.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:14.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:14.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:14.570 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:14.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:14 smithi099 bash[84576]: cluster 2024-09-13T15:12:13.075445+0000 mgr.y (mgr.26252) 1384 : cluster [DBG] pgmap v1962: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:12:14.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:14 smithi099 bash[84576]: cluster 2024-09-13T15:12:13.075445+0000 mgr.y (mgr.26252) 1384 : cluster [DBG] pgmap v1962: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:12:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:14 smithi099 bash[87147]: cluster 2024-09-13T15:12:13.075445+0000 mgr.y (mgr.26252) 1384 : cluster [DBG] pgmap v1962: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:12:14.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:14 smithi099 bash[87147]: cluster 2024-09-13T15:12:13.075445+0000 mgr.y (mgr.26252) 1384 : cluster [DBG] pgmap v1962: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:12:14.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:14.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:14.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:14.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:14.746 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:14.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:14.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:14.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:14.881 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:14.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:14.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:14.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:15.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:15.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:15.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:15.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:15.162 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:15.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:15.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:15.269 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:15.297 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:15.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:15.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:15.404 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:15.426 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:15.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:15.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:15.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:15.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:15.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:15.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:15.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:15.707 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:15.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:15.813 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:15.833 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:15.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:15.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:15.965 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:15.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:15.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:16.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:16.104 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:16.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:16.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:16.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:16.244 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:16.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:16.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:16.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:16.360 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:16 smithi099 bash[84576]: cluster 2024-09-13T15:12:15.076198+0000 mgr.y (mgr.26252) 1385 : cluster [DBG] pgmap v1963: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2024-09-13T15:12:16.360 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:16 smithi099 bash[84576]: cluster 2024-09-13T15:12:15.076198+0000 mgr.y (mgr.26252) 1385 : cluster [DBG] pgmap v1963: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2024-09-13T15:12:16.360 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:16 smithi099 bash[87147]: cluster 2024-09-13T15:12:15.076198+0000 mgr.y (mgr.26252) 1385 : cluster [DBG] pgmap v1963: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2024-09-13T15:12:16.360 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:16 smithi099 bash[87147]: cluster 2024-09-13T15:12:15.076198+0000 mgr.y (mgr.26252) 1385 : cluster [DBG] pgmap v1963: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2024-09-13T15:12:16.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:16 smithi177 bash[68594]: cluster 2024-09-13T15:12:15.076198+0000 mgr.y (mgr.26252) 1385 : cluster [DBG] pgmap v1963: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2024-09-13T15:12:16.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:16 smithi177 bash[68594]: cluster 2024-09-13T15:12:15.076198+0000 mgr.y (mgr.26252) 1385 : cluster [DBG] pgmap v1963: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2024-09-13T15:12:16.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:16.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:16.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:16.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:16.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:16.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:16.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:16.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:16.652 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:16.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:16.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:16.758 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:16.786 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:16.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:16.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:16.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:16.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:16.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:17.018 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:17.041 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:17.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:17.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:17.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:17.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:17.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:17.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:17.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:17.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:17.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:17.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:12:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:12:17] "GET /metrics HTTP/1.1" 200 36780 "" "Prometheus/2.51.0" 2024-09-13T15:12:17.396 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:17.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:17.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:17.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:17.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:17.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:17.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:17.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:17.664 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:17.689 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:17.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:17.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:17.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:17.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:17.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:17.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:17.932 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:17.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:17.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:17.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:18.066 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:18.100 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:18.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:18.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:18.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:18.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:18.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:18.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:18.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:18 smithi099 bash[84576]: cluster 2024-09-13T15:12:17.077024+0000 mgr.y (mgr.26252) 1386 : cluster [DBG] pgmap v1964: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2024-09-13T15:12:18.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:18 smithi099 bash[84576]: cluster 2024-09-13T15:12:17.077024+0000 mgr.y (mgr.26252) 1386 : cluster [DBG] pgmap v1964: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2024-09-13T15:12:18.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:18 smithi099 bash[87147]: cluster 2024-09-13T15:12:17.077024+0000 mgr.y (mgr.26252) 1386 : cluster [DBG] pgmap v1964: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2024-09-13T15:12:18.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:18 smithi099 bash[87147]: cluster 2024-09-13T15:12:17.077024+0000 mgr.y (mgr.26252) 1386 : cluster [DBG] pgmap v1964: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2024-09-13T15:12:18.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:18.362 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:18.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:18.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:18.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:18.524 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:18.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:18.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:18.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:18 smithi177 bash[68594]: cluster 2024-09-13T15:12:17.077024+0000 mgr.y (mgr.26252) 1386 : cluster [DBG] pgmap v1964: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2024-09-13T15:12:18.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:18 smithi177 bash[68594]: cluster 2024-09-13T15:12:17.077024+0000 mgr.y (mgr.26252) 1386 : cluster [DBG] pgmap v1964: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2024-09-13T15:12:18.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:18.656 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:18.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:18.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:18.759 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:18.796 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:18.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:18.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:18.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:18.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:18.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:19.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:19.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:19.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:19.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:19.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:19.187 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:19.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:19.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:19.191 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T15:12:19.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:19.299 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:19 smithi177 bash[68594]: audit 2024-09-13T15:12:18.309750+0000 mon.a (mon.0) 2538 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:19.299 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:19 smithi177 bash[68594]: audit 2024-09-13T15:12:18.309750+0000 mon.a (mon.0) 2538 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:19.299 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:19 smithi177 bash[68594]: audit 2024-09-13T15:12:19.162749+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.99:0/3387862211' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.299 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:19 smithi177 bash[68594]: audit 2024-09-13T15:12:19.162749+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.99:0/3387862211' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.299 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:19 smithi177 bash[68594]: audit 2024-09-13T15:12:19.162809+0000 mon.a (mon.0) 2539 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.299 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:19 smithi177 bash[68594]: audit 2024-09-13T15:12:19.162809+0000 mon.a (mon.0) 2539 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.312 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:19.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:19.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[84576]: audit 2024-09-13T15:12:18.309750+0000 mon.a (mon.0) 2538 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[84576]: audit 2024-09-13T15:12:18.309750+0000 mon.a (mon.0) 2538 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[84576]: audit 2024-09-13T15:12:19.162749+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.99:0/3387862211' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[84576]: audit 2024-09-13T15:12:19.162749+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.99:0/3387862211' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[84576]: audit 2024-09-13T15:12:19.162809+0000 mon.a (mon.0) 2539 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[84576]: audit 2024-09-13T15:12:19.162809+0000 mon.a (mon.0) 2539 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[87147]: audit 2024-09-13T15:12:18.309750+0000 mon.a (mon.0) 2538 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[87147]: audit 2024-09-13T15:12:18.309750+0000 mon.a (mon.0) 2538 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[87147]: audit 2024-09-13T15:12:19.162749+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.99:0/3387862211' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[87147]: audit 2024-09-13T15:12:19.162749+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.99:0/3387862211' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[87147]: audit 2024-09-13T15:12:19.162809+0000 mon.a (mon.0) 2539 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:19 smithi099 bash[87147]: audit 2024-09-13T15:12:19.162809+0000 mon.a (mon.0) 2539 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:12:19.415 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:19.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:19.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:19.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:19.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:19.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:19.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:19.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:19.695 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:19.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:19.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:19.801 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:19.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:19.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:19.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:19.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:19.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:19.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:19.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:20.056 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:20.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:20.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:20.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:20.182 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:20.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:20 smithi177 bash[68594]: cluster 2024-09-13T15:12:19.077577+0000 mgr.y (mgr.26252) 1387 : cluster [DBG] pgmap v1965: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:20.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:20 smithi177 bash[68594]: cluster 2024-09-13T15:12:19.077577+0000 mgr.y (mgr.26252) 1387 : cluster [DBG] pgmap v1965: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:20.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:20 smithi177 bash[68594]: cluster 2024-09-13T15:12:19.174894+0000 mon.a (mon.0) 2540 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:20.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:20 smithi177 bash[68594]: cluster 2024-09-13T15:12:19.174894+0000 mon.a (mon.0) 2540 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:20.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:20 smithi177 bash[68594]: audit 2024-09-13T15:12:19.185735+0000 mon.a (mon.0) 2541 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:12:20.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:20 smithi177 bash[68594]: audit 2024-09-13T15:12:19.185735+0000 mon.a (mon.0) 2541 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:12:20.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:20 smithi177 bash[68594]: cluster 2024-09-13T15:12:19.195695+0000 mon.a (mon.0) 2542 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2024-09-13T15:12:20.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:20 smithi177 bash[68594]: cluster 2024-09-13T15:12:19.195695+0000 mon.a (mon.0) 2542 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2024-09-13T15:12:20.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:20.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:20.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:20.322 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:20.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[84576]: cluster 2024-09-13T15:12:19.077577+0000 mgr.y (mgr.26252) 1387 : cluster [DBG] pgmap v1965: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:20.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[84576]: cluster 2024-09-13T15:12:19.077577+0000 mgr.y (mgr.26252) 1387 : cluster [DBG] pgmap v1965: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:20.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[84576]: cluster 2024-09-13T15:12:19.174894+0000 mon.a (mon.0) 2540 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:20.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[84576]: cluster 2024-09-13T15:12:19.174894+0000 mon.a (mon.0) 2540 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:20.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[84576]: audit 2024-09-13T15:12:19.185735+0000 mon.a (mon.0) 2541 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:12:20.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[84576]: audit 2024-09-13T15:12:19.185735+0000 mon.a (mon.0) 2541 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:12:20.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[84576]: cluster 2024-09-13T15:12:19.195695+0000 mon.a (mon.0) 2542 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2024-09-13T15:12:20.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[84576]: cluster 2024-09-13T15:12:19.195695+0000 mon.a (mon.0) 2542 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2024-09-13T15:12:20.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[87147]: cluster 2024-09-13T15:12:19.077577+0000 mgr.y (mgr.26252) 1387 : cluster [DBG] pgmap v1965: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:20.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[87147]: cluster 2024-09-13T15:12:19.077577+0000 mgr.y (mgr.26252) 1387 : cluster [DBG] pgmap v1965: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:20.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[87147]: cluster 2024-09-13T15:12:19.174894+0000 mon.a (mon.0) 2540 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:20.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[87147]: cluster 2024-09-13T15:12:19.174894+0000 mon.a (mon.0) 2540 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:20.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[87147]: audit 2024-09-13T15:12:19.185735+0000 mon.a (mon.0) 2541 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:12:20.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[87147]: audit 2024-09-13T15:12:19.185735+0000 mon.a (mon.0) 2541 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:12:20.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[87147]: cluster 2024-09-13T15:12:19.195695+0000 mon.a (mon.0) 2542 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2024-09-13T15:12:20.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:20 smithi099 bash[87147]: cluster 2024-09-13T15:12:19.195695+0000 mon.a (mon.0) 2542 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2024-09-13T15:12:20.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:20.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:20.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:20.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:20.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:20.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:20.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:20.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:20.600 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:12:20 smithi177 bash[61439]: ts=2024-09-13T15:12:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:12:20.617 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:20.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:20.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:20.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:20.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:20.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:20.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:20.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:20.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:20.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:21.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:21.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:21.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 reset_purged_snaps_last 2024-09-13T15:12:21.079 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:12:21.080 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:12:21.107 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:12:21.107 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:12:21.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:21.159 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:12:21.159 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:12:21.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:21.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:21.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:21.209 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:12:21.209 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:12:21.237 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:12:21.237 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:12:21.253 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:12:21.253 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:12:21.266 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:21.269 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:12:21.269 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:12:21.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:21.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:21.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:21.290 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:12:21.291 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:12:21.348 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:12:21.349 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:12:21.349 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:12:21.349 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:12:21.349 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:12:21.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:21.425 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:21.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:21.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:21.534 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:21.543 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:21 smithi177 bash[68594]: audit 2024-09-13T15:12:21.058955+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.99:0/1856490559' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:21.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:21 smithi177 bash[68594]: audit 2024-09-13T15:12:21.058955+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.99:0/1856490559' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:21.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:21.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:21.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:21.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:21.673 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:21 smithi099 bash[84576]: audit 2024-09-13T15:12:21.058955+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.99:0/1856490559' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:21.673 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:21 smithi099 bash[84576]: audit 2024-09-13T15:12:21.058955+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.99:0/1856490559' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:21.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:21 smithi099 bash[87147]: audit 2024-09-13T15:12:21.058955+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.99:0/1856490559' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:21.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:21 smithi099 bash[87147]: audit 2024-09-13T15:12:21.058955+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.99:0/1856490559' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:21.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:21.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:21.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:21.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:21.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:21.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:21.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:21.926 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:21.959 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:21.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:21.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:22.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:22.091 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:22.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:22.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:22.194 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:22.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:22.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:22.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:22.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:22 smithi099 bash[84576]: cluster 2024-09-13T15:12:21.078103+0000 mgr.y (mgr.26252) 1388 : cluster [DBG] pgmap v1967: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-09-13T15:12:22.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:22 smithi099 bash[84576]: cluster 2024-09-13T15:12:21.078103+0000 mgr.y (mgr.26252) 1388 : cluster [DBG] pgmap v1967: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-09-13T15:12:22.322 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:22.331 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:22 smithi177 bash[68594]: cluster 2024-09-13T15:12:21.078103+0000 mgr.y (mgr.26252) 1388 : cluster [DBG] pgmap v1967: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-09-13T15:12:22.331 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:22 smithi177 bash[68594]: cluster 2024-09-13T15:12:21.078103+0000 mgr.y (mgr.26252) 1388 : cluster [DBG] pgmap v1967: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-09-13T15:12:22.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:22.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:22.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:22.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:22.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:22.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:22.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:22.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:22 smithi099 bash[87147]: cluster 2024-09-13T15:12:21.078103+0000 mgr.y (mgr.26252) 1388 : cluster [DBG] pgmap v1967: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-09-13T15:12:22.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:22 smithi099 bash[87147]: cluster 2024-09-13T15:12:21.078103+0000 mgr.y (mgr.26252) 1388 : cluster [DBG] pgmap v1967: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-09-13T15:12:22.612 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:22.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:22.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:22.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:22.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:22.782 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:22.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:22.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:22.885 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:22.923 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:22.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:22.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:23.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:23.046 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:23.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:23.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:23.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:23.181 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:23.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:23.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:23.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:23.304 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:23.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:23.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:23.407 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:23.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:23 smithi099 bash[84576]: cluster 2024-09-13T15:12:23.078504+0000 mgr.y (mgr.26252) 1389 : cluster [DBG] pgmap v1968: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:23.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:23 smithi099 bash[84576]: cluster 2024-09-13T15:12:23.078504+0000 mgr.y (mgr.26252) 1389 : cluster [DBG] pgmap v1968: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:23.417 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:23 smithi099 bash[87147]: cluster 2024-09-13T15:12:23.078504+0000 mgr.y (mgr.26252) 1389 : cluster [DBG] pgmap v1968: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:23.418 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:23 smithi099 bash[87147]: cluster 2024-09-13T15:12:23.078504+0000 mgr.y (mgr.26252) 1389 : cluster [DBG] pgmap v1968: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:23.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:23.437 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:23.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:23.538 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:23.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:23 smithi177 bash[68594]: cluster 2024-09-13T15:12:23.078504+0000 mgr.y (mgr.26252) 1389 : cluster [DBG] pgmap v1968: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:23.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:23 smithi177 bash[68594]: cluster 2024-09-13T15:12:23.078504+0000 mgr.y (mgr.26252) 1389 : cluster [DBG] pgmap v1968: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T15:12:23.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:23.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:23.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:23.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:23.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:23.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:23.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:23.793 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:23.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:23.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:23.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:23.940 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:23.966 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:23.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:23.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:24.071 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:24.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:24.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:24.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:24.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:24.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:24.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:24.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:24.338 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:24.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:24.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:24.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:24.483 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:24.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:24.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:24.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:24.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:24.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:24.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:24.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:24.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:24.789 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:24.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:24.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:24.895 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:24.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:24.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:24.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:25.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:25.059 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:25.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:25.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:25.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:25.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:25.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:25.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:25.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T15:12:25.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:25.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:25.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:25.424 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:25.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:25.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:25.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:25.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:25.588 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:25.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:25.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:25.691 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:25.719 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:25.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:25.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:25.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:25.854 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:25.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:25.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:25.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:12:25.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:26.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:26.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:26.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:26.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:26.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:26.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:26.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:26.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:26.295 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:26.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:26.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:26.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:26 smithi099 bash[84576]: cluster 2024-09-13T15:12:25.079165+0000 mgr.y (mgr.26252) 1390 : cluster [DBG] pgmap v1969: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-09-13T15:12:26.407 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:26 smithi099 bash[84576]: cluster 2024-09-13T15:12:25.079165+0000 mgr.y (mgr.26252) 1390 : cluster [DBG] pgmap v1969: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-09-13T15:12:26.408 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:26 smithi099 bash[87147]: cluster 2024-09-13T15:12:25.079165+0000 mgr.y (mgr.26252) 1390 : cluster [DBG] pgmap v1969: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-09-13T15:12:26.408 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:26 smithi099 bash[87147]: cluster 2024-09-13T15:12:25.079165+0000 mgr.y (mgr.26252) 1390 : cluster [DBG] pgmap v1969: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-09-13T15:12:26.433 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:26.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:26.540 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:26.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:26.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:26.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:26.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:26 smithi177 bash[68594]: cluster 2024-09-13T15:12:25.079165+0000 mgr.y (mgr.26252) 1390 : cluster [DBG] pgmap v1969: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-09-13T15:12:26.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:26 smithi177 bash[68594]: cluster 2024-09-13T15:12:25.079165+0000 mgr.y (mgr.26252) 1390 : cluster [DBG] pgmap v1969: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-09-13T15:12:26.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:26.705 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:26.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:26.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:26.831 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:26.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:26.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:26.936 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:26.973 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:26.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:26.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:27.080 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:27.113 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:27.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:27.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:27.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:27.231 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:12:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:12:27] "GET /metrics HTTP/1.1" 200 36778 "" "Prometheus/2.51.0" 2024-09-13T15:12:27.251 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:27.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:27.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:27.330 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:12:27.356 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:27.397 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:27.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:27.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:27.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:27.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:27.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:27.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:27.635 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:27.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:27.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:27.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:27.775 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:27.801 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:27.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:27.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:27.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:27.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:27.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:28.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:28.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:28.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:28.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:28.131 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:12:28.166 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:28.173 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:28 smithi177 bash[68594]: cluster 2024-09-13T15:12:27.079662+0000 mgr.y (mgr.26252) 1391 : cluster [DBG] pgmap v1970: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-13T15:12:28.173 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:28 smithi177 bash[68594]: cluster 2024-09-13T15:12:27.079662+0000 mgr.y (mgr.26252) 1391 : cluster [DBG] pgmap v1970: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-13T15:12:28.173 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:28 smithi177 bash[68594]: audit 2024-09-13T15:12:27.571948+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:12:28.173 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:28 smithi177 bash[68594]: audit 2024-09-13T15:12:27.571948+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:12:28.204 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:28.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:28.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:28.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:28.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:28.434 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:28.444 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:28 smithi099 bash[84576]: cluster 2024-09-13T15:12:27.079662+0000 mgr.y (mgr.26252) 1391 : cluster [DBG] pgmap v1970: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-13T15:12:28.445 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:28 smithi099 bash[84576]: cluster 2024-09-13T15:12:27.079662+0000 mgr.y (mgr.26252) 1391 : cluster [DBG] pgmap v1970: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-13T15:12:28.445 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:28 smithi099 bash[84576]: audit 2024-09-13T15:12:27.571948+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:12:28.445 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:28 smithi099 bash[84576]: audit 2024-09-13T15:12:27.571948+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:12:28.445 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:28 smithi099 bash[87147]: cluster 2024-09-13T15:12:27.079662+0000 mgr.y (mgr.26252) 1391 : cluster [DBG] pgmap v1970: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-13T15:12:28.445 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:28 smithi099 bash[87147]: cluster 2024-09-13T15:12:27.079662+0000 mgr.y (mgr.26252) 1391 : cluster [DBG] pgmap v1970: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-09-13T15:12:28.445 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:28 smithi099 bash[87147]: audit 2024-09-13T15:12:27.571948+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:12:28.445 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:28 smithi099 bash[87147]: audit 2024-09-13T15:12:27.571948+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:12:28.462 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:28.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:28.565 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:28.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:28.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:28.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:28.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:28.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:28.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:28.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:28.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:28.862 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:28.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:28.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:28.967 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:28.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:28.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:28.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:29.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:29.116 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:29.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:29.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:29.219 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:29.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:29.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:29.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:29.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:29.377 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:29.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:29.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:29.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:29.511 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:29.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:29.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:29.651 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:29.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:29.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:29.755 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:29.788 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:29.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:29.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:29.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:29.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:29.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:30.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:30.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:30.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:30.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:30.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:30.178 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:12:30.180 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:12:30.205 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:30.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:30.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:30.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:30 smithi099 bash[84576]: cluster 2024-09-13T15:12:29.080360+0000 mgr.y (mgr.26252) 1392 : cluster [DBG] pgmap v1971: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:12:30.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:30 smithi099 bash[84576]: cluster 2024-09-13T15:12:29.080360+0000 mgr.y (mgr.26252) 1392 : cluster [DBG] pgmap v1971: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:12:30.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:30 smithi099 bash[87147]: cluster 2024-09-13T15:12:29.080360+0000 mgr.y (mgr.26252) 1392 : cluster [DBG] pgmap v1971: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:12:30.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:30 smithi099 bash[87147]: cluster 2024-09-13T15:12:29.080360+0000 mgr.y (mgr.26252) 1392 : cluster [DBG] pgmap v1971: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:12:30.354 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:30.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:30.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:30.459 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:30.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:30 smithi177 bash[68594]: cluster 2024-09-13T15:12:29.080360+0000 mgr.y (mgr.26252) 1392 : cluster [DBG] pgmap v1971: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:12:30.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:30 smithi177 bash[68594]: cluster 2024-09-13T15:12:29.080360+0000 mgr.y (mgr.26252) 1392 : cluster [DBG] pgmap v1971: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T15:12:30.468 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:12:30 smithi177 bash[61439]: ts=2024-09-13T15:12:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:12:30.484 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:30.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:30.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:30.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:30.611 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:30.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:30.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:30.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:30.738 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:30.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:30.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:30.876 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:30.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:30.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:30.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:31.011 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:31.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:31.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:31.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:31.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:31.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:31.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:31.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:31.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:31.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:31.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:31.363 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:31.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:31.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:31.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:31.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:31.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:31.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:31.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:31.653 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:31.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:31.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:31.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:31.780 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:31.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:31.883 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:31.913 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:31.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:31.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:32.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:32.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:32.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:32.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:32.148 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:32.171 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:32.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:32.222 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:12:32.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:32.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:32.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:32.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:32.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:32.431 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:32 smithi177 bash[68594]: cluster 2024-09-13T15:12:31.081008+0000 mgr.y (mgr.26252) 1393 : cluster [DBG] pgmap v1972: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-09-13T15:12:32.432 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:32 smithi177 bash[68594]: cluster 2024-09-13T15:12:31.081008+0000 mgr.y (mgr.26252) 1393 : cluster [DBG] pgmap v1972: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-09-13T15:12:32.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:32.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:32.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:32.603 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:32.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:32.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:32.646 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:32 smithi099 bash[84576]: cluster 2024-09-13T15:12:31.081008+0000 mgr.y (mgr.26252) 1393 : cluster [DBG] pgmap v1972: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-09-13T15:12:32.647 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:32 smithi099 bash[84576]: cluster 2024-09-13T15:12:31.081008+0000 mgr.y (mgr.26252) 1393 : cluster [DBG] pgmap v1972: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-09-13T15:12:32.647 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:32 smithi099 bash[87147]: cluster 2024-09-13T15:12:31.081008+0000 mgr.y (mgr.26252) 1393 : cluster [DBG] pgmap v1972: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-09-13T15:12:32.647 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:32 smithi099 bash[87147]: cluster 2024-09-13T15:12:31.081008+0000 mgr.y (mgr.26252) 1393 : cluster [DBG] pgmap v1972: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-09-13T15:12:32.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:32.725 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:32.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:32.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:32.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:32.853 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:32.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:32.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:32.957 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:32.986 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:32.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:32.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:33.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:33.118 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:33.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:33.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:33.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:33.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:33.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:33.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:33.425 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:33.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:33.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:33.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:33.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:33.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:33.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:33.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:33.705 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:33.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:33.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:33.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:33.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:33.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:33.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:33.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:33.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:33.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:33.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:34.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:34.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:34.120 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:34.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:34.222 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:34.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: cluster 2024-09-13T15:12:33.081499+0000 mgr.y (mgr.26252) 1394 : cluster [DBG] pgmap v1973: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: cluster 2024-09-13T15:12:33.081499+0000 mgr.y (mgr.26252) 1394 : cluster [DBG] pgmap v1973: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: audit 2024-09-13T15:12:33.309683+0000 mon.a (mon.0) 2544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: audit 2024-09-13T15:12:33.309683+0000 mon.a (mon.0) 2544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: audit 2024-09-13T15:12:33.517487+0000 mon.a (mon.0) 2545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: audit 2024-09-13T15:12:33.517487+0000 mon.a (mon.0) 2545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: audit 2024-09-13T15:12:33.526914+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: audit 2024-09-13T15:12:33.526914+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: audit 2024-09-13T15:12:34.119523+0000 mon.a (mon.0) 2547 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: audit 2024-09-13T15:12:34.119523+0000 mon.a (mon.0) 2547 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: audit 2024-09-13T15:12:34.133053+0000 mon.a (mon.0) 2548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:34 smithi177 bash[68594]: audit 2024-09-13T15:12:34.133053+0000 mon.a (mon.0) 2548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:34.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:34.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:34.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:34.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:34.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:34.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:34.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:34.521 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:34.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:34.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:34.625 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:34.626 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: cluster 2024-09-13T15:12:33.081499+0000 mgr.y (mgr.26252) 1394 : cluster [DBG] pgmap v1973: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2024-09-13T15:12:34.626 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: cluster 2024-09-13T15:12:33.081499+0000 mgr.y (mgr.26252) 1394 : cluster [DBG] pgmap v1973: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2024-09-13T15:12:34.626 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: audit 2024-09-13T15:12:33.309683+0000 mon.a (mon.0) 2544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:34.626 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: audit 2024-09-13T15:12:33.309683+0000 mon.a (mon.0) 2544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:34.627 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: audit 2024-09-13T15:12:33.517487+0000 mon.a (mon.0) 2545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.627 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: audit 2024-09-13T15:12:33.517487+0000 mon.a (mon.0) 2545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.627 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: audit 2024-09-13T15:12:33.526914+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.627 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: audit 2024-09-13T15:12:33.526914+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.627 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: audit 2024-09-13T15:12:34.119523+0000 mon.a (mon.0) 2547 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.627 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: audit 2024-09-13T15:12:34.119523+0000 mon.a (mon.0) 2547 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.627 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: audit 2024-09-13T15:12:34.133053+0000 mon.a (mon.0) 2548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.627 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[84576]: audit 2024-09-13T15:12:34.133053+0000 mon.a (mon.0) 2548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.627 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: cluster 2024-09-13T15:12:33.081499+0000 mgr.y (mgr.26252) 1394 : cluster [DBG] pgmap v1973: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: cluster 2024-09-13T15:12:33.081499+0000 mgr.y (mgr.26252) 1394 : cluster [DBG] pgmap v1973: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: audit 2024-09-13T15:12:33.309683+0000 mon.a (mon.0) 2544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: audit 2024-09-13T15:12:33.309683+0000 mon.a (mon.0) 2544 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: audit 2024-09-13T15:12:33.517487+0000 mon.a (mon.0) 2545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: audit 2024-09-13T15:12:33.517487+0000 mon.a (mon.0) 2545 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: audit 2024-09-13T15:12:33.526914+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: audit 2024-09-13T15:12:33.526914+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: audit 2024-09-13T15:12:34.119523+0000 mon.a (mon.0) 2547 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: audit 2024-09-13T15:12:34.119523+0000 mon.a (mon.0) 2547 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: audit 2024-09-13T15:12:34.133053+0000 mon.a (mon.0) 2548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.628 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:34 smithi099 bash[87147]: audit 2024-09-13T15:12:34.133053+0000 mon.a (mon.0) 2548 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:34.652 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:34.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:34.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:34.759 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:34.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:34.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:34.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:34.887 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:34.939 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:34.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:35.048 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:35.070 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:35.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:35.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:35.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:35.210 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:35.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:35.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:35.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:35.348 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:35.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:35.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:35 smithi177 bash[68594]: audit 2024-09-13T15:12:34.885890+0000 mon.a (mon.0) 2549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:12:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:35 smithi177 bash[68594]: audit 2024-09-13T15:12:34.885890+0000 mon.a (mon.0) 2549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:12:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:35 smithi177 bash[68594]: audit 2024-09-13T15:12:34.887530+0000 mon.a (mon.0) 2550 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:12:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:35 smithi177 bash[68594]: audit 2024-09-13T15:12:34.887530+0000 mon.a (mon.0) 2550 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:12:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:35 smithi177 bash[68594]: audit 2024-09-13T15:12:34.897239+0000 mon.a (mon.0) 2551 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:35.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:35 smithi177 bash[68594]: audit 2024-09-13T15:12:34.897239+0000 mon.a (mon.0) 2551 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:35.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:35.460 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[84576]: audit 2024-09-13T15:12:34.885890+0000 mon.a (mon.0) 2549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:12:35.461 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[84576]: audit 2024-09-13T15:12:34.885890+0000 mon.a (mon.0) 2549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:12:35.461 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[84576]: audit 2024-09-13T15:12:34.887530+0000 mon.a (mon.0) 2550 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:12:35.461 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[84576]: audit 2024-09-13T15:12:34.887530+0000 mon.a (mon.0) 2550 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:12:35.461 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[84576]: audit 2024-09-13T15:12:34.897239+0000 mon.a (mon.0) 2551 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:35.461 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[84576]: audit 2024-09-13T15:12:34.897239+0000 mon.a (mon.0) 2551 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:35.461 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[87147]: audit 2024-09-13T15:12:34.885890+0000 mon.a (mon.0) 2549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:12:35.461 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[87147]: audit 2024-09-13T15:12:34.885890+0000 mon.a (mon.0) 2549 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:12:35.461 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[87147]: audit 2024-09-13T15:12:34.887530+0000 mon.a (mon.0) 2550 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:12:35.461 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[87147]: audit 2024-09-13T15:12:34.887530+0000 mon.a (mon.0) 2550 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:12:35.461 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[87147]: audit 2024-09-13T15:12:34.897239+0000 mon.a (mon.0) 2551 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:35.462 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:35 smithi099 bash[87147]: audit 2024-09-13T15:12:34.897239+0000 mon.a (mon.0) 2551 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:12:35.482 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:35.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:35.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:35.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:35.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:35.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:35.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:35.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:35.736 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:35.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:35.839 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:35.877 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:35.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:35.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:35.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:36.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:36.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:36.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:36.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:36.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:36.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:36.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:36.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:36.289 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:36 smithi099 bash[84576]: cluster 2024-09-13T15:12:35.082249+0000 mgr.y (mgr.26252) 1395 : cluster [DBG] pgmap v1974: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-13T15:12:36.289 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:36 smithi099 bash[84576]: cluster 2024-09-13T15:12:35.082249+0000 mgr.y (mgr.26252) 1395 : cluster [DBG] pgmap v1974: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-13T15:12:36.290 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:36 smithi099 bash[87147]: cluster 2024-09-13T15:12:35.082249+0000 mgr.y (mgr.26252) 1395 : cluster [DBG] pgmap v1974: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-13T15:12:36.290 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:36 smithi099 bash[87147]: cluster 2024-09-13T15:12:35.082249+0000 mgr.y (mgr.26252) 1395 : cluster [DBG] pgmap v1974: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-13T15:12:36.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:36.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:36.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:36.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:36.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:36.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:36.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:36.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:36 smithi177 bash[68594]: cluster 2024-09-13T15:12:35.082249+0000 mgr.y (mgr.26252) 1395 : cluster [DBG] pgmap v1974: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-13T15:12:36.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:36 smithi177 bash[68594]: cluster 2024-09-13T15:12:35.082249+0000 mgr.y (mgr.26252) 1395 : cluster [DBG] pgmap v1974: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-09-13T15:12:36.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:36.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:36.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:36.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:36.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:36.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:36.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:36.811 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:36.832 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:36.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:36.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:36.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:36.977 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:36.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:36.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:37.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:37.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:37.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:37.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:37.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:37.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:37.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:37.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:37.352 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:37.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:37.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:37.367 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:12:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:12:37] "GET /metrics HTTP/1.1" 200 36778 "" "Prometheus/2.51.0" 2024-09-13T15:12:37.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:37.498 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:37.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:37.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:37.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:37.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:37.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:37.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:37.739 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:37.763 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:37.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:37.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:37.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:37.904 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:37.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:37.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:38.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:38.030 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:38.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:38.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:38.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:38.164 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:38.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:38.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:38.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:38.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:38.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:38.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:38.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:38 smithi177 bash[68594]: cluster 2024-09-13T15:12:37.082771+0000 mgr.y (mgr.26252) 1396 : cluster [DBG] pgmap v1975: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:38.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:38 smithi177 bash[68594]: cluster 2024-09-13T15:12:37.082771+0000 mgr.y (mgr.26252) 1396 : cluster [DBG] pgmap v1975: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:38.392 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:38.415 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:38.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:38.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:38.518 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:38.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:38.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:38.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:38.638 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:38.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:38.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:38.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:38.674 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:38 smithi099 bash[84576]: cluster 2024-09-13T15:12:37.082771+0000 mgr.y (mgr.26252) 1396 : cluster [DBG] pgmap v1975: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:38.674 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:38 smithi099 bash[84576]: cluster 2024-09-13T15:12:37.082771+0000 mgr.y (mgr.26252) 1396 : cluster [DBG] pgmap v1975: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:38.675 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:38 smithi099 bash[87147]: cluster 2024-09-13T15:12:37.082771+0000 mgr.y (mgr.26252) 1396 : cluster [DBG] pgmap v1975: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:38.675 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:38 smithi099 bash[87147]: cluster 2024-09-13T15:12:37.082771+0000 mgr.y (mgr.26252) 1396 : cluster [DBG] pgmap v1975: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-09-13T15:12:38.769 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:38.805 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:38.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:38.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:38.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:38.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:38.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:38.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:39.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:39.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:39.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:39.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:39.184 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:39.202 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:39.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:39.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:39.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:39.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:39.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:39.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:39.432 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:39.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:39.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:39.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:39.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:39.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:39.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:39.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:39.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:39.749 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:39.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:39.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:39.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:39.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:39.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:39.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:39.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:40.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:40.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:40.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:40.115 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:40.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:40.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:40.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:40.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:40.269 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:40.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:40.371 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:40.379 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:40 smithi099 bash[84576]: cluster 2024-09-13T15:12:39.083271+0000 mgr.y (mgr.26252) 1397 : cluster [DBG] pgmap v1976: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2024-09-13T15:12:40.379 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:40 smithi099 bash[84576]: cluster 2024-09-13T15:12:39.083271+0000 mgr.y (mgr.26252) 1397 : cluster [DBG] pgmap v1976: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2024-09-13T15:12:40.380 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:40 smithi099 bash[87147]: cluster 2024-09-13T15:12:39.083271+0000 mgr.y (mgr.26252) 1397 : cluster [DBG] pgmap v1976: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2024-09-13T15:12:40.380 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:40 smithi099 bash[87147]: cluster 2024-09-13T15:12:39.083271+0000 mgr.y (mgr.26252) 1397 : cluster [DBG] pgmap v1976: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2024-09-13T15:12:40.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:40.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:40.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:40.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:40.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:40.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:40.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:12:40 smithi177 bash[61439]: ts=2024-09-13T15:12:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:12:40.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:40 smithi177 bash[68594]: cluster 2024-09-13T15:12:39.083271+0000 mgr.y (mgr.26252) 1397 : cluster [DBG] pgmap v1976: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2024-09-13T15:12:40.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:40 smithi177 bash[68594]: cluster 2024-09-13T15:12:39.083271+0000 mgr.y (mgr.26252) 1397 : cluster [DBG] pgmap v1976: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2024-09-13T15:12:40.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:40.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:40.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:40.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:40.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:40.799 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:40.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:40.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:40.906 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:40.924 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:40.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:40.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:41.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:41.048 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:41.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:41.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:41.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:41.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:41.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:41.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:41.279 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:41.307 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:41.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:41.409 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:41.433 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:41.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:41.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:41.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:41.561 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:41.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:41.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:41.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:41.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:41.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:41.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:41.813 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:41.843 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:41.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:41.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:41.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:41.980 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:41.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:41.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:42.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:42.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:42.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:42.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:42.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:42.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:42 smithi099 bash[84576]: cluster 2024-09-13T15:12:41.083964+0000 mgr.y (mgr.26252) 1398 : cluster [DBG] pgmap v1977: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T15:12:42.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:42 smithi099 bash[84576]: cluster 2024-09-13T15:12:41.083964+0000 mgr.y (mgr.26252) 1398 : cluster [DBG] pgmap v1977: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T15:12:42.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:42 smithi099 bash[87147]: cluster 2024-09-13T15:12:41.083964+0000 mgr.y (mgr.26252) 1398 : cluster [DBG] pgmap v1977: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T15:12:42.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:42 smithi099 bash[87147]: cluster 2024-09-13T15:12:41.083964+0000 mgr.y (mgr.26252) 1398 : cluster [DBG] pgmap v1977: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T15:12:42.254 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:42.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:42.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:42.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:42.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:42.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:42.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:42.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:42.540 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:42.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:42.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:42.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:42 smithi177 bash[68594]: cluster 2024-09-13T15:12:41.083964+0000 mgr.y (mgr.26252) 1398 : cluster [DBG] pgmap v1977: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T15:12:42.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:42 smithi177 bash[68594]: cluster 2024-09-13T15:12:41.083964+0000 mgr.y (mgr.26252) 1398 : cluster [DBG] pgmap v1977: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-09-13T15:12:42.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:42.671 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:42.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:42.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:42.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:42.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:42.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:42.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:42.942 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:42.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:42.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:43.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:43.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:43.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:43.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:43.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:43.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:43.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:43.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:43.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:43.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:43.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:43.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:43.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:43.468 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:43.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:43.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:43.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:43.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:43.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:43.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:43.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:43.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:43.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:43.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:43.910 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:43.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:43.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:44.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:44.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:44.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:44.148 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:44.171 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:44.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:44.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:44.274 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:44.296 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:44.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:44.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:44.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:44 smithi177 bash[68594]: cluster 2024-09-13T15:12:43.084366+0000 mgr.y (mgr.26252) 1399 : cluster [DBG] pgmap v1978: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:44.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:44 smithi177 bash[68594]: cluster 2024-09-13T15:12:43.084366+0000 mgr.y (mgr.26252) 1399 : cluster [DBG] pgmap v1978: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:44.400 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:44.424 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:44.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:44.526 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:44.553 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:44.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:44.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:44.625 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:44 smithi099 bash[84576]: cluster 2024-09-13T15:12:43.084366+0000 mgr.y (mgr.26252) 1399 : cluster [DBG] pgmap v1978: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:44.626 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:44 smithi099 bash[84576]: cluster 2024-09-13T15:12:43.084366+0000 mgr.y (mgr.26252) 1399 : cluster [DBG] pgmap v1978: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:44.626 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:44 smithi099 bash[87147]: cluster 2024-09-13T15:12:43.084366+0000 mgr.y (mgr.26252) 1399 : cluster [DBG] pgmap v1978: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:44.626 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:44 smithi099 bash[87147]: cluster 2024-09-13T15:12:43.084366+0000 mgr.y (mgr.26252) 1399 : cluster [DBG] pgmap v1978: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:44.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:44.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:44.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:44.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:44.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:44.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:44.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:44.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:44.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:44.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:44.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:44.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:45.065 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:45.090 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:45.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:45.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:45.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:45.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:45.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:45.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:45.344 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:45.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:45.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:45.362 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:45 smithi099 bash[84576]: cluster 2024-09-13T15:12:45.084976+0000 mgr.y (mgr.26252) 1400 : cluster [DBG] pgmap v1979: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T15:12:45.447 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:45.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:45 smithi177 bash[68594]: cluster 2024-09-13T15:12:45.084976+0000 mgr.y (mgr.26252) 1400 : cluster [DBG] pgmap v1979: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T15:12:45.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:45 smithi177 bash[68594]: cluster 2024-09-13T15:12:45.084976+0000 mgr.y (mgr.26252) 1400 : cluster [DBG] pgmap v1979: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T15:12:45.481 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:45.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:45.555 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:45 smithi099 bash[87147]: cluster 2024-09-13T15:12:45.084976+0000 mgr.y (mgr.26252) 1400 : cluster [DBG] pgmap v1979: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T15:12:45.555 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:45 smithi099 bash[87147]: cluster 2024-09-13T15:12:45.084976+0000 mgr.y (mgr.26252) 1400 : cluster [DBG] pgmap v1979: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T15:12:45.583 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:45.605 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:45.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:45.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:45.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:45.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:45.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:45.823 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:45 smithi099 bash[84576]: cluster 2024-09-13T15:12:45.084976+0000 mgr.y (mgr.26252) 1400 : cluster [DBG] pgmap v1979: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T15:12:45.845 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:45.867 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:45.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:45.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:45.970 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:45.992 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:45.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:45.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:46.095 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:46.140 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:46.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:46.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:46.245 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:46.289 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:46.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:46.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:46.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:46.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:46.544 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:46.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:46.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:46.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:46.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:46.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:46.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:46.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:46.797 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:46.824 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:46.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:46.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:46.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:46.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:46.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:47.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:47.073 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:47.074 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:12:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:12:47] "GET /metrics HTTP/1.1" 200 36778 "" "Prometheus/2.51.0" 2024-09-13T15:12:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:47.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:47.176 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:47.215 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:47.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:47.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:47.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:47.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:47.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:47.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:47.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:47.503 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:47.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:47.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:47.608 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:47.644 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:47.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:47.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:47.746 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:47.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:47.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:47.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:47.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:47.899 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:47.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:47.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:48.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:48.032 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:48.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:48.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:48.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:48.160 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:48.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:48.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:48.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:48.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:48.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:48.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:48.385 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:48.398 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:48 smithi177 bash[68594]: cluster 2024-09-13T15:12:47.085492+0000 mgr.y (mgr.26252) 1401 : cluster [DBG] pgmap v1980: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2024-09-13T15:12:48.398 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:48 smithi177 bash[68594]: cluster 2024-09-13T15:12:47.085492+0000 mgr.y (mgr.26252) 1401 : cluster [DBG] pgmap v1980: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2024-09-13T15:12:48.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:48.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:48.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:48.522 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:48.532 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:48 smithi099 bash[84576]: cluster 2024-09-13T15:12:47.085492+0000 mgr.y (mgr.26252) 1401 : cluster [DBG] pgmap v1980: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2024-09-13T15:12:48.532 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:48 smithi099 bash[84576]: cluster 2024-09-13T15:12:47.085492+0000 mgr.y (mgr.26252) 1401 : cluster [DBG] pgmap v1980: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2024-09-13T15:12:48.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:48 smithi099 bash[87147]: cluster 2024-09-13T15:12:47.085492+0000 mgr.y (mgr.26252) 1401 : cluster [DBG] pgmap v1980: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2024-09-13T15:12:48.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:48 smithi099 bash[87147]: cluster 2024-09-13T15:12:47.085492+0000 mgr.y (mgr.26252) 1401 : cluster [DBG] pgmap v1980: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 533 op/s 2024-09-13T15:12:48.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:48.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:48.655 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:48.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:48.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:48.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:48.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:48.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:48.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:48.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:48.913 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:48.939 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:48.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:48.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:49.046 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:49.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:49.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:49.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:49.184 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:49.211 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:49.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:49.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:49.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:49.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:49.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:49.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:49.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:49.463 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:49 smithi099 bash[87147]: audit 2024-09-13T15:12:48.310431+0000 mon.a (mon.0) 2552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:49.464 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:49 smithi099 bash[87147]: audit 2024-09-13T15:12:48.310431+0000 mon.a (mon.0) 2552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:49.466 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:49 smithi099 bash[84576]: audit 2024-09-13T15:12:48.310431+0000 mon.a (mon.0) 2552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:49.466 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:49 smithi099 bash[84576]: audit 2024-09-13T15:12:48.310431+0000 mon.a (mon.0) 2552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:49.483 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:49.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:49.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:49.588 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:49.597 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:49 smithi177 bash[68594]: audit 2024-09-13T15:12:48.310431+0000 mon.a (mon.0) 2552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:49.597 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:49 smithi177 bash[68594]: audit 2024-09-13T15:12:48.310431+0000 mon.a (mon.0) 2552 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:49.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:49.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:49.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:49.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:49.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:49.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:49.866 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:49.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:49.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:49.994 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:50.024 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:50.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:50.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:50.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:50.161 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:50.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:50.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:50.265 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:50.274 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:12:50 smithi177 bash[61439]: ts=2024-09-13T15:12:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:12:50.275 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:50 smithi177 bash[68594]: cluster 2024-09-13T15:12:49.086034+0000 mgr.y (mgr.26252) 1402 : cluster [DBG] pgmap v1981: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-09-13T15:12:50.275 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:50 smithi177 bash[68594]: cluster 2024-09-13T15:12:49.086034+0000 mgr.y (mgr.26252) 1402 : cluster [DBG] pgmap v1981: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-09-13T15:12:50.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:50.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:50.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:50.387 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:50 smithi099 bash[84576]: cluster 2024-09-13T15:12:49.086034+0000 mgr.y (mgr.26252) 1402 : cluster [DBG] pgmap v1981: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-09-13T15:12:50.387 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:50 smithi099 bash[84576]: cluster 2024-09-13T15:12:49.086034+0000 mgr.y (mgr.26252) 1402 : cluster [DBG] pgmap v1981: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-09-13T15:12:50.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:50.427 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:50.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:50.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:50.532 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:50.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:50.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:50.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:50.663 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:50.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:50.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:50.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:50.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:50.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:50 smithi099 bash[87147]: cluster 2024-09-13T15:12:49.086034+0000 mgr.y (mgr.26252) 1402 : cluster [DBG] pgmap v1981: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-09-13T15:12:50.804 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:50 smithi099 bash[87147]: cluster 2024-09-13T15:12:49.086034+0000 mgr.y (mgr.26252) 1402 : cluster [DBG] pgmap v1981: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-09-13T15:12:50.819 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:50.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:50.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:50.927 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:50.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:50.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:50.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:51.065 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:51.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:51.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:51.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:51.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:51.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:51.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:51.382 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:51.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:51.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:51.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:51.521 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:51.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:51.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:51.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:51.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:51.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:51.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:51.760 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:51.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:51.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:51.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:51.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 reset_purged_snaps_last 2024-09-13T15:12:51.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:51.966 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:51.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:51.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:52.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:52.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:52.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:52.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:52.216 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:52.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:52.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:52.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:52.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:52.356 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:52.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:52.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:52.458 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:52.468 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:52 smithi099 bash[84576]: cluster 2024-09-13T15:12:51.090447+0000 mgr.y (mgr.26252) 1403 : cluster [DBG] pgmap v1982: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T15:12:52.469 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:52 smithi099 bash[84576]: cluster 2024-09-13T15:12:51.090447+0000 mgr.y (mgr.26252) 1403 : cluster [DBG] pgmap v1982: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T15:12:52.469 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:52 smithi099 bash[87147]: cluster 2024-09-13T15:12:51.090447+0000 mgr.y (mgr.26252) 1403 : cluster [DBG] pgmap v1982: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T15:12:52.469 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:52 smithi099 bash[87147]: cluster 2024-09-13T15:12:51.090447+0000 mgr.y (mgr.26252) 1403 : cluster [DBG] pgmap v1982: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T15:12:52.482 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:52.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:52.586 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:52.597 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:52 smithi177 bash[68594]: cluster 2024-09-13T15:12:51.090447+0000 mgr.y (mgr.26252) 1403 : cluster [DBG] pgmap v1982: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T15:12:52.598 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:52 smithi177 bash[68594]: cluster 2024-09-13T15:12:51.090447+0000 mgr.y (mgr.26252) 1403 : cluster [DBG] pgmap v1982: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T15:12:52.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:52.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:52.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:52.725 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:52.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:52.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:52.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:52.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:52.896 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:52.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:52.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:53.027 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:53.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:53.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:53.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:53.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:53.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:53.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:53.255 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:53.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:53.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:53.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:53.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:53.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:53.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:53.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:53.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:53.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:53.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:53.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:53.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:53.696 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:53.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:53.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:53.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:53.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:53.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:53.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:53.929 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:53.949 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:53.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:53.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:53.983 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:12:53.983 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T15:12:53.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T15:12:53.983 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T15:12:53.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.983 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45764.0:205884 44.4 44:259e5975:::benchmark_data_smithi099_188632_object98680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:12:53.983 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:08:01.440456+0000", 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 292.54139592299998, 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.77980412399999999, 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45764", 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 205884 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.440456+0000", 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.440456+0000", 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.440457+0000", 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.855e-06 2024-09-13T15:12:53.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.440463+0000", 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2159999999999997e-06 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.440474+0000", 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.112e-05 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:53.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.440484+0000", 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8740000000000007e-06 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.440609+0000", 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012501299999999999 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.440764+0000", 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015578900000000001 2024-09-13T15:12:53.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5", 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.440861+0000", 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6139999999999998e-05 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.445917+0000", 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0050564909999999998 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:53.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.447963+0000", 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020456799999999998 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:02.220051+0000", 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.77208842300000002 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:02.220168+0000", 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117075 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:02.220260+0000", 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1447999999999996e-05 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45764.0:205891 44.f 44:fa08976e:::benchmark_data_smithi099_188632_object98687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:08:01.456314+0000", 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 292.52553699200001, 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.76011521999999998, 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:53.990 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45764", 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 205891 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.456314+0000", 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:53.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.456314+0000", 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.456317+0000", 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5179999999999999e-06 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:53.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.456327+0000", 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5200000000000003e-06 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.456350+0000", 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3949999999999999e-05 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.456361+0000", 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1032999999999999e-05 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.456433+0000", 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1242000000000007e-05 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.456573+0000", 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000140703 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.456690+0000", 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116686 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.461904+0000", 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052141760000000001 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:01.461971+0000", 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6637000000000006e-05 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:02.216350+0000", 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.75437907800000004 2024-09-13T15:12:53.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:02.216398+0000", 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8038999999999999e-05 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:02.216430+0000", 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1637999999999998e-05 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:53.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45764.0:210364 44.0 e1073/1067 44:08ed2be5:::benchmark_data_smithi099_188632_object103160:head v 1073'12941, mlcod=1060'746)", 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:08:08.837706+0000", 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 285.14414532699999, 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.50511836899999996, 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:08.837706+0000", 2024-09-13T15:12:53.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:08.837706+0000", 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:08.837709+0000", 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0240000000000002e-06 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:08.837718+0000", 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1649999999999995e-06 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:09.308832+0000", 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.47111320099999998 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:09.308846+0000", 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4936000000000001e-05 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:53.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:09.316758+0000", 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079117500000000004 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:09.316798+0000", 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0031999999999999e-05 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:09.336010+0000", 2024-09-13T15:12:54.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019211395999999999 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:08:09.342824+0000", 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0068148649999999998 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36057.0:6222 45.f 45:fd7e05f7:::benchmark_data_smithi099_210648_object6221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1088)", 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:09:49.154669+0000", 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 184.827181981, 2024-09-13T15:12:54.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.66017228800000005, 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36057", 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 6222 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.154669+0000", 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.154669+0000", 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.154673+0000", 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4230000000000001e-06 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.154744+0000", 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1272000000000005e-05 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.154777+0000", 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3309000000000002e-05 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.154800+0000", 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2229999999999999e-05 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.154949+0000", 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014938000000000001 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.802623+0000", 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.64767404500000003 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.802997+0000", 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000374352 2024-09-13T15:12:54.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.806400+0000", 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034028800000000001 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.812093+0000", 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0056930210000000004 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.814730+0000", 2024-09-13T15:12:54.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002636738 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.814792+0000", 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2321000000000002e-05 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.814842+0000", 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9317000000000002e-05 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.007 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36057.0:6225 45.f 45:feeed9e6:::benchmark_data_smithi099_210648_object6224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1088)", 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:09:49.156408+0000", 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 184.825443681, 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.65856609700000002, 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36057", 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:12:54.008 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 6225 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.156408+0000", 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.156408+0000", 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.156410+0000", 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.26e-06 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.156452+0000", 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2413000000000002e-05 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.803139+0000", 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.64668630900000001 2024-09-13T15:12:54.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.803172+0000", 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3148999999999997e-05 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.804700+0000", 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015281349999999999 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.805053+0000", 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00035274599999999997 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.805444+0000", 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00039090199999999999 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.808752+0000", 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003308624 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.812133+0000", 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033805350000000001 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.814885+0000", 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027520790000000002 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.814934+0000", 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8887999999999997e-05 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.814974+0000", 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0057e-05 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.36057.0:6226 45.0 e1088/1084 45:0a02f4ca:::benchmark_data_smithi099_210648_object6225:head v 1088'443, mlcod=1088'441)", 2024-09-13T15:12:54.014 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:09:49.184182+0000", 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 184.79766972799999, 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62641408399999998, 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.184182+0000", 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.184182+0000", 2024-09-13T15:12:54.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.184183+0000", 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.395e-06 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.184212+0000", 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8632999999999998e-05 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.803103+0000", 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.61889123800000001 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.803125+0000", 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2096e-05 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.803715+0000", 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00058992500000000004 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.803771+0000", 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5791999999999997e-05 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.810479+0000", 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0067079979999999997 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.810596+0000", 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117007 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.36057.0:6228 45.1 e1088/1084 45:8eb374c8:::benchmark_data_smithi099_210648_object6227:head v 1088'393, mlcod=1088'391)", 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:09:49.185537+0000", 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 184.79631443100001, 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.61965081499999997, 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.019 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.185537+0000", 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.185537+0000", 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.185538+0000", 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4600000000000003e-07 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.185662+0000", 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012361199999999999 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.802235+0000", 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.61657326899999998 2024-09-13T15:12:54.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.802264+0000", 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8853e-05 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.802446+0000", 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000182459 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.802511+0000", 2024-09-13T15:12:54.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4646000000000004e-05 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.805129+0000", 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002617936 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:09:49.805188+0000", 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9094e-05 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36057.0:34442 45.5 45:a998a740:::benchmark_data_smithi099_210648_object34441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1088)", 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:10:02.994455+0000", 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.987396792, 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59255387699999995, 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36057", 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 34442 2024-09-13T15:12:54.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.994455+0000", 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.994455+0000", 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.994456+0000", 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4449999999999999e-06 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.994478+0000", 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1645e-05 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.994489+0000", 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1552999999999999e-05 2024-09-13T15:12:54.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.994496+0000", 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7920000000000004e-06 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.994553+0000", 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6495e-05 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.994693+0000", 2024-09-13T15:12:54.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013996500000000001 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.994822+0000", 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012955400000000001 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.996341+0000", 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015188840000000001 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997040+0000", 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00069873400000000001 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999515+0000", 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024754719999999998 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.586885+0000", 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58736966999999995 2024-09-13T15:12:54.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587009+0000", 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012366800000000001 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36057.0:34446 45.5 45:a34caa7a:::benchmark_data_smithi099_210648_object34445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1088)", 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:10:02.997417+0000", 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.98443468299999, 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59012823700000006, 2024-09-13T15:12:54.030 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36057", 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 34446 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997417+0000", 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997417+0000", 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997418+0000", 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4219999999999999e-06 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997479+0000", 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0868000000000001e-05 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997488+0000", 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3319999999999998e-06 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997493+0000", 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0980000000000001e-06 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997542+0000", 2024-09-13T15:12:54.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8059000000000002e-05 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997685+0000", 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143492 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997919+0000", 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000233713 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.034 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587145+0000", 2024-09-13T15:12:54.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58922647800000005 2024-09-13T15:12:54.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.035 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.035 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:54.035 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587183+0000", 2024-09-13T15:12:54.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7369000000000001e-05 2024-09-13T15:12:54.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.035 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.035 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587301+0000", 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117995 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587441+0000", 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014027700000000001 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587545+0000", 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104134 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.036 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36057.0:34447 45.f 45:fac1d9cb:::benchmark_data_smithi099_210648_object34446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1088)", 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:10:02.997507+0000", 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.98434456199999, 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59008817199999997, 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36057", 2024-09-13T15:12:54.037 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 34447 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997507+0000", 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997507+0000", 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997507+0000", 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9400000000000005e-07 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997550+0000", 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2565999999999999e-05 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997557+0000", 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9260000000000003e-06 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997563+0000", 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7060000000000001e-06 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997615+0000", 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2756000000000001e-05 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.040 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997744+0000", 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000128305 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.997919+0000", 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017494900000000001 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999419+0000", 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015006520000000001 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.042 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.042 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587210+0000", 2024-09-13T15:12:54.042 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58779088199999996 2024-09-13T15:12:54.042 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.042 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.042 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.042 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587376+0000", 2024-09-13T15:12:54.042 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016553299999999999 2024-09-13T15:12:54.042 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.042 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587507+0000", 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013107099999999999 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587595+0000", 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8232000000000005e-05 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.043 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.36057.0:34442 45.5 e1088/1084 ondisk, result = 0)", 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:10:02.999318+0000", 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.98253350300001, 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58770996099999995, 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999318+0000", 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.044 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999318+0000", 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999319+0000", 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6400000000000001e-07 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999324+0000", 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.921e-06 2024-09-13T15:12:54.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999333+0000", 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4590000000000004e-06 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999340+0000", 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7859999999999999e-06 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.046 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999503+0000", 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016291799999999999 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999510+0000", 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1149999999999996e-06 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587028+0000", 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58751789799999998 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.047 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.36057.0:34449 45.0 e1088/1084 45:0daa1ba4:::benchmark_data_smithi099_210648_object34448:head v 1088'2220, mlcod=1088'2218)", 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:10:02.999368+0000", 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.982483403, 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59526732500000001, 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.048 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999368+0000", 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999368+0000", 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999368+0000", 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9900000000000001e-07 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.049 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999393+0000", 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4304999999999999e-05 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999400+0000", 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6369999999999999e-06 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999406+0000", 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4190000000000001e-06 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.050 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999505+0000", 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9236999999999999e-05 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999523+0000", 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.772e-05 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.594487+0000", 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59496394900000005 2024-09-13T15:12:54.051 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.594635+0000", 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014865899999999999 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.36057.0:34446 45.5 e1088/1084 ondisk, result = 0)", 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:10:02.999876+0000", 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.98197566900001, 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58732335700000005, 2024-09-13T15:12:54.052 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999876+0000", 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999876+0000", 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.053 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999877+0000", 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2099999999999995e-07 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:02.999881+0000", 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.454e-06 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.586682+0000", 2024-09-13T15:12:54.054 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58680102599999995 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.586706+0000", 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3866000000000002e-05 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587091+0000", 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00038551500000000001 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.055 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587125+0000", 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3105999999999998e-05 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587199+0000", 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4568999999999997e-05 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.056 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.36057.0:34447 45.f e1088/1084 ondisk, result = 0)", 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:10:03.001949+0000", 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.979902708, 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58529402500000005, 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.001949+0000", 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.057 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.001949+0000", 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.001950+0000", 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9400000000000004e-07 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.001955+0000", 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4330000000000003e-06 2024-09-13T15:12:54.058 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.586620+0000", 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58466500799999999 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.586659+0000", 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8658999999999997e-05 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.059 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587169+0000", 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00051051899999999997 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587193+0000", 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3366e-05 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587243+0000", 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0145999999999997e-05 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.060 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.36057.0:34459 45.1 e1088/1084 45:8781907d:::benchmark_data_smithi099_210648_object34458:head v 1088'2151, mlcod=1088'2149)", 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:10:03.005193+0000", 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 170.97665841400001, 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59005181299999998, 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.061 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.005193+0000", 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.005193+0000", 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.005194+0000", 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7600000000000006e-07 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.005250+0000", 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6045000000000003e-05 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.586616+0000", 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58136591599999998 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.586659+0000", 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2759000000000002e-05 2024-09-13T15:12:54.063 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587294+0000", 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063532699999999998 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.587347+0000", 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3198999999999997e-05 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.064 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.595137+0000", 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0077893900000000002 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:03.595245+0000", 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108201 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36057.0:43450 45.5 45:a905afc1:::benchmark_data_smithi099_210648_object43449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1088)", 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:10:12.580406+0000", 2024-09-13T15:12:54.065 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 161.40144552699999, 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.51026849200000002, 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36057", 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 43450 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:12.580406+0000", 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.066 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:12.580406+0000", 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:12.580407+0000", 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.356e-06 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:12.580428+0000", 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0744e-05 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:12.580438+0000", 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0376e-05 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:12.580445+0000", 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9299999999999997e-06 2024-09-13T15:12:54.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:12.580490+0000", 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4270000000000001e-05 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:12.580659+0000", 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016956 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:12:54.069 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:12.580805+0000", 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014536700000000001 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:12.585326+0000", 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045217260000000002 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:13.090310+0000", 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.50498420700000002 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.070 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:13.090426+0000", 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011579899999999999 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:13.090575+0000", 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000148677 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:10:13.090674+0000", 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9480000000000003e-05 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.071 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36057.0:154959 45.f 45:fd9ca6af:::benchmark_data_smithi099_210648_object53226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1092)", 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:12:28.891612+0000", 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 25.090239401000002, 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.64206960400000002, 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36057", 2024-09-13T15:12:54.072 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154959 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.891612+0000", 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.891612+0000", 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.891614+0000", 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.66e-06 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.891620+0000", 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0070000000000003e-06 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.891634+0000", 2024-09-13T15:12:54.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4056999999999999e-05 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.891640+0000", 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2129999999999996e-06 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.891692+0000", 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1768000000000003e-05 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.891769+0000", 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7651000000000006e-05 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.891832+0000", 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2846000000000001e-05 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.895583+0000", 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037507270000000001 2024-09-13T15:12:54.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:28.895750+0000", 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000166797 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:29.533413+0000", 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.63766345999999996 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:29.533631+0000", 2024-09-13T15:12:54.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021798400000000001 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:29.533682+0000", 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0433999999999999e-05 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36057.0:164153 45.5 45:aebec3da:::benchmark_data_smithi099_210648_object62420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1092)", 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:12:44.098365+0000", 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 9.8834862260000005, 2024-09-13T15:12:54.078 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.53257168600000004, 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36057", 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 164153 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.098365+0000", 2024-09-13T15:12:54.079 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.098365+0000", 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.098366+0000", 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8200000000000008e-07 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.080 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.098370+0000", 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2130000000000002e-06 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.098386+0000", 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5333e-05 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.098394+0000", 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8590000000000005e-06 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.081 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.114729+0000", 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016335579999999999 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.114950+0000", 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022090099999999999 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.115201+0000", 2024-09-13T15:12:54.082 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025103400000000002 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.141993+0000", 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.026791748000000001 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.142291+0000", 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00029762799999999998 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.083 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.630817+0000", 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.48852602099999998 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.630884+0000", 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7049000000000001e-05 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.630937+0000", 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3338000000000003e-05 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.084 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36057.0:164160 45.5 45:aa39be8d:::benchmark_data_smithi099_210648_object62427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1092)", 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:12:44.110310+0000", 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 9.8715415750000002, 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.52077301300000001, 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36057", 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:12:54.085 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 164160 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110310+0000", 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110310+0000", 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.086 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110311+0000", 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3909999999999999e-06 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110316+0000", 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7210000000000002e-06 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110325+0000", 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.867e-06 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.087 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110333+0000", 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1019999999999997e-06 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.120001+0000", 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096682910000000007 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.124680+0000", 2024-09-13T15:12:54.088 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046786049999999997 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.124924+0000", 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00024403399999999999 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.142295+0000", 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017371373999999998 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.149499+0000", 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0072038279999999998 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.630983+0000", 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.481484264 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.631040+0000", 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6784999999999999e-05 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.090 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.631083+0000", 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2750999999999998e-05 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36057.0:164163 45.5 45:ab9bfc00:::benchmark_data_smithi099_210648_object62430:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1092)", 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:12:44.110434+0000", 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 9.8714178500000003, 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.52683469599999999, 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:12:54.091 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36057", 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 164163 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110434+0000", 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:12:54.092 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110434+0000", 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110434+0000", 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1699999999999997e-07 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110438+0000", 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7290000000000002e-06 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.093 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110447+0000", 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7190000000000007e-06 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.110454+0000", 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7170000000000002e-06 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.130409+0000", 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019954979000000001 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.094 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.130698+0000", 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00028964099999999999 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.131040+0000", 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00034225599999999998 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.142300+0000", 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011259106 2024-09-13T15:12:54.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.151672+0000", 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0093726839999999992 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.637154+0000", 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.485482201 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:12:54.096 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.637218+0000", 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3423999999999994e-05 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:12:44.637268+0000", 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0423000000000002e-05 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:12:54.097 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:12:54.098 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:54.122 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:54.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:54.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:54.225 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:54.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:54.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:54.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:54.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:54.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:54 smithi177 bash[68594]: cluster 2024-09-13T15:12:53.090941+0000 mgr.y (mgr.26252) 1404 : cluster [DBG] pgmap v1983: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:54.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:54 smithi177 bash[68594]: cluster 2024-09-13T15:12:53.090941+0000 mgr.y (mgr.26252) 1404 : cluster [DBG] pgmap v1983: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:54.373 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:54.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:54.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:54.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:54.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:54.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:54.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:54.542 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:54 smithi099 bash[84576]: cluster 2024-09-13T15:12:53.090941+0000 mgr.y (mgr.26252) 1404 : cluster [DBG] pgmap v1983: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:54.542 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:54 smithi099 bash[84576]: cluster 2024-09-13T15:12:53.090941+0000 mgr.y (mgr.26252) 1404 : cluster [DBG] pgmap v1983: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:54.542 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:54 smithi099 bash[87147]: cluster 2024-09-13T15:12:53.090941+0000 mgr.y (mgr.26252) 1404 : cluster [DBG] pgmap v1983: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:54.542 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:54 smithi099 bash[87147]: cluster 2024-09-13T15:12:53.090941+0000 mgr.y (mgr.26252) 1404 : cluster [DBG] pgmap v1983: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s 2024-09-13T15:12:54.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:54.633 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:54.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:54.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:54.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:54.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:54.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:54.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:54.875 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:54.902 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:54.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:54.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:55.010 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:55.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:55.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:55.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:55.146 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:55.175 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:55.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:55.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:55.279 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:55.307 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:55.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:55.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:55.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:55.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:55.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:55.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:55.597 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:55.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:55.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:55.703 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:55.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:55.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:55.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:55.786 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:12:55.786 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:12:55.820 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:12:55.820 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:12:55.841 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:55.849 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:12:55.849 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:12:55.871 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:55.878 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:12:55.879 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:12:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:55.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:55.929 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:12:55.930 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:12:55.958 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:12:55.958 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:12:55.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:55.991 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:12:55.991 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:12:56.001 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:56.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:56.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:56.024 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:12:56.024 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:12:56.108 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:56.133 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:56.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:56.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:56.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:56.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:56.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:56.367 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:56 smithi099 bash[84576]: cluster 2024-09-13T15:12:55.091576+0000 mgr.y (mgr.26252) 1405 : cluster [DBG] pgmap v1984: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-13T15:12:56.386 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:12:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:56.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:12:56.489 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:56.518 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:56.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:56.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:56.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:56 smithi177 bash[68594]: cluster 2024-09-13T15:12:55.091576+0000 mgr.y (mgr.26252) 1405 : cluster [DBG] pgmap v1984: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-13T15:12:56.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:56 smithi177 bash[68594]: cluster 2024-09-13T15:12:55.091576+0000 mgr.y (mgr.26252) 1405 : cluster [DBG] pgmap v1984: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-13T15:12:56.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:56 smithi177 bash[68594]: audit 2024-09-13T15:12:55.721521+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.99:0/2626933545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:56.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:56 smithi177 bash[68594]: audit 2024-09-13T15:12:55.721521+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.99:0/2626933545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:56.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:56.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:56.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:56.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:56.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:56.764 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:56.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:56.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:56.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:56 smithi099 bash[84576]: cluster 2024-09-13T15:12:55.091576+0000 mgr.y (mgr.26252) 1405 : cluster [DBG] pgmap v1984: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-13T15:12:56.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:56 smithi099 bash[84576]: audit 2024-09-13T15:12:55.721521+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.99:0/2626933545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:56.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:56 smithi099 bash[84576]: audit 2024-09-13T15:12:55.721521+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.99:0/2626933545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:56 smithi099 bash[87147]: cluster 2024-09-13T15:12:55.091576+0000 mgr.y (mgr.26252) 1405 : cluster [DBG] pgmap v1984: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-13T15:12:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:56 smithi099 bash[87147]: cluster 2024-09-13T15:12:55.091576+0000 mgr.y (mgr.26252) 1405 : cluster [DBG] pgmap v1984: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-13T15:12:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:56 smithi099 bash[87147]: audit 2024-09-13T15:12:55.721521+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.99:0/2626933545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:56 smithi099 bash[87147]: audit 2024-09-13T15:12:55.721521+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.99:0/2626933545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:12:56.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:56.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:56.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:56.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:57.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:57.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:57.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:57.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:57.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:57.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:57.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:57.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:57.188 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:12:57.193 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:12:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:12:57] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-09-13T15:12:57.281 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:57.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:12:57.316 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:57 smithi099 bash[84576]: audit 2024-09-13T15:12:56.584749+0000 mon.a (mon.0) 2553 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:12:57.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:57 smithi099 bash[84576]: audit 2024-09-13T15:12:56.584749+0000 mon.a (mon.0) 2553 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:12:57.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:57 smithi099 bash[87147]: audit 2024-09-13T15:12:56.584749+0000 mon.a (mon.0) 2553 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:12:57.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:57 smithi099 bash[87147]: audit 2024-09-13T15:12:56.584749+0000 mon.a (mon.0) 2553 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:12:57.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:57.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:12:57.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:57.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:57.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:57.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:57.552 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:57.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:57 smithi177 bash[68594]: audit 2024-09-13T15:12:56.584749+0000 mon.a (mon.0) 2553 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:12:57.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:57 smithi177 bash[68594]: audit 2024-09-13T15:12:56.584749+0000 mon.a (mon.0) 2553 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:12:57.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:57.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:57.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:57.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:57.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:57.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:57.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:57.814 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:57.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:57.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:57.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:57.940 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:57.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:57.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:57.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:58.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:58.116 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:12:58.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:58.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:12:58.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:58.244 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:58.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:58.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:58.330 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T15:12:58.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:58.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:58 smithi177 bash[68594]: cluster 2024-09-13T15:12:57.092082+0000 mgr.y (mgr.26252) 1406 : cluster [DBG] pgmap v1985: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2024-09-13T15:12:58.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:58 smithi177 bash[68594]: cluster 2024-09-13T15:12:57.092082+0000 mgr.y (mgr.26252) 1406 : cluster [DBG] pgmap v1985: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2024-09-13T15:12:58.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:58 smithi177 bash[68594]: cluster 2024-09-13T15:12:57.174806+0000 mon.a (mon.0) 2554 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:58.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:58 smithi177 bash[68594]: cluster 2024-09-13T15:12:57.174806+0000 mon.a (mon.0) 2554 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:58.359 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:58 smithi177 bash[68594]: audit 2024-09-13T15:12:57.188753+0000 mon.a (mon.0) 2555 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:12:58.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:58 smithi177 bash[68594]: audit 2024-09-13T15:12:57.188753+0000 mon.a (mon.0) 2555 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:12:58.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:58 smithi177 bash[68594]: cluster 2024-09-13T15:12:57.197394+0000 mon.a (mon.0) 2556 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2024-09-13T15:12:58.360 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:12:58 smithi177 bash[68594]: cluster 2024-09-13T15:12:57.197394+0000 mon.a (mon.0) 2556 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2024-09-13T15:12:58.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:58.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:58.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:12:58.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:58.507 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:58.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:58.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[87147]: cluster 2024-09-13T15:12:57.092082+0000 mgr.y (mgr.26252) 1406 : cluster [DBG] pgmap v1985: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2024-09-13T15:12:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[87147]: cluster 2024-09-13T15:12:57.092082+0000 mgr.y (mgr.26252) 1406 : cluster [DBG] pgmap v1985: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2024-09-13T15:12:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[87147]: cluster 2024-09-13T15:12:57.174806+0000 mon.a (mon.0) 2554 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[87147]: cluster 2024-09-13T15:12:57.174806+0000 mon.a (mon.0) 2554 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[87147]: audit 2024-09-13T15:12:57.188753+0000 mon.a (mon.0) 2555 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:12:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[87147]: audit 2024-09-13T15:12:57.188753+0000 mon.a (mon.0) 2555 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:12:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[87147]: cluster 2024-09-13T15:12:57.197394+0000 mon.a (mon.0) 2556 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2024-09-13T15:12:58.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[87147]: cluster 2024-09-13T15:12:57.197394+0000 mon.a (mon.0) 2556 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2024-09-13T15:12:58.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[84576]: cluster 2024-09-13T15:12:57.092082+0000 mgr.y (mgr.26252) 1406 : cluster [DBG] pgmap v1985: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2024-09-13T15:12:58.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[84576]: cluster 2024-09-13T15:12:57.092082+0000 mgr.y (mgr.26252) 1406 : cluster [DBG] pgmap v1985: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2024-09-13T15:12:58.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[84576]: cluster 2024-09-13T15:12:57.174806+0000 mon.a (mon.0) 2554 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:58.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[84576]: cluster 2024-09-13T15:12:57.174806+0000 mon.a (mon.0) 2554 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:12:58.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[84576]: audit 2024-09-13T15:12:57.188753+0000 mon.a (mon.0) 2555 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:12:58.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[84576]: audit 2024-09-13T15:12:57.188753+0000 mon.a (mon.0) 2555 : audit [INF] from='client.? 172.21.15.99:0/1402126585' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:12:58.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[84576]: cluster 2024-09-13T15:12:57.197394+0000 mon.a (mon.0) 2556 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2024-09-13T15:12:58.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:12:58 smithi099 bash[84576]: cluster 2024-09-13T15:12:57.197394+0000 mon.a (mon.0) 2556 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2024-09-13T15:12:58.609 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:58.631 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:58.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:58.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:58.777 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:58.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:58.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:58.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:58.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:58.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:58.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:59.017 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:59.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:12:59.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:59.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:12:59.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:59.173 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:59.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:59.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:59.281 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:59.317 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:59.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:59.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:59.422 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:59.440 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:12:59.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:59.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:12:59.542 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:59.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:59.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:59.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:59.677 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:59.705 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:12:59.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:59.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:12:59.810 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:59.838 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:12:59.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:59.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:12:59.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:59.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:12:59.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:12:59.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:00.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:00.109 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:00.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:00.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:00.217 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:00.240 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:00.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:00.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:00.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:00.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:00 smithi177 bash[68594]: cluster 2024-09-13T15:12:59.092706+0000 mgr.y (mgr.26252) 1407 : cluster [DBG] pgmap v1987: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-13T15:13:00.355 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:00 smithi177 bash[68594]: cluster 2024-09-13T15:12:59.092706+0000 mgr.y (mgr.26252) 1407 : cluster [DBG] pgmap v1987: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-13T15:13:00.355 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:13:00 smithi177 bash[61439]: ts=2024-09-13T15:13:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:13:00.371 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:00.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:00.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:00.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:00.487 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:00 smithi099 bash[84576]: cluster 2024-09-13T15:12:59.092706+0000 mgr.y (mgr.26252) 1407 : cluster [DBG] pgmap v1987: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-13T15:13:00.488 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:00 smithi099 bash[84576]: cluster 2024-09-13T15:12:59.092706+0000 mgr.y (mgr.26252) 1407 : cluster [DBG] pgmap v1987: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-13T15:13:00.488 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:00 smithi099 bash[87147]: cluster 2024-09-13T15:12:59.092706+0000 mgr.y (mgr.26252) 1407 : cluster [DBG] pgmap v1987: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-13T15:13:00.488 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:00 smithi099 bash[87147]: cluster 2024-09-13T15:12:59.092706+0000 mgr.y (mgr.26252) 1407 : cluster [DBG] pgmap v1987: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-09-13T15:13:00.505 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:00.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:00.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:00.609 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:00.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:00.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:00.746 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:00.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:00.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:00.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:00.831 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:13:00.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:00.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:00.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:00.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:01.010 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:01.043 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:01.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:01.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:01.148 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:01.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:01.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:01.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:01.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:01.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:01.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:01.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:01.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:01.419 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:01.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:01.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:01.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:01.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:01.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:01.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:01.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:01.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:01.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:01.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:01.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:01.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:01.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:01.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:01.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:01.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:01.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:02.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:02.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:02.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:02.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:02.187 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:02.219 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:02.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:02.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:02.312 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:13:02.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:02.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:02 smithi099 bash[87147]: cluster 2024-09-13T15:13:01.093220+0000 mgr.y (mgr.26252) 1408 : cluster [DBG] pgmap v1988: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-09-13T15:13:02.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:02 smithi099 bash[87147]: cluster 2024-09-13T15:13:01.093220+0000 mgr.y (mgr.26252) 1408 : cluster [DBG] pgmap v1988: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-09-13T15:13:02.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:02 smithi099 bash[84576]: cluster 2024-09-13T15:13:01.093220+0000 mgr.y (mgr.26252) 1408 : cluster [DBG] pgmap v1988: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-09-13T15:13:02.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:02 smithi099 bash[84576]: cluster 2024-09-13T15:13:01.093220+0000 mgr.y (mgr.26252) 1408 : cluster [DBG] pgmap v1988: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-09-13T15:13:02.366 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:02.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:02.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:02.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:13:02.471 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:02.527 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:02.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:02.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:02.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:02 smithi177 bash[68594]: cluster 2024-09-13T15:13:01.093220+0000 mgr.y (mgr.26252) 1408 : cluster [DBG] pgmap v1988: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-09-13T15:13:02.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:02 smithi177 bash[68594]: cluster 2024-09-13T15:13:01.093220+0000 mgr.y (mgr.26252) 1408 : cluster [DBG] pgmap v1988: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-09-13T15:13:02.634 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:02.685 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:02.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:02.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:02.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:02.831 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:02.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:02.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:02.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:02.963 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:02.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:02.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:03.068 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:03.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:03.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:03.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:03.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:03.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:03.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:03.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:03.372 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:03.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:03.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:03.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:03.504 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:03.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:03.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:03.609 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:03.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:03.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:03.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:03.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:03.789 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:03.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:03.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:03.893 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:03.917 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:03.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:03.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:04.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:04.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:04.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:04.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:04.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:04.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:04.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:04.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:04.314 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:04.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:04.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:04 smithi177 bash[68594]: cluster 2024-09-13T15:13:03.093731+0000 mgr.y (mgr.26252) 1409 : cluster [DBG] pgmap v1989: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:04 smithi177 bash[68594]: cluster 2024-09-13T15:13:03.093731+0000 mgr.y (mgr.26252) 1409 : cluster [DBG] pgmap v1989: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:04 smithi177 bash[68594]: audit 2024-09-13T15:13:03.310648+0000 mon.a (mon.0) 2557 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:04.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:04 smithi177 bash[68594]: audit 2024-09-13T15:13:03.310648+0000 mon.a (mon.0) 2557 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:04.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:04.427 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:04 smithi099 bash[84576]: cluster 2024-09-13T15:13:03.093731+0000 mgr.y (mgr.26252) 1409 : cluster [DBG] pgmap v1989: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:04.428 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:04 smithi099 bash[84576]: cluster 2024-09-13T15:13:03.093731+0000 mgr.y (mgr.26252) 1409 : cluster [DBG] pgmap v1989: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:04.428 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:04 smithi099 bash[84576]: audit 2024-09-13T15:13:03.310648+0000 mon.a (mon.0) 2557 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:04.428 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:04 smithi099 bash[84576]: audit 2024-09-13T15:13:03.310648+0000 mon.a (mon.0) 2557 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:04.428 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:04 smithi099 bash[87147]: cluster 2024-09-13T15:13:03.093731+0000 mgr.y (mgr.26252) 1409 : cluster [DBG] pgmap v1989: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:04.428 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:04 smithi099 bash[87147]: cluster 2024-09-13T15:13:03.093731+0000 mgr.y (mgr.26252) 1409 : cluster [DBG] pgmap v1989: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:04.428 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:04 smithi099 bash[87147]: audit 2024-09-13T15:13:03.310648+0000 mon.a (mon.0) 2557 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:04.428 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:04 smithi099 bash[87147]: audit 2024-09-13T15:13:03.310648+0000 mon.a (mon.0) 2557 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:04.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:04.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:04.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:04.547 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:04.590 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:04.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:04.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:04.693 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:04.720 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:04.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:04.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:04.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:04.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:04.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:04.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:04.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:04.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:04.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:05.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:05.117 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:05.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:05.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:05.219 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:05.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:05.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:05.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:05.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:05 smithi177 bash[68594]: cluster 2024-09-13T15:13:05.094458+0000 mgr.y (mgr.26252) 1410 : cluster [DBG] pgmap v1990: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:05.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:05 smithi177 bash[68594]: cluster 2024-09-13T15:13:05.094458+0000 mgr.y (mgr.26252) 1410 : cluster [DBG] pgmap v1990: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:05.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:05.406 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:05.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:05.512 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:05.533 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:05.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:05.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:05.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:05 smithi099 bash[84576]: cluster 2024-09-13T15:13:05.094458+0000 mgr.y (mgr.26252) 1410 : cluster [DBG] pgmap v1990: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:05.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:05 smithi099 bash[84576]: cluster 2024-09-13T15:13:05.094458+0000 mgr.y (mgr.26252) 1410 : cluster [DBG] pgmap v1990: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:05 smithi099 bash[87147]: cluster 2024-09-13T15:13:05.094458+0000 mgr.y (mgr.26252) 1410 : cluster [DBG] pgmap v1990: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:05.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:05 smithi099 bash[87147]: cluster 2024-09-13T15:13:05.094458+0000 mgr.y (mgr.26252) 1410 : cluster [DBG] pgmap v1990: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:05.637 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:05.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:05.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:05.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:05.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:05.787 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:05.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:05.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:05.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:05.915 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:05.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:05.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:06.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:06.053 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:06.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:06.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:06.098 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:13:06.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:06.181 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:06.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:06.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:06.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:06.319 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:06.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:06.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:06.425 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:06.451 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:06.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:06.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:06.556 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:06.583 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:06.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:06.690 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:06.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:06.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:06.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:06.824 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:06.855 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:06.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:06.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:06.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:06.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:06.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:07.094 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:07.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:07.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:07.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:07.218 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:07.242 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:07.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:07.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:07.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:13:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:13:07] "GET /metrics HTTP/1.1" 200 36773 "" "Prometheus/2.51.0" 2024-09-13T15:13:07.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:07.374 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:07.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:07.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:07.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:07.497 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:07.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:07.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:07.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:07.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:07.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:07.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:07.744 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:07.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:07.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:07.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:07.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:07.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:07.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:07.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:08.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:08.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:08.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:08.142 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:13:08.142 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:13:08.154 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:08.179 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:08.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:08.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:08.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:08.297 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:08 smithi099 bash[84576]: cluster 2024-09-13T15:13:07.095158+0000 mgr.y (mgr.26252) 1411 : cluster [DBG] pgmap v1991: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T15:13:08.297 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:08 smithi099 bash[84576]: cluster 2024-09-13T15:13:07.095158+0000 mgr.y (mgr.26252) 1411 : cluster [DBG] pgmap v1991: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T15:13:08.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:08 smithi099 bash[87147]: cluster 2024-09-13T15:13:07.095158+0000 mgr.y (mgr.26252) 1411 : cluster [DBG] pgmap v1991: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T15:13:08.302 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:08 smithi099 bash[87147]: cluster 2024-09-13T15:13:07.095158+0000 mgr.y (mgr.26252) 1411 : cluster [DBG] pgmap v1991: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T15:13:08.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:08.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:08.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:08 smithi177 bash[68594]: cluster 2024-09-13T15:13:07.095158+0000 mgr.y (mgr.26252) 1411 : cluster [DBG] pgmap v1991: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T15:13:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:08 smithi177 bash[68594]: cluster 2024-09-13T15:13:07.095158+0000 mgr.y (mgr.26252) 1411 : cluster [DBG] pgmap v1991: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T15:13:08.419 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:08.440 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:08.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:08.543 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:08.579 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:08.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:08.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:08.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:08.704 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:08.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:08.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:08.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:08.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:08.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:08.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:08.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:08.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:08.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:09.095 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:09.121 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:09.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:09.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:09.226 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:09.246 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:09.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:09.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:09.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:09.381 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:09.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:09.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:09.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:09.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:09.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:09.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:09.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:09.655 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:09.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:09.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:09.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:09.785 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:09.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:09.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:09.890 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:09.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:09.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:09.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:10.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:10.061 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:10.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:10.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:10.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:10.195 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:10.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:10.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:10.299 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:10.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:10.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:10.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:10.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:10.461 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:10 smithi099 bash[84576]: cluster 2024-09-13T15:13:09.098238+0000 mgr.y (mgr.26252) 1412 : cluster [DBG] pgmap v1992: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2024-09-13T15:13:10.461 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:10 smithi099 bash[84576]: cluster 2024-09-13T15:13:09.098238+0000 mgr.y (mgr.26252) 1412 : cluster [DBG] pgmap v1992: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2024-09-13T15:13:10.462 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:10 smithi099 bash[87147]: cluster 2024-09-13T15:13:09.098238+0000 mgr.y (mgr.26252) 1412 : cluster [DBG] pgmap v1992: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2024-09-13T15:13:10.462 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:10 smithi099 bash[87147]: cluster 2024-09-13T15:13:09.098238+0000 mgr.y (mgr.26252) 1412 : cluster [DBG] pgmap v1992: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2024-09-13T15:13:10.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:10.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:10.604 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:10.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:10 smithi177 bash[68594]: cluster 2024-09-13T15:13:09.098238+0000 mgr.y (mgr.26252) 1412 : cluster [DBG] pgmap v1992: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2024-09-13T15:13:10.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:10 smithi177 bash[68594]: cluster 2024-09-13T15:13:09.098238+0000 mgr.y (mgr.26252) 1412 : cluster [DBG] pgmap v1992: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2024-09-13T15:13:10.611 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:13:10 smithi177 bash[61439]: ts=2024-09-13T15:13:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:13:10.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:10.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:10.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:10.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:10.776 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:10.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:10.880 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:10.899 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:10.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:10.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:11.002 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:11.038 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:11.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:11.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:11.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:11.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:11.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:11.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:11.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:11.311 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:11.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:11.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:11.418 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:11.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:11.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:11.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:11.569 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:11.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:11.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:11.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:11.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:11.736 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:11.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:11.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:11.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:11.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:11.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:11.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:11.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:11.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:11.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:12.090 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:12.116 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:12.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:12.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:12.219 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:12.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:12 smithi177 bash[68594]: cluster 2024-09-13T15:13:11.098863+0000 mgr.y (mgr.26252) 1413 : cluster [DBG] pgmap v1993: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-13T15:13:12.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:12 smithi177 bash[68594]: cluster 2024-09-13T15:13:11.098863+0000 mgr.y (mgr.26252) 1413 : cluster [DBG] pgmap v1993: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-13T15:13:12.255 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:12.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:12.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:12.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:12.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:12.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:12.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:12.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:12.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:12.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:12.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:12.584 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:12 smithi099 bash[84576]: cluster 2024-09-13T15:13:11.098863+0000 mgr.y (mgr.26252) 1413 : cluster [DBG] pgmap v1993: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-13T15:13:12.584 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:12 smithi099 bash[84576]: cluster 2024-09-13T15:13:11.098863+0000 mgr.y (mgr.26252) 1413 : cluster [DBG] pgmap v1993: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-13T15:13:12.623 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:12.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:12.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:12.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:12.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:12.789 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:12.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:12.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:12.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:12.901 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:12 smithi099 bash[87147]: cluster 2024-09-13T15:13:11.098863+0000 mgr.y (mgr.26252) 1413 : cluster [DBG] pgmap v1993: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-13T15:13:12.901 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:12 smithi099 bash[87147]: cluster 2024-09-13T15:13:11.098863+0000 mgr.y (mgr.26252) 1413 : cluster [DBG] pgmap v1993: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2024-09-13T15:13:12.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:12.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:12.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:13.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:13.046 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:13.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:13.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:13.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:13.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:13.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:13.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:13.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:13.327 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:13.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:13.434 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:13.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:13.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:13.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:13.560 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:13.589 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:13.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:13.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:13.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:13.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:13.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:13.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:13.850 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:13.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:13.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:13.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:13.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:13.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:14.096 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:14.121 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:14.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:14.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:14.227 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:14.252 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:14.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:14.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:14 smithi099 bash[84576]: cluster 2024-09-13T15:13:13.099298+0000 mgr.y (mgr.26252) 1414 : cluster [DBG] pgmap v1994: 89 pgs: 89 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:14.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:14 smithi099 bash[84576]: cluster 2024-09-13T15:13:13.099298+0000 mgr.y (mgr.26252) 1414 : cluster [DBG] pgmap v1994: 89 pgs: 89 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:14.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:14 smithi099 bash[87147]: cluster 2024-09-13T15:13:13.099298+0000 mgr.y (mgr.26252) 1414 : cluster [DBG] pgmap v1994: 89 pgs: 89 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:14.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:14 smithi099 bash[87147]: cluster 2024-09-13T15:13:13.099298+0000 mgr.y (mgr.26252) 1414 : cluster [DBG] pgmap v1994: 89 pgs: 89 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:14.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:14 smithi177 bash[68594]: cluster 2024-09-13T15:13:13.099298+0000 mgr.y (mgr.26252) 1414 : cluster [DBG] pgmap v1994: 89 pgs: 89 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:14 smithi177 bash[68594]: cluster 2024-09-13T15:13:13.099298+0000 mgr.y (mgr.26252) 1414 : cluster [DBG] pgmap v1994: 89 pgs: 89 active+clean; 1.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:14.381 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:14.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:14.485 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:14.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:14.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:14.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:14.612 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:14.651 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:14.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:14.758 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:14.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:14.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:14.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:14.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:14.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:14.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:14.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:15.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:15.056 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:15.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:15.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:15.159 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:15.185 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:15.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:15.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:15.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:15.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:15.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:15.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:15.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:15.465 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:15.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:15.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:15.568 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:15.596 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:15.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:15.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:15.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:15.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:15.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:15.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:15.848 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:15.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:15.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:15.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:15.967 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:15.990 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:15.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:15.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:16.093 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:16.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:16.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:16.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:16.215 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:16.235 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:16.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:16.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:16.378 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:16.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:16.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:16.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:16.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:16.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:16.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:16.518 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:16 smithi099 bash[84576]: cluster 2024-09-13T15:13:15.100202+0000 mgr.y (mgr.26252) 1415 : cluster [DBG] pgmap v1995: 89 pgs: 89 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2024-09-13T15:13:16.518 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:16 smithi099 bash[84576]: cluster 2024-09-13T15:13:15.100202+0000 mgr.y (mgr.26252) 1415 : cluster [DBG] pgmap v1995: 89 pgs: 89 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2024-09-13T15:13:16.518 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:16 smithi099 bash[87147]: cluster 2024-09-13T15:13:15.100202+0000 mgr.y (mgr.26252) 1415 : cluster [DBG] pgmap v1995: 89 pgs: 89 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2024-09-13T15:13:16.518 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:16 smithi099 bash[87147]: cluster 2024-09-13T15:13:15.100202+0000 mgr.y (mgr.26252) 1415 : cluster [DBG] pgmap v1995: 89 pgs: 89 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2024-09-13T15:13:16.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:16 smithi177 bash[68594]: cluster 2024-09-13T15:13:15.100202+0000 mgr.y (mgr.26252) 1415 : cluster [DBG] pgmap v1995: 89 pgs: 89 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2024-09-13T15:13:16.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:16 smithi177 bash[68594]: cluster 2024-09-13T15:13:15.100202+0000 mgr.y (mgr.26252) 1415 : cluster [DBG] pgmap v1995: 89 pgs: 89 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2024-09-13T15:13:16.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:16.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:16.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:16.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:16.748 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:16.771 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:16.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:16.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:16.877 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:16.917 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:16.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:16.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:17.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:17.046 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:17.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:17.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:17.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:17.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:17.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:17.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:17.280 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:17.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:17.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:17.421 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:17.455 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:17.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:17.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:17.560 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:17.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:17.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:17.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:17.601 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:13:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:13:17] "GET /metrics HTTP/1.1" 200 36773 "" "Prometheus/2.51.0" 2024-09-13T15:13:17.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:17.734 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:17.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:17.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:17.840 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:17.871 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:17.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:17.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:17.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:17.993 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:17.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:18.095 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:18.118 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:18.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:18.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:18.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:18.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:18.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:18.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:18.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:18 smithi177 bash[68594]: cluster 2024-09-13T15:13:17.100753+0000 mgr.y (mgr.26252) 1416 : cluster [DBG] pgmap v1996: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:13:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:18 smithi177 bash[68594]: cluster 2024-09-13T15:13:17.100753+0000 mgr.y (mgr.26252) 1416 : cluster [DBG] pgmap v1996: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:13:18.373 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:18.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:18.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:18.476 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:18.486 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:18 smithi099 bash[84576]: cluster 2024-09-13T15:13:17.100753+0000 mgr.y (mgr.26252) 1416 : cluster [DBG] pgmap v1996: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:13:18.486 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:18 smithi099 bash[84576]: cluster 2024-09-13T15:13:17.100753+0000 mgr.y (mgr.26252) 1416 : cluster [DBG] pgmap v1996: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:13:18.486 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:18 smithi099 bash[87147]: cluster 2024-09-13T15:13:17.100753+0000 mgr.y (mgr.26252) 1416 : cluster [DBG] pgmap v1996: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:13:18.486 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:18 smithi099 bash[87147]: cluster 2024-09-13T15:13:17.100753+0000 mgr.y (mgr.26252) 1416 : cluster [DBG] pgmap v1996: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2024-09-13T15:13:18.519 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:18.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:18.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:18.627 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:18.645 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:18.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:18.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:18.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:18.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:18.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:18.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:18.901 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:18.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:18.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:19.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:19.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:19.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:19.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:19.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:19.161 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:19.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:19.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:19.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:19.273 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:19 smithi177 bash[68594]: audit 2024-09-13T15:13:18.311091+0000 mon.a (mon.0) 2558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:19.273 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:19 smithi177 bash[68594]: audit 2024-09-13T15:13:18.311091+0000 mon.a (mon.0) 2558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:19.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:19.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:19.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:19.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:19.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:19.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:19.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:19.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:19.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:19.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:19.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:19 smithi099 bash[84576]: audit 2024-09-13T15:13:18.311091+0000 mon.a (mon.0) 2558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:19 smithi099 bash[84576]: audit 2024-09-13T15:13:18.311091+0000 mon.a (mon.0) 2558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:19 smithi099 bash[87147]: audit 2024-09-13T15:13:18.311091+0000 mon.a (mon.0) 2558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:19.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:19 smithi099 bash[87147]: audit 2024-09-13T15:13:18.311091+0000 mon.a (mon.0) 2558 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:19.616 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:13:19.616 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:13:19.616 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T15:13:19.617 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:13:19.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:19.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:19.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:19.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:19.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:19.800 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:19.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:19.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:19.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:19.935 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:19.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:19.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:20.040 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:20.063 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:20.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:20.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:20.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:20.197 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:20.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:20.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:20.300 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:20.330 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:20.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:20.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:20.447 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:13:20 smithi177 bash[61439]: ts=2024-09-13T15:13:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:13:20.447 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:20 smithi177 bash[68594]: cluster 2024-09-13T15:13:19.101147+0000 mgr.y (mgr.26252) 1417 : cluster [DBG] pgmap v1997: 89 pgs: 89 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2024-09-13T15:13:20.447 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:20 smithi177 bash[68594]: cluster 2024-09-13T15:13:19.101147+0000 mgr.y (mgr.26252) 1417 : cluster [DBG] pgmap v1997: 89 pgs: 89 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2024-09-13T15:13:20.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:20.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:20.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:20.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:20 smithi099 bash[84576]: cluster 2024-09-13T15:13:19.101147+0000 mgr.y (mgr.26252) 1417 : cluster [DBG] pgmap v1997: 89 pgs: 89 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2024-09-13T15:13:20.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:20 smithi099 bash[84576]: cluster 2024-09-13T15:13:19.101147+0000 mgr.y (mgr.26252) 1417 : cluster [DBG] pgmap v1997: 89 pgs: 89 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2024-09-13T15:13:20.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:20 smithi099 bash[87147]: cluster 2024-09-13T15:13:19.101147+0000 mgr.y (mgr.26252) 1417 : cluster [DBG] pgmap v1997: 89 pgs: 89 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2024-09-13T15:13:20.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:20 smithi099 bash[87147]: cluster 2024-09-13T15:13:19.101147+0000 mgr.y (mgr.26252) 1417 : cluster [DBG] pgmap v1997: 89 pgs: 89 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2024-09-13T15:13:20.577 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:20.600 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:20.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:20.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:20.709 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:20.753 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:20.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:20.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:20.860 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:20.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:20.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:20.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:21.000 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:21.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:21.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:21.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:21.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:21.148 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:21.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:21.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:21.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:21.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:21.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:21.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:21.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:21.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:21.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:21.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:21.518 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:21.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:21.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:21.659 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:21.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:21.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:21.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:21.849 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:21.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:21.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:21.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:21.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:21.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:21.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:22.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:22.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:22.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:22.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:22.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:22.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:22 smithi177 bash[68594]: cluster 2024-09-13T15:13:21.101731+0000 mgr.y (mgr.26252) 1418 : cluster [DBG] pgmap v1998: 89 pgs: 89 active+clean; 1.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-09-13T15:13:22.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:22 smithi177 bash[68594]: cluster 2024-09-13T15:13:21.101731+0000 mgr.y (mgr.26252) 1418 : cluster [DBG] pgmap v1998: 89 pgs: 89 active+clean; 1.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-09-13T15:13:22.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:22.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:22.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:22.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:22.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:22.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:22.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:22.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:22.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:22.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:22.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:22.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:22 smithi099 bash[84576]: cluster 2024-09-13T15:13:21.101731+0000 mgr.y (mgr.26252) 1418 : cluster [DBG] pgmap v1998: 89 pgs: 89 active+clean; 1.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-09-13T15:13:22.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:22 smithi099 bash[84576]: cluster 2024-09-13T15:13:21.101731+0000 mgr.y (mgr.26252) 1418 : cluster [DBG] pgmap v1998: 89 pgs: 89 active+clean; 1.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-09-13T15:13:22.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:22 smithi099 bash[87147]: cluster 2024-09-13T15:13:21.101731+0000 mgr.y (mgr.26252) 1418 : cluster [DBG] pgmap v1998: 89 pgs: 89 active+clean; 1.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-09-13T15:13:22.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:22 smithi099 bash[87147]: cluster 2024-09-13T15:13:21.101731+0000 mgr.y (mgr.26252) 1418 : cluster [DBG] pgmap v1998: 89 pgs: 89 active+clean; 1.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2024-09-13T15:13:22.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:22.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:22.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:22.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:22.743 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:22.761 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:22.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:22.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:22.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:22.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:22.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:22.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:22.999 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:23.024 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:23.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:23.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:23.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:23.151 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:23.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:23.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:23.254 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:23.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:23.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:23.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:23.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:23.426 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:23.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:23.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:23.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:23.545 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:13:23.545 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:13:23.579 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:23.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:23.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:23.682 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:23.710 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:23.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:23.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:23.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:23.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:23.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:23.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:23.947 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:23.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:23.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:23.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:24.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:24.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:24.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:24.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:24.201 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:24.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:24.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:24.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:24.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:24.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:24.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:24.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:24.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:24.471 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:24 smithi177 bash[68594]: cluster 2024-09-13T15:13:23.102641+0000 mgr.y (mgr.26252) 1419 : cluster [DBG] pgmap v1999: 89 pgs: 89 active+clean; 1024 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T15:13:24.471 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:24 smithi177 bash[68594]: cluster 2024-09-13T15:13:23.102641+0000 mgr.y (mgr.26252) 1419 : cluster [DBG] pgmap v1999: 89 pgs: 89 active+clean; 1024 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T15:13:24.487 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:24.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:24.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:24.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:24 smithi099 bash[84576]: cluster 2024-09-13T15:13:23.102641+0000 mgr.y (mgr.26252) 1419 : cluster [DBG] pgmap v1999: 89 pgs: 89 active+clean; 1024 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T15:13:24.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:24 smithi099 bash[84576]: cluster 2024-09-13T15:13:23.102641+0000 mgr.y (mgr.26252) 1419 : cluster [DBG] pgmap v1999: 89 pgs: 89 active+clean; 1024 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T15:13:24.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:24 smithi099 bash[87147]: cluster 2024-09-13T15:13:23.102641+0000 mgr.y (mgr.26252) 1419 : cluster [DBG] pgmap v1999: 89 pgs: 89 active+clean; 1024 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T15:13:24.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:24 smithi099 bash[87147]: cluster 2024-09-13T15:13:23.102641+0000 mgr.y (mgr.26252) 1419 : cluster [DBG] pgmap v1999: 89 pgs: 89 active+clean; 1024 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2024-09-13T15:13:24.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:24.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:24.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:24.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:24.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:24.747 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:24.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:24.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:24.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:24.891 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:24.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:24.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:24.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:25.032 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:25.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:25.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:25.137 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:25.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:25.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:25.265 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:25.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:25.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:25.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:25.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:25.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:25.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:25.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:25.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:25.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:25.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:25.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:25.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:25.702 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:25.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:25.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:25.806 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:25.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:25.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:25.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:25.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:25.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:25.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:25.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:26.038 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T15:13:26.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:26.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:26.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:26.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:26.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:26.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:26.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:26.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:26 smithi099 bash[84576]: cluster 2024-09-13T15:13:25.106467+0000 mgr.y (mgr.26252) 1420 : cluster [DBG] pgmap v2000: 89 pgs: 89 active+clean; 875 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T15:13:26.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:26 smithi099 bash[84576]: cluster 2024-09-13T15:13:25.106467+0000 mgr.y (mgr.26252) 1420 : cluster [DBG] pgmap v2000: 89 pgs: 89 active+clean; 875 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T15:13:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:26 smithi099 bash[87147]: cluster 2024-09-13T15:13:25.106467+0000 mgr.y (mgr.26252) 1420 : cluster [DBG] pgmap v2000: 89 pgs: 89 active+clean; 875 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T15:13:26.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:26 smithi099 bash[87147]: cluster 2024-09-13T15:13:25.106467+0000 mgr.y (mgr.26252) 1420 : cluster [DBG] pgmap v2000: 89 pgs: 89 active+clean; 875 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T15:13:26.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:26.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:26.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:26.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:26.471 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:26.492 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:26.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:26.508 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:13:26.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:26.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:26 smithi177 bash[68594]: cluster 2024-09-13T15:13:25.106467+0000 mgr.y (mgr.26252) 1420 : cluster [DBG] pgmap v2000: 89 pgs: 89 active+clean; 875 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T15:13:26.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:26 smithi177 bash[68594]: cluster 2024-09-13T15:13:25.106467+0000 mgr.y (mgr.26252) 1420 : cluster [DBG] pgmap v2000: 89 pgs: 89 active+clean; 875 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T15:13:26.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:26.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:26.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:26.725 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:26.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:26.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:26.797 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:13:26.798 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:13:26.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:26.877 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:13:26.877 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:13:26.877 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:26.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:26.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:26.909 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:13:26.909 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:13:26.950 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:13:26.951 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:13:26.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:26.989 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:13:26.989 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:13:27.003 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:27.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:27.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:27.014 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:13:27.014 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:13:27.044 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:13:27.045 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:13:27.069 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:13:27.069 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:13:27.109 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:27.119 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:13:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:13:27] "GET /metrics HTTP/1.1" 200 36773 "" "Prometheus/2.51.0" 2024-09-13T15:13:27.146 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:27.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:27.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:27.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:27.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:27 smithi177 bash[68594]: audit 2024-09-13T15:13:26.738954+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.99:0/1093712982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:27.261 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:27 smithi177 bash[68594]: audit 2024-09-13T15:13:26.738954+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.99:0/1093712982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:27.272 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:27.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:27.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:27.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:27 smithi099 bash[84576]: audit 2024-09-13T15:13:26.738954+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.99:0/1093712982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:27.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:27 smithi099 bash[84576]: audit 2024-09-13T15:13:26.738954+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.99:0/1093712982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:27 smithi099 bash[87147]: audit 2024-09-13T15:13:26.738954+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.99:0/1093712982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:27 smithi099 bash[87147]: audit 2024-09-13T15:13:26.738954+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.99:0/1093712982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:27.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:27.397 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:27.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:27.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:27.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:27.542 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:27.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:27.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:27.686 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:27.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:27.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:27.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:27.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:27.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:27.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:27.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:27.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:27.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:27.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:28.037 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:28.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:28.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:28.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:28.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:28.200 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:28.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:28.217 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:13:28.304 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:28.322 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:28.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:28.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:28.425 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:28.439 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:28 smithi099 bash[84576]: cluster 2024-09-13T15:13:27.107145+0000 mgr.y (mgr.26252) 1421 : cluster [DBG] pgmap v2001: 89 pgs: 89 active+clean; 838 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2024-09-13T15:13:28.439 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:28 smithi099 bash[84576]: cluster 2024-09-13T15:13:27.107145+0000 mgr.y (mgr.26252) 1421 : cluster [DBG] pgmap v2001: 89 pgs: 89 active+clean; 838 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2024-09-13T15:13:28.439 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:28 smithi099 bash[84576]: audit 2024-09-13T15:13:27.278019+0000 mon.a (mon.0) 2559 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:13:28.439 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:28 smithi099 bash[84576]: audit 2024-09-13T15:13:27.278019+0000 mon.a (mon.0) 2559 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:13:28.440 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:28 smithi099 bash[87147]: cluster 2024-09-13T15:13:27.107145+0000 mgr.y (mgr.26252) 1421 : cluster [DBG] pgmap v2001: 89 pgs: 89 active+clean; 838 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2024-09-13T15:13:28.440 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:28 smithi099 bash[87147]: cluster 2024-09-13T15:13:27.107145+0000 mgr.y (mgr.26252) 1421 : cluster [DBG] pgmap v2001: 89 pgs: 89 active+clean; 838 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2024-09-13T15:13:28.440 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:28 smithi099 bash[87147]: audit 2024-09-13T15:13:27.278019+0000 mon.a (mon.0) 2559 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:13:28.440 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:28 smithi099 bash[87147]: audit 2024-09-13T15:13:27.278019+0000 mon.a (mon.0) 2559 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:13:28.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:28.464 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:28 smithi177 bash[68594]: cluster 2024-09-13T15:13:27.107145+0000 mgr.y (mgr.26252) 1421 : cluster [DBG] pgmap v2001: 89 pgs: 89 active+clean; 838 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2024-09-13T15:13:28.464 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:28 smithi177 bash[68594]: cluster 2024-09-13T15:13:27.107145+0000 mgr.y (mgr.26252) 1421 : cluster [DBG] pgmap v2001: 89 pgs: 89 active+clean; 838 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2024-09-13T15:13:28.464 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:28 smithi177 bash[68594]: audit 2024-09-13T15:13:27.278019+0000 mon.a (mon.0) 2559 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:13:28.464 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:28 smithi177 bash[68594]: audit 2024-09-13T15:13:27.278019+0000 mon.a (mon.0) 2559 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:13:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:28.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:28.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:28.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:28.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:28.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:28.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:28.718 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:28.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:28.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:28.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:28.846 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:28.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:28.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:28.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:28.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:28.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:28.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:29.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:29.111 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:29.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:29.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:29.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:29.223 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:29 smithi177 bash[68594]: cluster 2024-09-13T15:13:28.213830+0000 mon.a (mon.0) 2560 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:13:29.223 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:29 smithi177 bash[68594]: cluster 2024-09-13T15:13:28.213830+0000 mon.a (mon.0) 2560 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:13:29.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:29 smithi177 bash[68594]: cluster 2024-09-13T15:13:28.213856+0000 mon.a (mon.0) 2561 : cluster [INF] Cluster is now healthy 2024-09-13T15:13:29.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:29 smithi177 bash[68594]: cluster 2024-09-13T15:13:28.213856+0000 mon.a (mon.0) 2561 : cluster [INF] Cluster is now healthy 2024-09-13T15:13:29.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:29 smithi177 bash[68594]: audit 2024-09-13T15:13:28.215110+0000 mon.a (mon.0) 2562 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:13:29.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:29 smithi177 bash[68594]: audit 2024-09-13T15:13:28.215110+0000 mon.a (mon.0) 2562 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:13:29.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:29 smithi177 bash[68594]: cluster 2024-09-13T15:13:28.223074+0000 mon.a (mon.0) 2563 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2024-09-13T15:13:29.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:29 smithi177 bash[68594]: cluster 2024-09-13T15:13:28.223074+0000 mon.a (mon.0) 2563 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2024-09-13T15:13:29.251 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:29.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:29.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:29.354 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:29.384 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:29.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:29.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:29.529 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:29.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:29.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:29.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:29.648 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[84576]: cluster 2024-09-13T15:13:28.213830+0000 mon.a (mon.0) 2560 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:13:29.648 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[84576]: cluster 2024-09-13T15:13:28.213830+0000 mon.a (mon.0) 2560 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:13:29.648 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[84576]: cluster 2024-09-13T15:13:28.213856+0000 mon.a (mon.0) 2561 : cluster [INF] Cluster is now healthy 2024-09-13T15:13:29.648 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[84576]: cluster 2024-09-13T15:13:28.213856+0000 mon.a (mon.0) 2561 : cluster [INF] Cluster is now healthy 2024-09-13T15:13:29.648 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[84576]: audit 2024-09-13T15:13:28.215110+0000 mon.a (mon.0) 2562 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:13:29.648 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[84576]: audit 2024-09-13T15:13:28.215110+0000 mon.a (mon.0) 2562 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:13:29.648 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[84576]: cluster 2024-09-13T15:13:28.223074+0000 mon.a (mon.0) 2563 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2024-09-13T15:13:29.648 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[84576]: cluster 2024-09-13T15:13:28.223074+0000 mon.a (mon.0) 2563 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2024-09-13T15:13:29.648 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[87147]: cluster 2024-09-13T15:13:28.213830+0000 mon.a (mon.0) 2560 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:13:29.649 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[87147]: cluster 2024-09-13T15:13:28.213830+0000 mon.a (mon.0) 2560 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:13:29.649 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[87147]: cluster 2024-09-13T15:13:28.213856+0000 mon.a (mon.0) 2561 : cluster [INF] Cluster is now healthy 2024-09-13T15:13:29.649 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[87147]: cluster 2024-09-13T15:13:28.213856+0000 mon.a (mon.0) 2561 : cluster [INF] Cluster is now healthy 2024-09-13T15:13:29.649 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[87147]: audit 2024-09-13T15:13:28.215110+0000 mon.a (mon.0) 2562 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:13:29.649 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[87147]: audit 2024-09-13T15:13:28.215110+0000 mon.a (mon.0) 2562 : audit [INF] from='client.? 172.21.15.99:0/3420094080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:13:29.649 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[87147]: cluster 2024-09-13T15:13:28.223074+0000 mon.a (mon.0) 2563 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2024-09-13T15:13:29.649 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:29 smithi099 bash[87147]: cluster 2024-09-13T15:13:28.223074+0000 mon.a (mon.0) 2563 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2024-09-13T15:13:29.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:29.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:29.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:29.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:29.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:29.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:29.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:29.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:29.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:29.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:30.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:30.042 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:30.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:30.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:30.147 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:30.186 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:30.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:30.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:30.292 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:30.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:30 smithi177 bash[68594]: cluster 2024-09-13T15:13:29.107646+0000 mgr.y (mgr.26252) 1422 : cluster [DBG] pgmap v2003: 89 pgs: 89 active+clean; 801 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2024-09-13T15:13:30.301 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:30 smithi177 bash[68594]: cluster 2024-09-13T15:13:29.107646+0000 mgr.y (mgr.26252) 1422 : cluster [DBG] pgmap v2003: 89 pgs: 89 active+clean; 801 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2024-09-13T15:13:30.301 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:13:30 smithi177 bash[61439]: ts=2024-09-13T15:13:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:13:30.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:30.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:30.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:30 smithi099 bash[84576]: cluster 2024-09-13T15:13:29.107646+0000 mgr.y (mgr.26252) 1422 : cluster [DBG] pgmap v2003: 89 pgs: 89 active+clean; 801 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2024-09-13T15:13:30.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:30 smithi099 bash[84576]: cluster 2024-09-13T15:13:29.107646+0000 mgr.y (mgr.26252) 1422 : cluster [DBG] pgmap v2003: 89 pgs: 89 active+clean; 801 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2024-09-13T15:13:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:30 smithi099 bash[87147]: cluster 2024-09-13T15:13:29.107646+0000 mgr.y (mgr.26252) 1422 : cluster [DBG] pgmap v2003: 89 pgs: 89 active+clean; 801 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2024-09-13T15:13:30.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:30 smithi099 bash[87147]: cluster 2024-09-13T15:13:29.107646+0000 mgr.y (mgr.26252) 1422 : cluster [DBG] pgmap v2003: 89 pgs: 89 active+clean; 801 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2024-09-13T15:13:30.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:30.452 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:30.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:30.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:30.562 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:30.599 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:30.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:30.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:30.705 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:30.740 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:30.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:30.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:30.876 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:30.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:30.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:30.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:31.005 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:31.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:31.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:31.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:31.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:31.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:31.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:31.249 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:31.272 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:31.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:31.375 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:31.385 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:31 smithi177 bash[68594]: cluster 2024-09-13T15:13:31.108204+0000 mgr.y (mgr.26252) 1423 : cluster [DBG] pgmap v2004: 89 pgs: 89 active+clean; 657 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T15:13:31.385 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:31 smithi177 bash[68594]: cluster 2024-09-13T15:13:31.108204+0000 mgr.y (mgr.26252) 1423 : cluster [DBG] pgmap v2004: 89 pgs: 89 active+clean; 657 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T15:13:31.403 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:31.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:31.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:31.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:31.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:31.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:31.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:31 smithi099 bash[84576]: cluster 2024-09-13T15:13:31.108204+0000 mgr.y (mgr.26252) 1423 : cluster [DBG] pgmap v2004: 89 pgs: 89 active+clean; 657 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T15:13:31.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:31 smithi099 bash[84576]: cluster 2024-09-13T15:13:31.108204+0000 mgr.y (mgr.26252) 1423 : cluster [DBG] pgmap v2004: 89 pgs: 89 active+clean; 657 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T15:13:31.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:31 smithi099 bash[87147]: cluster 2024-09-13T15:13:31.108204+0000 mgr.y (mgr.26252) 1423 : cluster [DBG] pgmap v2004: 89 pgs: 89 active+clean; 657 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T15:13:31.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:31 smithi099 bash[87147]: cluster 2024-09-13T15:13:31.108204+0000 mgr.y (mgr.26252) 1423 : cluster [DBG] pgmap v2004: 89 pgs: 89 active+clean; 657 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2024-09-13T15:13:31.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:31.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:31.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:31.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:31.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:31.796 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:31.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:31.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:31.902 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:31.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:31.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:31.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:32.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:32.065 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:32.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:32.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:32.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:32.208 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:32.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:32.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:32.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:32.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:32.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:32.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:32.473 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:32.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:32.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:32.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:32.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:32.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:32.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:32.707 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:32.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:32.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:32.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:32.840 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:32.887 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:32.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:32.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:32.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:33.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:33.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:33.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:33.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:33.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:33.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:33.255 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:33.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:33.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:33.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:33.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:13:33.391 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:33.429 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:33.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:33.449 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:13:33.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:33.575 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:33.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:33.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:33.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:33.705 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:33.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:33.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:33.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:33.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:33.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:33.955 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:33.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:33.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:34.058 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:34.085 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:34.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:34.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:34.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[84576]: cluster 2024-09-13T15:13:33.108682+0000 mgr.y (mgr.26252) 1424 : cluster [DBG] pgmap v2005: 89 pgs: 89 active+clean; 634 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-09-13T15:13:34.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[84576]: cluster 2024-09-13T15:13:33.108682+0000 mgr.y (mgr.26252) 1424 : cluster [DBG] pgmap v2005: 89 pgs: 89 active+clean; 634 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-09-13T15:13:34.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[84576]: audit 2024-09-13T15:13:33.315643+0000 mon.a (mon.0) 2564 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:34.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[84576]: audit 2024-09-13T15:13:33.315643+0000 mon.a (mon.0) 2564 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:34.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[84576]: audit 2024-09-13T15:13:33.316816+0000 mon.a (mon.0) 2565 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:34.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[84576]: audit 2024-09-13T15:13:33.316816+0000 mon.a (mon.0) 2565 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:34.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[87147]: cluster 2024-09-13T15:13:33.108682+0000 mgr.y (mgr.26252) 1424 : cluster [DBG] pgmap v2005: 89 pgs: 89 active+clean; 634 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-09-13T15:13:34.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[87147]: cluster 2024-09-13T15:13:33.108682+0000 mgr.y (mgr.26252) 1424 : cluster [DBG] pgmap v2005: 89 pgs: 89 active+clean; 634 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-09-13T15:13:34.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[87147]: audit 2024-09-13T15:13:33.315643+0000 mon.a (mon.0) 2564 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:34.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[87147]: audit 2024-09-13T15:13:33.315643+0000 mon.a (mon.0) 2564 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:34.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[87147]: audit 2024-09-13T15:13:33.316816+0000 mon.a (mon.0) 2565 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:34.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:34 smithi099 bash[87147]: audit 2024-09-13T15:13:33.316816+0000 mon.a (mon.0) 2565 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:34.225 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:34.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:34.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:34.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:34.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:34.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:34.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:34.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:34.466 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:34 smithi177 bash[68594]: cluster 2024-09-13T15:13:33.108682+0000 mgr.y (mgr.26252) 1424 : cluster [DBG] pgmap v2005: 89 pgs: 89 active+clean; 634 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-09-13T15:13:34.466 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:34 smithi177 bash[68594]: cluster 2024-09-13T15:13:33.108682+0000 mgr.y (mgr.26252) 1424 : cluster [DBG] pgmap v2005: 89 pgs: 89 active+clean; 634 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2024-09-13T15:13:34.466 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:34 smithi177 bash[68594]: audit 2024-09-13T15:13:33.315643+0000 mon.a (mon.0) 2564 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:34.466 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:34 smithi177 bash[68594]: audit 2024-09-13T15:13:33.315643+0000 mon.a (mon.0) 2564 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:34.466 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:34 smithi177 bash[68594]: audit 2024-09-13T15:13:33.316816+0000 mon.a (mon.0) 2565 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:34.466 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:34 smithi177 bash[68594]: audit 2024-09-13T15:13:33.316816+0000 mon.a (mon.0) 2565 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:34.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:34.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:34.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:34.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:34.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:34.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:34.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:34.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:34.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:34.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:34.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:34.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:34.887 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:34.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:34.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:34.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:35.012 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:35.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:35.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:35.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:35.138 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:35.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:35.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:35.242 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:35.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:35.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:35.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:35 smithi177 bash[68594]: audit 2024-09-13T15:13:35.034749+0000 mon.a (mon.0) 2566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:13:35.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:35 smithi177 bash[68594]: audit 2024-09-13T15:13:35.034749+0000 mon.a (mon.0) 2566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:13:35.375 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:35.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:35.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:35.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:35.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:35.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:35.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:35.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:35.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:35 smithi099 bash[84576]: audit 2024-09-13T15:13:35.034749+0000 mon.a (mon.0) 2566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:13:35.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:35 smithi099 bash[84576]: audit 2024-09-13T15:13:35.034749+0000 mon.a (mon.0) 2566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:13:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:35 smithi099 bash[87147]: audit 2024-09-13T15:13:35.034749+0000 mon.a (mon.0) 2566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:13:35.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:35 smithi099 bash[87147]: audit 2024-09-13T15:13:35.034749+0000 mon.a (mon.0) 2566 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:13:35.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:35.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:35.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:35.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:35.831 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:35.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:35.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:35.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:35.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:35.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:35.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:36.070 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:36.074 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:13:36.090 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:36.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:36.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:36.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:36.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[84576]: cluster 2024-09-13T15:13:35.109304+0000 mgr.y (mgr.26252) 1425 : cluster [DBG] pgmap v2006: 89 pgs: 89 active+clean; 468 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:36.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[84576]: cluster 2024-09-13T15:13:35.109304+0000 mgr.y (mgr.26252) 1425 : cluster [DBG] pgmap v2006: 89 pgs: 89 active+clean; 468 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:36.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[84576]: audit 2024-09-13T15:13:35.893304+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[84576]: audit 2024-09-13T15:13:35.893304+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.207 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[84576]: audit 2024-09-13T15:13:35.900838+0000 mon.a (mon.0) 2568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.208 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[84576]: audit 2024-09-13T15:13:35.900838+0000 mon.a (mon.0) 2568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[87147]: cluster 2024-09-13T15:13:35.109304+0000 mgr.y (mgr.26252) 1425 : cluster [DBG] pgmap v2006: 89 pgs: 89 active+clean; 468 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:36.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[87147]: cluster 2024-09-13T15:13:35.109304+0000 mgr.y (mgr.26252) 1425 : cluster [DBG] pgmap v2006: 89 pgs: 89 active+clean; 468 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:36.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[87147]: audit 2024-09-13T15:13:35.893304+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[87147]: audit 2024-09-13T15:13:35.893304+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[87147]: audit 2024-09-13T15:13:35.900838+0000 mon.a (mon.0) 2568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.208 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:36 smithi099 bash[87147]: audit 2024-09-13T15:13:35.900838+0000 mon.a (mon.0) 2568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:36.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:36.352 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:36 smithi177 bash[68594]: cluster 2024-09-13T15:13:35.109304+0000 mgr.y (mgr.26252) 1425 : cluster [DBG] pgmap v2006: 89 pgs: 89 active+clean; 468 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:36 smithi177 bash[68594]: cluster 2024-09-13T15:13:35.109304+0000 mgr.y (mgr.26252) 1425 : cluster [DBG] pgmap v2006: 89 pgs: 89 active+clean; 468 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-09-13T15:13:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:36 smithi177 bash[68594]: audit 2024-09-13T15:13:35.893304+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:36 smithi177 bash[68594]: audit 2024-09-13T15:13:35.893304+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:36 smithi177 bash[68594]: audit 2024-09-13T15:13:35.900838+0000 mon.a (mon.0) 2568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:36 smithi177 bash[68594]: audit 2024-09-13T15:13:35.900838+0000 mon.a (mon.0) 2568 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:36.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:36.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:36.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:36.499 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:36.522 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:36.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:36.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:36.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:36.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:36.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:36.690 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T15:13:36.692 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T15:13:36.784 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:36.806 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:36.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:36.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:36.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:36.932 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:36.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:36.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:37.038 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:37.061 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:37.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:37.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:37.186 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:37.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:37.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:37.289 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:37.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:37.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:37.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:37.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:13:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:13:37] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:13:37.418 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:37.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:37.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:37.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:37.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:37.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:37.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:37.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:37.699 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:37.729 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:37.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:37.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:37.833 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:37.860 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:37.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:37.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:37.962 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:37.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:37.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:37.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:38.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:38.133 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:38.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:38.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:38.238 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:38.272 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:38.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:38.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:38.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:38 smithi177 bash[68594]: audit 2024-09-13T15:13:36.676778+0000 mgr.y (mgr.26252) 1426 : audit [DBG] from='client.36159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:13:38.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:38 smithi177 bash[68594]: audit 2024-09-13T15:13:36.676778+0000 mgr.y (mgr.26252) 1426 : audit [DBG] from='client.36159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:13:38.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:38 smithi177 bash[68594]: cluster 2024-09-13T15:13:37.110117+0000 mgr.y (mgr.26252) 1427 : cluster [DBG] pgmap v2007: 89 pgs: 89 active+clean; 425 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:38.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:38 smithi177 bash[68594]: cluster 2024-09-13T15:13:37.110117+0000 mgr.y (mgr.26252) 1427 : cluster [DBG] pgmap v2007: 89 pgs: 89 active+clean; 425 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:38.376 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:38.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:38.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:38.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:38.411 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:38 smithi099 bash[84576]: audit 2024-09-13T15:13:36.676778+0000 mgr.y (mgr.26252) 1426 : audit [DBG] from='client.36159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:13:38.411 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:38 smithi099 bash[84576]: audit 2024-09-13T15:13:36.676778+0000 mgr.y (mgr.26252) 1426 : audit [DBG] from='client.36159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:13:38.411 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:38 smithi099 bash[84576]: cluster 2024-09-13T15:13:37.110117+0000 mgr.y (mgr.26252) 1427 : cluster [DBG] pgmap v2007: 89 pgs: 89 active+clean; 425 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:38.411 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:38 smithi099 bash[84576]: cluster 2024-09-13T15:13:37.110117+0000 mgr.y (mgr.26252) 1427 : cluster [DBG] pgmap v2007: 89 pgs: 89 active+clean; 425 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:38.411 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:38 smithi099 bash[87147]: audit 2024-09-13T15:13:36.676778+0000 mgr.y (mgr.26252) 1426 : audit [DBG] from='client.36159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:13:38.411 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:38 smithi099 bash[87147]: audit 2024-09-13T15:13:36.676778+0000 mgr.y (mgr.26252) 1426 : audit [DBG] from='client.36159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:13:38.412 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:38 smithi099 bash[87147]: cluster 2024-09-13T15:13:37.110117+0000 mgr.y (mgr.26252) 1427 : cluster [DBG] pgmap v2007: 89 pgs: 89 active+clean; 425 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:38.412 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:38 smithi099 bash[87147]: cluster 2024-09-13T15:13:37.110117+0000 mgr.y (mgr.26252) 1427 : cluster [DBG] pgmap v2007: 89 pgs: 89 active+clean; 425 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2024-09-13T15:13:38.512 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:38.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:38.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:38.643 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:38.683 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:38.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:38.790 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:38.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:38.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:38.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:38.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:38.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:38.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:38.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:39.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:39.081 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:39.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:39.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:39.185 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:39.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:39.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:39.323 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:39.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:39.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:39.455 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:39.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:39.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:39.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:39.594 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:39.628 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:39.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:39.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:39.643 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":2006,"stamp":"2024-09-13T15:13:35.108860+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":490608613,"num_objects":7504,"num_object_clones":0,"num_object_copies":22512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7504,"num_whiteouts":0,"num_read":333,"num_read_kb":788,"num_write":196196,"num_write_kb":6512380,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":1787554,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138648,"ondisk_log_size":138648,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7211044,"kb_used_data":1614560,"kb_used_omap":397,"kb_used_meta":5596018,"kb_avail":742750172,"statfs":{"total":767960285184,"available":760576176128,"internally_reserved":0,"allocated":1653309440,"data_stored":1617901297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":406940,"internal_metadata":5730323044},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,1,4,4,5,2,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-426704896,"num_objects":-6511,"num_object_clones":0,"num_object_copies":-19533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6511,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002892"},"pg_stats":[{"pgid":"45.f","version":"1094'12131","reported_seq":12161,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:33.524515+0000","last_change":"2024-09-13T15:09:14.125167+0000","last_active":"2024-09-13T15:13:33.524515+0000","last_peered":"2024-09-13T15:13:33.524515+0000","last_clean":"2024-09-13T15:13:33.524515+0000","last_became_active":"2024-09-13T15:09:14.124779+0000","last_became_peered":"2024-09-13T15:09:14.124779+0000","last_unstale":"2024-09-13T15:13:33.524515+0000","last_undegraded":"2024-09-13T15:13:33.524515+0000","last_fullsized":"2024-09-13T15:13:33.524515+0000","mapping_epoch":1084,"log_start":"1088'2100","ondisk_log_start":"1088'2100","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":10031,"log_dups_size":0,"ondisk_log_size":10031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:25:04.199874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29687808,"num_objects":453,"num_object_clones":0,"num_object_copies":1359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12131,"num_write_kb":402688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1094'11974","reported_seq":11992,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:30.230328+0000","last_change":"2024-09-13T15:09:14.250241+0000","last_active":"2024-09-13T15:13:30.230328+0000","last_peered":"2024-09-13T15:13:30.230328+0000","last_clean":"2024-09-13T15:13:30.230328+0000","last_became_active":"2024-09-13T15:09:14.249362+0000","last_became_peered":"2024-09-13T15:09:14.249362+0000","last_unstale":"2024-09-13T15:13:30.230328+0000","last_undegraded":"2024-09-13T15:13:30.230328+0000","last_fullsized":"2024-09-13T15:13:30.230328+0000","mapping_epoch":1084,"log_start":"1088'3800","ondisk_log_start":"1088'3800","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8174,"log_dups_size":0,"ondisk_log_size":8174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:50:46.394646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39321600,"num_objects":600,"num_object_clones":0,"num_object_copies":1800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11974,"num_write_kb":402368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1094'12465","reported_seq":12498,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:34.950066+0000","last_change":"2024-09-13T15:09:14.121386+0000","last_active":"2024-09-13T15:13:34.950066+0000","last_peered":"2024-09-13T15:13:34.950066+0000","last_clean":"2024-09-13T15:13:34.950066+0000","last_became_active":"2024-09-13T15:09:14.120505+0000","last_became_peered":"2024-09-13T15:09:14.120505+0000","last_unstale":"2024-09-13T15:13:34.950066+0000","last_undegraded":"2024-09-13T15:13:34.950066+0000","last_fullsized":"2024-09-13T15:13:34.950066+0000","mapping_epoch":1084,"log_start":"1089'4300","ondisk_log_start":"1089'4300","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8165,"log_dups_size":0,"ondisk_log_size":8165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:03.545701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28442648,"num_objects":435,"num_object_clones":0,"num_object_copies":1305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":435,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12465,"num_write_kb":412737,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1094'12461","reported_seq":12478,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:34.812539+0000","last_change":"2024-09-13T15:09:14.118650+0000","last_active":"2024-09-13T15:13:34.812539+0000","last_peered":"2024-09-13T15:13:34.812539+0000","last_clean":"2024-09-13T15:13:34.812539+0000","last_became_active":"2024-09-13T15:09:14.117877+0000","last_became_peered":"2024-09-13T15:09:14.117877+0000","last_unstale":"2024-09-13T15:13:34.812539+0000","last_undegraded":"2024-09-13T15:13:34.812539+0000","last_fullsized":"2024-09-13T15:13:34.812539+0000","mapping_epoch":1084,"log_start":"1088'3600","ondisk_log_start":"1088'3600","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8861,"log_dups_size":0,"ondisk_log_size":8861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:45:40.704193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26542080,"num_objects":405,"num_object_clones":0,"num_object_copies":1215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12461,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1094'12558","reported_seq":12590,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:34.473727+0000","last_change":"2024-09-13T15:09:14.248423+0000","last_active":"2024-09-13T15:13:34.473727+0000","last_peered":"2024-09-13T15:13:34.473727+0000","last_clean":"2024-09-13T15:13:34.473727+0000","last_became_active":"2024-09-13T15:09:14.248033+0000","last_became_peered":"2024-09-13T15:09:14.248033+0000","last_unstale":"2024-09-13T15:13:34.473727+0000","last_undegraded":"2024-09-13T15:13:34.473727+0000","last_fullsized":"2024-09-13T15:13:34.473727+0000","mapping_epoch":1084,"log_start":"1089'4800","ondisk_log_start":"1089'4800","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":7758,"log_dups_size":0,"ondisk_log_size":7758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:42:41.935560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30408704,"num_objects":464,"num_object_clones":0,"num_object_copies":1392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12558,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1094'12093","reported_seq":12109,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:34.843514+0000","last_change":"2024-09-13T15:09:14.117758+0000","last_active":"2024-09-13T15:13:34.843514+0000","last_peered":"2024-09-13T15:13:34.843514+0000","last_clean":"2024-09-13T15:13:34.843514+0000","last_became_active":"2024-09-13T15:09:14.116937+0000","last_became_peered":"2024-09-13T15:09:14.116937+0000","last_unstale":"2024-09-13T15:13:34.843514+0000","last_undegraded":"2024-09-13T15:13:34.843514+0000","last_fullsized":"2024-09-13T15:13:34.843514+0000","mapping_epoch":1084,"log_start":"1088'3200","ondisk_log_start":"1088'3200","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8893,"log_dups_size":0,"ondisk_log_size":8893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:02:33.216646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26542080,"num_objects":405,"num_object_clones":0,"num_object_copies":1215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12093,"num_write_kb":399936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1094'11868","reported_seq":11886,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:34.806134+0000","last_change":"2024-09-13T15:09:14.116549+0000","last_active":"2024-09-13T15:13:34.806134+0000","last_peered":"2024-09-13T15:13:34.806134+0000","last_clean":"2024-09-13T15:13:34.806134+0000","last_became_active":"2024-09-13T15:09:14.115720+0000","last_became_peered":"2024-09-13T15:09:14.115720+0000","last_unstale":"2024-09-13T15:13:34.806134+0000","last_undegraded":"2024-09-13T15:13:34.806134+0000","last_fullsized":"2024-09-13T15:13:34.806134+0000","mapping_epoch":1084,"log_start":"1088'3000","ondisk_log_start":"1088'3000","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8868,"log_dups_size":0,"ondisk_log_size":8868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:11:55.430644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28442624,"num_objects":434,"num_object_clones":0,"num_object_copies":1302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11868,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1094'12156","reported_seq":12189,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:34.538903+0000","last_change":"2024-09-13T15:09:14.248418+0000","last_active":"2024-09-13T15:13:34.538903+0000","last_peered":"2024-09-13T15:13:34.538903+0000","last_clean":"2024-09-13T15:13:34.538903+0000","last_became_active":"2024-09-13T15:09:14.248010+0000","last_became_peered":"2024-09-13T15:09:14.248010+0000","last_unstale":"2024-09-13T15:13:34.538903+0000","last_undegraded":"2024-09-13T15:13:34.538903+0000","last_fullsized":"2024-09-13T15:13:34.538903+0000","mapping_epoch":1084,"log_start":"1089'4400","ondisk_log_start":"1089'4400","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":7756,"log_dups_size":0,"ondisk_log_size":7756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:23:13.892973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28311552,"num_objects":432,"num_object_clones":0,"num_object_copies":1296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12156,"num_write_kb":402816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1094'12285","reported_seq":12303,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:34.779303+0000","last_change":"2024-09-13T15:09:14.111841+0000","last_active":"2024-09-13T15:13:34.779303+0000","last_peered":"2024-09-13T15:13:34.779303+0000","last_clean":"2024-09-13T15:13:34.779303+0000","last_became_active":"2024-09-13T15:09:14.111065+0000","last_became_peered":"2024-09-13T15:09:14.111065+0000","last_unstale":"2024-09-13T15:13:34.779303+0000","last_undegraded":"2024-09-13T15:13:34.779303+0000","last_fullsized":"2024-09-13T15:13:34.779303+0000","mapping_epoch":1084,"log_start":"1088'3400","ondisk_log_start":"1088'3400","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8885,"log_dups_size":0,"ondisk_log_size":8885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:34:35.020415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27590656,"num_objects":421,"num_object_clones":0,"num_object_copies":1263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12285,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1094'11869","reported_seq":11899,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:34.911654+0000","last_change":"2024-09-13T15:09:14.248685+0000","last_active":"2024-09-13T15:13:34.911654+0000","last_peered":"2024-09-13T15:13:34.911654+0000","last_clean":"2024-09-13T15:13:34.911654+0000","last_became_active":"2024-09-13T15:09:14.248279+0000","last_became_peered":"2024-09-13T15:09:14.248279+0000","last_unstale":"2024-09-13T15:13:34.911654+0000","last_undegraded":"2024-09-13T15:13:34.911654+0000","last_fullsized":"2024-09-13T15:13:34.911654+0000","mapping_epoch":1084,"log_start":"1088'3700","ondisk_log_start":"1088'3700","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8169,"log_dups_size":0,"ondisk_log_size":8169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:57.809770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26279936,"num_objects":401,"num_object_clones":0,"num_object_copies":1203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11869,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1094'12176","reported_seq":12209,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:33.458958+0000","last_change":"2024-09-13T15:09:14.123642+0000","last_active":"2024-09-13T15:13:33.458958+0000","last_peered":"2024-09-13T15:13:33.458958+0000","last_clean":"2024-09-13T15:13:33.458958+0000","last_became_active":"2024-09-13T15:09:14.122841+0000","last_became_peered":"2024-09-13T15:09:14.122841+0000","last_unstale":"2024-09-13T15:13:33.458958+0000","last_undegraded":"2024-09-13T15:13:33.458958+0000","last_fullsized":"2024-09-13T15:13:33.458958+0000","mapping_epoch":1084,"log_start":"1088'2100","ondisk_log_start":"1088'2100","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":10076,"log_dups_size":0,"ondisk_log_size":10076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:03.699610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31326208,"num_objects":478,"num_object_clones":0,"num_object_copies":1434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12176,"num_write_kb":404928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.4","version":"1094'12446","reported_seq":12479,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:32.725255+0000","last_change":"2024-09-13T15:09:14.249192+0000","last_active":"2024-09-13T15:13:32.725255+0000","last_peered":"2024-09-13T15:13:32.725255+0000","last_clean":"2024-09-13T15:13:32.725255+0000","last_became_active":"2024-09-13T15:09:14.248794+0000","last_became_peered":"2024-09-13T15:09:14.248794+0000","last_unstale":"2024-09-13T15:13:32.725255+0000","last_undegraded":"2024-09-13T15:13:32.725255+0000","last_fullsized":"2024-09-13T15:13:32.725255+0000","mapping_epoch":1084,"log_start":"1088'2400","ondisk_log_start":"1088'2400","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":10046,"log_dups_size":0,"ondisk_log_size":10046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:42:50.874084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32768000,"num_objects":500,"num_object_clones":0,"num_object_copies":1500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12446,"num_write_kb":414272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1094'12444","reported_seq":12477,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:34.925219+0000","last_change":"2024-09-13T15:09:14.122677+0000","last_active":"2024-09-13T15:13:34.925219+0000","last_peered":"2024-09-13T15:13:34.925219+0000","last_clean":"2024-09-13T15:13:34.925219+0000","last_became_active":"2024-09-13T15:09:14.121849+0000","last_became_peered":"2024-09-13T15:09:14.121849+0000","last_unstale":"2024-09-13T15:13:34.925219+0000","last_undegraded":"2024-09-13T15:13:34.925219+0000","last_fullsized":"2024-09-13T15:13:34.925219+0000","mapping_epoch":1084,"log_start":"1089'4300","ondisk_log_start":"1089'4300","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8144,"log_dups_size":0,"ondisk_log_size":8144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:50:54.801961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25952256,"num_objects":396,"num_object_clones":0,"num_object_copies":1188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12444,"num_write_kb":410880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1094'12335","reported_seq":12353,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:30.217003+0000","last_change":"2024-09-13T15:09:14.120946+0000","last_active":"2024-09-13T15:13:30.217003+0000","last_peered":"2024-09-13T15:13:30.217003+0000","last_clean":"2024-09-13T15:13:30.217003+0000","last_became_active":"2024-09-13T15:09:14.120116+0000","last_became_peered":"2024-09-13T15:09:14.120116+0000","last_unstale":"2024-09-13T15:13:30.217003+0000","last_undegraded":"2024-09-13T15:13:30.217003+0000","last_fullsized":"2024-09-13T15:13:30.217003+0000","mapping_epoch":1084,"log_start":"1089'4200","ondisk_log_start":"1089'4200","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8135,"log_dups_size":0,"ondisk_log_size":8135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:18.231609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43057152,"num_objects":657,"num_object_clones":0,"num_object_copies":1971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12335,"num_write_kb":415744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1094'12280","reported_seq":12312,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:31.610890+0000","last_change":"2024-09-13T15:09:14.250659+0000","last_active":"2024-09-13T15:13:31.610890+0000","last_peered":"2024-09-13T15:13:31.610890+0000","last_clean":"2024-09-13T15:13:31.610890+0000","last_became_active":"2024-09-13T15:09:14.249762+0000","last_became_peered":"2024-09-13T15:09:14.249762+0000","last_unstale":"2024-09-13T15:13:31.610890+0000","last_undegraded":"2024-09-13T15:13:31.610890+0000","last_fullsized":"2024-09-13T15:13:31.610890+0000","mapping_epoch":1084,"log_start":"1088'3900","ondisk_log_start":"1088'3900","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8380,"log_dups_size":0,"ondisk_log_size":8380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:20.456400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38797312,"num_objects":592,"num_object_clones":0,"num_object_copies":1776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12280,"num_write_kb":411904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"45.0","version":"1094'12443","reported_seq":12475,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:34.904267+0000","last_change":"2024-09-13T15:09:14.248482+0000","last_active":"2024-09-13T15:13:34.904267+0000","last_peered":"2024-09-13T15:13:34.904267+0000","last_clean":"2024-09-13T15:13:34.904267+0000","last_became_active":"2024-09-13T15:09:14.248157+0000","last_became_peered":"2024-09-13T15:09:14.248157+0000","last_unstale":"2024-09-13T15:13:34.904267+0000","last_undegraded":"2024-09-13T15:13:34.904267+0000","last_fullsized":"2024-09-13T15:13:34.904267+0000","mapping_epoch":1084,"log_start":"1089'4300","ondisk_log_start":"1089'4300","created":1084,"last_epoch_clean":1085,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:09:13.086069+0000","last_clean_scrub_stamp":"2024-09-13T15:09:13.086069+0000","objects_scrubbed":0,"log_size":8143,"log_dups_size":0,"ondisk_log_size":8143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:46:36.089130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26542080,"num_objects":405,"num_object_clones":0,"num_object_copies":1215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12443,"num_write_kb":411136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":106,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.194604+0000","last_change":"2024-09-13T15:08:52.195462+0000","last_active":"2024-09-13T15:12:57.194604+0000","last_peered":"2024-09-13T15:12:57.194604+0000","last_clean":"2024-09-13T15:12:57.194604+0000","last_became_active":"2024-09-13T15:08:52.174050+0000","last_became_peered":"2024-09-13T15:08:52.174050+0000","last_unstale":"2024-09-13T15:12:57.194604+0000","last_undegraded":"2024-09-13T15:12:57.194604+0000","last_fullsized":"2024-09-13T15:12:57.194604+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:58.561367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":320,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.314434+0000","last_change":"2024-09-13T15:08:52.180022+0000","last_active":"2024-09-13T15:13:28.314434+0000","last_peered":"2024-09-13T15:13:28.314434+0000","last_clean":"2024-09-13T15:13:28.314434+0000","last_became_active":"2024-09-13T15:08:52.179546+0000","last_became_peered":"2024-09-13T15:08:52.179546+0000","last_unstale":"2024-09-13T15:13:28.314434+0000","last_undegraded":"2024-09-13T15:13:28.314434+0000","last_fullsized":"2024-09-13T15:13:28.314434+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:15:47.462895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1315,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442644+0000","last_change":"2024-09-13T15:08:52.175606+0000","last_active":"2024-09-13T15:13:28.442644+0000","last_peered":"2024-09-13T15:13:28.442644+0000","last_clean":"2024-09-13T15:13:28.442644+0000","last_became_active":"2024-09-13T15:08:52.175210+0000","last_became_peered":"2024-09-13T15:08:52.175210+0000","last_unstale":"2024-09-13T15:13:28.442644+0000","last_undegraded":"2024-09-13T15:13:28.442644+0000","last_fullsized":"2024-09-13T15:13:28.442644+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:16.692008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":125,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.194452+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T15:12:57.194452+0000","last_peered":"2024-09-13T15:12:57.194452+0000","last_clean":"2024-09-13T15:12:57.194452+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T15:12:57.194452+0000","last_undegraded":"2024-09-13T15:12:57.194452+0000","last_fullsized":"2024-09-13T15:12:57.194452+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1107,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.194476+0000","last_change":"2024-09-13T15:08:52.205951+0000","last_active":"2024-09-13T15:12:57.194476+0000","last_peered":"2024-09-13T15:12:57.194476+0000","last_clean":"2024-09-13T15:12:57.194476+0000","last_became_active":"2024-09-13T15:08:52.183627+0000","last_became_peered":"2024-09-13T15:08:52.183627+0000","last_unstale":"2024-09-13T15:12:57.194476+0000","last_undegraded":"2024-09-13T15:12:57.194476+0000","last_fullsized":"2024-09-13T15:12:57.194476+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:14:14.282531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":303,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.314927+0000","last_change":"2024-09-13T15:05:34.086551+0000","last_active":"2024-09-13T15:13:28.314927+0000","last_peered":"2024-09-13T15:13:28.314927+0000","last_clean":"2024-09-13T15:13:28.314927+0000","last_became_active":"2024-09-13T15:05:34.086077+0000","last_became_peered":"2024-09-13T15:05:34.086077+0000","last_unstale":"2024-09-13T15:13:28.314927+0000","last_undegraded":"2024-09-13T15:13:28.314927+0000","last_fullsized":"2024-09-13T15:13:28.314927+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:33:47.917777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1124,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.194361+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T15:12:57.194361+0000","last_peered":"2024-09-13T15:12:57.194361+0000","last_clean":"2024-09-13T15:12:57.194361+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T15:12:57.194361+0000","last_undegraded":"2024-09-13T15:12:57.194361+0000","last_fullsized":"2024-09-13T15:12:57.194361+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":317,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.315539+0000","last_change":"2024-09-13T15:05:34.075607+0000","last_active":"2024-09-13T15:13:28.315539+0000","last_peered":"2024-09-13T15:13:28.315539+0000","last_clean":"2024-09-13T15:13:28.315539+0000","last_became_active":"2024-09-13T15:05:34.075079+0000","last_became_peered":"2024-09-13T15:05:34.075079+0000","last_unstale":"2024-09-13T15:13:28.315539+0000","last_undegraded":"2024-09-13T15:13:28.315539+0000","last_fullsized":"2024-09-13T15:13:28.315539+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:05.384206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1360,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.315197+0000","last_change":"2024-09-13T15:07:56.108198+0000","last_active":"2024-09-13T15:13:28.315197+0000","last_peered":"2024-09-13T15:13:28.315197+0000","last_clean":"2024-09-13T15:13:28.315197+0000","last_became_active":"2024-09-13T15:07:56.107824+0000","last_became_peered":"2024-09-13T15:07:56.107824+0000","last_unstale":"2024-09-13T15:13:28.315197+0000","last_undegraded":"2024-09-13T15:13:28.315197+0000","last_fullsized":"2024-09-13T15:13:28.315197+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:19.128127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1508,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.455535+0000","last_change":"2024-09-13T15:05:34.082773+0000","last_active":"2024-09-13T15:13:28.455535+0000","last_peered":"2024-09-13T15:13:28.455535+0000","last_clean":"2024-09-13T15:13:28.455535+0000","last_became_active":"2024-09-13T15:05:34.082408+0000","last_became_peered":"2024-09-13T15:05:34.082408+0000","last_unstale":"2024-09-13T15:13:28.455535+0000","last_undegraded":"2024-09-13T15:13:28.455535+0000","last_fullsized":"2024-09-13T15:13:28.455535+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:13.447546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":306,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.314830+0000","last_change":"2024-09-13T14:52:43.597400+0000","last_active":"2024-09-13T15:13:28.314830+0000","last_peered":"2024-09-13T15:13:28.314830+0000","last_clean":"2024-09-13T15:13:28.314830+0000","last_became_active":"2024-09-13T14:52:43.596294+0000","last_became_peered":"2024-09-13T14:52:43.596294+0000","last_unstale":"2024-09-13T15:13:28.314830+0000","last_undegraded":"2024-09-13T15:13:28.314830+0000","last_fullsized":"2024-09-13T15:13:28.314830+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:35.218171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1370,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.439627+0000","last_change":"2024-09-13T15:05:34.070717+0000","last_active":"2024-09-13T15:13:28.439627+0000","last_peered":"2024-09-13T15:13:28.439627+0000","last_clean":"2024-09-13T15:13:28.439627+0000","last_became_active":"2024-09-13T15:05:34.069929+0000","last_became_peered":"2024-09-13T15:05:34.069929+0000","last_unstale":"2024-09-13T15:13:28.439627+0000","last_undegraded":"2024-09-13T15:13:28.439627+0000","last_fullsized":"2024-09-13T15:13:28.439627+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:44:19.208697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":207,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.440366+0000","last_change":"2024-09-13T15:05:34.071133+0000","last_active":"2024-09-13T15:13:28.440366+0000","last_peered":"2024-09-13T15:13:28.440366+0000","last_clean":"2024-09-13T15:13:28.440366+0000","last_became_active":"2024-09-13T15:05:34.070519+0000","last_became_peered":"2024-09-13T15:05:34.070519+0000","last_unstale":"2024-09-13T15:13:28.440366+0000","last_undegraded":"2024-09-13T15:13:28.440366+0000","last_fullsized":"2024-09-13T15:13:28.440366+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:02.617403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1584,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.313394+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T15:13:28.313394+0000","last_peered":"2024-09-13T15:13:28.313394+0000","last_clean":"2024-09-13T15:13:28.313394+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T15:13:28.313394+0000","last_undegraded":"2024-09-13T15:13:28.313394+0000","last_fullsized":"2024-09-13T15:13:28.313394+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1370,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442775+0000","last_change":"2024-09-13T15:07:56.095947+0000","last_active":"2024-09-13T15:13:28.442775+0000","last_peered":"2024-09-13T15:13:28.442775+0000","last_clean":"2024-09-13T15:13:28.442775+0000","last_became_active":"2024-09-13T15:07:56.094500+0000","last_became_peered":"2024-09-13T15:07:56.094500+0000","last_unstale":"2024-09-13T15:13:28.442775+0000","last_undegraded":"2024-09-13T15:13:28.442775+0000","last_fullsized":"2024-09-13T15:13:28.442775+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:20:31.568566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":23,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.204718+0000","last_change":"2024-09-13T15:08:52.176756+0000","last_active":"2024-09-13T15:12:57.204718+0000","last_peered":"2024-09-13T15:12:57.204718+0000","last_clean":"2024-09-13T15:12:57.204718+0000","last_became_active":"2024-09-13T15:08:52.176280+0000","last_became_peered":"2024-09-13T15:08:52.176280+0000","last_unstale":"2024-09-13T15:12:57.204718+0000","last_undegraded":"2024-09-13T15:12:57.204718+0000","last_fullsized":"2024-09-13T15:12:57.204718+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:41.217586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1357,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442497+0000","last_change":"2024-09-13T15:08:52.170737+0000","last_active":"2024-09-13T15:13:28.442497+0000","last_peered":"2024-09-13T15:13:28.442497+0000","last_clean":"2024-09-13T15:13:28.442497+0000","last_became_active":"2024-09-13T15:08:52.170217+0000","last_became_peered":"2024-09-13T15:08:52.170217+0000","last_unstale":"2024-09-13T15:13:28.442497+0000","last_undegraded":"2024-09-13T15:13:28.442497+0000","last_fullsized":"2024-09-13T15:13:28.442497+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:17.688936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1585,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.445368+0000","last_change":"2024-09-13T15:05:34.087376+0000","last_active":"2024-09-13T15:13:28.445368+0000","last_peered":"2024-09-13T15:13:28.445368+0000","last_clean":"2024-09-13T15:13:28.445368+0000","last_became_active":"2024-09-13T15:05:34.087046+0000","last_became_peered":"2024-09-13T15:05:34.087046+0000","last_unstale":"2024-09-13T15:13:28.445368+0000","last_undegraded":"2024-09-13T15:13:28.445368+0000","last_fullsized":"2024-09-13T15:13:28.445368+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:47.801431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1297,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.441807+0000","last_change":"2024-09-13T15:08:52.174279+0000","last_active":"2024-09-13T15:13:28.441807+0000","last_peered":"2024-09-13T15:13:28.441807+0000","last_clean":"2024-09-13T15:13:28.441807+0000","last_became_active":"2024-09-13T15:08:52.173877+0000","last_became_peered":"2024-09-13T15:08:52.173877+0000","last_unstale":"2024-09-13T15:13:28.441807+0000","last_undegraded":"2024-09-13T15:13:28.441807+0000","last_fullsized":"2024-09-13T15:13:28.441807+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:16.648765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1068,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.197121+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T15:12:57.197121+0000","last_peered":"2024-09-13T15:12:57.197121+0000","last_clean":"2024-09-13T15:12:57.197121+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T15:12:57.197121+0000","last_undegraded":"2024-09-13T15:12:57.197121+0000","last_fullsized":"2024-09-13T15:12:57.197121+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1122,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.206167+0000","last_change":"2024-09-13T15:05:34.089020+0000","last_active":"2024-09-13T15:12:57.206167+0000","last_peered":"2024-09-13T15:12:57.206167+0000","last_clean":"2024-09-13T15:12:57.206167+0000","last_became_active":"2024-09-13T15:05:34.088440+0000","last_became_peered":"2024-09-13T15:05:34.088440+0000","last_unstale":"2024-09-13T15:12:57.206167+0000","last_undegraded":"2024-09-13T15:12:57.206167+0000","last_fullsized":"2024-09-13T15:12:57.206167+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:24.826746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":35,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.313237+0000","last_change":"2024-09-13T15:08:54.446160+0000","last_active":"2024-09-13T15:13:28.313237+0000","last_peered":"2024-09-13T15:13:28.313237+0000","last_clean":"2024-09-13T15:13:28.313237+0000","last_became_active":"2024-09-13T15:08:54.445649+0000","last_became_peered":"2024-09-13T15:08:54.445649+0000","last_unstale":"2024-09-13T15:13:28.313237+0000","last_undegraded":"2024-09-13T15:13:28.313237+0000","last_fullsized":"2024-09-13T15:13:28.313237+0000","mapping_epoch":1082,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1083,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:17.407470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":336,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.315683+0000","last_change":"2024-09-13T15:08:52.203370+0000","last_active":"2024-09-13T15:13:28.315683+0000","last_peered":"2024-09-13T15:13:28.315683+0000","last_clean":"2024-09-13T15:13:28.315683+0000","last_became_active":"2024-09-13T15:08:52.179887+0000","last_became_peered":"2024-09-13T15:08:52.179887+0000","last_unstale":"2024-09-13T15:13:28.315683+0000","last_undegraded":"2024-09-13T15:13:28.315683+0000","last_fullsized":"2024-09-13T15:13:28.315683+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:57.293764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":321,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.315708+0000","last_change":"2024-09-13T15:08:52.180881+0000","last_active":"2024-09-13T15:13:28.315708+0000","last_peered":"2024-09-13T15:13:28.315708+0000","last_clean":"2024-09-13T15:13:28.315708+0000","last_became_active":"2024-09-13T15:08:52.180000+0000","last_became_peered":"2024-09-13T15:08:52.180000+0000","last_unstale":"2024-09-13T15:13:28.315708+0000","last_undegraded":"2024-09-13T15:13:28.315708+0000","last_fullsized":"2024-09-13T15:13:28.315708+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:32.622289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":308,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.314943+0000","last_change":"2024-09-13T15:08:52.188050+0000","last_active":"2024-09-13T15:13:28.314943+0000","last_peered":"2024-09-13T15:13:28.314943+0000","last_clean":"2024-09-13T15:13:28.314943+0000","last_became_active":"2024-09-13T15:08:52.187522+0000","last_became_peered":"2024-09-13T15:08:52.187522+0000","last_unstale":"2024-09-13T15:13:28.314943+0000","last_undegraded":"2024-09-13T15:13:28.314943+0000","last_fullsized":"2024-09-13T15:13:28.314943+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:29.314964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":23,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.204687+0000","last_change":"2024-09-13T15:08:52.187782+0000","last_active":"2024-09-13T15:12:57.204687+0000","last_peered":"2024-09-13T15:12:57.204687+0000","last_clean":"2024-09-13T15:12:57.204687+0000","last_became_active":"2024-09-13T15:08:52.186669+0000","last_became_peered":"2024-09-13T15:08:52.186669+0000","last_unstale":"2024-09-13T15:12:57.204687+0000","last_undegraded":"2024-09-13T15:12:57.204687+0000","last_fullsized":"2024-09-13T15:12:57.204687+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:43.000869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1636,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.440975+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T15:13:28.440975+0000","last_peered":"2024-09-13T15:13:28.440975+0000","last_clean":"2024-09-13T15:13:28.440975+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T15:13:28.440975+0000","last_undegraded":"2024-09-13T15:13:28.440975+0000","last_fullsized":"2024-09-13T15:13:28.440975+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1291,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442724+0000","last_change":"2024-09-13T15:07:56.109509+0000","last_active":"2024-09-13T15:13:28.442724+0000","last_peered":"2024-09-13T15:13:28.442724+0000","last_clean":"2024-09-13T15:13:28.442724+0000","last_became_active":"2024-09-13T15:07:56.108034+0000","last_became_peered":"2024-09-13T15:07:56.108034+0000","last_unstale":"2024-09-13T15:13:28.442724+0000","last_undegraded":"2024-09-13T15:13:28.442724+0000","last_fullsized":"2024-09-13T15:13:28.442724+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:57:02.300327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":211,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.443115+0000","last_change":"2024-09-13T15:08:52.186362+0000","last_active":"2024-09-13T15:13:28.443115+0000","last_peered":"2024-09-13T15:13:28.443115+0000","last_clean":"2024-09-13T15:13:28.443115+0000","last_became_active":"2024-09-13T15:08:52.185731+0000","last_became_peered":"2024-09-13T15:08:52.185731+0000","last_unstale":"2024-09-13T15:13:28.443115+0000","last_undegraded":"2024-09-13T15:13:28.443115+0000","last_fullsized":"2024-09-13T15:13:28.443115+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:26.408919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1286,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442768+0000","last_change":"2024-09-13T15:05:34.085917+0000","last_active":"2024-09-13T15:13:28.442768+0000","last_peered":"2024-09-13T15:13:28.442768+0000","last_clean":"2024-09-13T15:13:28.442768+0000","last_became_active":"2024-09-13T15:05:34.085156+0000","last_became_peered":"2024-09-13T15:05:34.085156+0000","last_unstale":"2024-09-13T15:13:28.442768+0000","last_undegraded":"2024-09-13T15:13:28.442768+0000","last_fullsized":"2024-09-13T15:13:28.442768+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:33.482732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1565,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.313248+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T15:13:28.313248+0000","last_peered":"2024-09-13T15:13:28.313248+0000","last_clean":"2024-09-13T15:13:28.313248+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T15:13:28.313248+0000","last_undegraded":"2024-09-13T15:13:28.313248+0000","last_fullsized":"2024-09-13T15:13:28.313248+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1100,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.206366+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T15:12:57.206366+0000","last_peered":"2024-09-13T15:12:57.206366+0000","last_clean":"2024-09-13T15:12:57.206366+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T15:12:57.206366+0000","last_undegraded":"2024-09-13T15:12:57.206366+0000","last_fullsized":"2024-09-13T15:12:57.206366+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":197,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.441435+0000","last_change":"2024-09-13T14:52:43.583860+0000","last_active":"2024-09-13T15:13:28.441435+0000","last_peered":"2024-09-13T15:13:28.441435+0000","last_clean":"2024-09-13T15:13:28.441435+0000","last_became_active":"2024-09-13T14:52:43.583055+0000","last_became_peered":"2024-09-13T14:52:43.583055+0000","last_unstale":"2024-09-13T15:13:28.441435+0000","last_undegraded":"2024-09-13T15:13:28.441435+0000","last_fullsized":"2024-09-13T15:13:28.441435+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:55.113648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":198,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442047+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T15:13:28.442047+0000","last_peered":"2024-09-13T15:13:28.442047+0000","last_clean":"2024-09-13T15:13:28.442047+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T15:13:28.442047+0000","last_undegraded":"2024-09-13T15:13:28.442047+0000","last_fullsized":"2024-09-13T15:13:28.442047+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":197,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.441687+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T15:13:28.441687+0000","last_peered":"2024-09-13T15:13:28.441687+0000","last_clean":"2024-09-13T15:13:28.441687+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T15:13:28.441687+0000","last_undegraded":"2024-09-13T15:13:28.441687+0000","last_fullsized":"2024-09-13T15:13:28.441687+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1082,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.194501+0000","last_change":"2024-09-13T15:05:34.086758+0000","last_active":"2024-09-13T15:12:57.194501+0000","last_peered":"2024-09-13T15:12:57.194501+0000","last_clean":"2024-09-13T15:12:57.194501+0000","last_became_active":"2024-09-13T15:05:34.086145+0000","last_became_peered":"2024-09-13T15:05:34.086145+0000","last_unstale":"2024-09-13T15:12:57.194501+0000","last_undegraded":"2024-09-13T15:12:57.194501+0000","last_fullsized":"2024-09-13T15:12:57.194501+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:33.764698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":208,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.441558+0000","last_change":"2024-09-13T15:05:34.085898+0000","last_active":"2024-09-13T15:13:28.441558+0000","last_peered":"2024-09-13T15:13:28.441558+0000","last_clean":"2024-09-13T15:13:28.441558+0000","last_became_active":"2024-09-13T15:05:34.085174+0000","last_became_peered":"2024-09-13T15:05:34.085174+0000","last_unstale":"2024-09-13T15:13:28.441558+0000","last_undegraded":"2024-09-13T15:13:28.441558+0000","last_fullsized":"2024-09-13T15:13:28.441558+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:17:39.204192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":308,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.315791+0000","last_change":"2024-09-13T15:08:52.188254+0000","last_active":"2024-09-13T15:13:28.315791+0000","last_peered":"2024-09-13T15:13:28.315791+0000","last_clean":"2024-09-13T15:13:28.315791+0000","last_became_active":"2024-09-13T15:08:52.187535+0000","last_became_peered":"2024-09-13T15:08:52.187535+0000","last_unstale":"2024-09-13T15:13:28.315791+0000","last_undegraded":"2024-09-13T15:13:28.315791+0000","last_fullsized":"2024-09-13T15:13:28.315791+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:32.024558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1121,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.194212+0000","last_change":"2024-09-13T15:08:52.209457+0000","last_active":"2024-09-13T15:12:57.194212+0000","last_peered":"2024-09-13T15:12:57.194212+0000","last_clean":"2024-09-13T15:12:57.194212+0000","last_became_active":"2024-09-13T15:08:52.171595+0000","last_became_peered":"2024-09-13T15:08:52.171595+0000","last_unstale":"2024-09-13T15:12:57.194212+0000","last_undegraded":"2024-09-13T15:12:57.194212+0000","last_fullsized":"2024-09-13T15:12:57.194212+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:05:42.960465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":197,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442035+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T15:13:28.442035+0000","last_peered":"2024-09-13T15:13:28.442035+0000","last_clean":"2024-09-13T15:13:28.442035+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T15:13:28.442035+0000","last_undegraded":"2024-09-13T15:13:28.442035+0000","last_fullsized":"2024-09-13T15:13:28.442035+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":212,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.441245+0000","last_change":"2024-09-13T15:08:52.186074+0000","last_active":"2024-09-13T15:13:28.441245+0000","last_peered":"2024-09-13T15:13:28.441245+0000","last_clean":"2024-09-13T15:13:28.441245+0000","last_became_active":"2024-09-13T15:08:52.185693+0000","last_became_peered":"2024-09-13T15:08:52.185693+0000","last_unstale":"2024-09-13T15:13:28.441245+0000","last_undegraded":"2024-09-13T15:13:28.441245+0000","last_fullsized":"2024-09-13T15:13:28.441245+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:39:06.391608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1485,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.221078+0000","last_change":"2024-09-13T15:05:34.082028+0000","last_active":"2024-09-13T15:13:28.221078+0000","last_peered":"2024-09-13T15:13:28.221078+0000","last_clean":"2024-09-13T15:13:28.221078+0000","last_became_active":"2024-09-13T15:05:34.081220+0000","last_became_peered":"2024-09-13T15:05:34.081220+0000","last_unstale":"2024-09-13T15:13:28.221078+0000","last_undegraded":"2024-09-13T15:13:28.221078+0000","last_fullsized":"2024-09-13T15:13:28.221078+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:12:26.484994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1462,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442965+0000","last_change":"2024-09-13T15:07:56.111014+0000","last_active":"2024-09-13T15:13:28.442965+0000","last_peered":"2024-09-13T15:13:28.442965+0000","last_clean":"2024-09-13T15:13:28.442965+0000","last_became_active":"2024-09-13T15:07:56.110698+0000","last_became_peered":"2024-09-13T15:07:56.110698+0000","last_unstale":"2024-09-13T15:13:28.442965+0000","last_undegraded":"2024-09-13T15:13:28.442965+0000","last_fullsized":"2024-09-13T15:13:28.442965+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:29.395253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1612,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.313140+0000","last_change":"2024-09-13T15:05:34.087817+0000","last_active":"2024-09-13T15:13:28.313140+0000","last_peered":"2024-09-13T15:13:28.313140+0000","last_clean":"2024-09-13T15:13:28.313140+0000","last_became_active":"2024-09-13T15:05:34.087450+0000","last_became_peered":"2024-09-13T15:05:34.087450+0000","last_unstale":"2024-09-13T15:13:28.313140+0000","last_undegraded":"2024-09-13T15:13:28.313140+0000","last_fullsized":"2024-09-13T15:13:28.313140+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:47.661935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1100,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.205955+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T15:12:57.205955+0000","last_peered":"2024-09-13T15:12:57.205955+0000","last_clean":"2024-09-13T15:12:57.205955+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T15:12:57.205955+0000","last_undegraded":"2024-09-13T15:12:57.205955+0000","last_fullsized":"2024-09-13T15:12:57.205955+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":23,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.204701+0000","last_change":"2024-09-13T15:08:52.188028+0000","last_active":"2024-09-13T15:12:57.204701+0000","last_peered":"2024-09-13T15:12:57.204701+0000","last_clean":"2024-09-13T15:12:57.204701+0000","last_became_active":"2024-09-13T15:08:52.187424+0000","last_became_peered":"2024-09-13T15:08:52.187424+0000","last_unstale":"2024-09-13T15:12:57.204701+0000","last_undegraded":"2024-09-13T15:12:57.204701+0000","last_fullsized":"2024-09-13T15:12:57.204701+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:31.790876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1299,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.440321+0000","last_change":"2024-09-13T15:08:52.170510+0000","last_active":"2024-09-13T15:13:28.440321+0000","last_peered":"2024-09-13T15:13:28.440321+0000","last_clean":"2024-09-13T15:13:28.440321+0000","last_became_active":"2024-09-13T15:08:52.170141+0000","last_became_peered":"2024-09-13T15:08:52.170141+0000","last_unstale":"2024-09-13T15:13:28.440321+0000","last_undegraded":"2024-09-13T15:13:28.440321+0000","last_fullsized":"2024-09-13T15:13:28.440321+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:20.674380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1061,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.194579+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T15:12:57.194579+0000","last_peered":"2024-09-13T15:12:57.194579+0000","last_clean":"2024-09-13T15:12:57.194579+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T15:12:57.194579+0000","last_undegraded":"2024-09-13T15:12:57.194579+0000","last_fullsized":"2024-09-13T15:12:57.194579+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1200,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.205976+0000","last_change":"2024-09-13T15:08:52.214302+0000","last_active":"2024-09-13T15:12:57.205976+0000","last_peered":"2024-09-13T15:12:57.205976+0000","last_clean":"2024-09-13T15:12:57.205976+0000","last_became_active":"2024-09-13T15:08:52.194638+0000","last_became_peered":"2024-09-13T15:08:52.194638+0000","last_unstale":"2024-09-13T15:12:57.205976+0000","last_undegraded":"2024-09-13T15:12:57.205976+0000","last_fullsized":"2024-09-13T15:12:57.205976+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:57.236719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":208,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442142+0000","last_change":"2024-09-13T15:05:34.078805+0000","last_active":"2024-09-13T15:13:28.442142+0000","last_peered":"2024-09-13T15:13:28.442142+0000","last_clean":"2024-09-13T15:13:28.442142+0000","last_became_active":"2024-09-13T15:05:34.078180+0000","last_became_peered":"2024-09-13T15:05:34.078180+0000","last_unstale":"2024-09-13T15:13:28.442142+0000","last_undegraded":"2024-09-13T15:13:28.442142+0000","last_fullsized":"2024-09-13T15:13:28.442142+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:47.240906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1347,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442864+0000","last_change":"2024-09-13T15:05:34.076149+0000","last_active":"2024-09-13T15:13:28.442864+0000","last_peered":"2024-09-13T15:13:28.442864+0000","last_clean":"2024-09-13T15:13:28.442864+0000","last_became_active":"2024-09-13T15:05:34.075623+0000","last_became_peered":"2024-09-13T15:05:34.075623+0000","last_unstale":"2024-09-13T15:13:28.442864+0000","last_undegraded":"2024-09-13T15:13:28.442864+0000","last_fullsized":"2024-09-13T15:13:28.442864+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:23.608262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":321,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.316684+0000","last_change":"2024-09-13T15:08:52.189786+0000","last_active":"2024-09-13T15:13:28.316684+0000","last_peered":"2024-09-13T15:13:28.316684+0000","last_clean":"2024-09-13T15:13:28.316684+0000","last_became_active":"2024-09-13T15:08:52.188997+0000","last_became_peered":"2024-09-13T15:08:52.188997+0000","last_unstale":"2024-09-13T15:13:28.316684+0000","last_undegraded":"2024-09-13T15:13:28.316684+0000","last_fullsized":"2024-09-13T15:13:28.316684+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:08.932986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":293,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.314798+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T15:13:28.314798+0000","last_peered":"2024-09-13T15:13:28.314798+0000","last_clean":"2024-09-13T15:13:28.314798+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T15:13:28.314798+0000","last_undegraded":"2024-09-13T15:13:28.314798+0000","last_fullsized":"2024-09-13T15:13:28.314798+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":307,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.316759+0000","last_change":"2024-09-13T14:52:43.597595+0000","last_active":"2024-09-13T15:13:28.316759+0000","last_peered":"2024-09-13T15:13:28.316759+0000","last_clean":"2024-09-13T15:13:28.316759+0000","last_became_active":"2024-09-13T14:52:43.596592+0000","last_became_peered":"2024-09-13T14:52:43.596592+0000","last_unstale":"2024-09-13T15:13:28.316759+0000","last_undegraded":"2024-09-13T15:13:28.316759+0000","last_fullsized":"2024-09-13T15:13:28.316759+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:56.532603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":23,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.204471+0000","last_change":"2024-09-13T15:08:52.182709+0000","last_active":"2024-09-13T15:12:57.204471+0000","last_peered":"2024-09-13T15:12:57.204471+0000","last_clean":"2024-09-13T15:12:57.204471+0000","last_became_active":"2024-09-13T15:08:52.181906+0000","last_became_peered":"2024-09-13T15:08:52.181906+0000","last_unstale":"2024-09-13T15:12:57.204471+0000","last_undegraded":"2024-09-13T15:12:57.204471+0000","last_fullsized":"2024-09-13T15:12:57.204471+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:36.317662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":318,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.315413+0000","last_change":"2024-09-13T15:08:52.182885+0000","last_active":"2024-09-13T15:13:28.315413+0000","last_peered":"2024-09-13T15:13:28.315413+0000","last_clean":"2024-09-13T15:13:28.315413+0000","last_became_active":"2024-09-13T15:08:52.180608+0000","last_became_peered":"2024-09-13T15:08:52.180608+0000","last_unstale":"2024-09-13T15:13:28.315413+0000","last_undegraded":"2024-09-13T15:13:28.315413+0000","last_fullsized":"2024-09-13T15:13:28.315413+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:12.517462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1331,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.450074+0000","last_change":"2024-09-13T15:05:34.085458+0000","last_active":"2024-09-13T15:13:28.450074+0000","last_peered":"2024-09-13T15:13:28.450074+0000","last_clean":"2024-09-13T15:13:28.450074+0000","last_became_active":"2024-09-13T15:05:34.084748+0000","last_became_peered":"2024-09-13T15:05:34.084748+0000","last_unstale":"2024-09-13T15:13:28.450074+0000","last_undegraded":"2024-09-13T15:13:28.450074+0000","last_fullsized":"2024-09-13T15:13:28.450074+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:21.898959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1621,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.313291+0000","last_change":"2024-09-13T14:52:54.185117+0000","last_active":"2024-09-13T15:13:28.313291+0000","last_peered":"2024-09-13T15:13:28.313291+0000","last_clean":"2024-09-13T15:13:28.313291+0000","last_became_active":"2024-09-13T14:52:54.184712+0000","last_became_peered":"2024-09-13T14:52:54.184712+0000","last_unstale":"2024-09-13T15:13:28.313291+0000","last_undegraded":"2024-09-13T15:13:28.313291+0000","last_fullsized":"2024-09-13T15:13:28.313291+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:32.654401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1085,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.194318+0000","last_change":"2024-09-13T15:08:52.175569+0000","last_active":"2024-09-13T15:12:57.194318+0000","last_peered":"2024-09-13T15:12:57.194318+0000","last_clean":"2024-09-13T15:12:57.194318+0000","last_became_active":"2024-09-13T15:08:52.174891+0000","last_became_peered":"2024-09-13T15:08:52.174891+0000","last_unstale":"2024-09-13T15:12:57.194318+0000","last_undegraded":"2024-09-13T15:12:57.194318+0000","last_fullsized":"2024-09-13T15:12:57.194318+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.933115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.204525+0000","last_change":"2024-09-13T15:08:52.188074+0000","last_active":"2024-09-13T15:12:57.204525+0000","last_peered":"2024-09-13T15:12:57.204525+0000","last_clean":"2024-09-13T15:12:57.204525+0000","last_became_active":"2024-09-13T15:08:52.186649+0000","last_became_peered":"2024-09-13T15:08:52.186649+0000","last_unstale":"2024-09-13T15:12:57.204525+0000","last_undegraded":"2024-09-13T15:12:57.204525+0000","last_fullsized":"2024-09-13T15:12:57.204525+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:23.478250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":1064,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.206313+0000","last_change":"2024-09-13T15:08:52.208388+0000","last_active":"2024-09-13T15:12:57.206313+0000","last_peered":"2024-09-13T15:12:57.206313+0000","last_clean":"2024-09-13T15:12:57.206313+0000","last_became_active":"2024-09-13T15:08:52.192253+0000","last_became_peered":"2024-09-13T15:08:52.192253+0000","last_unstale":"2024-09-13T15:12:57.206313+0000","last_undegraded":"2024-09-13T15:12:57.206313+0000","last_fullsized":"2024-09-13T15:12:57.206313+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:07.508774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":318,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.315082+0000","last_change":"2024-09-13T15:05:34.072278+0000","last_active":"2024-09-13T15:13:28.315082+0000","last_peered":"2024-09-13T15:13:28.315082+0000","last_clean":"2024-09-13T15:13:28.315082+0000","last_became_active":"2024-09-13T15:05:34.071723+0000","last_became_peered":"2024-09-13T15:05:34.071723+0000","last_unstale":"2024-09-13T15:13:28.315082+0000","last_undegraded":"2024-09-13T15:13:28.315082+0000","last_fullsized":"2024-09-13T15:13:28.315082+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:02.130890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1127,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.208316+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T15:12:57.208316+0000","last_peered":"2024-09-13T15:12:57.208316+0000","last_clean":"2024-09-13T15:12:57.208316+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T15:12:57.208316+0000","last_undegraded":"2024-09-13T15:12:57.208316+0000","last_fullsized":"2024-09-13T15:12:57.208316+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1050,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.194243+0000","last_change":"2024-09-13T15:08:52.176830+0000","last_active":"2024-09-13T15:12:57.194243+0000","last_peered":"2024-09-13T15:12:57.194243+0000","last_clean":"2024-09-13T15:12:57.194243+0000","last_became_active":"2024-09-13T15:08:52.176303+0000","last_became_peered":"2024-09-13T15:08:52.176303+0000","last_unstale":"2024-09-13T15:12:57.194243+0000","last_undegraded":"2024-09-13T15:12:57.194243+0000","last_fullsized":"2024-09-13T15:12:57.194243+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:59.553257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1098,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.206272+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T15:12:57.206272+0000","last_peered":"2024-09-13T15:12:57.206272+0000","last_clean":"2024-09-13T15:12:57.206272+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T15:12:57.206272+0000","last_undegraded":"2024-09-13T15:12:57.206272+0000","last_fullsized":"2024-09-13T15:12:57.206272+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":33,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.204316+0000","last_change":"2024-09-13T15:08:52.200695+0000","last_active":"2024-09-13T15:12:57.204316+0000","last_peered":"2024-09-13T15:12:57.204316+0000","last_clean":"2024-09-13T15:12:57.204316+0000","last_became_active":"2024-09-13T15:08:52.183473+0000","last_became_peered":"2024-09-13T15:08:52.183473+0000","last_unstale":"2024-09-13T15:12:57.204316+0000","last_undegraded":"2024-09-13T15:12:57.204316+0000","last_fullsized":"2024-09-13T15:12:57.204316+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:42.810003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1345,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.442676+0000","last_change":"2024-09-13T15:05:34.074069+0000","last_active":"2024-09-13T15:13:28.442676+0000","last_peered":"2024-09-13T15:13:28.442676+0000","last_clean":"2024-09-13T15:13:28.442676+0000","last_became_active":"2024-09-13T15:05:34.073282+0000","last_became_peered":"2024-09-13T15:05:34.073282+0000","last_unstale":"2024-09-13T15:13:28.442676+0000","last_undegraded":"2024-09-13T15:13:28.442676+0000","last_fullsized":"2024-09-13T15:13:28.442676+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:14.307582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1091,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.206077+0000","last_change":"2024-09-13T15:08:52.192368+0000","last_active":"2024-09-13T15:12:57.206077+0000","last_peered":"2024-09-13T15:12:57.206077+0000","last_clean":"2024-09-13T15:12:57.206077+0000","last_became_active":"2024-09-13T15:08:52.191668+0000","last_became_peered":"2024-09-13T15:08:52.191668+0000","last_unstale":"2024-09-13T15:12:57.206077+0000","last_undegraded":"2024-09-13T15:12:57.206077+0000","last_fullsized":"2024-09-13T15:12:57.206077+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:27.410222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1585,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.313402+0000","last_change":"2024-09-13T15:05:34.087636+0000","last_active":"2024-09-13T15:13:28.313402+0000","last_peered":"2024-09-13T15:13:28.313402+0000","last_clean":"2024-09-13T15:13:28.313402+0000","last_became_active":"2024-09-13T15:05:34.087255+0000","last_became_peered":"2024-09-13T15:05:34.087255+0000","last_unstale":"2024-09-13T15:13:28.313402+0000","last_undegraded":"2024-09-13T15:13:28.313402+0000","last_fullsized":"2024-09-13T15:13:28.313402+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:36.313581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1586,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.313437+0000","last_change":"2024-09-13T15:05:34.089162+0000","last_active":"2024-09-13T15:13:28.313437+0000","last_peered":"2024-09-13T15:13:28.313437+0000","last_clean":"2024-09-13T15:13:28.313437+0000","last_became_active":"2024-09-13T15:05:34.088727+0000","last_became_peered":"2024-09-13T15:05:34.088727+0000","last_unstale":"2024-09-13T15:13:28.313437+0000","last_undegraded":"2024-09-13T15:13:28.313437+0000","last_fullsized":"2024-09-13T15:13:28.313437+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:29.209082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1064,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.194386+0000","last_change":"2024-09-13T15:05:34.073363+0000","last_active":"2024-09-13T15:12:57.194386+0000","last_peered":"2024-09-13T15:12:57.194386+0000","last_clean":"2024-09-13T15:12:57.194386+0000","last_became_active":"2024-09-13T15:05:34.072906+0000","last_became_peered":"2024-09-13T15:05:34.072906+0000","last_unstale":"2024-09-13T15:12:57.194386+0000","last_undegraded":"2024-09-13T15:12:57.194386+0000","last_fullsized":"2024-09-13T15:12:57.194386+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:27.966654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1104,"reported_epoch":1093,"state":"active+clean","last_fresh":"2024-09-13T15:12:57.206506+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T15:12:57.206506+0000","last_peered":"2024-09-13T15:12:57.206506+0000","last_clean":"2024-09-13T15:12:57.206506+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T15:12:57.206506+0000","last_undegraded":"2024-09-13T15:12:57.206506+0000","last_fullsized":"2024-09-13T15:12:57.206506+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1356,"reported_epoch":1094,"state":"active+clean","last_fresh":"2024-09-13T15:13:28.439729+0000","last_change":"2024-09-13T15:05:34.075192+0000","last_active":"2024-09-13T15:13:28.439729+0000","last_peered":"2024-09-13T15:13:28.439729+0000","last_clean":"2024-09-13T15:13:28.439729+0000","last_became_active":"2024-09-13T15:05:34.073315+0000","last_became_peered":"2024-09-13T15:05:34.073315+0000","last_unstale":"2024-09-13T15:13:28.439729+0000","last_undegraded":"2024-09-13T15:13:28.439729+0000","last_fullsized":"2024-09-13T15:13:28.439729+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:38.699775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":490012696,"num_objects":7478,"num_object_clones":0,"num_object_copies":22434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7478,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":195984,"num_write_kb":6510721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1489383424,"data_stored":1489371208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138484,"ondisk_log_size":138484,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":26,"num_read_kb":28,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":16374,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398149,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1162440,"kb_used_data":258044,"kb_used_omap":1,"kb_used_meta":904382,"kb_avail":92582712,"statfs":{"total":95995035648,"available":94804697088,"internally_reserved":0,"allocated":264237056,"data_stored":259798120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":926087512},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594385,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1068804,"kb_used_data":180988,"kb_used_omap":1,"kb_used_meta":887806,"kb_avail":92676348,"statfs":{"total":95995035648,"available":94900580352,"internally_reserved":0,"allocated":185331712,"data_stored":180893235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":909113688},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790619,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1099696,"kb_used_data":213608,"kb_used_omap":1,"kb_used_meta":886078,"kb_avail":92645456,"statfs":{"total":95995035648,"available":94868946944,"internally_reserved":0,"allocated":218734592,"data_stored":214311275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":907344216},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980832147,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":786108,"kb_used_data":130228,"kb_used_omap":76,"kb_used_meta":655795,"kb_avail":92959044,"statfs":{"total":95995035648,"available":95190061056,"internally_reserved":0,"allocated":133353472,"data_stored":128922126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":671534912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1066,"seq":4578435137633,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":444844,"kb_used_data":229284,"kb_used_omap":121,"kb_used_meta":215430,"kb_avail":93300308,"statfs":{"total":95995035648,"available":95539515392,"internally_reserved":0,"allocated":234786816,"data_stored":230367086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124093,"internal_metadata":220601155},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,2,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023329,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":989292,"kb_used_data":247588,"kb_used_omap":71,"kb_used_meta":741624,"kb_avail":92755860,"statfs":{"total":95995035648,"available":94982000640,"internally_reserved":0,"allocated":253530112,"data_stored":249110880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":759423648},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004459,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":782572,"kb_used_data":179748,"kb_used_omap":66,"kb_used_meta":602749,"kb_avail":92962580,"statfs":{"total":95995035648,"available":95193681920,"internally_reserved":0,"allocated":184061952,"data_stored":179642151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":617215146},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195651,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":877288,"kb_used_data":175072,"kb_used_omap":56,"kb_used_meta":702151,"kb_avail":92867864,"statfs":{"total":95995035648,"available":95096692736,"internally_reserved":0,"allocated":179273728,"data_stored":174856424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":719002767},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,3,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":153944064,"data_stored":153944064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":171642880,"data_stored":171638808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":234029056,"data_stored":234029056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":212865024,"data_stored":212860952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":113377280,"data_stored":113377280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":192937984,"data_stored":192937984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":166137856,"data_stored":166133784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":244449280,"data_stored":244449280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:13:39.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-09-13T15:13:39.736 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:39.763 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:39.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:39.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:39.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:39.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:39.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:39.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:40.009 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:40.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:40.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:40.110 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:13:40.113 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:13:40.115 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:40.158 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:40.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:40.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:40.262 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:40.275 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:40 smithi099 bash[84576]: cluster 2024-09-13T15:13:39.114262+0000 mgr.y (mgr.26252) 1428 : cluster [DBG] pgmap v2008: 89 pgs: 89 active+clean; 388 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-09-13T15:13:40.275 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:40 smithi099 bash[84576]: cluster 2024-09-13T15:13:39.114262+0000 mgr.y (mgr.26252) 1428 : cluster [DBG] pgmap v2008: 89 pgs: 89 active+clean; 388 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-09-13T15:13:40.275 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:40 smithi099 bash[87147]: cluster 2024-09-13T15:13:39.114262+0000 mgr.y (mgr.26252) 1428 : cluster [DBG] pgmap v2008: 89 pgs: 89 active+clean; 388 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-09-13T15:13:40.275 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:40 smithi099 bash[87147]: cluster 2024-09-13T15:13:39.114262+0000 mgr.y (mgr.26252) 1428 : cluster [DBG] pgmap v2008: 89 pgs: 89 active+clean; 388 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-09-13T15:13:40.304 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:40.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:40.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:40.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:40.458 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:40.467 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:40 smithi177 bash[68594]: cluster 2024-09-13T15:13:39.114262+0000 mgr.y (mgr.26252) 1428 : cluster [DBG] pgmap v2008: 89 pgs: 89 active+clean; 388 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-09-13T15:13:40.467 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:40 smithi177 bash[68594]: cluster 2024-09-13T15:13:39.114262+0000 mgr.y (mgr.26252) 1428 : cluster [DBG] pgmap v2008: 89 pgs: 89 active+clean; 388 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-09-13T15:13:40.468 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:13:40 smithi177 bash[61439]: ts=2024-09-13T15:13:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:13:40.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:40.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:40.570 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:40.589 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:40.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:40.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:40.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:40.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:40.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:40.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:40.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:40.838 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:40.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:40.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:40.941 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:40.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:40.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:40.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:41.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:41.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:41.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:41.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:41.216 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:41.239 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:41.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:41.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:41.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:41.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:41.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:41.478 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:41.503 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:41.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:41.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:41.610 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:41.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:41.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:41.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:41.744 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:41.764 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:41.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:41.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:41.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:41.891 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:41.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:41.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:41.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:42.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:42.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:42.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:42.151 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:42.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:42.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:42.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:42.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:42.311 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:42.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:42.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:42.415 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:42.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:42.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:42.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:42.556 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:42.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:42 smithi177 bash[68594]: cluster 2024-09-13T15:13:41.114874+0000 mgr.y (mgr.26252) 1429 : cluster [DBG] pgmap v2009: 89 pgs: 89 active+clean; 235 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T15:13:42.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:42 smithi177 bash[68594]: cluster 2024-09-13T15:13:41.114874+0000 mgr.y (mgr.26252) 1429 : cluster [DBG] pgmap v2009: 89 pgs: 89 active+clean; 235 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T15:13:42.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:42 smithi177 bash[68594]: audit 2024-09-13T15:13:41.352654+0000 mon.a (mon.0) 2569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:42 smithi177 bash[68594]: audit 2024-09-13T15:13:41.352654+0000 mon.a (mon.0) 2569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:42 smithi177 bash[68594]: audit 2024-09-13T15:13:41.365553+0000 mon.a (mon.0) 2570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:42 smithi177 bash[68594]: audit 2024-09-13T15:13:41.365553+0000 mon.a (mon.0) 2570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[84576]: cluster 2024-09-13T15:13:41.114874+0000 mgr.y (mgr.26252) 1429 : cluster [DBG] pgmap v2009: 89 pgs: 89 active+clean; 235 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T15:13:42.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[84576]: cluster 2024-09-13T15:13:41.114874+0000 mgr.y (mgr.26252) 1429 : cluster [DBG] pgmap v2009: 89 pgs: 89 active+clean; 235 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T15:13:42.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[84576]: audit 2024-09-13T15:13:41.352654+0000 mon.a (mon.0) 2569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[84576]: audit 2024-09-13T15:13:41.352654+0000 mon.a (mon.0) 2569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[84576]: audit 2024-09-13T15:13:41.365553+0000 mon.a (mon.0) 2570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[84576]: audit 2024-09-13T15:13:41.365553+0000 mon.a (mon.0) 2570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[87147]: cluster 2024-09-13T15:13:41.114874+0000 mgr.y (mgr.26252) 1429 : cluster [DBG] pgmap v2009: 89 pgs: 89 active+clean; 235 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T15:13:42.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[87147]: cluster 2024-09-13T15:13:41.114874+0000 mgr.y (mgr.26252) 1429 : cluster [DBG] pgmap v2009: 89 pgs: 89 active+clean; 235 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T15:13:42.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[87147]: audit 2024-09-13T15:13:41.352654+0000 mon.a (mon.0) 2569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[87147]: audit 2024-09-13T15:13:41.352654+0000 mon.a (mon.0) 2569 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[87147]: audit 2024-09-13T15:13:41.365553+0000 mon.a (mon.0) 2570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:42 smithi099 bash[87147]: audit 2024-09-13T15:13:41.365553+0000 mon.a (mon.0) 2570 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:42.579 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:42.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:42.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:42.682 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:42.720 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:42.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:42.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:42.824 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:42.848 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:42.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:42.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:42.984 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:42.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:42.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:43.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:43.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:43.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:43.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:43.221 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:43.246 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:43.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:43.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:43.354 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:43.384 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:43.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:43.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:43.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:43.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:43.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:43.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:43.608 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:43.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:43.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:43.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:43.738 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:43.771 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:43.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:43.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:43.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:43.898 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:43.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:44.002 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:44.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:44.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:44.146 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:44.177 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:44.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:44.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:44.280 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:44.300 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:44.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:44.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:44.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:44.426 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:44.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:44.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:44.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:44.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:44 smithi177 bash[68594]: cluster 2024-09-13T15:13:43.118285+0000 mgr.y (mgr.26252) 1430 : cluster [DBG] pgmap v2010: 89 pgs: 89 active+clean; 209 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2024-09-13T15:13:44.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:44 smithi177 bash[68594]: cluster 2024-09-13T15:13:43.118285+0000 mgr.y (mgr.26252) 1430 : cluster [DBG] pgmap v2010: 89 pgs: 89 active+clean; 209 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2024-09-13T15:13:44.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:44.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:44.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:44.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:44 smithi099 bash[84576]: cluster 2024-09-13T15:13:43.118285+0000 mgr.y (mgr.26252) 1430 : cluster [DBG] pgmap v2010: 89 pgs: 89 active+clean; 209 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2024-09-13T15:13:44.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:44 smithi099 bash[84576]: cluster 2024-09-13T15:13:43.118285+0000 mgr.y (mgr.26252) 1430 : cluster [DBG] pgmap v2010: 89 pgs: 89 active+clean; 209 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2024-09-13T15:13:44.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:44 smithi099 bash[87147]: cluster 2024-09-13T15:13:43.118285+0000 mgr.y (mgr.26252) 1430 : cluster [DBG] pgmap v2010: 89 pgs: 89 active+clean; 209 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2024-09-13T15:13:44.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:44 smithi099 bash[87147]: cluster 2024-09-13T15:13:43.118285+0000 mgr.y (mgr.26252) 1430 : cluster [DBG] pgmap v2010: 89 pgs: 89 active+clean; 209 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2024-09-13T15:13:44.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:44.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:44.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:44.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:44.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:44.900 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:44.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:44.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:45.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:45.046 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:45.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:45.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:45.154 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:45.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:45.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:45.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:45.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:45.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:45.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:45.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:45.345 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Removed 101730 objects 2024-09-13T15:13:45.345 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Clean up completed and total clean up time :148.127 2024-09-13T15:13:45.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:45.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:45.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:45.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:45.540 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:45.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:45.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:45.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:45.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:45.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:45.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:45.792 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:45.819 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:45.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:45.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:45.922 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:45.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:45.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:45.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:46.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:46.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:46.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:46.172 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:46.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:46.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:46.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:46.249 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:13:46.304 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:46.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: cluster 2024-09-13T15:13:45.118918+0000 mgr.y (mgr.26252) 1431 : cluster [DBG] pgmap v2011: 89 pgs: 89 active+clean; 63 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-09-13T15:13:46.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: cluster 2024-09-13T15:13:45.118918+0000 mgr.y (mgr.26252) 1431 : cluster [DBG] pgmap v2011: 89 pgs: 89 active+clean; 63 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-09-13T15:13:46.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: audit 2024-09-13T15:13:45.132015+0000 mon.a (mon.0) 2571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: audit 2024-09-13T15:13:45.132015+0000 mon.a (mon.0) 2571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: audit 2024-09-13T15:13:45.141388+0000 mon.a (mon.0) 2572 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: audit 2024-09-13T15:13:45.141388+0000 mon.a (mon.0) 2572 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: audit 2024-09-13T15:13:45.844675+0000 mon.a (mon.0) 2573 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:13:46.314 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: audit 2024-09-13T15:13:45.844675+0000 mon.a (mon.0) 2573 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:13:46.315 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: audit 2024-09-13T15:13:45.845617+0000 mon.a (mon.0) 2574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:13:46.315 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: audit 2024-09-13T15:13:45.845617+0000 mon.a (mon.0) 2574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:13:46.315 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: audit 2024-09-13T15:13:45.850520+0000 mon.a (mon.0) 2575 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.315 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[84576]: audit 2024-09-13T15:13:45.850520+0000 mon.a (mon.0) 2575 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.315 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: cluster 2024-09-13T15:13:45.118918+0000 mgr.y (mgr.26252) 1431 : cluster [DBG] pgmap v2011: 89 pgs: 89 active+clean; 63 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-09-13T15:13:46.315 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: cluster 2024-09-13T15:13:45.118918+0000 mgr.y (mgr.26252) 1431 : cluster [DBG] pgmap v2011: 89 pgs: 89 active+clean; 63 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-09-13T15:13:46.315 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: audit 2024-09-13T15:13:45.132015+0000 mon.a (mon.0) 2571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.315 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: audit 2024-09-13T15:13:45.132015+0000 mon.a (mon.0) 2571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.315 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: audit 2024-09-13T15:13:45.141388+0000 mon.a (mon.0) 2572 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.316 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: audit 2024-09-13T15:13:45.141388+0000 mon.a (mon.0) 2572 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.316 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: audit 2024-09-13T15:13:45.844675+0000 mon.a (mon.0) 2573 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:13:46.316 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: audit 2024-09-13T15:13:45.844675+0000 mon.a (mon.0) 2573 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:13:46.316 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: audit 2024-09-13T15:13:45.845617+0000 mon.a (mon.0) 2574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:13:46.316 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: audit 2024-09-13T15:13:45.845617+0000 mon.a (mon.0) 2574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:13:46.316 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: audit 2024-09-13T15:13:45.850520+0000 mon.a (mon.0) 2575 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.316 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:46 smithi099 bash[87147]: audit 2024-09-13T15:13:45.850520+0000 mon.a (mon.0) 2575 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.328 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:46.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:46.430 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:46.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:46.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:46.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:46.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:46.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:46.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:46.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:46.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: cluster 2024-09-13T15:13:45.118918+0000 mgr.y (mgr.26252) 1431 : cluster [DBG] pgmap v2011: 89 pgs: 89 active+clean; 63 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-09-13T15:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: cluster 2024-09-13T15:13:45.118918+0000 mgr.y (mgr.26252) 1431 : cluster [DBG] pgmap v2011: 89 pgs: 89 active+clean; 63 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-09-13T15:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: audit 2024-09-13T15:13:45.132015+0000 mon.a (mon.0) 2571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: audit 2024-09-13T15:13:45.132015+0000 mon.a (mon.0) 2571 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: audit 2024-09-13T15:13:45.141388+0000 mon.a (mon.0) 2572 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: audit 2024-09-13T15:13:45.141388+0000 mon.a (mon.0) 2572 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: audit 2024-09-13T15:13:45.844675+0000 mon.a (mon.0) 2573 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: audit 2024-09-13T15:13:45.844675+0000 mon.a (mon.0) 2573 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: audit 2024-09-13T15:13:45.845617+0000 mon.a (mon.0) 2574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:13:46.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: audit 2024-09-13T15:13:45.845617+0000 mon.a (mon.0) 2574 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:13:46.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: audit 2024-09-13T15:13:45.850520+0000 mon.a (mon.0) 2575 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:46 smithi177 bash[68594]: audit 2024-09-13T15:13:45.850520+0000 mon.a (mon.0) 2575 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:13:46.688 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:46.717 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:46.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:46.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:46.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:46.839 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:46.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:46.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:46.943 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:46.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:46.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:47.070 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:47.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:47.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:47.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:47.155 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:13:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:13:47] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:13:47.201 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:47.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:47.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:47.332 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:47.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:47.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:47.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:47.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:47.491 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:47.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:47.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:47.594 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:47.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:47.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:47.724 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:47.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:47.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:47.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:47.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:47.885 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:47.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:47.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:47.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:48.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:48.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:48.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:48.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:48.143 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:48.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:48.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:48.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:48.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:48.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:48.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:48.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:48.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:48 smithi177 bash[68594]: cluster 2024-09-13T15:13:47.119653+0000 mgr.y (mgr.26252) 1432 : cluster [DBG] pgmap v2012: 89 pgs: 89 active+clean; 29 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:48.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:48 smithi177 bash[68594]: cluster 2024-09-13T15:13:47.119653+0000 mgr.y (mgr.26252) 1432 : cluster [DBG] pgmap v2012: 89 pgs: 89 active+clean; 29 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:48.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:48.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:48.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:48.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:48.533 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:48.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:48.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:48 smithi099 bash[84576]: cluster 2024-09-13T15:13:47.119653+0000 mgr.y (mgr.26252) 1432 : cluster [DBG] pgmap v2012: 89 pgs: 89 active+clean; 29 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:48.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:48 smithi099 bash[84576]: cluster 2024-09-13T15:13:47.119653+0000 mgr.y (mgr.26252) 1432 : cluster [DBG] pgmap v2012: 89 pgs: 89 active+clean; 29 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:48 smithi099 bash[87147]: cluster 2024-09-13T15:13:47.119653+0000 mgr.y (mgr.26252) 1432 : cluster [DBG] pgmap v2012: 89 pgs: 89 active+clean; 29 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:48.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:48 smithi099 bash[87147]: cluster 2024-09-13T15:13:47.119653+0000 mgr.y (mgr.26252) 1432 : cluster [DBG] pgmap v2012: 89 pgs: 89 active+clean; 29 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2024-09-13T15:13:48.636 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:48.664 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:48.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:48.767 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:48.789 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:48.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:48.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:48.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:48.918 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:48.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:48.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:49.021 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:49.046 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:49.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:49.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:49.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:49.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:49.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:49.279 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:49.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:49.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:49.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:49.355 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:49 smithi099 bash[84576]: audit 2024-09-13T15:13:48.311743+0000 mon.a (mon.0) 2576 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:49.355 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:49 smithi099 bash[84576]: audit 2024-09-13T15:13:48.311743+0000 mon.a (mon.0) 2576 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:49.355 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:49 smithi099 bash[87147]: audit 2024-09-13T15:13:48.311743+0000 mon.a (mon.0) 2576 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:49.355 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:49 smithi099 bash[87147]: audit 2024-09-13T15:13:48.311743+0000 mon.a (mon.0) 2576 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:49 smithi177 bash[68594]: audit 2024-09-13T15:13:48.311743+0000 mon.a (mon.0) 2576 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:49.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:49 smithi177 bash[68594]: audit 2024-09-13T15:13:48.311743+0000 mon.a (mon.0) 2576 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:13:49.409 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:49.436 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:49.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:49.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:49.527 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:13:49.527 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:13:49.527 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:13:49.527 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:13:49.527 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:13:49.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:49.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:49.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:49.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:49.671 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:49.699 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:49.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:49.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:49.802 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:49.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:49.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:49.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:49.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:49.961 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:49.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:50.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:50.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:50.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:50.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:50.192 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:50.202 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:50 smithi177 bash[68594]: cluster 2024-09-13T15:13:49.120399+0000 mgr.y (mgr.26252) 1433 : cluster [DBG] pgmap v2013: 89 pgs: 89 active+clean; 15 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2024-09-13T15:13:50.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:50 smithi177 bash[68594]: cluster 2024-09-13T15:13:49.120399+0000 mgr.y (mgr.26252) 1433 : cluster [DBG] pgmap v2013: 89 pgs: 89 active+clean; 15 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2024-09-13T15:13:50.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:50.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:50.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:50.322 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:50.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:50 smithi099 bash[84576]: cluster 2024-09-13T15:13:49.120399+0000 mgr.y (mgr.26252) 1433 : cluster [DBG] pgmap v2013: 89 pgs: 89 active+clean; 15 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2024-09-13T15:13:50.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:50 smithi099 bash[84576]: cluster 2024-09-13T15:13:49.120399+0000 mgr.y (mgr.26252) 1433 : cluster [DBG] pgmap v2013: 89 pgs: 89 active+clean; 15 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2024-09-13T15:13:50.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:50 smithi099 bash[87147]: cluster 2024-09-13T15:13:49.120399+0000 mgr.y (mgr.26252) 1433 : cluster [DBG] pgmap v2013: 89 pgs: 89 active+clean; 15 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2024-09-13T15:13:50.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:50 smithi099 bash[87147]: cluster 2024-09-13T15:13:49.120399+0000 mgr.y (mgr.26252) 1433 : cluster [DBG] pgmap v2013: 89 pgs: 89 active+clean; 15 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2024-09-13T15:13:50.350 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:50.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:50.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:50.465 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:13:50 smithi177 bash[61439]: ts=2024-09-13T15:13:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:13:50.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:50.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:50.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:50.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:50.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:50.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:50.713 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:50.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:50.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:50.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:50.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:50.872 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:50.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:50.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:50.975 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:51.003 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:51.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:51.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:51.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:51.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:51.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:51.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:51.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:51.264 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:51.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:51.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:51.368 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:51.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:51.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:51.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:51.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:51.523 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:51.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:51.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:51.628 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:51.654 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:51.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:51.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:51.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:51.778 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:51.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:51.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:51.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:51.904 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:51.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:51.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:52.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:52.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:52.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:52.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:52.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:52.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:52.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:52.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:52.267 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:52.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:52.296 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:52 smithi099 bash[84576]: cluster 2024-09-13T15:13:51.121500+0000 mgr.y (mgr.26252) 1434 : cluster [DBG] pgmap v2014: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2024-09-13T15:13:52.296 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:52 smithi099 bash[84576]: cluster 2024-09-13T15:13:51.121500+0000 mgr.y (mgr.26252) 1434 : cluster [DBG] pgmap v2014: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2024-09-13T15:13:52.296 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:52 smithi099 bash[87147]: cluster 2024-09-13T15:13:51.121500+0000 mgr.y (mgr.26252) 1434 : cluster [DBG] pgmap v2014: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2024-09-13T15:13:52.297 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:52 smithi099 bash[87147]: cluster 2024-09-13T15:13:51.121500+0000 mgr.y (mgr.26252) 1434 : cluster [DBG] pgmap v2014: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2024-09-13T15:13:52.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:52.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:52.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:52.426 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:52.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:52.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:52.529 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:52.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:52 smithi177 bash[68594]: cluster 2024-09-13T15:13:51.121500+0000 mgr.y (mgr.26252) 1434 : cluster [DBG] pgmap v2014: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2024-09-13T15:13:52.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:52 smithi177 bash[68594]: cluster 2024-09-13T15:13:51.121500+0000 mgr.y (mgr.26252) 1434 : cluster [DBG] pgmap v2014: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2024-09-13T15:13:52.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:52.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:52.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:52.658 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:52.685 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:52.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:52.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:52.816 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:52.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:52.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:52.946 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:52.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:53.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:53.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:53.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:53.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:53.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:53.207 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:53.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:53.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:53.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T15:13:53.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:53.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:53.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:53.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:53.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:53.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:53.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:53.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:53.593 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:53.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:53.696 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:53.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:53.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:53.819 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:53.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:53.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:53.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:53.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:53.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:54.082 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:54.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:54.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:54.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:54.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:54.224 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:54 smithi099 bash[84576]: cluster 2024-09-13T15:13:53.122028+0000 mgr.y (mgr.26252) 1435 : cluster [DBG] pgmap v2015: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 312 op/s 2024-09-13T15:13:54.224 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:54 smithi099 bash[84576]: cluster 2024-09-13T15:13:53.122028+0000 mgr.y (mgr.26252) 1435 : cluster [DBG] pgmap v2015: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 312 op/s 2024-09-13T15:13:54.225 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:54 smithi099 bash[87147]: cluster 2024-09-13T15:13:53.122028+0000 mgr.y (mgr.26252) 1435 : cluster [DBG] pgmap v2015: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 312 op/s 2024-09-13T15:13:54.225 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:54 smithi099 bash[87147]: cluster 2024-09-13T15:13:53.122028+0000 mgr.y (mgr.26252) 1435 : cluster [DBG] pgmap v2015: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 312 op/s 2024-09-13T15:13:54.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:54.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:54.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:54.372 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:54.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:54.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:54.475 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:54.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:54.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:54.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:54.606 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:54.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:54 smithi177 bash[68594]: cluster 2024-09-13T15:13:53.122028+0000 mgr.y (mgr.26252) 1435 : cluster [DBG] pgmap v2015: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 312 op/s 2024-09-13T15:13:54.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:54 smithi177 bash[68594]: cluster 2024-09-13T15:13:53.122028+0000 mgr.y (mgr.26252) 1435 : cluster [DBG] pgmap v2015: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 312 op/s 2024-09-13T15:13:54.633 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:54.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:54.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:54.736 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:54.764 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:54.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:54.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:54.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:54.895 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:54.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:54.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:54.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:55.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:55.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:55.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:55.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:55.158 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:55.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:55.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:55.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:55.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:13:55.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:55.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:13:55.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:55.420 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:55.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:55.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:55.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:55.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:55.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:55.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:55.682 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:55.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:55.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:55.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:55.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:55.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:55.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:55.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:55.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:56.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:56.075 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:56.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:56.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:56.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:56.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:56 smithi177 bash[68594]: cluster 2024-09-13T15:13:55.123099+0000 mgr.y (mgr.26252) 1436 : cluster [DBG] pgmap v2016: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 277 op/s 2024-09-13T15:13:56.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:56 smithi177 bash[68594]: cluster 2024-09-13T15:13:55.123099+0000 mgr.y (mgr.26252) 1436 : cluster [DBG] pgmap v2016: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 277 op/s 2024-09-13T15:13:56.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:56.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:56.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:56.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:56.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:56.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:56.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:56.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:56.450 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:56 smithi099 bash[84576]: cluster 2024-09-13T15:13:55.123099+0000 mgr.y (mgr.26252) 1436 : cluster [DBG] pgmap v2016: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 277 op/s 2024-09-13T15:13:56.450 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:56 smithi099 bash[84576]: cluster 2024-09-13T15:13:55.123099+0000 mgr.y (mgr.26252) 1436 : cluster [DBG] pgmap v2016: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 277 op/s 2024-09-13T15:13:56.451 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:56 smithi099 bash[87147]: cluster 2024-09-13T15:13:55.123099+0000 mgr.y (mgr.26252) 1436 : cluster [DBG] pgmap v2016: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 277 op/s 2024-09-13T15:13:56.451 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:56 smithi099 bash[87147]: cluster 2024-09-13T15:13:55.123099+0000 mgr.y (mgr.26252) 1436 : cluster [DBG] pgmap v2016: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 277 op/s 2024-09-13T15:13:56.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:56.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:56.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:56.570 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:56.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:56.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:56.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:56.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:56.728 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:56.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:56.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:56.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:56.853 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:56.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:56.956 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:56.983 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:56.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:56.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:57.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:57.114 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:57.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:57.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:57.218 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:57.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:57.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:57.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:57.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:13:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:13:57] "GET /metrics HTTP/1.1" 200 36768 "" "Prometheus/2.51.0" 2024-09-13T15:13:57.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:57.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:57.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:57.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:57.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:57.508 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:13:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:57.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:13:57.612 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:57.639 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:57.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:57.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:57.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:57.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:57.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:57.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:57.873 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:57.898 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:57.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:57.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:58.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:58.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:58.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:58.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:58.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:58.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:58.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:58.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:58.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:58.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:58.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:58.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:58.391 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:58.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:58.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:58.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:58.460 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:58 smithi099 bash[84576]: cluster 2024-09-13T15:13:57.123973+0000 mgr.y (mgr.26252) 1437 : cluster [DBG] pgmap v2017: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 83 op/s 2024-09-13T15:13:58.460 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:58 smithi099 bash[84576]: cluster 2024-09-13T15:13:57.123973+0000 mgr.y (mgr.26252) 1437 : cluster [DBG] pgmap v2017: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 83 op/s 2024-09-13T15:13:58.460 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:58 smithi099 bash[87147]: cluster 2024-09-13T15:13:57.123973+0000 mgr.y (mgr.26252) 1437 : cluster [DBG] pgmap v2017: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 83 op/s 2024-09-13T15:13:58.460 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:58 smithi099 bash[87147]: cluster 2024-09-13T15:13:57.123973+0000 mgr.y (mgr.26252) 1437 : cluster [DBG] pgmap v2017: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 83 op/s 2024-09-13T15:13:58.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:58.547 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:58.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:58.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:58.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:58 smithi177 bash[68594]: cluster 2024-09-13T15:13:57.123973+0000 mgr.y (mgr.26252) 1437 : cluster [DBG] pgmap v2017: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 83 op/s 2024-09-13T15:13:58.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:58 smithi177 bash[68594]: cluster 2024-09-13T15:13:57.123973+0000 mgr.y (mgr.26252) 1437 : cluster [DBG] pgmap v2017: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 83 op/s 2024-09-13T15:13:58.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:58.671 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:58.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:58.774 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:58.804 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:58.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:58.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:58.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:58.938 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:58.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:58.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:59.042 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:59.061 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:59.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:59.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:59.091 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:13:59.092 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:13:59.109 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:13:59.110 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:13:59.126 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:13:59.127 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:13:59.143 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:13:59.144 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:13:59.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:59.165 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:13:59.165 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:13:59.172 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T15:13:59.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:59.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:13:59.195 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:13:59.195 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:13:59.215 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:13:59.216 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:13:59.226 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:59 smithi099 bash[84576]: audit 2024-09-13T15:13:59.058412+0000 mon.a (mon.0) 2577 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:13:59.226 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:59 smithi099 bash[84576]: audit 2024-09-13T15:13:59.058412+0000 mon.a (mon.0) 2577 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:13:59.226 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:59 smithi099 bash[84576]: audit 2024-09-13T15:13:59.072745+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.99:0/1707629256' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:59.226 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:13:59 smithi099 bash[84576]: audit 2024-09-13T15:13:59.072745+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.99:0/1707629256' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:59.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:59 smithi099 bash[87147]: audit 2024-09-13T15:13:59.058412+0000 mon.a (mon.0) 2577 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:13:59.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:59 smithi099 bash[87147]: audit 2024-09-13T15:13:59.058412+0000 mon.a (mon.0) 2577 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:13:59.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:59 smithi099 bash[87147]: audit 2024-09-13T15:13:59.072745+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.99:0/1707629256' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:59.226 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:13:59 smithi099 bash[87147]: audit 2024-09-13T15:13:59.072745+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.99:0/1707629256' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:59.236 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:13:59.236 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:13:59.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:59.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:13:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:59.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:13:59.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:59.436 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:59 smithi177 bash[68594]: audit 2024-09-13T15:13:59.058412+0000 mon.a (mon.0) 2577 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:13:59.436 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:59 smithi177 bash[68594]: audit 2024-09-13T15:13:59.058412+0000 mon.a (mon.0) 2577 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:13:59.436 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:59 smithi177 bash[68594]: audit 2024-09-13T15:13:59.072745+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.99:0/1707629256' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:59.436 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:13:59 smithi177 bash[68594]: audit 2024-09-13T15:13:59.072745+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.99:0/1707629256' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:13:59.452 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:13:59.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:59.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:13:59.555 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:59.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:13:59.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:59.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:13:59.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:59.712 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:13:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:59.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:13:59.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:59.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:13:59.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:59.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:13:59.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:59.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:13:59.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:13:59.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:00.074 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:00.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:00.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:00.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:00.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:00.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:00.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:00.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:00.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:00.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:14:00 smithi177 bash[61439]: ts=2024-09-13T15:14:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:14:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:00 smithi177 bash[68594]: cluster 2024-09-13T15:13:59.124453+0000 mgr.y (mgr.26252) 1438 : cluster [DBG] pgmap v2018: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 37 op/s 2024-09-13T15:14:00.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:00 smithi177 bash[68594]: cluster 2024-09-13T15:13:59.124453+0000 mgr.y (mgr.26252) 1438 : cluster [DBG] pgmap v2018: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 37 op/s 2024-09-13T15:14:00.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:00 smithi177 bash[68594]: cluster 2024-09-13T15:13:59.164442+0000 mon.a (mon.0) 2578 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:00.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:00 smithi177 bash[68594]: cluster 2024-09-13T15:13:59.164442+0000 mon.a (mon.0) 2578 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:00.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:00 smithi177 bash[68594]: audit 2024-09-13T15:13:59.173298+0000 mon.a (mon.0) 2579 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:14:00.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:00 smithi177 bash[68594]: audit 2024-09-13T15:13:59.173298+0000 mon.a (mon.0) 2579 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:14:00.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:00 smithi177 bash[68594]: cluster 2024-09-13T15:13:59.177314+0000 mon.a (mon.0) 2580 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2024-09-13T15:14:00.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:00 smithi177 bash[68594]: cluster 2024-09-13T15:13:59.177314+0000 mon.a (mon.0) 2580 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2024-09-13T15:14:00.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:00.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:00.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:00.493 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:00.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:00.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[84576]: cluster 2024-09-13T15:13:59.124453+0000 mgr.y (mgr.26252) 1438 : cluster [DBG] pgmap v2018: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 37 op/s 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[84576]: cluster 2024-09-13T15:13:59.124453+0000 mgr.y (mgr.26252) 1438 : cluster [DBG] pgmap v2018: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 37 op/s 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[84576]: cluster 2024-09-13T15:13:59.164442+0000 mon.a (mon.0) 2578 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[84576]: cluster 2024-09-13T15:13:59.164442+0000 mon.a (mon.0) 2578 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[84576]: audit 2024-09-13T15:13:59.173298+0000 mon.a (mon.0) 2579 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[84576]: audit 2024-09-13T15:13:59.173298+0000 mon.a (mon.0) 2579 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[84576]: cluster 2024-09-13T15:13:59.177314+0000 mon.a (mon.0) 2580 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[84576]: cluster 2024-09-13T15:13:59.177314+0000 mon.a (mon.0) 2580 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[87147]: cluster 2024-09-13T15:13:59.124453+0000 mgr.y (mgr.26252) 1438 : cluster [DBG] pgmap v2018: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 37 op/s 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[87147]: cluster 2024-09-13T15:13:59.124453+0000 mgr.y (mgr.26252) 1438 : cluster [DBG] pgmap v2018: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 37 op/s 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[87147]: cluster 2024-09-13T15:13:59.164442+0000 mon.a (mon.0) 2578 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[87147]: cluster 2024-09-13T15:13:59.164442+0000 mon.a (mon.0) 2578 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:00.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[87147]: audit 2024-09-13T15:13:59.173298+0000 mon.a (mon.0) 2579 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:14:00.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[87147]: audit 2024-09-13T15:13:59.173298+0000 mon.a (mon.0) 2579 : audit [INF] from='client.? 172.21.15.99:0/389122209' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:14:00.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[87147]: cluster 2024-09-13T15:13:59.177314+0000 mon.a (mon.0) 2580 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2024-09-13T15:14:00.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:00 smithi099 bash[87147]: cluster 2024-09-13T15:13:59.177314+0000 mon.a (mon.0) 2580 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2024-09-13T15:14:00.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:00.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:00.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:00.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:00.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:00.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:00.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:00.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:00.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:00.885 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:00.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:00.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:00.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:01.017 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:01.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:01.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:01.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:01.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:01.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:01.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:01.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:01.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:01.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:01.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:01.409 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:01.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:01.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:01.512 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:01.539 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:01.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:01.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:01.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:01.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:01.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:01.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:01.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:01.800 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:01.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:01.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:01.903 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:01.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:01.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:02.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:02.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:02.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:02.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:02.141 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:14:02.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:02.182 INFO:teuthology.orchestra.run.smithi099.stderr:set pool 1 pgp_num to 1 2024-09-13T15:14:02.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:02.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:02.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:02.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:02.316 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:02.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:02.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:02.436 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[84576]: cluster 2024-09-13T15:14:01.125501+0000 mgr.y (mgr.26252) 1439 : cluster [DBG] pgmap v2020: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:02.436 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[84576]: cluster 2024-09-13T15:14:01.125501+0000 mgr.y (mgr.26252) 1439 : cluster [DBG] pgmap v2020: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:02.436 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[84576]: audit 2024-09-13T15:14:01.997438+0000 mon.c (mon.2) 108 : audit [INF] from='client.? 172.21.15.99:0/1900478501' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.436 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[84576]: audit 2024-09-13T15:14:01.997438+0000 mon.c (mon.2) 108 : audit [INF] from='client.? 172.21.15.99:0/1900478501' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.436 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[84576]: audit 2024-09-13T15:14:01.997988+0000 mon.a (mon.0) 2581 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.436 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[84576]: audit 2024-09-13T15:14:01.997988+0000 mon.a (mon.0) 2581 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.437 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[87147]: cluster 2024-09-13T15:14:01.125501+0000 mgr.y (mgr.26252) 1439 : cluster [DBG] pgmap v2020: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:02.437 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[87147]: cluster 2024-09-13T15:14:01.125501+0000 mgr.y (mgr.26252) 1439 : cluster [DBG] pgmap v2020: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:02.437 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[87147]: audit 2024-09-13T15:14:01.997438+0000 mon.c (mon.2) 108 : audit [INF] from='client.? 172.21.15.99:0/1900478501' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.437 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[87147]: audit 2024-09-13T15:14:01.997438+0000 mon.c (mon.2) 108 : audit [INF] from='client.? 172.21.15.99:0/1900478501' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.437 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[87147]: audit 2024-09-13T15:14:01.997988+0000 mon.a (mon.0) 2581 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.437 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:02 smithi099 bash[87147]: audit 2024-09-13T15:14:01.997988+0000 mon.a (mon.0) 2581 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.445 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:02.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:02.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:02.548 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:02.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:02 smithi177 bash[68594]: cluster 2024-09-13T15:14:01.125501+0000 mgr.y (mgr.26252) 1439 : cluster [DBG] pgmap v2020: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:02.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:02 smithi177 bash[68594]: cluster 2024-09-13T15:14:01.125501+0000 mgr.y (mgr.26252) 1439 : cluster [DBG] pgmap v2020: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:02.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:02 smithi177 bash[68594]: audit 2024-09-13T15:14:01.997438+0000 mon.c (mon.2) 108 : audit [INF] from='client.? 172.21.15.99:0/1900478501' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:02 smithi177 bash[68594]: audit 2024-09-13T15:14:01.997438+0000 mon.c (mon.2) 108 : audit [INF] from='client.? 172.21.15.99:0/1900478501' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:02 smithi177 bash[68594]: audit 2024-09-13T15:14:01.997988+0000 mon.a (mon.0) 2581 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:02 smithi177 bash[68594]: audit 2024-09-13T15:14:01.997988+0000 mon.a (mon.0) 2581 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-09-13T15:14:02.575 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:02.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:02.678 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:02.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:02.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:02.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:02.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:02.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:02.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:02.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:02.941 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:02.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:02.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:03.071 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:03.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:03.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:03.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:03.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:03.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:03.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:03.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:03.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:03 smithi099 bash[84576]: audit 2024-09-13T15:14:02.174976+0000 mon.a (mon.0) 2582 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:14:03.344 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:03 smithi099 bash[84576]: audit 2024-09-13T15:14:02.174976+0000 mon.a (mon.0) 2582 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:14:03.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:03 smithi099 bash[84576]: cluster 2024-09-13T15:14:02.182791+0000 mon.a (mon.0) 2583 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2024-09-13T15:14:03.345 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:03 smithi099 bash[84576]: cluster 2024-09-13T15:14:02.182791+0000 mon.a (mon.0) 2583 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2024-09-13T15:14:03.345 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:03 smithi099 bash[87147]: audit 2024-09-13T15:14:02.174976+0000 mon.a (mon.0) 2582 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:14:03.345 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:03 smithi099 bash[87147]: audit 2024-09-13T15:14:02.174976+0000 mon.a (mon.0) 2582 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:14:03.345 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:03 smithi099 bash[87147]: cluster 2024-09-13T15:14:02.182791+0000 mon.a (mon.0) 2583 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2024-09-13T15:14:03.345 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:03 smithi099 bash[87147]: cluster 2024-09-13T15:14:02.182791+0000 mon.a (mon.0) 2583 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2024-09-13T15:14:03.360 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:03.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:03.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:03.463 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:03.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:03 smithi177 bash[68594]: audit 2024-09-13T15:14:02.174976+0000 mon.a (mon.0) 2582 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:14:03.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:03 smithi177 bash[68594]: audit 2024-09-13T15:14:02.174976+0000 mon.a (mon.0) 2582 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-09-13T15:14:03.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:03 smithi177 bash[68594]: cluster 2024-09-13T15:14:02.182791+0000 mon.a (mon.0) 2583 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2024-09-13T15:14:03.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:03 smithi177 bash[68594]: cluster 2024-09-13T15:14:02.182791+0000 mon.a (mon.0) 2583 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2024-09-13T15:14:03.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:03.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:03.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:03.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:14:03.593 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:03.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:03.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:03.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:03.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:14:03.725 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:03.749 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:03.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:03.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:03.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:03.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:03.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:03.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:04.009 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:04.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:04.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:04.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:04.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:04.141 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:04.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:04.242 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:04.259 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-09-13T15:14:04.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-09-13T15:14:04.269 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:04.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:04.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:04.372 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:04.400 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:04.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:04.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:04.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:04.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:04 smithi177 bash[68594]: cluster 2024-09-13T15:14:03.126228+0000 mgr.y (mgr.26252) 1440 : cluster [DBG] pgmap v2022: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:04.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:04 smithi177 bash[68594]: cluster 2024-09-13T15:14:03.126228+0000 mgr.y (mgr.26252) 1440 : cluster [DBG] pgmap v2022: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:04.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:04 smithi177 bash[68594]: audit 2024-09-13T15:14:03.318542+0000 mon.a (mon.0) 2584 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:04.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:04 smithi177 bash[68594]: audit 2024-09-13T15:14:03.318542+0000 mon.a (mon.0) 2584 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:04.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:04 smithi177 bash[68594]: audit 2024-09-13T15:14:03.320868+0000 mon.a (mon.0) 2585 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:04.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:04 smithi177 bash[68594]: audit 2024-09-13T15:14:03.320868+0000 mon.a (mon.0) 2585 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:04.532 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:04.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:04.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:04.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[84576]: cluster 2024-09-13T15:14:03.126228+0000 mgr.y (mgr.26252) 1440 : cluster [DBG] pgmap v2022: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:04.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[84576]: cluster 2024-09-13T15:14:03.126228+0000 mgr.y (mgr.26252) 1440 : cluster [DBG] pgmap v2022: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:04.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[84576]: audit 2024-09-13T15:14:03.318542+0000 mon.a (mon.0) 2584 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:04.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[84576]: audit 2024-09-13T15:14:03.318542+0000 mon.a (mon.0) 2584 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:04.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[84576]: audit 2024-09-13T15:14:03.320868+0000 mon.a (mon.0) 2585 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:04.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[84576]: audit 2024-09-13T15:14:03.320868+0000 mon.a (mon.0) 2585 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[87147]: cluster 2024-09-13T15:14:03.126228+0000 mgr.y (mgr.26252) 1440 : cluster [DBG] pgmap v2022: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[87147]: cluster 2024-09-13T15:14:03.126228+0000 mgr.y (mgr.26252) 1440 : cluster [DBG] pgmap v2022: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[87147]: audit 2024-09-13T15:14:03.318542+0000 mon.a (mon.0) 2584 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[87147]: audit 2024-09-13T15:14:03.318542+0000 mon.a (mon.0) 2584 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[87147]: audit 2024-09-13T15:14:03.320868+0000 mon.a (mon.0) 2585 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:04 smithi099 bash[87147]: audit 2024-09-13T15:14:03.320868+0000 mon.a (mon.0) 2585 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:04.636 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:04.664 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:04.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:04.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:04.767 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:04.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:04.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:04.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:04.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:04.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:04.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:05.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:05.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:05.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:05.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:05.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:05.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:05.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:05.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:05.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:05.319 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:05.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:05.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:05.423 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:05.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:05.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:05.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:05.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:05.581 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:05.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:05.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:05.683 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:05.712 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:05.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:05.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:05.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:05.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:05.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:05.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:05.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:05.970 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:05.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:05.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:06.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:06.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:06.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:06.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:06.205 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:06.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:06 smithi177 bash[68594]: cluster 2024-09-13T15:14:05.127268+0000 mgr.y (mgr.26252) 1441 : cluster [DBG] pgmap v2023: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:06.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:06 smithi177 bash[68594]: cluster 2024-09-13T15:14:05.127268+0000 mgr.y (mgr.26252) 1441 : cluster [DBG] pgmap v2023: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:06.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:06.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:06.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:06.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:06.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:06 smithi099 bash[84576]: cluster 2024-09-13T15:14:05.127268+0000 mgr.y (mgr.26252) 1441 : cluster [DBG] pgmap v2023: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:06.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:06 smithi099 bash[84576]: cluster 2024-09-13T15:14:05.127268+0000 mgr.y (mgr.26252) 1441 : cluster [DBG] pgmap v2023: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:06.348 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:06 smithi099 bash[87147]: cluster 2024-09-13T15:14:05.127268+0000 mgr.y (mgr.26252) 1441 : cluster [DBG] pgmap v2023: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:06.348 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:06 smithi099 bash[87147]: cluster 2024-09-13T15:14:05.127268+0000 mgr.y (mgr.26252) 1441 : cluster [DBG] pgmap v2023: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:06.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:06.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:06.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:06.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:06.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:06.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:06.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:06.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:06.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:06.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:06.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:06.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:06.750 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:06.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:06.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:06.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:06.882 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:06.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:06.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:06.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:07.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:07.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:07.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:07.144 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:07.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:07.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:07.247 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:07.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:07.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:07.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:14:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:14:07] "GET /metrics HTTP/1.1" 200 36726 "" "Prometheus/2.51.0" 2024-09-13T15:14:07.376 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:07.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:07.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:07.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:07.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:07.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:07.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:07.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:07.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:07.663 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:07.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:07.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:07.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:07.794 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:07.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:07.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:07.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:07.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:07.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:08.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:08.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:08.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:08.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:08.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:08.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:08.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:08.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:08.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:08.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:08 smithi099 bash[84576]: cluster 2024-09-13T15:14:07.128139+0000 mgr.y (mgr.26252) 1442 : cluster [DBG] pgmap v2024: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:08.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:08 smithi099 bash[84576]: cluster 2024-09-13T15:14:07.128139+0000 mgr.y (mgr.26252) 1442 : cluster [DBG] pgmap v2024: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:08.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:08 smithi099 bash[87147]: cluster 2024-09-13T15:14:07.128139+0000 mgr.y (mgr.26252) 1442 : cluster [DBG] pgmap v2024: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:08.300 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:08 smithi099 bash[87147]: cluster 2024-09-13T15:14:07.128139+0000 mgr.y (mgr.26252) 1442 : cluster [DBG] pgmap v2024: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:08.316 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:08.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:08.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:08.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:08.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:08.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:08.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:08.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:08.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:08 smithi177 bash[68594]: cluster 2024-09-13T15:14:07.128139+0000 mgr.y (mgr.26252) 1442 : cluster [DBG] pgmap v2024: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:08.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:08 smithi177 bash[68594]: cluster 2024-09-13T15:14:07.128139+0000 mgr.y (mgr.26252) 1442 : cluster [DBG] pgmap v2024: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:08.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:08.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:08.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:08.612 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:14:08.612 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:14:08.680 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:08.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:08.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:08.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:08.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:08.838 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:08.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:08.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:08.941 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:08.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:08.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:08.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:09.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:09.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:09.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:09.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:09.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:09.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:09.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:09.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:09.259 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:14:09.260 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:14:09.260 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-13T15:14:09.260 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:14:09.260 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:14:09.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-13T15:14:09.330 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:09.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:09.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:09.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:09.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:09.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:09.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:09.582 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:09.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:09.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:09.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:09.686 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T15:14:09.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:09.756 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:09.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:09.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:09.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:09.877 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:09.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:09.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:09.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:10.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:10.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:10.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:10.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:10.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:10.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:10.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:10.167 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:14:10.167 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:14:10.168 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:14:10.240 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:10.260 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:10.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:10.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:10.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:14:10 smithi177 bash[61439]: ts=2024-09-13T15:14:10.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:14:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:10 smithi177 bash[68594]: cluster 2024-09-13T15:14:09.128935+0000 mgr.y (mgr.26252) 1443 : cluster [DBG] pgmap v2025: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:10 smithi177 bash[68594]: cluster 2024-09-13T15:14:09.128935+0000 mgr.y (mgr.26252) 1443 : cluster [DBG] pgmap v2025: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:10.363 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:10.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:10.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:10.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:10.493 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:10.520 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:10.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:10.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:10.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:10 smithi099 bash[84576]: cluster 2024-09-13T15:14:09.128935+0000 mgr.y (mgr.26252) 1443 : cluster [DBG] pgmap v2025: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:10.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:10 smithi099 bash[84576]: cluster 2024-09-13T15:14:09.128935+0000 mgr.y (mgr.26252) 1443 : cluster [DBG] pgmap v2025: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:10.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:10 smithi099 bash[87147]: cluster 2024-09-13T15:14:09.128935+0000 mgr.y (mgr.26252) 1443 : cluster [DBG] pgmap v2025: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:10.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:10 smithi099 bash[87147]: cluster 2024-09-13T15:14:09.128935+0000 mgr.y (mgr.26252) 1443 : cluster [DBG] pgmap v2025: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:10.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:10.651 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:10.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:10.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:10.754 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:10.781 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:10.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:10.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:10.884 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:10.912 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:10.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:10.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:11.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:11.042 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:11.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:11.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:11.145 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:11.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:11.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:11.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:11.274 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:11.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:11.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:11.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:11.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:11.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:11.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:11.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:11.537 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:11.563 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:11.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:11.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:11.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:11.692 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:11.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:11.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:11.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:11.820 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:11.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:11.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:11.923 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:11.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:11.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:11.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:12.053 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:12.080 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:12.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:12.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:12.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:12.209 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:12.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:12.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:12.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:12.324 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:12 smithi177 bash[68594]: cluster 2024-09-13T15:14:11.129840+0000 mgr.y (mgr.26252) 1444 : cluster [DBG] pgmap v2026: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:12.325 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:12 smithi177 bash[68594]: cluster 2024-09-13T15:14:11.129840+0000 mgr.y (mgr.26252) 1444 : cluster [DBG] pgmap v2026: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:12.341 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:12.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:12.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:12.443 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:12.455 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:12 smithi099 bash[84576]: cluster 2024-09-13T15:14:11.129840+0000 mgr.y (mgr.26252) 1444 : cluster [DBG] pgmap v2026: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:12.455 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:12 smithi099 bash[84576]: cluster 2024-09-13T15:14:11.129840+0000 mgr.y (mgr.26252) 1444 : cluster [DBG] pgmap v2026: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:12.455 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:12 smithi099 bash[87147]: cluster 2024-09-13T15:14:11.129840+0000 mgr.y (mgr.26252) 1444 : cluster [DBG] pgmap v2026: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:12.455 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:12 smithi099 bash[87147]: cluster 2024-09-13T15:14:11.129840+0000 mgr.y (mgr.26252) 1444 : cluster [DBG] pgmap v2026: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:12.470 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:12.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:12.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:12.573 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:12.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:12.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:12.704 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:12.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:12.731 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:12.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:12.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:12.858 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:12.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:12.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:12.961 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:12.989 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:12.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:13.092 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:13.120 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:13.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:13.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:13.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:13.249 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:13.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:13.353 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:13.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:13.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:13.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:13.483 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:13.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:13.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:13.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:13.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:13.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:13.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:13.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:13.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:13.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:13.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:13.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:13.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:13.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:13.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:13.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:13.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:14.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:14.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:14.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:14.128 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:14.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:14.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:14.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:14.230 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:14:14.232 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:14:14.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:14.278 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:14.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:14.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:14 smithi177 bash[68594]: cluster 2024-09-13T15:14:13.130503+0000 mgr.y (mgr.26252) 1445 : cluster [DBG] pgmap v2027: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:14 smithi177 bash[68594]: cluster 2024-09-13T15:14:13.130503+0000 mgr.y (mgr.26252) 1445 : cluster [DBG] pgmap v2027: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:14.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:14.393 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:14 smithi099 bash[84576]: cluster 2024-09-13T15:14:13.130503+0000 mgr.y (mgr.26252) 1445 : cluster [DBG] pgmap v2027: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:14.393 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:14 smithi099 bash[84576]: cluster 2024-09-13T15:14:13.130503+0000 mgr.y (mgr.26252) 1445 : cluster [DBG] pgmap v2027: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:14.393 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:14 smithi099 bash[87147]: cluster 2024-09-13T15:14:13.130503+0000 mgr.y (mgr.26252) 1445 : cluster [DBG] pgmap v2027: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:14.393 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:14 smithi099 bash[87147]: cluster 2024-09-13T15:14:13.130503+0000 mgr.y (mgr.26252) 1445 : cluster [DBG] pgmap v2027: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:14.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:14.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:14.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:14.512 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:14.539 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:14.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:14.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:14.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:14.667 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:14.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:14.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:14.770 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:14.800 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:14.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:14.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:14.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:14.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:14.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:14.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:15.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:15.060 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:15.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:15.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:15.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:15.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:15.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:15.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:15.293 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:15.317 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:15.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:15.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:15.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:15.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:15.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:15.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:15.578 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:15.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:15.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:15.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:15.710 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:15.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:15.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:15.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:15.839 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:15.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:15.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:15.969 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:15.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:15.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:16.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:16.099 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:16.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:16.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:16.202 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:16.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:16.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:16.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:16.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:16.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:16.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:16.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:16.463 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:16.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:16 smithi177 bash[68594]: cluster 2024-09-13T15:14:15.131559+0000 mgr.y (mgr.26252) 1446 : cluster [DBG] pgmap v2028: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:16.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:16 smithi177 bash[68594]: cluster 2024-09-13T15:14:15.131559+0000 mgr.y (mgr.26252) 1446 : cluster [DBG] pgmap v2028: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:16.488 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:16.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:16.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:16.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:16 smithi099 bash[84576]: cluster 2024-09-13T15:14:15.131559+0000 mgr.y (mgr.26252) 1446 : cluster [DBG] pgmap v2028: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:16.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:16 smithi099 bash[84576]: cluster 2024-09-13T15:14:15.131559+0000 mgr.y (mgr.26252) 1446 : cluster [DBG] pgmap v2028: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:16.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:16 smithi099 bash[87147]: cluster 2024-09-13T15:14:15.131559+0000 mgr.y (mgr.26252) 1446 : cluster [DBG] pgmap v2028: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:16.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:16 smithi099 bash[87147]: cluster 2024-09-13T15:14:15.131559+0000 mgr.y (mgr.26252) 1446 : cluster [DBG] pgmap v2028: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:16.591 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:16.622 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:16.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:16.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:16.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:16.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:16.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:16.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:16.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:16.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:16.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:16.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:16.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:17.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:17.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:17.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:17.118 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:17.129 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:14:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:14:17] "GET /metrics HTTP/1.1" 200 36726 "" "Prometheus/2.51.0" 2024-09-13T15:14:17.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:17.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:17.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:17.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:17.276 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:17.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:17.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:17.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:17.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:17.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:17.510 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:17.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:17.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:17.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:17.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:17.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:17.674 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:17.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:17.775 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:17.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:17.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:17.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:17.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:17.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:17.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:18.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:18.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:18.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:18.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:18.164 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:18.190 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:18.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:18.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:18.293 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:18.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:18.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:18.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:18 smithi177 bash[68594]: cluster 2024-09-13T15:14:17.132153+0000 mgr.y (mgr.26252) 1447 : cluster [DBG] pgmap v2029: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:18.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:18 smithi177 bash[68594]: cluster 2024-09-13T15:14:17.132153+0000 mgr.y (mgr.26252) 1447 : cluster [DBG] pgmap v2029: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:18.421 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:18 smithi099 bash[84576]: cluster 2024-09-13T15:14:17.132153+0000 mgr.y (mgr.26252) 1447 : cluster [DBG] pgmap v2029: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:18.422 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:18 smithi099 bash[84576]: cluster 2024-09-13T15:14:17.132153+0000 mgr.y (mgr.26252) 1447 : cluster [DBG] pgmap v2029: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:18.422 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:18 smithi099 bash[87147]: cluster 2024-09-13T15:14:17.132153+0000 mgr.y (mgr.26252) 1447 : cluster [DBG] pgmap v2029: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:18.422 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:18 smithi099 bash[87147]: cluster 2024-09-13T15:14:17.132153+0000 mgr.y (mgr.26252) 1447 : cluster [DBG] pgmap v2029: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:18.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:18.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:18.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:18.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:18.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:18.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:18.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:18.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:18.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:18.720 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:18.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:18.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:18.823 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:18.850 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:18.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:18.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:18.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:18.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:18.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:18.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:19.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:19.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:19.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:19.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:19.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:19.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:19.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:19.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:19.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:19.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:19 smithi177 bash[68594]: audit 2024-09-13T15:14:18.312501+0000 mon.a (mon.0) 2586 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:19.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:19 smithi177 bash[68594]: audit 2024-09-13T15:14:18.312501+0000 mon.a (mon.0) 2586 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:19.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:19.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:19.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:19.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:19.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:19.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:19 smithi099 bash[84576]: audit 2024-09-13T15:14:18.312501+0000 mon.a (mon.0) 2586 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:19 smithi099 bash[84576]: audit 2024-09-13T15:14:18.312501+0000 mon.a (mon.0) 2586 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:19 smithi099 bash[87147]: audit 2024-09-13T15:14:18.312501+0000 mon.a (mon.0) 2586 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:19 smithi099 bash[87147]: audit 2024-09-13T15:14:18.312501+0000 mon.a (mon.0) 2586 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:19.608 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:19.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:19.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:19.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:19.738 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:19.763 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:19.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:19.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:19.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:19.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:19.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:19.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:19.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:20.025 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:20.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:20.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:20.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:20.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:20.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:20.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:20.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:20.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:20 smithi099 bash[84576]: cluster 2024-09-13T15:14:19.132907+0000 mgr.y (mgr.26252) 1448 : cluster [DBG] pgmap v2030: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:20.270 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:20 smithi099 bash[84576]: cluster 2024-09-13T15:14:19.132907+0000 mgr.y (mgr.26252) 1448 : cluster [DBG] pgmap v2030: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:20.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:20 smithi099 bash[87147]: cluster 2024-09-13T15:14:19.132907+0000 mgr.y (mgr.26252) 1448 : cluster [DBG] pgmap v2030: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:20.271 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:20 smithi099 bash[87147]: cluster 2024-09-13T15:14:19.132907+0000 mgr.y (mgr.26252) 1448 : cluster [DBG] pgmap v2030: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:20.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:20.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:20.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:20.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:20.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:20.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:20.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:20.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:20.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:20.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:20.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:20.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:20 smithi177 bash[68594]: cluster 2024-09-13T15:14:19.132907+0000 mgr.y (mgr.26252) 1448 : cluster [DBG] pgmap v2030: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:20.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:20 smithi177 bash[68594]: cluster 2024-09-13T15:14:19.132907+0000 mgr.y (mgr.26252) 1448 : cluster [DBG] pgmap v2030: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:20.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:14:20 smithi177 bash[61439]: ts=2024-09-13T15:14:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:14:20.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:20.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:20.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:20.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:20.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:20.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:20.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:20.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:20.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:20.938 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:20.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:21.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:21.067 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:21.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:21.170 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:21.195 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:21.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:21.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:21.297 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:21.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:21.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:21.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:21.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:21.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:21.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:21.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:21.482 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:21 smithi099 bash[84576]: cluster 2024-09-13T15:14:21.133631+0000 mgr.y (mgr.26252) 1449 : cluster [DBG] pgmap v2031: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:21.482 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:21 smithi099 bash[84576]: cluster 2024-09-13T15:14:21.133631+0000 mgr.y (mgr.26252) 1449 : cluster [DBG] pgmap v2031: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:21.482 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:21 smithi099 bash[87147]: cluster 2024-09-13T15:14:21.133631+0000 mgr.y (mgr.26252) 1449 : cluster [DBG] pgmap v2031: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:21.482 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:21 smithi099 bash[87147]: cluster 2024-09-13T15:14:21.133631+0000 mgr.y (mgr.26252) 1449 : cluster [DBG] pgmap v2031: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:21.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:21.588 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:21.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:21.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:21.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:21 smithi177 bash[68594]: cluster 2024-09-13T15:14:21.133631+0000 mgr.y (mgr.26252) 1449 : cluster [DBG] pgmap v2031: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:21.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:21 smithi177 bash[68594]: cluster 2024-09-13T15:14:21.133631+0000 mgr.y (mgr.26252) 1449 : cluster [DBG] pgmap v2031: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:21.691 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:21.720 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:21.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:21.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:21.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:21.850 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:21.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:21.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:21.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:21.982 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:21.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:21.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:22.085 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:22.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:22.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:22.215 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:22.244 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:22.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:22.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:22.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:22.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:22.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:22.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:22.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:22.501 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:22.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:22.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:22.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:22.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:22.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:22.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:22.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:22.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:22.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:22.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:22.891 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:22.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:22.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:22.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:23.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:23.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:23.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:23.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:23.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:23.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:23.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:23.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:23.284 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:23.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:23.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:23.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:23.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:23.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:23.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:23.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:23.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:23.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:23.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:23.648 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:23.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:23.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:23.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:23.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:23.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:23.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:23.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:23.938 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:23.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:23.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:24.041 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:24.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:24.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:24.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:24.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:24.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:24.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:24.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:24 smithi177 bash[68594]: cluster 2024-09-13T15:14:23.134436+0000 mgr.y (mgr.26252) 1450 : cluster [DBG] pgmap v2032: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:24.313 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:24 smithi177 bash[68594]: cluster 2024-09-13T15:14:23.134436+0000 mgr.y (mgr.26252) 1450 : cluster [DBG] pgmap v2032: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:24.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:24.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:24.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:24.411 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:24 smithi099 bash[84576]: cluster 2024-09-13T15:14:23.134436+0000 mgr.y (mgr.26252) 1450 : cluster [DBG] pgmap v2032: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:24.411 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:24 smithi099 bash[84576]: cluster 2024-09-13T15:14:23.134436+0000 mgr.y (mgr.26252) 1450 : cluster [DBG] pgmap v2032: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:24.411 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:24 smithi099 bash[87147]: cluster 2024-09-13T15:14:23.134436+0000 mgr.y (mgr.26252) 1450 : cluster [DBG] pgmap v2032: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:24.411 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:24 smithi099 bash[87147]: cluster 2024-09-13T15:14:23.134436+0000 mgr.y (mgr.26252) 1450 : cluster [DBG] pgmap v2032: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:24.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:24.458 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:24.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:24.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:24.561 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:24.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:24.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:24.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:24.719 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:24.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:24.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:24.822 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:24.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:24.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:24.878 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:14:24.878 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:14:24.896 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:14:24.896 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:14:24.913 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:14:24.913 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:14:24.930 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:14:24.931 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:14:24.949 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:14:24.949 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:14:24.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:24.969 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:14:24.969 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:14:24.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:24.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:24.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:24.990 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:14:24.990 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:14:25.010 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:14:25.010 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:14:25.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:25.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:25.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:25.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:25.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:25.225 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:25.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:25.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:25.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:25 smithi099 bash[84576]: audit 2024-09-13T15:14:24.859857+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.99:0/751374789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:25.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:25 smithi099 bash[84576]: audit 2024-09-13T15:14:24.859857+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.99:0/751374789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:25.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:25 smithi099 bash[87147]: audit 2024-09-13T15:14:24.859857+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.99:0/751374789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:25 smithi099 bash[87147]: audit 2024-09-13T15:14:24.859857+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.99:0/751374789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:25.329 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:25.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:25.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:25.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:25.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:25.487 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:25.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:25.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:25.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:25.601 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:25 smithi177 bash[68594]: audit 2024-09-13T15:14:24.859857+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.99:0/751374789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:25.601 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:25 smithi177 bash[68594]: audit 2024-09-13T15:14:24.859857+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.99:0/751374789' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:25.617 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:25.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:25.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:25.720 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:25.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:25.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:25.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:25.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:25.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:25.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:25.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:25.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:26.003 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:26.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:26.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:26.106 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:26.127 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:26.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:26.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:26.207 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_4' removed 2024-09-13T15:14:26.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:26.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[84576]: cluster 2024-09-13T15:14:25.135073+0000 mgr.y (mgr.26252) 1451 : cluster [DBG] pgmap v2033: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:26.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[84576]: cluster 2024-09-13T15:14:25.135073+0000 mgr.y (mgr.26252) 1451 : cluster [DBG] pgmap v2033: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:26.247 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[84576]: audit 2024-09-13T15:14:26.117434+0000 mon.c (mon.2) 110 : audit [INF] from='client.? 172.21.15.99:0/920005725' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[84576]: audit 2024-09-13T15:14:26.117434+0000 mon.c (mon.2) 110 : audit [INF] from='client.? 172.21.15.99:0/920005725' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[84576]: audit 2024-09-13T15:14:26.117962+0000 mon.a (mon.0) 2587 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[84576]: audit 2024-09-13T15:14:26.117962+0000 mon.a (mon.0) 2587 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[87147]: cluster 2024-09-13T15:14:25.135073+0000 mgr.y (mgr.26252) 1451 : cluster [DBG] pgmap v2033: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:26.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[87147]: cluster 2024-09-13T15:14:25.135073+0000 mgr.y (mgr.26252) 1451 : cluster [DBG] pgmap v2033: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:26.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[87147]: audit 2024-09-13T15:14:26.117434+0000 mon.c (mon.2) 110 : audit [INF] from='client.? 172.21.15.99:0/920005725' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[87147]: audit 2024-09-13T15:14:26.117434+0000 mon.c (mon.2) 110 : audit [INF] from='client.? 172.21.15.99:0/920005725' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[87147]: audit 2024-09-13T15:14:26.117962+0000 mon.a (mon.0) 2587 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.248 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:26 smithi099 bash[87147]: audit 2024-09-13T15:14:26.117962+0000 mon.a (mon.0) 2587 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.255 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:26.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:26.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:26.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:26.385 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:26.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:26.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:26.498 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:26 smithi177 bash[68594]: cluster 2024-09-13T15:14:25.135073+0000 mgr.y (mgr.26252) 1451 : cluster [DBG] pgmap v2033: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:26.498 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:26 smithi177 bash[68594]: cluster 2024-09-13T15:14:25.135073+0000 mgr.y (mgr.26252) 1451 : cluster [DBG] pgmap v2033: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:26.498 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:26 smithi177 bash[68594]: audit 2024-09-13T15:14:26.117434+0000 mon.c (mon.2) 110 : audit [INF] from='client.? 172.21.15.99:0/920005725' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.498 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:26 smithi177 bash[68594]: audit 2024-09-13T15:14:26.117434+0000 mon.c (mon.2) 110 : audit [INF] from='client.? 172.21.15.99:0/920005725' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.498 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:26 smithi177 bash[68594]: audit 2024-09-13T15:14:26.117962+0000 mon.a (mon.0) 2587 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.498 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:26 smithi177 bash[68594]: audit 2024-09-13T15:14:26.117962+0000 mon.a (mon.0) 2587 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:14:26.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:26.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:26.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:26.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:26.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:26.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:26.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:26.744 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:26.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:26.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:26.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:26.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:26.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:26.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:26.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:27.002 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:27.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:27.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:27.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:27.133 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:27.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:27.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:27.229 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T15:14:27.264 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:27.292 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:27.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:27.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:14:27] "GET /metrics HTTP/1.1" 200 36724 "" "Prometheus/2.51.0" 2024-09-13T15:14:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[87147]: audit 2024-09-13T15:14:26.199110+0000 mon.a (mon.0) 2588 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:14:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[87147]: audit 2024-09-13T15:14:26.199110+0000 mon.a (mon.0) 2588 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:14:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[87147]: cluster 2024-09-13T15:14:26.202640+0000 mon.a (mon.0) 2589 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2024-09-13T15:14:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[87147]: cluster 2024-09-13T15:14:26.202640+0000 mon.a (mon.0) 2589 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2024-09-13T15:14:27.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[87147]: audit 2024-09-13T15:14:26.555677+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.99:0/870387146' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[87147]: audit 2024-09-13T15:14:26.555677+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.99:0/870387146' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[87147]: audit 2024-09-13T15:14:26.556174+0000 mon.a (mon.0) 2590 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[87147]: audit 2024-09-13T15:14:26.556174+0000 mon.a (mon.0) 2590 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[84576]: audit 2024-09-13T15:14:26.199110+0000 mon.a (mon.0) 2588 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:14:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[84576]: audit 2024-09-13T15:14:26.199110+0000 mon.a (mon.0) 2588 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:14:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[84576]: cluster 2024-09-13T15:14:26.202640+0000 mon.a (mon.0) 2589 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2024-09-13T15:14:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[84576]: cluster 2024-09-13T15:14:26.202640+0000 mon.a (mon.0) 2589 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2024-09-13T15:14:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[84576]: audit 2024-09-13T15:14:26.555677+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.99:0/870387146' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[84576]: audit 2024-09-13T15:14:26.555677+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.99:0/870387146' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[84576]: audit 2024-09-13T15:14:26.556174+0000 mon.a (mon.0) 2590 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:27 smithi099 bash[84576]: audit 2024-09-13T15:14:26.556174+0000 mon.a (mon.0) 2590 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:27.423 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:27.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:27.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:27.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:27.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:27 smithi177 bash[68594]: audit 2024-09-13T15:14:26.199110+0000 mon.a (mon.0) 2588 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:14:27.538 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:27 smithi177 bash[68594]: audit 2024-09-13T15:14:26.199110+0000 mon.a (mon.0) 2588 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:14:27.538 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:27 smithi177 bash[68594]: cluster 2024-09-13T15:14:26.202640+0000 mon.a (mon.0) 2589 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2024-09-13T15:14:27.538 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:27 smithi177 bash[68594]: cluster 2024-09-13T15:14:26.202640+0000 mon.a (mon.0) 2589 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2024-09-13T15:14:27.538 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:27 smithi177 bash[68594]: audit 2024-09-13T15:14:26.555677+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.99:0/870387146' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.538 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:27 smithi177 bash[68594]: audit 2024-09-13T15:14:26.555677+0000 mon.c (mon.2) 111 : audit [INF] from='client.? 172.21.15.99:0/870387146' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.538 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:27 smithi177 bash[68594]: audit 2024-09-13T15:14:26.556174+0000 mon.a (mon.0) 2590 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.538 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:27 smithi177 bash[68594]: audit 2024-09-13T15:14:26.556174+0000 mon.a (mon.0) 2590 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:14:27.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:27.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:27.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:27.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:27.682 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:27.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:27.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:27.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:27.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:27.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:27.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:27.916 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:27.946 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:27.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:27.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:28.049 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:28.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:28.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:28.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:28.179 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:28.206 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:28.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:28.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:28.310 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:28.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:28.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:28.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:28.467 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:28.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:28 smithi177 bash[68594]: cluster 2024-09-13T15:14:27.135776+0000 mgr.y (mgr.26252) 1452 : cluster [DBG] pgmap v2035: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:28.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:28 smithi177 bash[68594]: cluster 2024-09-13T15:14:27.135776+0000 mgr.y (mgr.26252) 1452 : cluster [DBG] pgmap v2035: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:28.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:28 smithi177 bash[68594]: cluster 2024-09-13T15:14:27.207396+0000 mon.a (mon.0) 2591 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:28.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:28 smithi177 bash[68594]: cluster 2024-09-13T15:14:27.207396+0000 mon.a (mon.0) 2591 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:28.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:28 smithi177 bash[68594]: audit 2024-09-13T15:14:27.220224+0000 mon.a (mon.0) 2592 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:14:28.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:28 smithi177 bash[68594]: audit 2024-09-13T15:14:27.220224+0000 mon.a (mon.0) 2592 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:14:28.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:28 smithi177 bash[68594]: cluster 2024-09-13T15:14:27.228884+0000 mon.a (mon.0) 2593 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2024-09-13T15:14:28.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:28 smithi177 bash[68594]: cluster 2024-09-13T15:14:27.228884+0000 mon.a (mon.0) 2593 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2024-09-13T15:14:28.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:28.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:28.493 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[84576]: cluster 2024-09-13T15:14:27.135776+0000 mgr.y (mgr.26252) 1452 : cluster [DBG] pgmap v2035: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:28.493 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[84576]: cluster 2024-09-13T15:14:27.135776+0000 mgr.y (mgr.26252) 1452 : cluster [DBG] pgmap v2035: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:28.494 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[84576]: cluster 2024-09-13T15:14:27.207396+0000 mon.a (mon.0) 2591 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:28.494 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[84576]: cluster 2024-09-13T15:14:27.207396+0000 mon.a (mon.0) 2591 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:28.494 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[84576]: audit 2024-09-13T15:14:27.220224+0000 mon.a (mon.0) 2592 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:14:28.494 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[84576]: audit 2024-09-13T15:14:27.220224+0000 mon.a (mon.0) 2592 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:14:28.494 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[84576]: cluster 2024-09-13T15:14:27.228884+0000 mon.a (mon.0) 2593 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2024-09-13T15:14:28.494 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[84576]: cluster 2024-09-13T15:14:27.228884+0000 mon.a (mon.0) 2593 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2024-09-13T15:14:28.494 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[87147]: cluster 2024-09-13T15:14:27.135776+0000 mgr.y (mgr.26252) 1452 : cluster [DBG] pgmap v2035: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:28.494 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[87147]: cluster 2024-09-13T15:14:27.135776+0000 mgr.y (mgr.26252) 1452 : cluster [DBG] pgmap v2035: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:28.495 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[87147]: cluster 2024-09-13T15:14:27.207396+0000 mon.a (mon.0) 2591 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:28.495 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[87147]: cluster 2024-09-13T15:14:27.207396+0000 mon.a (mon.0) 2591 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:28.495 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[87147]: audit 2024-09-13T15:14:27.220224+0000 mon.a (mon.0) 2592 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:14:28.495 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[87147]: audit 2024-09-13T15:14:27.220224+0000 mon.a (mon.0) 2592 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:14:28.495 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[87147]: cluster 2024-09-13T15:14:27.228884+0000 mon.a (mon.0) 2593 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2024-09-13T15:14:28.495 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:28 smithi099 bash[87147]: cluster 2024-09-13T15:14:27.228884+0000 mon.a (mon.0) 2593 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2024-09-13T15:14:28.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:28.600 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:28.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:28.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:28.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:28.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:28.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:28.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:28.833 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:28.861 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:28.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:28.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:28.964 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:28.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:28.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:28.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:29.094 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:29.122 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:29.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:29.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:29.226 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:29.253 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:29.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:29.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:29.356 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:29.383 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:29.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:29.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:29.486 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:29.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:29.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:29.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:29.646 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:29.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:29.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:29.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:29.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:29.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:29.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:29.907 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:29.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:29.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:30.010 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:30.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:30.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:30.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:30.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:30.169 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:30.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:30.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:30.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:30.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:30 smithi099 bash[84576]: cluster 2024-09-13T15:14:29.136329+0000 mgr.y (mgr.26252) 1453 : cluster [DBG] pgmap v2037: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:30.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:30 smithi099 bash[84576]: cluster 2024-09-13T15:14:29.136329+0000 mgr.y (mgr.26252) 1453 : cluster [DBG] pgmap v2037: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:30.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:30 smithi099 bash[87147]: cluster 2024-09-13T15:14:29.136329+0000 mgr.y (mgr.26252) 1453 : cluster [DBG] pgmap v2037: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:30.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:30 smithi099 bash[87147]: cluster 2024-09-13T15:14:29.136329+0000 mgr.y (mgr.26252) 1453 : cluster [DBG] pgmap v2037: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:30.300 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:30.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:30.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:30.404 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:30.415 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:14:30 smithi177 bash[61439]: ts=2024-09-13T15:14:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:14:30.416 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:30 smithi177 bash[68594]: cluster 2024-09-13T15:14:29.136329+0000 mgr.y (mgr.26252) 1453 : cluster [DBG] pgmap v2037: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:30.416 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:30 smithi177 bash[68594]: cluster 2024-09-13T15:14:29.136329+0000 mgr.y (mgr.26252) 1453 : cluster [DBG] pgmap v2037: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:30.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:30.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:30.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:30.537 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:30.562 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:30.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:30.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:30.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:30.692 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:30.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:30.796 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:30.810 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T15:14:30.811 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T15:14:30.811 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-09-13T15:14:30.812 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-09-13T15:14:30.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:30.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:30.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:30.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:30.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:30.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:30.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:31.056 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:31.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:31.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:31.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:31.216 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:31.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:31.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:31.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:31.347 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:31.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:31.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:31.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:31.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:31.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:31.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:31.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:31.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:31.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:31.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:31.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:31.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:31.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:31.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:31.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:31.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:31.968 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:31.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:31.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:31.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:32.099 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:32.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:32.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:32.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:32.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:32.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:32 smithi177 bash[68594]: cluster 2024-09-13T15:14:31.137286+0000 mgr.y (mgr.26252) 1454 : cluster [DBG] pgmap v2038: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:32.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:32 smithi177 bash[68594]: cluster 2024-09-13T15:14:31.137286+0000 mgr.y (mgr.26252) 1454 : cluster [DBG] pgmap v2038: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:32.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:32.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:32.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:32.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:32.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:32.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:32.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:14:32.401 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:32 smithi099 bash[87147]: cluster 2024-09-13T15:14:31.137286+0000 mgr.y (mgr.26252) 1454 : cluster [DBG] pgmap v2038: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:32.401 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:32 smithi099 bash[87147]: cluster 2024-09-13T15:14:31.137286+0000 mgr.y (mgr.26252) 1454 : cluster [DBG] pgmap v2038: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:32.401 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:32 smithi099 bash[84576]: cluster 2024-09-13T15:14:31.137286+0000 mgr.y (mgr.26252) 1454 : cluster [DBG] pgmap v2038: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:32.401 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:32 smithi099 bash[84576]: cluster 2024-09-13T15:14:31.137286+0000 mgr.y (mgr.26252) 1454 : cluster [DBG] pgmap v2038: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:32.489 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:32.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:32.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:32.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:32.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:32.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:32.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:32.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:32.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:14:32.742 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:32.762 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:32.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:32.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:32.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:32.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:32.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:32.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:33.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:33.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:33.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:33.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:33.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:33.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:33.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:33.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:33.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:33.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:33.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:33.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:33.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:33.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:33.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:33.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:33.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:33.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:33.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:33.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:33.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:33.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:33.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:33.793 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:14:33.793 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T15:14:33.793 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:14:33.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:33.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:33.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:33.822 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:14:33.822 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:14:33.822 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2024-09-13T15:14:33.822 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:14:33.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:33.933 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:33.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:33.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:34.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:34.065 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:34.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:34.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:34.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:34.193 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:34.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:34.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:34.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:34.324 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:34.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:34.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:34.427 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:34.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:34.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:34.560 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:34 smithi099 bash[84576]: cluster 2024-09-13T15:14:33.138123+0000 mgr.y (mgr.26252) 1455 : cluster [DBG] pgmap v2039: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:34 smithi099 bash[84576]: cluster 2024-09-13T15:14:33.138123+0000 mgr.y (mgr.26252) 1455 : cluster [DBG] pgmap v2039: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:34 smithi099 bash[84576]: audit 2024-09-13T15:14:33.313149+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:34.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:34 smithi099 bash[84576]: audit 2024-09-13T15:14:33.313149+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:34.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:34 smithi099 bash[87147]: cluster 2024-09-13T15:14:33.138123+0000 mgr.y (mgr.26252) 1455 : cluster [DBG] pgmap v2039: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:34.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:34 smithi099 bash[87147]: cluster 2024-09-13T15:14:33.138123+0000 mgr.y (mgr.26252) 1455 : cluster [DBG] pgmap v2039: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:34.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:34 smithi099 bash[87147]: audit 2024-09-13T15:14:33.313149+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:34.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:34 smithi099 bash[87147]: audit 2024-09-13T15:14:33.313149+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:34.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:34.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:34.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:34.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:34 smithi177 bash[68594]: cluster 2024-09-13T15:14:33.138123+0000 mgr.y (mgr.26252) 1455 : cluster [DBG] pgmap v2039: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:34.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:34 smithi177 bash[68594]: cluster 2024-09-13T15:14:33.138123+0000 mgr.y (mgr.26252) 1455 : cluster [DBG] pgmap v2039: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:34.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:34 smithi177 bash[68594]: audit 2024-09-13T15:14:33.313149+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:34.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:34 smithi177 bash[68594]: audit 2024-09-13T15:14:33.313149+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:34.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:34.717 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:34.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:34.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:34.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:34.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:34.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:34.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:34.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:34.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:34.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:34.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:35.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:35.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:35.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:35.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:35.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:35.237 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:35.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:35.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:35.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:35.367 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:35.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:35.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:35.470 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:35.497 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:35.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:35.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:35.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:35.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:35.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:35.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:35.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T15:14:35.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:35.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:35.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:35.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:35.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:35.880 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:35.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:35.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:35.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:36.006 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:36.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:36.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:36.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:36.138 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:36.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:36.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:36.242 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:36.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:36.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:36.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:36.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:36 smithi099 bash[84576]: cluster 2024-09-13T15:14:35.139164+0000 mgr.y (mgr.26252) 1456 : cluster [DBG] pgmap v2040: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:36.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:36 smithi099 bash[84576]: cluster 2024-09-13T15:14:35.139164+0000 mgr.y (mgr.26252) 1456 : cluster [DBG] pgmap v2040: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:36.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:36 smithi099 bash[87147]: cluster 2024-09-13T15:14:35.139164+0000 mgr.y (mgr.26252) 1456 : cluster [DBG] pgmap v2040: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:36.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:36 smithi099 bash[87147]: cluster 2024-09-13T15:14:35.139164+0000 mgr.y (mgr.26252) 1456 : cluster [DBG] pgmap v2040: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:36.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:36.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:36.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:36.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:36.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:36 smithi177 bash[68594]: cluster 2024-09-13T15:14:35.139164+0000 mgr.y (mgr.26252) 1456 : cluster [DBG] pgmap v2040: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:36.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:36 smithi177 bash[68594]: cluster 2024-09-13T15:14:35.139164+0000 mgr.y (mgr.26252) 1456 : cluster [DBG] pgmap v2040: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:36.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:36.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:36.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:36.633 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:36.661 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:36.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:36.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:36.764 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:36.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:36.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:36.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:36.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:36.921 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:36.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:36.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:37.024 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:37.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:37.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:37.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:37.154 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:37.162 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:14:37.181 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:37.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:37.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:37.284 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:37.294 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:14:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:14:37] "GET /metrics HTTP/1.1" 200 34375 "" "Prometheus/2.51.0" 2024-09-13T15:14:37.310 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:37.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:37.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:37.413 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:37.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:37.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:37.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:37.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:37.571 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:37.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:37.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:37.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:37.697 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:37.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:37.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:37.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:37.827 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:37.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:37.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:37.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:37.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:37.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:37.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:38.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:38.088 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:38.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:38.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:38.176 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:14:38.176 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:14:38.192 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:38.220 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:38.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:38.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:38.323 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:38.350 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:38.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:38.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:38.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:38.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:38 smithi177 bash[68594]: cluster 2024-09-13T15:14:37.139915+0000 mgr.y (mgr.26252) 1457 : cluster [DBG] pgmap v2041: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:38.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:38 smithi177 bash[68594]: cluster 2024-09-13T15:14:37.139915+0000 mgr.y (mgr.26252) 1457 : cluster [DBG] pgmap v2041: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:38.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:38.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:38.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:38.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:38 smithi099 bash[84576]: cluster 2024-09-13T15:14:37.139915+0000 mgr.y (mgr.26252) 1457 : cluster [DBG] pgmap v2041: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:38.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:38 smithi099 bash[84576]: cluster 2024-09-13T15:14:37.139915+0000 mgr.y (mgr.26252) 1457 : cluster [DBG] pgmap v2041: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:38.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:38 smithi099 bash[87147]: cluster 2024-09-13T15:14:37.139915+0000 mgr.y (mgr.26252) 1457 : cluster [DBG] pgmap v2041: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:38.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:38 smithi099 bash[87147]: cluster 2024-09-13T15:14:37.139915+0000 mgr.y (mgr.26252) 1457 : cluster [DBG] pgmap v2041: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:38.586 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:38.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:38.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:38.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:38.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:38.755 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:38.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:38.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:38.858 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:38.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:38.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:38.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:38.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:39.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:39.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:39.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:39.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:39.144 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:39.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:39.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:39.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:39.276 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:39.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:39.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:39.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:39.390 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:39 smithi177 bash[68594]: cluster 2024-09-13T15:14:39.140649+0000 mgr.y (mgr.26252) 1458 : cluster [DBG] pgmap v2042: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:39.391 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:39 smithi177 bash[68594]: cluster 2024-09-13T15:14:39.140649+0000 mgr.y (mgr.26252) 1458 : cluster [DBG] pgmap v2042: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:39.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:39.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:39.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:39.536 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:39.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:39.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:39.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:39.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:39.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:39.669 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:39 smithi099 bash[84576]: cluster 2024-09-13T15:14:39.140649+0000 mgr.y (mgr.26252) 1458 : cluster [DBG] pgmap v2042: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:39.669 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:39 smithi099 bash[84576]: cluster 2024-09-13T15:14:39.140649+0000 mgr.y (mgr.26252) 1458 : cluster [DBG] pgmap v2042: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:39.669 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:39 smithi099 bash[87147]: cluster 2024-09-13T15:14:39.140649+0000 mgr.y (mgr.26252) 1458 : cluster [DBG] pgmap v2042: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:39.670 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:39 smithi099 bash[87147]: cluster 2024-09-13T15:14:39.140649+0000 mgr.y (mgr.26252) 1458 : cluster [DBG] pgmap v2042: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:39.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:39.797 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:39.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:39.902 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:39.928 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:39.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:40.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:40.058 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:40.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:40.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:40.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:40.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:40.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:40.229 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:14:40.292 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:40.320 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:40.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:40.423 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:40.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:40.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:40.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:40.564 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:14:40 smithi177 bash[61439]: ts=2024-09-13T15:14:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:14:40.581 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:40.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:40.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:40.672 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:14:40.672 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:14:40.672 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-13T15:14:40.672 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:14:40.672 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:14:40.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-13T15:14:40.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:40.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:40.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:40.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:40.847 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:40.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:40.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:40.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:40.973 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:41.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:41.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:41.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:41.103 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:41.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:41.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:41.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:41.234 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:41.260 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:41.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:41.363 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:41.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:41.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:41.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:41.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:41.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:41.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:41.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:41.625 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:41.650 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:41.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:41.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:41.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:41.782 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:41.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:41.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:41.884 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:41.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:41.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:41.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:42.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:42.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:42.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:42.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:42.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:42.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:42.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:42.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:42.274 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:42.286 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:42 smithi099 bash[84576]: cluster 2024-09-13T15:14:41.141365+0000 mgr.y (mgr.26252) 1459 : cluster [DBG] pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:42.286 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:42 smithi099 bash[84576]: cluster 2024-09-13T15:14:41.141365+0000 mgr.y (mgr.26252) 1459 : cluster [DBG] pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:42.286 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:42 smithi099 bash[87147]: cluster 2024-09-13T15:14:41.141365+0000 mgr.y (mgr.26252) 1459 : cluster [DBG] pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:42.286 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:42 smithi099 bash[87147]: cluster 2024-09-13T15:14:41.141365+0000 mgr.y (mgr.26252) 1459 : cluster [DBG] pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:42.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:42.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:42.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:42.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:42.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:42.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:42.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:42.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:42.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:42 smithi177 bash[68594]: cluster 2024-09-13T15:14:41.141365+0000 mgr.y (mgr.26252) 1459 : cluster [DBG] pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:42.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:42 smithi177 bash[68594]: cluster 2024-09-13T15:14:41.141365+0000 mgr.y (mgr.26252) 1459 : cluster [DBG] pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:42.564 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:42.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:42.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:42.667 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:42.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:42.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:42.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:42.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:42.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:42.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:42.929 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:42.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:42.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:42.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:43.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:43.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:43.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:43.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:43.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:43.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:43.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:43.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:43.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:43.354 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:43.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:43.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:43.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:43.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:43.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:43.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:43.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:43.617 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:43.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:43.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:43.721 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:43.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:43.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:43.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:43.853 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:43.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:43.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:43.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:43.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:44.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:44.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:44.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:44.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:44.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:44.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:44.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:44.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:44.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:44 smithi177 bash[68594]: cluster 2024-09-13T15:14:43.142139+0000 mgr.y (mgr.26252) 1460 : cluster [DBG] pgmap v2044: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:44.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:44 smithi177 bash[68594]: cluster 2024-09-13T15:14:43.142139+0000 mgr.y (mgr.26252) 1460 : cluster [DBG] pgmap v2044: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:44.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:44.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:44.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:44.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:44.410 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:44.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:44.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:44.499 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:44 smithi099 bash[84576]: cluster 2024-09-13T15:14:43.142139+0000 mgr.y (mgr.26252) 1460 : cluster [DBG] pgmap v2044: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:44.499 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:44 smithi099 bash[84576]: cluster 2024-09-13T15:14:43.142139+0000 mgr.y (mgr.26252) 1460 : cluster [DBG] pgmap v2044: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:44.500 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:44 smithi099 bash[87147]: cluster 2024-09-13T15:14:43.142139+0000 mgr.y (mgr.26252) 1460 : cluster [DBG] pgmap v2044: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:44.500 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:44 smithi099 bash[87147]: cluster 2024-09-13T15:14:43.142139+0000 mgr.y (mgr.26252) 1460 : cluster [DBG] pgmap v2044: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:44.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:44.537 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:44.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:44.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:44.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:44.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:44.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:44.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:44.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:44.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:44.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:44.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:44.889 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:44.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:44.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:44.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:45.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:45.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:45.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:45.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:45.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:45.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:45.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:45.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:45.273 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:45.294 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:45.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:45.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:45.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:45.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:45.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:45.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:45.455 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:14:45.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:45.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:45.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:45.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:45.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:45.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:45.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:45.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:45.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:45.808 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:45.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:45.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:45.911 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:45.938 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:45.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:46.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:46.068 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:46.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:46.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:46.197 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:46.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:46.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:46.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:46 smithi099 bash[84576]: cluster 2024-09-13T15:14:45.142946+0000 mgr.y (mgr.26252) 1461 : cluster [DBG] pgmap v2045: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:46.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:46 smithi099 bash[84576]: cluster 2024-09-13T15:14:45.142946+0000 mgr.y (mgr.26252) 1461 : cluster [DBG] pgmap v2045: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:46.334 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:46 smithi099 bash[84576]: audit 2024-09-13T15:14:45.958890+0000 mon.a (mon.0) 2595 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:14:46.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:46 smithi099 bash[84576]: audit 2024-09-13T15:14:45.958890+0000 mon.a (mon.0) 2595 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:14:46.335 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:46 smithi099 bash[87147]: cluster 2024-09-13T15:14:45.142946+0000 mgr.y (mgr.26252) 1461 : cluster [DBG] pgmap v2045: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:46.335 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:46 smithi099 bash[87147]: cluster 2024-09-13T15:14:45.142946+0000 mgr.y (mgr.26252) 1461 : cluster [DBG] pgmap v2045: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:46.335 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:46 smithi099 bash[87147]: audit 2024-09-13T15:14:45.958890+0000 mon.a (mon.0) 2595 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:14:46.335 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:46 smithi099 bash[87147]: audit 2024-09-13T15:14:45.958890+0000 mon.a (mon.0) 2595 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:14:46.356 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:46.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:46 smithi177 bash[68594]: cluster 2024-09-13T15:14:45.142946+0000 mgr.y (mgr.26252) 1461 : cluster [DBG] pgmap v2045: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:46 smithi177 bash[68594]: cluster 2024-09-13T15:14:45.142946+0000 mgr.y (mgr.26252) 1461 : cluster [DBG] pgmap v2045: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:46 smithi177 bash[68594]: audit 2024-09-13T15:14:45.958890+0000 mon.a (mon.0) 2595 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:14:46.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:46 smithi177 bash[68594]: audit 2024-09-13T15:14:45.958890+0000 mon.a (mon.0) 2595 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:14:46.459 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:46.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:46.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:46.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:46.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:46.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:46.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:46.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:46.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:46.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:46.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:46.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:46.837 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:46.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:46.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:46.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:46.960 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:46.978 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:46.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:46.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:47.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:47.103 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:47.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:47.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:47.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:47.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:47.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:47.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:47.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:14:47] "GET /metrics HTTP/1.1" 200 34375 "" "Prometheus/2.51.0" 2024-09-13T15:14:47.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:47.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:47.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:47.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:47.462 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:47.490 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:47.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:47.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:47.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:47.607 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:47 smithi177 bash[68594]: audit 2024-09-13T15:14:46.529422+0000 mon.a (mon.0) 2596 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:47 smithi177 bash[68594]: audit 2024-09-13T15:14:46.529422+0000 mon.a (mon.0) 2596 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:47 smithi177 bash[68594]: audit 2024-09-13T15:14:46.533516+0000 mon.a (mon.0) 2597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:47 smithi177 bash[68594]: audit 2024-09-13T15:14:46.533516+0000 mon.a (mon.0) 2597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:47 smithi177 bash[68594]: cluster 2024-09-13T15:14:47.143744+0000 mgr.y (mgr.26252) 1462 : cluster [DBG] pgmap v2046: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:47.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:47 smithi177 bash[68594]: cluster 2024-09-13T15:14:47.143744+0000 mgr.y (mgr.26252) 1462 : cluster [DBG] pgmap v2046: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:47.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:47.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:47.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:47.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:47.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:47.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:47.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:47.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[84576]: audit 2024-09-13T15:14:46.529422+0000 mon.a (mon.0) 2596 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[84576]: audit 2024-09-13T15:14:46.529422+0000 mon.a (mon.0) 2596 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[84576]: audit 2024-09-13T15:14:46.533516+0000 mon.a (mon.0) 2597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[84576]: audit 2024-09-13T15:14:46.533516+0000 mon.a (mon.0) 2597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[84576]: cluster 2024-09-13T15:14:47.143744+0000 mgr.y (mgr.26252) 1462 : cluster [DBG] pgmap v2046: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:47.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[84576]: cluster 2024-09-13T15:14:47.143744+0000 mgr.y (mgr.26252) 1462 : cluster [DBG] pgmap v2046: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:47.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[87147]: audit 2024-09-13T15:14:46.529422+0000 mon.a (mon.0) 2596 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[87147]: audit 2024-09-13T15:14:46.529422+0000 mon.a (mon.0) 2596 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[87147]: audit 2024-09-13T15:14:46.533516+0000 mon.a (mon.0) 2597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[87147]: audit 2024-09-13T15:14:46.533516+0000 mon.a (mon.0) 2597 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:47.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[87147]: cluster 2024-09-13T15:14:47.143744+0000 mgr.y (mgr.26252) 1462 : cluster [DBG] pgmap v2046: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:47.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:47 smithi099 bash[87147]: cluster 2024-09-13T15:14:47.143744+0000 mgr.y (mgr.26252) 1462 : cluster [DBG] pgmap v2046: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:47.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:47.885 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:47.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:47.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:47.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:48.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:48.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:48.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:48.119 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:48.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:48.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:48.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:48.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:48.272 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:48.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:48.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:48.376 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:48.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:48.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:48.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:48.505 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:48.532 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:48.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:48.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:48.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:48.660 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:48.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:48.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:48.774 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:48 smithi177 bash[68594]: audit 2024-09-13T15:14:48.312939+0000 mon.a (mon.0) 2598 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:48.774 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:48 smithi177 bash[68594]: audit 2024-09-13T15:14:48.312939+0000 mon.a (mon.0) 2598 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:48.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:48.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:48.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:48.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:48 smithi099 bash[84576]: audit 2024-09-13T15:14:48.312939+0000 mon.a (mon.0) 2598 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:48.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:48 smithi099 bash[84576]: audit 2024-09-13T15:14:48.312939+0000 mon.a (mon.0) 2598 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:48.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:48 smithi099 bash[87147]: audit 2024-09-13T15:14:48.312939+0000 mon.a (mon.0) 2598 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:48.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:48 smithi099 bash[87147]: audit 2024-09-13T15:14:48.312939+0000 mon.a (mon.0) 2598 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:14:48.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:48.922 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:48.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:48.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:49.025 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:49.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:49.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:49.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:49.152 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:49.178 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:49.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:49.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:49.281 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:49.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:49.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:49.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:49.440 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:49.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:49.543 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:49.564 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:49.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:49.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:49.667 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:49.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:49 smithi099 bash[84576]: cluster 2024-09-13T15:14:49.144591+0000 mgr.y (mgr.26252) 1463 : cluster [DBG] pgmap v2047: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:49.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:49 smithi099 bash[84576]: cluster 2024-09-13T15:14:49.144591+0000 mgr.y (mgr.26252) 1463 : cluster [DBG] pgmap v2047: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:49.676 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:49 smithi099 bash[87147]: cluster 2024-09-13T15:14:49.144591+0000 mgr.y (mgr.26252) 1463 : cluster [DBG] pgmap v2047: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:49.676 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:49 smithi099 bash[87147]: cluster 2024-09-13T15:14:49.144591+0000 mgr.y (mgr.26252) 1463 : cluster [DBG] pgmap v2047: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:49.688 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:49.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:49.792 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:49.803 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:49 smithi177 bash[68594]: cluster 2024-09-13T15:14:49.144591+0000 mgr.y (mgr.26252) 1463 : cluster [DBG] pgmap v2047: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:49.803 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:49 smithi177 bash[68594]: cluster 2024-09-13T15:14:49.144591+0000 mgr.y (mgr.26252) 1463 : cluster [DBG] pgmap v2047: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:49.819 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:49.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:49.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:49.922 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:49.951 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:49.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:49.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:50.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:50.075 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:50.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:50.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:50.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:50.189 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:14:50.189 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T15:14:50.189 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:14:50.199 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:50.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:50.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:50.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:50.330 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:50.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:50.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:50.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:14:50 smithi177 bash[61439]: ts=2024-09-13T15:14:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:14:50.433 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:50.460 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:50.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:50.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:50.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:50.590 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:50.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:50.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:50.693 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:50.719 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:50.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:50.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:50.822 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:50.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:50.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:50.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:50.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:50.977 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:50.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:51.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:51.107 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:51.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:51.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:51.236 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:51.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:51.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:51.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:51.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:51.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:51.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:51.490 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:51.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:51.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:51.593 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:51.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:51.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:51.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:51.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:51.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:51.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:51.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:51.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:51.881 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:51.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:51.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:51.984 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:52.012 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:52.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:52.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:52.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:52.138 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:52.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:52.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:52.221 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_5' created 2024-09-13T15:14:52.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:52.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:52.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:52.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:52.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:52.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:52.385 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:52 smithi177 bash[68594]: cluster 2024-09-13T15:14:51.145556+0000 mgr.y (mgr.26252) 1464 : cluster [DBG] pgmap v2048: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:52.385 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:52 smithi177 bash[68594]: cluster 2024-09-13T15:14:51.145556+0000 mgr.y (mgr.26252) 1464 : cluster [DBG] pgmap v2048: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:52.385 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:52 smithi177 bash[68594]: audit 2024-09-13T15:14:51.783044+0000 mon.c (mon.2) 112 : audit [INF] from='client.? 172.21.15.99:0/3871228999' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.385 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:52 smithi177 bash[68594]: audit 2024-09-13T15:14:51.783044+0000 mon.c (mon.2) 112 : audit [INF] from='client.? 172.21.15.99:0/3871228999' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.385 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:52 smithi177 bash[68594]: audit 2024-09-13T15:14:51.783764+0000 mon.a (mon.0) 2599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.385 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:52 smithi177 bash[68594]: audit 2024-09-13T15:14:51.783764+0000 mon.a (mon.0) 2599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:52.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:52.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:52.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:52.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:52.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[84576]: cluster 2024-09-13T15:14:51.145556+0000 mgr.y (mgr.26252) 1464 : cluster [DBG] pgmap v2048: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:52.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[84576]: cluster 2024-09-13T15:14:51.145556+0000 mgr.y (mgr.26252) 1464 : cluster [DBG] pgmap v2048: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:52.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[84576]: audit 2024-09-13T15:14:51.783044+0000 mon.c (mon.2) 112 : audit [INF] from='client.? 172.21.15.99:0/3871228999' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[84576]: audit 2024-09-13T15:14:51.783044+0000 mon.c (mon.2) 112 : audit [INF] from='client.? 172.21.15.99:0/3871228999' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[84576]: audit 2024-09-13T15:14:51.783764+0000 mon.a (mon.0) 2599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[84576]: audit 2024-09-13T15:14:51.783764+0000 mon.a (mon.0) 2599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[87147]: cluster 2024-09-13T15:14:51.145556+0000 mgr.y (mgr.26252) 1464 : cluster [DBG] pgmap v2048: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:52.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[87147]: cluster 2024-09-13T15:14:51.145556+0000 mgr.y (mgr.26252) 1464 : cluster [DBG] pgmap v2048: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:52.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[87147]: audit 2024-09-13T15:14:51.783044+0000 mon.c (mon.2) 112 : audit [INF] from='client.? 172.21.15.99:0/3871228999' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[87147]: audit 2024-09-13T15:14:51.783044+0000 mon.c (mon.2) 112 : audit [INF] from='client.? 172.21.15.99:0/3871228999' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[87147]: audit 2024-09-13T15:14:51.783764+0000 mon.a (mon.0) 2599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:52 smithi099 bash[87147]: audit 2024-09-13T15:14:51.783764+0000 mon.a (mon.0) 2599 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T15:14:52.608 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:52.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:52.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:52.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:52.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:52.766 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:52.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:52.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:52.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:52.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:52.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:52.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:52.996 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:53.021 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:53.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:53.124 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:53.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:53.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:53.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:53.254 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:53.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:53.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:53.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:53 smithi177 bash[68594]: audit 2024-09-13T15:14:52.212789+0000 mon.a (mon.0) 2600 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T15:14:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:53 smithi177 bash[68594]: audit 2024-09-13T15:14:52.212789+0000 mon.a (mon.0) 2600 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T15:14:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:53 smithi177 bash[68594]: cluster 2024-09-13T15:14:52.220233+0000 mon.a (mon.0) 2601 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2024-09-13T15:14:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:53 smithi177 bash[68594]: cluster 2024-09-13T15:14:52.220233+0000 mon.a (mon.0) 2601 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2024-09-13T15:14:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:53 smithi177 bash[68594]: audit 2024-09-13T15:14:52.671041+0000 mon.a (mon.0) 2602 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:53 smithi177 bash[68594]: audit 2024-09-13T15:14:52.671041+0000 mon.a (mon.0) 2602 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:53 smithi177 bash[68594]: audit 2024-09-13T15:14:52.675873+0000 mon.a (mon.0) 2603 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:53 smithi177 bash[68594]: audit 2024-09-13T15:14:52.675873+0000 mon.a (mon.0) 2603 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.380 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:53.403 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:53.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:53.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:53.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:53.532 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:53.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:53.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:53.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[84576]: audit 2024-09-13T15:14:52.212789+0000 mon.a (mon.0) 2600 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T15:14:53.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[84576]: audit 2024-09-13T15:14:52.212789+0000 mon.a (mon.0) 2600 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T15:14:53.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[84576]: cluster 2024-09-13T15:14:52.220233+0000 mon.a (mon.0) 2601 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2024-09-13T15:14:53.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[84576]: cluster 2024-09-13T15:14:52.220233+0000 mon.a (mon.0) 2601 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2024-09-13T15:14:53.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[84576]: audit 2024-09-13T15:14:52.671041+0000 mon.a (mon.0) 2602 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[84576]: audit 2024-09-13T15:14:52.671041+0000 mon.a (mon.0) 2602 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[84576]: audit 2024-09-13T15:14:52.675873+0000 mon.a (mon.0) 2603 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[84576]: audit 2024-09-13T15:14:52.675873+0000 mon.a (mon.0) 2603 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[87147]: audit 2024-09-13T15:14:52.212789+0000 mon.a (mon.0) 2600 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T15:14:53.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[87147]: audit 2024-09-13T15:14:52.212789+0000 mon.a (mon.0) 2600 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T15:14:53.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[87147]: cluster 2024-09-13T15:14:52.220233+0000 mon.a (mon.0) 2601 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2024-09-13T15:14:53.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[87147]: cluster 2024-09-13T15:14:52.220233+0000 mon.a (mon.0) 2601 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2024-09-13T15:14:53.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[87147]: audit 2024-09-13T15:14:52.671041+0000 mon.a (mon.0) 2602 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[87147]: audit 2024-09-13T15:14:52.671041+0000 mon.a (mon.0) 2602 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[87147]: audit 2024-09-13T15:14:52.675873+0000 mon.a (mon.0) 2603 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:53 smithi099 bash[87147]: audit 2024-09-13T15:14:52.675873+0000 mon.a (mon.0) 2603 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:53.635 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:53.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:53.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:53.764 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:53.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:53.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:53.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:53.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:53.922 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:53.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:53.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:54.025 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:54.053 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:54.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:54.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:54.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:54.183 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:54.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:54.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:54.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:54.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:54.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:54.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:54.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[84576]: cluster 2024-09-13T15:14:53.146266+0000 mgr.y (mgr.26252) 1465 : cluster [DBG] pgmap v2050: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:54.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[84576]: cluster 2024-09-13T15:14:53.146266+0000 mgr.y (mgr.26252) 1465 : cluster [DBG] pgmap v2050: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:54.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[84576]: cluster 2024-09-13T15:14:53.219462+0000 mon.a (mon.0) 2604 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2024-09-13T15:14:54.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[84576]: cluster 2024-09-13T15:14:53.219462+0000 mon.a (mon.0) 2604 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2024-09-13T15:14:54.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[84576]: cluster 2024-09-13T15:14:53.674013+0000 mon.a (mon.0) 2605 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:14:54.335 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[84576]: cluster 2024-09-13T15:14:53.674013+0000 mon.a (mon.0) 2605 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:14:54.335 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[87147]: cluster 2024-09-13T15:14:53.146266+0000 mgr.y (mgr.26252) 1465 : cluster [DBG] pgmap v2050: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:54.335 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[87147]: cluster 2024-09-13T15:14:53.146266+0000 mgr.y (mgr.26252) 1465 : cluster [DBG] pgmap v2050: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:54.335 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[87147]: cluster 2024-09-13T15:14:53.219462+0000 mon.a (mon.0) 2604 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2024-09-13T15:14:54.336 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[87147]: cluster 2024-09-13T15:14:53.219462+0000 mon.a (mon.0) 2604 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2024-09-13T15:14:54.336 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[87147]: cluster 2024-09-13T15:14:53.674013+0000 mon.a (mon.0) 2605 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:14:54.336 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:54 smithi099 bash[87147]: cluster 2024-09-13T15:14:53.674013+0000 mon.a (mon.0) 2605 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:14:54.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:54.444 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:14:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:54.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:14:54.547 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:54.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:54 smithi177 bash[68594]: cluster 2024-09-13T15:14:53.146266+0000 mgr.y (mgr.26252) 1465 : cluster [DBG] pgmap v2050: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:54.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:54 smithi177 bash[68594]: cluster 2024-09-13T15:14:53.146266+0000 mgr.y (mgr.26252) 1465 : cluster [DBG] pgmap v2050: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:54.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:54 smithi177 bash[68594]: cluster 2024-09-13T15:14:53.219462+0000 mon.a (mon.0) 2604 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2024-09-13T15:14:54.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:54 smithi177 bash[68594]: cluster 2024-09-13T15:14:53.219462+0000 mon.a (mon.0) 2604 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2024-09-13T15:14:54.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:54 smithi177 bash[68594]: cluster 2024-09-13T15:14:53.674013+0000 mon.a (mon.0) 2605 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:14:54.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:54 smithi177 bash[68594]: cluster 2024-09-13T15:14:53.674013+0000 mon.a (mon.0) 2605 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:14:54.572 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-09-13T15:14:54.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:54.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:54.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:54.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:54.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:54.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:54.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:54.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:54.833 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:54.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:54.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:54.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:54.965 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:54.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:54.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:55.068 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:55.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:55.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:55.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:55.190 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:14:55.190 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:14:55.190 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:14:55.190 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:14:55.190 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:14:55.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:55.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:55.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:55.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:55.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:55.323 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:55 smithi099 bash[84576]: cluster 2024-09-13T15:14:54.242930+0000 mon.a (mon.0) 2606 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2024-09-13T15:14:55.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:55 smithi099 bash[84576]: cluster 2024-09-13T15:14:54.242930+0000 mon.a (mon.0) 2606 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2024-09-13T15:14:55.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:55 smithi099 bash[84576]: cluster 2024-09-13T15:14:55.146956+0000 mgr.y (mgr.26252) 1466 : cluster [DBG] pgmap v2053: 89 pgs: 3 creating+peering, 2 unknown, 84 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:55.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:55 smithi099 bash[84576]: cluster 2024-09-13T15:14:55.146956+0000 mgr.y (mgr.26252) 1466 : cluster [DBG] pgmap v2053: 89 pgs: 3 creating+peering, 2 unknown, 84 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:55.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:55 smithi099 bash[87147]: cluster 2024-09-13T15:14:54.242930+0000 mon.a (mon.0) 2606 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2024-09-13T15:14:55.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:55 smithi099 bash[87147]: cluster 2024-09-13T15:14:54.242930+0000 mon.a (mon.0) 2606 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2024-09-13T15:14:55.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:55 smithi099 bash[87147]: cluster 2024-09-13T15:14:55.146956+0000 mgr.y (mgr.26252) 1466 : cluster [DBG] pgmap v2053: 89 pgs: 3 creating+peering, 2 unknown, 84 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:55.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:55 smithi099 bash[87147]: cluster 2024-09-13T15:14:55.146956+0000 mgr.y (mgr.26252) 1466 : cluster [DBG] pgmap v2053: 89 pgs: 3 creating+peering, 2 unknown, 84 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:55.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:55.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:55.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:55.352 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:14:55.353 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:14:55.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:55 smithi177 bash[68594]: cluster 2024-09-13T15:14:54.242930+0000 mon.a (mon.0) 2606 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2024-09-13T15:14:55.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:55 smithi177 bash[68594]: cluster 2024-09-13T15:14:54.242930+0000 mon.a (mon.0) 2606 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2024-09-13T15:14:55.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:55 smithi177 bash[68594]: cluster 2024-09-13T15:14:55.146956+0000 mgr.y (mgr.26252) 1466 : cluster [DBG] pgmap v2053: 89 pgs: 3 creating+peering, 2 unknown, 84 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:55.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:55 smithi177 bash[68594]: cluster 2024-09-13T15:14:55.146956+0000 mgr.y (mgr.26252) 1466 : cluster [DBG] pgmap v2053: 89 pgs: 3 creating+peering, 2 unknown, 84 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:55.371 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:14:55.371 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:14:55.395 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:14:55.395 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:14:55.414 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:14:55.414 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:14:55.445 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:14:55.446 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:14:55.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:55.466 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:14:55.467 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:14:55.468 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:55.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:55.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:55.486 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:14:55.486 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:14:55.506 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:14:55.506 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:14:55.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:55.591 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:55.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:55.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:55.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:55.717 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:55.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:55.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:55.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:55.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:55.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:55.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:55.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:55.978 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:55.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:55.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:56.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:56.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:56.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:56.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:56.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:56.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:56.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:56.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:56.250 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:14:56.304 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:14:56.304 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:14:56.304 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-09-13T15:14:56.305 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:14:56.305 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:14:56.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-09-13T15:14:56.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:55.326132+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.99:0/2167654146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:55.326132+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.99:0/2167654146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:55.470987+0000 mon.c (mon.2) 114 : audit [INF] from='client.? 172.21.15.99:0/1722317924' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:55.470987+0000 mon.c (mon.2) 114 : audit [INF] from='client.? 172.21.15.99:0/1722317924' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:55.471344+0000 mon.a (mon.0) 2607 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:55.471344+0000 mon.a (mon.0) 2607 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:55.559916+0000 mon.a (mon.0) 2608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:55.559916+0000 mon.a (mon.0) 2608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:55.566308+0000 mon.a (mon.0) 2609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:55.566308+0000 mon.a (mon.0) 2609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:56.166268+0000 mon.a (mon.0) 2610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:14:56.342 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:56.166268+0000 mon.a (mon.0) 2610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:56.167105+0000 mon.a (mon.0) 2611 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:56.167105+0000 mon.a (mon.0) 2611 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:56.173846+0000 mon.a (mon.0) 2612 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[84576]: audit 2024-09-13T15:14:56.173846+0000 mon.a (mon.0) 2612 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:55.326132+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.99:0/2167654146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:55.326132+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.99:0/2167654146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:55.470987+0000 mon.c (mon.2) 114 : audit [INF] from='client.? 172.21.15.99:0/1722317924' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:55.470987+0000 mon.c (mon.2) 114 : audit [INF] from='client.? 172.21.15.99:0/1722317924' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:55.471344+0000 mon.a (mon.0) 2607 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:55.471344+0000 mon.a (mon.0) 2607 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.343 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:55.559916+0000 mon.a (mon.0) 2608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:55.559916+0000 mon.a (mon.0) 2608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:55.566308+0000 mon.a (mon.0) 2609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:55.566308+0000 mon.a (mon.0) 2609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:56.166268+0000 mon.a (mon.0) 2610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:14:56.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:56.166268+0000 mon.a (mon.0) 2610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:14:56.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:56.167105+0000 mon.a (mon.0) 2611 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:14:56.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:56.167105+0000 mon.a (mon.0) 2611 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:14:56.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:56.173846+0000 mon.a (mon.0) 2612 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.344 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:56 smithi099 bash[87147]: audit 2024-09-13T15:14:56.173846+0000 mon.a (mon.0) 2612 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.352 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:56.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:56.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:56.455 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:56.480 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:56.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:56.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:56.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:56.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:56.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:55.326132+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.99:0/2167654146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:55.326132+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.99:0/2167654146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:55.470987+0000 mon.c (mon.2) 114 : audit [INF] from='client.? 172.21.15.99:0/1722317924' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:55.470987+0000 mon.c (mon.2) 114 : audit [INF] from='client.? 172.21.15.99:0/1722317924' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:55.471344+0000 mon.a (mon.0) 2607 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:55.471344+0000 mon.a (mon.0) 2607 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:55.559916+0000 mon.a (mon.0) 2608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:55.559916+0000 mon.a (mon.0) 2608 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:55.566308+0000 mon.a (mon.0) 2609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:55.566308+0000 mon.a (mon.0) 2609 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:56.166268+0000 mon.a (mon.0) 2610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:14:56.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:56.166268+0000 mon.a (mon.0) 2610 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:14:56.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:56.167105+0000 mon.a (mon.0) 2611 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:14:56.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:56.167105+0000 mon.a (mon.0) 2611 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:14:56.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:56.173846+0000 mon.a (mon.0) 2612 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:56 smithi177 bash[68594]: audit 2024-09-13T15:14:56.173846+0000 mon.a (mon.0) 2612 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:14:56.710 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:56.731 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:56.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:56.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:56.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:56.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:56.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:56.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:56.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:56.992 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:56.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:57.095 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:57.106 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:14:57] "GET /metrics HTTP/1.1" 200 34358 "" "Prometheus/2.51.0" 2024-09-13T15:14:57.122 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:14:57.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:57.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:14:57.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:57.251 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:57.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:57.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[87147]: cluster 2024-09-13T15:14:56.245967+0000 mon.a (mon.0) 2613 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[87147]: cluster 2024-09-13T15:14:56.245967+0000 mon.a (mon.0) 2613 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[87147]: audit 2024-09-13T15:14:56.247589+0000 mon.a (mon.0) 2614 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:14:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[87147]: audit 2024-09-13T15:14:56.247589+0000 mon.a (mon.0) 2614 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:14:57.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[87147]: cluster 2024-09-13T15:14:56.251380+0000 mon.a (mon.0) 2615 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[87147]: cluster 2024-09-13T15:14:56.251380+0000 mon.a (mon.0) 2615 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[87147]: cluster 2024-09-13T15:14:57.147478+0000 mgr.y (mgr.26252) 1467 : cluster [DBG] pgmap v2055: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[87147]: cluster 2024-09-13T15:14:57.147478+0000 mgr.y (mgr.26252) 1467 : cluster [DBG] pgmap v2055: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[84576]: cluster 2024-09-13T15:14:56.245967+0000 mon.a (mon.0) 2613 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[84576]: cluster 2024-09-13T15:14:56.245967+0000 mon.a (mon.0) 2613 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[84576]: audit 2024-09-13T15:14:56.247589+0000 mon.a (mon.0) 2614 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[84576]: audit 2024-09-13T15:14:56.247589+0000 mon.a (mon.0) 2614 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[84576]: cluster 2024-09-13T15:14:56.251380+0000 mon.a (mon.0) 2615 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[84576]: cluster 2024-09-13T15:14:56.251380+0000 mon.a (mon.0) 2615 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[84576]: cluster 2024-09-13T15:14:57.147478+0000 mgr.y (mgr.26252) 1467 : cluster [DBG] pgmap v2055: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:14:57 smithi099 bash[84576]: cluster 2024-09-13T15:14:57.147478+0000 mgr.y (mgr.26252) 1467 : cluster [DBG] pgmap v2055: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:57.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:57.382 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:57.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:57.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:57.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:57.514 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:57.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:57 smithi177 bash[68594]: cluster 2024-09-13T15:14:56.245967+0000 mon.a (mon.0) 2613 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:57.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:57 smithi177 bash[68594]: cluster 2024-09-13T15:14:56.245967+0000 mon.a (mon.0) 2613 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:14:57.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:57 smithi177 bash[68594]: audit 2024-09-13T15:14:56.247589+0000 mon.a (mon.0) 2614 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:14:57.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:57 smithi177 bash[68594]: audit 2024-09-13T15:14:56.247589+0000 mon.a (mon.0) 2614 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:14:57.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:57 smithi177 bash[68594]: cluster 2024-09-13T15:14:56.251380+0000 mon.a (mon.0) 2615 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2024-09-13T15:14:57.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:57 smithi177 bash[68594]: cluster 2024-09-13T15:14:56.251380+0000 mon.a (mon.0) 2615 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2024-09-13T15:14:57.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:57 smithi177 bash[68594]: cluster 2024-09-13T15:14:57.147478+0000 mgr.y (mgr.26252) 1467 : cluster [DBG] pgmap v2055: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:57.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:14:57 smithi177 bash[68594]: cluster 2024-09-13T15:14:57.147478+0000 mgr.y (mgr.26252) 1467 : cluster [DBG] pgmap v2055: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:14:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:57.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:57.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:57.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:57.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:57.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:57.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:57.771 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:57.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:57.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:57.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:57.902 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:57.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:58.005 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:58.033 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:58.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:58.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:58.136 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:58.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:58.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:58.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:58.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:58.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:58.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:58.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:14:58.396 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:58.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:58.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:58.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:58.527 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:58.553 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:58.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:58.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:58.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:58.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:58.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:58.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:58.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T15:14:58.787 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:58.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:58.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:58.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:58.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:14:58.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:58.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:14:59.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:59.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:59.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:59.172 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:59.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:59.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:59.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:59.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:59.330 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:14:59.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:59.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:14:59.433 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:59.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:14:59.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:59.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:14:59.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:59.590 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:14:59.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:59.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:14:59.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:59.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:59.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:59.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:59.826 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:59.853 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:14:59.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:59.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:14:59.956 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:59.985 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:14:59.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:14:59.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:00.088 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:00.116 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:00.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:00.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:00.219 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:00.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:00.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:00.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:00.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:00 smithi177 bash[68594]: cluster 2024-09-13T15:14:59.148211+0000 mgr.y (mgr.26252) 1468 : cluster [DBG] pgmap v2056: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:00.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:00 smithi177 bash[68594]: cluster 2024-09-13T15:14:59.148211+0000 mgr.y (mgr.26252) 1468 : cluster [DBG] pgmap v2056: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:00.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:15:00 smithi177 bash[61439]: ts=2024-09-13T15:15:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:15:00.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:00.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:00.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:00.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:00 smithi099 bash[87147]: cluster 2024-09-13T15:14:59.148211+0000 mgr.y (mgr.26252) 1468 : cluster [DBG] pgmap v2056: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:00.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:00 smithi099 bash[87147]: cluster 2024-09-13T15:14:59.148211+0000 mgr.y (mgr.26252) 1468 : cluster [DBG] pgmap v2056: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:00.491 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:00 smithi099 bash[84576]: cluster 2024-09-13T15:14:59.148211+0000 mgr.y (mgr.26252) 1468 : cluster [DBG] pgmap v2056: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:00.491 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:00 smithi099 bash[84576]: cluster 2024-09-13T15:14:59.148211+0000 mgr.y (mgr.26252) 1468 : cluster [DBG] pgmap v2056: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:00.506 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:00.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:00.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:00.609 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:00.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:00.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:00.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:00.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:00.766 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:00.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:00.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:00.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:00.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:00.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:15:00.928 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:15:00.999 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:01.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:01.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:01.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:01.085 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:15:01.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:01.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:01.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:01.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:01.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:01.276 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:01.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:01.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:01.380 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:01.408 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:01.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:01.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:01.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:01.536 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:01.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:01.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:01.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:01.668 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:01.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:01.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:01.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:01.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:01.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:01.902 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:01.926 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:01.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:02.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:02.057 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:02.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:02.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:02.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:02.188 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:02.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:02.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:02.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:02.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:02 smithi099 bash[84576]: cluster 2024-09-13T15:15:01.148960+0000 mgr.y (mgr.26252) 1469 : cluster [DBG] pgmap v2057: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:02.303 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:02 smithi099 bash[84576]: cluster 2024-09-13T15:15:01.148960+0000 mgr.y (mgr.26252) 1469 : cluster [DBG] pgmap v2057: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:02.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:02 smithi099 bash[87147]: cluster 2024-09-13T15:15:01.148960+0000 mgr.y (mgr.26252) 1469 : cluster [DBG] pgmap v2057: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:02.304 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:02 smithi099 bash[87147]: cluster 2024-09-13T15:15:01.148960+0000 mgr.y (mgr.26252) 1469 : cluster [DBG] pgmap v2057: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:02.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:02.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:02.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:02.421 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:02.442 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:15:02.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:02.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:02.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:02.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:02.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:02 smithi177 bash[68594]: cluster 2024-09-13T15:15:01.148960+0000 mgr.y (mgr.26252) 1469 : cluster [DBG] pgmap v2057: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:02.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:02 smithi177 bash[68594]: cluster 2024-09-13T15:15:01.148960+0000 mgr.y (mgr.26252) 1469 : cluster [DBG] pgmap v2057: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:02.575 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:02.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:02.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:02.678 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:02.705 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:02.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:02.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:02.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:02.835 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:02.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:02.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:02.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:02.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:02.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:02.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:03.067 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:03.094 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:03.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:03.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:03.198 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:03.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:03.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:03.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:03.322 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:03.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:03.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:03.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:03.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:03.464 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:15:03.482 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:03.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:03.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:03.586 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:03.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:03.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:03.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:03.746 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:03.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:03.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:03.849 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:03.877 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:03.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:03.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:03.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:04.005 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:04.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:04.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:04.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:04.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:04.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:04.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:04.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:04.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:04.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:04.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[84576]: cluster 2024-09-13T15:15:03.149574+0000 mgr.y (mgr.26252) 1470 : cluster [DBG] pgmap v2058: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:04.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[84576]: cluster 2024-09-13T15:15:03.149574+0000 mgr.y (mgr.26252) 1470 : cluster [DBG] pgmap v2058: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:04.300 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[84576]: audit 2024-09-13T15:15:03.322127+0000 mon.a (mon.0) 2616 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:04.301 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[84576]: audit 2024-09-13T15:15:03.322127+0000 mon.a (mon.0) 2616 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:04.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:04.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:04.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:04.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:04.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:04 smithi177 bash[68594]: cluster 2024-09-13T15:15:03.149574+0000 mgr.y (mgr.26252) 1470 : cluster [DBG] pgmap v2058: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:04.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:04 smithi177 bash[68594]: cluster 2024-09-13T15:15:03.149574+0000 mgr.y (mgr.26252) 1470 : cluster [DBG] pgmap v2058: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:04.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:04 smithi177 bash[68594]: audit 2024-09-13T15:15:03.322127+0000 mon.a (mon.0) 2616 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:04.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:04 smithi177 bash[68594]: audit 2024-09-13T15:15:03.322127+0000 mon.a (mon.0) 2616 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:04.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:04 smithi177 bash[68594]: audit 2024-09-13T15:15:03.324217+0000 mon.a (mon.0) 2617 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:04.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:04 smithi177 bash[68594]: audit 2024-09-13T15:15:03.324217+0000 mon.a (mon.0) 2617 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:04.524 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:04.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:04.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:04.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[84576]: audit 2024-09-13T15:15:03.324217+0000 mon.a (mon.0) 2617 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:04.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[84576]: audit 2024-09-13T15:15:03.324217+0000 mon.a (mon.0) 2617 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:04.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[87147]: cluster 2024-09-13T15:15:03.149574+0000 mgr.y (mgr.26252) 1470 : cluster [DBG] pgmap v2058: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[87147]: cluster 2024-09-13T15:15:03.149574+0000 mgr.y (mgr.26252) 1470 : cluster [DBG] pgmap v2058: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[87147]: audit 2024-09-13T15:15:03.322127+0000 mon.a (mon.0) 2616 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[87147]: audit 2024-09-13T15:15:03.322127+0000 mon.a (mon.0) 2616 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[87147]: audit 2024-09-13T15:15:03.324217+0000 mon.a (mon.0) 2617 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:04 smithi099 bash[87147]: audit 2024-09-13T15:15:03.324217+0000 mon.a (mon.0) 2617 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:04.627 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:04.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:04.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:04.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:04.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:04.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:04.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:04.886 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:04.915 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:04.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:04.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:05.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:05.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:05.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:05.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:05.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:05.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:05.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:05.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:05.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:05.304 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:05.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:05.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:05.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:05.434 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:05.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:05.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:05.510 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:15:05.510 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:15:05.537 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:05.561 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:05.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:05.664 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:05.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:05 smithi177 bash[68594]: audit 2024-09-13T15:15:04.624750+0000 mon.a (mon.0) 2618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:05.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:05 smithi177 bash[68594]: audit 2024-09-13T15:15:04.624750+0000 mon.a (mon.0) 2618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:05.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:05 smithi177 bash[68594]: cluster 2024-09-13T15:15:05.150952+0000 mgr.y (mgr.26252) 1471 : cluster [DBG] pgmap v2059: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:05.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:05 smithi177 bash[68594]: cluster 2024-09-13T15:15:05.150952+0000 mgr.y (mgr.26252) 1471 : cluster [DBG] pgmap v2059: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:05.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:05.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:05.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:05.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:05.805 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:15:05.805 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T15:15:05.805 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:15:05.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:05.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:05.911 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:05.924 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:05 smithi099 bash[84576]: audit 2024-09-13T15:15:04.624750+0000 mon.a (mon.0) 2618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:05.924 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:05 smithi099 bash[84576]: audit 2024-09-13T15:15:04.624750+0000 mon.a (mon.0) 2618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:05.924 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:05 smithi099 bash[84576]: cluster 2024-09-13T15:15:05.150952+0000 mgr.y (mgr.26252) 1471 : cluster [DBG] pgmap v2059: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:05.924 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:05 smithi099 bash[84576]: cluster 2024-09-13T15:15:05.150952+0000 mgr.y (mgr.26252) 1471 : cluster [DBG] pgmap v2059: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:05.924 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:05 smithi099 bash[87147]: audit 2024-09-13T15:15:04.624750+0000 mon.a (mon.0) 2618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:05.925 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:05 smithi099 bash[87147]: audit 2024-09-13T15:15:04.624750+0000 mon.a (mon.0) 2618 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:05.925 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:05 smithi099 bash[87147]: cluster 2024-09-13T15:15:05.150952+0000 mgr.y (mgr.26252) 1471 : cluster [DBG] pgmap v2059: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:05.925 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:05 smithi099 bash[87147]: cluster 2024-09-13T15:15:05.150952+0000 mgr.y (mgr.26252) 1471 : cluster [DBG] pgmap v2059: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:05.938 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:05.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:05.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:06.041 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:06.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:06.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:06.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:06.174 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:06.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:06.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:06.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:06.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:06.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:06.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:06.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:06.432 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:06.460 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:06.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:06.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:06.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:06.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:06.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:06.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:06.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:06.720 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:06.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:06.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:06.823 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:06.850 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:06.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:06.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:06.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:06.978 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:06.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:07.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:07.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:07.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:07.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:07.212 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:07.240 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:07.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:07.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:07.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:15:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:15:07] "GET /metrics HTTP/1.1" 200 36712 "" "Prometheus/2.51.0" 2024-09-13T15:15:07.345 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:07.372 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:07.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:07.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:07.475 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:07.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:07.506 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:07.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:07.607 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:07.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:07.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:07.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:07.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:07.766 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:07.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:07.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:07.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:07.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:07.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:07.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:07.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:08.027 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:08.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:08.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:08.130 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:08.158 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:08.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:08.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:08.290 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:08.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:08.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:08.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:08 smithi177 bash[68594]: cluster 2024-09-13T15:15:07.151878+0000 mgr.y (mgr.26252) 1472 : cluster [DBG] pgmap v2060: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:08.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:08 smithi177 bash[68594]: cluster 2024-09-13T15:15:07.151878+0000 mgr.y (mgr.26252) 1472 : cluster [DBG] pgmap v2060: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:08.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:08.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:08.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:08.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:08.486 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:08 smithi099 bash[87147]: cluster 2024-09-13T15:15:07.151878+0000 mgr.y (mgr.26252) 1472 : cluster [DBG] pgmap v2060: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:08.487 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:08 smithi099 bash[87147]: cluster 2024-09-13T15:15:07.151878+0000 mgr.y (mgr.26252) 1472 : cluster [DBG] pgmap v2060: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:08.487 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:08 smithi099 bash[84576]: cluster 2024-09-13T15:15:07.151878+0000 mgr.y (mgr.26252) 1472 : cluster [DBG] pgmap v2060: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:08.487 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:08 smithi099 bash[84576]: cluster 2024-09-13T15:15:07.151878+0000 mgr.y (mgr.26252) 1472 : cluster [DBG] pgmap v2060: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:08.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:08.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:08.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:08.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:08.653 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:08.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:08.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:08.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:08.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:08.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:08.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:08.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:08.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:08.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:08.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:08.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:09.045 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:09.073 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:09.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:09.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:09.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:09.204 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:09.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:09.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:09.334 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:09.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:09.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:09.436 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:09.462 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:09.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:09.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:09.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:09.592 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:09.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:09.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:09.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:09.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:09.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:09.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:09.825 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:09.853 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:09.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:09.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:09.957 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:09.983 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:09.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:10.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:10.114 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:10.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:10.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:10.217 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:10.228 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:10 smithi099 bash[84576]: cluster 2024-09-13T15:15:09.152707+0000 mgr.y (mgr.26252) 1473 : cluster [DBG] pgmap v2061: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:10.228 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:10 smithi099 bash[84576]: cluster 2024-09-13T15:15:09.152707+0000 mgr.y (mgr.26252) 1473 : cluster [DBG] pgmap v2061: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:10.228 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:10 smithi099 bash[87147]: cluster 2024-09-13T15:15:09.152707+0000 mgr.y (mgr.26252) 1473 : cluster [DBG] pgmap v2061: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:10.229 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:10 smithi099 bash[87147]: cluster 2024-09-13T15:15:09.152707+0000 mgr.y (mgr.26252) 1473 : cluster [DBG] pgmap v2061: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:10.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:10.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:10.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:10.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:10.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:10 smithi177 bash[68594]: cluster 2024-09-13T15:15:09.152707+0000 mgr.y (mgr.26252) 1473 : cluster [DBG] pgmap v2061: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:10 smithi177 bash[68594]: cluster 2024-09-13T15:15:09.152707+0000 mgr.y (mgr.26252) 1473 : cluster [DBG] pgmap v2061: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:10.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:15:10 smithi177 bash[61439]: ts=2024-09-13T15:15:10.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:15:10.369 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:10.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:10.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:10.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:10.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:10.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:10.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:10.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:10.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:10.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:10.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:10.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:10.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:10.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:10.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:10.889 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:10.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:10.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:10.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:11.022 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:11.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:11.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:11.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:11.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:11.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:11.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:11.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:11.284 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:11.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:11.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:11.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:11.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:11.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:11.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:11.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:11.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:11.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:11.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:11.675 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:11.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:11.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:11.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:11.804 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:11.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:11.907 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:11.934 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:11.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:11.952 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:15:11.952 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:15:11.952 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-13T15:15:11.952 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:15:11.952 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:15:11.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-13T15:15:12.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:12.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:12.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:12.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:12.166 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:12.194 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:12.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:12.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:12.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:12.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:12 smithi177 bash[68594]: cluster 2024-09-13T15:15:11.153600+0000 mgr.y (mgr.26252) 1474 : cluster [DBG] pgmap v2062: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:12.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:12 smithi177 bash[68594]: cluster 2024-09-13T15:15:11.153600+0000 mgr.y (mgr.26252) 1474 : cluster [DBG] pgmap v2062: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:12.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:12.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:12.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:12.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:12.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:12.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:12.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:12.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:12.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:12.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:12.587 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:12 smithi099 bash[84576]: cluster 2024-09-13T15:15:11.153600+0000 mgr.y (mgr.26252) 1474 : cluster [DBG] pgmap v2062: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:12.587 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:12 smithi099 bash[84576]: cluster 2024-09-13T15:15:11.153600+0000 mgr.y (mgr.26252) 1474 : cluster [DBG] pgmap v2062: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:12.587 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:12 smithi099 bash[87147]: cluster 2024-09-13T15:15:11.153600+0000 mgr.y (mgr.26252) 1474 : cluster [DBG] pgmap v2062: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:12.587 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:12 smithi099 bash[87147]: cluster 2024-09-13T15:15:11.153600+0000 mgr.y (mgr.26252) 1474 : cluster [DBG] pgmap v2062: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:12.688 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:12.717 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:12.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:12.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:12.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:12.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:12.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:12.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:12.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:12.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:12.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:12.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:13.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:13.107 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:13.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:13.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:13.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:13.236 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:13.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:13.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:13.363 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:13.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:13.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:13.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:13.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:13.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:13.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:13.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:13.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:13.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:13.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:13.725 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:13.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:13.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:13.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:13.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:13.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:13.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:13.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:13.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:14.007 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:14.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:14.111 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:14.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:14.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:14.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:14.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:14.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:14 smithi099 bash[84576]: cluster 2024-09-13T15:15:13.154342+0000 mgr.y (mgr.26252) 1475 : cluster [DBG] pgmap v2063: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:14.248 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:14 smithi099 bash[84576]: cluster 2024-09-13T15:15:13.154342+0000 mgr.y (mgr.26252) 1475 : cluster [DBG] pgmap v2063: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:14.249 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:14 smithi099 bash[87147]: cluster 2024-09-13T15:15:13.154342+0000 mgr.y (mgr.26252) 1475 : cluster [DBG] pgmap v2063: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:14.249 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:14 smithi099 bash[87147]: cluster 2024-09-13T15:15:13.154342+0000 mgr.y (mgr.26252) 1475 : cluster [DBG] pgmap v2063: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:14.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:14.265 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:14.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:14.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:14.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:14.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:14.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:14.523 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:14.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:14.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:14.626 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:14.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:14.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:14.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:14.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:14.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:14.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:14.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:14.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:14 smithi177 bash[68594]: cluster 2024-09-13T15:15:13.154342+0000 mgr.y (mgr.26252) 1475 : cluster [DBG] pgmap v2063: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:14.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:14 smithi177 bash[68594]: cluster 2024-09-13T15:15:13.154342+0000 mgr.y (mgr.26252) 1475 : cluster [DBG] pgmap v2063: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:14.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:14.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:14.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:14.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:15.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:15.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:15.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:15.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:15.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:15.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:15.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:15.230 INFO:teuthology.orchestra.run.smithi099.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-09-13T15:15:15.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:15.289 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:15.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:15.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:15 smithi177 bash[68594]: audit 2024-09-13T15:15:15.022098+0000 mon.a (mon.0) 2619 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:15:15.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:15 smithi177 bash[68594]: audit 2024-09-13T15:15:15.022098+0000 mon.a (mon.0) 2619 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:15:15.392 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:15.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:15.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:15.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:15.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:15.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:15.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:15.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:15.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:15 smithi099 bash[84576]: audit 2024-09-13T15:15:15.022098+0000 mon.a (mon.0) 2619 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:15:15.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:15 smithi099 bash[84576]: audit 2024-09-13T15:15:15.022098+0000 mon.a (mon.0) 2619 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:15:15.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:15 smithi099 bash[87147]: audit 2024-09-13T15:15:15.022098+0000 mon.a (mon.0) 2619 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:15:15.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:15 smithi099 bash[87147]: audit 2024-09-13T15:15:15.022098+0000 mon.a (mon.0) 2619 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:15:15.653 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:15.680 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:15.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:15.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:15.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:15.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:15.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:15.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:15.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:15.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:15.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:16.043 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:16.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:16.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:16.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:16.201 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:16.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:16.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:16.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:16.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[87147]: cluster 2024-09-13T15:15:15.155408+0000 mgr.y (mgr.26252) 1476 : cluster [DBG] pgmap v2064: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:16.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[87147]: cluster 2024-09-13T15:15:15.155408+0000 mgr.y (mgr.26252) 1476 : cluster [DBG] pgmap v2064: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:16.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[87147]: audit 2024-09-13T15:15:15.230422+0000 mon.a (mon.0) 2620 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:15:16.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[87147]: audit 2024-09-13T15:15:15.230422+0000 mon.a (mon.0) 2620 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:15:16.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[87147]: cluster 2024-09-13T15:15:15.237531+0000 mon.a (mon.0) 2621 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2024-09-13T15:15:16.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[87147]: cluster 2024-09-13T15:15:15.237531+0000 mon.a (mon.0) 2621 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2024-09-13T15:15:16.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[84576]: cluster 2024-09-13T15:15:15.155408+0000 mgr.y (mgr.26252) 1476 : cluster [DBG] pgmap v2064: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:16.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[84576]: cluster 2024-09-13T15:15:15.155408+0000 mgr.y (mgr.26252) 1476 : cluster [DBG] pgmap v2064: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:16.317 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[84576]: audit 2024-09-13T15:15:15.230422+0000 mon.a (mon.0) 2620 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:15:16.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[84576]: audit 2024-09-13T15:15:15.230422+0000 mon.a (mon.0) 2620 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:15:16.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[84576]: cluster 2024-09-13T15:15:15.237531+0000 mon.a (mon.0) 2621 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2024-09-13T15:15:16.318 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:16 smithi099 bash[84576]: cluster 2024-09-13T15:15:15.237531+0000 mon.a (mon.0) 2621 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2024-09-13T15:15:16.333 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:16.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:16.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:16.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:16.448 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:16 smithi177 bash[68594]: cluster 2024-09-13T15:15:15.155408+0000 mgr.y (mgr.26252) 1476 : cluster [DBG] pgmap v2064: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:16.448 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:16 smithi177 bash[68594]: cluster 2024-09-13T15:15:15.155408+0000 mgr.y (mgr.26252) 1476 : cluster [DBG] pgmap v2064: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:16.448 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:16 smithi177 bash[68594]: audit 2024-09-13T15:15:15.230422+0000 mon.a (mon.0) 2620 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:15:16.448 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:16 smithi177 bash[68594]: audit 2024-09-13T15:15:15.230422+0000 mon.a (mon.0) 2620 : audit [INF] from='client.? 172.21.15.99:0/193484527' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:15:16.448 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:16 smithi177 bash[68594]: cluster 2024-09-13T15:15:15.237531+0000 mon.a (mon.0) 2621 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2024-09-13T15:15:16.448 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:16 smithi177 bash[68594]: cluster 2024-09-13T15:15:15.237531+0000 mon.a (mon.0) 2621 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2024-09-13T15:15:16.465 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:16.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:16.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:16.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:16.596 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:16.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:16.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:16.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:16.717 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:15:16.730 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:16.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:16.834 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:16.860 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:16.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:16.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:16.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:16.990 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:16.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:16.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:17.093 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:17.121 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:17.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:17.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:17.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:17.251 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:17.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:17.353 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:17.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:17 smithi177 bash[68594]: cluster 2024-09-13T15:15:17.156322+0000 mgr.y (mgr.26252) 1477 : cluster [DBG] pgmap v2066: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:17.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:17 smithi177 bash[68594]: cluster 2024-09-13T15:15:17.156322+0000 mgr.y (mgr.26252) 1477 : cluster [DBG] pgmap v2066: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:17.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:17 smithi177 bash[68594]: cluster 2024-09-13T15:15:17.232306+0000 mon.a (mon.0) 2622 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:15:17.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:17 smithi177 bash[68594]: cluster 2024-09-13T15:15:17.232306+0000 mon.a (mon.0) 2622 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:15:17.382 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:17.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:17.433 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:15:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:15:17] "GET /metrics HTTP/1.1" 200 36712 "" "Prometheus/2.51.0" 2024-09-13T15:15:17.433 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:17 smithi099 bash[84576]: cluster 2024-09-13T15:15:17.156322+0000 mgr.y (mgr.26252) 1477 : cluster [DBG] pgmap v2066: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:17.434 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:17 smithi099 bash[84576]: cluster 2024-09-13T15:15:17.156322+0000 mgr.y (mgr.26252) 1477 : cluster [DBG] pgmap v2066: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:17.485 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:17.496 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:17 smithi099 bash[84576]: cluster 2024-09-13T15:15:17.232306+0000 mon.a (mon.0) 2622 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:15:17.496 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:17 smithi099 bash[84576]: cluster 2024-09-13T15:15:17.232306+0000 mon.a (mon.0) 2622 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:15:17.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:17 smithi099 bash[87147]: cluster 2024-09-13T15:15:17.156322+0000 mgr.y (mgr.26252) 1477 : cluster [DBG] pgmap v2066: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:17.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:17 smithi099 bash[87147]: cluster 2024-09-13T15:15:17.156322+0000 mgr.y (mgr.26252) 1477 : cluster [DBG] pgmap v2066: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:17.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:17 smithi099 bash[87147]: cluster 2024-09-13T15:15:17.232306+0000 mon.a (mon.0) 2622 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:15:17.497 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:17 smithi099 bash[87147]: cluster 2024-09-13T15:15:17.232306+0000 mon.a (mon.0) 2622 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:15:17.512 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:17.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:17.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:17.615 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:17.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:17.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:17.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:17.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:17.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:17.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:17.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:17.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:17.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:18.003 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:18.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:18.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:18.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:18.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:18.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:18.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:18.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:18.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:18.295 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:18.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:18.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:18.398 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:18.426 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:18.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:18.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:18.529 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:18.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:18.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:18.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:18.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:18.677 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:18.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:18.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:18.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:18.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:18.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:18.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:18.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:18.939 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:18.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:18.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:19.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:19.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:19.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:19.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:19.172 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:19.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:19.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:19.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:19.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:19.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:19.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:19.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:19.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:19.458 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:19.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:19.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:19.561 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[84576]: audit 2024-09-13T15:15:18.322875+0000 mon.a (mon.0) 2623 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[84576]: audit 2024-09-13T15:15:18.322875+0000 mon.a (mon.0) 2623 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[84576]: audit 2024-09-13T15:15:18.324903+0000 mon.a (mon.0) 2624 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[84576]: audit 2024-09-13T15:15:18.324903+0000 mon.a (mon.0) 2624 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[84576]: cluster 2024-09-13T15:15:19.157173+0000 mgr.y (mgr.26252) 1478 : cluster [DBG] pgmap v2067: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[84576]: cluster 2024-09-13T15:15:19.157173+0000 mgr.y (mgr.26252) 1478 : cluster [DBG] pgmap v2067: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[87147]: audit 2024-09-13T15:15:18.322875+0000 mon.a (mon.0) 2623 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[87147]: audit 2024-09-13T15:15:18.322875+0000 mon.a (mon.0) 2623 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[87147]: audit 2024-09-13T15:15:18.324903+0000 mon.a (mon.0) 2624 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[87147]: audit 2024-09-13T15:15:18.324903+0000 mon.a (mon.0) 2624 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[87147]: cluster 2024-09-13T15:15:19.157173+0000 mgr.y (mgr.26252) 1478 : cluster [DBG] pgmap v2067: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:19 smithi099 bash[87147]: cluster 2024-09-13T15:15:19.157173+0000 mgr.y (mgr.26252) 1478 : cluster [DBG] pgmap v2067: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:19.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:19.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:19.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:19 smithi177 bash[68594]: audit 2024-09-13T15:15:18.322875+0000 mon.a (mon.0) 2623 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:19 smithi177 bash[68594]: audit 2024-09-13T15:15:18.322875+0000 mon.a (mon.0) 2623 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:19 smithi177 bash[68594]: audit 2024-09-13T15:15:18.324903+0000 mon.a (mon.0) 2624 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:19 smithi177 bash[68594]: audit 2024-09-13T15:15:18.324903+0000 mon.a (mon.0) 2624 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:19 smithi177 bash[68594]: cluster 2024-09-13T15:15:19.157173+0000 mgr.y (mgr.26252) 1478 : cluster [DBG] pgmap v2067: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:19 smithi177 bash[68594]: cluster 2024-09-13T15:15:19.157173+0000 mgr.y (mgr.26252) 1478 : cluster [DBG] pgmap v2067: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-13T15:15:19.693 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:19.718 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:19.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:19.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:19.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:19.848 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:19.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:19.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:19.950 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:19.977 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:19.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:20.012 DEBUG:teuthology.orchestra.run.smithi099:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-09-13T15:15:20.022 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T15:15:20.023 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T15:15:20.063 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:hints = 1 2024-09-13T15:15:20.063 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T15:15:20.063 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object prefix: benchmark_data_smithi099_233884 2024-09-13T15:15:20.064 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:15:20.064 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 0 0 0 0 0 0 - 0 2024-09-13T15:15:20.080 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:20.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:20.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:20.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:20.219 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:20.251 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:20.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:20.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:20.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:20.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:20.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:20.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:20.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:20.500 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:15:20 smithi177 bash[61439]: ts=2024-09-13T15:15:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:15:20.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:20.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:20.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:20.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:20.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:20.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:20.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:20.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:20.787 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:20.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:20.840 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:15:20.840 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:15:20.891 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:20.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:20.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:20.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:21.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:21.041 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:21.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:21.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:21.064 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 1 16 2936 2920 182.49 182.5 0.00297407 0.00545721 2024-09-13T15:15:21.145 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:21.172 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:21.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:21.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:21.275 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:21.307 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:21.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:21.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:21.411 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:21.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:21.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:21.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:21.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:21.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:21.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:21.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:21.662 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:21.692 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:21.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:21.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:21.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:21.822 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:21.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:21.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:21.926 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:21.927 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:15:21.927 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "" 2024-09-13T15:15:21.927 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:15:21.954 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:21.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:21.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:22.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:22.064 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 2 16 6062 6046 188.923 195.375 0.00204274 0.00528029 2024-09-13T15:15:22.085 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:22.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:22.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:22.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:22.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:22.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:22.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:22.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:22.342 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:22.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:22.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:22.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:22 smithi177 bash[68594]: cluster 2024-09-13T15:15:21.158266+0000 mgr.y (mgr.26252) 1479 : cluster [DBG] pgmap v2068: 89 pgs: 89 active+clean; 5.3 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 486 KiB/s wr, 7 op/s 2024-09-13T15:15:22.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:22 smithi177 bash[68594]: cluster 2024-09-13T15:15:21.158266+0000 mgr.y (mgr.26252) 1479 : cluster [DBG] pgmap v2068: 89 pgs: 89 active+clean; 5.3 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 486 KiB/s wr, 7 op/s 2024-09-13T15:15:22.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:22.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:22.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:22.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:22.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:22 smithi099 bash[84576]: cluster 2024-09-13T15:15:21.158266+0000 mgr.y (mgr.26252) 1479 : cluster [DBG] pgmap v2068: 89 pgs: 89 active+clean; 5.3 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 486 KiB/s wr, 7 op/s 2024-09-13T15:15:22.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:22 smithi099 bash[84576]: cluster 2024-09-13T15:15:21.158266+0000 mgr.y (mgr.26252) 1479 : cluster [DBG] pgmap v2068: 89 pgs: 89 active+clean; 5.3 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 486 KiB/s wr, 7 op/s 2024-09-13T15:15:22.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:22 smithi099 bash[87147]: cluster 2024-09-13T15:15:21.158266+0000 mgr.y (mgr.26252) 1479 : cluster [DBG] pgmap v2068: 89 pgs: 89 active+clean; 5.3 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 486 KiB/s wr, 7 op/s 2024-09-13T15:15:22.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:22 smithi099 bash[87147]: cluster 2024-09-13T15:15:21.158266+0000 mgr.y (mgr.26252) 1479 : cluster [DBG] pgmap v2068: 89 pgs: 89 active+clean; 5.3 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 486 KiB/s wr, 7 op/s 2024-09-13T15:15:22.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:22.601 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:22.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:22.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:22.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:22.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:22.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:22.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:22.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:22.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:22.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:22.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:22.967 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:22.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:22.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:23.064 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 3 15 8505 8490 176.861 152.75 0.0117796 0.00564406 2024-09-13T15:15:23.092 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:23.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:23.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:23.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:23.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:23.238 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:23.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:23.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:23.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:23.367 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:23.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:23.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:23.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:23.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:23.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:23.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:23.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:23.634 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:23.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:23.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:23.737 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:23.763 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:23.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:23.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:23.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:23.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:23.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:23.996 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:24.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:24.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:24.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:24.064 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 4 16 9804 9788 152.925 81.125 0.0035515 0.00653083 2024-09-13T15:15:24.130 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:24.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:24.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:24.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:24.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:24.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:24.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:24.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:24.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:24.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:24 smithi099 bash[84576]: cluster 2024-09-13T15:15:23.158688+0000 mgr.y (mgr.26252) 1480 : cluster [DBG] pgmap v2069: 89 pgs: 89 active+clean; 101 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s wr, 159 op/s 2024-09-13T15:15:24.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:24 smithi099 bash[84576]: cluster 2024-09-13T15:15:23.158688+0000 mgr.y (mgr.26252) 1480 : cluster [DBG] pgmap v2069: 89 pgs: 89 active+clean; 101 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s wr, 159 op/s 2024-09-13T15:15:24.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:24 smithi099 bash[87147]: cluster 2024-09-13T15:15:23.158688+0000 mgr.y (mgr.26252) 1480 : cluster [DBG] pgmap v2069: 89 pgs: 89 active+clean; 101 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s wr, 159 op/s 2024-09-13T15:15:24.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:24 smithi099 bash[87147]: cluster 2024-09-13T15:15:23.158688+0000 mgr.y (mgr.26252) 1480 : cluster [DBG] pgmap v2069: 89 pgs: 89 active+clean; 101 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s wr, 159 op/s 2024-09-13T15:15:24.419 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:24.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:24.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:24.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:24.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:24 smithi177 bash[68594]: cluster 2024-09-13T15:15:23.158688+0000 mgr.y (mgr.26252) 1480 : cluster [DBG] pgmap v2069: 89 pgs: 89 active+clean; 101 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s wr, 159 op/s 2024-09-13T15:15:24.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:24 smithi177 bash[68594]: cluster 2024-09-13T15:15:23.158688+0000 mgr.y (mgr.26252) 1480 : cluster [DBG] pgmap v2069: 89 pgs: 89 active+clean; 101 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s wr, 159 op/s 2024-09-13T15:15:24.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:24.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:24.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:24.647 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:24.672 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:24.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:24.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:24.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:24.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:24.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:24.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:24.906 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:24.933 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:24.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:24.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:25.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:25.064 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 5 16 12384 12368 154.585 161.25 0.017381 0.00646128 2024-09-13T15:15:25.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:25.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:25.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:25.194 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:25.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:25.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:25.297 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:25.323 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:25.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:25.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:25.426 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:25.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:25.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:25.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:25.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:25.580 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:25.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:25.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:25.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:25.710 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:25.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:25.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:25.813 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:25.838 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:25.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:25.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:25.942 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:25.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:25.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:25.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:26.064 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 6 16 14921 14905 155.246 158.562 0.00601425 0.00643289 2024-09-13T15:15:26.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:26.109 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:26.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:26.212 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:26.241 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:26.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:26.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:26.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:26.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T15:15:26.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:26 smithi177 bash[68594]: cluster 2024-09-13T15:15:25.159366+0000 mgr.y (mgr.26252) 1481 : cluster [DBG] pgmap v2070: 89 pgs: 89 active+clean; 552 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-09-13T15:15:26.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:26 smithi177 bash[68594]: cluster 2024-09-13T15:15:25.159366+0000 mgr.y (mgr.26252) 1481 : cluster [DBG] pgmap v2070: 89 pgs: 89 active+clean; 552 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-09-13T15:15:26.369 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:26.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:26.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:26.485 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:26 smithi099 bash[84576]: cluster 2024-09-13T15:15:25.159366+0000 mgr.y (mgr.26252) 1481 : cluster [DBG] pgmap v2070: 89 pgs: 89 active+clean; 552 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-09-13T15:15:26.486 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:26 smithi099 bash[84576]: cluster 2024-09-13T15:15:25.159366+0000 mgr.y (mgr.26252) 1481 : cluster [DBG] pgmap v2070: 89 pgs: 89 active+clean; 552 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-09-13T15:15:26.486 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:26 smithi099 bash[87147]: cluster 2024-09-13T15:15:25.159366+0000 mgr.y (mgr.26252) 1481 : cluster [DBG] pgmap v2070: 89 pgs: 89 active+clean; 552 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-09-13T15:15:26.486 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:26 smithi099 bash[87147]: cluster 2024-09-13T15:15:25.159366+0000 mgr.y (mgr.26252) 1481 : cluster [DBG] pgmap v2070: 89 pgs: 89 active+clean; 552 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2024-09-13T15:15:26.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:26.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:26.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:26.602 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:26.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:26.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:26.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:26.734 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:26.762 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:26.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:26.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:26.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:26.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:26.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:26.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:27.017 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:27.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:27.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:27.046 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:15:27.046 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:15:27.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 7 16 17172 17156 153.164 140.688 0.00756606 0.00652057 2024-09-13T15:15:27.090 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:15:27.090 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:15:27.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:27.124 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:15:27.124 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:15:27.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:27.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:27.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:27.165 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:15:27.165 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:15:27.204 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:15:27.204 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:15:27.235 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:15:27.235 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:15:27.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:27.263 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:15:27.263 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:15:27.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:27.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:27.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:27.298 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:15:27.298 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:15:27.319 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:27 smithi099 bash[84576]: audit 2024-09-13T15:15:26.999256+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.99:0/2584886205' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:27.320 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:27 smithi099 bash[84576]: audit 2024-09-13T15:15:26.999256+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.99:0/2584886205' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:27.320 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:15:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:15:27] "GET /metrics HTTP/1.1" 200 36709 "" "Prometheus/2.51.0" 2024-09-13T15:15:27.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:27 smithi099 bash[87147]: audit 2024-09-13T15:15:26.999256+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.99:0/2584886205' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:27.320 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:27 smithi099 bash[87147]: audit 2024-09-13T15:15:26.999256+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.99:0/2584886205' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:27.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:27.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:27.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:27.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:27.515 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:27.525 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:27 smithi177 bash[68594]: audit 2024-09-13T15:15:26.999256+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.99:0/2584886205' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:27.525 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:27 smithi177 bash[68594]: audit 2024-09-13T15:15:26.999256+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.99:0/2584886205' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:27.540 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:27.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:27.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:27.643 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:27.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:27.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:27.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:27.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:27.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:27.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:27.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:27.906 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:27.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:27.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:28.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:28.062 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:28.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:28.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:28.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 8 16 19344 19328 150.986 135.75 0.00503436 0.00661573 2024-09-13T15:15:28.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:28.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:28.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:28.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:28.256 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:15:28.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:28.312 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[84576]: cluster 2024-09-13T15:15:27.160496+0000 mgr.y (mgr.26252) 1482 : cluster [DBG] pgmap v2071: 89 pgs: 89 active+clean; 651 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T15:15:28.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[84576]: cluster 2024-09-13T15:15:27.160496+0000 mgr.y (mgr.26252) 1482 : cluster [DBG] pgmap v2071: 89 pgs: 89 active+clean; 651 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T15:15:28.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[84576]: audit 2024-09-13T15:15:27.383030+0000 mon.c (mon.2) 116 : audit [INF] from='client.? 172.21.15.99:0/2631420988' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[84576]: audit 2024-09-13T15:15:27.383030+0000 mon.c (mon.2) 116 : audit [INF] from='client.? 172.21.15.99:0/2631420988' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[84576]: audit 2024-09-13T15:15:27.383663+0000 mon.a (mon.0) 2625 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.313 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[84576]: audit 2024-09-13T15:15:27.383663+0000 mon.a (mon.0) 2625 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.313 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[87147]: cluster 2024-09-13T15:15:27.160496+0000 mgr.y (mgr.26252) 1482 : cluster [DBG] pgmap v2071: 89 pgs: 89 active+clean; 651 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T15:15:28.313 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[87147]: cluster 2024-09-13T15:15:27.160496+0000 mgr.y (mgr.26252) 1482 : cluster [DBG] pgmap v2071: 89 pgs: 89 active+clean; 651 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T15:15:28.313 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[87147]: audit 2024-09-13T15:15:27.383030+0000 mon.c (mon.2) 116 : audit [INF] from='client.? 172.21.15.99:0/2631420988' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.313 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[87147]: audit 2024-09-13T15:15:27.383030+0000 mon.c (mon.2) 116 : audit [INF] from='client.? 172.21.15.99:0/2631420988' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.313 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[87147]: audit 2024-09-13T15:15:27.383663+0000 mon.a (mon.0) 2625 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.314 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:28 smithi099 bash[87147]: audit 2024-09-13T15:15:27.383663+0000 mon.a (mon.0) 2625 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:28.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:28.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:28.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:28.456 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:28.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:28.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:28.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:28.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:28 smithi177 bash[68594]: cluster 2024-09-13T15:15:27.160496+0000 mgr.y (mgr.26252) 1482 : cluster [DBG] pgmap v2071: 89 pgs: 89 active+clean; 651 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T15:15:28.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:28 smithi177 bash[68594]: cluster 2024-09-13T15:15:27.160496+0000 mgr.y (mgr.26252) 1482 : cluster [DBG] pgmap v2071: 89 pgs: 89 active+clean; 651 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2024-09-13T15:15:28.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:28 smithi177 bash[68594]: audit 2024-09-13T15:15:27.383030+0000 mon.c (mon.2) 116 : audit [INF] from='client.? 172.21.15.99:0/2631420988' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:28 smithi177 bash[68594]: audit 2024-09-13T15:15:27.383030+0000 mon.c (mon.2) 116 : audit [INF] from='client.? 172.21.15.99:0/2631420988' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:28 smithi177 bash[68594]: audit 2024-09-13T15:15:27.383663+0000 mon.a (mon.0) 2625 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:28 smithi177 bash[68594]: audit 2024-09-13T15:15:27.383663+0000 mon.a (mon.0) 2625 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:15:28.587 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:28.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:28.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:28.690 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:28.715 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:28.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:28.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:28.846 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:28.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:28.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:28.950 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:28.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:28.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:28.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:29.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 9 16 21613 21597 149.966 141.812 0.00436189 0.00666019 2024-09-13T15:15:29.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:29.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:29.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:29.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:29.209 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:15:29.209 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:15:29.209 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-13T15:15:29.210 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:15:29.210 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:15:29.210 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-13T15:15:29.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:29.233 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:29.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:29.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:29.339 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:29.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[84576]: cluster 2024-09-13T15:15:28.235942+0000 mon.a (mon.0) 2626 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:15:29.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[84576]: cluster 2024-09-13T15:15:28.235942+0000 mon.a (mon.0) 2626 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:15:29.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[84576]: cluster 2024-09-13T15:15:28.235986+0000 mon.a (mon.0) 2627 : cluster [INF] Cluster is now healthy 2024-09-13T15:15:29.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[84576]: cluster 2024-09-13T15:15:28.235986+0000 mon.a (mon.0) 2627 : cluster [INF] Cluster is now healthy 2024-09-13T15:15:29.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[84576]: audit 2024-09-13T15:15:28.247887+0000 mon.a (mon.0) 2628 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:15:29.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[84576]: audit 2024-09-13T15:15:28.247887+0000 mon.a (mon.0) 2628 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:15:29.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[84576]: cluster 2024-09-13T15:15:28.251661+0000 mon.a (mon.0) 2629 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2024-09-13T15:15:29.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[84576]: cluster 2024-09-13T15:15:28.251661+0000 mon.a (mon.0) 2629 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2024-09-13T15:15:29.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[87147]: cluster 2024-09-13T15:15:28.235942+0000 mon.a (mon.0) 2626 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:15:29.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[87147]: cluster 2024-09-13T15:15:28.235942+0000 mon.a (mon.0) 2626 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:15:29.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[87147]: cluster 2024-09-13T15:15:28.235986+0000 mon.a (mon.0) 2627 : cluster [INF] Cluster is now healthy 2024-09-13T15:15:29.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[87147]: cluster 2024-09-13T15:15:28.235986+0000 mon.a (mon.0) 2627 : cluster [INF] Cluster is now healthy 2024-09-13T15:15:29.350 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[87147]: audit 2024-09-13T15:15:28.247887+0000 mon.a (mon.0) 2628 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:15:29.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[87147]: audit 2024-09-13T15:15:28.247887+0000 mon.a (mon.0) 2628 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:15:29.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[87147]: cluster 2024-09-13T15:15:28.251661+0000 mon.a (mon.0) 2629 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2024-09-13T15:15:29.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:29 smithi099 bash[87147]: cluster 2024-09-13T15:15:28.251661+0000 mon.a (mon.0) 2629 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2024-09-13T15:15:29.369 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:29.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:29.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:29.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:29.486 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:29 smithi177 bash[68594]: cluster 2024-09-13T15:15:28.235942+0000 mon.a (mon.0) 2626 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:15:29.486 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:29 smithi177 bash[68594]: cluster 2024-09-13T15:15:28.235942+0000 mon.a (mon.0) 2626 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:15:29.486 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:29 smithi177 bash[68594]: cluster 2024-09-13T15:15:28.235986+0000 mon.a (mon.0) 2627 : cluster [INF] Cluster is now healthy 2024-09-13T15:15:29.486 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:29 smithi177 bash[68594]: cluster 2024-09-13T15:15:28.235986+0000 mon.a (mon.0) 2627 : cluster [INF] Cluster is now healthy 2024-09-13T15:15:29.486 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:29 smithi177 bash[68594]: audit 2024-09-13T15:15:28.247887+0000 mon.a (mon.0) 2628 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:15:29.486 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:29 smithi177 bash[68594]: audit 2024-09-13T15:15:28.247887+0000 mon.a (mon.0) 2628 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:15:29.486 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:29 smithi177 bash[68594]: cluster 2024-09-13T15:15:28.251661+0000 mon.a (mon.0) 2629 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2024-09-13T15:15:29.486 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:29 smithi177 bash[68594]: cluster 2024-09-13T15:15:28.251661+0000 mon.a (mon.0) 2629 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2024-09-13T15:15:29.506 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:29.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:29.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:29.608 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:29.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:29.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:29.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:29.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:29.766 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:29.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:29.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:29.869 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:29.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:29.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:29.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:30.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:30.027 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:30.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:30.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:30.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 10 16 23742 23726 148.274 133.062 0.00390353 0.00673874 2024-09-13T15:15:30.133 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:30.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:30.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:30.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:30.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:30.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:30.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:30.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:30 smithi099 bash[84576]: cluster 2024-09-13T15:15:29.160977+0000 mgr.y (mgr.26252) 1483 : cluster [DBG] pgmap v2073: 89 pgs: 89 active+clean; 955 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2024-09-13T15:15:30.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:30 smithi099 bash[84576]: cluster 2024-09-13T15:15:29.160977+0000 mgr.y (mgr.26252) 1483 : cluster [DBG] pgmap v2073: 89 pgs: 89 active+clean; 955 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2024-09-13T15:15:30.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:30 smithi099 bash[87147]: cluster 2024-09-13T15:15:29.160977+0000 mgr.y (mgr.26252) 1483 : cluster [DBG] pgmap v2073: 89 pgs: 89 active+clean; 955 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2024-09-13T15:15:30.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:30 smithi099 bash[87147]: cluster 2024-09-13T15:15:29.160977+0000 mgr.y (mgr.26252) 1483 : cluster [DBG] pgmap v2073: 89 pgs: 89 active+clean; 955 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2024-09-13T15:15:30.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:30.400 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:15:30 smithi177 bash[61439]: ts=2024-09-13T15:15:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:15:30.401 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:30 smithi177 bash[68594]: cluster 2024-09-13T15:15:29.160977+0000 mgr.y (mgr.26252) 1483 : cluster [DBG] pgmap v2073: 89 pgs: 89 active+clean; 955 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2024-09-13T15:15:30.401 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:30 smithi177 bash[68594]: cluster 2024-09-13T15:15:29.160977+0000 mgr.y (mgr.26252) 1483 : cluster [DBG] pgmap v2073: 89 pgs: 89 active+clean; 955 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2024-09-13T15:15:30.423 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:30.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:30.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:30.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:30.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:30.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:30.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:30.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:30.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:30.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:30.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:30.814 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:30.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:30.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:30.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:30.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:30.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:30.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:31.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:31.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 11 16 25871 25855 146.89 133.062 0.00751906 0.00680079 2024-09-13T15:15:31.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:31.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:31.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:31.167 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:15:31.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:31.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:31.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:31.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:31.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:31.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:31 smithi099 bash[84576]: cluster 2024-09-13T15:15:31.161583+0000 mgr.y (mgr.26252) 1484 : cluster [DBG] pgmap v2074: 89 pgs: 89 active+clean; 1.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-09-13T15:15:31.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:31 smithi099 bash[84576]: cluster 2024-09-13T15:15:31.161583+0000 mgr.y (mgr.26252) 1484 : cluster [DBG] pgmap v2074: 89 pgs: 89 active+clean; 1.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-09-13T15:15:31.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:31 smithi099 bash[87147]: cluster 2024-09-13T15:15:31.161583+0000 mgr.y (mgr.26252) 1484 : cluster [DBG] pgmap v2074: 89 pgs: 89 active+clean; 1.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-09-13T15:15:31.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:31 smithi099 bash[87147]: cluster 2024-09-13T15:15:31.161583+0000 mgr.y (mgr.26252) 1484 : cluster [DBG] pgmap v2074: 89 pgs: 89 active+clean; 1.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-09-13T15:15:31.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:31.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:31.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:31.484 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:31.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:31.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:31.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:31.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:31 smithi177 bash[68594]: cluster 2024-09-13T15:15:31.161583+0000 mgr.y (mgr.26252) 1484 : cluster [DBG] pgmap v2074: 89 pgs: 89 active+clean; 1.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-09-13T15:15:31.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:31 smithi177 bash[68594]: cluster 2024-09-13T15:15:31.161583+0000 mgr.y (mgr.26252) 1484 : cluster [DBG] pgmap v2074: 89 pgs: 89 active+clean; 1.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-09-13T15:15:31.617 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:31.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:31.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:31.720 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:31.746 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:31.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:31.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:31.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:31.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:31.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:31.978 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:32.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:32.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:32.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:32.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 12 16 27865 27849 145.034 124.625 0.0121158 0.00688849 2024-09-13T15:15:32.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:32.138 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:32.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:32.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:32.241 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:32.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:32.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:32.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:32.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:32.393 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:32.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:32.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:32.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:32.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:32.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:32.627 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:32.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:32.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:32.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:32.759 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:32.786 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:32.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:32.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:32.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:32.917 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:32.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:32.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:33.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:33.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:33.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:33.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 13 16 28119 28103 135.098 15.875 0.0114029 0.0073951 2024-09-13T15:15:33.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:33.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:33.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:33.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:33.195 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:15:33.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:33.311 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:33.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:33.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:33.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:33.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:33.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:33.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:33.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:33.575 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:33.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:33.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:33.682 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:33.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:33.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:33.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:33.839 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:33.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:33.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:33.942 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:33.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:33.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:33.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:34.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 14 16 29857 29841 133.207 108.625 0.00538082 0.00750062 2024-09-13T15:15:34.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:34.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:34.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:34.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:34.207 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:34.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:34.237 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:34.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:34.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:34.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:34.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:34.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:34.467 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:34.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[84576]: cluster 2024-09-13T15:15:33.162344+0000 mgr.y (mgr.26252) 1485 : cluster [DBG] pgmap v2075: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-13T15:15:34.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[84576]: cluster 2024-09-13T15:15:33.162344+0000 mgr.y (mgr.26252) 1485 : cluster [DBG] pgmap v2075: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-13T15:15:34.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[84576]: audit 2024-09-13T15:15:33.316910+0000 mon.a (mon.0) 2630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:34.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[84576]: audit 2024-09-13T15:15:33.316910+0000 mon.a (mon.0) 2630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:34.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[84576]: audit 2024-09-13T15:15:33.317826+0000 mon.a (mon.0) 2631 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:34.478 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[84576]: audit 2024-09-13T15:15:33.317826+0000 mon.a (mon.0) 2631 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:34.478 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[87147]: cluster 2024-09-13T15:15:33.162344+0000 mgr.y (mgr.26252) 1485 : cluster [DBG] pgmap v2075: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-13T15:15:34.478 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[87147]: cluster 2024-09-13T15:15:33.162344+0000 mgr.y (mgr.26252) 1485 : cluster [DBG] pgmap v2075: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-13T15:15:34.479 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[87147]: audit 2024-09-13T15:15:33.316910+0000 mon.a (mon.0) 2630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:34.479 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[87147]: audit 2024-09-13T15:15:33.316910+0000 mon.a (mon.0) 2630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:34.479 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[87147]: audit 2024-09-13T15:15:33.317826+0000 mon.a (mon.0) 2631 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:34.479 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:34 smithi099 bash[87147]: audit 2024-09-13T15:15:33.317826+0000 mon.a (mon.0) 2631 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:34.496 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:34.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:34.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:34.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:34 smithi177 bash[68594]: cluster 2024-09-13T15:15:33.162344+0000 mgr.y (mgr.26252) 1485 : cluster [DBG] pgmap v2075: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-13T15:15:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:34 smithi177 bash[68594]: cluster 2024-09-13T15:15:33.162344+0000 mgr.y (mgr.26252) 1485 : cluster [DBG] pgmap v2075: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-13T15:15:34.609 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:34 smithi177 bash[68594]: audit 2024-09-13T15:15:33.316910+0000 mon.a (mon.0) 2630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:34 smithi177 bash[68594]: audit 2024-09-13T15:15:33.316910+0000 mon.a (mon.0) 2630 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:15:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:34 smithi177 bash[68594]: audit 2024-09-13T15:15:33.317826+0000 mon.a (mon.0) 2631 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:34.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:34 smithi177 bash[68594]: audit 2024-09-13T15:15:33.317826+0000 mon.a (mon.0) 2631 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:34.627 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:34.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:34.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:34.732 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:34.757 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:34.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:34.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:34.860 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:34.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:34.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:34.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:34.981 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:35.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:35.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:35.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:35.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 15 16 31455 31439 130.984 99.875 0.00978341 0.00762927 2024-09-13T15:15:35.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:35.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:35.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:35.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:35.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:35.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:35.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:35.377 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:35.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:35.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:35.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:35.505 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:35.531 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:35.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:35.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:35.635 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:35.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:35.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:35.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:35.767 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:35.798 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:35.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:35.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:35.903 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:35.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:35.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:35.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:36.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:15:36.025 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:15:36.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:36.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:36.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 16 16 33273 33257 129.898 113.625 0.0105153 0.00769301 2024-09-13T15:15:36.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:36.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:36.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:36.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:36.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:36.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:36.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:36.313 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:36 smithi099 bash[87147]: cluster 2024-09-13T15:15:35.163267+0000 mgr.y (mgr.26252) 1486 : cluster [DBG] pgmap v2076: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T15:15:36.313 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:36 smithi099 bash[87147]: cluster 2024-09-13T15:15:35.163267+0000 mgr.y (mgr.26252) 1486 : cluster [DBG] pgmap v2076: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T15:15:36.316 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:36 smithi099 bash[84576]: cluster 2024-09-13T15:15:35.163267+0000 mgr.y (mgr.26252) 1486 : cluster [DBG] pgmap v2076: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T15:15:36.316 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:36 smithi099 bash[84576]: cluster 2024-09-13T15:15:35.163267+0000 mgr.y (mgr.26252) 1486 : cluster [DBG] pgmap v2076: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T15:15:36.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:36.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:36.432 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:36.442 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:36 smithi177 bash[68594]: cluster 2024-09-13T15:15:35.163267+0000 mgr.y (mgr.26252) 1486 : cluster [DBG] pgmap v2076: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T15:15:36.442 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:36 smithi177 bash[68594]: cluster 2024-09-13T15:15:35.163267+0000 mgr.y (mgr.26252) 1486 : cluster [DBG] pgmap v2076: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T15:15:36.458 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:36.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:36.564 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:36.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:36.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:36.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:36.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:36.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:36.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:36.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:36.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:36.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:36.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:36.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:36.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:36.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:36.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:36.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:37.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 17 16 34553 34537 126.963 80 0.00192437 0.00777006 2024-09-13T15:15:37.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:37.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:37.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:37.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:37.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:37.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:37.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:37.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:37.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:15:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:15:37] "GET /metrics HTTP/1.1" 200 36771 "" "Prometheus/2.51.0" 2024-09-13T15:15:37.328 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:37.356 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:37.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:37.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:37.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:37.492 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:37.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:37.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:37.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:37.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:37.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:37.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:37.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:37.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:37.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:37.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:37.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:38.024 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:38.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:38.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:38.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 18 16 35948 35932 124.752 87.1875 0.00720817 0.00801115 2024-09-13T15:15:38.128 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:38.152 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:38.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:38.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:38.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:38.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:38.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:38.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:38.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:38 smithi177 bash[68594]: cluster 2024-09-13T15:15:37.164136+0000 mgr.y (mgr.26252) 1487 : cluster [DBG] pgmap v2077: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-09-13T15:15:38.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:38 smithi177 bash[68594]: cluster 2024-09-13T15:15:37.164136+0000 mgr.y (mgr.26252) 1487 : cluster [DBG] pgmap v2077: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-09-13T15:15:38.391 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:38.401 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:38 smithi099 bash[84576]: cluster 2024-09-13T15:15:37.164136+0000 mgr.y (mgr.26252) 1487 : cluster [DBG] pgmap v2077: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-09-13T15:15:38.401 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:38 smithi099 bash[84576]: cluster 2024-09-13T15:15:37.164136+0000 mgr.y (mgr.26252) 1487 : cluster [DBG] pgmap v2077: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-09-13T15:15:38.401 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:38 smithi099 bash[87147]: cluster 2024-09-13T15:15:37.164136+0000 mgr.y (mgr.26252) 1487 : cluster [DBG] pgmap v2077: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-09-13T15:15:38.401 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:38 smithi099 bash[87147]: cluster 2024-09-13T15:15:37.164136+0000 mgr.y (mgr.26252) 1487 : cluster [DBG] pgmap v2077: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-09-13T15:15:38.416 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:38.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:38.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:38.519 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:38.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:38.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:38.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:38.671 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:38.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:38.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:38.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:38.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:38.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:38.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:38.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:38.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:38.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:38.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:39.037 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:39.064 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:39.065 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 19 16 37655 37639 123.801 106.688 0.00483982 0.00807078 2024-09-13T15:15:39.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:39.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:39.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:39.199 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:39.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:39.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:39.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:39.330 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:39.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:39.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:39.434 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:39.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:39.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:39.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:39.562 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:39.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:39.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:39.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:39.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:39.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:39.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:39.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:39.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:39.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:39.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:39.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:39.986 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:39.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:39.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:40.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:15:40.066622+0000 min lat: 0.00120222 max lat: 0.869931 avg lat: 0.00812541 2024-09-13T15:15:40.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:15:40.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 20 16 39370 39354 122.97 107.188 0.00403863 0.00812541 2024-09-13T15:15:40.093 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:40.120 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:40.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:40.223 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:40.249 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:40.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:40.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:40.378 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:40.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:40.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:40.491 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:15:40 smithi177 bash[61439]: ts=2024-09-13T15:15:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:15:40.491 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:40 smithi177 bash[68594]: cluster 2024-09-13T15:15:39.164704+0000 mgr.y (mgr.26252) 1488 : cluster [DBG] pgmap v2078: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-09-13T15:15:40.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:40 smithi177 bash[68594]: cluster 2024-09-13T15:15:39.164704+0000 mgr.y (mgr.26252) 1488 : cluster [DBG] pgmap v2078: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-09-13T15:15:40.508 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:40.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:40.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:40 smithi099 bash[84576]: cluster 2024-09-13T15:15:39.164704+0000 mgr.y (mgr.26252) 1488 : cluster [DBG] pgmap v2078: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-09-13T15:15:40.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:40 smithi099 bash[84576]: cluster 2024-09-13T15:15:39.164704+0000 mgr.y (mgr.26252) 1488 : cluster [DBG] pgmap v2078: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-09-13T15:15:40.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:40 smithi099 bash[87147]: cluster 2024-09-13T15:15:39.164704+0000 mgr.y (mgr.26252) 1488 : cluster [DBG] pgmap v2078: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-09-13T15:15:40.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:40 smithi099 bash[87147]: cluster 2024-09-13T15:15:39.164704+0000 mgr.y (mgr.26252) 1488 : cluster [DBG] pgmap v2078: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2024-09-13T15:15:40.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:40.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:40.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:40.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:40.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:40.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:40.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:40.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:40.891 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:15:40.891 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:15:40.901 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:40.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:40.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:41.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:41.027 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:41.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:41.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 21 16 40939 40923 121.783 98.0625 0.00728981 0.00820006 2024-09-13T15:15:41.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:41.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:41.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:41.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:41.262 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:41.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:41.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:41.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:41.419 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:41.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:41.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:41.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:41.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:41.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:41.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:41.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:41.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:41.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:41.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:41.821 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:41.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:41.924 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:41.944 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:41.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:41.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:42.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:42.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 22 16 42297 42281 120.105 84.875 0.00230438 0.00826535 2024-09-13T15:15:42.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:42.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:42.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:42.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:42.197 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:42.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:42.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:42.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:42.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:42.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:42.433 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:42.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:42.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:42.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:42.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:42.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:42 smithi099 bash[84576]: cluster 2024-09-13T15:15:41.165231+0000 mgr.y (mgr.26252) 1489 : cluster [DBG] pgmap v2079: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T15:15:42.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:42 smithi099 bash[84576]: cluster 2024-09-13T15:15:41.165231+0000 mgr.y (mgr.26252) 1489 : cluster [DBG] pgmap v2079: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T15:15:42.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:42 smithi099 bash[87147]: cluster 2024-09-13T15:15:41.165231+0000 mgr.y (mgr.26252) 1489 : cluster [DBG] pgmap v2079: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T15:15:42.573 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:42 smithi099 bash[87147]: cluster 2024-09-13T15:15:41.165231+0000 mgr.y (mgr.26252) 1489 : cluster [DBG] pgmap v2079: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T15:15:42.589 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:42.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:42.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:42 smithi177 bash[68594]: cluster 2024-09-13T15:15:41.165231+0000 mgr.y (mgr.26252) 1489 : cluster [DBG] pgmap v2079: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T15:15:42.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:42 smithi177 bash[68594]: cluster 2024-09-13T15:15:41.165231+0000 mgr.y (mgr.26252) 1489 : cluster [DBG] pgmap v2079: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T15:15:42.693 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:42.724 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:42.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:42.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:42.828 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:42.854 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:42.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:42.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:42.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:42.985 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:42.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:43.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 23 16 43715 43699 118.736 88.625 0.00718833 0.00841669 2024-09-13T15:15:43.091 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:43.117 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:43.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:43.220 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:43.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:43.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:43.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:43.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:43.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:43.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:43.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:43.474 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:43.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:43.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:43.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:43.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:43.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:43.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:43.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:43.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:43.774 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:43.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:43.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:43.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:43.913 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:43.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:44.018 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:44.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 24 16 45336 45320 118.01 101.312 0.0203953 0.00846843 2024-09-13T15:15:44.067 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:44.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:44.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:44.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:44.241 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:15:44.242 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "" 2024-09-13T15:15:44.242 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:15:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:44.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:44.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:44.271 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:44 smithi177 bash[68594]: cluster 2024-09-13T15:15:43.165769+0000 mgr.y (mgr.26252) 1490 : cluster [DBG] pgmap v2080: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:44.272 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:44 smithi177 bash[68594]: cluster 2024-09-13T15:15:43.165769+0000 mgr.y (mgr.26252) 1490 : cluster [DBG] pgmap v2080: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:44.328 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:15:44.328 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:15:44.328 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2024-09-13T15:15:44.328 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:15:44.343 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:44.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:44.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:44.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:44.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:44.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:44 smithi099 bash[84576]: cluster 2024-09-13T15:15:43.165769+0000 mgr.y (mgr.26252) 1490 : cluster [DBG] pgmap v2080: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:44.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:44 smithi099 bash[84576]: cluster 2024-09-13T15:15:43.165769+0000 mgr.y (mgr.26252) 1490 : cluster [DBG] pgmap v2080: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:44.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:44 smithi099 bash[87147]: cluster 2024-09-13T15:15:43.165769+0000 mgr.y (mgr.26252) 1490 : cluster [DBG] pgmap v2080: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:44.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:44 smithi099 bash[87147]: cluster 2024-09-13T15:15:43.165769+0000 mgr.y (mgr.26252) 1490 : cluster [DBG] pgmap v2080: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:44.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:44.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:44.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:44.678 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:44.723 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:44.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:44.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:44.826 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:44.852 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:44.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:44.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:44.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:44.972 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:44.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:44.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:45.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 25 16 46643 46627 116.557 81.6875 0.00242625 0.0085314 2024-09-13T15:15:45.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:45.095 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:45.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:45.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:45.198 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:45.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:45.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:45.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:45.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:45.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:45.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:45.459 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:45.491 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:45.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:45.602 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:45.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:45.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:45.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:45.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:45.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:45.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:45.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:45.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:45.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:45.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:45.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:46.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:46.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:46.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 26 16 47473 47457 114.069 51.875 0.00999774 0.00876157 2024-09-13T15:15:46.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:46.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:46.172 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:46.225 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:46.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:46.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:46.328 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:46.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:46.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:46.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:46.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:46.494 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:46.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:46.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:46.501 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:46 smithi099 bash[84576]: cluster 2024-09-13T15:15:45.166780+0000 mgr.y (mgr.26252) 1491 : cluster [DBG] pgmap v2081: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-09-13T15:15:46.501 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:46 smithi099 bash[84576]: cluster 2024-09-13T15:15:45.166780+0000 mgr.y (mgr.26252) 1491 : cluster [DBG] pgmap v2081: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-09-13T15:15:46.501 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:46 smithi099 bash[87147]: cluster 2024-09-13T15:15:45.166780+0000 mgr.y (mgr.26252) 1491 : cluster [DBG] pgmap v2081: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-09-13T15:15:46.501 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:46 smithi099 bash[87147]: cluster 2024-09-13T15:15:45.166780+0000 mgr.y (mgr.26252) 1491 : cluster [DBG] pgmap v2081: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-09-13T15:15:46.597 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:46.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:46 smithi177 bash[68594]: cluster 2024-09-13T15:15:45.166780+0000 mgr.y (mgr.26252) 1491 : cluster [DBG] pgmap v2081: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-09-13T15:15:46.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:46 smithi177 bash[68594]: cluster 2024-09-13T15:15:45.166780+0000 mgr.y (mgr.26252) 1491 : cluster [DBG] pgmap v2081: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-09-13T15:15:46.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:46.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:46.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:46.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:46.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:46.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:46.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:46.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:46.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:46.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:46.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:47.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:47.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:47.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:47.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:47.068 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 27 16 48480 48464 112.175 62.9375 0.00888793 0.00891004 2024-09-13T15:15:47.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:47.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:47.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T15:15:47.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:47.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:47.271 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:47.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:47.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:47.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:15:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:15:47] "GET /metrics HTTP/1.1" 200 36771 "" "Prometheus/2.51.0" 2024-09-13T15:15:47.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:47.442 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:47.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:47.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:47.547 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:47.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:47.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:47.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:47.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:47.720 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:47.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:47.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:47.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:47.834 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:47 smithi177 bash[68594]: cluster 2024-09-13T15:15:47.167290+0000 mgr.y (mgr.26252) 1492 : cluster [DBG] pgmap v2082: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:15:47.834 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:47 smithi177 bash[68594]: cluster 2024-09-13T15:15:47.167290+0000 mgr.y (mgr.26252) 1492 : cluster [DBG] pgmap v2082: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:15:47.847 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:47.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:47.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:47.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:47.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:47.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:47.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:48.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 28 16 49936 49920 111.418 91 0.0134064 0.00897067 2024-09-13T15:15:48.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:47 smithi099 bash[84576]: cluster 2024-09-13T15:15:47.167290+0000 mgr.y (mgr.26252) 1492 : cluster [DBG] pgmap v2082: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:15:48.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:47 smithi099 bash[84576]: cluster 2024-09-13T15:15:47.167290+0000 mgr.y (mgr.26252) 1492 : cluster [DBG] pgmap v2082: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:15:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:47 smithi099 bash[87147]: cluster 2024-09-13T15:15:47.167290+0000 mgr.y (mgr.26252) 1492 : cluster [DBG] pgmap v2082: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:15:48.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:47 smithi099 bash[87147]: cluster 2024-09-13T15:15:47.167290+0000 mgr.y (mgr.26252) 1492 : cluster [DBG] pgmap v2082: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:15:48.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:48.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:48.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:48.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:48.216 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:48.240 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:48.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:48.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:48.349 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:48.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:48.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:48.478 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:48.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:48.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:48.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:48.615 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:48.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:48.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:48.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:48.752 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:48.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:48 smithi099 bash[84576]: audit 2024-09-13T15:15:48.314526+0000 mon.a (mon.0) 2632 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:48.763 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:48 smithi099 bash[84576]: audit 2024-09-13T15:15:48.314526+0000 mon.a (mon.0) 2632 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:48.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:48 smithi099 bash[87147]: audit 2024-09-13T15:15:48.314526+0000 mon.a (mon.0) 2632 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:48.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:48 smithi099 bash[87147]: audit 2024-09-13T15:15:48.314526+0000 mon.a (mon.0) 2632 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:48.788 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:48.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:48.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:48.891 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:48.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:48.916 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:48 smithi177 bash[68594]: audit 2024-09-13T15:15:48.314526+0000 mon.a (mon.0) 2632 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:48.916 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:48 smithi177 bash[68594]: audit 2024-09-13T15:15:48.314526+0000 mon.a (mon.0) 2632 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:15:48.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:48.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:49.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:49.044 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:49.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:49.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:49.066 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 29 16 51346 51330 110.615 88.125 0.00813969 0.00903394 2024-09-13T15:15:49.148 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:49.175 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:49.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:49.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:49.279 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:49.309 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:49.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:49.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:49.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:49.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:49.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:49.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:49.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:49.587 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:49.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:49.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:49.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:49.703 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:49 smithi177 bash[68594]: cluster 2024-09-13T15:15:49.170102+0000 mgr.y (mgr.26252) 1493 : cluster [DBG] pgmap v2083: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T15:15:49.703 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:49 smithi177 bash[68594]: cluster 2024-09-13T15:15:49.170102+0000 mgr.y (mgr.26252) 1493 : cluster [DBG] pgmap v2083: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T15:15:49.725 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:49.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:49.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:49.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:49 smithi099 bash[84576]: cluster 2024-09-13T15:15:49.170102+0000 mgr.y (mgr.26252) 1493 : cluster [DBG] pgmap v2083: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T15:15:49.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:49 smithi099 bash[84576]: cluster 2024-09-13T15:15:49.170102+0000 mgr.y (mgr.26252) 1493 : cluster [DBG] pgmap v2083: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T15:15:49.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:49 smithi099 bash[87147]: cluster 2024-09-13T15:15:49.170102+0000 mgr.y (mgr.26252) 1493 : cluster [DBG] pgmap v2083: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T15:15:49.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:49 smithi099 bash[87147]: cluster 2024-09-13T15:15:49.170102+0000 mgr.y (mgr.26252) 1493 : cluster [DBG] pgmap v2083: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T15:15:49.829 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:49.854 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:49.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:49.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:49.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:49.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:49.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:49.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:50.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 30 16 52783 52767 109.921 89.8125 0.0183022 0.00909228 2024-09-13T15:15:50.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:50.113 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:50.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:50.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:50.217 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:50.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:50.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:50.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:50.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:50.357 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:15:50 smithi177 bash[61439]: ts=2024-09-13T15:15:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:15:50.370 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:50.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:50.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:50.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:50.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:50.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:50.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:50.610 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:50.632 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:50.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:50.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:50.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:50.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:50.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:50.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:50.869 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:50.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:50.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:50.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:50.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:51.017 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:51.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:51.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 31 16 53831 53815 108.488 65.5 0.00352794 0.00913385 2024-09-13T15:15:51.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:51.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:51.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:51.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:51.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:51.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:51.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:51.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:51.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:51.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:51.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:51.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:51.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:51.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:51.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:51.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:51.642 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:51.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:51.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:51.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:51.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:51.794 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:51.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:51.900 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:51.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:51.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:51.989 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:15:52.034 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:52.067 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:52.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 32 16 54939 54923 107.262 69.25 0.0109437 0.00931767 2024-09-13T15:15:52.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:52.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:52.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:52.180 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:15:52.181 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:15:52.181 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2024-09-13T15:15:52.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:15:52.208 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:52.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:52.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:52.312 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:52 smithi177 bash[68594]: cluster 2024-09-13T15:15:51.170662+0000 mgr.y (mgr.26252) 1494 : cluster [DBG] pgmap v2084: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:15:52.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:52 smithi177 bash[68594]: cluster 2024-09-13T15:15:51.170662+0000 mgr.y (mgr.26252) 1494 : cluster [DBG] pgmap v2084: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:15:52.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:52.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:52.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:52.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:52.529 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:52 smithi099 bash[84576]: cluster 2024-09-13T15:15:51.170662+0000 mgr.y (mgr.26252) 1494 : cluster [DBG] pgmap v2084: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:15:52.529 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:52 smithi099 bash[84576]: cluster 2024-09-13T15:15:51.170662+0000 mgr.y (mgr.26252) 1494 : cluster [DBG] pgmap v2084: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:15:52.529 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:52 smithi099 bash[87147]: cluster 2024-09-13T15:15:51.170662+0000 mgr.y (mgr.26252) 1494 : cluster [DBG] pgmap v2084: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:15:52.529 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:52 smithi099 bash[87147]: cluster 2024-09-13T15:15:51.170662+0000 mgr.y (mgr.26252) 1494 : cluster [DBG] pgmap v2084: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:15:52.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:52.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:52.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:52.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:52.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:52.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:52.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:52.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:52.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:52.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:52.913 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:52.945 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:52.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:52.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:53.051 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:53.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 33 16 56269 56253 106.53 83.125 0.021333 0.009382 2024-09-13T15:15:53.091 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:53.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:53.195 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:53.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:53.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:53.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:53.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:53.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:53.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:53.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:53.462 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:53.492 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:53.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:53.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:53.598 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:53.616 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:53.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:53.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:53.743 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:53.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:53.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:53.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:53.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:53.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:53.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:54.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:54.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:54.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:54.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 34 16 57030 57014 104.796 47.5625 0.0150356 0.00953781 2024-09-13T15:15:54.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:54.136 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:54.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:54.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:54.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:54.248 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:54 smithi177 bash[68594]: cluster 2024-09-13T15:15:53.171065+0000 mgr.y (mgr.26252) 1495 : cluster [DBG] pgmap v2085: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-13T15:15:54.248 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:54 smithi177 bash[68594]: cluster 2024-09-13T15:15:53.171065+0000 mgr.y (mgr.26252) 1495 : cluster [DBG] pgmap v2085: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-13T15:15:54.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:54.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:54.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:54.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:54.378 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:54 smithi099 bash[84576]: cluster 2024-09-13T15:15:53.171065+0000 mgr.y (mgr.26252) 1495 : cluster [DBG] pgmap v2085: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-13T15:15:54.378 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:54 smithi099 bash[84576]: cluster 2024-09-13T15:15:53.171065+0000 mgr.y (mgr.26252) 1495 : cluster [DBG] pgmap v2085: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-13T15:15:54.379 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:54 smithi099 bash[87147]: cluster 2024-09-13T15:15:53.171065+0000 mgr.y (mgr.26252) 1495 : cluster [DBG] pgmap v2085: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-13T15:15:54.379 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:54 smithi099 bash[87147]: cluster 2024-09-13T15:15:53.171065+0000 mgr.y (mgr.26252) 1495 : cluster [DBG] pgmap v2085: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-13T15:15:54.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:15:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:54.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:15:54.497 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:54.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:54.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:54.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:54.633 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:54.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:54.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:54.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:54.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:54.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:54.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:54.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:54.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:54.934 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:54.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:54.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:55.040 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:55.068 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 35 15 58401 58386 104.251 85.75 0.0136478 0.00958627 2024-09-13T15:15:55.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:55.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:55.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:55.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:55.210 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:55.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:55.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:55.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:55.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:55.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:55.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:55.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:55.480 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:55.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:55.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:55.583 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:55.614 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:55.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:55.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:55.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:55.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:55.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:55.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:15:55.853 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:55.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:55.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:55.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:55.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:56.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 36 16 59739 59723 103.676 83.5625 0.00439798 0.00963986 2024-09-13T15:15:56.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:56 smithi177 bash[68594]: cluster 2024-09-13T15:15:55.172531+0000 mgr.y (mgr.26252) 1496 : cluster [DBG] pgmap v2086: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:56.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:56 smithi177 bash[68594]: cluster 2024-09-13T15:15:55.172531+0000 mgr.y (mgr.26252) 1496 : cluster [DBG] pgmap v2086: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:56.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:56.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:56.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:56.500 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:56 smithi099 bash[84576]: cluster 2024-09-13T15:15:55.172531+0000 mgr.y (mgr.26252) 1496 : cluster [DBG] pgmap v2086: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:56.500 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:56 smithi099 bash[84576]: cluster 2024-09-13T15:15:55.172531+0000 mgr.y (mgr.26252) 1496 : cluster [DBG] pgmap v2086: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:56.500 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:56 smithi099 bash[87147]: cluster 2024-09-13T15:15:55.172531+0000 mgr.y (mgr.26252) 1496 : cluster [DBG] pgmap v2086: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:56.500 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:56 smithi099 bash[87147]: cluster 2024-09-13T15:15:55.172531+0000 mgr.y (mgr.26252) 1496 : cluster [DBG] pgmap v2086: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T15:15:56.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:56.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:56.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:56.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:56.774 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:56.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:56.844 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:15:56.844 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:15:56.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:56.892 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:15:56.892 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:15:56.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:56.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:56.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:56.945 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:15:56.946 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:15:56.993 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:15:56.994 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:15:57.010 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:57.038 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:15:57.038 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:15:57.059 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:15:57.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 37 16 60811 60795 102.685 67 0.015623 0.00973257 2024-09-13T15:15:57.080 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:15:57.080 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:15:57.105 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:15:57.105 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:15:57.135 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:15:57.136 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:15:57.157 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:15:57] "GET /metrics HTTP/1.1" 200 36773 "" "Prometheus/2.51.0" 2024-09-13T15:15:57.208 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:57.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:57.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:57.255 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T15:15:57.311 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:57.319 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:57 smithi177 bash[68594]: audit 2024-09-13T15:15:56.251667+0000 mon.a (mon.0) 2633 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:15:57.319 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:57 smithi177 bash[68594]: audit 2024-09-13T15:15:56.251667+0000 mon.a (mon.0) 2633 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:15:57.320 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:57 smithi177 bash[68594]: audit 2024-09-13T15:15:56.785515+0000 mon.a (mon.0) 2634 : audit [DBG] from='client.? 172.21.15.99:0/798208289' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:57.320 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:57 smithi177 bash[68594]: audit 2024-09-13T15:15:56.785515+0000 mon.a (mon.0) 2634 : audit [DBG] from='client.? 172.21.15.99:0/798208289' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:57.320 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:57 smithi177 bash[68594]: audit 2024-09-13T15:15:57.111717+0000 mon.c (mon.2) 117 : audit [INF] from='client.? 172.21.15.99:0/2069741976' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.320 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:57 smithi177 bash[68594]: audit 2024-09-13T15:15:57.111717+0000 mon.c (mon.2) 117 : audit [INF] from='client.? 172.21.15.99:0/2069741976' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.320 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:57 smithi177 bash[68594]: audit 2024-09-13T15:15:57.112286+0000 mon.a (mon.0) 2635 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.320 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:57 smithi177 bash[68594]: audit 2024-09-13T15:15:57.112286+0000 mon.a (mon.0) 2635 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[84576]: audit 2024-09-13T15:15:56.251667+0000 mon.a (mon.0) 2633 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:15:57.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[84576]: audit 2024-09-13T15:15:56.251667+0000 mon.a (mon.0) 2633 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:15:57.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[84576]: audit 2024-09-13T15:15:56.785515+0000 mon.a (mon.0) 2634 : audit [DBG] from='client.? 172.21.15.99:0/798208289' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:57.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[84576]: audit 2024-09-13T15:15:56.785515+0000 mon.a (mon.0) 2634 : audit [DBG] from='client.? 172.21.15.99:0/798208289' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:57.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[84576]: audit 2024-09-13T15:15:57.111717+0000 mon.c (mon.2) 117 : audit [INF] from='client.? 172.21.15.99:0/2069741976' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[84576]: audit 2024-09-13T15:15:57.111717+0000 mon.c (mon.2) 117 : audit [INF] from='client.? 172.21.15.99:0/2069741976' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[84576]: audit 2024-09-13T15:15:57.112286+0000 mon.a (mon.0) 2635 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[84576]: audit 2024-09-13T15:15:57.112286+0000 mon.a (mon.0) 2635 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.570 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:57.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:57.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:57.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[87147]: audit 2024-09-13T15:15:56.251667+0000 mon.a (mon.0) 2633 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:15:57.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[87147]: audit 2024-09-13T15:15:56.251667+0000 mon.a (mon.0) 2633 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:15:57.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[87147]: audit 2024-09-13T15:15:56.785515+0000 mon.a (mon.0) 2634 : audit [DBG] from='client.? 172.21.15.99:0/798208289' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:57.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[87147]: audit 2024-09-13T15:15:56.785515+0000 mon.a (mon.0) 2634 : audit [DBG] from='client.? 172.21.15.99:0/798208289' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:15:57.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[87147]: audit 2024-09-13T15:15:57.111717+0000 mon.c (mon.2) 117 : audit [INF] from='client.? 172.21.15.99:0/2069741976' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[87147]: audit 2024-09-13T15:15:57.111717+0000 mon.c (mon.2) 117 : audit [INF] from='client.? 172.21.15.99:0/2069741976' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[87147]: audit 2024-09-13T15:15:57.112286+0000 mon.a (mon.0) 2635 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:57 smithi099 bash[87147]: audit 2024-09-13T15:15:57.112286+0000 mon.a (mon.0) 2635 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:15:57.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:57.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:57.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:58.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:58.058 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:58.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:58.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 38 16 61704 61688 101.451 55.8125 0.0230048 0.00985143 2024-09-13T15:15:58.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:58.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:58.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:58.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:58.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:58.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:58.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:58.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:58.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:58.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:58.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:58 smithi177 bash[68594]: cluster 2024-09-13T15:15:57.173033+0000 mgr.y (mgr.26252) 1497 : cluster [DBG] pgmap v2087: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:15:58.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:58 smithi177 bash[68594]: cluster 2024-09-13T15:15:57.173033+0000 mgr.y (mgr.26252) 1497 : cluster [DBG] pgmap v2087: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:15:58.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:58 smithi177 bash[68594]: cluster 2024-09-13T15:15:57.236520+0000 mon.a (mon.0) 2636 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:15:58.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:58 smithi177 bash[68594]: cluster 2024-09-13T15:15:57.236520+0000 mon.a (mon.0) 2636 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:15:58.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:58 smithi177 bash[68594]: audit 2024-09-13T15:15:57.242420+0000 mon.a (mon.0) 2637 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:15:58.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:58 smithi177 bash[68594]: audit 2024-09-13T15:15:57.242420+0000 mon.a (mon.0) 2637 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:15:58.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:58 smithi177 bash[68594]: cluster 2024-09-13T15:15:57.250916+0000 mon.a (mon.0) 2638 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2024-09-13T15:15:58.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:58 smithi177 bash[68594]: cluster 2024-09-13T15:15:57.250916+0000 mon.a (mon.0) 2638 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2024-09-13T15:15:58.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:58.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:58.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:58.628 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:58.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:58.689 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[84576]: cluster 2024-09-13T15:15:57.173033+0000 mgr.y (mgr.26252) 1497 : cluster [DBG] pgmap v2087: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:15:58.689 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[84576]: cluster 2024-09-13T15:15:57.173033+0000 mgr.y (mgr.26252) 1497 : cluster [DBG] pgmap v2087: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:15:58.689 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[84576]: cluster 2024-09-13T15:15:57.236520+0000 mon.a (mon.0) 2636 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:15:58.689 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[84576]: cluster 2024-09-13T15:15:57.236520+0000 mon.a (mon.0) 2636 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:15:58.689 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[84576]: audit 2024-09-13T15:15:57.242420+0000 mon.a (mon.0) 2637 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:15:58.689 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[84576]: audit 2024-09-13T15:15:57.242420+0000 mon.a (mon.0) 2637 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:15:58.689 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[84576]: cluster 2024-09-13T15:15:57.250916+0000 mon.a (mon.0) 2638 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2024-09-13T15:15:58.689 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[84576]: cluster 2024-09-13T15:15:57.250916+0000 mon.a (mon.0) 2638 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2024-09-13T15:15:58.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[87147]: cluster 2024-09-13T15:15:57.173033+0000 mgr.y (mgr.26252) 1497 : cluster [DBG] pgmap v2087: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:15:58.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[87147]: cluster 2024-09-13T15:15:57.173033+0000 mgr.y (mgr.26252) 1497 : cluster [DBG] pgmap v2087: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-13T15:15:58.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[87147]: cluster 2024-09-13T15:15:57.236520+0000 mon.a (mon.0) 2636 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:15:58.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[87147]: cluster 2024-09-13T15:15:57.236520+0000 mon.a (mon.0) 2636 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:15:58.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[87147]: audit 2024-09-13T15:15:57.242420+0000 mon.a (mon.0) 2637 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:15:58.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[87147]: audit 2024-09-13T15:15:57.242420+0000 mon.a (mon.0) 2637 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:15:58.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[87147]: cluster 2024-09-13T15:15:57.250916+0000 mon.a (mon.0) 2638 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2024-09-13T15:15:58.690 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:58 smithi099 bash[87147]: cluster 2024-09-13T15:15:57.250916+0000 mon.a (mon.0) 2638 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2024-09-13T15:15:58.731 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:58.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:58.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:58.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:58.863 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:58.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:58.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:58.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:58.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:59.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:59.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:59.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:59.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 39 16 62421 62405 99.999 44.8125 0.292526 0.0099724 2024-09-13T15:15:59.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:59.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:15:59.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:59.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:15:59.220 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:15:59.221 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:15:59.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:59.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:15:59.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:59.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:15:59.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:59 smithi177 bash[68594]: cluster 2024-09-13T15:15:59.173650+0000 mgr.y (mgr.26252) 1498 : cluster [DBG] pgmap v2089: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T15:15:59.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:15:59 smithi177 bash[68594]: cluster 2024-09-13T15:15:59.173650+0000 mgr.y (mgr.26252) 1498 : cluster [DBG] pgmap v2089: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T15:15:59.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:59.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:15:59.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:59.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:15:59.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:59.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:15:59.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:59.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:15:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:59 smithi099 bash[84576]: cluster 2024-09-13T15:15:59.173650+0000 mgr.y (mgr.26252) 1498 : cluster [DBG] pgmap v2089: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T15:15:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:15:59 smithi099 bash[84576]: cluster 2024-09-13T15:15:59.173650+0000 mgr.y (mgr.26252) 1498 : cluster [DBG] pgmap v2089: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T15:15:59.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:59 smithi099 bash[87147]: cluster 2024-09-13T15:15:59.173650+0000 mgr.y (mgr.26252) 1498 : cluster [DBG] pgmap v2089: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T15:15:59.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:15:59 smithi099 bash[87147]: cluster 2024-09-13T15:15:59.173650+0000 mgr.y (mgr.26252) 1498 : cluster [DBG] pgmap v2089: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T15:15:59.647 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:59.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:15:59.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:59.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:15:59.778 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:59.804 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:15:59.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:59.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:15:59.911 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:59.928 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:15:59.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:15:59.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:00.032 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:00.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:00.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:00.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:00.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:16:00.068434+0000 min lat: 0.00120222 max lat: 0.869931 avg lat: 0.0100608 2024-09-13T15:16:00.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:16:00.067 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 40 16 63307 63291 98.8832 55.375 0.00313335 0.0100608 2024-09-13T15:16:00.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:00.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:00.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:00.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:00.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:00.324 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:00.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:00.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:00.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:16:00 smithi177 bash[61439]: ts=2024-09-13T15:16:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:16:00.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:00.454 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:00.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:00.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:16:00.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:00.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:00.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:00.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:00.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:00.742 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:00.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:00.845 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:00.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:00.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:00.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:00.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:01.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:01.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:01.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:01.068 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 41 16 64036 64020 97.5826 45.5625 0.0179332 0.0102432 2024-09-13T15:16:01.124 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:01.144 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:01.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:01.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:01.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:01.287 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:01.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:01.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:01.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:01.420 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:01.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:01.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:01.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:01.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:01.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:01.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:01.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:01.692 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:01.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:01.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:01.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:01.832 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:01.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:01.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:01.937 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:01.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:01.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:01.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:02.068 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 42 16 64900 64884 96.5448 54 0.0236437 0.0103525 2024-09-13T15:16:02.070 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:02.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:02.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:02.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:02.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:02.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:02.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:02.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:02.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:02.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:02.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:02.412 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:02 smithi099 bash[84576]: cluster 2024-09-13T15:16:01.174207+0000 mgr.y (mgr.26252) 1499 : cluster [DBG] pgmap v2090: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T15:16:02.412 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:02 smithi099 bash[84576]: cluster 2024-09-13T15:16:01.174207+0000 mgr.y (mgr.26252) 1499 : cluster [DBG] pgmap v2090: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T15:16:02.413 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:02 smithi099 bash[87147]: cluster 2024-09-13T15:16:01.174207+0000 mgr.y (mgr.26252) 1499 : cluster [DBG] pgmap v2090: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T15:16:02.413 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:02 smithi099 bash[87147]: cluster 2024-09-13T15:16:01.174207+0000 mgr.y (mgr.26252) 1499 : cluster [DBG] pgmap v2090: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T15:16:02.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:02.488 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:02 smithi177 bash[68594]: cluster 2024-09-13T15:16:01.174207+0000 mgr.y (mgr.26252) 1499 : cluster [DBG] pgmap v2090: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T15:16:02.488 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:02 smithi177 bash[68594]: cluster 2024-09-13T15:16:01.174207+0000 mgr.y (mgr.26252) 1499 : cluster [DBG] pgmap v2090: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T15:16:02.499 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:02.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:02.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:02.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:02.628 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:02.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:02.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:02.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:02.772 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:02.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:02.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:02.877 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:02.902 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:02.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:02.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:03.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:03.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:03.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:03.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:03.068 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 43 16 66059 66043 95.984 72.4375 0.0117707 0.0104131 2024-09-13T15:16:03.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:03.177 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:03.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:03.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:03.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:03.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:03.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:03.424 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:03.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:03.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:03.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:03.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:03.574 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:03.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:03.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:03.677 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:03.697 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:03.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:03.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:03.800 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:03.822 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:03.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:03.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:03.925 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:03.950 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:03.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:03.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:04.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:04.068 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 44 16 66631 66615 94.615 35.75 0.00515625 0.0105636 2024-09-13T15:16:04.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:04.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:04.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:04.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:04.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:04.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:04 smithi177 bash[68594]: audit 2024-09-13T15:16:03.032596+0000 mon.a (mon.0) 2639 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:04 smithi177 bash[68594]: audit 2024-09-13T15:16:03.032596+0000 mon.a (mon.0) 2639 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:04 smithi177 bash[68594]: audit 2024-09-13T15:16:03.039315+0000 mon.a (mon.0) 2640 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:04 smithi177 bash[68594]: audit 2024-09-13T15:16:03.039315+0000 mon.a (mon.0) 2640 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:04 smithi177 bash[68594]: cluster 2024-09-13T15:16:03.174744+0000 mgr.y (mgr.26252) 1500 : cluster [DBG] pgmap v2091: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:16:04.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:04 smithi177 bash[68594]: cluster 2024-09-13T15:16:03.174744+0000 mgr.y (mgr.26252) 1500 : cluster [DBG] pgmap v2091: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:16:04.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:04 smithi177 bash[68594]: audit 2024-09-13T15:16:03.319676+0000 mon.a (mon.0) 2641 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:04 smithi177 bash[68594]: audit 2024-09-13T15:16:03.319676+0000 mon.a (mon.0) 2641 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:04 smithi177 bash[68594]: audit 2024-09-13T15:16:03.320662+0000 mon.a (mon.0) 2642 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:04.231 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:04 smithi177 bash[68594]: audit 2024-09-13T15:16:03.320662+0000 mon.a (mon.0) 2642 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:04.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:04.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:04.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[84576]: audit 2024-09-13T15:16:03.032596+0000 mon.a (mon.0) 2639 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[84576]: audit 2024-09-13T15:16:03.032596+0000 mon.a (mon.0) 2639 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[84576]: audit 2024-09-13T15:16:03.039315+0000 mon.a (mon.0) 2640 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[84576]: audit 2024-09-13T15:16:03.039315+0000 mon.a (mon.0) 2640 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[84576]: cluster 2024-09-13T15:16:03.174744+0000 mgr.y (mgr.26252) 1500 : cluster [DBG] pgmap v2091: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:16:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[84576]: cluster 2024-09-13T15:16:03.174744+0000 mgr.y (mgr.26252) 1500 : cluster [DBG] pgmap v2091: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:16:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[84576]: audit 2024-09-13T15:16:03.319676+0000 mon.a (mon.0) 2641 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[84576]: audit 2024-09-13T15:16:03.319676+0000 mon.a (mon.0) 2641 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[84576]: audit 2024-09-13T15:16:03.320662+0000 mon.a (mon.0) 2642 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[84576]: audit 2024-09-13T15:16:03.320662+0000 mon.a (mon.0) 2642 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[87147]: audit 2024-09-13T15:16:03.032596+0000 mon.a (mon.0) 2639 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[87147]: audit 2024-09-13T15:16:03.032596+0000 mon.a (mon.0) 2639 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[87147]: audit 2024-09-13T15:16:03.039315+0000 mon.a (mon.0) 2640 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[87147]: audit 2024-09-13T15:16:03.039315+0000 mon.a (mon.0) 2640 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[87147]: cluster 2024-09-13T15:16:03.174744+0000 mgr.y (mgr.26252) 1500 : cluster [DBG] pgmap v2091: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:16:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[87147]: cluster 2024-09-13T15:16:03.174744+0000 mgr.y (mgr.26252) 1500 : cluster [DBG] pgmap v2091: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T15:16:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[87147]: audit 2024-09-13T15:16:03.319676+0000 mon.a (mon.0) 2641 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[87147]: audit 2024-09-13T15:16:03.319676+0000 mon.a (mon.0) 2641 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[87147]: audit 2024-09-13T15:16:03.320662+0000 mon.a (mon.0) 2642 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:04 smithi099 bash[87147]: audit 2024-09-13T15:16:03.320662+0000 mon.a (mon.0) 2642 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:04.333 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:04.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:04.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:04.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:04.415 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:16:04.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:16:04.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:04.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:04.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:04.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:04.581 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:16:04.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:04.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:04.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:04.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:04.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:04.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:04.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:04.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:04.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:04.899 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:04.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:04.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:05.005 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:05.068 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 45 16 67660 67644 93.9414 64.3125 0.0036793 0.0106204 2024-09-13T15:16:05.107 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:05.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:05.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:05.210 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:05.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:05.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:05.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:05.330 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:05.353 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:05.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:05.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:05.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:05.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:05.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:05.585 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:05.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:05.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:05.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:05.713 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:05.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:05.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:05.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:05.844 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:05.876 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:05.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:05.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:05.981 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:06.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:06.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:06.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:06.068 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 46 16 68038 68022 92.4127 23.625 0.0115583 0.0108157 2024-09-13T15:16:06.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:06.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:06.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:06.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:06.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:06 smithi099 bash[84576]: cluster 2024-09-13T15:16:05.175500+0000 mgr.y (mgr.26252) 1501 : cluster [DBG] pgmap v2092: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T15:16:06.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:06 smithi099 bash[84576]: cluster 2024-09-13T15:16:05.175500+0000 mgr.y (mgr.26252) 1501 : cluster [DBG] pgmap v2092: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T15:16:06.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:06 smithi099 bash[87147]: cluster 2024-09-13T15:16:05.175500+0000 mgr.y (mgr.26252) 1501 : cluster [DBG] pgmap v2092: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T15:16:06.328 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:06 smithi099 bash[87147]: cluster 2024-09-13T15:16:05.175500+0000 mgr.y (mgr.26252) 1501 : cluster [DBG] pgmap v2092: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T15:16:06.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:06.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:06.392 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:06 smithi177 bash[68594]: cluster 2024-09-13T15:16:05.175500+0000 mgr.y (mgr.26252) 1501 : cluster [DBG] pgmap v2092: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T15:16:06.392 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:06 smithi177 bash[68594]: cluster 2024-09-13T15:16:05.175500+0000 mgr.y (mgr.26252) 1501 : cluster [DBG] pgmap v2092: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2024-09-13T15:16:06.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:06.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:06.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:06.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:06.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:06.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:06.746 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:06.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:06.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:06.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:06.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:06.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:06.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:06.948 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T15:16:06.952 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T15:16:07.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:07.068 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 47 16 69113 69097 91.8759 67.1875 0.0177558 0.0108785 2024-09-13T15:16:07.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:07.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:07.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:07.184 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:07.196 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:16:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:16:07] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:16:07.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:07.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:07.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:07.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:07.346 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:07.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:07.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:07.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:07.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:07.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:07.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:07.582 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:07.615 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:07.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:07.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:07.764 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:07.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:07.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:07.869 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:07.891 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:07.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:07.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:08.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:08.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:08.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:08.068 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 48 16 70162 70146 91.3275 65.5625 0.0141136 0.0109444 2024-09-13T15:16:08.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:08.164 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:08.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:08.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:08.268 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[84576]: audit 2024-09-13T15:16:06.937228+0000 mgr.y (mgr.26252) 1502 : audit [DBG] from='client.36249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:16:08.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[84576]: audit 2024-09-13T15:16:06.937228+0000 mgr.y (mgr.26252) 1502 : audit [DBG] from='client.36249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:16:08.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[84576]: cluster 2024-09-13T15:16:07.175980+0000 mgr.y (mgr.26252) 1503 : cluster [DBG] pgmap v2093: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-13T15:16:08.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[84576]: cluster 2024-09-13T15:16:07.175980+0000 mgr.y (mgr.26252) 1503 : cluster [DBG] pgmap v2093: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-13T15:16:08.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[84576]: audit 2024-09-13T15:16:07.863109+0000 mon.a (mon.0) 2643 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[84576]: audit 2024-09-13T15:16:07.863109+0000 mon.a (mon.0) 2643 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[84576]: audit 2024-09-13T15:16:07.870454+0000 mon.a (mon.0) 2644 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.269 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[84576]: audit 2024-09-13T15:16:07.870454+0000 mon.a (mon.0) 2644 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[87147]: audit 2024-09-13T15:16:06.937228+0000 mgr.y (mgr.26252) 1502 : audit [DBG] from='client.36249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:16:08.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[87147]: audit 2024-09-13T15:16:06.937228+0000 mgr.y (mgr.26252) 1502 : audit [DBG] from='client.36249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:16:08.269 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[87147]: cluster 2024-09-13T15:16:07.175980+0000 mgr.y (mgr.26252) 1503 : cluster [DBG] pgmap v2093: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-13T15:16:08.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[87147]: cluster 2024-09-13T15:16:07.175980+0000 mgr.y (mgr.26252) 1503 : cluster [DBG] pgmap v2093: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-13T15:16:08.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[87147]: audit 2024-09-13T15:16:07.863109+0000 mon.a (mon.0) 2643 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[87147]: audit 2024-09-13T15:16:07.863109+0000 mon.a (mon.0) 2643 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[87147]: audit 2024-09-13T15:16:07.870454+0000 mon.a (mon.0) 2644 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.270 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:08 smithi099 bash[87147]: audit 2024-09-13T15:16:07.870454+0000 mon.a (mon.0) 2644 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:08.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:08.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:08.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:08.406 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:08.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:08.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:08.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:08.526 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":2092,"stamp":"2024-09-13T15:16:05.174929+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4349564877,"num_objects":66386,"num_object_clones":0,"num_object_copies":199158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66386,"num_whiteouts":0,"num_read":332,"num_read_kb":787,"num_write":66572,"num_write_kb":4248699,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":1787554,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":66524,"ondisk_log_size":66524,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18634804,"kb_used_data":12813428,"kb_used_omap":397,"kb_used_meta":5820914,"kb_avail":731326412,"statfs":{"total":767960285184,"available":748878245888,"internally_reserved":0,"allocated":13120950272,"data_stored":13084873017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":406940,"internal_metadata":5960616548},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":627376128,"num_objects":9573,"num_object_clones":0,"num_object_copies":28719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9573,"num_write_kb":612672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003073"},"pg_stats":[{"pgid":"46.f","version":"1105'4185","reported_seq":4199,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:04.933088+0000","last_change":"2024-09-13T15:14:53.230762+0000","last_active":"2024-09-13T15:16:04.933088+0000","last_peered":"2024-09-13T15:16:04.933088+0000","last_clean":"2024-09-13T15:16:04.933088+0000","last_became_active":"2024-09-13T15:14:53.230270+0000","last_became_peered":"2024-09-13T15:14:53.230270+0000","last_unstale":"2024-09-13T15:16:04.933088+0000","last_undegraded":"2024-09-13T15:16:04.933088+0000","last_fullsized":"2024-09-13T15:16:04.933088+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4185,"log_dups_size":0,"ondisk_log_size":4185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:24:33.738184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274268160,"num_objects":4185,"num_object_clones":0,"num_object_copies":12555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4185,"num_write_kb":267840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.e","version":"1105'4061","reported_seq":4084,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:02.832217+0000","last_change":"2024-09-13T15:14:53.225301+0000","last_active":"2024-09-13T15:16:02.832217+0000","last_peered":"2024-09-13T15:16:02.832217+0000","last_clean":"2024-09-13T15:16:02.832217+0000","last_became_active":"2024-09-13T15:14:53.224979+0000","last_became_peered":"2024-09-13T15:14:53.224979+0000","last_unstale":"2024-09-13T15:16:02.832217+0000","last_undegraded":"2024-09-13T15:16:02.832217+0000","last_fullsized":"2024-09-13T15:16:02.832217+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4061,"log_dups_size":0,"ondisk_log_size":4061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:07.690830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266141696,"num_objects":4061,"num_object_clones":0,"num_object_copies":12183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4061,"num_write_kb":259904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"46.d","version":"1105'4158","reported_seq":4181,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:03.312312+0000","last_change":"2024-09-13T15:14:53.236183+0000","last_active":"2024-09-13T15:16:03.312312+0000","last_peered":"2024-09-13T15:16:03.312312+0000","last_clean":"2024-09-13T15:16:03.312312+0000","last_became_active":"2024-09-13T15:14:53.235681+0000","last_became_peered":"2024-09-13T15:14:53.235681+0000","last_unstale":"2024-09-13T15:16:03.312312+0000","last_undegraded":"2024-09-13T15:16:03.312312+0000","last_fullsized":"2024-09-13T15:16:03.312312+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4158,"log_dups_size":0,"ondisk_log_size":4158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:27.855424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272498688,"num_objects":4158,"num_object_clones":0,"num_object_copies":12474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4158,"num_write_kb":266112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.c","version":"1105'4212","reported_seq":4226,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:04.943187+0000","last_change":"2024-09-13T15:14:53.234121+0000","last_active":"2024-09-13T15:16:04.943187+0000","last_peered":"2024-09-13T15:16:04.943187+0000","last_clean":"2024-09-13T15:16:04.943187+0000","last_became_active":"2024-09-13T15:14:53.233879+0000","last_became_peered":"2024-09-13T15:14:53.233879+0000","last_unstale":"2024-09-13T15:16:04.943187+0000","last_undegraded":"2024-09-13T15:16:04.943187+0000","last_fullsized":"2024-09-13T15:16:04.943187+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4212,"log_dups_size":0,"ondisk_log_size":4212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:16:57.111464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276037632,"num_objects":4212,"num_object_clones":0,"num_object_copies":12636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4212,"num_write_kb":269568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.b","version":"1105'4212","reported_seq":4235,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:03.254414+0000","last_change":"2024-09-13T15:14:53.235572+0000","last_active":"2024-09-13T15:16:03.254414+0000","last_peered":"2024-09-13T15:16:03.254414+0000","last_clean":"2024-09-13T15:16:03.254414+0000","last_became_active":"2024-09-13T15:14:53.235203+0000","last_became_peered":"2024-09-13T15:14:53.235203+0000","last_unstale":"2024-09-13T15:16:03.254414+0000","last_undegraded":"2024-09-13T15:16:03.254414+0000","last_fullsized":"2024-09-13T15:16:03.254414+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4212,"log_dups_size":0,"ondisk_log_size":4212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:40.468656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276037632,"num_objects":4212,"num_object_clones":0,"num_object_copies":12636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4212,"num_write_kb":269568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.a","version":"1105'4042","reported_seq":4056,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:00.240189+0000","last_change":"2024-09-13T15:14:53.226603+0000","last_active":"2024-09-13T15:16:00.240189+0000","last_peered":"2024-09-13T15:16:00.240189+0000","last_clean":"2024-09-13T15:16:00.240189+0000","last_became_active":"2024-09-13T15:14:53.226024+0000","last_became_peered":"2024-09-13T15:14:53.226024+0000","last_unstale":"2024-09-13T15:16:00.240189+0000","last_undegraded":"2024-09-13T15:16:00.240189+0000","last_fullsized":"2024-09-13T15:16:00.240189+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4042,"log_dups_size":0,"ondisk_log_size":4042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:24:35.671044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264896512,"num_objects":4042,"num_object_clones":0,"num_object_copies":12126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4042,"num_write_kb":258688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.9","version":"1105'4107","reported_seq":4130,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:03.336253+0000","last_change":"2024-09-13T15:14:53.236042+0000","last_active":"2024-09-13T15:16:03.336253+0000","last_peered":"2024-09-13T15:16:03.336253+0000","last_clean":"2024-09-13T15:16:03.336253+0000","last_became_active":"2024-09-13T15:14:53.235572+0000","last_became_peered":"2024-09-13T15:14:53.235572+0000","last_unstale":"2024-09-13T15:16:03.336253+0000","last_undegraded":"2024-09-13T15:16:03.336253+0000","last_fullsized":"2024-09-13T15:16:03.336253+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4107,"log_dups_size":0,"ondisk_log_size":4107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:03:58.967002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269156352,"num_objects":4107,"num_object_clones":0,"num_object_copies":12321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4107,"num_write_kb":262848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.8","version":"1105'4234","reported_seq":4257,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:04.595415+0000","last_change":"2024-09-13T15:14:53.235231+0000","last_active":"2024-09-13T15:16:04.595415+0000","last_peered":"2024-09-13T15:16:04.595415+0000","last_clean":"2024-09-13T15:16:04.595415+0000","last_became_active":"2024-09-13T15:14:53.234899+0000","last_became_peered":"2024-09-13T15:14:53.234899+0000","last_unstale":"2024-09-13T15:16:04.595415+0000","last_undegraded":"2024-09-13T15:16:04.595415+0000","last_fullsized":"2024-09-13T15:16:04.595415+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4234,"log_dups_size":0,"ondisk_log_size":4234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:14:36.983655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277479424,"num_objects":4234,"num_object_clones":0,"num_object_copies":12702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4234,"num_write_kb":270976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.7","version":"1105'4096","reported_seq":4119,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:02.767719+0000","last_change":"2024-09-13T15:14:53.239551+0000","last_active":"2024-09-13T15:16:02.767719+0000","last_peered":"2024-09-13T15:16:02.767719+0000","last_clean":"2024-09-13T15:16:02.767719+0000","last_became_active":"2024-09-13T15:14:53.239097+0000","last_became_peered":"2024-09-13T15:14:53.239097+0000","last_unstale":"2024-09-13T15:16:02.767719+0000","last_undegraded":"2024-09-13T15:16:02.767719+0000","last_fullsized":"2024-09-13T15:16:02.767719+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4096,"log_dups_size":0,"ondisk_log_size":4096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:38.413699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268435456,"num_objects":4096,"num_object_clones":0,"num_object_copies":12288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4096,"num_write_kb":262144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"46.6","version":"1105'4210","reported_seq":4224,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:04.863673+0000","last_change":"2024-09-13T15:14:53.227517+0000","last_active":"2024-09-13T15:16:04.863673+0000","last_peered":"2024-09-13T15:16:04.863673+0000","last_clean":"2024-09-13T15:16:04.863673+0000","last_became_active":"2024-09-13T15:14:53.227170+0000","last_became_peered":"2024-09-13T15:14:53.227170+0000","last_unstale":"2024-09-13T15:16:04.863673+0000","last_undegraded":"2024-09-13T15:16:04.863673+0000","last_fullsized":"2024-09-13T15:16:04.863673+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4210,"log_dups_size":0,"ondisk_log_size":4210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:11.329111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275906560,"num_objects":4210,"num_object_clones":0,"num_object_copies":12630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4210,"num_write_kb":269440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"46.5","version":"1105'4157","reported_seq":4180,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:04.964857+0000","last_change":"2024-09-13T15:14:53.232412+0000","last_active":"2024-09-13T15:16:04.964857+0000","last_peered":"2024-09-13T15:16:04.964857+0000","last_clean":"2024-09-13T15:16:04.964857+0000","last_became_active":"2024-09-13T15:14:53.231986+0000","last_became_peered":"2024-09-13T15:14:53.231986+0000","last_unstale":"2024-09-13T15:16:04.964857+0000","last_undegraded":"2024-09-13T15:16:04.964857+0000","last_fullsized":"2024-09-13T15:16:04.964857+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4157,"log_dups_size":0,"ondisk_log_size":4157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:10.157866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272433152,"num_objects":4157,"num_object_clones":0,"num_object_copies":12471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4157,"num_write_kb":266048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"46.4","version":"1105'4038","reported_seq":4052,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:00.230784+0000","last_change":"2024-09-13T15:14:53.230142+0000","last_active":"2024-09-13T15:16:00.230784+0000","last_peered":"2024-09-13T15:16:00.230784+0000","last_clean":"2024-09-13T15:16:00.230784+0000","last_became_active":"2024-09-13T15:14:53.229814+0000","last_became_peered":"2024-09-13T15:14:53.229814+0000","last_unstale":"2024-09-13T15:16:00.230784+0000","last_undegraded":"2024-09-13T15:16:00.230784+0000","last_fullsized":"2024-09-13T15:16:00.230784+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4038,"log_dups_size":0,"ondisk_log_size":4038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:58:35.282823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264634368,"num_objects":4038,"num_object_clones":0,"num_object_copies":12114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4038,"num_write_kb":258432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.3","version":"1105'4160","reported_seq":4183,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:04.606999+0000","last_change":"2024-09-13T15:14:53.231627+0000","last_active":"2024-09-13T15:16:04.606999+0000","last_peered":"2024-09-13T15:16:04.606999+0000","last_clean":"2024-09-13T15:16:04.606999+0000","last_became_active":"2024-09-13T15:14:53.231137+0000","last_became_peered":"2024-09-13T15:14:53.231137+0000","last_unstale":"2024-09-13T15:16:04.606999+0000","last_undegraded":"2024-09-13T15:16:04.606999+0000","last_fullsized":"2024-09-13T15:16:04.606999+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4160,"log_dups_size":0,"ondisk_log_size":4160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:51:32.851635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272629760,"num_objects":4160,"num_object_clones":0,"num_object_copies":12480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4160,"num_write_kb":266240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.2","version":"1105'4152","reported_seq":4175,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:03.332827+0000","last_change":"2024-09-13T15:14:53.234530+0000","last_active":"2024-09-13T15:16:03.332827+0000","last_peered":"2024-09-13T15:16:03.332827+0000","last_clean":"2024-09-13T15:16:03.332827+0000","last_became_active":"2024-09-13T15:14:53.234069+0000","last_became_peered":"2024-09-13T15:14:53.234069+0000","last_unstale":"2024-09-13T15:16:03.332827+0000","last_undegraded":"2024-09-13T15:16:03.332827+0000","last_fullsized":"2024-09-13T15:16:03.332827+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4152,"log_dups_size":0,"ondisk_log_size":4152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:00:52.481732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272105472,"num_objects":4152,"num_object_clones":0,"num_object_copies":12456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4152,"num_write_kb":265728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.1","version":"1105'4209","reported_seq":4223,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:04.949159+0000","last_change":"2024-09-13T15:14:53.235408+0000","last_active":"2024-09-13T15:16:04.949159+0000","last_peered":"2024-09-13T15:16:04.949159+0000","last_clean":"2024-09-13T15:16:04.949159+0000","last_became_active":"2024-09-13T15:14:53.234537+0000","last_became_peered":"2024-09-13T15:14:53.234537+0000","last_unstale":"2024-09-13T15:16:04.949159+0000","last_undegraded":"2024-09-13T15:16:04.949159+0000","last_fullsized":"2024-09-13T15:16:04.949159+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4209,"log_dups_size":0,"ondisk_log_size":4209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:45.203591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275841024,"num_objects":4209,"num_object_clones":0,"num_object_copies":12627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4209,"num_write_kb":269376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.0","version":"1105'4127","reported_seq":4150,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:16:02.836057+0000","last_change":"2024-09-13T15:14:53.239483+0000","last_active":"2024-09-13T15:16:02.836057+0000","last_peered":"2024-09-13T15:16:02.836057+0000","last_clean":"2024-09-13T15:16:02.836057+0000","last_became_active":"2024-09-13T15:14:53.239009+0000","last_became_peered":"2024-09-13T15:14:53.239009+0000","last_unstale":"2024-09-13T15:16:02.836057+0000","last_undegraded":"2024-09-13T15:16:02.836057+0000","last_fullsized":"2024-09-13T15:16:02.836057+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":4127,"log_dups_size":0,"ondisk_log_size":4127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:35:02.068622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270467072,"num_objects":4127,"num_object_clones":0,"num_object_copies":12381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4127,"num_write_kb":264128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":117,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.257316+0000","last_change":"2024-09-13T15:08:52.195462+0000","last_active":"2024-09-13T15:15:28.257316+0000","last_peered":"2024-09-13T15:15:28.257316+0000","last_clean":"2024-09-13T15:15:28.257316+0000","last_became_active":"2024-09-13T15:08:52.174050+0000","last_became_peered":"2024-09-13T15:08:52.174050+0000","last_unstale":"2024-09-13T15:15:28.257316+0000","last_undegraded":"2024-09-13T15:15:28.257316+0000","last_fullsized":"2024-09-13T15:15:28.257316+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:58.561367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":343,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.259364+0000","last_change":"2024-09-13T15:08:52.180022+0000","last_active":"2024-09-13T15:15:57.259364+0000","last_peered":"2024-09-13T15:15:57.259364+0000","last_clean":"2024-09-13T15:15:57.259364+0000","last_became_active":"2024-09-13T15:08:52.179546+0000","last_became_peered":"2024-09-13T15:08:52.179546+0000","last_unstale":"2024-09-13T15:15:57.259364+0000","last_undegraded":"2024-09-13T15:15:57.259364+0000","last_fullsized":"2024-09-13T15:15:57.259364+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:15:47.462895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1338,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.246390+0000","last_change":"2024-09-13T15:08:52.175606+0000","last_active":"2024-09-13T15:15:57.246390+0000","last_peered":"2024-09-13T15:15:57.246390+0000","last_clean":"2024-09-13T15:15:57.246390+0000","last_became_active":"2024-09-13T15:08:52.175210+0000","last_became_peered":"2024-09-13T15:08:52.175210+0000","last_unstale":"2024-09-13T15:15:57.246390+0000","last_undegraded":"2024-09-13T15:15:57.246390+0000","last_fullsized":"2024-09-13T15:15:57.246390+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:16.692008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":136,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.253664+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T15:15:28.253664+0000","last_peered":"2024-09-13T15:15:28.253664+0000","last_clean":"2024-09-13T15:15:28.253664+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T15:15:28.253664+0000","last_undegraded":"2024-09-13T15:15:28.253664+0000","last_fullsized":"2024-09-13T15:15:28.253664+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1118,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.257783+0000","last_change":"2024-09-13T15:08:52.205951+0000","last_active":"2024-09-13T15:15:28.257783+0000","last_peered":"2024-09-13T15:15:28.257783+0000","last_clean":"2024-09-13T15:15:28.257783+0000","last_became_active":"2024-09-13T15:08:52.183627+0000","last_became_peered":"2024-09-13T15:08:52.183627+0000","last_unstale":"2024-09-13T15:15:28.257783+0000","last_undegraded":"2024-09-13T15:15:28.257783+0000","last_fullsized":"2024-09-13T15:15:28.257783+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:14:14.282531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":326,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258203+0000","last_change":"2024-09-13T15:05:34.086551+0000","last_active":"2024-09-13T15:15:57.258203+0000","last_peered":"2024-09-13T15:15:57.258203+0000","last_clean":"2024-09-13T15:15:57.258203+0000","last_became_active":"2024-09-13T15:05:34.086077+0000","last_became_peered":"2024-09-13T15:05:34.086077+0000","last_unstale":"2024-09-13T15:15:57.258203+0000","last_undegraded":"2024-09-13T15:15:57.258203+0000","last_fullsized":"2024-09-13T15:15:57.258203+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:33:47.917777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1135,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.256366+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T15:15:28.256366+0000","last_peered":"2024-09-13T15:15:28.256366+0000","last_clean":"2024-09-13T15:15:28.256366+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T15:15:28.256366+0000","last_undegraded":"2024-09-13T15:15:28.256366+0000","last_fullsized":"2024-09-13T15:15:28.256366+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":340,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258504+0000","last_change":"2024-09-13T15:05:34.075607+0000","last_active":"2024-09-13T15:15:57.258504+0000","last_peered":"2024-09-13T15:15:57.258504+0000","last_clean":"2024-09-13T15:15:57.258504+0000","last_became_active":"2024-09-13T15:05:34.075079+0000","last_became_peered":"2024-09-13T15:05:34.075079+0000","last_unstale":"2024-09-13T15:15:57.258504+0000","last_undegraded":"2024-09-13T15:15:57.258504+0000","last_fullsized":"2024-09-13T15:15:57.258504+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:05.384206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1383,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258377+0000","last_change":"2024-09-13T15:07:56.108198+0000","last_active":"2024-09-13T15:15:57.258377+0000","last_peered":"2024-09-13T15:15:57.258377+0000","last_clean":"2024-09-13T15:15:57.258377+0000","last_became_active":"2024-09-13T15:07:56.107824+0000","last_became_peered":"2024-09-13T15:07:56.107824+0000","last_unstale":"2024-09-13T15:15:57.258377+0000","last_undegraded":"2024-09-13T15:15:57.258377+0000","last_fullsized":"2024-09-13T15:15:57.258377+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:19.128127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1531,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.256973+0000","last_change":"2024-09-13T15:05:34.082773+0000","last_active":"2024-09-13T15:15:57.256973+0000","last_peered":"2024-09-13T15:15:57.256973+0000","last_clean":"2024-09-13T15:15:57.256973+0000","last_became_active":"2024-09-13T15:05:34.082408+0000","last_became_peered":"2024-09-13T15:05:34.082408+0000","last_unstale":"2024-09-13T15:15:57.256973+0000","last_undegraded":"2024-09-13T15:15:57.256973+0000","last_fullsized":"2024-09-13T15:15:57.256973+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:13.447546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":329,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258202+0000","last_change":"2024-09-13T14:52:43.597400+0000","last_active":"2024-09-13T15:15:57.258202+0000","last_peered":"2024-09-13T15:15:57.258202+0000","last_clean":"2024-09-13T15:15:57.258202+0000","last_became_active":"2024-09-13T14:52:43.596294+0000","last_became_peered":"2024-09-13T14:52:43.596294+0000","last_unstale":"2024-09-13T15:15:57.258202+0000","last_undegraded":"2024-09-13T15:15:57.258202+0000","last_fullsized":"2024-09-13T15:15:57.258202+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:35.218171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1393,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.246633+0000","last_change":"2024-09-13T15:05:34.070717+0000","last_active":"2024-09-13T15:15:57.246633+0000","last_peered":"2024-09-13T15:15:57.246633+0000","last_clean":"2024-09-13T15:15:57.246633+0000","last_became_active":"2024-09-13T15:05:34.069929+0000","last_became_peered":"2024-09-13T15:05:34.069929+0000","last_unstale":"2024-09-13T15:15:57.246633+0000","last_undegraded":"2024-09-13T15:15:57.246633+0000","last_fullsized":"2024-09-13T15:15:57.246633+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:44:19.208697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":230,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.256861+0000","last_change":"2024-09-13T15:05:34.071133+0000","last_active":"2024-09-13T15:15:57.256861+0000","last_peered":"2024-09-13T15:15:57.256861+0000","last_clean":"2024-09-13T15:15:57.256861+0000","last_became_active":"2024-09-13T15:05:34.070519+0000","last_became_peered":"2024-09-13T15:05:34.070519+0000","last_unstale":"2024-09-13T15:15:57.256861+0000","last_undegraded":"2024-09-13T15:15:57.256861+0000","last_fullsized":"2024-09-13T15:15:57.256861+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:02.617403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1607,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.249952+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T15:15:57.249952+0000","last_peered":"2024-09-13T15:15:57.249952+0000","last_clean":"2024-09-13T15:15:57.249952+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T15:15:57.249952+0000","last_undegraded":"2024-09-13T15:15:57.249952+0000","last_fullsized":"2024-09-13T15:15:57.249952+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1393,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.245556+0000","last_change":"2024-09-13T15:07:56.095947+0000","last_active":"2024-09-13T15:15:57.245556+0000","last_peered":"2024-09-13T15:15:57.245556+0000","last_clean":"2024-09-13T15:15:57.245556+0000","last_became_active":"2024-09-13T15:07:56.094500+0000","last_became_peered":"2024-09-13T15:07:56.094500+0000","last_unstale":"2024-09-13T15:15:57.245556+0000","last_undegraded":"2024-09-13T15:15:57.245556+0000","last_fullsized":"2024-09-13T15:15:57.245556+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:20:31.568566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":34,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.260212+0000","last_change":"2024-09-13T15:08:52.176756+0000","last_active":"2024-09-13T15:15:28.260212+0000","last_peered":"2024-09-13T15:15:28.260212+0000","last_clean":"2024-09-13T15:15:28.260212+0000","last_became_active":"2024-09-13T15:08:52.176280+0000","last_became_peered":"2024-09-13T15:08:52.176280+0000","last_unstale":"2024-09-13T15:15:28.260212+0000","last_undegraded":"2024-09-13T15:15:28.260212+0000","last_fullsized":"2024-09-13T15:15:28.260212+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:41.217586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1380,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.245447+0000","last_change":"2024-09-13T15:08:52.170737+0000","last_active":"2024-09-13T15:15:57.245447+0000","last_peered":"2024-09-13T15:15:57.245447+0000","last_clean":"2024-09-13T15:15:57.245447+0000","last_became_active":"2024-09-13T15:08:52.170217+0000","last_became_peered":"2024-09-13T15:08:52.170217+0000","last_unstale":"2024-09-13T15:15:57.245447+0000","last_undegraded":"2024-09-13T15:15:57.245447+0000","last_fullsized":"2024-09-13T15:15:57.245447+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:17.688936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1608,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.250118+0000","last_change":"2024-09-13T15:05:34.087376+0000","last_active":"2024-09-13T15:15:57.250118+0000","last_peered":"2024-09-13T15:15:57.250118+0000","last_clean":"2024-09-13T15:15:57.250118+0000","last_became_active":"2024-09-13T15:05:34.087046+0000","last_became_peered":"2024-09-13T15:05:34.087046+0000","last_unstale":"2024-09-13T15:15:57.250118+0000","last_undegraded":"2024-09-13T15:15:57.250118+0000","last_fullsized":"2024-09-13T15:15:57.250118+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:47.801431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1320,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.245766+0000","last_change":"2024-09-13T15:08:52.174279+0000","last_active":"2024-09-13T15:15:57.245766+0000","last_peered":"2024-09-13T15:15:57.245766+0000","last_clean":"2024-09-13T15:15:57.245766+0000","last_became_active":"2024-09-13T15:08:52.173877+0000","last_became_peered":"2024-09-13T15:08:52.173877+0000","last_unstale":"2024-09-13T15:15:57.245766+0000","last_undegraded":"2024-09-13T15:15:57.245766+0000","last_fullsized":"2024-09-13T15:15:57.245766+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:16.648765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1079,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.257111+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T15:15:28.257111+0000","last_peered":"2024-09-13T15:15:28.257111+0000","last_clean":"2024-09-13T15:15:28.257111+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T15:15:28.257111+0000","last_undegraded":"2024-09-13T15:15:28.257111+0000","last_fullsized":"2024-09-13T15:15:28.257111+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1133,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.257729+0000","last_change":"2024-09-13T15:05:34.089020+0000","last_active":"2024-09-13T15:15:28.257729+0000","last_peered":"2024-09-13T15:15:28.257729+0000","last_clean":"2024-09-13T15:15:28.257729+0000","last_became_active":"2024-09-13T15:05:34.088440+0000","last_became_peered":"2024-09-13T15:05:34.088440+0000","last_unstale":"2024-09-13T15:15:28.257729+0000","last_undegraded":"2024-09-13T15:15:28.257729+0000","last_fullsized":"2024-09-13T15:15:28.257729+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:24.826746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.249958+0000","last_change":"2024-09-13T15:08:54.446160+0000","last_active":"2024-09-13T15:15:57.249958+0000","last_peered":"2024-09-13T15:15:57.249958+0000","last_clean":"2024-09-13T15:15:57.249958+0000","last_became_active":"2024-09-13T15:08:54.445649+0000","last_became_peered":"2024-09-13T15:08:54.445649+0000","last_unstale":"2024-09-13T15:15:57.249958+0000","last_undegraded":"2024-09-13T15:15:57.249958+0000","last_fullsized":"2024-09-13T15:15:57.249958+0000","mapping_epoch":1082,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1083,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:17.407470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":360,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258804+0000","last_change":"2024-09-13T15:08:52.203370+0000","last_active":"2024-09-13T15:15:57.258804+0000","last_peered":"2024-09-13T15:15:57.258804+0000","last_clean":"2024-09-13T15:15:57.258804+0000","last_became_active":"2024-09-13T15:08:52.179887+0000","last_became_peered":"2024-09-13T15:08:52.179887+0000","last_unstale":"2024-09-13T15:15:57.258804+0000","last_undegraded":"2024-09-13T15:15:57.258804+0000","last_fullsized":"2024-09-13T15:15:57.258804+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:57.293764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":344,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258548+0000","last_change":"2024-09-13T15:08:52.180881+0000","last_active":"2024-09-13T15:15:57.258548+0000","last_peered":"2024-09-13T15:15:57.258548+0000","last_clean":"2024-09-13T15:15:57.258548+0000","last_became_active":"2024-09-13T15:08:52.180000+0000","last_became_peered":"2024-09-13T15:08:52.180000+0000","last_unstale":"2024-09-13T15:15:57.258548+0000","last_undegraded":"2024-09-13T15:15:57.258548+0000","last_fullsized":"2024-09-13T15:15:57.258548+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:32.622289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":331,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258299+0000","last_change":"2024-09-13T15:08:52.188050+0000","last_active":"2024-09-13T15:15:57.258299+0000","last_peered":"2024-09-13T15:15:57.258299+0000","last_clean":"2024-09-13T15:15:57.258299+0000","last_became_active":"2024-09-13T15:08:52.187522+0000","last_became_peered":"2024-09-13T15:08:52.187522+0000","last_unstale":"2024-09-13T15:15:57.258299+0000","last_undegraded":"2024-09-13T15:15:57.258299+0000","last_fullsized":"2024-09-13T15:15:57.258299+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:29.314964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":34,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.260142+0000","last_change":"2024-09-13T15:08:52.187782+0000","last_active":"2024-09-13T15:15:28.260142+0000","last_peered":"2024-09-13T15:15:28.260142+0000","last_clean":"2024-09-13T15:15:28.260142+0000","last_became_active":"2024-09-13T15:08:52.186669+0000","last_became_peered":"2024-09-13T15:08:52.186669+0000","last_unstale":"2024-09-13T15:15:28.260142+0000","last_undegraded":"2024-09-13T15:15:28.260142+0000","last_fullsized":"2024-09-13T15:15:28.260142+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:43.000869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1659,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.250123+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T15:15:57.250123+0000","last_peered":"2024-09-13T15:15:57.250123+0000","last_clean":"2024-09-13T15:15:57.250123+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T15:15:57.250123+0000","last_undegraded":"2024-09-13T15:15:57.250123+0000","last_fullsized":"2024-09-13T15:15:57.250123+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1314,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.245528+0000","last_change":"2024-09-13T15:07:56.109509+0000","last_active":"2024-09-13T15:15:57.245528+0000","last_peered":"2024-09-13T15:15:57.245528+0000","last_clean":"2024-09-13T15:15:57.245528+0000","last_became_active":"2024-09-13T15:07:56.108034+0000","last_became_peered":"2024-09-13T15:07:56.108034+0000","last_unstale":"2024-09-13T15:15:57.245528+0000","last_undegraded":"2024-09-13T15:15:57.245528+0000","last_fullsized":"2024-09-13T15:15:57.245528+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:57:02.300327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":234,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.255564+0000","last_change":"2024-09-13T15:08:52.186362+0000","last_active":"2024-09-13T15:15:57.255564+0000","last_peered":"2024-09-13T15:15:57.255564+0000","last_clean":"2024-09-13T15:15:57.255564+0000","last_became_active":"2024-09-13T15:08:52.185731+0000","last_became_peered":"2024-09-13T15:08:52.185731+0000","last_unstale":"2024-09-13T15:15:57.255564+0000","last_undegraded":"2024-09-13T15:15:57.255564+0000","last_fullsized":"2024-09-13T15:15:57.255564+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:26.408919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1309,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.246466+0000","last_change":"2024-09-13T15:05:34.085917+0000","last_active":"2024-09-13T15:15:57.246466+0000","last_peered":"2024-09-13T15:15:57.246466+0000","last_clean":"2024-09-13T15:15:57.246466+0000","last_became_active":"2024-09-13T15:05:34.085156+0000","last_became_peered":"2024-09-13T15:05:34.085156+0000","last_unstale":"2024-09-13T15:15:57.246466+0000","last_undegraded":"2024-09-13T15:15:57.246466+0000","last_fullsized":"2024-09-13T15:15:57.246466+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:33.482732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1589,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.250297+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T15:15:57.250297+0000","last_peered":"2024-09-13T15:15:57.250297+0000","last_clean":"2024-09-13T15:15:57.250297+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T15:15:57.250297+0000","last_undegraded":"2024-09-13T15:15:57.250297+0000","last_fullsized":"2024-09-13T15:15:57.250297+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1111,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.257814+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T15:15:28.257814+0000","last_peered":"2024-09-13T15:15:28.257814+0000","last_clean":"2024-09-13T15:15:28.257814+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T15:15:28.257814+0000","last_undegraded":"2024-09-13T15:15:28.257814+0000","last_fullsized":"2024-09-13T15:15:28.257814+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":221,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.255042+0000","last_change":"2024-09-13T14:52:43.583860+0000","last_active":"2024-09-13T15:15:57.255042+0000","last_peered":"2024-09-13T15:15:57.255042+0000","last_clean":"2024-09-13T15:15:57.255042+0000","last_became_active":"2024-09-13T14:52:43.583055+0000","last_became_peered":"2024-09-13T14:52:43.583055+0000","last_unstale":"2024-09-13T15:15:57.255042+0000","last_undegraded":"2024-09-13T15:15:57.255042+0000","last_fullsized":"2024-09-13T15:15:57.255042+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:55.113648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":221,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.254822+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T15:15:57.254822+0000","last_peered":"2024-09-13T15:15:57.254822+0000","last_clean":"2024-09-13T15:15:57.254822+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T15:15:57.254822+0000","last_undegraded":"2024-09-13T15:15:57.254822+0000","last_fullsized":"2024-09-13T15:15:57.254822+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":221,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.255212+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T15:15:57.255212+0000","last_peered":"2024-09-13T15:15:57.255212+0000","last_clean":"2024-09-13T15:15:57.255212+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T15:15:57.255212+0000","last_undegraded":"2024-09-13T15:15:57.255212+0000","last_fullsized":"2024-09-13T15:15:57.255212+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1093,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.257836+0000","last_change":"2024-09-13T15:05:34.086758+0000","last_active":"2024-09-13T15:15:28.257836+0000","last_peered":"2024-09-13T15:15:28.257836+0000","last_clean":"2024-09-13T15:15:28.257836+0000","last_became_active":"2024-09-13T15:05:34.086145+0000","last_became_peered":"2024-09-13T15:05:34.086145+0000","last_unstale":"2024-09-13T15:15:28.257836+0000","last_undegraded":"2024-09-13T15:15:28.257836+0000","last_fullsized":"2024-09-13T15:15:28.257836+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:33.764698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":232,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.255123+0000","last_change":"2024-09-13T15:05:34.085898+0000","last_active":"2024-09-13T15:15:57.255123+0000","last_peered":"2024-09-13T15:15:57.255123+0000","last_clean":"2024-09-13T15:15:57.255123+0000","last_became_active":"2024-09-13T15:05:34.085174+0000","last_became_peered":"2024-09-13T15:05:34.085174+0000","last_unstale":"2024-09-13T15:15:57.255123+0000","last_undegraded":"2024-09-13T15:15:57.255123+0000","last_fullsized":"2024-09-13T15:15:57.255123+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:17:39.204192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":331,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258603+0000","last_change":"2024-09-13T15:08:52.188254+0000","last_active":"2024-09-13T15:15:57.258603+0000","last_peered":"2024-09-13T15:15:57.258603+0000","last_clean":"2024-09-13T15:15:57.258603+0000","last_became_active":"2024-09-13T15:08:52.187535+0000","last_became_peered":"2024-09-13T15:08:52.187535+0000","last_unstale":"2024-09-13T15:15:57.258603+0000","last_undegraded":"2024-09-13T15:15:57.258603+0000","last_fullsized":"2024-09-13T15:15:57.258603+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:32.024558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1132,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.253859+0000","last_change":"2024-09-13T15:08:52.209457+0000","last_active":"2024-09-13T15:15:28.253859+0000","last_peered":"2024-09-13T15:15:28.253859+0000","last_clean":"2024-09-13T15:15:28.253859+0000","last_became_active":"2024-09-13T15:08:52.171595+0000","last_became_peered":"2024-09-13T15:08:52.171595+0000","last_unstale":"2024-09-13T15:15:28.253859+0000","last_undegraded":"2024-09-13T15:15:28.253859+0000","last_fullsized":"2024-09-13T15:15:28.253859+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:05:42.960465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":220,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.255705+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T15:15:57.255705+0000","last_peered":"2024-09-13T15:15:57.255705+0000","last_clean":"2024-09-13T15:15:57.255705+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T15:15:57.255705+0000","last_undegraded":"2024-09-13T15:15:57.255705+0000","last_fullsized":"2024-09-13T15:15:57.255705+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":235,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.260740+0000","last_change":"2024-09-13T15:08:52.186074+0000","last_active":"2024-09-13T15:15:57.260740+0000","last_peered":"2024-09-13T15:15:57.260740+0000","last_clean":"2024-09-13T15:15:57.260740+0000","last_became_active":"2024-09-13T15:08:52.185693+0000","last_became_peered":"2024-09-13T15:08:52.185693+0000","last_unstale":"2024-09-13T15:15:57.260740+0000","last_undegraded":"2024-09-13T15:15:57.260740+0000","last_fullsized":"2024-09-13T15:15:57.260740+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:39:06.391608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1508,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.256847+0000","last_change":"2024-09-13T15:05:34.082028+0000","last_active":"2024-09-13T15:15:57.256847+0000","last_peered":"2024-09-13T15:15:57.256847+0000","last_clean":"2024-09-13T15:15:57.256847+0000","last_became_active":"2024-09-13T15:05:34.081220+0000","last_became_peered":"2024-09-13T15:05:34.081220+0000","last_unstale":"2024-09-13T15:15:57.256847+0000","last_undegraded":"2024-09-13T15:15:57.256847+0000","last_fullsized":"2024-09-13T15:15:57.256847+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:12:26.484994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1485,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.255448+0000","last_change":"2024-09-13T15:07:56.111014+0000","last_active":"2024-09-13T15:15:57.255448+0000","last_peered":"2024-09-13T15:15:57.255448+0000","last_clean":"2024-09-13T15:15:57.255448+0000","last_became_active":"2024-09-13T15:07:56.110698+0000","last_became_peered":"2024-09-13T15:07:56.110698+0000","last_unstale":"2024-09-13T15:15:57.255448+0000","last_undegraded":"2024-09-13T15:15:57.255448+0000","last_fullsized":"2024-09-13T15:15:57.255448+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:29.395253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1636,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.250268+0000","last_change":"2024-09-13T15:05:34.087817+0000","last_active":"2024-09-13T15:15:57.250268+0000","last_peered":"2024-09-13T15:15:57.250268+0000","last_clean":"2024-09-13T15:15:57.250268+0000","last_became_active":"2024-09-13T15:05:34.087450+0000","last_became_peered":"2024-09-13T15:05:34.087450+0000","last_unstale":"2024-09-13T15:15:57.250268+0000","last_undegraded":"2024-09-13T15:15:57.250268+0000","last_fullsized":"2024-09-13T15:15:57.250268+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:47.661935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1111,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.255578+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T15:15:28.255578+0000","last_peered":"2024-09-13T15:15:28.255578+0000","last_clean":"2024-09-13T15:15:28.255578+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T15:15:28.255578+0000","last_undegraded":"2024-09-13T15:15:28.255578+0000","last_fullsized":"2024-09-13T15:15:28.255578+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":34,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.259396+0000","last_change":"2024-09-13T15:08:52.188028+0000","last_active":"2024-09-13T15:15:28.259396+0000","last_peered":"2024-09-13T15:15:28.259396+0000","last_clean":"2024-09-13T15:15:28.259396+0000","last_became_active":"2024-09-13T15:08:52.187424+0000","last_became_peered":"2024-09-13T15:08:52.187424+0000","last_unstale":"2024-09-13T15:15:28.259396+0000","last_undegraded":"2024-09-13T15:15:28.259396+0000","last_fullsized":"2024-09-13T15:15:28.259396+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:31.790876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1323,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.247315+0000","last_change":"2024-09-13T15:08:52.170510+0000","last_active":"2024-09-13T15:15:57.247315+0000","last_peered":"2024-09-13T15:15:57.247315+0000","last_clean":"2024-09-13T15:15:57.247315+0000","last_became_active":"2024-09-13T15:08:52.170141+0000","last_became_peered":"2024-09-13T15:08:52.170141+0000","last_unstale":"2024-09-13T15:15:57.247315+0000","last_undegraded":"2024-09-13T15:15:57.247315+0000","last_fullsized":"2024-09-13T15:15:57.247315+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:20.674380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1072,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.254116+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T15:15:28.254116+0000","last_peered":"2024-09-13T15:15:28.254116+0000","last_clean":"2024-09-13T15:15:28.254116+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T15:15:28.254116+0000","last_undegraded":"2024-09-13T15:15:28.254116+0000","last_fullsized":"2024-09-13T15:15:28.254116+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1211,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.255481+0000","last_change":"2024-09-13T15:08:52.214302+0000","last_active":"2024-09-13T15:15:28.255481+0000","last_peered":"2024-09-13T15:15:28.255481+0000","last_clean":"2024-09-13T15:15:28.255481+0000","last_became_active":"2024-09-13T15:08:52.194638+0000","last_became_peered":"2024-09-13T15:08:52.194638+0000","last_unstale":"2024-09-13T15:15:28.255481+0000","last_undegraded":"2024-09-13T15:15:28.255481+0000","last_fullsized":"2024-09-13T15:15:28.255481+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:57.236719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":231,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.255722+0000","last_change":"2024-09-13T15:05:34.078805+0000","last_active":"2024-09-13T15:15:57.255722+0000","last_peered":"2024-09-13T15:15:57.255722+0000","last_clean":"2024-09-13T15:15:57.255722+0000","last_became_active":"2024-09-13T15:05:34.078180+0000","last_became_peered":"2024-09-13T15:05:34.078180+0000","last_unstale":"2024-09-13T15:15:57.255722+0000","last_undegraded":"2024-09-13T15:15:57.255722+0000","last_fullsized":"2024-09-13T15:15:57.255722+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:47.240906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1370,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.245615+0000","last_change":"2024-09-13T15:05:34.076149+0000","last_active":"2024-09-13T15:15:57.245615+0000","last_peered":"2024-09-13T15:15:57.245615+0000","last_clean":"2024-09-13T15:15:57.245615+0000","last_became_active":"2024-09-13T15:05:34.075623+0000","last_became_peered":"2024-09-13T15:05:34.075623+0000","last_unstale":"2024-09-13T15:15:57.245615+0000","last_undegraded":"2024-09-13T15:15:57.245615+0000","last_fullsized":"2024-09-13T15:15:57.245615+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:23.608262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":344,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.259157+0000","last_change":"2024-09-13T15:08:52.189786+0000","last_active":"2024-09-13T15:15:57.259157+0000","last_peered":"2024-09-13T15:15:57.259157+0000","last_clean":"2024-09-13T15:15:57.259157+0000","last_became_active":"2024-09-13T15:08:52.188997+0000","last_became_peered":"2024-09-13T15:08:52.188997+0000","last_unstale":"2024-09-13T15:15:57.259157+0000","last_undegraded":"2024-09-13T15:15:57.259157+0000","last_fullsized":"2024-09-13T15:15:57.259157+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:08.932986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":317,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258203+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T15:15:57.258203+0000","last_peered":"2024-09-13T15:15:57.258203+0000","last_clean":"2024-09-13T15:15:57.258203+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T15:15:57.258203+0000","last_undegraded":"2024-09-13T15:15:57.258203+0000","last_fullsized":"2024-09-13T15:15:57.258203+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":330,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258971+0000","last_change":"2024-09-13T14:52:43.597595+0000","last_active":"2024-09-13T15:15:57.258971+0000","last_peered":"2024-09-13T15:15:57.258971+0000","last_clean":"2024-09-13T15:15:57.258971+0000","last_became_active":"2024-09-13T14:52:43.596592+0000","last_became_peered":"2024-09-13T14:52:43.596592+0000","last_unstale":"2024-09-13T15:15:57.258971+0000","last_undegraded":"2024-09-13T15:15:57.258971+0000","last_fullsized":"2024-09-13T15:15:57.258971+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:56.532603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":34,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.259069+0000","last_change":"2024-09-13T15:08:52.182709+0000","last_active":"2024-09-13T15:15:28.259069+0000","last_peered":"2024-09-13T15:15:28.259069+0000","last_clean":"2024-09-13T15:15:28.259069+0000","last_became_active":"2024-09-13T15:08:52.181906+0000","last_became_peered":"2024-09-13T15:08:52.181906+0000","last_unstale":"2024-09-13T15:15:28.259069+0000","last_undegraded":"2024-09-13T15:15:28.259069+0000","last_fullsized":"2024-09-13T15:15:28.259069+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:36.317662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":341,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258756+0000","last_change":"2024-09-13T15:08:52.182885+0000","last_active":"2024-09-13T15:15:57.258756+0000","last_peered":"2024-09-13T15:15:57.258756+0000","last_clean":"2024-09-13T15:15:57.258756+0000","last_became_active":"2024-09-13T15:08:52.180608+0000","last_became_peered":"2024-09-13T15:08:52.180608+0000","last_unstale":"2024-09-13T15:15:57.258756+0000","last_undegraded":"2024-09-13T15:15:57.258756+0000","last_fullsized":"2024-09-13T15:15:57.258756+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:12.517462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1354,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.245758+0000","last_change":"2024-09-13T15:05:34.085458+0000","last_active":"2024-09-13T15:15:57.245758+0000","last_peered":"2024-09-13T15:15:57.245758+0000","last_clean":"2024-09-13T15:15:57.245758+0000","last_became_active":"2024-09-13T15:05:34.084748+0000","last_became_peered":"2024-09-13T15:05:34.084748+0000","last_unstale":"2024-09-13T15:15:57.245758+0000","last_undegraded":"2024-09-13T15:15:57.245758+0000","last_fullsized":"2024-09-13T15:15:57.245758+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:21.898959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1644,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.250352+0000","last_change":"2024-09-13T14:52:54.185117+0000","last_active":"2024-09-13T15:15:57.250352+0000","last_peered":"2024-09-13T15:15:57.250352+0000","last_clean":"2024-09-13T15:15:57.250352+0000","last_became_active":"2024-09-13T14:52:54.184712+0000","last_became_peered":"2024-09-13T14:52:54.184712+0000","last_unstale":"2024-09-13T15:15:57.250352+0000","last_undegraded":"2024-09-13T15:15:57.250352+0000","last_fullsized":"2024-09-13T15:15:57.250352+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:32.654401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1096,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.255195+0000","last_change":"2024-09-13T15:08:52.175569+0000","last_active":"2024-09-13T15:15:28.255195+0000","last_peered":"2024-09-13T15:15:28.255195+0000","last_clean":"2024-09-13T15:15:28.255195+0000","last_became_active":"2024-09-13T15:08:52.174891+0000","last_became_peered":"2024-09-13T15:08:52.174891+0000","last_unstale":"2024-09-13T15:15:28.255195+0000","last_undegraded":"2024-09-13T15:15:28.255195+0000","last_fullsized":"2024-09-13T15:15:28.255195+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.933115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":34,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.259159+0000","last_change":"2024-09-13T15:08:52.188074+0000","last_active":"2024-09-13T15:15:28.259159+0000","last_peered":"2024-09-13T15:15:28.259159+0000","last_clean":"2024-09-13T15:15:28.259159+0000","last_became_active":"2024-09-13T15:08:52.186649+0000","last_became_peered":"2024-09-13T15:08:52.186649+0000","last_unstale":"2024-09-13T15:15:28.259159+0000","last_undegraded":"2024-09-13T15:15:28.259159+0000","last_fullsized":"2024-09-13T15:15:28.259159+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:23.478250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":1075,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.257828+0000","last_change":"2024-09-13T15:08:52.208388+0000","last_active":"2024-09-13T15:15:28.257828+0000","last_peered":"2024-09-13T15:15:28.257828+0000","last_clean":"2024-09-13T15:15:28.257828+0000","last_became_active":"2024-09-13T15:08:52.192253+0000","last_became_peered":"2024-09-13T15:08:52.192253+0000","last_unstale":"2024-09-13T15:15:28.257828+0000","last_undegraded":"2024-09-13T15:15:28.257828+0000","last_fullsized":"2024-09-13T15:15:28.257828+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:07.508774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":342,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.258373+0000","last_change":"2024-09-13T15:05:34.072278+0000","last_active":"2024-09-13T15:15:57.258373+0000","last_peered":"2024-09-13T15:15:57.258373+0000","last_clean":"2024-09-13T15:15:57.258373+0000","last_became_active":"2024-09-13T15:05:34.071723+0000","last_became_peered":"2024-09-13T15:05:34.071723+0000","last_unstale":"2024-09-13T15:15:57.258373+0000","last_undegraded":"2024-09-13T15:15:57.258373+0000","last_fullsized":"2024-09-13T15:15:57.258373+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:02.130890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1138,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.257393+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T15:15:28.257393+0000","last_peered":"2024-09-13T15:15:28.257393+0000","last_clean":"2024-09-13T15:15:28.257393+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T15:15:28.257393+0000","last_undegraded":"2024-09-13T15:15:28.257393+0000","last_fullsized":"2024-09-13T15:15:28.257393+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1061,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.253653+0000","last_change":"2024-09-13T15:08:52.176830+0000","last_active":"2024-09-13T15:15:28.253653+0000","last_peered":"2024-09-13T15:15:28.253653+0000","last_clean":"2024-09-13T15:15:28.253653+0000","last_became_active":"2024-09-13T15:08:52.176303+0000","last_became_peered":"2024-09-13T15:08:52.176303+0000","last_unstale":"2024-09-13T15:15:28.253653+0000","last_undegraded":"2024-09-13T15:15:28.253653+0000","last_fullsized":"2024-09-13T15:15:28.253653+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:59.553257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1109,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.257382+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T15:15:28.257382+0000","last_peered":"2024-09-13T15:15:28.257382+0000","last_clean":"2024-09-13T15:15:28.257382+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T15:15:28.257382+0000","last_undegraded":"2024-09-13T15:15:28.257382+0000","last_fullsized":"2024-09-13T15:15:28.257382+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":44,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.258904+0000","last_change":"2024-09-13T15:08:52.200695+0000","last_active":"2024-09-13T15:15:28.258904+0000","last_peered":"2024-09-13T15:15:28.258904+0000","last_clean":"2024-09-13T15:15:28.258904+0000","last_became_active":"2024-09-13T15:08:52.183473+0000","last_became_peered":"2024-09-13T15:08:52.183473+0000","last_unstale":"2024-09-13T15:15:28.258904+0000","last_undegraded":"2024-09-13T15:15:28.258904+0000","last_fullsized":"2024-09-13T15:15:28.258904+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:42.810003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1368,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.246306+0000","last_change":"2024-09-13T15:05:34.074069+0000","last_active":"2024-09-13T15:15:57.246306+0000","last_peered":"2024-09-13T15:15:57.246306+0000","last_clean":"2024-09-13T15:15:57.246306+0000","last_became_active":"2024-09-13T15:05:34.073282+0000","last_became_peered":"2024-09-13T15:05:34.073282+0000","last_unstale":"2024-09-13T15:15:57.246306+0000","last_undegraded":"2024-09-13T15:15:57.246306+0000","last_fullsized":"2024-09-13T15:15:57.246306+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:14.307582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1102,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.255584+0000","last_change":"2024-09-13T15:08:52.192368+0000","last_active":"2024-09-13T15:15:28.255584+0000","last_peered":"2024-09-13T15:15:28.255584+0000","last_clean":"2024-09-13T15:15:28.255584+0000","last_became_active":"2024-09-13T15:08:52.191668+0000","last_became_peered":"2024-09-13T15:08:52.191668+0000","last_unstale":"2024-09-13T15:15:28.255584+0000","last_undegraded":"2024-09-13T15:15:28.255584+0000","last_fullsized":"2024-09-13T15:15:28.255584+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:27.410222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1608,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.250031+0000","last_change":"2024-09-13T15:05:34.087636+0000","last_active":"2024-09-13T15:15:57.250031+0000","last_peered":"2024-09-13T15:15:57.250031+0000","last_clean":"2024-09-13T15:15:57.250031+0000","last_became_active":"2024-09-13T15:05:34.087255+0000","last_became_peered":"2024-09-13T15:05:34.087255+0000","last_unstale":"2024-09-13T15:15:57.250031+0000","last_undegraded":"2024-09-13T15:15:57.250031+0000","last_fullsized":"2024-09-13T15:15:57.250031+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:36.313581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1609,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.250067+0000","last_change":"2024-09-13T15:05:34.089162+0000","last_active":"2024-09-13T15:15:57.250067+0000","last_peered":"2024-09-13T15:15:57.250067+0000","last_clean":"2024-09-13T15:15:57.250067+0000","last_became_active":"2024-09-13T15:05:34.088727+0000","last_became_peered":"2024-09-13T15:05:34.088727+0000","last_unstale":"2024-09-13T15:15:57.250067+0000","last_undegraded":"2024-09-13T15:15:57.250067+0000","last_fullsized":"2024-09-13T15:15:57.250067+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:29.209082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1075,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.257699+0000","last_change":"2024-09-13T15:05:34.073363+0000","last_active":"2024-09-13T15:15:28.257699+0000","last_peered":"2024-09-13T15:15:28.257699+0000","last_clean":"2024-09-13T15:15:28.257699+0000","last_became_active":"2024-09-13T15:05:34.072906+0000","last_became_peered":"2024-09-13T15:05:34.072906+0000","last_unstale":"2024-09-13T15:15:28.257699+0000","last_undegraded":"2024-09-13T15:15:28.257699+0000","last_fullsized":"2024-09-13T15:15:28.257699+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:27.966654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1115,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-13T15:15:28.256709+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T15:15:28.256709+0000","last_peered":"2024-09-13T15:15:28.256709+0000","last_clean":"2024-09-13T15:15:28.256709+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T15:15:28.256709+0000","last_undegraded":"2024-09-13T15:15:28.256709+0000","last_fullsized":"2024-09-13T15:15:28.256709+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1379,"reported_epoch":1105,"state":"active+clean","last_fresh":"2024-09-13T15:15:57.248773+0000","last_change":"2024-09-13T15:05:34.075192+0000","last_active":"2024-09-13T15:15:57.248773+0000","last_peered":"2024-09-13T15:15:57.248773+0000","last_clean":"2024-09-13T15:15:57.248773+0000","last_became_active":"2024-09-13T15:05:34.073315+0000","last_became_peered":"2024-09-13T15:05:34.073315+0000","last_unstale":"2024-09-13T15:15:57.248773+0000","last_undegraded":"2024-09-13T15:15:57.248773+0000","last_fullsized":"2024-09-13T15:15:57.248773+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:38.699775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":46,"num_pg":16,"stat_sum":{"num_bytes":4348968960,"num_objects":66360,"num_object_clones":0,"num_object_copies":199080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":66360,"num_write_kb":4247040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13084852224,"data_stored":13084852224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":66360,"ondisk_log_size":66360,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":26,"num_read_kb":28,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":16374,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398179,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2181880,"kb_used_data":1289968,"kb_used_omap":1,"kb_used_meta":891902,"kb_avail":91563272,"statfs":{"total":95995035648,"available":93760790528,"internally_reserved":0,"allocated":1320927232,"data_stored":1316403130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913307992},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594415,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2785332,"kb_used_data":1880876,"kb_used_omap":1,"kb_used_meta":904446,"kb_avail":90959820,"statfs":{"total":95995035648,"available":93142855680,"internally_reserved":0,"allocated":1926017024,"data_stored":1921497453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":926153048},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790649,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3308964,"kb_used_data":2414556,"kb_used_omap":1,"kb_used_meta":894398,"kb_avail":90436188,"statfs":{"total":95995035648,"available":92606656512,"internally_reserved":0,"allocated":2472505344,"data_stored":2467996861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915863896},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980832177,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1174448,"kb_used_data":526888,"kb_used_omap":76,"kb_used_meta":647475,"kb_avail":92570704,"statfs":{"total":95995035648,"available":94792400896,"internally_reserved":0,"allocated":539533312,"data_stored":535016800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":663015232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1066,"seq":4578435137663,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1611804,"kb_used_data":1355988,"kb_used_omap":121,"kb_used_meta":255686,"kb_avail":92133348,"statfs":{"total":95995035648,"available":94344548352,"internally_reserved":0,"allocated":1388531712,"data_stored":1384030888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124093,"internal_metadata":261823299},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023359,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2374752,"kb_used_data":1596952,"kb_used_omap":71,"kb_used_meta":777720,"kb_avail":91370400,"statfs":{"total":95995035648,"available":93563289600,"internally_reserved":0,"allocated":1635278848,"data_stored":1630774450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":796385952},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004489,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2249628,"kb_used_data":1610708,"kb_used_omap":66,"kb_used_meta":638845,"kb_avail":91495524,"statfs":{"total":95995035648,"available":93691416576,"internally_reserved":0,"allocated":1649364992,"data_stored":1644864097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":654177450},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195681,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2947996,"kb_used_data":2137492,"kb_used_omap":56,"kb_used_meta":810439,"kb_avail":90797156,"statfs":{"total":95995035648,"available":92976287744,"internally_reserved":0,"allocated":2188791808,"data_stored":2184289338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":829889679},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2176188416,"data_stored":2176188416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1652293632,"data_stored":1652293632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1627979776,"data_stored":1627979776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1382285312,"data_stored":1382285312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":529006592,"data_stored":529006592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2482307072,"data_stored":2482307072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1926299648,"data_stored":1926299648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1308491776,"data_stored":1308491776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:16:08.530 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T15:16:08.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_5 pg_num 26 2024-09-13T15:16:08.537 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:08 smithi177 bash[68594]: audit 2024-09-13T15:16:06.937228+0000 mgr.y (mgr.26252) 1502 : audit [DBG] from='client.36249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:16:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:08 smithi177 bash[68594]: audit 2024-09-13T15:16:06.937228+0000 mgr.y (mgr.26252) 1502 : audit [DBG] from='client.36249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:16:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:08 smithi177 bash[68594]: cluster 2024-09-13T15:16:07.175980+0000 mgr.y (mgr.26252) 1503 : cluster [DBG] pgmap v2093: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-13T15:16:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:08 smithi177 bash[68594]: cluster 2024-09-13T15:16:07.175980+0000 mgr.y (mgr.26252) 1503 : cluster [DBG] pgmap v2093: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-13T15:16:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:08 smithi177 bash[68594]: audit 2024-09-13T15:16:07.863109+0000 mon.a (mon.0) 2643 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:08 smithi177 bash[68594]: audit 2024-09-13T15:16:07.863109+0000 mon.a (mon.0) 2643 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:08 smithi177 bash[68594]: audit 2024-09-13T15:16:07.870454+0000 mon.a (mon.0) 2644 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:08 smithi177 bash[68594]: audit 2024-09-13T15:16:07.870454+0000 mon.a (mon.0) 2644 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:08.615 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:08.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:08.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:08.721 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:08.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:08.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:08.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:08.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:08.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:08.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:08.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:08.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:09.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:09.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:09.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:09.069 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 49 16 71210 71194 90.8003 65.5 0.0136804 0.0110087 2024-09-13T15:16:09.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:09.156 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:09.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:09.259 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:09.292 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:09.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:09.398 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:09.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:09.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:09.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:09.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:09.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:09 smithi177 bash[68594]: audit 2024-09-13T15:16:08.848098+0000 mon.a (mon.0) 2645 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:16:09.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:09 smithi177 bash[68594]: audit 2024-09-13T15:16:08.848098+0000 mon.a (mon.0) 2645 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:16:09.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:09 smithi177 bash[68594]: audit 2024-09-13T15:16:08.849350+0000 mon.a (mon.0) 2646 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:16:09.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:09 smithi177 bash[68594]: audit 2024-09-13T15:16:08.849350+0000 mon.a (mon.0) 2646 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:16:09.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:09 smithi177 bash[68594]: audit 2024-09-13T15:16:08.857110+0000 mon.a (mon.0) 2647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:09.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:09 smithi177 bash[68594]: audit 2024-09-13T15:16:08.857110+0000 mon.a (mon.0) 2647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:09.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:09.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:09.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[84576]: audit 2024-09-13T15:16:08.848098+0000 mon.a (mon.0) 2645 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:16:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[84576]: audit 2024-09-13T15:16:08.848098+0000 mon.a (mon.0) 2645 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:16:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[84576]: audit 2024-09-13T15:16:08.849350+0000 mon.a (mon.0) 2646 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:16:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[84576]: audit 2024-09-13T15:16:08.849350+0000 mon.a (mon.0) 2646 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:16:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[84576]: audit 2024-09-13T15:16:08.857110+0000 mon.a (mon.0) 2647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[84576]: audit 2024-09-13T15:16:08.857110+0000 mon.a (mon.0) 2647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[87147]: audit 2024-09-13T15:16:08.848098+0000 mon.a (mon.0) 2645 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:16:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[87147]: audit 2024-09-13T15:16:08.848098+0000 mon.a (mon.0) 2645 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:16:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[87147]: audit 2024-09-13T15:16:08.849350+0000 mon.a (mon.0) 2646 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:16:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[87147]: audit 2024-09-13T15:16:08.849350+0000 mon.a (mon.0) 2646 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:16:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[87147]: audit 2024-09-13T15:16:08.857110+0000 mon.a (mon.0) 2647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:09 smithi099 bash[87147]: audit 2024-09-13T15:16:08.857110+0000 mon.a (mon.0) 2647 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:09.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:09.690 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:09.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:09.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:09.795 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:09.827 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:09.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:09.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:09.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:09.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:10.009 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:16:10.009 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:16:10.069 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 50 16 72424 72408 90.5017 75.875 0.00362598 0.0110437 2024-09-13T15:16:10.071 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:10.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:10.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:10.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:10.206 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:10.250 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:10.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:10.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:10.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:10 smithi177 bash[68594]: cluster 2024-09-13T15:16:09.176444+0000 mgr.y (mgr.26252) 1504 : cluster [DBG] pgmap v2094: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T15:16:10.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:10 smithi177 bash[68594]: cluster 2024-09-13T15:16:09.176444+0000 mgr.y (mgr.26252) 1504 : cluster [DBG] pgmap v2094: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T15:16:10.363 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:16:10 smithi177 bash[61439]: ts=2024-09-13T15:16:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:16:10.381 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:10.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:10.484 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:10.508 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:10.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:10.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:10.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:10 smithi099 bash[84576]: cluster 2024-09-13T15:16:09.176444+0000 mgr.y (mgr.26252) 1504 : cluster [DBG] pgmap v2094: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T15:16:10.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:10 smithi099 bash[84576]: cluster 2024-09-13T15:16:09.176444+0000 mgr.y (mgr.26252) 1504 : cluster [DBG] pgmap v2094: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T15:16:10.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:10 smithi099 bash[87147]: cluster 2024-09-13T15:16:09.176444+0000 mgr.y (mgr.26252) 1504 : cluster [DBG] pgmap v2094: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T15:16:10.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:10 smithi099 bash[87147]: cluster 2024-09-13T15:16:09.176444+0000 mgr.y (mgr.26252) 1504 : cluster [DBG] pgmap v2094: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T15:16:10.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:10.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:10.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:10.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:10.758 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:10.786 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:10.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:10.890 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:16:10.890 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:16:10.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:10.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:10.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:10.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:11.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:11.052 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:11.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:11.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:11.070 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 51 16 73549 73533 90.1023 70.3125 0.00590096 0.0110935 2024-09-13T15:16:11.156 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:11.178 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:11.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:11.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:11.281 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:11.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:11.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:11.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:11.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:11.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:11.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:11.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:11.557 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:11.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:11.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:11.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:11.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:11.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:11.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:11.792 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:11.820 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:11.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:11.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:11.925 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:11.951 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:11.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:11.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:12.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:12.071 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 52 16 74046 74030 88.9669 31.0625 0.0167698 0.0112342 2024-09-13T15:16:12.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:12.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:12.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:12.123 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:16:12.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:12.248 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:12.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:12.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:12.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:12.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:12.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:12.483 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:12.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:12 smithi177 bash[68594]: cluster 2024-09-13T15:16:11.176979+0000 mgr.y (mgr.26252) 1505 : cluster [DBG] pgmap v2095: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-09-13T15:16:12.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:12 smithi177 bash[68594]: cluster 2024-09-13T15:16:11.176979+0000 mgr.y (mgr.26252) 1505 : cluster [DBG] pgmap v2095: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-09-13T15:16:12.512 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:12.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:12.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:12 smithi099 bash[84576]: cluster 2024-09-13T15:16:11.176979+0000 mgr.y (mgr.26252) 1505 : cluster [DBG] pgmap v2095: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-09-13T15:16:12.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:12 smithi099 bash[84576]: cluster 2024-09-13T15:16:11.176979+0000 mgr.y (mgr.26252) 1505 : cluster [DBG] pgmap v2095: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-09-13T15:16:12.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:12 smithi099 bash[87147]: cluster 2024-09-13T15:16:11.176979+0000 mgr.y (mgr.26252) 1505 : cluster [DBG] pgmap v2095: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-09-13T15:16:12.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:12 smithi099 bash[87147]: cluster 2024-09-13T15:16:11.176979+0000 mgr.y (mgr.26252) 1505 : cluster [DBG] pgmap v2095: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-09-13T15:16:12.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:12.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:12.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:12.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:12.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:12.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:12.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:12.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:12.905 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:12.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:12.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:13.009 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:13.033 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:13.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:13.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:13.071 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 53 16 75081 75065 88.5087 64.6875 0.0109276 0.0112932 2024-09-13T15:16:13.076 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:16:13.138 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:13.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:13.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:13.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:13.273 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:13.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:13.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:13.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:13.437 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:13.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:13.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:13.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:13.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:13.571 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:13.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:13.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:13.700 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:13.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:13.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:13.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:13.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:13.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:13.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:13.934 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:13.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:13.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:14.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:14.071 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 54 16 76211 76195 88.1774 70.625 0.00626308 0.0113352 2024-09-13T15:16:14.090 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:14.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:14.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:14.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:14.221 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:14.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:14.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:14.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:14.352 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:14 smithi099 bash[84576]: cluster 2024-09-13T15:16:13.177442+0000 mgr.y (mgr.26252) 1506 : cluster [DBG] pgmap v2096: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2024-09-13T15:16:14.352 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:14 smithi099 bash[84576]: cluster 2024-09-13T15:16:13.177442+0000 mgr.y (mgr.26252) 1506 : cluster [DBG] pgmap v2096: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2024-09-13T15:16:14.352 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:14 smithi099 bash[87147]: cluster 2024-09-13T15:16:13.177442+0000 mgr.y (mgr.26252) 1506 : cluster [DBG] pgmap v2096: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2024-09-13T15:16:14.352 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:14 smithi099 bash[87147]: cluster 2024-09-13T15:16:13.177442+0000 mgr.y (mgr.26252) 1506 : cluster [DBG] pgmap v2096: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2024-09-13T15:16:14.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:14.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:14 smithi177 bash[68594]: cluster 2024-09-13T15:16:13.177442+0000 mgr.y (mgr.26252) 1506 : cluster [DBG] pgmap v2096: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2024-09-13T15:16:14.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:14 smithi177 bash[68594]: cluster 2024-09-13T15:16:13.177442+0000 mgr.y (mgr.26252) 1506 : cluster [DBG] pgmap v2096: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2024-09-13T15:16:14.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:14.478 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:14.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:14.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:14.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:14.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:14.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:14.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:14.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:14.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:14.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:14.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:14.838 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:14.855 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:14.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:14.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:14.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:14.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:14.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:15.071 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 55 16 76855 76839 87.3059 40.25 0.00611838 0.0114484 2024-09-13T15:16:15.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:15.123 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:15.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:15.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:15.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:15.264 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:15.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:15.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:15.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:15 smithi099 bash[84576]: cluster 2024-09-13T15:16:15.178077+0000 mgr.y (mgr.26252) 1507 : cluster [DBG] pgmap v2097: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:16:15.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:15 smithi099 bash[84576]: cluster 2024-09-13T15:16:15.178077+0000 mgr.y (mgr.26252) 1507 : cluster [DBG] pgmap v2097: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:16:15.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:15 smithi099 bash[87147]: cluster 2024-09-13T15:16:15.178077+0000 mgr.y (mgr.26252) 1507 : cluster [DBG] pgmap v2097: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:16:15.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:15 smithi099 bash[87147]: cluster 2024-09-13T15:16:15.178077+0000 mgr.y (mgr.26252) 1507 : cluster [DBG] pgmap v2097: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:16:15.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:15 smithi177 bash[68594]: cluster 2024-09-13T15:16:15.178077+0000 mgr.y (mgr.26252) 1507 : cluster [DBG] pgmap v2097: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:16:15.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:15 smithi177 bash[68594]: cluster 2024-09-13T15:16:15.178077+0000 mgr.y (mgr.26252) 1507 : cluster [DBG] pgmap v2097: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:16:15.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:15.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:15.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:15.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:15.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:15.520 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:15.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:15.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:15.626 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:15.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:15.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:15.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:15.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:15.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:15.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:15.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:15.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:15.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:15.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:15.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:16.025 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:16.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:16.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:16.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:16.071 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 56 16 77776 77760 86.7747 57.5625 0.0223016 0.011519 2024-09-13T15:16:16.164 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:16.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:16.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:16.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:16.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:16.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:16.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:16.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:16.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:16.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:16.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:16.590 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:16.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:16.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:16.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:16.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:16.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:16.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:16.831 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:16.867 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:16.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:16.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:16.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:17.007 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:16:17.018 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:17.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:17.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:17.071 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 57 15 78548 78533 86.0999 48.3125 0.00884396 0.011609 2024-09-13T15:16:17.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:17.164 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:17.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:17.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:17.269 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:17.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:17.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:17.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:17.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:16:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:16:17] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:16:17.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:17.442 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:17.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:17.543 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:17.575 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:17.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:17.679 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:17.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:17.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:17.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:17.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:17.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:17.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:17.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:17.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:17.956 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:17.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:17.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:18.059 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:18.071 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 58 16 79230 79214 85.3492 42.5625 0.00372519 0.011639 2024-09-13T15:16:18.085 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:18.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:18.187 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:18.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:18.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:18.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:18.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:18.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:18.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:18.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:18.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:18.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:18.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:18.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:18.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:18.600 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:18.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:18.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:18.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:18 smithi099 bash[84576]: cluster 2024-09-13T15:16:17.178584+0000 mgr.y (mgr.26252) 1508 : cluster [DBG] pgmap v2098: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2024-09-13T15:16:18.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:18 smithi099 bash[84576]: cluster 2024-09-13T15:16:17.178584+0000 mgr.y (mgr.26252) 1508 : cluster [DBG] pgmap v2098: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2024-09-13T15:16:18.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:18 smithi099 bash[87147]: cluster 2024-09-13T15:16:17.178584+0000 mgr.y (mgr.26252) 1508 : cluster [DBG] pgmap v2098: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2024-09-13T15:16:18.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:18 smithi099 bash[87147]: cluster 2024-09-13T15:16:17.178584+0000 mgr.y (mgr.26252) 1508 : cluster [DBG] pgmap v2098: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2024-09-13T15:16:18.705 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:18.731 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:18.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:18.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:18.835 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:18.847 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:18 smithi177 bash[68594]: cluster 2024-09-13T15:16:17.178584+0000 mgr.y (mgr.26252) 1508 : cluster [DBG] pgmap v2098: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2024-09-13T15:16:18.847 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:18 smithi177 bash[68594]: cluster 2024-09-13T15:16:17.178584+0000 mgr.y (mgr.26252) 1508 : cluster [DBG] pgmap v2098: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2024-09-13T15:16:18.862 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:18.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:18.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:18.965 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:18.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:18.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:18.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:19.071 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 59 16 79740 79724 84.4429 31.875 0.0197411 0.0118365 2024-09-13T15:16:19.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:19.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:19.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:19.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:19.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:19.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:19.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:19.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:19.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:19 smithi177 bash[68594]: audit 2024-09-13T15:16:18.348675+0000 mon.a (mon.0) 2648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:19.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:19 smithi177 bash[68594]: audit 2024-09-13T15:16:18.348675+0000 mon.a (mon.0) 2648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:19.373 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:19.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:19.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:19.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:19.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:19.533 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:19.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:19 smithi099 bash[84576]: audit 2024-09-13T15:16:18.348675+0000 mon.a (mon.0) 2648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:19.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:19 smithi099 bash[84576]: audit 2024-09-13T15:16:18.348675+0000 mon.a (mon.0) 2648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:19 smithi099 bash[87147]: audit 2024-09-13T15:16:18.348675+0000 mon.a (mon.0) 2648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:19.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:19 smithi099 bash[87147]: audit 2024-09-13T15:16:18.348675+0000 mon.a (mon.0) 2648 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:19.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:19.674 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:19.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:19.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:19.779 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:19.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:19.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:19.948 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:19.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:19.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:20.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:20.070 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:20.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:20.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:16:20.072232+0000 min lat: 0.00120222 max lat: 0.869931 avg lat: 0.0118899 2024-09-13T15:16:20.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:16:20.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 60 16 80706 80690 84.0417 60.375 0.004558 0.0118899 2024-09-13T15:16:20.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:20.197 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:20.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:20.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:20.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:20.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:20.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:20.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:20.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:20.440 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:20 smithi099 bash[84576]: cluster 2024-09-13T15:16:19.179066+0000 mgr.y (mgr.26252) 1509 : cluster [DBG] pgmap v2099: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:16:20.440 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:20 smithi099 bash[84576]: cluster 2024-09-13T15:16:19.179066+0000 mgr.y (mgr.26252) 1509 : cluster [DBG] pgmap v2099: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:16:20.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:20 smithi099 bash[87147]: cluster 2024-09-13T15:16:19.179066+0000 mgr.y (mgr.26252) 1509 : cluster [DBG] pgmap v2099: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:16:20.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:20 smithi099 bash[87147]: cluster 2024-09-13T15:16:19.179066+0000 mgr.y (mgr.26252) 1509 : cluster [DBG] pgmap v2099: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:16:20.455 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:20.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:20.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:20.558 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:20.571 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:20 smithi177 bash[68594]: cluster 2024-09-13T15:16:19.179066+0000 mgr.y (mgr.26252) 1509 : cluster [DBG] pgmap v2099: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:16:20.571 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:20 smithi177 bash[68594]: cluster 2024-09-13T15:16:19.179066+0000 mgr.y (mgr.26252) 1509 : cluster [DBG] pgmap v2099: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:16:20.572 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:16:20 smithi177 bash[61439]: ts=2024-09-13T15:16:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:16:20.585 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:20.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:20.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:20.688 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:20.715 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:20.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:20.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:20.823 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:20.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:20.858 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:20.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:20.959 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:20.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:20.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:20.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:21.071 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 61 16 81201 81185 83.1711 30.9375 0.0179086 0.0120181 2024-09-13T15:16:21.091 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:21.119 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:21.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:21.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:21.223 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:21.260 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:21.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:21.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:21.368 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:21.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:21.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:21.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:21.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:21.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:21.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:21.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:21.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:21.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:21.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:21.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:21.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:21.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:21.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:21.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:21.929 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:21.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:21.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:22.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:22.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:22.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:22.071 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 62 16 82284 82268 82.9212 67.6875 0.0135874 0.0120547 2024-09-13T15:16:22.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:22.177 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:22.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:22.279 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:22.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:22.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:22.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:22 smithi177 bash[68594]: cluster 2024-09-13T15:16:21.181199+0000 mgr.y (mgr.26252) 1510 : cluster [DBG] pgmap v2100: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T15:16:22.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:22 smithi177 bash[68594]: cluster 2024-09-13T15:16:21.181199+0000 mgr.y (mgr.26252) 1510 : cluster [DBG] pgmap v2100: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T15:16:22.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:22.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:22.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:22.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:22.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:22 smithi099 bash[84576]: cluster 2024-09-13T15:16:21.181199+0000 mgr.y (mgr.26252) 1510 : cluster [DBG] pgmap v2100: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T15:16:22.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:22 smithi099 bash[84576]: cluster 2024-09-13T15:16:21.181199+0000 mgr.y (mgr.26252) 1510 : cluster [DBG] pgmap v2100: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T15:16:22.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:22 smithi099 bash[87147]: cluster 2024-09-13T15:16:21.181199+0000 mgr.y (mgr.26252) 1510 : cluster [DBG] pgmap v2100: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T15:16:22.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:22 smithi099 bash[87147]: cluster 2024-09-13T15:16:21.181199+0000 mgr.y (mgr.26252) 1510 : cluster [DBG] pgmap v2100: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-13T15:16:22.578 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:22.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:22.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:22.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:22.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:22.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:22.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:22.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:22.830 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:22.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:22.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:22.966 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:22.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:22.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:23.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 63 16 82989 82973 82.3044 44.0625 0.00772532 0.0121448 2024-09-13T15:16:23.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:23.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:23.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:23.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:23.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:23.226 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:16:23.226 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:16:23.240 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:23.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:23.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:23.347 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:23.371 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:23.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:23.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:23.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:23.494 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:23.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:23.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:23.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:23.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:23.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:23.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:23.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:23.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:23.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:23.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:23.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:23.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:23.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:23.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:23.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:24.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:24.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:24.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:24.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 64 16 83655 83639 81.6688 41.625 0.016784 0.0122391 2024-09-13T15:16:24.114 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:24.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:24.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:24.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:24.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:24.288 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:24.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:24.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:24.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:24 smithi177 bash[68594]: cluster 2024-09-13T15:16:23.181649+0000 mgr.y (mgr.26252) 1511 : cluster [DBG] pgmap v2101: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:16:24.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:24 smithi177 bash[68594]: cluster 2024-09-13T15:16:23.181649+0000 mgr.y (mgr.26252) 1511 : cluster [DBG] pgmap v2101: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:16:24.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:24.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:24.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:24.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:24.484 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:16:24.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:24.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:24.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:24.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:24.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:24 smithi099 bash[84576]: cluster 2024-09-13T15:16:23.181649+0000 mgr.y (mgr.26252) 1511 : cluster [DBG] pgmap v2101: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:16:24.590 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:24 smithi099 bash[84576]: cluster 2024-09-13T15:16:23.181649+0000 mgr.y (mgr.26252) 1511 : cluster [DBG] pgmap v2101: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:16:24.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:24 smithi099 bash[87147]: cluster 2024-09-13T15:16:23.181649+0000 mgr.y (mgr.26252) 1511 : cluster [DBG] pgmap v2101: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:16:24.591 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:24 smithi099 bash[87147]: cluster 2024-09-13T15:16:23.181649+0000 mgr.y (mgr.26252) 1511 : cluster [DBG] pgmap v2101: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2024-09-13T15:16:24.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:24.677 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:24.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:24.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:24.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:24.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:24.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:24.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:24.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:24.962 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:24.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:24.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:25.067 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:25.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 65 16 84556 84540 81.2786 56.3125 0.0129568 0.0122842 2024-09-13T15:16:25.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:25.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:25.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:25.216 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:25.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:25.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:25.322 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:25.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:25.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:25.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:25.455 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:25.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:25.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:25.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:25.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:25 smithi099 bash[84576]: cluster 2024-09-13T15:16:25.182237+0000 mgr.y (mgr.26252) 1512 : cluster [DBG] pgmap v2102: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T15:16:25.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:25 smithi099 bash[84576]: cluster 2024-09-13T15:16:25.182237+0000 mgr.y (mgr.26252) 1512 : cluster [DBG] pgmap v2102: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T15:16:25.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:25 smithi099 bash[87147]: cluster 2024-09-13T15:16:25.182237+0000 mgr.y (mgr.26252) 1512 : cluster [DBG] pgmap v2102: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T15:16:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:25 smithi099 bash[87147]: cluster 2024-09-13T15:16:25.182237+0000 mgr.y (mgr.26252) 1512 : cluster [DBG] pgmap v2102: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T15:16:25.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:25.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:25 smithi177 bash[68594]: cluster 2024-09-13T15:16:25.182237+0000 mgr.y (mgr.26252) 1512 : cluster [DBG] pgmap v2102: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T15:16:25.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:25 smithi177 bash[68594]: cluster 2024-09-13T15:16:25.182237+0000 mgr.y (mgr.26252) 1512 : cluster [DBG] pgmap v2102: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T15:16:25.631 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:25.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:25.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:25.764 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:25.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:25.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:25.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:25.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:25.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:25.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:26.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:26.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:26.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:26.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:26.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 66 16 85612 85596 81.047 66 0.0158009 0.0123333 2024-09-13T15:16:26.136 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:26.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:26.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:26.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:26.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:26.303 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:26.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:26.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:26.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:26.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:26.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:26.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:26.548 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:26.579 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:26.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:26.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:26.686 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:26.705 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:26.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:26.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:26.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:26.832 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:26.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:26.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:26.935 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:26.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:26.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:26.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:27.059 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:27.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 67 16 86309 86293 80.4875 43.5625 0.00409199 0.0123938 2024-09-13T15:16:27.082 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:27.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:27.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:27.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:27.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:27.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:27.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:27.249 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:16:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:16:27] "GET /metrics HTTP/1.1" 200 36777 "" "Prometheus/2.51.0" 2024-09-13T15:16:27.312 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:27.348 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:27.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:27.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:27.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:27.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:27.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:27.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:27.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:27.608 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:27.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:27.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:27.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:27.720 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:27 smithi177 bash[68594]: cluster 2024-09-13T15:16:27.183854+0000 mgr.y (mgr.26252) 1513 : cluster [DBG] pgmap v2103: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2024-09-13T15:16:27.721 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:27 smithi177 bash[68594]: cluster 2024-09-13T15:16:27.183854+0000 mgr.y (mgr.26252) 1513 : cluster [DBG] pgmap v2103: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2024-09-13T15:16:27.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:27.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:27.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:27.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:27.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:27.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:27.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:27.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:27.986 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:27 smithi099 bash[84576]: cluster 2024-09-13T15:16:27.183854+0000 mgr.y (mgr.26252) 1513 : cluster [DBG] pgmap v2103: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2024-09-13T15:16:27.986 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:27 smithi099 bash[84576]: cluster 2024-09-13T15:16:27.183854+0000 mgr.y (mgr.26252) 1513 : cluster [DBG] pgmap v2103: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2024-09-13T15:16:27.987 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:27 smithi099 bash[87147]: cluster 2024-09-13T15:16:27.183854+0000 mgr.y (mgr.26252) 1513 : cluster [DBG] pgmap v2103: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2024-09-13T15:16:27.987 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:27 smithi099 bash[87147]: cluster 2024-09-13T15:16:27.183854+0000 mgr.y (mgr.26252) 1513 : cluster [DBG] pgmap v2103: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2024-09-13T15:16:28.001 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:28.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:28.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:28.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 68 16 87051 87035 79.9858 46.375 0.0189819 0.0124959 2024-09-13T15:16:28.105 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:28.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:28.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:28.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:28.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:28.264 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:28.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:28.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:28.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:28.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:28.499 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:28.522 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:28.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:28.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:28.626 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:28.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:28.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:28.684 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:16:28.684 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:16:28.718 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:16:28.718 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:16:28.750 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:16:28.750 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:16:28.759 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:28.780 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:16:28.780 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:16:28.784 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:28.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:28.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:28.807 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:16:28.807 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:16:28.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:28 smithi099 bash[84576]: audit 2024-09-13T15:16:28.641539+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.99:0/2137103929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:16:28.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:28 smithi099 bash[84576]: audit 2024-09-13T15:16:28.641539+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.99:0/2137103929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:16:28.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:28 smithi099 bash[87147]: audit 2024-09-13T15:16:28.641539+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.99:0/2137103929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:16:28.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:28 smithi099 bash[87147]: audit 2024-09-13T15:16:28.641539+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.99:0/2137103929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:16:28.841 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:16:28.841 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:16:28.874 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:16:28.874 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:16:28.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:28.897 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:16:28.897 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:16:28.898 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:28 smithi177 bash[68594]: audit 2024-09-13T15:16:28.641539+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.99:0/2137103929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:16:28.898 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:28 smithi177 bash[68594]: audit 2024-09-13T15:16:28.641539+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.99:0/2137103929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:16:28.919 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:28.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:28.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:29.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:29.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:29.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:29.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:29.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 69 16 88020 88004 79.7042 60.5625 0.00610659 0.0125407 2024-09-13T15:16:29.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:29.177 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:29.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:29.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:29.280 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:29.308 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:16:29.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:29.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:29.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:29.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:29.438 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:29.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:29.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:29.541 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:29.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:29.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:29.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:29.675 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T15:16:29.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:29.717 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:29.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:29.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:29.823 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:29.832 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:29 smithi177 bash[68594]: cluster 2024-09-13T15:16:29.184400+0000 mgr.y (mgr.26252) 1514 : cluster [DBG] pgmap v2104: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2024-09-13T15:16:29.833 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:29 smithi177 bash[68594]: cluster 2024-09-13T15:16:29.184400+0000 mgr.y (mgr.26252) 1514 : cluster [DBG] pgmap v2104: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2024-09-13T15:16:29.833 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:29 smithi177 bash[68594]: audit 2024-09-13T15:16:29.465209+0000 mon.a (mon.0) 2649 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:16:29.833 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:29 smithi177 bash[68594]: audit 2024-09-13T15:16:29.465209+0000 mon.a (mon.0) 2649 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:16:29.852 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:29.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:29.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:29.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:29.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:29.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:29.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:30.032 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:29 smithi099 bash[84576]: cluster 2024-09-13T15:16:29.184400+0000 mgr.y (mgr.26252) 1514 : cluster [DBG] pgmap v2104: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2024-09-13T15:16:30.032 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:29 smithi099 bash[84576]: cluster 2024-09-13T15:16:29.184400+0000 mgr.y (mgr.26252) 1514 : cluster [DBG] pgmap v2104: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2024-09-13T15:16:30.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:29 smithi099 bash[84576]: audit 2024-09-13T15:16:29.465209+0000 mon.a (mon.0) 2649 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:16:30.033 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:29 smithi099 bash[84576]: audit 2024-09-13T15:16:29.465209+0000 mon.a (mon.0) 2649 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:16:30.072 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 70 16 88702 88686 79.1744 42.625 0.0115422 0.0126259 2024-09-13T15:16:30.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:30.104 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:30.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:30.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:30.206 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:30.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:30.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:30.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:30.332 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:30.361 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:30.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:30.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:30.374 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: cluster 2024-09-13T15:16:29.184400+0000 mgr.y (mgr.26252) 1514 : cluster [DBG] pgmap v2104: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2024-09-13T15:16:30.374 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: cluster 2024-09-13T15:16:29.184400+0000 mgr.y (mgr.26252) 1514 : cluster [DBG] pgmap v2104: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2024-09-13T15:16:30.374 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: audit 2024-09-13T15:16:29.465209+0000 mon.a (mon.0) 2649 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:16:30.374 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: audit 2024-09-13T15:16:29.465209+0000 mon.a (mon.0) 2649 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:16:30.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:30.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:30.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:30.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:30.593 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:30.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:30.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:30.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:30.688 INFO:teuthology.orchestra.run.smithi099.stderr:set pool 46 pg_num to 26 2024-09-13T15:16:30.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:30.764 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:30.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:30.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:30.862 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[61439]: ts=2024-09-13T15:16:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:16:30.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[68594]: cluster 2024-09-13T15:16:29.662516+0000 mon.a (mon.0) 2650 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:16:30.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[68594]: cluster 2024-09-13T15:16:29.662516+0000 mon.a (mon.0) 2650 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:16:30.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[68594]: audit 2024-09-13T15:16:29.674828+0000 mon.a (mon.0) 2651 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:16:30.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[68594]: audit 2024-09-13T15:16:29.674828+0000 mon.a (mon.0) 2651 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:16:30.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[68594]: cluster 2024-09-13T15:16:29.683847+0000 mon.a (mon.0) 2652 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2024-09-13T15:16:30.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[68594]: cluster 2024-09-13T15:16:29.683847+0000 mon.a (mon.0) 2652 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2024-09-13T15:16:30.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[68594]: audit 2024-09-13T15:16:29.778206+0000 mon.c (mon.2) 119 : audit [INF] from='client.? 172.21.15.99:0/3880599476' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:30.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[68594]: audit 2024-09-13T15:16:29.778206+0000 mon.c (mon.2) 119 : audit [INF] from='client.? 172.21.15.99:0/3880599476' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:30.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[68594]: audit 2024-09-13T15:16:29.778718+0000 mon.a (mon.0) 2653 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:30.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:30 smithi177 bash[68594]: audit 2024-09-13T15:16:29.778718+0000 mon.a (mon.0) 2653 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:30.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:30.882 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: cluster 2024-09-13T15:16:29.662516+0000 mon.a (mon.0) 2650 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:16:30.883 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: cluster 2024-09-13T15:16:29.662516+0000 mon.a (mon.0) 2650 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:16:30.897 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:30.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:30.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:31.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:31.028 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:31.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:31.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:31.073 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 71 16 88970 88954 78.2952 16.75 0.0115569 0.012767 2024-09-13T15:16:31.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[84576]: cluster 2024-09-13T15:16:29.662516+0000 mon.a (mon.0) 2650 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:16:31.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[84576]: cluster 2024-09-13T15:16:29.662516+0000 mon.a (mon.0) 2650 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:16:31.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[84576]: audit 2024-09-13T15:16:29.674828+0000 mon.a (mon.0) 2651 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:16:31.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[84576]: audit 2024-09-13T15:16:29.674828+0000 mon.a (mon.0) 2651 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:16:31.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[84576]: cluster 2024-09-13T15:16:29.683847+0000 mon.a (mon.0) 2652 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2024-09-13T15:16:31.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[84576]: cluster 2024-09-13T15:16:29.683847+0000 mon.a (mon.0) 2652 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2024-09-13T15:16:31.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[84576]: audit 2024-09-13T15:16:29.778206+0000 mon.c (mon.2) 119 : audit [INF] from='client.? 172.21.15.99:0/3880599476' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:31.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[84576]: audit 2024-09-13T15:16:29.778206+0000 mon.c (mon.2) 119 : audit [INF] from='client.? 172.21.15.99:0/3880599476' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:31.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[84576]: audit 2024-09-13T15:16:29.778718+0000 mon.a (mon.0) 2653 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:31.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[84576]: audit 2024-09-13T15:16:29.778718+0000 mon.a (mon.0) 2653 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: audit 2024-09-13T15:16:29.674828+0000 mon.a (mon.0) 2651 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:16:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: audit 2024-09-13T15:16:29.674828+0000 mon.a (mon.0) 2651 : audit [INF] from='client.? 172.21.15.99:0/541610821' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:16:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: cluster 2024-09-13T15:16:29.683847+0000 mon.a (mon.0) 2652 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2024-09-13T15:16:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: cluster 2024-09-13T15:16:29.683847+0000 mon.a (mon.0) 2652 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2024-09-13T15:16:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: audit 2024-09-13T15:16:29.778206+0000 mon.c (mon.2) 119 : audit [INF] from='client.? 172.21.15.99:0/3880599476' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: audit 2024-09-13T15:16:29.778206+0000 mon.c (mon.2) 119 : audit [INF] from='client.? 172.21.15.99:0/3880599476' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: audit 2024-09-13T15:16:29.778718+0000 mon.a (mon.0) 2653 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:31.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:30 smithi099 bash[87147]: audit 2024-09-13T15:16:29.778718+0000 mon.a (mon.0) 2653 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:16:31.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:31.164 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:31.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:31.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:31.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:31.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:31.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:31.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:31.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:31.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:31.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:31.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:31.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:31.571 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:31.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:31.678 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:31.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:31.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:31.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:31.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:31.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:31.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:31.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:32.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:32.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:31 smithi177 bash[68594]: audit 2024-09-13T15:16:30.677906+0000 mon.a (mon.0) 2654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:16:32.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:31 smithi177 bash[68594]: audit 2024-09-13T15:16:30.677906+0000 mon.a (mon.0) 2654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:16:32.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:31 smithi177 bash[68594]: cluster 2024-09-13T15:16:30.683360+0000 mon.a (mon.0) 2655 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2024-09-13T15:16:32.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:31 smithi177 bash[68594]: cluster 2024-09-13T15:16:30.683360+0000 mon.a (mon.0) 2655 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2024-09-13T15:16:32.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:31 smithi177 bash[68594]: cluster 2024-09-13T15:16:31.184782+0000 mgr.y (mgr.26252) 1515 : cluster [DBG] pgmap v2107: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2024-09-13T15:16:32.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:31 smithi177 bash[68594]: cluster 2024-09-13T15:16:31.184782+0000 mgr.y (mgr.26252) 1515 : cluster [DBG] pgmap v2107: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2024-09-13T15:16:32.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:31 smithi177 bash[68594]: audit 2024-09-13T15:16:31.185418+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:16:32.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:31 smithi177 bash[68594]: audit 2024-09-13T15:16:31.185418+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:16:32.042 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:32.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:32.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:32.073 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 72 16 89333 89317 77.5229 22.6875 0.00570845 0.0128469 2024-09-13T15:16:32.122 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[84576]: audit 2024-09-13T15:16:30.677906+0000 mon.a (mon.0) 2654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:16:32.123 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[84576]: audit 2024-09-13T15:16:30.677906+0000 mon.a (mon.0) 2654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:16:32.123 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[84576]: cluster 2024-09-13T15:16:30.683360+0000 mon.a (mon.0) 2655 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2024-09-13T15:16:32.123 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[84576]: cluster 2024-09-13T15:16:30.683360+0000 mon.a (mon.0) 2655 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2024-09-13T15:16:32.123 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[84576]: cluster 2024-09-13T15:16:31.184782+0000 mgr.y (mgr.26252) 1515 : cluster [DBG] pgmap v2107: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2024-09-13T15:16:32.123 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[84576]: cluster 2024-09-13T15:16:31.184782+0000 mgr.y (mgr.26252) 1515 : cluster [DBG] pgmap v2107: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2024-09-13T15:16:32.123 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[84576]: audit 2024-09-13T15:16:31.185418+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:16:32.123 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[84576]: audit 2024-09-13T15:16:31.185418+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:16:32.123 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[87147]: audit 2024-09-13T15:16:30.677906+0000 mon.a (mon.0) 2654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:16:32.123 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[87147]: audit 2024-09-13T15:16:30.677906+0000 mon.a (mon.0) 2654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:16:32.124 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[87147]: cluster 2024-09-13T15:16:30.683360+0000 mon.a (mon.0) 2655 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2024-09-13T15:16:32.124 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[87147]: cluster 2024-09-13T15:16:30.683360+0000 mon.a (mon.0) 2655 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2024-09-13T15:16:32.124 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[87147]: cluster 2024-09-13T15:16:31.184782+0000 mgr.y (mgr.26252) 1515 : cluster [DBG] pgmap v2107: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2024-09-13T15:16:32.124 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[87147]: cluster 2024-09-13T15:16:31.184782+0000 mgr.y (mgr.26252) 1515 : cluster [DBG] pgmap v2107: 89 pgs: 89 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2024-09-13T15:16:32.124 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[87147]: audit 2024-09-13T15:16:31.185418+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:16:32.124 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:31 smithi099 bash[87147]: audit 2024-09-13T15:16:31.185418+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:16:32.145 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:32.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:32.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:32.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:32.271 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:32.296 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:32.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:32.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:32.400 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:32.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:32.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:32.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:32.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:32.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:32.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:32.661 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:32.685 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:32.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:32.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:32.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:32.806 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:32.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:32.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:32.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:32.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:32.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:33.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:33.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:32 smithi099 bash[84576]: audit 2024-09-13T15:16:31.705014+0000 mon.a (mon.0) 2657 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:16:33.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:32 smithi099 bash[84576]: audit 2024-09-13T15:16:31.705014+0000 mon.a (mon.0) 2657 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:16:33.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:32 smithi099 bash[84576]: cluster 2024-09-13T15:16:31.709756+0000 mon.a (mon.0) 2658 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2024-09-13T15:16:33.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:32 smithi099 bash[84576]: cluster 2024-09-13T15:16:31.709756+0000 mon.a (mon.0) 2658 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2024-09-13T15:16:33.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:32 smithi099 bash[87147]: audit 2024-09-13T15:16:31.705014+0000 mon.a (mon.0) 2657 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:16:33.046 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:32 smithi099 bash[87147]: audit 2024-09-13T15:16:31.705014+0000 mon.a (mon.0) 2657 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:16:33.046 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:32 smithi099 bash[87147]: cluster 2024-09-13T15:16:31.709756+0000 mon.a (mon.0) 2658 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2024-09-13T15:16:33.046 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:32 smithi099 bash[87147]: cluster 2024-09-13T15:16:31.709756+0000 mon.a (mon.0) 2658 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2024-09-13T15:16:33.073 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 73 16 89687 89671 76.764 22.125 0.00742252 0.0130218 2024-09-13T15:16:33.078 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:33.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:33.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:33.180 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:33.206 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:33.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:33.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:33.309 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:33.319 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:32 smithi177 bash[68594]: audit 2024-09-13T15:16:31.705014+0000 mon.a (mon.0) 2657 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:16:33.319 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:32 smithi177 bash[68594]: audit 2024-09-13T15:16:31.705014+0000 mon.a (mon.0) 2657 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:16:33.319 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:32 smithi177 bash[68594]: cluster 2024-09-13T15:16:31.709756+0000 mon.a (mon.0) 2658 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2024-09-13T15:16:33.319 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:32 smithi177 bash[68594]: cluster 2024-09-13T15:16:31.709756+0000 mon.a (mon.0) 2658 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2024-09-13T15:16:33.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:33.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:33.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:33.441 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:33.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:33.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:33.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:33.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:33.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:33.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:33.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:33.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:33.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:33.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:33.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:33.840 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:33.864 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:33.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:33.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:33.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[84576]: cluster 2024-09-13T15:16:32.718968+0000 mon.a (mon.0) 2659 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2024-09-13T15:16:33.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[84576]: cluster 2024-09-13T15:16:32.718968+0000 mon.a (mon.0) 2659 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2024-09-13T15:16:33.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.185315+0000 mgr.y (mgr.26252) 1516 : cluster [DBG] pgmap v2110: 99 pgs: 8 unknown, 91 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s 2024-09-13T15:16:33.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.185315+0000 mgr.y (mgr.26252) 1516 : cluster [DBG] pgmap v2110: 99 pgs: 8 unknown, 91 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s 2024-09-13T15:16:33.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.235871+0000 osd.2 (osd.2) 26 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:33.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.235871+0000 osd.2 (osd.2) 26 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:33.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[84576]: audit 2024-09-13T15:16:33.316628+0000 mon.a (mon.0) 2660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:33.910 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[84576]: audit 2024-09-13T15:16:33.316628+0000 mon.a (mon.0) 2660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:33.968 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:33.980 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:33 smithi177 bash[68594]: cluster 2024-09-13T15:16:32.718968+0000 mon.a (mon.0) 2659 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2024-09-13T15:16:33.980 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:33 smithi177 bash[68594]: cluster 2024-09-13T15:16:32.718968+0000 mon.a (mon.0) 2659 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2024-09-13T15:16:33.980 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:33 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.185315+0000 mgr.y (mgr.26252) 1516 : cluster [DBG] pgmap v2110: 99 pgs: 8 unknown, 91 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s 2024-09-13T15:16:33.981 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:33 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.185315+0000 mgr.y (mgr.26252) 1516 : cluster [DBG] pgmap v2110: 99 pgs: 8 unknown, 91 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s 2024-09-13T15:16:33.981 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:33 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.235871+0000 osd.2 (osd.2) 26 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:33.981 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:33 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.235871+0000 osd.2 (osd.2) 26 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:33.981 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:33 smithi177 bash[68594]: audit 2024-09-13T15:16:33.316628+0000 mon.a (mon.0) 2660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:33.981 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:33 smithi177 bash[68594]: audit 2024-09-13T15:16:33.316628+0000 mon.a (mon.0) 2660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:34.005 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:34.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:34.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:34.073 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 74 16 90626 90610 76.5197 58.6875 0.0147489 0.0130634 2024-09-13T15:16:34.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:34.131 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:34.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:34.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:34.234 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:34.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:34.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:34.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[87147]: cluster 2024-09-13T15:16:32.718968+0000 mon.a (mon.0) 2659 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2024-09-13T15:16:34.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[87147]: cluster 2024-09-13T15:16:32.718968+0000 mon.a (mon.0) 2659 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2024-09-13T15:16:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.185315+0000 mgr.y (mgr.26252) 1516 : cluster [DBG] pgmap v2110: 99 pgs: 8 unknown, 91 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s 2024-09-13T15:16:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.185315+0000 mgr.y (mgr.26252) 1516 : cluster [DBG] pgmap v2110: 99 pgs: 8 unknown, 91 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s 2024-09-13T15:16:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.235871+0000 osd.2 (osd.2) 26 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.235871+0000 osd.2 (osd.2) 26 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[87147]: audit 2024-09-13T15:16:33.316628+0000 mon.a (mon.0) 2660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:33 smithi099 bash[87147]: audit 2024-09-13T15:16:33.316628+0000 mon.a (mon.0) 2660 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:34.368 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:34.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:34.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:34.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:34.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:34.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:34.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:34.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:34.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:34.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:34.775 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:34.800 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:34.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:34.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:34.848 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:16:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:34 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.002943+0000 osd.3 (osd.3) 3 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:34 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.002943+0000 osd.3 (osd.3) 3 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:34 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.292254+0000 osd.1 (osd.1) 6 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:34 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.292254+0000 osd.1 (osd.1) 6 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:34 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.575685+0000 osd.0 (osd.0) 20 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:34 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.575685+0000 osd.0 (osd.0) 20 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:34 smithi177 bash[68594]: cluster 2024-09-13T15:16:34.240418+0000 osd.2 (osd.2) 27 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:34 smithi177 bash[68594]: cluster 2024-09-13T15:16:34.240418+0000 osd.2 (osd.2) 27 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:34.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:34.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:34.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:34.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:35.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:35.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.002943+0000 osd.3 (osd.3) 3 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:35.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.002943+0000 osd.3 (osd.3) 3 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:35.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.292254+0000 osd.1 (osd.1) 6 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:35.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.292254+0000 osd.1 (osd.1) 6 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:35.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.575685+0000 osd.0 (osd.0) 20 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:35.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.575685+0000 osd.0 (osd.0) 20 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:35.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[84576]: cluster 2024-09-13T15:16:34.240418+0000 osd.2 (osd.2) 27 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:35.049 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[84576]: cluster 2024-09-13T15:16:34.240418+0000 osd.2 (osd.2) 27 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:35.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.002943+0000 osd.3 (osd.3) 3 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:35.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.002943+0000 osd.3 (osd.3) 3 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:35.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.292254+0000 osd.1 (osd.1) 6 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:35.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.292254+0000 osd.1 (osd.1) 6 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:35.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.575685+0000 osd.0 (osd.0) 20 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:35.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.575685+0000 osd.0 (osd.0) 20 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:35.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[87147]: cluster 2024-09-13T15:16:34.240418+0000 osd.2 (osd.2) 27 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:35.050 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:34 smithi099 bash[87147]: cluster 2024-09-13T15:16:34.240418+0000 osd.2 (osd.2) 27 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:35.073 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 75 16 91931 91915 76.5865 81.5625 0.00631426 0.013052 2024-09-13T15:16:35.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:35.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:35.181 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:35.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:35.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:35.317 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:35.348 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:35.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:35.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:35.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:35.480 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:35.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:35.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:35.526 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:16:35.526 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:16:35.526 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:16:35.526 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:16:35.526 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:16:35.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:35.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:35.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:35.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:35.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:35.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:35.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:35.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:35.849 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:35.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:35 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.985702+0000 osd.3 (osd.3) 4 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:35.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:35 smithi177 bash[68594]: cluster 2024-09-13T15:16:33.985702+0000 osd.3 (osd.3) 4 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:35.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:35 smithi177 bash[68594]: cluster 2024-09-13T15:16:34.257007+0000 osd.1 (osd.1) 7 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:35.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:35 smithi177 bash[68594]: cluster 2024-09-13T15:16:34.257007+0000 osd.1 (osd.1) 7 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:35.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:35 smithi177 bash[68594]: cluster 2024-09-13T15:16:34.604717+0000 osd.0 (osd.0) 21 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:35.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:35 smithi177 bash[68594]: cluster 2024-09-13T15:16:34.604717+0000 osd.0 (osd.0) 21 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:35.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:35 smithi177 bash[68594]: cluster 2024-09-13T15:16:35.186021+0000 mgr.y (mgr.26252) 1517 : cluster [DBG] pgmap v2111: 99 pgs: 1 unknown, 98 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 12 KiB/s, 0 objects/s recovering 2024-09-13T15:16:35.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:35 smithi177 bash[68594]: cluster 2024-09-13T15:16:35.186021+0000 mgr.y (mgr.26252) 1517 : cluster [DBG] pgmap v2111: 99 pgs: 1 unknown, 98 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 12 KiB/s, 0 objects/s recovering 2024-09-13T15:16:35.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:35 smithi177 bash[68594]: cluster 2024-09-13T15:16:35.214167+0000 osd.2 (osd.2) 28 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:35.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:35 smithi177 bash[68594]: cluster 2024-09-13T15:16:35.214167+0000 osd.2 (osd.2) 28 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:35.877 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:35.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:35.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:35.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:36.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:36.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:36.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:36.073 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 76 16 93239 93223 76.6544 81.75 0.00896618 0.0130406 2024-09-13T15:16:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.985702+0000 osd.3 (osd.3) 4 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[84576]: cluster 2024-09-13T15:16:33.985702+0000 osd.3 (osd.3) 4 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[84576]: cluster 2024-09-13T15:16:34.257007+0000 osd.1 (osd.1) 7 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[84576]: cluster 2024-09-13T15:16:34.257007+0000 osd.1 (osd.1) 7 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[84576]: cluster 2024-09-13T15:16:34.604717+0000 osd.0 (osd.0) 21 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[84576]: cluster 2024-09-13T15:16:34.604717+0000 osd.0 (osd.0) 21 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[84576]: cluster 2024-09-13T15:16:35.186021+0000 mgr.y (mgr.26252) 1517 : cluster [DBG] pgmap v2111: 99 pgs: 1 unknown, 98 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 12 KiB/s, 0 objects/s recovering 2024-09-13T15:16:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[84576]: cluster 2024-09-13T15:16:35.186021+0000 mgr.y (mgr.26252) 1517 : cluster [DBG] pgmap v2111: 99 pgs: 1 unknown, 98 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 12 KiB/s, 0 objects/s recovering 2024-09-13T15:16:36.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[84576]: cluster 2024-09-13T15:16:35.214167+0000 osd.2 (osd.2) 28 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:36.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[84576]: cluster 2024-09-13T15:16:35.214167+0000 osd.2 (osd.2) 28 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:36.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.985702+0000 osd.3 (osd.3) 4 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:36.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[87147]: cluster 2024-09-13T15:16:33.985702+0000 osd.3 (osd.3) 4 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:36.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[87147]: cluster 2024-09-13T15:16:34.257007+0000 osd.1 (osd.1) 7 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:36.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[87147]: cluster 2024-09-13T15:16:34.257007+0000 osd.1 (osd.1) 7 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:36.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[87147]: cluster 2024-09-13T15:16:34.604717+0000 osd.0 (osd.0) 21 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:36.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[87147]: cluster 2024-09-13T15:16:34.604717+0000 osd.0 (osd.0) 21 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:36.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[87147]: cluster 2024-09-13T15:16:35.186021+0000 mgr.y (mgr.26252) 1517 : cluster [DBG] pgmap v2111: 99 pgs: 1 unknown, 98 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 12 KiB/s, 0 objects/s recovering 2024-09-13T15:16:36.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[87147]: cluster 2024-09-13T15:16:35.186021+0000 mgr.y (mgr.26252) 1517 : cluster [DBG] pgmap v2111: 99 pgs: 1 unknown, 98 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 12 KiB/s, 0 objects/s recovering 2024-09-13T15:16:36.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[87147]: cluster 2024-09-13T15:16:35.214167+0000 osd.2 (osd.2) 28 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:36.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:35 smithi099 bash[87147]: cluster 2024-09-13T15:16:35.214167+0000 osd.2 (osd.2) 28 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:36.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:36.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:36.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:36.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:36.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:36.288 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:36.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:36.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:36.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:36.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:36.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:36.436 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:16:36.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:36.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:36.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:36.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:36.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:36.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:36.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:36.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:36.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:36.821 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:36.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:36.833 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:16:36.926 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:36.952 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:36.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:36.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:37.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[84576]: cluster 2024-09-13T15:16:35.290223+0000 osd.1 (osd.1) 8 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:37.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[84576]: cluster 2024-09-13T15:16:35.290223+0000 osd.1 (osd.1) 8 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:37.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[84576]: cluster 2024-09-13T15:16:35.629748+0000 osd.0 (osd.0) 22 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[84576]: cluster 2024-09-13T15:16:35.629748+0000 osd.0 (osd.0) 22 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[84576]: cluster 2024-09-13T15:16:36.254243+0000 osd.2 (osd.2) 29 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:16:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[84576]: cluster 2024-09-13T15:16:36.254243+0000 osd.2 (osd.2) 29 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:16:37.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[87147]: cluster 2024-09-13T15:16:35.290223+0000 osd.1 (osd.1) 8 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:37.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[87147]: cluster 2024-09-13T15:16:35.290223+0000 osd.1 (osd.1) 8 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:37.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[87147]: cluster 2024-09-13T15:16:35.629748+0000 osd.0 (osd.0) 22 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:37.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[87147]: cluster 2024-09-13T15:16:35.629748+0000 osd.0 (osd.0) 22 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:37.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[87147]: cluster 2024-09-13T15:16:36.254243+0000 osd.2 (osd.2) 29 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:16:37.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:36 smithi099 bash[87147]: cluster 2024-09-13T15:16:36.254243+0000 osd.2 (osd.2) 29 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:16:37.056 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:37.065 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:36 smithi177 bash[68594]: cluster 2024-09-13T15:16:35.290223+0000 osd.1 (osd.1) 8 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:37.065 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:36 smithi177 bash[68594]: cluster 2024-09-13T15:16:35.290223+0000 osd.1 (osd.1) 8 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:37.065 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:36 smithi177 bash[68594]: cluster 2024-09-13T15:16:35.629748+0000 osd.0 (osd.0) 22 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:37.065 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:36 smithi177 bash[68594]: cluster 2024-09-13T15:16:35.629748+0000 osd.0 (osd.0) 22 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:37.065 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:36 smithi177 bash[68594]: cluster 2024-09-13T15:16:36.254243+0000 osd.2 (osd.2) 29 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:16:37.065 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:36 smithi177 bash[68594]: cluster 2024-09-13T15:16:36.254243+0000 osd.2 (osd.2) 29 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:16:37.073 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 77 16 94504 94488 76.6855 79.0625 0.0136918 0.0130345 2024-09-13T15:16:37.088 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:37.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:37.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:37.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:37.220 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:37.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:37.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:16:37] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:16:37.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:37.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:37.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:37.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:37.489 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:37.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:37.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:37.593 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:37.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:37.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:37.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:37.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:37.752 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:37.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:37.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:37.855 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:37.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:37.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:37.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:37.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:38.009 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:38.010 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[84576]: cluster 2024-09-13T15:16:36.253929+0000 osd.1 (osd.1) 9 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:38.010 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[84576]: cluster 2024-09-13T15:16:36.253929+0000 osd.1 (osd.1) 9 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:38.010 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[84576]: cluster 2024-09-13T15:16:36.581626+0000 osd.0 (osd.0) 23 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:38.010 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[84576]: cluster 2024-09-13T15:16:36.581626+0000 osd.0 (osd.0) 23 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:38.010 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[84576]: cluster 2024-09-13T15:16:37.186576+0000 mgr.y (mgr.26252) 1518 : cluster [DBG] pgmap v2112: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 535 op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-13T15:16:38.010 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[84576]: cluster 2024-09-13T15:16:37.186576+0000 mgr.y (mgr.26252) 1518 : cluster [DBG] pgmap v2112: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 535 op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-13T15:16:38.010 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[84576]: audit 2024-09-13T15:16:37.189176+0000 mon.a (mon.0) 2661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T15:16:38.010 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[84576]: audit 2024-09-13T15:16:37.189176+0000 mon.a (mon.0) 2661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T15:16:38.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:38.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:38.012 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[87147]: cluster 2024-09-13T15:16:36.253929+0000 osd.1 (osd.1) 9 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:38.012 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[87147]: cluster 2024-09-13T15:16:36.253929+0000 osd.1 (osd.1) 9 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:38.012 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[87147]: cluster 2024-09-13T15:16:36.581626+0000 osd.0 (osd.0) 23 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:38.012 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[87147]: cluster 2024-09-13T15:16:36.581626+0000 osd.0 (osd.0) 23 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:38.012 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[87147]: cluster 2024-09-13T15:16:37.186576+0000 mgr.y (mgr.26252) 1518 : cluster [DBG] pgmap v2112: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 535 op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-13T15:16:38.012 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[87147]: cluster 2024-09-13T15:16:37.186576+0000 mgr.y (mgr.26252) 1518 : cluster [DBG] pgmap v2112: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 535 op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-13T15:16:38.012 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[87147]: audit 2024-09-13T15:16:37.189176+0000 mon.a (mon.0) 2661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T15:16:38.012 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:37 smithi099 bash[87147]: audit 2024-09-13T15:16:37.189176+0000 mon.a (mon.0) 2661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T15:16:38.073 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 78 16 95745 95729 76.6967 77.5625 0.0030934 0.0130076 2024-09-13T15:16:38.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:37 smithi177 bash[68594]: cluster 2024-09-13T15:16:36.253929+0000 osd.1 (osd.1) 9 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:38.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:37 smithi177 bash[68594]: cluster 2024-09-13T15:16:36.253929+0000 osd.1 (osd.1) 9 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:38.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:37 smithi177 bash[68594]: cluster 2024-09-13T15:16:36.581626+0000 osd.0 (osd.0) 23 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:38.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:37 smithi177 bash[68594]: cluster 2024-09-13T15:16:36.581626+0000 osd.0 (osd.0) 23 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:38.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:37 smithi177 bash[68594]: cluster 2024-09-13T15:16:37.186576+0000 mgr.y (mgr.26252) 1518 : cluster [DBG] pgmap v2112: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 535 op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-13T15:16:38.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:37 smithi177 bash[68594]: cluster 2024-09-13T15:16:37.186576+0000 mgr.y (mgr.26252) 1518 : cluster [DBG] pgmap v2112: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 535 op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-13T15:16:38.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:37 smithi177 bash[68594]: audit 2024-09-13T15:16:37.189176+0000 mon.a (mon.0) 2661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T15:16:38.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:37 smithi177 bash[68594]: audit 2024-09-13T15:16:37.189176+0000 mon.a (mon.0) 2661 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T15:16:38.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:38.134 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:38.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:38.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:38.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:38.258 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:38.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:38.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:38.361 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:38.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:38.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:38.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:38.483 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:38.509 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:38.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:38.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:38.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:38.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:38.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:38.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:38.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:38.768 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:38.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:38.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:38.872 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:38.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:38.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:38.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:39.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:39.018 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:38 smithi177 bash[68594]: audit 2024-09-13T15:16:37.747813+0000 mon.a (mon.0) 2662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T15:16:39.018 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:38 smithi177 bash[68594]: audit 2024-09-13T15:16:37.747813+0000 mon.a (mon.0) 2662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T15:16:39.018 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:38 smithi177 bash[68594]: cluster 2024-09-13T15:16:37.759713+0000 mon.a (mon.0) 2663 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2024-09-13T15:16:39.018 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:38 smithi177 bash[68594]: cluster 2024-09-13T15:16:37.759713+0000 mon.a (mon.0) 2663 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2024-09-13T15:16:39.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:38 smithi177 bash[68594]: cluster 2024-09-13T15:16:38.189827+0000 osd.2 (osd.2) 30 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:39.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:38 smithi177 bash[68594]: cluster 2024-09-13T15:16:38.189827+0000 osd.2 (osd.2) 30 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:39.032 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:39.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:39.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:39.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 79 16 95921 95905 75.8651 11 0.00322821 0.0129913 2024-09-13T15:16:39.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[84576]: audit 2024-09-13T15:16:37.747813+0000 mon.a (mon.0) 2662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[84576]: audit 2024-09-13T15:16:37.747813+0000 mon.a (mon.0) 2662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[84576]: cluster 2024-09-13T15:16:37.759713+0000 mon.a (mon.0) 2663 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[84576]: cluster 2024-09-13T15:16:37.759713+0000 mon.a (mon.0) 2663 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[84576]: cluster 2024-09-13T15:16:38.189827+0000 osd.2 (osd.2) 30 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[84576]: cluster 2024-09-13T15:16:38.189827+0000 osd.2 (osd.2) 30 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[87147]: audit 2024-09-13T15:16:37.747813+0000 mon.a (mon.0) 2662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[87147]: audit 2024-09-13T15:16:37.747813+0000 mon.a (mon.0) 2662 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[87147]: cluster 2024-09-13T15:16:37.759713+0000 mon.a (mon.0) 2663 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[87147]: cluster 2024-09-13T15:16:37.759713+0000 mon.a (mon.0) 2663 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[87147]: cluster 2024-09-13T15:16:38.189827+0000 osd.2 (osd.2) 30 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:39.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:38 smithi099 bash[87147]: cluster 2024-09-13T15:16:38.189827+0000 osd.2 (osd.2) 30 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:39.135 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:39.164 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:39.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:39.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:39.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:39.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:39.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:39.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:39.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:39.423 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:39.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:39.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:39.526 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:39.537 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:16:39.537 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:16:39.537 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-13T15:16:39.537 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:16:39.554 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:39.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:39.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:39.657 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:39.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:39.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:39.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:39.696 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:16:39.784 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:39.808 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:39.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:39.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:39.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:39.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:39.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:40.034 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:40.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:37.916215+0000 osd.3 (osd.3) 5 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:40.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:37.916215+0000 osd.3 (osd.3) 5 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:40.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:38.172143+0000 osd.1 (osd.1) 10 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:40.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:38.172143+0000 osd.1 (osd.1) 10 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:40.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:38.658372+0000 osd.0 (osd.0) 24 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:40.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:38.658372+0000 osd.0 (osd.0) 24 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:40.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:38.754797+0000 mon.a (mon.0) 2664 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2024-09-13T15:16:40.045 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:38.754797+0000 mon.a (mon.0) 2664 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2024-09-13T15:16:40.046 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:39.187329+0000 mgr.y (mgr.26252) 1519 : cluster [DBG] pgmap v2115: 99 pgs: 1 remapped+peering, 98 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9.9 KiB/s, 0 objects/s recovering 2024-09-13T15:16:40.046 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:39.187329+0000 mgr.y (mgr.26252) 1519 : cluster [DBG] pgmap v2115: 99 pgs: 1 remapped+peering, 98 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9.9 KiB/s, 0 objects/s recovering 2024-09-13T15:16:40.046 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:39.224306+0000 osd.2 (osd.2) 31 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:40.046 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[87147]: cluster 2024-09-13T15:16:39.224306+0000 osd.2 (osd.2) 31 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:40.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:37.916215+0000 osd.3 (osd.3) 5 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:40.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:37.916215+0000 osd.3 (osd.3) 5 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:40.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:38.172143+0000 osd.1 (osd.1) 10 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:40.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:38.172143+0000 osd.1 (osd.1) 10 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:40.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:38.658372+0000 osd.0 (osd.0) 24 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:40.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:38.658372+0000 osd.0 (osd.0) 24 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:40.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:38.754797+0000 mon.a (mon.0) 2664 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2024-09-13T15:16:40.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:38.754797+0000 mon.a (mon.0) 2664 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2024-09-13T15:16:40.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:39.187329+0000 mgr.y (mgr.26252) 1519 : cluster [DBG] pgmap v2115: 99 pgs: 1 remapped+peering, 98 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9.9 KiB/s, 0 objects/s recovering 2024-09-13T15:16:40.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:39.187329+0000 mgr.y (mgr.26252) 1519 : cluster [DBG] pgmap v2115: 99 pgs: 1 remapped+peering, 98 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9.9 KiB/s, 0 objects/s recovering 2024-09-13T15:16:40.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:39.224306+0000 osd.2 (osd.2) 31 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:40.047 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:39 smithi099 bash[84576]: cluster 2024-09-13T15:16:39.224306+0000 osd.2 (osd.2) 31 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:40.057 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:40.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:40.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:40.074 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:2024-09-13T15:16:40.074375+0000 min lat: 0.00120222 max lat: 1.14026 avg lat: 0.0129913 2024-09-13T15:16:40.074 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:16:40.074 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 80 16 95921 95905 74.9168 0 - 0.0129913 2024-09-13T15:16:40.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:37.916215+0000 osd.3 (osd.3) 5 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:40.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:37.916215+0000 osd.3 (osd.3) 5 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:40.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:38.172143+0000 osd.1 (osd.1) 10 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:40.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:38.172143+0000 osd.1 (osd.1) 10 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:40.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:38.658372+0000 osd.0 (osd.0) 24 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:40.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:38.658372+0000 osd.0 (osd.0) 24 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:40.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:38.754797+0000 mon.a (mon.0) 2664 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2024-09-13T15:16:40.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:38.754797+0000 mon.a (mon.0) 2664 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2024-09-13T15:16:40.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:39.187329+0000 mgr.y (mgr.26252) 1519 : cluster [DBG] pgmap v2115: 99 pgs: 1 remapped+peering, 98 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9.9 KiB/s, 0 objects/s recovering 2024-09-13T15:16:40.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:39.187329+0000 mgr.y (mgr.26252) 1519 : cluster [DBG] pgmap v2115: 99 pgs: 1 remapped+peering, 98 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9.9 KiB/s, 0 objects/s recovering 2024-09-13T15:16:40.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:39.224306+0000 osd.2 (osd.2) 31 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:40.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:39 smithi177 bash[68594]: cluster 2024-09-13T15:16:39.224306+0000 osd.2 (osd.2) 31 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:40.161 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:40.190 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:40.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:40.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:40.293 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:40.320 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:40.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:40.423 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:40.435 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[61439]: ts=2024-09-13T15:16:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:16:40.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:40.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:40.553 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:40.583 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:40.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:40.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:40.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:40.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:40.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:40.720 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:16:40.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:40.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:38.925095+0000 osd.3 (osd.3) 6 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:40.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:38.925095+0000 osd.3 (osd.3) 6 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:40.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:39.138860+0000 osd.1 (osd.1) 11 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:40.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:39.138860+0000 osd.1 (osd.1) 11 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:40.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:39.661425+0000 osd.0 (osd.0) 25 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:39.661425+0000 osd.0 (osd.0) 25 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:39.780661+0000 mon.a (mon.0) 2665 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:39.780661+0000 mon.a (mon.0) 2665 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:40.097637+0000 osd.1 (osd.1) 12 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:40.097637+0000 osd.1 (osd.1) 12 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:40.197270+0000 osd.2 (osd.2) 32 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[84576]: cluster 2024-09-13T15:16:40.197270+0000 osd.2 (osd.2) 32 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:38.925095+0000 osd.3 (osd.3) 6 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:38.925095+0000 osd.3 (osd.3) 6 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:39.138860+0000 osd.1 (osd.1) 11 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:39.138860+0000 osd.1 (osd.1) 11 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:39.661425+0000 osd.0 (osd.0) 25 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:40.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:39.661425+0000 osd.0 (osd.0) 25 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:40.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:39.780661+0000 mon.a (mon.0) 2665 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2024-09-13T15:16:40.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:39.780661+0000 mon.a (mon.0) 2665 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2024-09-13T15:16:40.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:40.097637+0000 osd.1 (osd.1) 12 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:40.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:40.097637+0000 osd.1 (osd.1) 12 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:40.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:40.197270+0000 osd.2 (osd.2) 32 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:40.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:40 smithi099 bash[87147]: cluster 2024-09-13T15:16:40.197270+0000 osd.2 (osd.2) 32 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:40.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:40.858 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:40.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:40.959 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:40.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:38.925095+0000 osd.3 (osd.3) 6 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:40.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:38.925095+0000 osd.3 (osd.3) 6 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:40.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:39.138860+0000 osd.1 (osd.1) 11 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:40.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:39.138860+0000 osd.1 (osd.1) 11 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:40.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:39.661425+0000 osd.0 (osd.0) 25 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:40.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:39.661425+0000 osd.0 (osd.0) 25 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:40.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:39.780661+0000 mon.a (mon.0) 2665 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2024-09-13T15:16:40.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:39.780661+0000 mon.a (mon.0) 2665 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2024-09-13T15:16:40.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:40.097637+0000 osd.1 (osd.1) 12 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:40.968 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:40.097637+0000 osd.1 (osd.1) 12 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:40.968 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:40.197270+0000 osd.2 (osd.2) 32 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:40.968 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:40 smithi177 bash[68594]: cluster 2024-09-13T15:16:40.197270+0000 osd.2 (osd.2) 32 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:40.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:40.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:41.074 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 81 16 96389 96373 74.353 14.625 0.0106177 0.0134444 2024-09-13T15:16:41.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:41.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:41.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:41.211 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:41.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:41.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:41.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:41.339 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:41.368 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:41.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:41.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:41.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:41.504 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:41.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:41.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:41.610 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:41.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:41.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:41.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:41.708 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:16:41.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:41.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:41.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:41.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:41.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:41.898 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:41.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:41.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:42.001 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:42.010 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:41 smithi177 bash[68594]: cluster 2024-09-13T15:16:40.662938+0000 osd.0 (osd.0) 26 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:42.010 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:41 smithi177 bash[68594]: cluster 2024-09-13T15:16:40.662938+0000 osd.0 (osd.0) 26 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:42.010 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:41 smithi177 bash[68594]: cluster 2024-09-13T15:16:41.052668+0000 osd.1 (osd.1) 13 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:42.010 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:41 smithi177 bash[68594]: cluster 2024-09-13T15:16:41.052668+0000 osd.1 (osd.1) 13 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:42.010 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:41 smithi177 bash[68594]: cluster 2024-09-13T15:16:41.187875+0000 mgr.y (mgr.26252) 1520 : cluster [DBG] pgmap v2117: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T15:16:42.010 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:41 smithi177 bash[68594]: cluster 2024-09-13T15:16:41.187875+0000 mgr.y (mgr.26252) 1520 : cluster [DBG] pgmap v2117: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T15:16:42.023 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:42.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:42.074 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 82 16 97567 97551 74.344 73.625 0.00699171 0.0134459 2024-09-13T15:16:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[84576]: cluster 2024-09-13T15:16:40.662938+0000 osd.0 (osd.0) 26 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[84576]: cluster 2024-09-13T15:16:40.662938+0000 osd.0 (osd.0) 26 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[84576]: cluster 2024-09-13T15:16:41.052668+0000 osd.1 (osd.1) 13 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[84576]: cluster 2024-09-13T15:16:41.052668+0000 osd.1 (osd.1) 13 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[84576]: cluster 2024-09-13T15:16:41.187875+0000 mgr.y (mgr.26252) 1520 : cluster [DBG] pgmap v2117: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T15:16:42.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[84576]: cluster 2024-09-13T15:16:41.187875+0000 mgr.y (mgr.26252) 1520 : cluster [DBG] pgmap v2117: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T15:16:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[87147]: cluster 2024-09-13T15:16:40.662938+0000 osd.0 (osd.0) 26 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[87147]: cluster 2024-09-13T15:16:40.662938+0000 osd.0 (osd.0) 26 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:16:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[87147]: cluster 2024-09-13T15:16:41.052668+0000 osd.1 (osd.1) 13 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[87147]: cluster 2024-09-13T15:16:41.052668+0000 osd.1 (osd.1) 13 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[87147]: cluster 2024-09-13T15:16:41.187875+0000 mgr.y (mgr.26252) 1520 : cluster [DBG] pgmap v2117: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T15:16:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:41 smithi099 bash[87147]: cluster 2024-09-13T15:16:41.187875+0000 mgr.y (mgr.26252) 1520 : cluster [DBG] pgmap v2117: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T15:16:42.131 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:42.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:42.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:42.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:42.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:42.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:42.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:42.388 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:42.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:42.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:42.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:42.521 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:42.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:42.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:42.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:42.647 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:42.668 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:42.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:42.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:42.771 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:42.798 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:42.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:42.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:42.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:42.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:42.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:42.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:43.034 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:43.045 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:42 smithi099 bash[84576]: cluster 2024-09-13T15:16:41.680896+0000 osd.0 (osd.0) 27 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:43.046 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:42 smithi099 bash[84576]: cluster 2024-09-13T15:16:41.680896+0000 osd.0 (osd.0) 27 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:43.047 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:42 smithi099 bash[87147]: cluster 2024-09-13T15:16:41.680896+0000 osd.0 (osd.0) 27 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:43.048 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:42 smithi099 bash[87147]: cluster 2024-09-13T15:16:41.680896+0000 osd.0 (osd.0) 27 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:43.067 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:43.074 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 83 16 98300 98284 74.0002 45.8125 0.0101138 0.0135082 2024-09-13T15:16:43.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:43.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:43.175 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:43.201 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:43.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:43.304 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:43.314 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:42 smithi177 bash[68594]: cluster 2024-09-13T15:16:41.680896+0000 osd.0 (osd.0) 27 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:43.314 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:42 smithi177 bash[68594]: cluster 2024-09-13T15:16:41.680896+0000 osd.0 (osd.0) 27 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:43.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:43.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:43.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:43.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:43.460 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:43.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:43.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:43.561 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:43.594 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:43.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:43.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:43.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:43.729 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:43.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:43.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:43.776 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:16:43.833 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:43.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:43 smithi177 bash[68594]: cluster 2024-09-13T15:16:43.188426+0000 mgr.y (mgr.26252) 1521 : cluster [DBG] pgmap v2118: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T15:16:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:43 smithi177 bash[68594]: cluster 2024-09-13T15:16:43.188426+0000 mgr.y (mgr.26252) 1521 : cluster [DBG] pgmap v2118: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T15:16:43.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:43.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:43.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:43.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:43.998 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:44.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:44.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:44.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:43 smithi099 bash[84576]: cluster 2024-09-13T15:16:43.188426+0000 mgr.y (mgr.26252) 1521 : cluster [DBG] pgmap v2118: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T15:16:44.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:43 smithi099 bash[84576]: cluster 2024-09-13T15:16:43.188426+0000 mgr.y (mgr.26252) 1521 : cluster [DBG] pgmap v2118: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T15:16:44.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 84 16 99431 99415 73.9607 70.6875 0.0168668 0.0135153 2024-09-13T15:16:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:43 smithi099 bash[87147]: cluster 2024-09-13T15:16:43.188426+0000 mgr.y (mgr.26252) 1521 : cluster [DBG] pgmap v2118: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T15:16:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:43 smithi099 bash[87147]: cluster 2024-09-13T15:16:43.188426+0000 mgr.y (mgr.26252) 1521 : cluster [DBG] pgmap v2118: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 97 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T15:16:44.101 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:44.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:44.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:44.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:44.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:44.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:44.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:44.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:44.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:44.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:44.510 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:44.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:44.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:44.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:44.646 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:44.672 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:44.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:44.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:44.778 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:44.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:44.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:44.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:44.906 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:44.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:44.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:44.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:45.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:45.050 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:44 smithi177 bash[68594]: cluster 2024-09-13T15:16:43.682055+0000 osd.0 (osd.0) 28 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:45.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:44 smithi177 bash[68594]: cluster 2024-09-13T15:16:43.682055+0000 osd.0 (osd.0) 28 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:45.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:44 smithi177 bash[68594]: cluster 2024-09-13T15:16:43.820556+0000 osd.3 (osd.3) 7 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:45.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:44 smithi177 bash[68594]: cluster 2024-09-13T15:16:43.820556+0000 osd.3 (osd.3) 7 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:45.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:44 smithi177 bash[68594]: cluster 2024-09-13T15:16:44.102880+0000 osd.1 (osd.1) 14 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:45.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:44 smithi177 bash[68594]: cluster 2024-09-13T15:16:44.102880+0000 osd.1 (osd.1) 14 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:45.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:44 smithi177 bash[68594]: cluster 2024-09-13T15:16:44.166574+0000 osd.2 (osd.2) 33 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:45.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:44 smithi177 bash[68594]: cluster 2024-09-13T15:16:44.166574+0000 osd.2 (osd.2) 33 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:45.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:44 smithi177 bash[68594]: audit 2024-09-13T15:16:44.653766+0000 mon.a (mon.0) 2666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:45.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:44 smithi177 bash[68594]: audit 2024-09-13T15:16:44.653766+0000 mon.a (mon.0) 2666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:45.072 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[84576]: cluster 2024-09-13T15:16:43.682055+0000 osd.0 (osd.0) 28 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[84576]: cluster 2024-09-13T15:16:43.682055+0000 osd.0 (osd.0) 28 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[84576]: cluster 2024-09-13T15:16:43.820556+0000 osd.3 (osd.3) 7 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[84576]: cluster 2024-09-13T15:16:43.820556+0000 osd.3 (osd.3) 7 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:45.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[84576]: cluster 2024-09-13T15:16:44.102880+0000 osd.1 (osd.1) 14 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[84576]: cluster 2024-09-13T15:16:44.102880+0000 osd.1 (osd.1) 14 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[84576]: cluster 2024-09-13T15:16:44.166574+0000 osd.2 (osd.2) 33 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[84576]: cluster 2024-09-13T15:16:44.166574+0000 osd.2 (osd.2) 33 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[84576]: audit 2024-09-13T15:16:44.653766+0000 mon.a (mon.0) 2666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[84576]: audit 2024-09-13T15:16:44.653766+0000 mon.a (mon.0) 2666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[87147]: cluster 2024-09-13T15:16:43.682055+0000 osd.0 (osd.0) 28 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[87147]: cluster 2024-09-13T15:16:43.682055+0000 osd.0 (osd.0) 28 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[87147]: cluster 2024-09-13T15:16:43.820556+0000 osd.3 (osd.3) 7 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[87147]: cluster 2024-09-13T15:16:43.820556+0000 osd.3 (osd.3) 7 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[87147]: cluster 2024-09-13T15:16:44.102880+0000 osd.1 (osd.1) 14 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[87147]: cluster 2024-09-13T15:16:44.102880+0000 osd.1 (osd.1) 14 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:45.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[87147]: cluster 2024-09-13T15:16:44.166574+0000 osd.2 (osd.2) 33 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:45.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[87147]: cluster 2024-09-13T15:16:44.166574+0000 osd.2 (osd.2) 33 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:45.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[87147]: audit 2024-09-13T15:16:44.653766+0000 mon.a (mon.0) 2666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:45.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:44 smithi099 bash[87147]: audit 2024-09-13T15:16:44.653766+0000 mon.a (mon.0) 2666 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:45.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 85 16 100487 100471 73.8669 66 0.0137519 0.0135323 2024-09-13T15:16:45.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:45.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:45.178 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:45.204 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:45.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:45.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:45.311 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:45.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:45.338 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:45.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:45.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:45.473 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:45.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:45.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:45.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:45.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:45.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:45.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:45.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:45.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:45.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:45.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:45.866 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:45.868 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[84576]: cluster 2024-09-13T15:16:44.812832+0000 osd.3 (osd.3) 8 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:45.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:45.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:45.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:46.002 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:46.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:46.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:46.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:46.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[84576]: cluster 2024-09-13T15:16:44.812832+0000 osd.3 (osd.3) 8 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:46.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[84576]: cluster 2024-09-13T15:16:45.082411+0000 osd.1 (osd.1) 15 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:46.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[84576]: cluster 2024-09-13T15:16:45.082411+0000 osd.1 (osd.1) 15 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:46.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[84576]: cluster 2024-09-13T15:16:45.189144+0000 mgr.y (mgr.26252) 1522 : cluster [DBG] pgmap v2119: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5736/297570 objects misplaced (1.928%); 723 KiB/s, 11 objects/s recovering 2024-09-13T15:16:46.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[84576]: cluster 2024-09-13T15:16:45.189144+0000 mgr.y (mgr.26252) 1522 : cluster [DBG] pgmap v2119: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5736/297570 objects misplaced (1.928%); 723 KiB/s, 11 objects/s recovering 2024-09-13T15:16:46.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[84576]: audit 2024-09-13T15:16:45.189687+0000 mon.a (mon.0) 2667 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T15:16:46.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[84576]: audit 2024-09-13T15:16:45.189687+0000 mon.a (mon.0) 2667 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T15:16:46.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[84576]: cluster 2024-09-13T15:16:45.218443+0000 osd.2 (osd.2) 34 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:46.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[84576]: cluster 2024-09-13T15:16:45.218443+0000 osd.2 (osd.2) 34 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:46.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 86 16 101564 101548 73.7906 67.3125 0.00574632 0.0135395 2024-09-13T15:16:46.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[87147]: cluster 2024-09-13T15:16:44.812832+0000 osd.3 (osd.3) 8 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:46.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[87147]: cluster 2024-09-13T15:16:44.812832+0000 osd.3 (osd.3) 8 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:46.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[87147]: cluster 2024-09-13T15:16:45.082411+0000 osd.1 (osd.1) 15 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:46.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[87147]: cluster 2024-09-13T15:16:45.082411+0000 osd.1 (osd.1) 15 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:46.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[87147]: cluster 2024-09-13T15:16:45.189144+0000 mgr.y (mgr.26252) 1522 : cluster [DBG] pgmap v2119: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5736/297570 objects misplaced (1.928%); 723 KiB/s, 11 objects/s recovering 2024-09-13T15:16:46.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[87147]: cluster 2024-09-13T15:16:45.189144+0000 mgr.y (mgr.26252) 1522 : cluster [DBG] pgmap v2119: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5736/297570 objects misplaced (1.928%); 723 KiB/s, 11 objects/s recovering 2024-09-13T15:16:46.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[87147]: audit 2024-09-13T15:16:45.189687+0000 mon.a (mon.0) 2667 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T15:16:46.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[87147]: audit 2024-09-13T15:16:45.189687+0000 mon.a (mon.0) 2667 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T15:16:46.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[87147]: cluster 2024-09-13T15:16:45.218443+0000 osd.2 (osd.2) 34 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:46.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:45 smithi099 bash[87147]: cluster 2024-09-13T15:16:45.218443+0000 osd.2 (osd.2) 34 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:46.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:45 smithi177 bash[68594]: cluster 2024-09-13T15:16:44.812832+0000 osd.3 (osd.3) 8 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:46.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:45 smithi177 bash[68594]: cluster 2024-09-13T15:16:44.812832+0000 osd.3 (osd.3) 8 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:46.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:45 smithi177 bash[68594]: cluster 2024-09-13T15:16:45.082411+0000 osd.1 (osd.1) 15 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:46.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:45 smithi177 bash[68594]: cluster 2024-09-13T15:16:45.082411+0000 osd.1 (osd.1) 15 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:46.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:45 smithi177 bash[68594]: cluster 2024-09-13T15:16:45.189144+0000 mgr.y (mgr.26252) 1522 : cluster [DBG] pgmap v2119: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5736/297570 objects misplaced (1.928%); 723 KiB/s, 11 objects/s recovering 2024-09-13T15:16:46.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:45 smithi177 bash[68594]: cluster 2024-09-13T15:16:45.189144+0000 mgr.y (mgr.26252) 1522 : cluster [DBG] pgmap v2119: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 5736/297570 objects misplaced (1.928%); 723 KiB/s, 11 objects/s recovering 2024-09-13T15:16:46.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:45 smithi177 bash[68594]: audit 2024-09-13T15:16:45.189687+0000 mon.a (mon.0) 2667 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T15:16:46.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:45 smithi177 bash[68594]: audit 2024-09-13T15:16:45.189687+0000 mon.a (mon.0) 2667 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T15:16:46.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:45 smithi177 bash[68594]: cluster 2024-09-13T15:16:45.218443+0000 osd.2 (osd.2) 34 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:46.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:45 smithi177 bash[68594]: cluster 2024-09-13T15:16:45.218443+0000 osd.2 (osd.2) 34 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:46.133 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:46.156 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:46.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:46.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:46.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:46.277 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:46.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:46.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:46.380 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:46.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:46.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:46.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:46.505 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:46.527 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:46.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:46.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:46.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:46.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:46.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:46.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:46.756 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:46.777 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:46.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:46.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:46.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: cluster 2024-09-13T15:16:45.673318+0000 osd.0 (osd.0) 29 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:46.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: cluster 2024-09-13T15:16:45.673318+0000 osd.0 (osd.0) 29 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:46.819 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: audit 2024-09-13T15:16:45.817563+0000 mon.a (mon.0) 2668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T15:16:46.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:46.900 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:46.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:46.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:47.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:47.013 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: cluster 2024-09-13T15:16:45.673318+0000 osd.0 (osd.0) 29 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:47.013 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: cluster 2024-09-13T15:16:45.673318+0000 osd.0 (osd.0) 29 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:47.013 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: audit 2024-09-13T15:16:45.817563+0000 mon.a (mon.0) 2668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T15:16:47.013 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: audit 2024-09-13T15:16:45.817563+0000 mon.a (mon.0) 2668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T15:16:47.013 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: cluster 2024-09-13T15:16:45.876343+0000 mon.a (mon.0) 2669 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2024-09-13T15:16:47.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: cluster 2024-09-13T15:16:45.876343+0000 mon.a (mon.0) 2669 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2024-09-13T15:16:47.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: cluster 2024-09-13T15:16:46.127918+0000 osd.1 (osd.1) 16 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:47.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: cluster 2024-09-13T15:16:46.127918+0000 osd.1 (osd.1) 16 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:47.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: cluster 2024-09-13T15:16:46.193633+0000 osd.2 (osd.2) 35 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:47.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: cluster 2024-09-13T15:16:46.193633+0000 osd.2 (osd.2) 35 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:47.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: cluster 2024-09-13T15:16:46.658339+0000 mon.a (mon.0) 2670 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2024-09-13T15:16:47.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:46 smithi177 bash[68594]: cluster 2024-09-13T15:16:46.658339+0000 mon.a (mon.0) 2670 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2024-09-13T15:16:47.024 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:47.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:47.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:47.074 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 87 16 101863 101847 73.1573 18.6875 0.0046967 0.0135269 2024-09-13T15:16:47.082 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:16:47] "GET /metrics HTTP/1.1" 200 36774 "" "Prometheus/2.51.0" 2024-09-13T15:16:47.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: audit 2024-09-13T15:16:45.817563+0000 mon.a (mon.0) 2668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T15:16:47.082 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: cluster 2024-09-13T15:16:45.876343+0000 mon.a (mon.0) 2669 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2024-09-13T15:16:47.083 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: cluster 2024-09-13T15:16:45.876343+0000 mon.a (mon.0) 2669 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2024-09-13T15:16:47.083 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: cluster 2024-09-13T15:16:46.127918+0000 osd.1 (osd.1) 16 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:47.083 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: cluster 2024-09-13T15:16:46.127918+0000 osd.1 (osd.1) 16 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:47.083 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: cluster 2024-09-13T15:16:46.193633+0000 osd.2 (osd.2) 35 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:47.083 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: cluster 2024-09-13T15:16:46.193633+0000 osd.2 (osd.2) 35 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:47.083 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: cluster 2024-09-13T15:16:46.658339+0000 mon.a (mon.0) 2670 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2024-09-13T15:16:47.083 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[84576]: cluster 2024-09-13T15:16:46.658339+0000 mon.a (mon.0) 2670 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2024-09-13T15:16:47.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: cluster 2024-09-13T15:16:45.673318+0000 osd.0 (osd.0) 29 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:47.083 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: cluster 2024-09-13T15:16:45.673318+0000 osd.0 (osd.0) 29 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:47.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: audit 2024-09-13T15:16:45.817563+0000 mon.a (mon.0) 2668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T15:16:47.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: audit 2024-09-13T15:16:45.817563+0000 mon.a (mon.0) 2668 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T15:16:47.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: cluster 2024-09-13T15:16:45.876343+0000 mon.a (mon.0) 2669 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2024-09-13T15:16:47.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: cluster 2024-09-13T15:16:45.876343+0000 mon.a (mon.0) 2669 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2024-09-13T15:16:47.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: cluster 2024-09-13T15:16:46.127918+0000 osd.1 (osd.1) 16 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:47.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: cluster 2024-09-13T15:16:46.127918+0000 osd.1 (osd.1) 16 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:47.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: cluster 2024-09-13T15:16:46.193633+0000 osd.2 (osd.2) 35 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:47.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: cluster 2024-09-13T15:16:46.193633+0000 osd.2 (osd.2) 35 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:47.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: cluster 2024-09-13T15:16:46.658339+0000 mon.a (mon.0) 2670 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2024-09-13T15:16:47.084 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:46 smithi099 bash[87147]: cluster 2024-09-13T15:16:46.658339+0000 mon.a (mon.0) 2670 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2024-09-13T15:16:47.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:47.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:47.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:47.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:47.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:47.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:47.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:47.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:47.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:47.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:47.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:47.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:47.515 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:47.539 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:47.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:47.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:47.642 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:47.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:47.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:47.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:47.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:47.800 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:47.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:47.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:47.916 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:46.696559+0000 osd.0 (osd.0) 30 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:47.916 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:46.696559+0000 osd.0 (osd.0) 30 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:47.916 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:46.780768+0000 osd.3 (osd.3) 9 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:47.916 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:46.780768+0000 osd.3 (osd.3) 9 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:47.917 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.124217+0000 osd.1 (osd.1) 17 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:47.917 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.124217+0000 osd.1 (osd.1) 17 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:47.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:47.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:47.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:48.023 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:16:48.023 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T15:16:48.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T15:16:48.023 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:16:48.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.023 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45764.0:170115 44.e e1070/1056 44:7152e06c:::benchmark_data_smithi099_188632_object62911:head v 1070'10589, mlcod=1070'10585)", 2024-09-13T15:16:48.023 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:59.895062+0000", 2024-09-13T15:16:48.023 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 588.12351622999995, 2024-09-13T15:16:48.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.47213490499999999, 2024-09-13T15:16:48.023 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:59.895062+0000", 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:59.895062+0000", 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:59.895062+0000", 2024-09-13T15:16:48.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:16:48.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:59.895070+0000", 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.259e-06 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:59.895079+0000", 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2629999999999992e-06 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:59.895087+0000", 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2470000000000002e-06 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.321375+0000", 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.426288679 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.321434+0000", 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8742000000000001e-05 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.367138+0000", 2024-09-13T15:16:48.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.045704423000000001 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.367197+0000", 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8897999999999998e-05 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:170119 44.5 44:ad6f5617:::benchmark_data_smithi099_188632_object62915:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1070)", 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:06:59.896850+0000", 2024-09-13T15:16:48.028 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 588.12172876, 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45211088700000002, 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 170119 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:59.896850+0000", 2024-09-13T15:16:48.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:59.896850+0000", 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:59.896849+0000", 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:06:59.896865+0000", 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6640999999999999e-05 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.321194+0000", 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.42432902099999997 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.321217+0000", 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2255999999999998e-05 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.321560+0000", 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00034380000000000001 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.321997+0000", 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000436377 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.322318+0000", 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032143699999999999 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.343664+0000", 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021345876 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.344515+0000", 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00085125900000000004 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.345017+0000", 2024-09-13T15:16:48.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00050155499999999999 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.345171+0000", 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015365299999999999 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.348961+0000", 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037901800000000002 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:170176 44.6 44:68d7c2f4:::benchmark_data_smithi099_188632_object62972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1070)", 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:07:00.383839+0000", 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 587.63473981200002, 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51185867600000001, 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 170176 2024-09-13T15:16:48.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.383839+0000", 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.383839+0000", 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.383837+0000", 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.383846+0000", 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7520000000000002e-06 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.383869+0000", 2024-09-13T15:16:48.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3023999999999998e-05 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.383878+0000", 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2040000000000003e-06 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.383939+0000", 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1706999999999997e-05 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.384049+0000", 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109832 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.384182+0000", 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013246199999999999 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.392389+0000", 2024-09-13T15:16:48.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0082077130000000002 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.849010+0000", 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45662076899999998 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.895438+0000", 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.046428125000000001 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.040 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.895585+0000", 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014619500000000001 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:07:00.895697+0000", 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011291 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:210347 44.6 44:6c56b776:::benchmark_data_smithi099_188632_object103143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:16:48.041 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:08.817288+0000", 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 519.20129036699996, 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50000899200000004, 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 210347 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.817288+0000", 2024-09-13T15:16:48.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.817288+0000", 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.817287+0000", 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.817296+0000", 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9099999999999994e-06 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.817310+0000", 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4443e-05 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.817316+0000", 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6489999999999996e-06 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.817350+0000", 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4530000000000003e-05 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.817441+0000", 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0791999999999994e-05 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2024-09-13T15:16:48.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.817517+0000", 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6309000000000004e-05 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.830490+0000", 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01297302 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.832562+0000", 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020716440000000001 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:09.317199+0000", 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.48463701399999998 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:09.317247+0000", 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7587999999999999e-05 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:09.317297+0000", 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0593000000000002e-05 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.047 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:210357 44.5 44:ac8662d2:::benchmark_data_smithi099_188632_object103153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:08.828788+0000", 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 519.18979066899999, 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.48855737700000001, 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 210357 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.828788+0000", 2024-09-13T15:16:48.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.828788+0000", 2024-09-13T15:16:48.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.828786+0000", 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.828791+0000", 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0950000000000003e-06 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.828801+0000", 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1859999999999998e-06 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.828805+0000", 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5120000000000002e-06 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.829011+0000", 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020588299999999999 2024-09-13T15:16:48.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.829124+0000", 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011268399999999999 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.829233+0000", 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109188 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.838743+0000", 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0095104089999999992 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.839950+0000", 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012067250000000001 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:09.317231+0000", 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.47728114199999999 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:09.317304+0000", 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2643000000000002e-05 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:09.317345+0000", 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1572000000000001e-05 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.054 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:210358 44.5 44:ad8a40db:::benchmark_data_smithi099_188632_object103154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:08.829258+0000", 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 519.18932054200002, 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.48822728799999998, 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 210358 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.829258+0000", 2024-09-13T15:16:48.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.829258+0000", 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.829257+0000", 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.829262+0000", 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4399999999999996e-06 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.829274+0000", 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1756000000000001e-05 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.829291+0000", 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7193999999999998e-05 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.833604+0000", 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043129759999999996 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.833742+0000", 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013757900000000001 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.833894+0000", 2024-09-13T15:16:48.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000152292 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.839074+0000", 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051794659999999998 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:08.840054+0000", 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00097987399999999994 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:09.317399+0000", 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.47734543699999998 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:09.317460+0000", 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0902000000000001e-05 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:09.317485+0000", 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5511e-05 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.060 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213429 44.6 44:65cbf58a:::benchmark_data_smithi099_188632_object106225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.119444+0000", 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 513.89913457199998, 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.55019501400000004, 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:16:48.061 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213429 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.119444+0000", 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.119444+0000", 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.119443+0000", 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.119448+0000", 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7489999999999998e-06 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.119455+0000", 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0550000000000002e-06 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.119458+0000", 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4759999999999998e-06 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.119486+0000", 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8010000000000001e-05 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.119570+0000", 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3715999999999996e-05 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.119641+0000", 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1415000000000007e-05 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.123647+0000", 2024-09-13T15:16:48.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0040061760000000002 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.126124+0000", 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024767299999999999 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669536+0000", 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.54341233499999997 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669598+0000", 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1268999999999999e-05 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669639+0000", 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1316999999999998e-05 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213433 44.6 44:68d51522:::benchmark_data_smithi099_188632_object106229:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.126898+0000", 2024-09-13T15:16:48.067 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 513.89168028899996, 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.54291539799999999, 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213433 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.126898+0000", 2024-09-13T15:16:48.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.126898+0000", 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.126897+0000", 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.126902+0000", 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0089999999999998e-06 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.126912+0000", 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3330000000000003e-06 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.126917+0000", 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.366e-06 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.126955+0000", 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8322999999999999e-05 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.127080+0000", 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012417400000000001 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.127190+0000", 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110721 2024-09-13T15:16:48.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.132213+0000", 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050228479999999999 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.134015+0000", 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018020919999999999 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669708+0000", 2024-09-13T15:16:48.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53569275100000002 2024-09-13T15:16:48.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669762+0000", 2024-09-13T15:16:48.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3874e-05 2024-09-13T15:16:48.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669814+0000", 2024-09-13T15:16:48.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.189e-05 2024-09-13T15:16:48.074 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.074 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.074 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.074 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213435 44.5 44:a881811e:::benchmark_data_smithi099_188632_object106231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:16:48.074 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.129702+0000", 2024-09-13T15:16:48.074 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 513.88887676800005, 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.54007933100000005, 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213435 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.129702+0000", 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.129702+0000", 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.129701+0000", 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.129707+0000", 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0800000000000002e-06 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.129719+0000", 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1678999999999999e-05 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.129724+0000", 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0209999999999999e-06 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.129761+0000", 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7777000000000001e-05 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.129842+0000", 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0306e-05 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.129910+0000", 2024-09-13T15:16:48.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8683000000000006e-05 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.134832+0000", 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0049212819999999999 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.135198+0000", 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00036663 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669725+0000", 2024-09-13T15:16:48.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53452635400000004 2024-09-13T15:16:48.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669756+0000", 2024-09-13T15:16:48.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1896000000000001e-05 2024-09-13T15:16:48.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669781+0000", 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4712999999999999e-05 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop_reply(client.45764.0:213429 44.6 e1073/1056 ondisk, result = 0)", 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.139068+0000", 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 513.87951069099995, 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53057910200000002, 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.081 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.139068+0000", 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.139068+0000", 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.139067+0000", 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.139073+0000", 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2879999999999998e-06 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.139082+0000", 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5650000000000007e-06 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.139088+0000", 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8540000000000002e-06 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669494+0000", 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53040599200000005 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669526+0000", 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2206999999999998e-05 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669647+0000", 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120574 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop_reply(client.45764.0:213433 44.6 e1073/1056 ondisk, result = 0)", 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.139126+0000", 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 513.87945282099997, 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53069329700000001, 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.139126+0000", 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.139126+0000", 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.139125+0000", 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.139128+0000", 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2110000000000001e-06 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669496+0000", 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53036860500000005 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669518+0000", 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1752000000000001e-05 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669690+0000", 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000171865 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669700+0000", 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8700000000000004e-06 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.669819+0000", 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011935200000000001 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.089 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:34526 46.6 46:6cf47109:::benchmark_data_smithi099_233884_object34525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:36.837485+0000", 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 71.181094004000002, 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44961954100000001, 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 34526 2024-09-13T15:16:48.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.837485+0000", 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.837485+0000", 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.837483+0000", 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.837541+0000", 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7419e-05 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.837550+0000", 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1800000000000002e-06 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.837555+0000", 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8980000000000004e-06 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.837596+0000", 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1307999999999998e-05 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.837724+0000", 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012777 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.837842+0000", 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000118307 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.839202+0000", 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001359407 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.839564+0000", 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00036256899999999998 2024-09-13T15:16:48.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:37.286995+0000", 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44743073700000002 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:37.287052+0000", 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6538e-05 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:37.287104+0000", 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2571999999999997e-05 2024-09-13T15:16:48.095 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:34529 46.1 e1104/1099 46:8f331030:::benchmark_data_smithi099_233884_object34528:head v 1104'2133, mlcod=1104'2131)", 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:36.840306+0000", 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 71.178272520999997, 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45398951700000001, 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840306+0000", 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840306+0000", 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840305+0000", 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:16:48.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840366+0000", 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0238999999999997e-05 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840376+0000", 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0508e-05 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840381+0000", 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0390000000000004e-06 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840430+0000", 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8621000000000001e-05 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840449+0000", 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9717000000000001e-05 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:37.294192+0000", 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45374277800000001 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:37.294296+0000", 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000103385 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:34532 46.f e1104/1099 46:f3fa30e4:::benchmark_data_smithi099_233884_object34531:head v 1104'2109, mlcod=1104'2107)", 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:36.840405+0000", 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 71.178173237999999, 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45391646499999999, 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840405+0000", 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840405+0000", 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840405+0000", 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840427+0000", 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2277e-05 2024-09-13T15:16:48.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840434+0000", 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0489999999999998e-06 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840438+0000", 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9110000000000003e-06 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:36.840487+0000", 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8613999999999999e-05 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:37.286964+0000", 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44647756199999999 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:37.294291+0000", 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073269520000000003 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:37.294322+0000", 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0507000000000001e-05 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:53829 46.6 46:6eaa05fc:::benchmark_data_smithi099_233884_object53828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:50.811735+0000", 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 57.206843280999998, 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.48090934099999999, 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.105 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 53829 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811735+0000", 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811735+0000", 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811734+0000", 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811796+0000", 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2223000000000004e-05 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811811+0000", 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4396000000000001e-05 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811817+0000", 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8290000000000002e-06 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811864+0000", 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6749999999999998e-05 2024-09-13T15:16:48.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811975+0000", 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110584 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.812070+0000", 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5697000000000002e-05 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.813604+0000", 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015338890000000001 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.814008+0000", 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00040333300000000002 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.292548+0000", 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.478540309 2024-09-13T15:16:48.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.292599+0000", 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1319000000000001e-05 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.292645+0000", 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5453e-05 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.111 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:88714 46.5 e1106/1099 46:ab9030a0:::benchmark_data_smithi099_233884_object88713:head v 1106'5491, mlcod=1106'5487)", 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:30.090741+0000", 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.927837692000001, 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.524941883, 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090741+0000", 2024-09-13T15:16:48.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090741+0000", 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090740+0000", 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090803+0000", 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2986999999999999e-05 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090813+0000", 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4220000000000006e-06 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090818+0000", 2024-09-13T15:16:48.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9590000000000003e-06 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090870+0000", 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2339999999999997e-05 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585133+0000", 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.494263491 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.615617+0000", 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.030484003999999999 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.615683+0000", 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5455000000000003e-05 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:88711 46.e e1106/1099 46:768b3877:::benchmark_data_smithi099_233884_object88710:head v 1106'5512, mlcod=1106'5509)", 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:30.092814+0000", 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.92576481, 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50584367299999999, 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.092814+0000", 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.092814+0000", 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.092813+0000", 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.092886+0000", 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3215999999999998e-05 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585069+0000", 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.492183174 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585100+0000", 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0576e-05 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585401+0000", 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000301434 2024-09-13T15:16:48.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585471+0000", 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9888000000000006e-05 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.598592+0000", 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013120427 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.598657+0000", 2024-09-13T15:16:48.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5864000000000005e-05 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:88715 46.5 e1106/1099 46:ad92bd9f:::benchmark_data_smithi099_233884_object88714:head v 1106'5492, mlcod=1106'5487)", 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:30.093743+0000", 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.924835308999999, 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.521982485, 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.093743+0000", 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.093743+0000", 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.093742+0000", 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:16:48.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.093855+0000", 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011267199999999999 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585142+0000", 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49128675799999999 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585161+0000", 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8848999999999999e-05 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.591914+0000", 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067532299999999998 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.591962+0000", 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8696000000000003e-05 2024-09-13T15:16:48.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.615696+0000", 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023733204000000001 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.615726+0000", 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0085000000000001e-05 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.125 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89298 46.6 46:618b79f4:::benchmark_data_smithi099_233884_object89297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.716825+0000", 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 16.301753171000001, 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.033399011, 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89298 2024-09-13T15:16:48.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716825+0000", 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716825+0000", 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716824+0000", 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716876+0000", 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2182000000000003e-05 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716888+0000", 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1572e-05 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716893+0000", 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9559999999999996e-06 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.974099+0000", 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25720631100000002 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.974111+0000", 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1948e-05 2024-09-13T15:16:48.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.750142+0000", 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.77603100599999997 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.750224+0000", 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2245999999999998e-05 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89316 46.16 46:6f82b5f7:::benchmark_data_smithi099_233884_object89315:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.744659+0000", 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 16.273919251999999, 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0223355249999999, 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89316 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:16:48.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744659+0000", 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744659+0000", 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744657+0000", 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744719+0000", 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2253000000000002e-05 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744729+0000", 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0032e-05 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744735+0000", 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3159999999999996e-06 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.974133+0000", 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.229398243 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.974138+0000", 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9359999999999998e-06 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.759984+0000", 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.78584600699999996 2024-09-13T15:16:48.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.760180+0000", 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019581500000000001 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.760329+0000", 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014914099999999999 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.762793+0000", 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024643849999999999 2024-09-13T15:16:48.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.763201+0000", 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00040765000000000002 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.766794+0000", 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035933530000000001 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.766873+0000", 2024-09-13T15:16:48.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8777999999999997e-05 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.766995+0000", 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012185599999999999 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:16:48.137 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:16:48.137 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:48.138 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 88 16 102105 102089 72.4978 15.125 0.012257 0.0137836 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:46.696559+0000 osd.0 (osd.0) 30 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:46.696559+0000 osd.0 (osd.0) 30 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:46.780768+0000 osd.3 (osd.3) 9 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:46.780768+0000 osd.3 (osd.3) 9 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.124217+0000 osd.1 (osd.1) 17 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.124217+0000 osd.1 (osd.1) 17 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.189759+0000 mgr.y (mgr.26252) 1523 : cluster [DBG] pgmap v2122: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 5736/299433 objects misplaced (1.916%); 725 KiB/s, 11 objects/s recovering 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.189759+0000 mgr.y (mgr.26252) 1523 : cluster [DBG] pgmap v2122: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 5736/299433 objects misplaced (1.916%); 725 KiB/s, 11 objects/s recovering 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: audit 2024-09-13T15:16:47.190882+0000 mon.a (mon.0) 2671 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: audit 2024-09-13T15:16:47.190882+0000 mon.a (mon.0) 2671 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.243486+0000 osd.2 (osd.2) 36 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.243486+0000 osd.2 (osd.2) 36 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: audit 2024-09-13T15:16:47.657795+0000 mon.a (mon.0) 2672 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T15:16:48.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: audit 2024-09-13T15:16:47.657795+0000 mon.a (mon.0) 2672 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T15:16:48.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.664894+0000 mon.a (mon.0) 2673 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2024-09-13T15:16:48.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:47 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.664894+0000 mon.a (mon.0) 2673 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2024-09-13T15:16:48.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:48.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:48.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:48.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:48.292 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:48.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:48.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:48.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:46.696559+0000 osd.0 (osd.0) 30 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:48.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:46.696559+0000 osd.0 (osd.0) 30 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:48.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:46.780768+0000 osd.3 (osd.3) 9 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:46.780768+0000 osd.3 (osd.3) 9 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.124217+0000 osd.1 (osd.1) 17 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.124217+0000 osd.1 (osd.1) 17 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.189759+0000 mgr.y (mgr.26252) 1523 : cluster [DBG] pgmap v2122: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 5736/299433 objects misplaced (1.916%); 725 KiB/s, 11 objects/s recovering 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.189759+0000 mgr.y (mgr.26252) 1523 : cluster [DBG] pgmap v2122: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 5736/299433 objects misplaced (1.916%); 725 KiB/s, 11 objects/s recovering 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: audit 2024-09-13T15:16:47.190882+0000 mon.a (mon.0) 2671 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: audit 2024-09-13T15:16:47.190882+0000 mon.a (mon.0) 2671 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.243486+0000 osd.2 (osd.2) 36 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.243486+0000 osd.2 (osd.2) 36 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: audit 2024-09-13T15:16:47.657795+0000 mon.a (mon.0) 2672 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: audit 2024-09-13T15:16:47.657795+0000 mon.a (mon.0) 2672 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.664894+0000 mon.a (mon.0) 2673 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2024-09-13T15:16:48.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.664894+0000 mon.a (mon.0) 2673 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2024-09-13T15:16:48.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.189759+0000 mgr.y (mgr.26252) 1523 : cluster [DBG] pgmap v2122: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 5736/299433 objects misplaced (1.916%); 725 KiB/s, 11 objects/s recovering 2024-09-13T15:16:48.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.189759+0000 mgr.y (mgr.26252) 1523 : cluster [DBG] pgmap v2122: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 5736/299433 objects misplaced (1.916%); 725 KiB/s, 11 objects/s recovering 2024-09-13T15:16:48.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: audit 2024-09-13T15:16:47.190882+0000 mon.a (mon.0) 2671 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T15:16:48.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: audit 2024-09-13T15:16:47.190882+0000 mon.a (mon.0) 2671 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T15:16:48.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.243486+0000 osd.2 (osd.2) 36 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:48.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.243486+0000 osd.2 (osd.2) 36 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:48.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: audit 2024-09-13T15:16:47.657795+0000 mon.a (mon.0) 2672 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T15:16:48.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: audit 2024-09-13T15:16:47.657795+0000 mon.a (mon.0) 2672 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T15:16:48.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.664894+0000 mon.a (mon.0) 2673 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2024-09-13T15:16:48.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:47 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.664894+0000 mon.a (mon.0) 2673 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2024-09-13T15:16:48.397 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:48.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:48.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:48.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:48.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:48.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:48.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:48.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:48.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:48.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:48.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:48.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:48.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:48.799 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:48.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:48.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:48.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:48.930 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:48.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:48.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:49.033 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:49.045 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:48 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.649649+0000 osd.0 (osd.0) 31 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:49.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:48 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.649649+0000 osd.0 (osd.0) 31 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:49.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:48 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.739875+0000 osd.3 (osd.3) 10 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:49.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:48 smithi177 bash[68594]: cluster 2024-09-13T15:16:47.739875+0000 osd.3 (osd.3) 10 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:49.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:48 smithi177 bash[68594]: cluster 2024-09-13T15:16:48.253286+0000 osd.2 (osd.2) 37 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:49.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:48 smithi177 bash[68594]: cluster 2024-09-13T15:16:48.253286+0000 osd.2 (osd.2) 37 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:49.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:48 smithi177 bash[68594]: audit 2024-09-13T15:16:48.316762+0000 mon.a (mon.0) 2674 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:49.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:48 smithi177 bash[68594]: audit 2024-09-13T15:16:48.316762+0000 mon.a (mon.0) 2674 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:49.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:48 smithi177 bash[68594]: cluster 2024-09-13T15:16:48.662342+0000 mon.a (mon.0) 2675 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2024-09-13T15:16:49.046 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:48 smithi177 bash[68594]: cluster 2024-09-13T15:16:48.662342+0000 mon.a (mon.0) 2675 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2024-09-13T15:16:49.060 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:49.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:49.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 89 16 102410 102394 71.8974 19.0625 0.00640317 0.0137671 2024-09-13T15:16:49.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:16:49.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:49.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.649649+0000 osd.0 (osd.0) 31 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:49.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.649649+0000 osd.0 (osd.0) 31 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:49.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.739875+0000 osd.3 (osd.3) 10 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:49.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:47.739875+0000 osd.3 (osd.3) 10 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:49.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:48.253286+0000 osd.2 (osd.2) 37 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:49.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:48.253286+0000 osd.2 (osd.2) 37 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:49.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: audit 2024-09-13T15:16:48.316762+0000 mon.a (mon.0) 2674 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:49.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: audit 2024-09-13T15:16:48.316762+0000 mon.a (mon.0) 2674 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:49.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:48.662342+0000 mon.a (mon.0) 2675 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2024-09-13T15:16:49.176 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:48.662342+0000 mon.a (mon.0) 2675 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2024-09-13T15:16:49.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:48 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.649649+0000 osd.0 (osd.0) 31 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:49.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:48 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.649649+0000 osd.0 (osd.0) 31 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:49.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:48 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.739875+0000 osd.3 (osd.3) 10 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:49.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:48 smithi099 bash[87147]: cluster 2024-09-13T15:16:47.739875+0000 osd.3 (osd.3) 10 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:49.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:48 smithi099 bash[87147]: cluster 2024-09-13T15:16:48.253286+0000 osd.2 (osd.2) 37 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:49.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:48 smithi099 bash[87147]: cluster 2024-09-13T15:16:48.253286+0000 osd.2 (osd.2) 37 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:49.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:48 smithi099 bash[87147]: audit 2024-09-13T15:16:48.316762+0000 mon.a (mon.0) 2674 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:49.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:48 smithi099 bash[87147]: audit 2024-09-13T15:16:48.316762+0000 mon.a (mon.0) 2674 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:16:49.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:48.662342+0000 mon.a (mon.0) 2675 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2024-09-13T15:16:49.177 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:48.662342+0000 mon.a (mon.0) 2675 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2024-09-13T15:16:49.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:49.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:49.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:49.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:49.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:49.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:49.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:49.423 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:49.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:49.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:49.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:49.553 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:49.579 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:49.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:49.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:49.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:49.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:49.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:49.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:49.813 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:49.841 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:49.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:49.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:49.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:49.956 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:48.157466+0000 osd.1 (osd.1) 18 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:49.956 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:48.157466+0000 osd.1 (osd.1) 18 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:49.956 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:48.670914+0000 osd.0 (osd.0) 32 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:49.956 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:48.670914+0000 osd.0 (osd.0) 32 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:49.956 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:48.699389+0000 osd.3 (osd.3) 11 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:49.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:48.699389+0000 osd.3 (osd.3) 11 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:49.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:49.190508+0000 mgr.y (mgr.26252) 1524 : cluster [DBG] pgmap v2125: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 5298/305238 objects misplaced (1.736%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T15:16:49.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:49.190508+0000 mgr.y (mgr.26252) 1524 : cluster [DBG] pgmap v2125: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 5298/305238 objects misplaced (1.736%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T15:16:49.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:49.211965+0000 osd.2 (osd.2) 38 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:49.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:49.211965+0000 osd.2 (osd.2) 38 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:49.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:49.666143+0000 mon.a (mon.0) 2676 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2024-09-13T15:16:49.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:49 smithi177 bash[68594]: cluster 2024-09-13T15:16:49.666143+0000 mon.a (mon.0) 2676 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2024-09-13T15:16:49.970 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:49.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:49.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:50.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:50.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout: 90 16 102410 102394 71.0985 0 - 0.0137671 2024-09-13T15:16:50.096 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:50.097 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:48.157466+0000 osd.1 (osd.1) 18 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:48.157466+0000 osd.1 (osd.1) 18 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:48.670914+0000 osd.0 (osd.0) 32 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:48.670914+0000 osd.0 (osd.0) 32 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:48.699389+0000 osd.3 (osd.3) 11 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:48.699389+0000 osd.3 (osd.3) 11 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:49.190508+0000 mgr.y (mgr.26252) 1524 : cluster [DBG] pgmap v2125: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 5298/305238 objects misplaced (1.736%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:49.190508+0000 mgr.y (mgr.26252) 1524 : cluster [DBG] pgmap v2125: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 5298/305238 objects misplaced (1.736%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:49.211965+0000 osd.2 (osd.2) 38 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:49.211965+0000 osd.2 (osd.2) 38 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:49.666143+0000 mon.a (mon.0) 2676 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2024-09-13T15:16:50.098 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[87147]: cluster 2024-09-13T15:16:49.666143+0000 mon.a (mon.0) 2676 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:48.157466+0000 osd.1 (osd.1) 18 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:48.157466+0000 osd.1 (osd.1) 18 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:48.670914+0000 osd.0 (osd.0) 32 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:48.670914+0000 osd.0 (osd.0) 32 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:48.699389+0000 osd.3 (osd.3) 11 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:48.699389+0000 osd.3 (osd.3) 11 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:49.190508+0000 mgr.y (mgr.26252) 1524 : cluster [DBG] pgmap v2125: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 5298/305238 objects misplaced (1.736%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:49.190508+0000 mgr.y (mgr.26252) 1524 : cluster [DBG] pgmap v2125: 99 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 1 active+recovering+undersized+remapped, 96 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 5298/305238 objects misplaced (1.736%); 3.4 MiB/s, 54 objects/s recovering 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:49.211965+0000 osd.2 (osd.2) 38 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:49.211965+0000 osd.2 (osd.2) 38 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:50.099 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:49.666143+0000 mon.a (mon.0) 2676 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2024-09-13T15:16:50.100 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:49 smithi099 bash[84576]: cluster 2024-09-13T15:16:49.666143+0000 mon.a (mon.0) 2676 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2024-09-13T15:16:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:50.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:50.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:50.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:50.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:50.325 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:50.348 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:50.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:50.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:50.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:50.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:50.472 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:16:50 smithi177 bash[61439]: ts=2024-09-13T15:16:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:16:50.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:50.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:50.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:50.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:50.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:50.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:50.698 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:50.721 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:50.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:50.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:50.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:50.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:50.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:50.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:50.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:50.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[84576]: cluster 2024-09-13T15:16:49.173055+0000 osd.1 (osd.1) 19 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:50.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[84576]: cluster 2024-09-13T15:16:49.173055+0000 osd.1 (osd.1) 19 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:50.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[84576]: cluster 2024-09-13T15:16:49.658397+0000 osd.0 (osd.0) 33 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:50.969 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[84576]: cluster 2024-09-13T15:16:49.658397+0000 osd.0 (osd.0) 33 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:50.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[84576]: cluster 2024-09-13T15:16:50.137932+0000 osd.1 (osd.1) 20 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:50.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[84576]: cluster 2024-09-13T15:16:50.137932+0000 osd.1 (osd.1) 20 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:50.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[84576]: cluster 2024-09-13T15:16:50.242138+0000 osd.2 (osd.2) 39 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:50.970 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[84576]: cluster 2024-09-13T15:16:50.242138+0000 osd.2 (osd.2) 39 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:50.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[87147]: cluster 2024-09-13T15:16:49.173055+0000 osd.1 (osd.1) 19 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:50.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[87147]: cluster 2024-09-13T15:16:49.173055+0000 osd.1 (osd.1) 19 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:50.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[87147]: cluster 2024-09-13T15:16:49.658397+0000 osd.0 (osd.0) 33 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:50.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[87147]: cluster 2024-09-13T15:16:49.658397+0000 osd.0 (osd.0) 33 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:50.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[87147]: cluster 2024-09-13T15:16:50.137932+0000 osd.1 (osd.1) 20 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:50.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[87147]: cluster 2024-09-13T15:16:50.137932+0000 osd.1 (osd.1) 20 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:50.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[87147]: cluster 2024-09-13T15:16:50.242138+0000 osd.2 (osd.2) 39 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:50.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:50 smithi099 bash[87147]: cluster 2024-09-13T15:16:50.242138+0000 osd.2 (osd.2) 39 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:50.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:50.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:51.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total time run: 90.0992 2024-09-13T15:16:51.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Total writes made: 102410 2024-09-13T15:16:51.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Write size: 65536 2024-09-13T15:16:51.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Object size: 65536 2024-09-13T15:16:51.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Bandwidth (MB/sec): 71.0398 2024-09-13T15:16:51.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Bandwidth: 39.8684 2024-09-13T15:16:51.075 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max bandwidth (MB/sec): 195.375 2024-09-13T15:16:51.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min bandwidth (MB/sec): 0 2024-09-13T15:16:51.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average IOPS: 1136 2024-09-13T15:16:51.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev IOPS: 637.895 2024-09-13T15:16:51.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max IOPS: 3126 2024-09-13T15:16:51.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min IOPS: 0 2024-09-13T15:16:51.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Average Latency(s): 0.0140713 2024-09-13T15:16:51.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Stddev Latency(s): 0.0555869 2024-09-13T15:16:51.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Max latency(s): 2.80921 2024-09-13T15:16:51.076 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Min latency(s): 0.00120222 2024-09-13T15:16:51.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:51.081 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T15:16:51.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:50 smithi177 bash[68594]: cluster 2024-09-13T15:16:49.173055+0000 osd.1 (osd.1) 19 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:51.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:50 smithi177 bash[68594]: cluster 2024-09-13T15:16:49.173055+0000 osd.1 (osd.1) 19 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:51.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:50 smithi177 bash[68594]: cluster 2024-09-13T15:16:49.658397+0000 osd.0 (osd.0) 33 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:51.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:50 smithi177 bash[68594]: cluster 2024-09-13T15:16:49.658397+0000 osd.0 (osd.0) 33 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:51.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:50 smithi177 bash[68594]: cluster 2024-09-13T15:16:50.137932+0000 osd.1 (osd.1) 20 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:51.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:50 smithi177 bash[68594]: cluster 2024-09-13T15:16:50.137932+0000 osd.1 (osd.1) 20 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:51.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:50 smithi177 bash[68594]: cluster 2024-09-13T15:16:50.242138+0000 osd.2 (osd.2) 39 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:51.093 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:50 smithi177 bash[68594]: cluster 2024-09-13T15:16:50.242138+0000 osd.2 (osd.2) 39 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:51.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:51.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:51.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:51.216 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:51.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:51.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:51.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:51.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:51.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:51.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:51.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:51.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:51.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:51.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:51.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:51.651 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:51.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:51.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:51.754 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:51.779 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:51.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:51.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:51.882 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:51.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:51.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:51.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:52.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:52.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:51 smithi177 bash[68594]: cluster 2024-09-13T15:16:50.646141+0000 osd.0 (osd.0) 34 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:52.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:51 smithi177 bash[68594]: cluster 2024-09-13T15:16:50.646141+0000 osd.0 (osd.0) 34 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:52.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:51 smithi177 bash[68594]: cluster 2024-09-13T15:16:51.191400+0000 mgr.y (mgr.26252) 1525 : cluster [DBG] pgmap v2127: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s; 8268/307050 objects misplaced (2.693%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:16:52.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:51 smithi177 bash[68594]: cluster 2024-09-13T15:16:51.191400+0000 mgr.y (mgr.26252) 1525 : cluster [DBG] pgmap v2127: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s; 8268/307050 objects misplaced (2.693%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:16:52.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:52.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:52.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:52.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:51 smithi099 bash[84576]: cluster 2024-09-13T15:16:50.646141+0000 osd.0 (osd.0) 34 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:52.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:51 smithi099 bash[84576]: cluster 2024-09-13T15:16:50.646141+0000 osd.0 (osd.0) 34 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:52.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:51 smithi099 bash[84576]: cluster 2024-09-13T15:16:51.191400+0000 mgr.y (mgr.26252) 1525 : cluster [DBG] pgmap v2127: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s; 8268/307050 objects misplaced (2.693%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:16:52.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:51 smithi099 bash[84576]: cluster 2024-09-13T15:16:51.191400+0000 mgr.y (mgr.26252) 1525 : cluster [DBG] pgmap v2127: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s; 8268/307050 objects misplaced (2.693%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:16:52.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:51 smithi099 bash[87147]: cluster 2024-09-13T15:16:50.646141+0000 osd.0 (osd.0) 34 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:52.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:51 smithi099 bash[87147]: cluster 2024-09-13T15:16:50.646141+0000 osd.0 (osd.0) 34 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:16:52.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:51 smithi099 bash[87147]: cluster 2024-09-13T15:16:51.191400+0000 mgr.y (mgr.26252) 1525 : cluster [DBG] pgmap v2127: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s; 8268/307050 objects misplaced (2.693%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:16:52.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:51 smithi099 bash[87147]: cluster 2024-09-13T15:16:51.191400+0000 mgr.y (mgr.26252) 1525 : cluster [DBG] pgmap v2127: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s; 8268/307050 objects misplaced (2.693%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:16:52.154 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:52.192 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:52.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:52.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:52.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:52.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:52.319 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:52.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:52.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:52.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:52.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:52.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:52.559 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:52.590 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:52.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:52.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:52.696 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:52.724 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:52.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:52.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:52.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:52.859 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:52.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:52.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:52.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:52.988 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:52.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:53.092 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:53.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:52 smithi099 bash[84576]: cluster 2024-09-13T15:16:51.260587+0000 osd.1 (osd.1) 21 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:53.101 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:52 smithi099 bash[84576]: cluster 2024-09-13T15:16:51.260587+0000 osd.1 (osd.1) 21 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:53.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:52 smithi099 bash[87147]: cluster 2024-09-13T15:16:51.260587+0000 osd.1 (osd.1) 21 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:53.102 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:52 smithi099 bash[87147]: cluster 2024-09-13T15:16:51.260587+0000 osd.1 (osd.1) 21 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:53.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:52 smithi177 bash[68594]: cluster 2024-09-13T15:16:51.260587+0000 osd.1 (osd.1) 21 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:53.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:52 smithi177 bash[68594]: cluster 2024-09-13T15:16:51.260587+0000 osd.1 (osd.1) 21 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:53.118 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:53.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:53.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:53.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:53.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:53.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:53.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:53.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:53.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:53.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:53.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:53.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:53.504 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:53.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:53.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:53.608 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:53.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:53.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:53.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:53.744 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:53.770 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:53.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:53.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:53.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:53.896 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:53.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:53.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:53.906 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:16:54.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:54.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:54.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:54.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:54.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:54.144 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:53 smithi177 bash[68594]: cluster 2024-09-13T15:16:53.191970+0000 mgr.y (mgr.26252) 1526 : cluster [DBG] pgmap v2128: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 27 MiB/s wr, 480 op/s; 8268/306429 objects misplaced (2.698%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T15:16:54.144 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:53 smithi177 bash[68594]: cluster 2024-09-13T15:16:53.191970+0000 mgr.y (mgr.26252) 1526 : cluster [DBG] pgmap v2128: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 27 MiB/s wr, 480 op/s; 8268/306429 objects misplaced (2.698%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T15:16:54.144 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:53 smithi177 bash[68594]: cluster 2024-09-13T15:16:53.292927+0000 osd.2 (osd.2) 40 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:54.145 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:53 smithi177 bash[68594]: cluster 2024-09-13T15:16:53.292927+0000 osd.2 (osd.2) 40 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:54.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:54.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:54.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:54.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:54.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:54.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:54.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:53 smithi099 bash[84576]: cluster 2024-09-13T15:16:53.191970+0000 mgr.y (mgr.26252) 1526 : cluster [DBG] pgmap v2128: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 27 MiB/s wr, 480 op/s; 8268/306429 objects misplaced (2.698%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T15:16:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:53 smithi099 bash[84576]: cluster 2024-09-13T15:16:53.191970+0000 mgr.y (mgr.26252) 1526 : cluster [DBG] pgmap v2128: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 27 MiB/s wr, 480 op/s; 8268/306429 objects misplaced (2.698%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T15:16:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:53 smithi099 bash[84576]: cluster 2024-09-13T15:16:53.292927+0000 osd.2 (osd.2) 40 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:54.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:53 smithi099 bash[84576]: cluster 2024-09-13T15:16:53.292927+0000 osd.2 (osd.2) 40 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:54.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:53 smithi099 bash[87147]: cluster 2024-09-13T15:16:53.191970+0000 mgr.y (mgr.26252) 1526 : cluster [DBG] pgmap v2128: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 27 MiB/s wr, 480 op/s; 8268/306429 objects misplaced (2.698%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T15:16:54.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:53 smithi099 bash[87147]: cluster 2024-09-13T15:16:53.191970+0000 mgr.y (mgr.26252) 1526 : cluster [DBG] pgmap v2128: 99 pgs: 2 active+clean+scrubbing, 1 remapped+peering, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 27 MiB/s wr, 480 op/s; 8268/306429 objects misplaced (2.698%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T15:16:54.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:53 smithi099 bash[87147]: cluster 2024-09-13T15:16:53.292927+0000 osd.2 (osd.2) 40 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:54.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:53 smithi099 bash[87147]: cluster 2024-09-13T15:16:53.292927+0000 osd.2 (osd.2) 40 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:54.398 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:54.430 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:54.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:54.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:54.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:54.563 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:54.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:54.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:54.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:54.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:54.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:54.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:54.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:54.831 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:54.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:54.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:54.934 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:54.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:54.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:55.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:55.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:55.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:55.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:55.189 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:55.204 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:54 smithi099 bash[84576]: cluster 2024-09-13T15:16:53.288645+0000 osd.1 (osd.1) 22 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:55.204 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:54 smithi099 bash[84576]: cluster 2024-09-13T15:16:53.288645+0000 osd.1 (osd.1) 22 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:55.204 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:54 smithi099 bash[84576]: cluster 2024-09-13T15:16:53.632366+0000 osd.3 (osd.3) 12 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:55.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:55.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:55.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:55.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:55.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:54 smithi177 bash[68594]: cluster 2024-09-13T15:16:53.288645+0000 osd.1 (osd.1) 22 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:55.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:54 smithi177 bash[68594]: cluster 2024-09-13T15:16:53.288645+0000 osd.1 (osd.1) 22 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:55.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:54 smithi177 bash[68594]: cluster 2024-09-13T15:16:53.632366+0000 osd.3 (osd.3) 12 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:55.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:54 smithi177 bash[68594]: cluster 2024-09-13T15:16:53.632366+0000 osd.3 (osd.3) 12 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:55.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:54 smithi177 bash[68594]: cluster 2024-09-13T15:16:54.247982+0000 osd.1 (osd.1) 23 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:55.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:54 smithi177 bash[68594]: cluster 2024-09-13T15:16:54.247982+0000 osd.1 (osd.1) 23 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:55.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:54 smithi177 bash[68594]: cluster 2024-09-13T15:16:54.323523+0000 osd.2 (osd.2) 41 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:55.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:54 smithi177 bash[68594]: cluster 2024-09-13T15:16:54.323523+0000 osd.2 (osd.2) 41 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:55.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:54 smithi177 bash[68594]: audit 2024-09-13T15:16:54.666763+0000 mon.a (mon.0) 2677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:55.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:54 smithi177 bash[68594]: audit 2024-09-13T15:16:54.666763+0000 mon.a (mon.0) 2677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:55.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:55.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:55.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:55.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:55.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:53.632366+0000 osd.3 (osd.3) 12 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:55.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:54.247982+0000 osd.1 (osd.1) 23 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:55.480 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:54.247982+0000 osd.1 (osd.1) 23 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:55.481 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:54.323523+0000 osd.2 (osd.2) 41 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:55.481 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:54.323523+0000 osd.2 (osd.2) 41 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:55.481 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: audit 2024-09-13T15:16:54.666763+0000 mon.a (mon.0) 2677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:55.481 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: audit 2024-09-13T15:16:54.666763+0000 mon.a (mon.0) 2677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:55.481 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:53.288645+0000 osd.1 (osd.1) 22 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:55.481 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:53.288645+0000 osd.1 (osd.1) 22 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:55.481 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:53.632366+0000 osd.3 (osd.3) 12 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:55.481 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:53.632366+0000 osd.3 (osd.3) 12 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:55.481 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:54.247982+0000 osd.1 (osd.1) 23 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:55.481 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:54.247982+0000 osd.1 (osd.1) 23 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:16:55.482 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:54.323523+0000 osd.2 (osd.2) 41 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:55.482 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:54.323523+0000 osd.2 (osd.2) 41 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:55.482 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: audit 2024-09-13T15:16:54.666763+0000 mon.a (mon.0) 2677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:55.482 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: audit 2024-09-13T15:16:54.666763+0000 mon.a (mon.0) 2677 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:16:55.491 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:55.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:55.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:55.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:55.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:55.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:55.717 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:55.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:55.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:55.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:55.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:55.864 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:55.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:55.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:55.986 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:55.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:55.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:56.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:56.102 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: cluster 2024-09-13T15:16:54.620748+0000 osd.3 (osd.3) 13 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: cluster 2024-09-13T15:16:54.620748+0000 osd.3 (osd.3) 13 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: cluster 2024-09-13T15:16:54.694011+0000 osd.0 (osd.0) 35 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: cluster 2024-09-13T15:16:54.694011+0000 osd.0 (osd.0) 35 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: cluster 2024-09-13T15:16:55.192717+0000 mgr.y (mgr.26252) 1527 : cluster [DBG] pgmap v2129: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 22 MiB/s wr, 745 op/s; 10385/298338 objects misplaced (3.481%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:16:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: cluster 2024-09-13T15:16:55.192717+0000 mgr.y (mgr.26252) 1527 : cluster [DBG] pgmap v2129: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 22 MiB/s wr, 745 op/s; 10385/298338 objects misplaced (3.481%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:16:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: audit 2024-09-13T15:16:55.193564+0000 mon.a (mon.0) 2678 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T15:16:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: audit 2024-09-13T15:16:55.193564+0000 mon.a (mon.0) 2678 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T15:16:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: cluster 2024-09-13T15:16:55.245575+0000 osd.1 (osd.1) 24 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: cluster 2024-09-13T15:16:55.245575+0000 osd.1 (osd.1) 24 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:56.103 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: cluster 2024-09-13T15:16:55.305789+0000 osd.2 (osd.2) 42 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:56.104 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:55 smithi177 bash[68594]: cluster 2024-09-13T15:16:55.305789+0000 osd.2 (osd.2) 42 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:56.117 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:56.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:56.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:56.221 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:56.231 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:54.620748+0000 osd.3 (osd.3) 13 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:56.231 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:54.620748+0000 osd.3 (osd.3) 13 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:56.231 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:54.694011+0000 osd.0 (osd.0) 35 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:56.231 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:54.694011+0000 osd.0 (osd.0) 35 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:56.231 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:55.192717+0000 mgr.y (mgr.26252) 1527 : cluster [DBG] pgmap v2129: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 22 MiB/s wr, 745 op/s; 10385/298338 objects misplaced (3.481%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:16:56.232 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:55.192717+0000 mgr.y (mgr.26252) 1527 : cluster [DBG] pgmap v2129: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 22 MiB/s wr, 745 op/s; 10385/298338 objects misplaced (3.481%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:16:56.232 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: audit 2024-09-13T15:16:55.193564+0000 mon.a (mon.0) 2678 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T15:16:56.232 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: audit 2024-09-13T15:16:55.193564+0000 mon.a (mon.0) 2678 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T15:16:56.232 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:55.245575+0000 osd.1 (osd.1) 24 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:56.232 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:55.245575+0000 osd.1 (osd.1) 24 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:56.232 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:55.305789+0000 osd.2 (osd.2) 42 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:56.232 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[84576]: cluster 2024-09-13T15:16:55.305789+0000 osd.2 (osd.2) 42 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:54.620748+0000 osd.3 (osd.3) 13 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:54.620748+0000 osd.3 (osd.3) 13 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:54.694011+0000 osd.0 (osd.0) 35 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:54.694011+0000 osd.0 (osd.0) 35 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:55.192717+0000 mgr.y (mgr.26252) 1527 : cluster [DBG] pgmap v2129: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 22 MiB/s wr, 745 op/s; 10385/298338 objects misplaced (3.481%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:55.192717+0000 mgr.y (mgr.26252) 1527 : cluster [DBG] pgmap v2129: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 22 MiB/s wr, 745 op/s; 10385/298338 objects misplaced (3.481%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: audit 2024-09-13T15:16:55.193564+0000 mon.a (mon.0) 2678 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: audit 2024-09-13T15:16:55.193564+0000 mon.a (mon.0) 2678 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:55.245575+0000 osd.1 (osd.1) 24 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:55.245575+0000 osd.1 (osd.1) 24 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:55.305789+0000 osd.2 (osd.2) 42 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:56.233 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:55 smithi099 bash[87147]: cluster 2024-09-13T15:16:55.305789+0000 osd.2 (osd.2) 42 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:16:56.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:56.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:56.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:56.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:56.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:56.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:56.483 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:56.504 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:56.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:56.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:56.608 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:56.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:56.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:56.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:56.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:56.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:56.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:56.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:56.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:56.871 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:56.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:56.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:56.974 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:56.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:56.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:56.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:57.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:57.119 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:57.120 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:16:57] "GET /metrics HTTP/1.1" 200 36788 "" "Prometheus/2.51.0" 2024-09-13T15:16:57.120 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: cluster 2024-09-13T15:16:55.734446+0000 osd.0 (osd.0) 36 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:57.120 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: cluster 2024-09-13T15:16:55.734446+0000 osd.0 (osd.0) 36 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:57.120 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: audit 2024-09-13T15:16:55.876741+0000 mon.a (mon.0) 2679 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T15:16:57.120 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: audit 2024-09-13T15:16:55.876741+0000 mon.a (mon.0) 2679 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T15:16:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: cluster 2024-09-13T15:16:55.881816+0000 mon.a (mon.0) 2680 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2024-09-13T15:16:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: cluster 2024-09-13T15:16:55.881816+0000 mon.a (mon.0) 2680 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2024-09-13T15:16:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: cluster 2024-09-13T15:16:56.252591+0000 osd.1 (osd.1) 25 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: cluster 2024-09-13T15:16:56.252591+0000 osd.1 (osd.1) 25 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: cluster 2024-09-13T15:16:56.348335+0000 osd.2 (osd.2) 43 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: cluster 2024-09-13T15:16:56.348335+0000 osd.2 (osd.2) 43 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: cluster 2024-09-13T15:16:56.660749+0000 mon.a (mon.0) 2681 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2024-09-13T15:16:57.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[84576]: cluster 2024-09-13T15:16:56.660749+0000 mon.a (mon.0) 2681 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2024-09-13T15:16:57.121 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: cluster 2024-09-13T15:16:55.734446+0000 osd.0 (osd.0) 36 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:57.121 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: cluster 2024-09-13T15:16:55.734446+0000 osd.0 (osd.0) 36 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: audit 2024-09-13T15:16:55.876741+0000 mon.a (mon.0) 2679 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T15:16:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: audit 2024-09-13T15:16:55.876741+0000 mon.a (mon.0) 2679 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T15:16:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: cluster 2024-09-13T15:16:55.881816+0000 mon.a (mon.0) 2680 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2024-09-13T15:16:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: cluster 2024-09-13T15:16:55.881816+0000 mon.a (mon.0) 2680 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2024-09-13T15:16:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: cluster 2024-09-13T15:16:56.252591+0000 osd.1 (osd.1) 25 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: cluster 2024-09-13T15:16:56.252591+0000 osd.1 (osd.1) 25 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: cluster 2024-09-13T15:16:56.348335+0000 osd.2 (osd.2) 43 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: cluster 2024-09-13T15:16:56.348335+0000 osd.2 (osd.2) 43 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: cluster 2024-09-13T15:16:56.660749+0000 mon.a (mon.0) 2681 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2024-09-13T15:16:57.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:56 smithi099 bash[87147]: cluster 2024-09-13T15:16:56.660749+0000 mon.a (mon.0) 2681 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2024-09-13T15:16:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:57.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:57.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: cluster 2024-09-13T15:16:55.734446+0000 osd.0 (osd.0) 36 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: cluster 2024-09-13T15:16:55.734446+0000 osd.0 (osd.0) 36 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: audit 2024-09-13T15:16:55.876741+0000 mon.a (mon.0) 2679 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: audit 2024-09-13T15:16:55.876741+0000 mon.a (mon.0) 2679 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: cluster 2024-09-13T15:16:55.881816+0000 mon.a (mon.0) 2680 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: cluster 2024-09-13T15:16:55.881816+0000 mon.a (mon.0) 2680 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: cluster 2024-09-13T15:16:56.252591+0000 osd.1 (osd.1) 25 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: cluster 2024-09-13T15:16:56.252591+0000 osd.1 (osd.1) 25 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: cluster 2024-09-13T15:16:56.348335+0000 osd.2 (osd.2) 43 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: cluster 2024-09-13T15:16:56.348335+0000 osd.2 (osd.2) 43 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:16:57.237 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: cluster 2024-09-13T15:16:56.660749+0000 mon.a (mon.0) 2681 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2024-09-13T15:16:57.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:56 smithi177 bash[68594]: cluster 2024-09-13T15:16:56.660749+0000 mon.a (mon.0) 2681 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2024-09-13T15:16:57.252 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:57.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:57.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:57.355 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:57.377 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:57.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:57.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:57.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:57.502 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:57.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:57.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:57.605 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:57.633 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:57.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:57.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:57.737 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:57.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:16:57.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:57.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:16:57.861 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:57.899 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:57.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:57.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:58.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:58.023 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:58.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:58.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:58.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: cluster 2024-09-13T15:16:56.620488+0000 osd.3 (osd.3) 14 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: cluster 2024-09-13T15:16:56.620488+0000 osd.3 (osd.3) 14 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: cluster 2024-09-13T15:16:56.732341+0000 osd.0 (osd.0) 37 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: cluster 2024-09-13T15:16:56.732341+0000 osd.0 (osd.0) 37 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.193133+0000 mgr.y (mgr.26252) 1528 : cluster [DBG] pgmap v2132: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 2.7 MiB/s wr, 440 op/s; 10385/298338 objects misplaced (3.481%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.193133+0000 mgr.y (mgr.26252) 1528 : cluster [DBG] pgmap v2132: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 2.7 MiB/s wr, 440 op/s; 10385/298338 objects misplaced (3.481%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: audit 2024-09-13T15:16:57.193820+0000 mon.a (mon.0) 2682 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: audit 2024-09-13T15:16:57.193820+0000 mon.a (mon.0) 2682 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.380336+0000 osd.2 (osd.2) 44 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.380336+0000 osd.2 (osd.2) 44 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: audit 2024-09-13T15:16:57.658605+0000 mon.a (mon.0) 2683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: audit 2024-09-13T15:16:57.658605+0000 mon.a (mon.0) 2683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T15:16:58.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.666183+0000 mon.a (mon.0) 2684 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.666183+0000 mon.a (mon.0) 2684 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: cluster 2024-09-13T15:16:56.620488+0000 osd.3 (osd.3) 14 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: cluster 2024-09-13T15:16:56.620488+0000 osd.3 (osd.3) 14 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: cluster 2024-09-13T15:16:56.732341+0000 osd.0 (osd.0) 37 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: cluster 2024-09-13T15:16:56.732341+0000 osd.0 (osd.0) 37 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.193133+0000 mgr.y (mgr.26252) 1528 : cluster [DBG] pgmap v2132: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 2.7 MiB/s wr, 440 op/s; 10385/298338 objects misplaced (3.481%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.193133+0000 mgr.y (mgr.26252) 1528 : cluster [DBG] pgmap v2132: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 2.7 MiB/s wr, 440 op/s; 10385/298338 objects misplaced (3.481%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: audit 2024-09-13T15:16:57.193820+0000 mon.a (mon.0) 2682 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: audit 2024-09-13T15:16:57.193820+0000 mon.a (mon.0) 2682 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.380336+0000 osd.2 (osd.2) 44 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:58.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.380336+0000 osd.2 (osd.2) 44 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:58.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: audit 2024-09-13T15:16:57.658605+0000 mon.a (mon.0) 2683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T15:16:58.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: audit 2024-09-13T15:16:57.658605+0000 mon.a (mon.0) 2683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T15:16:58.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.666183+0000 mon.a (mon.0) 2684 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2024-09-13T15:16:58.077 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:57 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.666183+0000 mon.a (mon.0) 2684 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2024-09-13T15:16:58.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:58.148 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:58.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:58.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:58.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:58.261 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: cluster 2024-09-13T15:16:56.620488+0000 osd.3 (osd.3) 14 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: cluster 2024-09-13T15:16:56.620488+0000 osd.3 (osd.3) 14 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: cluster 2024-09-13T15:16:56.732341+0000 osd.0 (osd.0) 37 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: cluster 2024-09-13T15:16:56.732341+0000 osd.0 (osd.0) 37 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.193133+0000 mgr.y (mgr.26252) 1528 : cluster [DBG] pgmap v2132: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 2.7 MiB/s wr, 440 op/s; 10385/298338 objects misplaced (3.481%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.193133+0000 mgr.y (mgr.26252) 1528 : cluster [DBG] pgmap v2132: 99 pgs: 3 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 135 B/s rd, 2.7 MiB/s wr, 440 op/s; 10385/298338 objects misplaced (3.481%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: audit 2024-09-13T15:16:57.193820+0000 mon.a (mon.0) 2682 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: audit 2024-09-13T15:16:57.193820+0000 mon.a (mon.0) 2682 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.380336+0000 osd.2 (osd.2) 44 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.380336+0000 osd.2 (osd.2) 44 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: audit 2024-09-13T15:16:57.658605+0000 mon.a (mon.0) 2683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: audit 2024-09-13T15:16:57.658605+0000 mon.a (mon.0) 2683 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.666183+0000 mon.a (mon.0) 2684 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2024-09-13T15:16:58.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:57 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.666183+0000 mon.a (mon.0) 2684 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2024-09-13T15:16:58.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:58.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:58.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:58.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:58.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:58.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:58.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:58.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:58.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:58.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:58.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:58.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:58.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:58.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:58.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:16:58.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:58.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:16:58.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:58.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:16:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:58.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:16:59.051 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:59.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.285286+0000 osd.1 (osd.1) 26 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:59.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.285286+0000 osd.1 (osd.1) 26 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:59.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.633402+0000 osd.3 (osd.3) 15 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:59.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.633402+0000 osd.3 (osd.3) 15 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:59.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.703383+0000 osd.0 (osd.0) 38 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:59.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[84576]: cluster 2024-09-13T15:16:57.703383+0000 osd.0 (osd.0) 38 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:59.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[84576]: cluster 2024-09-13T15:16:58.667259+0000 mon.a (mon.0) 2685 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2024-09-13T15:16:59.061 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[84576]: cluster 2024-09-13T15:16:58.667259+0000 mon.a (mon.0) 2685 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2024-09-13T15:16:59.062 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.285286+0000 osd.1 (osd.1) 26 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:59.062 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.285286+0000 osd.1 (osd.1) 26 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:59.062 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.633402+0000 osd.3 (osd.3) 15 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:59.062 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.633402+0000 osd.3 (osd.3) 15 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:59.062 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.703383+0000 osd.0 (osd.0) 38 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:59.062 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[87147]: cluster 2024-09-13T15:16:57.703383+0000 osd.0 (osd.0) 38 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:59.062 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[87147]: cluster 2024-09-13T15:16:58.667259+0000 mon.a (mon.0) 2685 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2024-09-13T15:16:59.062 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:58 smithi099 bash[87147]: cluster 2024-09-13T15:16:58.667259+0000 mon.a (mon.0) 2685 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2024-09-13T15:16:59.082 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:16:59.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:59.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:16:59.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:58 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.285286+0000 osd.1 (osd.1) 26 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:59.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:58 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.285286+0000 osd.1 (osd.1) 26 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:16:59.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:58 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.633402+0000 osd.3 (osd.3) 15 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:59.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:58 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.633402+0000 osd.3 (osd.3) 15 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:16:59.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:58 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.703383+0000 osd.0 (osd.0) 38 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:59.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:58 smithi177 bash[68594]: cluster 2024-09-13T15:16:57.703383+0000 osd.0 (osd.0) 38 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:16:59.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:58 smithi177 bash[68594]: cluster 2024-09-13T15:16:58.667259+0000 mon.a (mon.0) 2685 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2024-09-13T15:16:59.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:58 smithi177 bash[68594]: cluster 2024-09-13T15:16:58.667259+0000 mon.a (mon.0) 2685 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2024-09-13T15:16:59.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:59.209 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:16:59.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:59.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:16:59.312 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:59.332 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:59.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:59.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:59.542 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:59.568 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:16:59.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:59.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:16:59.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:59.698 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:16:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:59.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:16:59.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:59.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:59.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:16:59.929 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:59.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:16:59.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:16:59.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:00.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:00.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:58.324306+0000 osd.1 (osd.1) 27 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:00.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:58.324306+0000 osd.1 (osd.1) 27 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:00.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:58.622448+0000 osd.2 (osd.2) 45 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:00.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:58.622448+0000 osd.2 (osd.2) 45 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:00.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:58.643995+0000 osd.3 (osd.3) 16 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:00.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:58.643995+0000 osd.3 (osd.3) 16 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:00.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:58.710821+0000 osd.0 (osd.0) 39 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:00.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:58.710821+0000 osd.0 (osd.0) 39 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:00.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.193700+0000 mgr.y (mgr.26252) 1529 : cluster [DBG] pgmap v2135: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 3203/292800 objects degraded (1.094%); 9304/292800 objects misplaced (3.178%); 12 MiB/s, 189 objects/s recovering 2024-09-13T15:17:00.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.193700+0000 mgr.y (mgr.26252) 1529 : cluster [DBG] pgmap v2135: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 3203/292800 objects degraded (1.094%); 9304/292800 objects misplaced (3.178%); 12 MiB/s, 189 objects/s recovering 2024-09-13T15:17:00.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.668078+0000 mon.a (mon.0) 2686 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2024-09-13T15:17:00.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.668078+0000 mon.a (mon.0) 2686 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2024-09-13T15:17:00.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:58.324306+0000 osd.1 (osd.1) 27 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:58.324306+0000 osd.1 (osd.1) 27 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:58.622448+0000 osd.2 (osd.2) 45 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:58.622448+0000 osd.2 (osd.2) 45 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:58.643995+0000 osd.3 (osd.3) 16 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:58.643995+0000 osd.3 (osd.3) 16 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:58.710821+0000 osd.0 (osd.0) 39 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:58.710821+0000 osd.0 (osd.0) 39 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.193700+0000 mgr.y (mgr.26252) 1529 : cluster [DBG] pgmap v2135: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 3203/292800 objects degraded (1.094%); 9304/292800 objects misplaced (3.178%); 12 MiB/s, 189 objects/s recovering 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.193700+0000 mgr.y (mgr.26252) 1529 : cluster [DBG] pgmap v2135: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 3203/292800 objects degraded (1.094%); 9304/292800 objects misplaced (3.178%); 12 MiB/s, 189 objects/s recovering 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.668078+0000 mon.a (mon.0) 2686 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2024-09-13T15:17:00.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:16:59 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.668078+0000 mon.a (mon.0) 2686 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2024-09-13T15:17:00.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:58.324306+0000 osd.1 (osd.1) 27 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:00.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:58.324306+0000 osd.1 (osd.1) 27 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:00.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:58.622448+0000 osd.2 (osd.2) 45 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:00.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:58.622448+0000 osd.2 (osd.2) 45 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:00.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:58.643995+0000 osd.3 (osd.3) 16 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:00.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:58.643995+0000 osd.3 (osd.3) 16 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:00.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:58.710821+0000 osd.0 (osd.0) 39 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:00.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:58.710821+0000 osd.0 (osd.0) 39 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:00.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.193700+0000 mgr.y (mgr.26252) 1529 : cluster [DBG] pgmap v2135: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 3203/292800 objects degraded (1.094%); 9304/292800 objects misplaced (3.178%); 12 MiB/s, 189 objects/s recovering 2024-09-13T15:17:00.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.193700+0000 mgr.y (mgr.26252) 1529 : cluster [DBG] pgmap v2135: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 3203/292800 objects degraded (1.094%); 9304/292800 objects misplaced (3.178%); 12 MiB/s, 189 objects/s recovering 2024-09-13T15:17:00.078 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.668078+0000 mon.a (mon.0) 2686 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2024-09-13T15:17:00.078 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:16:59 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.668078+0000 mon.a (mon.0) 2686 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2024-09-13T15:17:00.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:00.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:00.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:00.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:00.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:00.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:00.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:00.361 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[61439]: ts=2024-09-13T15:17:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:17:00.373 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:00.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:00.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:00.506 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:00.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:00.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:00.612 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:00.638 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:00.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:00.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:00.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:00.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:00.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:00.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:00.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:00.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:00.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:00.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:01.010 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:01.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.292511+0000 osd.1 (osd.1) 28 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:01.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.292511+0000 osd.1 (osd.1) 28 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:01.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.655241+0000 osd.2 (osd.2) 46 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:01.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.655241+0000 osd.2 (osd.2) 46 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:01.019 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.673449+0000 osd.3 (osd.3) 17 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:01.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.673449+0000 osd.3 (osd.3) 17 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:01.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.755329+0000 osd.0 (osd.0) 40 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:01.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.755329+0000 osd.0 (osd.0) 40 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:01.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.891277+0000 mon.a (mon.0) 2687 : cluster [WRN] Health check failed: Degraded data redundancy: 3203/292800 objects degraded (1.094%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:17:01.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:00 smithi177 bash[68594]: cluster 2024-09-13T15:16:59.891277+0000 mon.a (mon.0) 2687 : cluster [WRN] Health check failed: Degraded data redundancy: 3203/292800 objects degraded (1.094%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:17:01.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:01.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:01.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:01.153 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:01.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:01.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:01.274 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:01.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.292511+0000 osd.1 (osd.1) 28 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:01.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.292511+0000 osd.1 (osd.1) 28 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:01.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.655241+0000 osd.2 (osd.2) 46 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:01.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.655241+0000 osd.2 (osd.2) 46 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:01.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.673449+0000 osd.3 (osd.3) 17 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:01.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.673449+0000 osd.3 (osd.3) 17 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:01.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.755329+0000 osd.0 (osd.0) 40 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:01.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.755329+0000 osd.0 (osd.0) 40 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:01.284 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.891277+0000 mon.a (mon.0) 2687 : cluster [WRN] Health check failed: Degraded data redundancy: 3203/292800 objects degraded (1.094%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[84576]: cluster 2024-09-13T15:16:59.891277+0000 mon.a (mon.0) 2687 : cluster [WRN] Health check failed: Degraded data redundancy: 3203/292800 objects degraded (1.094%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.292511+0000 osd.1 (osd.1) 28 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.292511+0000 osd.1 (osd.1) 28 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.655241+0000 osd.2 (osd.2) 46 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.655241+0000 osd.2 (osd.2) 46 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.673449+0000 osd.3 (osd.3) 17 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.673449+0000 osd.3 (osd.3) 17 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.755329+0000 osd.0 (osd.0) 40 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.755329+0000 osd.0 (osd.0) 40 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.891277+0000 mon.a (mon.0) 2687 : cluster [WRN] Health check failed: Degraded data redundancy: 3203/292800 objects degraded (1.094%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:17:01.285 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:00 smithi099 bash[87147]: cluster 2024-09-13T15:16:59.891277+0000 mon.a (mon.0) 2687 : cluster [WRN] Health check failed: Degraded data redundancy: 3203/292800 objects degraded (1.094%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:17:01.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:01.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:01.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:01.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:01.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:01.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:01.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:01.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:01.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:01.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:01.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:01.653 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:01.680 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:01.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:01.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:01.784 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:01.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:01.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:01.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:01.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:01.925 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[84576]: cluster 2024-09-13T15:17:00.301930+0000 osd.1 (osd.1) 29 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:01.925 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[84576]: cluster 2024-09-13T15:17:00.301930+0000 osd.1 (osd.1) 29 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:01.925 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[84576]: cluster 2024-09-13T15:17:00.678699+0000 osd.2 (osd.2) 47 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:01.925 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[84576]: cluster 2024-09-13T15:17:00.678699+0000 osd.2 (osd.2) 47 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:01.925 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[84576]: cluster 2024-09-13T15:17:01.194499+0000 mgr.y (mgr.26252) 1530 : cluster [DBG] pgmap v2137: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 3203/291669 objects degraded (1.098%); 8762/291669 objects misplaced (3.004%); 18 MiB/s, 286 objects/s recovering 2024-09-13T15:17:01.925 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[84576]: cluster 2024-09-13T15:17:01.194499+0000 mgr.y (mgr.26252) 1530 : cluster [DBG] pgmap v2137: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 3203/291669 objects degraded (1.098%); 8762/291669 objects misplaced (3.004%); 18 MiB/s, 286 objects/s recovering 2024-09-13T15:17:01.926 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[87147]: cluster 2024-09-13T15:17:00.301930+0000 osd.1 (osd.1) 29 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:01.926 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[87147]: cluster 2024-09-13T15:17:00.301930+0000 osd.1 (osd.1) 29 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:01.926 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[87147]: cluster 2024-09-13T15:17:00.678699+0000 osd.2 (osd.2) 47 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:01.926 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[87147]: cluster 2024-09-13T15:17:00.678699+0000 osd.2 (osd.2) 47 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:01.926 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[87147]: cluster 2024-09-13T15:17:01.194499+0000 mgr.y (mgr.26252) 1530 : cluster [DBG] pgmap v2137: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 3203/291669 objects degraded (1.098%); 8762/291669 objects misplaced (3.004%); 18 MiB/s, 286 objects/s recovering 2024-09-13T15:17:01.926 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:01 smithi099 bash[87147]: cluster 2024-09-13T15:17:01.194499+0000 mgr.y (mgr.26252) 1530 : cluster [DBG] pgmap v2137: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 3203/291669 objects degraded (1.098%); 8762/291669 objects misplaced (3.004%); 18 MiB/s, 286 objects/s recovering 2024-09-13T15:17:01.949 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:01.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:01.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:02.052 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:02.074 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:02.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:02.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:02.176 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:02.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:02.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:02.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:02.327 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:02.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:01 smithi177 bash[68594]: cluster 2024-09-13T15:17:00.301930+0000 osd.1 (osd.1) 29 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:02.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:01 smithi177 bash[68594]: cluster 2024-09-13T15:17:00.301930+0000 osd.1 (osd.1) 29 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:02.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:01 smithi177 bash[68594]: cluster 2024-09-13T15:17:00.678699+0000 osd.2 (osd.2) 47 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:02.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:01 smithi177 bash[68594]: cluster 2024-09-13T15:17:00.678699+0000 osd.2 (osd.2) 47 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:02.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:01 smithi177 bash[68594]: cluster 2024-09-13T15:17:01.194499+0000 mgr.y (mgr.26252) 1530 : cluster [DBG] pgmap v2137: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 3203/291669 objects degraded (1.098%); 8762/291669 objects misplaced (3.004%); 18 MiB/s, 286 objects/s recovering 2024-09-13T15:17:02.337 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:01 smithi177 bash[68594]: cluster 2024-09-13T15:17:01.194499+0000 mgr.y (mgr.26252) 1530 : cluster [DBG] pgmap v2137: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 3203/291669 objects degraded (1.098%); 8762/291669 objects misplaced (3.004%); 18 MiB/s, 286 objects/s recovering 2024-09-13T15:17:02.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:02.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:02.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:02.460 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:02.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:02.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:02.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:02.593 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:02.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:02.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:02.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:02.729 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:02.746 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:02.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:02.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:02.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:02.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:02.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:02.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:03.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:03.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:03.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:03.103 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:03.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:03.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:03.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:03.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:03.277 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:03.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:03.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:03.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:03.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:03.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:03.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:03.521 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:03.548 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:03.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:03.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:03.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:03.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:03.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:03.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:03.727 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:110711 46.f 46:f1ba1e28:::benchmark_data_smithi099_233884_object8298:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1122)", 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:17:03.720460+0000", 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0038467649999999998, 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038689190000000002, 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 110711 2024-09-13T15:17:03.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.720460+0000", 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.720460+0000", 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:17:03.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.720459+0000", 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.720464+0000", 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5639999999999996e-06 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.720476+0000", 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.201e-05 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:17:03.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.720481+0000", 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3510000000000004e-06 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.720538+0000", 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6892999999999999e-05 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.720684+0000", 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014669999999999999 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.720827+0000", 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014283300000000001 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:110712 46.12 46:4cce32a9:::benchmark_data_smithi099_233884_object8299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1122)", 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:17:03.721112+0000", 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0031943129999999998, 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003297295, 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:17:03.732 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 110712 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.721112+0000", 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:17:03.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.721112+0000", 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.721112+0000", 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.721116+0000", 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9400000000000004e-06 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.721125+0000", 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1500000000000005e-06 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.721129+0000", 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.878e-06 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.721646+0000", 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00051716900000000005 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.721721+0000", 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4975000000000006e-05 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.721798+0000", 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7255000000000005e-05 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:110717 46.15 e1122/1108)", 2024-09-13T15:17:03.736 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:17:03.724437+0000", 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4294967295.9998703, 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.354e-05, 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "queued for pg", 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.724437+0000", 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.724437+0000", 2024-09-13T15:17:03.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.724436+0000", 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.724443+0000", 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3919999999999997e-06 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.724454+0000", 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1e-05 2024-09-13T15:17:03.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:03.724460+0000", 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4160000000000003e-06 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 3 2024-09-13T15:17:03.739 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:17:03.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:03.816 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:03.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:03.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:03.922 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:03.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:03.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:03.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:04.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:04.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:04.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:04.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:04.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:04.221 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:04.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:04.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:04.375 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:04.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:04.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:04.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:04.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:04 smithi177 bash[68594]: cluster 2024-09-13T15:17:03.012674+0000 osd.0 (osd.0) 41 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:04.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:04 smithi177 bash[68594]: cluster 2024-09-13T15:17:03.012674+0000 osd.0 (osd.0) 41 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:04.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:04 smithi177 bash[68594]: cluster 2024-09-13T15:17:03.195119+0000 mgr.y (mgr.26252) 1531 : cluster [DBG] pgmap v2138: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 3203/290391 objects degraded (1.103%); 8762/290391 objects misplaced (3.017%); 14 MiB/s, 216 objects/s recovering 2024-09-13T15:17:04.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:04 smithi177 bash[68594]: cluster 2024-09-13T15:17:03.195119+0000 mgr.y (mgr.26252) 1531 : cluster [DBG] pgmap v2138: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 3203/290391 objects degraded (1.103%); 8762/290391 objects misplaced (3.017%); 14 MiB/s, 216 objects/s recovering 2024-09-13T15:17:04.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:04 smithi177 bash[68594]: audit 2024-09-13T15:17:03.323924+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:04.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:04 smithi177 bash[68594]: audit 2024-09-13T15:17:03.323924+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:04.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:04 smithi177 bash[68594]: audit 2024-09-13T15:17:03.325177+0000 mon.a (mon.0) 2689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:04.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:04 smithi177 bash[68594]: audit 2024-09-13T15:17:03.325177+0000 mon.a (mon.0) 2689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:04.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:04 smithi177 bash[68594]: cluster 2024-09-13T15:17:03.986726+0000 osd.3 (osd.3) 18 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:04.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:04 smithi177 bash[68594]: cluster 2024-09-13T15:17:03.986726+0000 osd.3 (osd.3) 18 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:04.506 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:04.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:04.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[84576]: cluster 2024-09-13T15:17:03.012674+0000 osd.0 (osd.0) 41 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:04.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[84576]: cluster 2024-09-13T15:17:03.012674+0000 osd.0 (osd.0) 41 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:04.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[84576]: cluster 2024-09-13T15:17:03.195119+0000 mgr.y (mgr.26252) 1531 : cluster [DBG] pgmap v2138: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 3203/290391 objects degraded (1.103%); 8762/290391 objects misplaced (3.017%); 14 MiB/s, 216 objects/s recovering 2024-09-13T15:17:04.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[84576]: cluster 2024-09-13T15:17:03.195119+0000 mgr.y (mgr.26252) 1531 : cluster [DBG] pgmap v2138: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 3203/290391 objects degraded (1.103%); 8762/290391 objects misplaced (3.017%); 14 MiB/s, 216 objects/s recovering 2024-09-13T15:17:04.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[84576]: audit 2024-09-13T15:17:03.323924+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:04.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[84576]: audit 2024-09-13T15:17:03.323924+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:04.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[84576]: audit 2024-09-13T15:17:03.325177+0000 mon.a (mon.0) 2689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:04.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[84576]: audit 2024-09-13T15:17:03.325177+0000 mon.a (mon.0) 2689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:04.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[84576]: cluster 2024-09-13T15:17:03.986726+0000 osd.3 (osd.3) 18 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:04.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[84576]: cluster 2024-09-13T15:17:03.986726+0000 osd.3 (osd.3) 18 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:04.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[87147]: cluster 2024-09-13T15:17:03.012674+0000 osd.0 (osd.0) 41 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:04.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[87147]: cluster 2024-09-13T15:17:03.012674+0000 osd.0 (osd.0) 41 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:04.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[87147]: cluster 2024-09-13T15:17:03.195119+0000 mgr.y (mgr.26252) 1531 : cluster [DBG] pgmap v2138: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 3203/290391 objects degraded (1.103%); 8762/290391 objects misplaced (3.017%); 14 MiB/s, 216 objects/s recovering 2024-09-13T15:17:04.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[87147]: cluster 2024-09-13T15:17:03.195119+0000 mgr.y (mgr.26252) 1531 : cluster [DBG] pgmap v2138: 99 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 3203/290391 objects degraded (1.103%); 8762/290391 objects misplaced (3.017%); 14 MiB/s, 216 objects/s recovering 2024-09-13T15:17:04.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[87147]: audit 2024-09-13T15:17:03.323924+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:04.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[87147]: audit 2024-09-13T15:17:03.323924+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:04.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[87147]: audit 2024-09-13T15:17:03.325177+0000 mon.a (mon.0) 2689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:04.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[87147]: audit 2024-09-13T15:17:03.325177+0000 mon.a (mon.0) 2689 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:04.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[87147]: cluster 2024-09-13T15:17:03.986726+0000 osd.3 (osd.3) 18 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:04.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:04 smithi099 bash[87147]: cluster 2024-09-13T15:17:03.986726+0000 osd.3 (osd.3) 18 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:04.612 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:04.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:04.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:04.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:04.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:04.783 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:04.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:04.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:04.913 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:04.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:04.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:05.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:05.048 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:05.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:05.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:05.152 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:05.179 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:05.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:05.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:05.286 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:05.312 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[84576]: cluster 2024-09-13T15:17:03.414447+0000 osd.1 (osd.1) 30 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:05.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:05.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:05.418 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:05.443 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:05.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:05.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:05.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:17:05.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:05 smithi177 bash[68594]: cluster 2024-09-13T15:17:03.414447+0000 osd.1 (osd.1) 30 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:05.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:05 smithi177 bash[68594]: cluster 2024-09-13T15:17:03.414447+0000 osd.1 (osd.1) 30 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:05.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:05 smithi177 bash[68594]: cluster 2024-09-13T15:17:03.654517+0000 osd.2 (osd.2) 48 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:05.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:05 smithi177 bash[68594]: cluster 2024-09-13T15:17:03.654517+0000 osd.2 (osd.2) 48 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:05.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:05 smithi177 bash[68594]: cluster 2024-09-13T15:17:04.023840+0000 osd.0 (osd.0) 42 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:05.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:05 smithi177 bash[68594]: cluster 2024-09-13T15:17:04.023840+0000 osd.0 (osd.0) 42 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:05.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:05 smithi177 bash[68594]: audit 2024-09-13T15:17:04.678222+0000 mon.a (mon.0) 2690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:05.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:05 smithi177 bash[68594]: audit 2024-09-13T15:17:04.678222+0000 mon.a (mon.0) 2690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:05.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:05 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.031102+0000 osd.3 (osd.3) 19 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:05.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:05 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.031102+0000 osd.3 (osd.3) 19 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:05.547 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:05.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[84576]: cluster 2024-09-13T15:17:03.414447+0000 osd.1 (osd.1) 30 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:05.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[84576]: cluster 2024-09-13T15:17:03.654517+0000 osd.2 (osd.2) 48 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:05.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[84576]: cluster 2024-09-13T15:17:03.654517+0000 osd.2 (osd.2) 48 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:05.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[84576]: cluster 2024-09-13T15:17:04.023840+0000 osd.0 (osd.0) 42 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:05.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[84576]: cluster 2024-09-13T15:17:04.023840+0000 osd.0 (osd.0) 42 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:05.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[84576]: audit 2024-09-13T15:17:04.678222+0000 mon.a (mon.0) 2690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:05.559 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[84576]: audit 2024-09-13T15:17:04.678222+0000 mon.a (mon.0) 2690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:05.560 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.031102+0000 osd.3 (osd.3) 19 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:05.560 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.031102+0000 osd.3 (osd.3) 19 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:05.561 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[87147]: cluster 2024-09-13T15:17:03.414447+0000 osd.1 (osd.1) 30 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:05.561 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[87147]: cluster 2024-09-13T15:17:03.414447+0000 osd.1 (osd.1) 30 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:05.562 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[87147]: cluster 2024-09-13T15:17:03.654517+0000 osd.2 (osd.2) 48 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:05.562 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[87147]: cluster 2024-09-13T15:17:03.654517+0000 osd.2 (osd.2) 48 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:05.562 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[87147]: cluster 2024-09-13T15:17:04.023840+0000 osd.0 (osd.0) 42 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:05.562 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[87147]: cluster 2024-09-13T15:17:04.023840+0000 osd.0 (osd.0) 42 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:05.562 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[87147]: audit 2024-09-13T15:17:04.678222+0000 mon.a (mon.0) 2690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:05.562 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[87147]: audit 2024-09-13T15:17:04.678222+0000 mon.a (mon.0) 2690 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:05.562 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.031102+0000 osd.3 (osd.3) 19 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:05.562 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:05 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.031102+0000 osd.3 (osd.3) 19 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:05.572 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:05.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:05.676 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:05.699 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:05.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:05.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:05.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:05.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:05.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:05.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:05.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:05.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:05.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:05.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:06.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:06.118 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:06.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:06.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:06.222 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:06.249 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:06.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:06.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:06.352 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:06.370 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:06.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:06.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:06.473 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:06.484 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:06 smithi177 bash[68594]: cluster 2024-09-13T15:17:04.370577+0000 osd.1 (osd.1) 31 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:06.484 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:06 smithi177 bash[68594]: cluster 2024-09-13T15:17:04.370577+0000 osd.1 (osd.1) 31 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:06.484 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:06 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.049268+0000 osd.0 (osd.0) 43 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:06.484 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:06 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.049268+0000 osd.0 (osd.0) 43 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:06.485 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:06 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.195912+0000 mgr.y (mgr.26252) 1532 : cluster [DBG] pgmap v2139: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 20 MiB/s, 321 objects/s recovering 2024-09-13T15:17:06.485 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:06 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.195912+0000 mgr.y (mgr.26252) 1532 : cluster [DBG] pgmap v2139: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 20 MiB/s, 321 objects/s recovering 2024-09-13T15:17:06.485 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:06 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.674976+0000 mon.a (mon.0) 2691 : cluster [WRN] Health check update: Degraded data redundancy: 9425/281547 objects degraded (3.348%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:06.485 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:06 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.674976+0000 mon.a (mon.0) 2691 : cluster [WRN] Health check update: Degraded data redundancy: 9425/281547 objects degraded (3.348%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:06.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:06.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:06.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:06.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[84576]: cluster 2024-09-13T15:17:04.370577+0000 osd.1 (osd.1) 31 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:06.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[84576]: cluster 2024-09-13T15:17:04.370577+0000 osd.1 (osd.1) 31 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:06.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.049268+0000 osd.0 (osd.0) 43 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:06.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.049268+0000 osd.0 (osd.0) 43 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:06.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.195912+0000 mgr.y (mgr.26252) 1532 : cluster [DBG] pgmap v2139: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 20 MiB/s, 321 objects/s recovering 2024-09-13T15:17:06.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.195912+0000 mgr.y (mgr.26252) 1532 : cluster [DBG] pgmap v2139: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 20 MiB/s, 321 objects/s recovering 2024-09-13T15:17:06.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.674976+0000 mon.a (mon.0) 2691 : cluster [WRN] Health check update: Degraded data redundancy: 9425/281547 objects degraded (3.348%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:06.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.674976+0000 mon.a (mon.0) 2691 : cluster [WRN] Health check update: Degraded data redundancy: 9425/281547 objects degraded (3.348%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:06.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[87147]: cluster 2024-09-13T15:17:04.370577+0000 osd.1 (osd.1) 31 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:06.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[87147]: cluster 2024-09-13T15:17:04.370577+0000 osd.1 (osd.1) 31 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:06.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.049268+0000 osd.0 (osd.0) 43 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:06.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.049268+0000 osd.0 (osd.0) 43 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:06.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.195912+0000 mgr.y (mgr.26252) 1532 : cluster [DBG] pgmap v2139: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 20 MiB/s, 321 objects/s recovering 2024-09-13T15:17:06.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.195912+0000 mgr.y (mgr.26252) 1532 : cluster [DBG] pgmap v2139: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 20 MiB/s, 321 objects/s recovering 2024-09-13T15:17:06.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.674976+0000 mon.a (mon.0) 2691 : cluster [WRN] Health check update: Degraded data redundancy: 9425/281547 objects degraded (3.348%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:06.578 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:06 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.674976+0000 mon.a (mon.0) 2691 : cluster [WRN] Health check update: Degraded data redundancy: 9425/281547 objects degraded (3.348%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:06.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:06.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:06.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:06.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:06.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:06.783 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:06.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:06.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:06.886 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:06.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:06.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:06.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:07.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:07.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:07.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:07.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:07.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:07.165 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:17:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:17:07] "GET /metrics HTTP/1.1" 200 36805 "" "Prometheus/2.51.0" 2024-09-13T15:17:07.177 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:07.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:07.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:07.280 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:07.307 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:07.310 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:07 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.365508+0000 osd.1 (osd.1) 32 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:07.310 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:07 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.365508+0000 osd.1 (osd.1) 32 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:07.311 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:07 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.892592+0000 osd.2 (osd.2) 49 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:07.311 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:07 smithi099 bash[84576]: cluster 2024-09-13T15:17:05.892592+0000 osd.2 (osd.2) 49 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:07.311 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:07 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.365508+0000 osd.1 (osd.1) 32 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:07.311 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:07 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.365508+0000 osd.1 (osd.1) 32 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:07.311 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:07 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.892592+0000 osd.2 (osd.2) 49 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:07.311 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:07 smithi099 bash[87147]: cluster 2024-09-13T15:17:05.892592+0000 osd.2 (osd.2) 49 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:07.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:07.412 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:07.444 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:07.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:07.547 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:07.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:07 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.365508+0000 osd.1 (osd.1) 32 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:07.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:07 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.365508+0000 osd.1 (osd.1) 32 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:07.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:07 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.892592+0000 osd.2 (osd.2) 49 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:07.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:07 smithi177 bash[68594]: cluster 2024-09-13T15:17:05.892592+0000 osd.2 (osd.2) 49 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:07.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:07.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:07.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:07.678 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:07.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:07.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:07.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:07.810 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:07.839 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:07.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:07.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:07.941 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:07.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:07.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:08.075 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:08.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:08.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:08.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:08.205 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:08.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:08.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:08.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:08.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:08.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:08 smithi177 bash[68594]: cluster 2024-09-13T15:17:06.884547+0000 osd.2 (osd.2) 50 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:08.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:08 smithi177 bash[68594]: cluster 2024-09-13T15:17:06.884547+0000 osd.2 (osd.2) 50 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:08.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:08 smithi177 bash[68594]: cluster 2024-09-13T15:17:07.196398+0000 mgr.y (mgr.26252) 1533 : cluster [DBG] pgmap v2140: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 18 MiB/s, 283 objects/s recovering 2024-09-13T15:17:08.344 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:08 smithi177 bash[68594]: cluster 2024-09-13T15:17:07.196398+0000 mgr.y (mgr.26252) 1533 : cluster [DBG] pgmap v2140: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 18 MiB/s, 283 objects/s recovering 2024-09-13T15:17:08.368 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:08.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:08.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:08.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:08.483 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:08 smithi099 bash[84576]: cluster 2024-09-13T15:17:06.884547+0000 osd.2 (osd.2) 50 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:08.483 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:08 smithi099 bash[84576]: cluster 2024-09-13T15:17:06.884547+0000 osd.2 (osd.2) 50 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:08.483 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:08 smithi099 bash[84576]: cluster 2024-09-13T15:17:07.196398+0000 mgr.y (mgr.26252) 1533 : cluster [DBG] pgmap v2140: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 18 MiB/s, 283 objects/s recovering 2024-09-13T15:17:08.483 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:08 smithi099 bash[84576]: cluster 2024-09-13T15:17:07.196398+0000 mgr.y (mgr.26252) 1533 : cluster [DBG] pgmap v2140: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 18 MiB/s, 283 objects/s recovering 2024-09-13T15:17:08.483 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:08 smithi099 bash[87147]: cluster 2024-09-13T15:17:06.884547+0000 osd.2 (osd.2) 50 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:08.484 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:08 smithi099 bash[87147]: cluster 2024-09-13T15:17:06.884547+0000 osd.2 (osd.2) 50 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:08.484 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:08 smithi099 bash[87147]: cluster 2024-09-13T15:17:07.196398+0000 mgr.y (mgr.26252) 1533 : cluster [DBG] pgmap v2140: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 18 MiB/s, 283 objects/s recovering 2024-09-13T15:17:08.484 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:08 smithi099 bash[87147]: cluster 2024-09-13T15:17:07.196398+0000 mgr.y (mgr.26252) 1533 : cluster [DBG] pgmap v2140: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 9425/281547 objects degraded (3.348%); 7536/281547 objects misplaced (2.677%); 18 MiB/s, 283 objects/s recovering 2024-09-13T15:17:08.501 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:08.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:08.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:08.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:08.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:08.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:08.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:08.760 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:08.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:08.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:08.863 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:08.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:08.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:08.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:08.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:09.023 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:09.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:09.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:09.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:09.157 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:09.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:09.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:09.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:09.295 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:09.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:09.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:09.399 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:09.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:09.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:09.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:09.529 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:09.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:09 smithi177 bash[68594]: cluster 2024-09-13T15:17:08.033476+0000 osd.0 (osd.0) 44 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:09.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:09 smithi177 bash[68594]: cluster 2024-09-13T15:17:08.033476+0000 osd.0 (osd.0) 44 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:09.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:09 smithi177 bash[68594]: cluster 2024-09-13T15:17:08.988224+0000 osd.3 (osd.3) 20 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:09.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:09 smithi177 bash[68594]: cluster 2024-09-13T15:17:08.988224+0000 osd.3 (osd.3) 20 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:09.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:09 smithi177 bash[68594]: audit 2024-09-13T15:17:08.991983+0000 mon.a (mon.0) 2692 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:17:09.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:09 smithi177 bash[68594]: audit 2024-09-13T15:17:08.991983+0000 mon.a (mon.0) 2692 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:17:09.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:09 smithi177 bash[68594]: cluster 2024-09-13T15:17:09.197493+0000 mgr.y (mgr.26252) 1534 : cluster [DBG] pgmap v2141: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 9157/272337 objects degraded (3.362%); 6648/272337 objects misplaced (2.441%); 16 MiB/s, 249 objects/s recovering 2024-09-13T15:17:09.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:09 smithi177 bash[68594]: cluster 2024-09-13T15:17:09.197493+0000 mgr.y (mgr.26252) 1534 : cluster [DBG] pgmap v2141: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 9157/272337 objects degraded (3.362%); 6648/272337 objects misplaced (2.441%); 16 MiB/s, 249 objects/s recovering 2024-09-13T15:17:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[84576]: cluster 2024-09-13T15:17:08.033476+0000 osd.0 (osd.0) 44 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[84576]: cluster 2024-09-13T15:17:08.033476+0000 osd.0 (osd.0) 44 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[84576]: cluster 2024-09-13T15:17:08.988224+0000 osd.3 (osd.3) 20 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[84576]: cluster 2024-09-13T15:17:08.988224+0000 osd.3 (osd.3) 20 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[84576]: audit 2024-09-13T15:17:08.991983+0000 mon.a (mon.0) 2692 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[84576]: audit 2024-09-13T15:17:08.991983+0000 mon.a (mon.0) 2692 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[84576]: cluster 2024-09-13T15:17:09.197493+0000 mgr.y (mgr.26252) 1534 : cluster [DBG] pgmap v2141: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 9157/272337 objects degraded (3.362%); 6648/272337 objects misplaced (2.441%); 16 MiB/s, 249 objects/s recovering 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[84576]: cluster 2024-09-13T15:17:09.197493+0000 mgr.y (mgr.26252) 1534 : cluster [DBG] pgmap v2141: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 9157/272337 objects degraded (3.362%); 6648/272337 objects misplaced (2.441%); 16 MiB/s, 249 objects/s recovering 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[87147]: cluster 2024-09-13T15:17:08.033476+0000 osd.0 (osd.0) 44 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[87147]: cluster 2024-09-13T15:17:08.033476+0000 osd.0 (osd.0) 44 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[87147]: cluster 2024-09-13T15:17:08.988224+0000 osd.3 (osd.3) 20 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[87147]: cluster 2024-09-13T15:17:08.988224+0000 osd.3 (osd.3) 20 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[87147]: audit 2024-09-13T15:17:08.991983+0000 mon.a (mon.0) 2692 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[87147]: audit 2024-09-13T15:17:08.991983+0000 mon.a (mon.0) 2692 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:17:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[87147]: cluster 2024-09-13T15:17:09.197493+0000 mgr.y (mgr.26252) 1534 : cluster [DBG] pgmap v2141: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 9157/272337 objects degraded (3.362%); 6648/272337 objects misplaced (2.441%); 16 MiB/s, 249 objects/s recovering 2024-09-13T15:17:09.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:09 smithi099 bash[87147]: cluster 2024-09-13T15:17:09.197493+0000 mgr.y (mgr.26252) 1534 : cluster [DBG] pgmap v2141: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 9157/272337 objects degraded (3.362%); 6648/272337 objects misplaced (2.441%); 16 MiB/s, 249 objects/s recovering 2024-09-13T15:17:09.585 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:09.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:09.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:09.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:09.736 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:09.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:09.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:09.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:09.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:09.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:09.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:09.975 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:10.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:10.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:10.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:10.115 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:10.146 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:10.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:10.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:10.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:10.259 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:17:10 smithi177 bash[61439]: ts=2024-09-13T15:17:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:17:10.270 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:10.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:10.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:10.309 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:17:10.372 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:10.403 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:10.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:10.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:10.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:10.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[84576]: cluster 2024-09-13T15:17:08.600916+0000 osd.1 (osd.1) 33 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:10.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[84576]: cluster 2024-09-13T15:17:08.600916+0000 osd.1 (osd.1) 33 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:10.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[84576]: cluster 2024-09-13T15:17:08.857727+0000 osd.2 (osd.2) 51 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:10.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[84576]: cluster 2024-09-13T15:17:08.857727+0000 osd.2 (osd.2) 51 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:10.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[84576]: cluster 2024-09-13T15:17:09.046565+0000 osd.0 (osd.0) 45 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:10.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[84576]: cluster 2024-09-13T15:17:09.046565+0000 osd.0 (osd.0) 45 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:10.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[84576]: cluster 2024-09-13T15:17:09.573808+0000 osd.1 (osd.1) 34 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:10.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[84576]: cluster 2024-09-13T15:17:09.573808+0000 osd.1 (osd.1) 34 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:10.524 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[87147]: cluster 2024-09-13T15:17:08.600916+0000 osd.1 (osd.1) 33 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:10.524 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[87147]: cluster 2024-09-13T15:17:08.600916+0000 osd.1 (osd.1) 33 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:10.524 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[87147]: cluster 2024-09-13T15:17:08.857727+0000 osd.2 (osd.2) 51 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:10.524 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[87147]: cluster 2024-09-13T15:17:08.857727+0000 osd.2 (osd.2) 51 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:10.524 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[87147]: cluster 2024-09-13T15:17:09.046565+0000 osd.0 (osd.0) 45 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:10.524 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[87147]: cluster 2024-09-13T15:17:09.046565+0000 osd.0 (osd.0) 45 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:10.524 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[87147]: cluster 2024-09-13T15:17:09.573808+0000 osd.1 (osd.1) 34 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:10.524 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:10 smithi099 bash[87147]: cluster 2024-09-13T15:17:09.573808+0000 osd.1 (osd.1) 34 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:10.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:10.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:10.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:10.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:10 smithi177 bash[68594]: cluster 2024-09-13T15:17:08.600916+0000 osd.1 (osd.1) 33 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:10.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:10 smithi177 bash[68594]: cluster 2024-09-13T15:17:08.600916+0000 osd.1 (osd.1) 33 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:10 smithi177 bash[68594]: cluster 2024-09-13T15:17:08.857727+0000 osd.2 (osd.2) 51 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:10 smithi177 bash[68594]: cluster 2024-09-13T15:17:08.857727+0000 osd.2 (osd.2) 51 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:10 smithi177 bash[68594]: cluster 2024-09-13T15:17:09.046565+0000 osd.0 (osd.0) 45 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:10 smithi177 bash[68594]: cluster 2024-09-13T15:17:09.046565+0000 osd.0 (osd.0) 45 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:10 smithi177 bash[68594]: cluster 2024-09-13T15:17:09.573808+0000 osd.1 (osd.1) 34 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:10 smithi177 bash[68594]: cluster 2024-09-13T15:17:09.573808+0000 osd.1 (osd.1) 34 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:10.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:10.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:10.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:10.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:10.768 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:10.786 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:10.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:10.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:10.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:10.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:10.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:11.009 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:11.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:11.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:11.130 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:11.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:11.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:11.254 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:11.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:11.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:11.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:11.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:11.440 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[84576]: cluster 2024-09-13T15:17:10.530285+0000 osd.1 (osd.1) 35 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:11.440 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[84576]: cluster 2024-09-13T15:17:10.530285+0000 osd.1 (osd.1) 35 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:11.440 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[84576]: cluster 2024-09-13T15:17:10.902812+0000 osd.3 (osd.3) 21 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:11.440 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[84576]: cluster 2024-09-13T15:17:10.902812+0000 osd.3 (osd.3) 21 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:11.440 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[84576]: cluster 2024-09-13T15:17:11.198084+0000 mgr.y (mgr.26252) 1535 : cluster [DBG] pgmap v2142: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 9157/269313 objects degraded (3.400%); 6295/269313 objects misplaced (2.337%); 15 MiB/s, 246 objects/s recovering 2024-09-13T15:17:11.440 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[84576]: cluster 2024-09-13T15:17:11.198084+0000 mgr.y (mgr.26252) 1535 : cluster [DBG] pgmap v2142: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 9157/269313 objects degraded (3.400%); 6295/269313 objects misplaced (2.337%); 15 MiB/s, 246 objects/s recovering 2024-09-13T15:17:11.440 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[87147]: cluster 2024-09-13T15:17:10.530285+0000 osd.1 (osd.1) 35 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:11.440 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[87147]: cluster 2024-09-13T15:17:10.530285+0000 osd.1 (osd.1) 35 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:11.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[87147]: cluster 2024-09-13T15:17:10.902812+0000 osd.3 (osd.3) 21 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:11.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[87147]: cluster 2024-09-13T15:17:10.902812+0000 osd.3 (osd.3) 21 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:11.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[87147]: cluster 2024-09-13T15:17:11.198084+0000 mgr.y (mgr.26252) 1535 : cluster [DBG] pgmap v2142: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 9157/269313 objects degraded (3.400%); 6295/269313 objects misplaced (2.337%); 15 MiB/s, 246 objects/s recovering 2024-09-13T15:17:11.441 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:11 smithi099 bash[87147]: cluster 2024-09-13T15:17:11.198084+0000 mgr.y (mgr.26252) 1535 : cluster [DBG] pgmap v2142: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 9157/269313 objects degraded (3.400%); 6295/269313 objects misplaced (2.337%); 15 MiB/s, 246 objects/s recovering 2024-09-13T15:17:11.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:11.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:11.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:11.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:11 smithi177 bash[68594]: cluster 2024-09-13T15:17:10.530285+0000 osd.1 (osd.1) 35 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:11.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:11 smithi177 bash[68594]: cluster 2024-09-13T15:17:10.530285+0000 osd.1 (osd.1) 35 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:11.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:11 smithi177 bash[68594]: cluster 2024-09-13T15:17:10.902812+0000 osd.3 (osd.3) 21 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:11.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:11 smithi177 bash[68594]: cluster 2024-09-13T15:17:10.902812+0000 osd.3 (osd.3) 21 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:11.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:11 smithi177 bash[68594]: cluster 2024-09-13T15:17:11.198084+0000 mgr.y (mgr.26252) 1535 : cluster [DBG] pgmap v2142: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 9157/269313 objects degraded (3.400%); 6295/269313 objects misplaced (2.337%); 15 MiB/s, 246 objects/s recovering 2024-09-13T15:17:11.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:11 smithi177 bash[68594]: cluster 2024-09-13T15:17:11.198084+0000 mgr.y (mgr.26252) 1535 : cluster [DBG] pgmap v2142: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 9157/269313 objects degraded (3.400%); 6295/269313 objects misplaced (2.337%); 15 MiB/s, 246 objects/s recovering 2024-09-13T15:17:11.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:11.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:11.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:11.677 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:11.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:11.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:11.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:11.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:11.832 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:11.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:11.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:11.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:11.954 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:11.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:12.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:12.082 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:12.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:12.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:12.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:12.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:12.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:12.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:12.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:12.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:12.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:12.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:12.452 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:12.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:12.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:12.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:12.602 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:12 smithi177 bash[68594]: cluster 2024-09-13T15:17:11.022795+0000 osd.0 (osd.0) 46 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:12.602 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:12 smithi177 bash[68594]: cluster 2024-09-13T15:17:11.022795+0000 osd.0 (osd.0) 46 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:12.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:12 smithi177 bash[68594]: cluster 2024-09-13T15:17:11.276546+0000 mon.a (mon.0) 2693 : cluster [WRN] Health check update: Degraded data redundancy: 9157/269313 objects degraded (3.400%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:12.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:12 smithi177 bash[68594]: cluster 2024-09-13T15:17:11.276546+0000 mon.a (mon.0) 2693 : cluster [WRN] Health check update: Degraded data redundancy: 9157/269313 objects degraded (3.400%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:12.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:12.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:12.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:12.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:12.775 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:12.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:12.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:12.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:12.915 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:12 smithi099 bash[84576]: cluster 2024-09-13T15:17:11.022795+0000 osd.0 (osd.0) 46 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:12.915 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:12 smithi099 bash[84576]: cluster 2024-09-13T15:17:11.022795+0000 osd.0 (osd.0) 46 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:12.915 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:12 smithi099 bash[84576]: cluster 2024-09-13T15:17:11.276546+0000 mon.a (mon.0) 2693 : cluster [WRN] Health check update: Degraded data redundancy: 9157/269313 objects degraded (3.400%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:12.915 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:12 smithi099 bash[84576]: cluster 2024-09-13T15:17:11.276546+0000 mon.a (mon.0) 2693 : cluster [WRN] Health check update: Degraded data redundancy: 9157/269313 objects degraded (3.400%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:12.915 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:12 smithi099 bash[87147]: cluster 2024-09-13T15:17:11.022795+0000 osd.0 (osd.0) 46 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:12.916 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:12 smithi099 bash[87147]: cluster 2024-09-13T15:17:11.022795+0000 osd.0 (osd.0) 46 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:12.916 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:12 smithi099 bash[87147]: cluster 2024-09-13T15:17:11.276546+0000 mon.a (mon.0) 2693 : cluster [WRN] Health check update: Degraded data redundancy: 9157/269313 objects degraded (3.400%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:12.916 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:12 smithi099 bash[87147]: cluster 2024-09-13T15:17:11.276546+0000 mon.a (mon.0) 2693 : cluster [WRN] Health check update: Degraded data redundancy: 9157/269313 objects degraded (3.400%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:12.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:12.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:13.018 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:13.058 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:13.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:13.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:13.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:13.189 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:13.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:13.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:13.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:13.319 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:13.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:13.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:13.425 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:13.446 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:13.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:13.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:13.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:13.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:13 smithi099 bash[84576]: cluster 2024-09-13T15:17:11.905056+0000 osd.2 (osd.2) 52 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:13.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:13 smithi099 bash[84576]: cluster 2024-09-13T15:17:11.905056+0000 osd.2 (osd.2) 52 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:13.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:13 smithi099 bash[84576]: cluster 2024-09-13T15:17:13.198661+0000 mgr.y (mgr.26252) 1536 : cluster [DBG] pgmap v2143: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 9157/267465 objects degraded (3.424%); 6295/267465 objects misplaced (2.354%); 12 MiB/s, 192 objects/s recovering 2024-09-13T15:17:13.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:13 smithi099 bash[84576]: cluster 2024-09-13T15:17:13.198661+0000 mgr.y (mgr.26252) 1536 : cluster [DBG] pgmap v2143: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 9157/267465 objects degraded (3.424%); 6295/267465 objects misplaced (2.354%); 12 MiB/s, 192 objects/s recovering 2024-09-13T15:17:13.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:13 smithi099 bash[87147]: cluster 2024-09-13T15:17:11.905056+0000 osd.2 (osd.2) 52 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:13.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:13 smithi099 bash[87147]: cluster 2024-09-13T15:17:11.905056+0000 osd.2 (osd.2) 52 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:13.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:13 smithi099 bash[87147]: cluster 2024-09-13T15:17:13.198661+0000 mgr.y (mgr.26252) 1536 : cluster [DBG] pgmap v2143: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 9157/267465 objects degraded (3.424%); 6295/267465 objects misplaced (2.354%); 12 MiB/s, 192 objects/s recovering 2024-09-13T15:17:13.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:13 smithi099 bash[87147]: cluster 2024-09-13T15:17:13.198661+0000 mgr.y (mgr.26252) 1536 : cluster [DBG] pgmap v2143: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 9157/267465 objects degraded (3.424%); 6295/267465 objects misplaced (2.354%); 12 MiB/s, 192 objects/s recovering 2024-09-13T15:17:13.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:13.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:13.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:13.707 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:13.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:13.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:13.814 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:13.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:13.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:13.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:13.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:13.989 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:13.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:13.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:14.094 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:14.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:13 smithi177 bash[68594]: cluster 2024-09-13T15:17:11.905056+0000 osd.2 (osd.2) 52 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:14.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:13 smithi177 bash[68594]: cluster 2024-09-13T15:17:11.905056+0000 osd.2 (osd.2) 52 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:14.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:13 smithi177 bash[68594]: cluster 2024-09-13T15:17:13.198661+0000 mgr.y (mgr.26252) 1536 : cluster [DBG] pgmap v2143: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 9157/267465 objects degraded (3.424%); 6295/267465 objects misplaced (2.354%); 12 MiB/s, 192 objects/s recovering 2024-09-13T15:17:14.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:13 smithi177 bash[68594]: cluster 2024-09-13T15:17:13.198661+0000 mgr.y (mgr.26252) 1536 : cluster [DBG] pgmap v2143: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 9157/267465 objects degraded (3.424%); 6295/267465 objects misplaced (2.354%); 12 MiB/s, 192 objects/s recovering 2024-09-13T15:17:14.120 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:14.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:14.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:14.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:14.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:14.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:14.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:14.362 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:14.382 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:14.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:14.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:14.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:14.494 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:14 smithi177 bash[68594]: cluster 2024-09-13T15:17:12.937710+0000 osd.2 (osd.2) 53 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:14.494 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:14 smithi177 bash[68594]: cluster 2024-09-13T15:17:12.937710+0000 osd.2 (osd.2) 53 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:14.507 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:14.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:14.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:14.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:14 smithi099 bash[84576]: cluster 2024-09-13T15:17:12.937710+0000 osd.2 (osd.2) 53 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:14.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:14 smithi099 bash[84576]: cluster 2024-09-13T15:17:12.937710+0000 osd.2 (osd.2) 53 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:14 smithi099 bash[87147]: cluster 2024-09-13T15:17:12.937710+0000 osd.2 (osd.2) 53 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:14 smithi099 bash[87147]: cluster 2024-09-13T15:17:12.937710+0000 osd.2 (osd.2) 53 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:14.610 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:14.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:14.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:14.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:14.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:14.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:14.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:14.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:14.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:14.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:14.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:14.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:14.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:15.032 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:15.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:15.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:15.136 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:15.167 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:15.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:15.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:15.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:15.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:15.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:15.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:15.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:15.442 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:15.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:15.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:15.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:15.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[87147]: cluster 2024-09-13T15:17:13.951015+0000 osd.2 (osd.2) 54 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:15.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[87147]: cluster 2024-09-13T15:17:13.951015+0000 osd.2 (osd.2) 54 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:15.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[87147]: cluster 2024-09-13T15:17:14.047507+0000 osd.0 (osd.0) 47 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:15.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[87147]: cluster 2024-09-13T15:17:14.047507+0000 osd.0 (osd.0) 47 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:15.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[87147]: cluster 2024-09-13T15:17:14.386145+0000 osd.1 (osd.1) 36 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:15.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[87147]: cluster 2024-09-13T15:17:14.386145+0000 osd.1 (osd.1) 36 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:15.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[87147]: cluster 2024-09-13T15:17:14.934233+0000 osd.3 (osd.3) 22 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:15.559 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[87147]: cluster 2024-09-13T15:17:14.934233+0000 osd.3 (osd.3) 22 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:15.560 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[87147]: cluster 2024-09-13T15:17:15.200351+0000 mgr.y (mgr.26252) 1537 : cluster [DBG] pgmap v2144: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 19 MiB/s, 299 objects/s recovering 2024-09-13T15:17:15.560 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[87147]: cluster 2024-09-13T15:17:15.200351+0000 mgr.y (mgr.26252) 1537 : cluster [DBG] pgmap v2144: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 19 MiB/s, 299 objects/s recovering 2024-09-13T15:17:15.560 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[84576]: cluster 2024-09-13T15:17:13.951015+0000 osd.2 (osd.2) 54 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:15.560 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[84576]: cluster 2024-09-13T15:17:13.951015+0000 osd.2 (osd.2) 54 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:15.560 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[84576]: cluster 2024-09-13T15:17:14.047507+0000 osd.0 (osd.0) 47 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:15.560 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[84576]: cluster 2024-09-13T15:17:14.047507+0000 osd.0 (osd.0) 47 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:15.560 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[84576]: cluster 2024-09-13T15:17:14.386145+0000 osd.1 (osd.1) 36 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:15.561 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[84576]: cluster 2024-09-13T15:17:14.386145+0000 osd.1 (osd.1) 36 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:15.561 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[84576]: cluster 2024-09-13T15:17:14.934233+0000 osd.3 (osd.3) 22 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:15.561 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[84576]: cluster 2024-09-13T15:17:14.934233+0000 osd.3 (osd.3) 22 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:15.561 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[84576]: cluster 2024-09-13T15:17:15.200351+0000 mgr.y (mgr.26252) 1537 : cluster [DBG] pgmap v2144: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 19 MiB/s, 299 objects/s recovering 2024-09-13T15:17:15.561 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:15 smithi099 bash[84576]: cluster 2024-09-13T15:17:15.200351+0000 mgr.y (mgr.26252) 1537 : cluster [DBG] pgmap v2144: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 19 MiB/s, 299 objects/s recovering 2024-09-13T15:17:15.575 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:15.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:15.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:15 smithi177 bash[68594]: cluster 2024-09-13T15:17:13.951015+0000 osd.2 (osd.2) 54 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:15.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:15 smithi177 bash[68594]: cluster 2024-09-13T15:17:13.951015+0000 osd.2 (osd.2) 54 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:15.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:15 smithi177 bash[68594]: cluster 2024-09-13T15:17:14.047507+0000 osd.0 (osd.0) 47 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:15.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:15 smithi177 bash[68594]: cluster 2024-09-13T15:17:14.047507+0000 osd.0 (osd.0) 47 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:15.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:15 smithi177 bash[68594]: cluster 2024-09-13T15:17:14.386145+0000 osd.1 (osd.1) 36 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:15.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:15 smithi177 bash[68594]: cluster 2024-09-13T15:17:14.386145+0000 osd.1 (osd.1) 36 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:15.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:15 smithi177 bash[68594]: cluster 2024-09-13T15:17:14.934233+0000 osd.3 (osd.3) 22 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:15.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:15 smithi177 bash[68594]: cluster 2024-09-13T15:17:14.934233+0000 osd.3 (osd.3) 22 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:15.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:15 smithi177 bash[68594]: cluster 2024-09-13T15:17:15.200351+0000 mgr.y (mgr.26252) 1537 : cluster [DBG] pgmap v2144: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 19 MiB/s, 299 objects/s recovering 2024-09-13T15:17:15.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:15 smithi177 bash[68594]: cluster 2024-09-13T15:17:15.200351+0000 mgr.y (mgr.26252) 1537 : cluster [DBG] pgmap v2144: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 19 MiB/s, 299 objects/s recovering 2024-09-13T15:17:15.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:15.712 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:15.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:15.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:15.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:15.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:15.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:15.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:15.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:15.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:15.983 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:15.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:16.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:16.121 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:16.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:16.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:16.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:16.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:16.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:16.349 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: cluster 2024-09-13T15:17:15.089435+0000 osd.0 (osd.0) 48 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: cluster 2024-09-13T15:17:15.089435+0000 osd.0 (osd.0) 48 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: cluster 2024-09-13T15:17:15.373031+0000 osd.1 (osd.1) 37 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: cluster 2024-09-13T15:17:15.373031+0000 osd.1 (osd.1) 37 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: cluster 2024-09-13T15:17:15.918357+0000 osd.3 (osd.3) 23 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: cluster 2024-09-13T15:17:15.918357+0000 osd.3 (osd.3) 23 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: audit 2024-09-13T15:17:16.266838+0000 mon.a (mon.0) 2694 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: audit 2024-09-13T15:17:16.266838+0000 mon.a (mon.0) 2694 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: audit 2024-09-13T15:17:16.274883+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: audit 2024-09-13T15:17:16.274883+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: audit 2024-09-13T15:17:16.297238+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: audit 2024-09-13T15:17:16.297238+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.350 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: audit 2024-09-13T15:17:16.308133+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.351 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[84576]: audit 2024-09-13T15:17:16.308133+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: cluster 2024-09-13T15:17:15.089435+0000 osd.0 (osd.0) 48 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:16.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: cluster 2024-09-13T15:17:15.089435+0000 osd.0 (osd.0) 48 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:16.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: cluster 2024-09-13T15:17:15.373031+0000 osd.1 (osd.1) 37 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:16.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: cluster 2024-09-13T15:17:15.373031+0000 osd.1 (osd.1) 37 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:16.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: cluster 2024-09-13T15:17:15.918357+0000 osd.3 (osd.3) 23 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:16.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: cluster 2024-09-13T15:17:15.918357+0000 osd.3 (osd.3) 23 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:16.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: audit 2024-09-13T15:17:16.266838+0000 mon.a (mon.0) 2694 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: audit 2024-09-13T15:17:16.266838+0000 mon.a (mon.0) 2694 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.351 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: audit 2024-09-13T15:17:16.274883+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.352 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: audit 2024-09-13T15:17:16.274883+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.352 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: audit 2024-09-13T15:17:16.297238+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.352 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: audit 2024-09-13T15:17:16.297238+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.352 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: audit 2024-09-13T15:17:16.308133+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.352 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:16 smithi099 bash[87147]: audit 2024-09-13T15:17:16.308133+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.352 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:17:16.352 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:17:16.352 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:17:16.353 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:17:16.353 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:17:16.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:16.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: cluster 2024-09-13T15:17:15.089435+0000 osd.0 (osd.0) 48 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:16.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: cluster 2024-09-13T15:17:15.089435+0000 osd.0 (osd.0) 48 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:16.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: cluster 2024-09-13T15:17:15.373031+0000 osd.1 (osd.1) 37 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:16.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: cluster 2024-09-13T15:17:15.373031+0000 osd.1 (osd.1) 37 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: cluster 2024-09-13T15:17:15.918357+0000 osd.3 (osd.3) 23 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: cluster 2024-09-13T15:17:15.918357+0000 osd.3 (osd.3) 23 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: audit 2024-09-13T15:17:16.266838+0000 mon.a (mon.0) 2694 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: audit 2024-09-13T15:17:16.266838+0000 mon.a (mon.0) 2694 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: audit 2024-09-13T15:17:16.274883+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: audit 2024-09-13T15:17:16.274883+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: audit 2024-09-13T15:17:16.297238+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: audit 2024-09-13T15:17:16.297238+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: audit 2024-09-13T15:17:16.308133+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:16 smithi177 bash[68594]: audit 2024-09-13T15:17:16.308133+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:16.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:16.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:16.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:16.516 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:16.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:16.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:16.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:16.583 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:17:16.583 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:17:16.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:16.657 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:17:16.657 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:17:16.694 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:17:16.694 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:17:16.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:16.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:16.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:16.720 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:17:16.720 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:17:16.755 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:17:16.755 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:17:16.792 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:17:16.793 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:17:16.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:16.826 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:17:16.826 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:17:16.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:16.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:16.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:16.860 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:17:16.860 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:17:16.940 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:16.960 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:16.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:17.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:17.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:17.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:17.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:17.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:17.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:17.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:17.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:17.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:17:17] "GET /metrics HTTP/1.1" 200 36805 "" "Prometheus/2.51.0" 2024-09-13T15:17:17.336 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:17.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:17.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:17.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:17.490 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:17.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:17.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:17.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: cluster 2024-09-13T15:17:16.076255+0000 osd.0 (osd.0) 49 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:17.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: cluster 2024-09-13T15:17:16.076255+0000 osd.0 (osd.0) 49 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: cluster 2024-09-13T15:17:16.420274+0000 osd.1 (osd.1) 38 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: cluster 2024-09-13T15:17:16.420274+0000 osd.1 (osd.1) 38 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: audit 2024-09-13T15:17:16.540515+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.99:0/278744994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: audit 2024-09-13T15:17:16.540515+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.99:0/278744994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: cluster 2024-09-13T15:17:16.658857+0000 mon.a (mon.0) 2698 : cluster [WRN] Health check update: Degraded data redundancy: 8943/258834 objects degraded (3.455%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: cluster 2024-09-13T15:17:16.658857+0000 mon.a (mon.0) 2698 : cluster [WRN] Health check update: Degraded data redundancy: 8943/258834 objects degraded (3.455%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: cluster 2024-09-13T15:17:17.201607+0000 mgr.y (mgr.26252) 1538 : cluster [DBG] pgmap v2145: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 13 MiB/s, 206 objects/s recovering 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: cluster 2024-09-13T15:17:17.201607+0000 mgr.y (mgr.26252) 1538 : cluster [DBG] pgmap v2145: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 13 MiB/s, 206 objects/s recovering 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: audit 2024-09-13T15:17:17.252593+0000 mon.a (mon.0) 2699 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: audit 2024-09-13T15:17:17.252593+0000 mon.a (mon.0) 2699 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:17:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: audit 2024-09-13T15:17:17.253705+0000 mon.a (mon.0) 2700 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:17:17.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: audit 2024-09-13T15:17:17.253705+0000 mon.a (mon.0) 2700 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:17:17.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: audit 2024-09-13T15:17:17.265706+0000 mon.a (mon.0) 2701 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:17.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[84576]: audit 2024-09-13T15:17:17.265706+0000 mon.a (mon.0) 2701 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:17.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: cluster 2024-09-13T15:17:16.076255+0000 osd.0 (osd.0) 49 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:17.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: cluster 2024-09-13T15:17:16.076255+0000 osd.0 (osd.0) 49 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:17.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: cluster 2024-09-13T15:17:16.420274+0000 osd.1 (osd.1) 38 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:17.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: cluster 2024-09-13T15:17:16.420274+0000 osd.1 (osd.1) 38 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:17.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: audit 2024-09-13T15:17:16.540515+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.99:0/278744994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:17.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: audit 2024-09-13T15:17:16.540515+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.99:0/278744994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:17.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: cluster 2024-09-13T15:17:16.658857+0000 mon.a (mon.0) 2698 : cluster [WRN] Health check update: Degraded data redundancy: 8943/258834 objects degraded (3.455%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:17.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: cluster 2024-09-13T15:17:16.658857+0000 mon.a (mon.0) 2698 : cluster [WRN] Health check update: Degraded data redundancy: 8943/258834 objects degraded (3.455%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:17.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: cluster 2024-09-13T15:17:17.201607+0000 mgr.y (mgr.26252) 1538 : cluster [DBG] pgmap v2145: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 13 MiB/s, 206 objects/s recovering 2024-09-13T15:17:17.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: cluster 2024-09-13T15:17:17.201607+0000 mgr.y (mgr.26252) 1538 : cluster [DBG] pgmap v2145: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 13 MiB/s, 206 objects/s recovering 2024-09-13T15:17:17.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: audit 2024-09-13T15:17:17.252593+0000 mon.a (mon.0) 2699 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:17:17.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: audit 2024-09-13T15:17:17.252593+0000 mon.a (mon.0) 2699 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:17:17.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: audit 2024-09-13T15:17:17.253705+0000 mon.a (mon.0) 2700 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:17:17.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: audit 2024-09-13T15:17:17.253705+0000 mon.a (mon.0) 2700 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:17:17.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: audit 2024-09-13T15:17:17.265706+0000 mon.a (mon.0) 2701 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:17.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:17 smithi099 bash[87147]: audit 2024-09-13T15:17:17.265706+0000 mon.a (mon.0) 2701 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:17.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: cluster 2024-09-13T15:17:16.076255+0000 osd.0 (osd.0) 49 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: cluster 2024-09-13T15:17:16.076255+0000 osd.0 (osd.0) 49 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: cluster 2024-09-13T15:17:16.420274+0000 osd.1 (osd.1) 38 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: cluster 2024-09-13T15:17:16.420274+0000 osd.1 (osd.1) 38 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: audit 2024-09-13T15:17:16.540515+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.99:0/278744994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: audit 2024-09-13T15:17:16.540515+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.99:0/278744994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: cluster 2024-09-13T15:17:16.658857+0000 mon.a (mon.0) 2698 : cluster [WRN] Health check update: Degraded data redundancy: 8943/258834 objects degraded (3.455%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: cluster 2024-09-13T15:17:16.658857+0000 mon.a (mon.0) 2698 : cluster [WRN] Health check update: Degraded data redundancy: 8943/258834 objects degraded (3.455%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: cluster 2024-09-13T15:17:17.201607+0000 mgr.y (mgr.26252) 1538 : cluster [DBG] pgmap v2145: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 13 MiB/s, 206 objects/s recovering 2024-09-13T15:17:17.605 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: cluster 2024-09-13T15:17:17.201607+0000 mgr.y (mgr.26252) 1538 : cluster [DBG] pgmap v2145: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8943/258834 objects degraded (3.455%); 4911/258834 objects misplaced (1.897%); 13 MiB/s, 206 objects/s recovering 2024-09-13T15:17:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: audit 2024-09-13T15:17:17.252593+0000 mon.a (mon.0) 2699 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:17:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: audit 2024-09-13T15:17:17.252593+0000 mon.a (mon.0) 2699 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:17:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: audit 2024-09-13T15:17:17.253705+0000 mon.a (mon.0) 2700 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:17:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: audit 2024-09-13T15:17:17.253705+0000 mon.a (mon.0) 2700 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:17:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: audit 2024-09-13T15:17:17.265706+0000 mon.a (mon.0) 2701 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:17.606 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:17 smithi177 bash[68594]: audit 2024-09-13T15:17:17.265706+0000 mon.a (mon.0) 2701 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:17:17.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:17.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:17.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:17.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:17.748 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:17.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:17.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:17.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:17.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:17.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:17:17.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:17.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:17.999 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:18.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:18.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:18.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:18.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:18.177 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:18.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:18.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:18.280 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:18.307 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:18.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:18.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:18.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:18.420 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:18 smithi099 bash[84576]: cluster 2024-09-13T15:17:17.058159+0000 osd.2 (osd.2) 55 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:18.420 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:18 smithi099 bash[84576]: cluster 2024-09-13T15:17:17.058159+0000 osd.2 (osd.2) 55 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:18.420 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:18 smithi099 bash[84576]: audit 2024-09-13T15:17:18.317696+0000 mon.a (mon.0) 2702 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:18.420 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:18 smithi099 bash[84576]: audit 2024-09-13T15:17:18.317696+0000 mon.a (mon.0) 2702 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:18.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:18 smithi099 bash[87147]: cluster 2024-09-13T15:17:17.058159+0000 osd.2 (osd.2) 55 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:18.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:18 smithi099 bash[87147]: cluster 2024-09-13T15:17:17.058159+0000 osd.2 (osd.2) 55 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:18.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:18 smithi099 bash[87147]: audit 2024-09-13T15:17:18.317696+0000 mon.a (mon.0) 2702 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:18.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:18 smithi099 bash[87147]: audit 2024-09-13T15:17:18.317696+0000 mon.a (mon.0) 2702 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:18.444 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:18.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:18.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:18.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:18.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:18 smithi177 bash[68594]: cluster 2024-09-13T15:17:17.058159+0000 osd.2 (osd.2) 55 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:18.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:18 smithi177 bash[68594]: cluster 2024-09-13T15:17:17.058159+0000 osd.2 (osd.2) 55 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:18.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:18 smithi177 bash[68594]: audit 2024-09-13T15:17:18.317696+0000 mon.a (mon.0) 2702 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:18.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:18 smithi177 bash[68594]: audit 2024-09-13T15:17:18.317696+0000 mon.a (mon.0) 2702 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:18.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:18.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:18.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:18.677 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:18.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:18.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:18.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:18.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:18.823 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:18.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:18.892 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T15:17:18.897 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T15:17:18.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:18.951 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:18.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:18.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:19.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:19.082 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:19.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:19.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:19.186 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:19.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:19.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:19.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:19.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:19.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:19.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:19.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:19.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:19.467 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:19.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:19.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:19.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:19.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:19.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:19.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:19.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:19 smithi177 bash[68594]: cluster 2024-09-13T15:17:18.062829+0000 osd.2 (osd.2) 56 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:19 smithi177 bash[68594]: cluster 2024-09-13T15:17:18.062829+0000 osd.2 (osd.2) 56 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:19 smithi177 bash[68594]: audit 2024-09-13T15:17:18.879261+0000 mgr.y (mgr.26252) 1539 : audit [DBG] from='client.46160 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:17:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:19 smithi177 bash[68594]: audit 2024-09-13T15:17:18.879261+0000 mgr.y (mgr.26252) 1539 : audit [DBG] from='client.46160 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:17:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:19 smithi177 bash[68594]: cluster 2024-09-13T15:17:19.202532+0000 mgr.y (mgr.26252) 1540 : cluster [DBG] pgmap v2146: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 8739/250914 objects degraded (3.483%); 4003/250914 objects misplaced (1.595%); 17 MiB/s, 273 objects/s recovering 2024-09-13T15:17:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:19 smithi177 bash[68594]: cluster 2024-09-13T15:17:19.202532+0000 mgr.y (mgr.26252) 1540 : cluster [DBG] pgmap v2146: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 8739/250914 objects degraded (3.483%); 4003/250914 objects misplaced (1.595%); 17 MiB/s, 273 objects/s recovering 2024-09-13T15:17:19.701 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:19.718 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[84576]: cluster 2024-09-13T15:17:18.062829+0000 osd.2 (osd.2) 56 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:19.718 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[84576]: cluster 2024-09-13T15:17:18.062829+0000 osd.2 (osd.2) 56 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:19.718 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[84576]: audit 2024-09-13T15:17:18.879261+0000 mgr.y (mgr.26252) 1539 : audit [DBG] from='client.46160 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:17:19.718 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[84576]: audit 2024-09-13T15:17:18.879261+0000 mgr.y (mgr.26252) 1539 : audit [DBG] from='client.46160 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:17:19.719 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[84576]: cluster 2024-09-13T15:17:19.202532+0000 mgr.y (mgr.26252) 1540 : cluster [DBG] pgmap v2146: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 8739/250914 objects degraded (3.483%); 4003/250914 objects misplaced (1.595%); 17 MiB/s, 273 objects/s recovering 2024-09-13T15:17:19.719 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[84576]: cluster 2024-09-13T15:17:19.202532+0000 mgr.y (mgr.26252) 1540 : cluster [DBG] pgmap v2146: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 8739/250914 objects degraded (3.483%); 4003/250914 objects misplaced (1.595%); 17 MiB/s, 273 objects/s recovering 2024-09-13T15:17:19.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[87147]: cluster 2024-09-13T15:17:18.062829+0000 osd.2 (osd.2) 56 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:19.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[87147]: cluster 2024-09-13T15:17:18.062829+0000 osd.2 (osd.2) 56 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:19.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[87147]: audit 2024-09-13T15:17:18.879261+0000 mgr.y (mgr.26252) 1539 : audit [DBG] from='client.46160 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:17:19.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[87147]: audit 2024-09-13T15:17:18.879261+0000 mgr.y (mgr.26252) 1539 : audit [DBG] from='client.46160 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:17:19.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[87147]: cluster 2024-09-13T15:17:19.202532+0000 mgr.y (mgr.26252) 1540 : cluster [DBG] pgmap v2146: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 8739/250914 objects degraded (3.483%); 4003/250914 objects misplaced (1.595%); 17 MiB/s, 273 objects/s recovering 2024-09-13T15:17:19.719 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:19 smithi099 bash[87147]: cluster 2024-09-13T15:17:19.202532+0000 mgr.y (mgr.26252) 1540 : cluster [DBG] pgmap v2146: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 8739/250914 objects degraded (3.483%); 4003/250914 objects misplaced (1.595%); 17 MiB/s, 273 objects/s recovering 2024-09-13T15:17:19.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:19.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:19.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:19.833 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:19.853 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:19.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:19.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:19.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:20.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:20.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:20.137 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:20.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:20.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:20.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:20.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:20.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:20.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:20.392 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:17:20.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:20.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:20.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:20.416 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[87147]: cluster 2024-09-13T15:17:19.075613+0000 osd.2 (osd.2) 57 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:20.416 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[87147]: cluster 2024-09-13T15:17:19.075613+0000 osd.2 (osd.2) 57 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:20.416 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[87147]: cluster 2024-09-13T15:17:19.079875+0000 osd.0 (osd.0) 50 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:20.416 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[87147]: cluster 2024-09-13T15:17:19.079875+0000 osd.0 (osd.0) 50 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:20.416 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[87147]: audit 2024-09-13T15:17:19.464581+0000 mon.a (mon.0) 2703 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:17:20.416 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[87147]: audit 2024-09-13T15:17:19.464581+0000 mon.a (mon.0) 2703 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:17:20.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[84576]: cluster 2024-09-13T15:17:19.075613+0000 osd.2 (osd.2) 57 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:20.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[84576]: cluster 2024-09-13T15:17:19.075613+0000 osd.2 (osd.2) 57 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:20.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[84576]: cluster 2024-09-13T15:17:19.079875+0000 osd.0 (osd.0) 50 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:20.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[84576]: cluster 2024-09-13T15:17:19.079875+0000 osd.0 (osd.0) 50 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:20.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[84576]: audit 2024-09-13T15:17:19.464581+0000 mon.a (mon.0) 2703 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:17:20.417 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:20 smithi099 bash[84576]: audit 2024-09-13T15:17:19.464581+0000 mon.a (mon.0) 2703 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:17:20.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:20.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:20.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:20.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:20.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:17:20 smithi177 bash[61439]: ts=2024-09-13T15:17:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:17:20.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:20 smithi177 bash[68594]: cluster 2024-09-13T15:17:19.075613+0000 osd.2 (osd.2) 57 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:20.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:20 smithi177 bash[68594]: cluster 2024-09-13T15:17:19.075613+0000 osd.2 (osd.2) 57 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:20.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:20 smithi177 bash[68594]: cluster 2024-09-13T15:17:19.079875+0000 osd.0 (osd.0) 50 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:20.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:20 smithi177 bash[68594]: cluster 2024-09-13T15:17:19.079875+0000 osd.0 (osd.0) 50 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:20.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:20 smithi177 bash[68594]: audit 2024-09-13T15:17:19.464581+0000 mon.a (mon.0) 2703 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:17:20.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:20 smithi177 bash[68594]: audit 2024-09-13T15:17:19.464581+0000 mon.a (mon.0) 2703 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:17:20.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:20.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:20.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:20.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:20.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:20.817 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:20.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:20.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:20.920 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:20.939 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:20.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:20.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:21.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:21.072 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:21.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:21.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:21.176 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:21.199 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:21.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:21.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:21.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:21.327 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:21.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:21.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:21.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:21.440 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:19.425605+0000 osd.1 (osd.1) 39 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:21.440 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:19.425605+0000 osd.1 (osd.1) 39 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:21.440 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:20.098875+0000 osd.0 (osd.0) 51 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:21.440 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:20.098875+0000 osd.0 (osd.0) 51 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:21.440 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:20.373266+0000 mon.a (mon.0) 2704 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:17:21.440 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:20.373266+0000 mon.a (mon.0) 2704 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:17:21.440 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: audit 2024-09-13T15:17:20.386702+0000 mon.a (mon.0) 2705 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: audit 2024-09-13T15:17:20.386702+0000 mon.a (mon.0) 2705 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:20.394097+0000 mon.a (mon.0) 2706 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:20.394097+0000 mon.a (mon.0) 2706 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:20.472127+0000 osd.1 (osd.1) 40 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:20.472127+0000 osd.1 (osd.1) 40 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:20.962235+0000 osd.3 (osd.3) 24 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:20.962235+0000 osd.3 (osd.3) 24 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.100940+0000 osd.1 (osd.1) 41 : cluster [DBG] 46.8 scrub ok 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.100940+0000 osd.1 (osd.1) 41 : cluster [DBG] 46.8 scrub ok 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.203003+0000 mgr.y (mgr.26252) 1541 : cluster [DBG] pgmap v2148: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 8739/247899 objects degraded (3.525%); 3544/247899 objects misplaced (1.430%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:21 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.203003+0000 mgr.y (mgr.26252) 1541 : cluster [DBG] pgmap v2148: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 8739/247899 objects degraded (3.525%); 3544/247899 objects misplaced (1.430%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:21.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:21.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:21.560 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:21.592 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:21.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:21.700 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:21.708 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:19.425605+0000 osd.1 (osd.1) 39 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:21.708 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:19.425605+0000 osd.1 (osd.1) 39 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:21.708 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:20.098875+0000 osd.0 (osd.0) 51 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:21.708 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:20.098875+0000 osd.0 (osd.0) 51 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:21.708 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:20.373266+0000 mon.a (mon.0) 2704 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:17:21.708 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:20.373266+0000 mon.a (mon.0) 2704 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:17:21.708 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: audit 2024-09-13T15:17:20.386702+0000 mon.a (mon.0) 2705 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: audit 2024-09-13T15:17:20.386702+0000 mon.a (mon.0) 2705 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:20.394097+0000 mon.a (mon.0) 2706 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:20.394097+0000 mon.a (mon.0) 2706 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:20.472127+0000 osd.1 (osd.1) 40 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:20.472127+0000 osd.1 (osd.1) 40 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:20.962235+0000 osd.3 (osd.3) 24 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:20.962235+0000 osd.3 (osd.3) 24 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.100940+0000 osd.1 (osd.1) 41 : cluster [DBG] 46.8 scrub ok 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.100940+0000 osd.1 (osd.1) 41 : cluster [DBG] 46.8 scrub ok 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.203003+0000 mgr.y (mgr.26252) 1541 : cluster [DBG] pgmap v2148: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 8739/247899 objects degraded (3.525%); 3544/247899 objects misplaced (1.430%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:21.709 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.203003+0000 mgr.y (mgr.26252) 1541 : cluster [DBG] pgmap v2148: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 8739/247899 objects degraded (3.525%); 3544/247899 objects misplaced (1.430%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:19.425605+0000 osd.1 (osd.1) 39 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:19.425605+0000 osd.1 (osd.1) 39 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:20.098875+0000 osd.0 (osd.0) 51 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:20.098875+0000 osd.0 (osd.0) 51 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:20.373266+0000 mon.a (mon.0) 2704 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:20.373266+0000 mon.a (mon.0) 2704 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: audit 2024-09-13T15:17:20.386702+0000 mon.a (mon.0) 2705 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: audit 2024-09-13T15:17:20.386702+0000 mon.a (mon.0) 2705 : audit [INF] from='client.? 172.21.15.99:0/3804834435' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:20.394097+0000 mon.a (mon.0) 2706 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:20.394097+0000 mon.a (mon.0) 2706 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2024-09-13T15:17:21.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:20.472127+0000 osd.1 (osd.1) 40 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:21.711 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:20.472127+0000 osd.1 (osd.1) 40 : cluster [DBG] 46.8 scrub starts 2024-09-13T15:17:21.711 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:20.962235+0000 osd.3 (osd.3) 24 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:21.711 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:20.962235+0000 osd.3 (osd.3) 24 : cluster [DBG] 46.15 scrub starts 2024-09-13T15:17:21.711 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.100940+0000 osd.1 (osd.1) 41 : cluster [DBG] 46.8 scrub ok 2024-09-13T15:17:21.711 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.100940+0000 osd.1 (osd.1) 41 : cluster [DBG] 46.8 scrub ok 2024-09-13T15:17:21.711 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.203003+0000 mgr.y (mgr.26252) 1541 : cluster [DBG] pgmap v2148: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 8739/247899 objects degraded (3.525%); 3544/247899 objects misplaced (1.430%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:21.711 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:21 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.203003+0000 mgr.y (mgr.26252) 1541 : cluster [DBG] pgmap v2148: 99 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 8739/247899 objects degraded (3.525%); 3544/247899 objects misplaced (1.430%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:21.728 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:21.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:21.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:21.831 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:21.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:21.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:21.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:21.962 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:21.997 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:22.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:22.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:22.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:22.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:22.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:22.257 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:22.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:22.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:22.362 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:22.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:22.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:22.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:22.526 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:22.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:22.634 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:22.645 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.131857+0000 osd.0 (osd.0) 52 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:22.645 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.131857+0000 osd.0 (osd.0) 52 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:22.645 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.519432+0000 osd.1 (osd.1) 42 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:22.645 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.519432+0000 osd.1 (osd.1) 42 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:22.645 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.640161+0000 osd.3 (osd.3) 25 : cluster [DBG] 46.15 scrub ok 2024-09-13T15:17:22.645 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.640161+0000 osd.3 (osd.3) 25 : cluster [DBG] 46.15 scrub ok 2024-09-13T15:17:22.645 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.659646+0000 mon.a (mon.0) 2707 : cluster [WRN] Health check update: Degraded data redundancy: 8739/247899 objects degraded (3.525%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:22.645 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.659646+0000 mon.a (mon.0) 2707 : cluster [WRN] Health check update: Degraded data redundancy: 8739/247899 objects degraded (3.525%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:22.645 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.990222+0000 osd.3 (osd.3) 26 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:21.990222+0000 osd.3 (osd.3) 26 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:22.285999+0000 osd.1 (osd.1) 43 : cluster [DBG] 46.3 scrub ok 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[84576]: cluster 2024-09-13T15:17:22.285999+0000 osd.1 (osd.1) 43 : cluster [DBG] 46.3 scrub ok 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.131857+0000 osd.0 (osd.0) 52 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.131857+0000 osd.0 (osd.0) 52 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.519432+0000 osd.1 (osd.1) 42 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.519432+0000 osd.1 (osd.1) 42 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.640161+0000 osd.3 (osd.3) 25 : cluster [DBG] 46.15 scrub ok 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.640161+0000 osd.3 (osd.3) 25 : cluster [DBG] 46.15 scrub ok 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.659646+0000 mon.a (mon.0) 2707 : cluster [WRN] Health check update: Degraded data redundancy: 8739/247899 objects degraded (3.525%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:22.646 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.659646+0000 mon.a (mon.0) 2707 : cluster [WRN] Health check update: Degraded data redundancy: 8739/247899 objects degraded (3.525%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:22.647 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.990222+0000 osd.3 (osd.3) 26 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:22.647 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:21.990222+0000 osd.3 (osd.3) 26 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:22.647 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:22.285999+0000 osd.1 (osd.1) 43 : cluster [DBG] 46.3 scrub ok 2024-09-13T15:17:22.647 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:22 smithi099 bash[87147]: cluster 2024-09-13T15:17:22.285999+0000 osd.1 (osd.1) 43 : cluster [DBG] 46.3 scrub ok 2024-09-13T15:17:22.666 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:22.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:22.750 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:17:22.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:22.784 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.131857+0000 osd.0 (osd.0) 52 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:22.784 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.131857+0000 osd.0 (osd.0) 52 : cluster [DBG] 46.9 scrub starts 2024-09-13T15:17:22.784 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.519432+0000 osd.1 (osd.1) 42 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:22.784 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.519432+0000 osd.1 (osd.1) 42 : cluster [DBG] 46.3 scrub starts 2024-09-13T15:17:22.784 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.640161+0000 osd.3 (osd.3) 25 : cluster [DBG] 46.15 scrub ok 2024-09-13T15:17:22.785 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.640161+0000 osd.3 (osd.3) 25 : cluster [DBG] 46.15 scrub ok 2024-09-13T15:17:22.785 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.659646+0000 mon.a (mon.0) 2707 : cluster [WRN] Health check update: Degraded data redundancy: 8739/247899 objects degraded (3.525%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:22.785 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.659646+0000 mon.a (mon.0) 2707 : cluster [WRN] Health check update: Degraded data redundancy: 8739/247899 objects degraded (3.525%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:22.785 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.990222+0000 osd.3 (osd.3) 26 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:22.785 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:21.990222+0000 osd.3 (osd.3) 26 : cluster [DBG] 46.5 scrub starts 2024-09-13T15:17:22.785 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:22.285999+0000 osd.1 (osd.1) 43 : cluster [DBG] 46.3 scrub ok 2024-09-13T15:17:22.785 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:22 smithi177 bash[68594]: cluster 2024-09-13T15:17:22.285999+0000 osd.1 (osd.1) 43 : cluster [DBG] 46.3 scrub ok 2024-09-13T15:17:22.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:22.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:22.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:22.917 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:22.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:22.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:22.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:23.043 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:23.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:23.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:23.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:23.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:23.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:23.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:23.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:23.324 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:23.350 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:23.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:23.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:23.455 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:23.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:23.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:23.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:23.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:23.592 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:23 smithi177 bash[68594]: cluster 2024-09-13T15:17:22.066098+0000 osd.2 (osd.2) 58 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:23.592 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:23 smithi177 bash[68594]: cluster 2024-09-13T15:17:22.066098+0000 osd.2 (osd.2) 58 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:23.592 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:23 smithi177 bash[68594]: cluster 2024-09-13T15:17:22.136886+0000 osd.0 (osd.0) 53 : cluster [DBG] 46.9 scrub ok 2024-09-13T15:17:23.593 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:23 smithi177 bash[68594]: cluster 2024-09-13T15:17:22.136886+0000 osd.0 (osd.0) 53 : cluster [DBG] 46.9 scrub ok 2024-09-13T15:17:23.593 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:23 smithi177 bash[68594]: cluster 2024-09-13T15:17:22.381294+0000 osd.3 (osd.3) 27 : cluster [DBG] 46.5 scrub ok 2024-09-13T15:17:23.593 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:23 smithi177 bash[68594]: cluster 2024-09-13T15:17:22.381294+0000 osd.3 (osd.3) 27 : cluster [DBG] 46.5 scrub ok 2024-09-13T15:17:23.593 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:23 smithi177 bash[68594]: cluster 2024-09-13T15:17:23.203687+0000 mgr.y (mgr.26252) 1542 : cluster [DBG] pgmap v2149: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 8739/246108 objects degraded (3.551%); 3544/246108 objects misplaced (1.440%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:23.593 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:23 smithi177 bash[68594]: cluster 2024-09-13T15:17:23.203687+0000 mgr.y (mgr.26252) 1542 : cluster [DBG] pgmap v2149: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 8739/246108 objects degraded (3.551%); 3544/246108 objects misplaced (1.440%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:23.604 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:23.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:23.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:23.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:23.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[84576]: cluster 2024-09-13T15:17:22.066098+0000 osd.2 (osd.2) 58 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:23.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[84576]: cluster 2024-09-13T15:17:22.066098+0000 osd.2 (osd.2) 58 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:23.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[84576]: cluster 2024-09-13T15:17:22.136886+0000 osd.0 (osd.0) 53 : cluster [DBG] 46.9 scrub ok 2024-09-13T15:17:23.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[84576]: cluster 2024-09-13T15:17:22.136886+0000 osd.0 (osd.0) 53 : cluster [DBG] 46.9 scrub ok 2024-09-13T15:17:23.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[84576]: cluster 2024-09-13T15:17:22.381294+0000 osd.3 (osd.3) 27 : cluster [DBG] 46.5 scrub ok 2024-09-13T15:17:23.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[84576]: cluster 2024-09-13T15:17:22.381294+0000 osd.3 (osd.3) 27 : cluster [DBG] 46.5 scrub ok 2024-09-13T15:17:23.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[84576]: cluster 2024-09-13T15:17:23.203687+0000 mgr.y (mgr.26252) 1542 : cluster [DBG] pgmap v2149: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 8739/246108 objects degraded (3.551%); 3544/246108 objects misplaced (1.440%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:23.717 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[84576]: cluster 2024-09-13T15:17:23.203687+0000 mgr.y (mgr.26252) 1542 : cluster [DBG] pgmap v2149: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 8739/246108 objects degraded (3.551%); 3544/246108 objects misplaced (1.440%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:23.717 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[87147]: cluster 2024-09-13T15:17:22.066098+0000 osd.2 (osd.2) 58 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:23.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[87147]: cluster 2024-09-13T15:17:22.066098+0000 osd.2 (osd.2) 58 : cluster [DBG] 46.17 scrub starts 2024-09-13T15:17:23.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[87147]: cluster 2024-09-13T15:17:22.136886+0000 osd.0 (osd.0) 53 : cluster [DBG] 46.9 scrub ok 2024-09-13T15:17:23.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[87147]: cluster 2024-09-13T15:17:22.136886+0000 osd.0 (osd.0) 53 : cluster [DBG] 46.9 scrub ok 2024-09-13T15:17:23.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[87147]: cluster 2024-09-13T15:17:22.381294+0000 osd.3 (osd.3) 27 : cluster [DBG] 46.5 scrub ok 2024-09-13T15:17:23.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[87147]: cluster 2024-09-13T15:17:22.381294+0000 osd.3 (osd.3) 27 : cluster [DBG] 46.5 scrub ok 2024-09-13T15:17:23.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[87147]: cluster 2024-09-13T15:17:23.203687+0000 mgr.y (mgr.26252) 1542 : cluster [DBG] pgmap v2149: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 8739/246108 objects degraded (3.551%); 3544/246108 objects misplaced (1.440%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:23.718 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:23 smithi099 bash[87147]: cluster 2024-09-13T15:17:23.203687+0000 mgr.y (mgr.26252) 1542 : cluster [DBG] pgmap v2149: 99 pgs: 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 92 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 8739/246108 objects degraded (3.551%); 3544/246108 objects misplaced (1.440%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:17:23.732 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:23.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:23.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:23.834 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:23.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:23.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:23.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:23.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:23.982 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:23.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:23.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:24.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:24.114 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:24.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:24.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:24.217 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:24.242 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:24.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:24.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:24.345 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:24.370 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:24.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:24.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:24.476 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:24.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:24.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:24.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:24.601 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:24 smithi177 bash[68594]: cluster 2024-09-13T15:17:23.160511+0000 osd.2 (osd.2) 59 : cluster [DBG] 46.17 scrub ok 2024-09-13T15:17:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:24 smithi177 bash[68594]: cluster 2024-09-13T15:17:23.160511+0000 osd.2 (osd.2) 59 : cluster [DBG] 46.17 scrub ok 2024-09-13T15:17:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:24 smithi177 bash[68594]: cluster 2024-09-13T15:17:23.340051+0000 osd.5 (osd.5) 16 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:24 smithi177 bash[68594]: cluster 2024-09-13T15:17:23.340051+0000 osd.5 (osd.5) 16 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:24 smithi177 bash[68594]: cluster 2024-09-13T15:17:23.407873+0000 mon.a (mon.0) 2708 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2024-09-13T15:17:24.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:24 smithi177 bash[68594]: cluster 2024-09-13T15:17:23.407873+0000 mon.a (mon.0) 2708 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2024-09-13T15:17:24.628 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:24.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:24.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:24.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:24.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:24.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:24.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:24.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[84576]: cluster 2024-09-13T15:17:23.160511+0000 osd.2 (osd.2) 59 : cluster [DBG] 46.17 scrub ok 2024-09-13T15:17:24.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[84576]: cluster 2024-09-13T15:17:23.160511+0000 osd.2 (osd.2) 59 : cluster [DBG] 46.17 scrub ok 2024-09-13T15:17:24.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[84576]: cluster 2024-09-13T15:17:23.340051+0000 osd.5 (osd.5) 16 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:24.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[84576]: cluster 2024-09-13T15:17:23.340051+0000 osd.5 (osd.5) 16 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:24.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[84576]: cluster 2024-09-13T15:17:23.407873+0000 mon.a (mon.0) 2708 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2024-09-13T15:17:24.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[84576]: cluster 2024-09-13T15:17:23.407873+0000 mon.a (mon.0) 2708 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2024-09-13T15:17:24.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[87147]: cluster 2024-09-13T15:17:23.160511+0000 osd.2 (osd.2) 59 : cluster [DBG] 46.17 scrub ok 2024-09-13T15:17:24.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[87147]: cluster 2024-09-13T15:17:23.160511+0000 osd.2 (osd.2) 59 : cluster [DBG] 46.17 scrub ok 2024-09-13T15:17:24.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[87147]: cluster 2024-09-13T15:17:23.340051+0000 osd.5 (osd.5) 16 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:24.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[87147]: cluster 2024-09-13T15:17:23.340051+0000 osd.5 (osd.5) 16 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:24.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[87147]: cluster 2024-09-13T15:17:23.407873+0000 mon.a (mon.0) 2708 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2024-09-13T15:17:24.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:24 smithi099 bash[87147]: cluster 2024-09-13T15:17:23.407873+0000 mon.a (mon.0) 2708 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2024-09-13T15:17:24.864 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:24.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:24.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:24.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:24.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:25.022 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:25.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:25.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:25.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:25.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:25.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:25.255 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:25.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:25.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:25.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:25.391 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:25.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:25.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:25.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:25.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:25.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:25.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:25.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:23.932699+0000 osd.7 (osd.7) 9 : cluster [DBG] 46.4 scrub starts 2024-09-13T15:17:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:23.932699+0000 osd.7 (osd.7) 9 : cluster [DBG] 46.4 scrub starts 2024-09-13T15:17:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:24.053047+0000 osd.2 (osd.2) 60 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:24.053047+0000 osd.2 (osd.2) 60 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:24.101091+0000 osd.0 (osd.0) 54 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:24.101091+0000 osd.0 (osd.0) 54 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:24.297650+0000 osd.5 (osd.5) 17 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:24.297650+0000 osd.5 (osd.5) 17 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:24.344726+0000 osd.7 (osd.7) 10 : cluster [DBG] 46.4 scrub ok 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:24.344726+0000 osd.7 (osd.7) 10 : cluster [DBG] 46.4 scrub ok 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:24.408388+0000 mon.a (mon.0) 2709 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:24.408388+0000 mon.a (mon.0) 2709 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.204565+0000 mgr.y (mgr.26252) 1543 : cluster [DBG] pgmap v2152: 99 pgs: 1 peering, 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8509/238530 objects degraded (3.567%); 2186/238530 objects misplaced (0.916%); 17 MiB/s, 269 objects/s recovering 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.204565+0000 mgr.y (mgr.26252) 1543 : cluster [DBG] pgmap v2152: 99 pgs: 1 peering, 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8509/238530 objects degraded (3.567%); 2186/238530 objects misplaced (0.916%); 17 MiB/s, 269 objects/s recovering 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.279178+0000 osd.1 (osd.1) 44 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.279178+0000 osd.1 (osd.1) 44 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:23.932699+0000 osd.7 (osd.7) 9 : cluster [DBG] 46.4 scrub starts 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:23.932699+0000 osd.7 (osd.7) 9 : cluster [DBG] 46.4 scrub starts 2024-09-13T15:17:25.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:24.053047+0000 osd.2 (osd.2) 60 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:24.053047+0000 osd.2 (osd.2) 60 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:24.101091+0000 osd.0 (osd.0) 54 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:24.101091+0000 osd.0 (osd.0) 54 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:24.297650+0000 osd.5 (osd.5) 17 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:24.297650+0000 osd.5 (osd.5) 17 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:24.344726+0000 osd.7 (osd.7) 10 : cluster [DBG] 46.4 scrub ok 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:24.344726+0000 osd.7 (osd.7) 10 : cluster [DBG] 46.4 scrub ok 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:24.408388+0000 mon.a (mon.0) 2709 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:24.408388+0000 mon.a (mon.0) 2709 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.204565+0000 mgr.y (mgr.26252) 1543 : cluster [DBG] pgmap v2152: 99 pgs: 1 peering, 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8509/238530 objects degraded (3.567%); 2186/238530 objects misplaced (0.916%); 17 MiB/s, 269 objects/s recovering 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.204565+0000 mgr.y (mgr.26252) 1543 : cluster [DBG] pgmap v2152: 99 pgs: 1 peering, 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8509/238530 objects degraded (3.567%); 2186/238530 objects misplaced (0.916%); 17 MiB/s, 269 objects/s recovering 2024-09-13T15:17:25.577 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.279178+0000 osd.1 (osd.1) 44 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:25.578 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:25 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.279178+0000 osd.1 (osd.1) 44 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:25.675 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:25.702 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:25.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:25.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:25.808 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:25.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:25.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:25.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:23.932699+0000 osd.7 (osd.7) 9 : cluster [DBG] 46.4 scrub starts 2024-09-13T15:17:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:23.932699+0000 osd.7 (osd.7) 9 : cluster [DBG] 46.4 scrub starts 2024-09-13T15:17:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:24.053047+0000 osd.2 (osd.2) 60 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:24.053047+0000 osd.2 (osd.2) 60 : cluster [DBG] 46.7 scrub starts 2024-09-13T15:17:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:24.101091+0000 osd.0 (osd.0) 54 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:24.101091+0000 osd.0 (osd.0) 54 : cluster [DBG] 46.19 scrub starts 2024-09-13T15:17:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:24.297650+0000 osd.5 (osd.5) 17 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:25.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:24.297650+0000 osd.5 (osd.5) 17 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:25.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:24.344726+0000 osd.7 (osd.7) 10 : cluster [DBG] 46.4 scrub ok 2024-09-13T15:17:25.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:24.344726+0000 osd.7 (osd.7) 10 : cluster [DBG] 46.4 scrub ok 2024-09-13T15:17:25.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:24.408388+0000 mon.a (mon.0) 2709 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2024-09-13T15:17:25.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:24.408388+0000 mon.a (mon.0) 2709 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2024-09-13T15:17:25.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.204565+0000 mgr.y (mgr.26252) 1543 : cluster [DBG] pgmap v2152: 99 pgs: 1 peering, 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8509/238530 objects degraded (3.567%); 2186/238530 objects misplaced (0.916%); 17 MiB/s, 269 objects/s recovering 2024-09-13T15:17:25.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.204565+0000 mgr.y (mgr.26252) 1543 : cluster [DBG] pgmap v2152: 99 pgs: 1 peering, 2 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8509/238530 objects degraded (3.567%); 2186/238530 objects misplaced (0.916%); 17 MiB/s, 269 objects/s recovering 2024-09-13T15:17:25.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.279178+0000 osd.1 (osd.1) 44 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:25.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:25 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.279178+0000 osd.1 (osd.1) 44 : cluster [DBG] 46.18 scrub starts 2024-09-13T15:17:25.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:25.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:25.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:26.068 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:26.104 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:26.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:26.210 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:26.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:26.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:26.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:26.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:26.388 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:26.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:26.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:26.493 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:26.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.009546+0000 osd.2 (osd.2) 61 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:26.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.009546+0000 osd.2 (osd.2) 61 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:26.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.131570+0000 osd.0 (osd.0) 55 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:26.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.131570+0000 osd.0 (osd.0) 55 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:26.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.229741+0000 osd.0 (osd.0) 56 : cluster [DBG] 46.19 scrub ok 2024-09-13T15:17:26.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.229741+0000 osd.0 (osd.0) 56 : cluster [DBG] 46.19 scrub ok 2024-09-13T15:17:26.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.284840+0000 osd.2 (osd.2) 62 : cluster [DBG] 46.7 scrub ok 2024-09-13T15:17:26.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.284840+0000 osd.2 (osd.2) 62 : cluster [DBG] 46.7 scrub ok 2024-09-13T15:17:26.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.829654+0000 osd.0 (osd.0) 57 : cluster [DBG] 46.2 scrub ok 2024-09-13T15:17:26.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:25.829654+0000 osd.0 (osd.0) 57 : cluster [DBG] 46.2 scrub ok 2024-09-13T15:17:26.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:26.079308+0000 osd.1 (osd.1) 45 : cluster [DBG] 46.18 scrub ok 2024-09-13T15:17:26.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:26.079308+0000 osd.1 (osd.1) 45 : cluster [DBG] 46.18 scrub ok 2024-09-13T15:17:26.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:26.329392+0000 osd.1 (osd.1) 46 : cluster [DBG] 46.11 scrub starts 2024-09-13T15:17:26.503 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:26 smithi177 bash[68594]: cluster 2024-09-13T15:17:26.329392+0000 osd.1 (osd.1) 46 : cluster [DBG] 46.11 scrub starts 2024-09-13T15:17:26.519 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:26.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:26.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:26.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:26.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:26.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:26.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:26.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.009546+0000 osd.2 (osd.2) 61 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.009546+0000 osd.2 (osd.2) 61 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.131570+0000 osd.0 (osd.0) 55 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.131570+0000 osd.0 (osd.0) 55 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.229741+0000 osd.0 (osd.0) 56 : cluster [DBG] 46.19 scrub ok 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.229741+0000 osd.0 (osd.0) 56 : cluster [DBG] 46.19 scrub ok 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.284840+0000 osd.2 (osd.2) 62 : cluster [DBG] 46.7 scrub ok 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.284840+0000 osd.2 (osd.2) 62 : cluster [DBG] 46.7 scrub ok 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.829654+0000 osd.0 (osd.0) 57 : cluster [DBG] 46.2 scrub ok 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:25.829654+0000 osd.0 (osd.0) 57 : cluster [DBG] 46.2 scrub ok 2024-09-13T15:17:26.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:26.079308+0000 osd.1 (osd.1) 45 : cluster [DBG] 46.18 scrub ok 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:26.079308+0000 osd.1 (osd.1) 45 : cluster [DBG] 46.18 scrub ok 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:26.329392+0000 osd.1 (osd.1) 46 : cluster [DBG] 46.11 scrub starts 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[84576]: cluster 2024-09-13T15:17:26.329392+0000 osd.1 (osd.1) 46 : cluster [DBG] 46.11 scrub starts 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.009546+0000 osd.2 (osd.2) 61 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.009546+0000 osd.2 (osd.2) 61 : cluster [DBG] 46.0 scrub starts 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.131570+0000 osd.0 (osd.0) 55 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.131570+0000 osd.0 (osd.0) 55 : cluster [DBG] 46.2 scrub starts 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.229741+0000 osd.0 (osd.0) 56 : cluster [DBG] 46.19 scrub ok 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.229741+0000 osd.0 (osd.0) 56 : cluster [DBG] 46.19 scrub ok 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.284840+0000 osd.2 (osd.2) 62 : cluster [DBG] 46.7 scrub ok 2024-09-13T15:17:26.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.284840+0000 osd.2 (osd.2) 62 : cluster [DBG] 46.7 scrub ok 2024-09-13T15:17:26.764 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.829654+0000 osd.0 (osd.0) 57 : cluster [DBG] 46.2 scrub ok 2024-09-13T15:17:26.764 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:25.829654+0000 osd.0 (osd.0) 57 : cluster [DBG] 46.2 scrub ok 2024-09-13T15:17:26.764 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:26.079308+0000 osd.1 (osd.1) 45 : cluster [DBG] 46.18 scrub ok 2024-09-13T15:17:26.764 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:26.079308+0000 osd.1 (osd.1) 45 : cluster [DBG] 46.18 scrub ok 2024-09-13T15:17:26.764 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:26.329392+0000 osd.1 (osd.1) 46 : cluster [DBG] 46.11 scrub starts 2024-09-13T15:17:26.764 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:26 smithi099 bash[87147]: cluster 2024-09-13T15:17:26.329392+0000 osd.1 (osd.1) 46 : cluster [DBG] 46.11 scrub starts 2024-09-13T15:17:26.776 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:26.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:26.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:26.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:26.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:26.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:26.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:27.009 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:27.052 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:27.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:27.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:27.157 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:27.190 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:27.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:27.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:27.295 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:27.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:27.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:27.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:27.380 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:17:27] "GET /metrics HTTP/1.1" 200 36805 "" "Prometheus/2.51.0" 2024-09-13T15:17:27.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:27.452 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:27.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:27.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:27.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:27.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:27 smithi177 bash[68594]: cluster 2024-09-13T15:17:26.246031+0000 osd.2 (osd.2) 63 : cluster [DBG] 46.0 scrub ok 2024-09-13T15:17:27.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:27 smithi177 bash[68594]: cluster 2024-09-13T15:17:26.246031+0000 osd.2 (osd.2) 63 : cluster [DBG] 46.0 scrub ok 2024-09-13T15:17:27.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:27 smithi177 bash[68594]: cluster 2024-09-13T15:17:26.302533+0000 osd.5 (osd.5) 18 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:27.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:27 smithi177 bash[68594]: cluster 2024-09-13T15:17:26.302533+0000 osd.5 (osd.5) 18 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:27.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:27 smithi177 bash[68594]: cluster 2024-09-13T15:17:26.660289+0000 mon.a (mon.0) 2710 : cluster [WRN] Health check update: Degraded data redundancy: 8509/238530 objects degraded (3.567%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:27.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:27 smithi177 bash[68594]: cluster 2024-09-13T15:17:26.660289+0000 mon.a (mon.0) 2710 : cluster [WRN] Health check update: Degraded data redundancy: 8509/238530 objects degraded (3.567%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:27.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:27 smithi177 bash[68594]: cluster 2024-09-13T15:17:27.077674+0000 osd.1 (osd.1) 47 : cluster [DBG] 46.11 scrub ok 2024-09-13T15:17:27.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:27 smithi177 bash[68594]: cluster 2024-09-13T15:17:27.077674+0000 osd.1 (osd.1) 47 : cluster [DBG] 46.11 scrub ok 2024-09-13T15:17:27.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:27 smithi177 bash[68594]: cluster 2024-09-13T15:17:27.205209+0000 mgr.y (mgr.26252) 1544 : cluster [DBG] pgmap v2153: 99 pgs: 1 activating+degraded, 1 peering, 3 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 91 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8510/237465 objects degraded (3.584%); 1882/237465 objects misplaced (0.793%); 13 MiB/s, 214 objects/s recovering 2024-09-13T15:17:27.569 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:27 smithi177 bash[68594]: cluster 2024-09-13T15:17:27.205209+0000 mgr.y (mgr.26252) 1544 : cluster [DBG] pgmap v2153: 99 pgs: 1 activating+degraded, 1 peering, 3 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 91 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8510/237465 objects degraded (3.584%); 1882/237465 objects misplaced (0.793%); 13 MiB/s, 214 objects/s recovering 2024-09-13T15:17:27.583 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:27.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:27.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:27.686 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:27.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[84576]: cluster 2024-09-13T15:17:26.246031+0000 osd.2 (osd.2) 63 : cluster [DBG] 46.0 scrub ok 2024-09-13T15:17:27.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[84576]: cluster 2024-09-13T15:17:26.246031+0000 osd.2 (osd.2) 63 : cluster [DBG] 46.0 scrub ok 2024-09-13T15:17:27.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[84576]: cluster 2024-09-13T15:17:26.302533+0000 osd.5 (osd.5) 18 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:27.696 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[84576]: cluster 2024-09-13T15:17:26.302533+0000 osd.5 (osd.5) 18 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:27.697 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[84576]: cluster 2024-09-13T15:17:26.660289+0000 mon.a (mon.0) 2710 : cluster [WRN] Health check update: Degraded data redundancy: 8509/238530 objects degraded (3.567%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:27.697 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[84576]: cluster 2024-09-13T15:17:26.660289+0000 mon.a (mon.0) 2710 : cluster [WRN] Health check update: Degraded data redundancy: 8509/238530 objects degraded (3.567%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:27.697 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[84576]: cluster 2024-09-13T15:17:27.077674+0000 osd.1 (osd.1) 47 : cluster [DBG] 46.11 scrub ok 2024-09-13T15:17:27.697 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[84576]: cluster 2024-09-13T15:17:27.077674+0000 osd.1 (osd.1) 47 : cluster [DBG] 46.11 scrub ok 2024-09-13T15:17:27.697 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[84576]: cluster 2024-09-13T15:17:27.205209+0000 mgr.y (mgr.26252) 1544 : cluster [DBG] pgmap v2153: 99 pgs: 1 activating+degraded, 1 peering, 3 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 91 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8510/237465 objects degraded (3.584%); 1882/237465 objects misplaced (0.793%); 13 MiB/s, 214 objects/s recovering 2024-09-13T15:17:27.697 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[84576]: cluster 2024-09-13T15:17:27.205209+0000 mgr.y (mgr.26252) 1544 : cluster [DBG] pgmap v2153: 99 pgs: 1 activating+degraded, 1 peering, 3 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 91 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8510/237465 objects degraded (3.584%); 1882/237465 objects misplaced (0.793%); 13 MiB/s, 214 objects/s recovering 2024-09-13T15:17:27.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[87147]: cluster 2024-09-13T15:17:26.246031+0000 osd.2 (osd.2) 63 : cluster [DBG] 46.0 scrub ok 2024-09-13T15:17:27.697 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[87147]: cluster 2024-09-13T15:17:26.246031+0000 osd.2 (osd.2) 63 : cluster [DBG] 46.0 scrub ok 2024-09-13T15:17:27.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[87147]: cluster 2024-09-13T15:17:26.302533+0000 osd.5 (osd.5) 18 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:27.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[87147]: cluster 2024-09-13T15:17:26.302533+0000 osd.5 (osd.5) 18 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:27.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[87147]: cluster 2024-09-13T15:17:26.660289+0000 mon.a (mon.0) 2710 : cluster [WRN] Health check update: Degraded data redundancy: 8509/238530 objects degraded (3.567%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:27.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[87147]: cluster 2024-09-13T15:17:26.660289+0000 mon.a (mon.0) 2710 : cluster [WRN] Health check update: Degraded data redundancy: 8509/238530 objects degraded (3.567%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:27.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[87147]: cluster 2024-09-13T15:17:27.077674+0000 osd.1 (osd.1) 47 : cluster [DBG] 46.11 scrub ok 2024-09-13T15:17:27.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[87147]: cluster 2024-09-13T15:17:27.077674+0000 osd.1 (osd.1) 47 : cluster [DBG] 46.11 scrub ok 2024-09-13T15:17:27.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[87147]: cluster 2024-09-13T15:17:27.205209+0000 mgr.y (mgr.26252) 1544 : cluster [DBG] pgmap v2153: 99 pgs: 1 activating+degraded, 1 peering, 3 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 91 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8510/237465 objects degraded (3.584%); 1882/237465 objects misplaced (0.793%); 13 MiB/s, 214 objects/s recovering 2024-09-13T15:17:27.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:27 smithi099 bash[87147]: cluster 2024-09-13T15:17:27.205209+0000 mgr.y (mgr.26252) 1544 : cluster [DBG] pgmap v2153: 99 pgs: 1 activating+degraded, 1 peering, 3 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 91 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8510/237465 objects degraded (3.584%); 1882/237465 objects misplaced (0.793%); 13 MiB/s, 214 objects/s recovering 2024-09-13T15:17:27.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:27.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:27.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:27.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:27.840 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:27.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:27.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:27.944 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:27.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:27.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:27.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:28.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:28.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:28.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:28.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:28.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:28.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:28.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:28.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:28.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:28.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:28.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:28.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:28.471 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:28.497 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:28.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:28.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:28.503 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":2145,"stamp":"2024-09-13T15:17:17.201029+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5653141477,"num_objects":86278,"num_object_clones":0,"num_object_copies":258834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8943,"num_objects_misplaced":4911,"num_objects_unfound":0,"num_objects_dirty":86278,"num_whiteouts":0,"num_read":333,"num_read_kb":788,"num_write":118782,"num_write_kb":6555900,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6269,"num_bytes_recovered":408438434,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":174133,"ondisk_log_size":174133,"up":297,"acting":292,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":304,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23263168,"kb_used_data":17168640,"kb_used_omap":397,"kb_used_meta":6093682,"kb_avail":726698048,"statfs":{"total":767960285184,"available":744138801152,"internally_reserved":0,"allocated":17580687360,"data_stored":17543611753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":406940,"internal_metadata":6239930980},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,2,4,7,2,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-496173056,"num_objects":-7571,"num_object_clones":0,"num_object_copies":-22713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-482,"num_objects_misplaced":-2625,"num_objects_unfound":0,"num_objects_dirty":-7571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7571,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2477,"num_bytes_recovered":162332672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005721"},"pg_stats":[{"pgid":"46.19","version":"1122'6444","reported_seq":6548,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:14.047569+0000","last_change":"2024-09-13T15:17:14.054041+0000","last_active":"2024-09-13T15:17:14.047569+0000","last_peered":"2024-09-13T15:17:14.047569+0000","last_clean":"2024-09-13T15:17:14.047569+0000","last_became_active":"2024-09-13T15:16:32.765145+0000","last_became_peered":"2024-09-13T15:16:32.765145+0000","last_unstale":"2024-09-13T15:17:14.047569+0000","last_undegraded":"2024-09-13T15:17:14.047569+0000","last_fullsized":"2024-09-13T15:17:14.047569+0000","mapping_epoch":1108,"log_start":"0'0","ondisk_log_start":"0'0","created":1108,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:17:19.047653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175865856,"num_objects":2684,"num_object_clones":0,"num_object_copies":8052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3672,"num_write_kb":203360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.18","version":"1122'6534","reported_seq":6632,"reported_epoch":1122,"state":"active+clean+scrubbing","last_fresh":"2024-09-13T15:17:14.386148+0000","last_change":"2024-09-13T15:17:14.386066+0000","last_active":"2024-09-13T15:17:14.386148+0000","last_peered":"2024-09-13T15:17:14.386148+0000","last_clean":"2024-09-13T15:17:14.386148+0000","last_became_active":"2024-09-13T15:16:32.761319+0000","last_became_peered":"2024-09-13T15:16:32.761319+0000","last_unstale":"2024-09-13T15:17:14.386148+0000","last_undegraded":"2024-09-13T15:17:14.386148+0000","last_fullsized":"2024-09-13T15:17:14.386148+0000","mapping_epoch":1108,"log_start":"0'0","ondisk_log_start":"0'0","created":1108,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6534,"log_dups_size":0,"ondisk_log_size":6534,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"scrubbing for 0s","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177209344,"num_objects":2704,"num_object_clones":0,"num_object_copies":8112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3714,"num_write_kb":205376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.17","version":"1122'6432","reported_seq":6533,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:12.869868+0000","last_change":"2024-09-13T15:17:12.084256+0000","last_active":"2024-09-13T15:17:12.869868+0000","last_peered":"2024-09-13T15:17:12.869868+0000","last_clean":"2024-09-13T15:17:12.869868+0000","last_became_active":"2024-09-13T15:16:32.833550+0000","last_became_peered":"2024-09-13T15:16:32.833550+0000","last_unstale":"2024-09-13T15:17:12.869868+0000","last_undegraded":"2024-09-13T15:17:12.869868+0000","last_fullsized":"2024-09-13T15:17:12.869868+0000","mapping_epoch":1108,"log_start":"0'0","ondisk_log_start":"0'0","created":1108,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6432,"log_dups_size":0,"ondisk_log_size":6432,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:17:16.905224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176586752,"num_objects":2695,"num_object_clones":0,"num_object_copies":8085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3651,"num_write_kb":203040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"46.16","version":"1122'6462","reported_seq":6502,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:14.788022+0000","last_change":"2024-09-13T15:16:32.761385+0000","last_active":"2024-09-13T15:17:14.788022+0000","last_peered":"2024-09-13T15:17:14.788022+0000","last_clean":"2024-09-13T15:17:14.788022+0000","last_became_active":"2024-09-13T15:16:32.757942+0000","last_became_peered":"2024-09-13T15:16:32.757942+0000","last_unstale":"2024-09-13T15:17:14.788022+0000","last_undegraded":"2024-09-13T15:17:14.788022+0000","last_fullsized":"2024-09-13T15:17:14.788022+0000","mapping_epoch":1108,"log_start":"0'0","ondisk_log_start":"0'0","created":1108,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6462,"log_dups_size":0,"ondisk_log_size":6462,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175308800,"num_objects":2675,"num_object_clones":0,"num_object_copies":8025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3691,"num_write_kb":203712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"46.15","version":"1122'6456","reported_seq":6559,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:15.059353+0000","last_change":"2024-09-13T15:17:14.945176+0000","last_active":"2024-09-13T15:17:15.059353+0000","last_peered":"2024-09-13T15:17:15.059353+0000","last_clean":"2024-09-13T15:17:15.059353+0000","last_became_active":"2024-09-13T15:16:32.757413+0000","last_became_peered":"2024-09-13T15:16:32.757413+0000","last_unstale":"2024-09-13T15:17:15.059353+0000","last_undegraded":"2024-09-13T15:17:15.059353+0000","last_fullsized":"2024-09-13T15:17:15.059353+0000","mapping_epoch":1108,"log_start":"0'0","ondisk_log_start":"0'0","created":1108,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6456,"log_dups_size":0,"ondisk_log_size":6456,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:17:19.934384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175407104,"num_objects":2677,"num_object_clones":0,"num_object_copies":8031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3687,"num_write_kb":203616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"46.14","version":"1122'6601","reported_seq":7779,"reported_epoch":1122,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:17:14.039707+0000","last_change":"2024-09-13T15:16:59.839402+0000","last_active":"2024-09-13T15:17:14.039707+0000","last_peered":"2024-09-13T15:17:14.039707+0000","last_clean":"2024-09-13T15:16:56.306901+0000","last_became_active":"2024-09-13T15:16:59.833374+0000","last_became_peered":"2024-09-13T15:16:59.833374+0000","last_unstale":"2024-09-13T15:17:14.039707+0000","last_undegraded":"2024-09-13T15:16:59.775737+0000","last_fullsized":"2024-09-13T15:16:59.669688+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":1108,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179011584,"num_objects":2732,"num_object_clones":0,"num_object_copies":8196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5740,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3766,"num_write_kb":207904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":378,"num_bytes_recovered":24772608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[0,1],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,1","objects":1},{"shards":"0,2,7","objects":2731}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.13","version":"1122'6498","reported_seq":6582,"reported_epoch":1122,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:17:14.041328+0000","last_change":"2024-09-13T15:16:57.775596+0000","last_active":"2024-09-13T15:17:14.041328+0000","last_peered":"2024-09-13T15:17:14.041328+0000","last_clean":"2024-09-13T15:16:55.317208+0000","last_became_active":"2024-09-13T15:16:57.768325+0000","last_became_peered":"2024-09-13T15:16:57.768325+0000","last_unstale":"2024-09-13T15:17:14.041328+0000","last_undegraded":"2024-09-13T15:16:57.683869+0000","last_fullsized":"2024-09-13T15:16:57.663167+0000","mapping_epoch":1119,"log_start":"0'0","ondisk_log_start":"0'0","created":1108,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6498,"log_dups_size":0,"ondisk_log_size":6498,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176816128,"num_objects":2698,"num_object_clones":0,"num_object_copies":8094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3203,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3722,"num_write_kb":205440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1,5","objects":2691},{"shards":"0,5","objects":-365}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.12","version":"1122'6490","reported_seq":10524,"reported_epoch":1122,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-13T15:17:13.967292+0000","last_change":"2024-09-13T15:16:50.174907+0000","last_active":"2024-09-13T15:17:13.967292+0000","last_peered":"2024-09-13T15:17:13.967292+0000","last_clean":"2024-09-13T15:16:46.812863+0000","last_became_active":"2024-09-13T15:16:50.108018+0000","last_became_peered":"2024-09-13T15:16:50.108018+0000","last_unstale":"2024-09-13T15:17:13.967292+0000","last_undegraded":"2024-09-13T15:17:13.967292+0000","last_fullsized":"2024-09-13T15:16:49.669958+0000","mapping_epoch":1116,"log_start":"0'0","ondisk_log_start":"0'0","created":1108,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6490,"log_dups_size":0,"ondisk_log_size":6490,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173506560,"num_objects":2648,"num_object_clones":0,"num_object_copies":7944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1232,"num_objects_unfound":0,"num_objects_dirty":2648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3710,"num_write_kb":203424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1979,"num_bytes_recovered":129695744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,0],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":1232},{"shards":"0,6","objects":1416}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.11","version":"1122'6451","reported_seq":11210,"reported_epoch":1122,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-13T15:17:14.736177+0000","last_change":"2024-09-13T15:16:47.847358+0000","last_active":"2024-09-13T15:17:14.736177+0000","last_peered":"2024-09-13T15:17:14.736177+0000","last_clean":"2024-09-13T15:16:45.769126+0000","last_became_active":"2024-09-13T15:16:47.783886+0000","last_became_peered":"2024-09-13T15:16:47.783886+0000","last_unstale":"2024-09-13T15:17:14.736177+0000","last_undegraded":"2024-09-13T15:17:14.736177+0000","last_fullsized":"2024-09-13T15:16:47.665570+0000","mapping_epoch":1114,"log_start":"0'0","ondisk_log_start":"0'0","created":1108,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6451,"log_dups_size":0,"ondisk_log_size":6451,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175964160,"num_objects":2685,"num_object_clones":0,"num_object_copies":8055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":763,"num_objects_unfound":0,"num_objects_dirty":2685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3669,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2353,"num_bytes_recovered":154206208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5","objects":1922},{"shards":"1,5,6","objects":763}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.10","version":"1122'6536","reported_seq":11091,"reported_epoch":1122,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-13T15:17:14.046885+0000","last_change":"2024-09-13T15:16:40.635907+0000","last_active":"2024-09-13T15:17:14.046885+0000","last_peered":"2024-09-13T15:17:14.046885+0000","last_clean":"2024-09-13T15:16:37.615426+0000","last_became_active":"2024-09-13T15:16:40.546265+0000","last_became_peered":"2024-09-13T15:16:40.546265+0000","last_unstale":"2024-09-13T15:17:14.046885+0000","last_undegraded":"2024-09-13T15:17:14.046885+0000","last_fullsized":"2024-09-13T15:16:39.777653+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":1108,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6536,"log_dups_size":0,"ondisk_log_size":6536,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179372032,"num_objects":2737,"num_object_clones":0,"num_object_copies":8211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2916,"num_objects_unfound":0,"num_objects_dirty":2737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3725,"num_write_kb":206784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1494,"num_bytes_recovered":97910784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[0,1],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1","objects":1279},{"shards":"0,1,2","objects":1458}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.f","version":"1122'7400","reported_seq":7431,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:13.889642+0000","last_change":"2024-09-13T15:14:53.230762+0000","last_active":"2024-09-13T15:17:13.889642+0000","last_peered":"2024-09-13T15:17:13.889642+0000","last_clean":"2024-09-13T15:17:13.889642+0000","last_became_active":"2024-09-13T15:14:53.230270+0000","last_became_peered":"2024-09-13T15:14:53.230270+0000","last_unstale":"2024-09-13T15:17:13.889642+0000","last_undegraded":"2024-09-13T15:17:13.889642+0000","last_fullsized":"2024-09-13T15:17:13.889642+0000","mapping_epoch":1099,"log_start":"1103'200","ondisk_log_start":"1103'200","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":7200,"log_dups_size":0,"ondisk_log_size":7200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:24:33.738184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349569024,"num_objects":5334,"num_object_clones":0,"num_object_copies":16002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7400,"num_write_kb":407488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.e","version":"1122'7345","reported_seq":7410,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:12.929621+0000","last_change":"2024-09-13T15:14:53.225301+0000","last_active":"2024-09-13T15:17:12.929621+0000","last_peered":"2024-09-13T15:17:12.929621+0000","last_clean":"2024-09-13T15:17:12.929621+0000","last_became_active":"2024-09-13T15:14:53.224979+0000","last_became_peered":"2024-09-13T15:14:53.224979+0000","last_unstale":"2024-09-13T15:17:12.929621+0000","last_undegraded":"2024-09-13T15:17:12.929621+0000","last_fullsized":"2024-09-13T15:17:12.929621+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":7345,"log_dups_size":0,"ondisk_log_size":7345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:32:42.548466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355532800,"num_objects":5425,"num_object_clones":0,"num_object_copies":16275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7345,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"46.d","version":"1122'7397","reported_seq":7463,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:13.996255+0000","last_change":"2024-09-13T15:14:53.236183+0000","last_active":"2024-09-13T15:17:13.996255+0000","last_peered":"2024-09-13T15:17:13.996255+0000","last_clean":"2024-09-13T15:17:13.996255+0000","last_became_active":"2024-09-13T15:14:53.235681+0000","last_became_peered":"2024-09-13T15:14:53.235681+0000","last_unstale":"2024-09-13T15:17:13.996255+0000","last_undegraded":"2024-09-13T15:17:13.996255+0000","last_fullsized":"2024-09-13T15:17:13.996255+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":7397,"log_dups_size":0,"ondisk_log_size":7397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:36.109059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355205144,"num_objects":5421,"num_object_clones":0,"num_object_copies":16263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5421,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7397,"num_write_kb":410113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.c","version":"1122'7419","reported_seq":7450,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:13.950053+0000","last_change":"2024-09-13T15:14:53.234121+0000","last_active":"2024-09-13T15:17:13.950053+0000","last_peered":"2024-09-13T15:17:13.950053+0000","last_clean":"2024-09-13T15:17:13.950053+0000","last_became_active":"2024-09-13T15:14:53.233879+0000","last_became_peered":"2024-09-13T15:14:53.233879+0000","last_unstale":"2024-09-13T15:17:13.950053+0000","last_undegraded":"2024-09-13T15:17:13.950053+0000","last_fullsized":"2024-09-13T15:17:13.950053+0000","mapping_epoch":1099,"log_start":"1103'200","ondisk_log_start":"1103'200","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":7219,"log_dups_size":0,"ondisk_log_size":7219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:16:57.111464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351600640,"num_objects":5365,"num_object_clones":0,"num_object_copies":16095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7419,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.b","version":"1122'7452","reported_seq":7516,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:14.043539+0000","last_change":"2024-09-13T15:14:53.235572+0000","last_active":"2024-09-13T15:17:14.043539+0000","last_peered":"2024-09-13T15:17:14.043539+0000","last_clean":"2024-09-13T15:17:14.043539+0000","last_became_active":"2024-09-13T15:14:53.235203+0000","last_became_peered":"2024-09-13T15:14:53.235203+0000","last_unstale":"2024-09-13T15:17:14.043539+0000","last_undegraded":"2024-09-13T15:17:14.043539+0000","last_fullsized":"2024-09-13T15:17:14.043539+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":7452,"log_dups_size":0,"ondisk_log_size":7452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:17.187189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356646912,"num_objects":5442,"num_object_clones":0,"num_object_copies":16326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7452,"num_write_kb":412608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.a","version":"1122'7474","reported_seq":7505,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:13.899601+0000","last_change":"2024-09-13T15:14:53.226603+0000","last_active":"2024-09-13T15:17:13.899601+0000","last_peered":"2024-09-13T15:17:13.899601+0000","last_clean":"2024-09-13T15:17:13.899601+0000","last_became_active":"2024-09-13T15:14:53.226024+0000","last_became_peered":"2024-09-13T15:14:53.226024+0000","last_unstale":"2024-09-13T15:17:13.899601+0000","last_undegraded":"2024-09-13T15:17:13.899601+0000","last_fullsized":"2024-09-13T15:17:13.899601+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":7474,"log_dups_size":0,"ondisk_log_size":7474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:24:35.671044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356515840,"num_objects":5440,"num_object_clones":0,"num_object_copies":16320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7474,"num_write_kb":413248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.9","version":"1122'6438","reported_seq":6537,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:14.051242+0000","last_change":"2024-09-13T15:17:11.277084+0000","last_active":"2024-09-13T15:17:14.051242+0000","last_peered":"2024-09-13T15:17:14.051242+0000","last_clean":"2024-09-13T15:17:14.051242+0000","last_became_active":"2024-09-13T15:16:32.737301+0000","last_became_peered":"2024-09-13T15:16:32.737301+0000","last_unstale":"2024-09-13T15:17:14.051242+0000","last_undegraded":"2024-09-13T15:17:14.051242+0000","last_fullsized":"2024-09-13T15:17:14.051242+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6438,"log_dups_size":0,"ondisk_log_size":6438,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:17:16.022961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176521216,"num_objects":2693,"num_object_clones":0,"num_object_copies":8079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3665,"num_write_kb":203488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.8","version":"1122'6575","reported_seq":6677,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:14.381780+0000","last_change":"2024-09-13T15:17:09.728640+0000","last_active":"2024-09-13T15:17:14.381780+0000","last_peered":"2024-09-13T15:17:14.381780+0000","last_clean":"2024-09-13T15:17:14.381780+0000","last_became_active":"2024-09-13T15:16:32.752946+0000","last_became_peered":"2024-09-13T15:16:32.752946+0000","last_unstale":"2024-09-13T15:17:14.381780+0000","last_undegraded":"2024-09-13T15:17:14.381780+0000","last_fullsized":"2024-09-13T15:17:14.381780+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6575,"log_dups_size":0,"ondisk_log_size":6575,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:17:14.574122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173867008,"num_objects":2653,"num_object_clones":0,"num_object_copies":7959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3755,"num_write_kb":205056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.7","version":"1122'6456","reported_seq":6549,"reported_epoch":1122,"state":"active+clean+scrubbing","last_fresh":"2024-09-13T15:17:12.937787+0000","last_change":"2024-09-13T15:17:12.937556+0000","last_active":"2024-09-13T15:17:12.937787+0000","last_peered":"2024-09-13T15:17:12.937787+0000","last_clean":"2024-09-13T15:17:12.937787+0000","last_became_active":"2024-09-13T15:16:32.818365+0000","last_became_peered":"2024-09-13T15:16:32.818365+0000","last_unstale":"2024-09-13T15:17:12.937787+0000","last_undegraded":"2024-09-13T15:17:12.937787+0000","last_fullsized":"2024-09-13T15:17:12.937787+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6456,"log_dups_size":0,"ondisk_log_size":6456,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"scrubbing for 0s","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178290688,"num_objects":2720,"num_object_clones":0,"num_object_copies":8160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3674,"num_write_kb":204640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"46.6","version":"1122'6447","reported_seq":6488,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:14.907863+0000","last_change":"2024-09-13T15:16:32.750016+0000","last_active":"2024-09-13T15:17:14.907863+0000","last_peered":"2024-09-13T15:17:14.907863+0000","last_clean":"2024-09-13T15:17:14.907863+0000","last_became_active":"2024-09-13T15:16:32.731856+0000","last_became_peered":"2024-09-13T15:16:32.731856+0000","last_unstale":"2024-09-13T15:17:14.907863+0000","last_undegraded":"2024-09-13T15:17:14.907863+0000","last_fullsized":"2024-09-13T15:17:14.907863+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174718976,"num_objects":2666,"num_object_clones":0,"num_object_copies":7998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3676,"num_write_kb":202944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"46.5","version":"1122'6448","reported_seq":6546,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:15.046759+0000","last_change":"2024-09-13T15:17:11.291681+0000","last_active":"2024-09-13T15:17:15.046759+0000","last_peered":"2024-09-13T15:17:15.046759+0000","last_clean":"2024-09-13T15:17:15.046759+0000","last_became_active":"2024-09-13T15:16:32.752140+0000","last_became_peered":"2024-09-13T15:16:32.752140+0000","last_unstale":"2024-09-13T15:17:15.046759+0000","last_undegraded":"2024-09-13T15:17:15.046759+0000","last_fullsized":"2024-09-13T15:17:15.046759+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6448,"log_dups_size":0,"ondisk_log_size":6448,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:17:15.903009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171999232,"num_objects":2624,"num_object_clones":0,"num_object_copies":7872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3678,"num_write_kb":201696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"46.4","version":"1122'6580","reported_seq":6621,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:13.919816+0000","last_change":"2024-09-13T15:16:32.732035+0000","last_active":"2024-09-13T15:17:13.919816+0000","last_peered":"2024-09-13T15:17:13.919816+0000","last_clean":"2024-09-13T15:17:13.919816+0000","last_became_active":"2024-09-13T15:16:32.726095+0000","last_became_peered":"2024-09-13T15:16:32.726095+0000","last_unstale":"2024-09-13T15:17:13.919816+0000","last_undegraded":"2024-09-13T15:17:13.919816+0000","last_fullsized":"2024-09-13T15:17:13.919816+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6580,"log_dups_size":0,"ondisk_log_size":6580,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177242112,"num_objects":2704,"num_object_clones":0,"num_object_copies":8112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3744,"num_write_kb":206368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.3","version":"1122'6488","reported_seq":6580,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:14.339113+0000","last_change":"2024-09-13T15:17:10.602254+0000","last_active":"2024-09-13T15:17:14.339113+0000","last_peered":"2024-09-13T15:17:14.339113+0000","last_clean":"2024-09-13T15:17:14.339113+0000","last_became_active":"2024-09-13T15:16:32.724562+0000","last_became_peered":"2024-09-13T15:16:32.724562+0000","last_unstale":"2024-09-13T15:17:14.339113+0000","last_undegraded":"2024-09-13T15:17:14.339113+0000","last_fullsized":"2024-09-13T15:17:14.339113+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6488,"log_dups_size":0,"ondisk_log_size":6488,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:17:15.530431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177471488,"num_objects":2708,"num_object_clones":0,"num_object_copies":8124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3712,"num_write_kb":205440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.2","version":"1122'6491","reported_seq":6581,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:14.008112+0000","last_change":"2024-09-13T15:17:09.117670+0000","last_active":"2024-09-13T15:17:14.008112+0000","last_peered":"2024-09-13T15:17:14.008112+0000","last_clean":"2024-09-13T15:17:14.008112+0000","last_became_active":"2024-09-13T15:16:32.730461+0000","last_became_peered":"2024-09-13T15:16:32.730461+0000","last_unstale":"2024-09-13T15:17:14.008112+0000","last_undegraded":"2024-09-13T15:17:14.008112+0000","last_fullsized":"2024-09-13T15:17:14.008112+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6491,"log_dups_size":0,"ondisk_log_size":6491,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:17:14.052969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176455680,"num_objects":2692,"num_object_clones":0,"num_object_copies":8076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3710,"num_write_kb":204896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.1","version":"1122'6499","reported_seq":6544,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:13.907138+0000","last_change":"2024-09-13T15:16:32.763045+0000","last_active":"2024-09-13T15:17:13.907138+0000","last_peered":"2024-09-13T15:17:13.907138+0000","last_clean":"2024-09-13T15:17:13.907138+0000","last_became_active":"2024-09-13T15:16:32.748108+0000","last_became_peered":"2024-09-13T15:16:32.748108+0000","last_unstale":"2024-09-13T15:17:13.907138+0000","last_undegraded":"2024-09-13T15:17:13.907138+0000","last_fullsized":"2024-09-13T15:17:13.907138+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6499,"log_dups_size":0,"ondisk_log_size":6499,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178323456,"num_objects":2721,"num_object_clones":0,"num_object_copies":8163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3717,"num_write_kb":206016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.0","version":"1122'6556","reported_seq":6657,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:17:12.930144+0000","last_change":"2024-09-13T15:17:08.881285+0000","last_active":"2024-09-13T15:17:12.930144+0000","last_peered":"2024-09-13T15:17:12.930144+0000","last_clean":"2024-09-13T15:17:12.930144+0000","last_became_active":"2024-09-13T15:16:32.814134+0000","last_became_peered":"2024-09-13T15:16:32.814134+0000","last_unstale":"2024-09-13T15:17:12.930144+0000","last_undegraded":"2024-09-13T15:17:12.930144+0000","last_fullsized":"2024-09-13T15:17:12.930144+0000","mapping_epoch":1099,"log_start":"0'0","ondisk_log_start":"0'0","created":1099,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:14:52.199036+0000","last_clean_scrub_stamp":"2024-09-13T15:14:52.199036+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:17:13.857981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177537024,"num_objects":2709,"num_object_clones":0,"num_object_copies":8127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3745,"num_write_kb":206528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"65'5","reported_seq":134,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.666265+0000","last_change":"2024-09-13T15:08:52.195462+0000","last_active":"2024-09-13T15:16:58.666265+0000","last_peered":"2024-09-13T15:16:58.666265+0000","last_clean":"2024-09-13T15:16:58.666265+0000","last_became_active":"2024-09-13T15:08:52.174050+0000","last_became_peered":"2024-09-13T15:08:52.174050+0000","last_unstale":"2024-09-13T15:16:58.666265+0000","last_undegraded":"2024-09-13T15:16:58.666265+0000","last_fullsized":"2024-09-13T15:16:58.666265+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:58.561367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":378,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.690085+0000","last_change":"2024-09-13T15:08:52.180022+0000","last_active":"2024-09-13T15:16:59.690085+0000","last_peered":"2024-09-13T15:16:59.690085+0000","last_clean":"2024-09-13T15:16:59.690085+0000","last_became_active":"2024-09-13T15:08:52.179546+0000","last_became_peered":"2024-09-13T15:08:52.179546+0000","last_unstale":"2024-09-13T15:16:59.690085+0000","last_undegraded":"2024-09-13T15:16:59.690085+0000","last_fullsized":"2024-09-13T15:16:59.690085+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:15:47.462895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1372,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674929+0000","last_change":"2024-09-13T15:08:52.175606+0000","last_active":"2024-09-13T15:16:59.674929+0000","last_peered":"2024-09-13T15:16:59.674929+0000","last_clean":"2024-09-13T15:16:59.674929+0000","last_became_active":"2024-09-13T15:08:52.175210+0000","last_became_peered":"2024-09-13T15:08:52.175210+0000","last_unstale":"2024-09-13T15:16:59.674929+0000","last_undegraded":"2024-09-13T15:16:59.674929+0000","last_fullsized":"2024-09-13T15:16:59.674929+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:16.692008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":153,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.665749+0000","last_change":"2024-09-13T14:48:53.344778+0000","last_active":"2024-09-13T15:16:58.665749+0000","last_peered":"2024-09-13T15:16:58.665749+0000","last_clean":"2024-09-13T15:16:58.665749+0000","last_became_active":"2024-09-13T14:48:53.344525+0000","last_became_peered":"2024-09-13T14:48:53.344525+0000","last_unstale":"2024-09-13T15:16:58.665749+0000","last_undegraded":"2024-09-13T15:16:58.665749+0000","last_fullsized":"2024-09-13T15:16:58.665749+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:26.417252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1135,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.665975+0000","last_change":"2024-09-13T15:08:52.205951+0000","last_active":"2024-09-13T15:16:58.665975+0000","last_peered":"2024-09-13T15:16:58.665975+0000","last_clean":"2024-09-13T15:16:58.665975+0000","last_became_active":"2024-09-13T15:08:52.183627+0000","last_became_peered":"2024-09-13T15:08:52.183627+0000","last_unstale":"2024-09-13T15:16:58.665975+0000","last_undegraded":"2024-09-13T15:16:58.665975+0000","last_fullsized":"2024-09-13T15:16:58.665975+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:14:14.282531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":360,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689858+0000","last_change":"2024-09-13T15:05:34.086551+0000","last_active":"2024-09-13T15:16:59.689858+0000","last_peered":"2024-09-13T15:16:59.689858+0000","last_clean":"2024-09-13T15:16:59.689858+0000","last_became_active":"2024-09-13T15:05:34.086077+0000","last_became_peered":"2024-09-13T15:05:34.086077+0000","last_unstale":"2024-09-13T15:16:59.689858+0000","last_undegraded":"2024-09-13T15:16:59.689858+0000","last_fullsized":"2024-09-13T15:16:59.689858+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:33:47.917777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1152,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.665734+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T15:16:58.665734+0000","last_peered":"2024-09-13T15:16:58.665734+0000","last_clean":"2024-09-13T15:16:58.665734+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T15:16:58.665734+0000","last_undegraded":"2024-09-13T15:16:58.665734+0000","last_fullsized":"2024-09-13T15:16:58.665734+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":374,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689670+0000","last_change":"2024-09-13T15:05:34.075607+0000","last_active":"2024-09-13T15:16:59.689670+0000","last_peered":"2024-09-13T15:16:59.689670+0000","last_clean":"2024-09-13T15:16:59.689670+0000","last_became_active":"2024-09-13T15:05:34.075079+0000","last_became_peered":"2024-09-13T15:05:34.075079+0000","last_unstale":"2024-09-13T15:16:59.689670+0000","last_undegraded":"2024-09-13T15:16:59.689670+0000","last_fullsized":"2024-09-13T15:16:59.689670+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:05.384206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1417,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689444+0000","last_change":"2024-09-13T15:07:56.108198+0000","last_active":"2024-09-13T15:16:59.689444+0000","last_peered":"2024-09-13T15:16:59.689444+0000","last_clean":"2024-09-13T15:16:59.689444+0000","last_became_active":"2024-09-13T15:07:56.107824+0000","last_became_peered":"2024-09-13T15:07:56.107824+0000","last_unstale":"2024-09-13T15:16:59.689444+0000","last_undegraded":"2024-09-13T15:16:59.689444+0000","last_fullsized":"2024-09-13T15:16:59.689444+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:19.128127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1566,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.667477+0000","last_change":"2024-09-13T15:05:34.082773+0000","last_active":"2024-09-13T15:16:59.667477+0000","last_peered":"2024-09-13T15:16:59.667477+0000","last_clean":"2024-09-13T15:16:59.667477+0000","last_became_active":"2024-09-13T15:05:34.082408+0000","last_became_peered":"2024-09-13T15:05:34.082408+0000","last_unstale":"2024-09-13T15:16:59.667477+0000","last_undegraded":"2024-09-13T15:16:59.667477+0000","last_fullsized":"2024-09-13T15:16:59.667477+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:13.447546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":363,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689695+0000","last_change":"2024-09-13T14:52:43.597400+0000","last_active":"2024-09-13T15:16:59.689695+0000","last_peered":"2024-09-13T15:16:59.689695+0000","last_clean":"2024-09-13T15:16:59.689695+0000","last_became_active":"2024-09-13T14:52:43.596294+0000","last_became_peered":"2024-09-13T14:52:43.596294+0000","last_unstale":"2024-09-13T15:16:59.689695+0000","last_undegraded":"2024-09-13T15:16:59.689695+0000","last_fullsized":"2024-09-13T15:16:59.689695+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:35.218171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1427,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674758+0000","last_change":"2024-09-13T15:05:34.070717+0000","last_active":"2024-09-13T15:16:59.674758+0000","last_peered":"2024-09-13T15:16:59.674758+0000","last_clean":"2024-09-13T15:16:59.674758+0000","last_became_active":"2024-09-13T15:05:34.069929+0000","last_became_peered":"2024-09-13T15:05:34.069929+0000","last_unstale":"2024-09-13T15:16:59.674758+0000","last_undegraded":"2024-09-13T15:16:59.674758+0000","last_fullsized":"2024-09-13T15:16:59.674758+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:44:19.208697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":265,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.686204+0000","last_change":"2024-09-13T15:05:34.071133+0000","last_active":"2024-09-13T15:16:59.686204+0000","last_peered":"2024-09-13T15:16:59.686204+0000","last_clean":"2024-09-13T15:16:59.686204+0000","last_became_active":"2024-09-13T15:05:34.070519+0000","last_became_peered":"2024-09-13T15:05:34.070519+0000","last_unstale":"2024-09-13T15:16:59.686204+0000","last_undegraded":"2024-09-13T15:16:59.686204+0000","last_fullsized":"2024-09-13T15:16:59.686204+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:02.617403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1642,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674126+0000","last_change":"2024-09-13T14:48:53.354832+0000","last_active":"2024-09-13T15:16:59.674126+0000","last_peered":"2024-09-13T15:16:59.674126+0000","last_clean":"2024-09-13T15:16:59.674126+0000","last_became_active":"2024-09-13T14:48:53.354511+0000","last_became_peered":"2024-09-13T14:48:53.354511+0000","last_unstale":"2024-09-13T15:16:59.674126+0000","last_undegraded":"2024-09-13T15:16:59.674126+0000","last_fullsized":"2024-09-13T15:16:59.674126+0000","mapping_epoch":978,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":979,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:11.570545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1427,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.675586+0000","last_change":"2024-09-13T15:07:56.095947+0000","last_active":"2024-09-13T15:16:59.675586+0000","last_peered":"2024-09-13T15:16:59.675586+0000","last_clean":"2024-09-13T15:16:59.675586+0000","last_became_active":"2024-09-13T15:07:56.094500+0000","last_became_peered":"2024-09-13T15:07:56.094500+0000","last_unstale":"2024-09-13T15:16:59.675586+0000","last_undegraded":"2024-09-13T15:16:59.675586+0000","last_fullsized":"2024-09-13T15:16:59.675586+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:20:31.568566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":51,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.671101+0000","last_change":"2024-09-13T15:08:52.176756+0000","last_active":"2024-09-13T15:16:58.671101+0000","last_peered":"2024-09-13T15:16:58.671101+0000","last_clean":"2024-09-13T15:16:58.671101+0000","last_became_active":"2024-09-13T15:08:52.176280+0000","last_became_peered":"2024-09-13T15:08:52.176280+0000","last_unstale":"2024-09-13T15:16:58.671101+0000","last_undegraded":"2024-09-13T15:16:58.671101+0000","last_fullsized":"2024-09-13T15:16:58.671101+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:41.217586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1415,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674405+0000","last_change":"2024-09-13T15:08:52.170737+0000","last_active":"2024-09-13T15:16:59.674405+0000","last_peered":"2024-09-13T15:16:59.674405+0000","last_clean":"2024-09-13T15:16:59.674405+0000","last_became_active":"2024-09-13T15:08:52.170217+0000","last_became_peered":"2024-09-13T15:08:52.170217+0000","last_unstale":"2024-09-13T15:16:59.674405+0000","last_undegraded":"2024-09-13T15:16:59.674405+0000","last_fullsized":"2024-09-13T15:16:59.674405+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:17.688936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1642,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.670392+0000","last_change":"2024-09-13T15:05:34.087376+0000","last_active":"2024-09-13T15:16:59.670392+0000","last_peered":"2024-09-13T15:16:59.670392+0000","last_clean":"2024-09-13T15:16:59.670392+0000","last_became_active":"2024-09-13T15:05:34.087046+0000","last_became_peered":"2024-09-13T15:05:34.087046+0000","last_unstale":"2024-09-13T15:16:59.670392+0000","last_undegraded":"2024-09-13T15:16:59.670392+0000","last_fullsized":"2024-09-13T15:16:59.670392+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:47.801431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1355,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674568+0000","last_change":"2024-09-13T15:08:52.174279+0000","last_active":"2024-09-13T15:16:59.674568+0000","last_peered":"2024-09-13T15:16:59.674568+0000","last_clean":"2024-09-13T15:16:59.674568+0000","last_became_active":"2024-09-13T15:08:52.173877+0000","last_became_peered":"2024-09-13T15:08:52.173877+0000","last_unstale":"2024-09-13T15:16:59.674568+0000","last_undegraded":"2024-09-13T15:16:59.674568+0000","last_fullsized":"2024-09-13T15:16:59.674568+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:16.648765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1096,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.666136+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T15:16:58.666136+0000","last_peered":"2024-09-13T15:16:58.666136+0000","last_clean":"2024-09-13T15:16:58.666136+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T15:16:58.666136+0000","last_undegraded":"2024-09-13T15:16:58.666136+0000","last_fullsized":"2024-09-13T15:16:58.666136+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1150,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.676700+0000","last_change":"2024-09-13T15:05:34.089020+0000","last_active":"2024-09-13T15:16:58.676700+0000","last_peered":"2024-09-13T15:16:58.676700+0000","last_clean":"2024-09-13T15:16:58.676700+0000","last_became_active":"2024-09-13T15:05:34.088440+0000","last_became_peered":"2024-09-13T15:05:34.088440+0000","last_unstale":"2024-09-13T15:16:58.676700+0000","last_undegraded":"2024-09-13T15:16:58.676700+0000","last_fullsized":"2024-09-13T15:16:58.676700+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:24.826746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":93,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.669956+0000","last_change":"2024-09-13T15:08:54.446160+0000","last_active":"2024-09-13T15:16:59.669956+0000","last_peered":"2024-09-13T15:16:59.669956+0000","last_clean":"2024-09-13T15:16:59.669956+0000","last_became_active":"2024-09-13T15:08:54.445649+0000","last_became_peered":"2024-09-13T15:08:54.445649+0000","last_unstale":"2024-09-13T15:16:59.669956+0000","last_undegraded":"2024-09-13T15:16:59.669956+0000","last_fullsized":"2024-09-13T15:16:59.669956+0000","mapping_epoch":1082,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1083,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:17.407470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":394,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689612+0000","last_change":"2024-09-13T15:08:52.203370+0000","last_active":"2024-09-13T15:16:59.689612+0000","last_peered":"2024-09-13T15:16:59.689612+0000","last_clean":"2024-09-13T15:16:59.689612+0000","last_became_active":"2024-09-13T15:08:52.179887+0000","last_became_peered":"2024-09-13T15:08:52.179887+0000","last_unstale":"2024-09-13T15:16:59.689612+0000","last_undegraded":"2024-09-13T15:16:59.689612+0000","last_fullsized":"2024-09-13T15:16:59.689612+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:57.293764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":378,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689768+0000","last_change":"2024-09-13T15:08:52.180881+0000","last_active":"2024-09-13T15:16:59.689768+0000","last_peered":"2024-09-13T15:16:59.689768+0000","last_clean":"2024-09-13T15:16:59.689768+0000","last_became_active":"2024-09-13T15:08:52.180000+0000","last_became_peered":"2024-09-13T15:08:52.180000+0000","last_unstale":"2024-09-13T15:16:59.689768+0000","last_undegraded":"2024-09-13T15:16:59.689768+0000","last_fullsized":"2024-09-13T15:16:59.689768+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:32.622289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":366,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689333+0000","last_change":"2024-09-13T15:08:52.188050+0000","last_active":"2024-09-13T15:16:59.689333+0000","last_peered":"2024-09-13T15:16:59.689333+0000","last_clean":"2024-09-13T15:16:59.689333+0000","last_became_active":"2024-09-13T15:08:52.187522+0000","last_became_peered":"2024-09-13T15:08:52.187522+0000","last_unstale":"2024-09-13T15:16:59.689333+0000","last_undegraded":"2024-09-13T15:16:59.689333+0000","last_fullsized":"2024-09-13T15:16:59.689333+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:29.314964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":51,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.671021+0000","last_change":"2024-09-13T15:08:52.187782+0000","last_active":"2024-09-13T15:16:58.671021+0000","last_peered":"2024-09-13T15:16:58.671021+0000","last_clean":"2024-09-13T15:16:58.671021+0000","last_became_active":"2024-09-13T15:08:52.186669+0000","last_became_peered":"2024-09-13T15:08:52.186669+0000","last_unstale":"2024-09-13T15:16:58.671021+0000","last_undegraded":"2024-09-13T15:16:58.671021+0000","last_fullsized":"2024-09-13T15:16:58.671021+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:43.000869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1693,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674409+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T15:16:59.674409+0000","last_peered":"2024-09-13T15:16:59.674409+0000","last_clean":"2024-09-13T15:16:59.674409+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T15:16:59.674409+0000","last_undegraded":"2024-09-13T15:16:59.674409+0000","last_fullsized":"2024-09-13T15:16:59.674409+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1348,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674500+0000","last_change":"2024-09-13T15:07:56.109509+0000","last_active":"2024-09-13T15:16:59.674500+0000","last_peered":"2024-09-13T15:16:59.674500+0000","last_clean":"2024-09-13T15:16:59.674500+0000","last_became_active":"2024-09-13T15:07:56.108034+0000","last_became_peered":"2024-09-13T15:07:56.108034+0000","last_unstale":"2024-09-13T15:16:59.674500+0000","last_undegraded":"2024-09-13T15:16:59.674500+0000","last_fullsized":"2024-09-13T15:16:59.674500+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:57:02.300327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":269,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.687997+0000","last_change":"2024-09-13T15:08:52.186362+0000","last_active":"2024-09-13T15:16:59.687997+0000","last_peered":"2024-09-13T15:16:59.687997+0000","last_clean":"2024-09-13T15:16:59.687997+0000","last_became_active":"2024-09-13T15:08:52.185731+0000","last_became_peered":"2024-09-13T15:08:52.185731+0000","last_unstale":"2024-09-13T15:16:59.687997+0000","last_undegraded":"2024-09-13T15:16:59.687997+0000","last_fullsized":"2024-09-13T15:16:59.687997+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:26.408919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1344,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.675093+0000","last_change":"2024-09-13T15:05:34.085917+0000","last_active":"2024-09-13T15:16:59.675093+0000","last_peered":"2024-09-13T15:16:59.675093+0000","last_clean":"2024-09-13T15:16:59.675093+0000","last_became_active":"2024-09-13T15:05:34.085156+0000","last_became_peered":"2024-09-13T15:05:34.085156+0000","last_unstale":"2024-09-13T15:16:59.675093+0000","last_undegraded":"2024-09-13T15:16:59.675093+0000","last_fullsized":"2024-09-13T15:16:59.675093+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:33.482732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1623,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.671047+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T15:16:59.671047+0000","last_peered":"2024-09-13T15:16:59.671047+0000","last_clean":"2024-09-13T15:16:59.671047+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T15:16:59.671047+0000","last_undegraded":"2024-09-13T15:16:59.671047+0000","last_fullsized":"2024-09-13T15:16:59.671047+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1128,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.676776+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T15:16:58.676776+0000","last_peered":"2024-09-13T15:16:58.676776+0000","last_clean":"2024-09-13T15:16:58.676776+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T15:16:58.676776+0000","last_undegraded":"2024-09-13T15:16:58.676776+0000","last_fullsized":"2024-09-13T15:16:58.676776+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":255,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.687113+0000","last_change":"2024-09-13T14:52:43.583860+0000","last_active":"2024-09-13T15:16:59.687113+0000","last_peered":"2024-09-13T15:16:59.687113+0000","last_clean":"2024-09-13T15:16:59.687113+0000","last_became_active":"2024-09-13T14:52:43.583055+0000","last_became_peered":"2024-09-13T14:52:43.583055+0000","last_unstale":"2024-09-13T15:16:59.687113+0000","last_undegraded":"2024-09-13T15:16:59.687113+0000","last_fullsized":"2024-09-13T15:16:59.687113+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:25:55.113648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":255,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.686739+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T15:16:59.686739+0000","last_peered":"2024-09-13T15:16:59.686739+0000","last_clean":"2024-09-13T15:16:59.686739+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T15:16:59.686739+0000","last_undegraded":"2024-09-13T15:16:59.686739+0000","last_fullsized":"2024-09-13T15:16:59.686739+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":255,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.687181+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T15:16:59.687181+0000","last_peered":"2024-09-13T15:16:59.687181+0000","last_clean":"2024-09-13T15:16:59.687181+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T15:16:59.687181+0000","last_undegraded":"2024-09-13T15:16:59.687181+0000","last_fullsized":"2024-09-13T15:16:59.687181+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1110,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.666024+0000","last_change":"2024-09-13T15:05:34.086758+0000","last_active":"2024-09-13T15:16:58.666024+0000","last_peered":"2024-09-13T15:16:58.666024+0000","last_clean":"2024-09-13T15:16:58.666024+0000","last_became_active":"2024-09-13T15:05:34.086145+0000","last_became_peered":"2024-09-13T15:05:34.086145+0000","last_unstale":"2024-09-13T15:16:58.666024+0000","last_undegraded":"2024-09-13T15:16:58.666024+0000","last_fullsized":"2024-09-13T15:16:58.666024+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:33.764698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":266,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.687128+0000","last_change":"2024-09-13T15:05:34.085898+0000","last_active":"2024-09-13T15:16:59.687128+0000","last_peered":"2024-09-13T15:16:59.687128+0000","last_clean":"2024-09-13T15:16:59.687128+0000","last_became_active":"2024-09-13T15:05:34.085174+0000","last_became_peered":"2024-09-13T15:05:34.085174+0000","last_unstale":"2024-09-13T15:16:59.687128+0000","last_undegraded":"2024-09-13T15:16:59.687128+0000","last_fullsized":"2024-09-13T15:16:59.687128+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:17:39.204192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":366,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689821+0000","last_change":"2024-09-13T15:08:52.188254+0000","last_active":"2024-09-13T15:16:59.689821+0000","last_peered":"2024-09-13T15:16:59.689821+0000","last_clean":"2024-09-13T15:16:59.689821+0000","last_became_active":"2024-09-13T15:08:52.187535+0000","last_became_peered":"2024-09-13T15:08:52.187535+0000","last_unstale":"2024-09-13T15:16:59.689821+0000","last_undegraded":"2024-09-13T15:16:59.689821+0000","last_fullsized":"2024-09-13T15:16:59.689821+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:32.024558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1149,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.665655+0000","last_change":"2024-09-13T15:08:52.209457+0000","last_active":"2024-09-13T15:16:58.665655+0000","last_peered":"2024-09-13T15:16:58.665655+0000","last_clean":"2024-09-13T15:16:58.665655+0000","last_became_active":"2024-09-13T15:08:52.171595+0000","last_became_peered":"2024-09-13T15:08:52.171595+0000","last_unstale":"2024-09-13T15:16:58.665655+0000","last_undegraded":"2024-09-13T15:16:58.665655+0000","last_fullsized":"2024-09-13T15:16:58.665655+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:05:42.960465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":255,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.688159+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T15:16:59.688159+0000","last_peered":"2024-09-13T15:16:59.688159+0000","last_clean":"2024-09-13T15:16:59.688159+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T15:16:59.688159+0000","last_undegraded":"2024-09-13T15:16:59.688159+0000","last_fullsized":"2024-09-13T15:16:59.688159+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":270,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.687695+0000","last_change":"2024-09-13T15:08:52.186074+0000","last_active":"2024-09-13T15:16:59.687695+0000","last_peered":"2024-09-13T15:16:59.687695+0000","last_clean":"2024-09-13T15:16:59.687695+0000","last_became_active":"2024-09-13T15:08:52.185693+0000","last_became_peered":"2024-09-13T15:08:52.185693+0000","last_unstale":"2024-09-13T15:16:59.687695+0000","last_undegraded":"2024-09-13T15:16:59.687695+0000","last_fullsized":"2024-09-13T15:16:59.687695+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:39:06.391608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1543,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.666931+0000","last_change":"2024-09-13T15:05:34.082028+0000","last_active":"2024-09-13T15:16:59.666931+0000","last_peered":"2024-09-13T15:16:59.666931+0000","last_clean":"2024-09-13T15:16:59.666931+0000","last_became_active":"2024-09-13T15:05:34.081220+0000","last_became_peered":"2024-09-13T15:05:34.081220+0000","last_unstale":"2024-09-13T15:16:59.666931+0000","last_undegraded":"2024-09-13T15:16:59.666931+0000","last_fullsized":"2024-09-13T15:16:59.666931+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:12:26.484994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1520,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.687897+0000","last_change":"2024-09-13T15:07:56.111014+0000","last_active":"2024-09-13T15:16:59.687897+0000","last_peered":"2024-09-13T15:16:59.687897+0000","last_clean":"2024-09-13T15:16:59.687897+0000","last_became_active":"2024-09-13T15:07:56.110698+0000","last_became_peered":"2024-09-13T15:07:56.110698+0000","last_unstale":"2024-09-13T15:16:59.687897+0000","last_undegraded":"2024-09-13T15:16:59.687897+0000","last_fullsized":"2024-09-13T15:16:59.687897+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:29.395253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1670,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.670007+0000","last_change":"2024-09-13T15:05:34.087817+0000","last_active":"2024-09-13T15:16:59.670007+0000","last_peered":"2024-09-13T15:16:59.670007+0000","last_clean":"2024-09-13T15:16:59.670007+0000","last_became_active":"2024-09-13T15:05:34.087450+0000","last_became_peered":"2024-09-13T15:05:34.087450+0000","last_unstale":"2024-09-13T15:16:59.670007+0000","last_undegraded":"2024-09-13T15:16:59.670007+0000","last_fullsized":"2024-09-13T15:16:59.670007+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:47.661935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1128,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.673375+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T15:16:58.673375+0000","last_peered":"2024-09-13T15:16:58.673375+0000","last_clean":"2024-09-13T15:16:58.673375+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T15:16:58.673375+0000","last_undegraded":"2024-09-13T15:16:58.673375+0000","last_fullsized":"2024-09-13T15:16:58.673375+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.670397+0000","last_change":"2024-09-13T15:08:52.188028+0000","last_active":"2024-09-13T15:16:58.670397+0000","last_peered":"2024-09-13T15:16:58.670397+0000","last_clean":"2024-09-13T15:16:58.670397+0000","last_became_active":"2024-09-13T15:08:52.187424+0000","last_became_peered":"2024-09-13T15:08:52.187424+0000","last_unstale":"2024-09-13T15:16:58.670397+0000","last_undegraded":"2024-09-13T15:16:58.670397+0000","last_fullsized":"2024-09-13T15:16:58.670397+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:31.790876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1357,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674660+0000","last_change":"2024-09-13T15:08:52.170510+0000","last_active":"2024-09-13T15:16:59.674660+0000","last_peered":"2024-09-13T15:16:59.674660+0000","last_clean":"2024-09-13T15:16:59.674660+0000","last_became_active":"2024-09-13T15:08:52.170141+0000","last_became_peered":"2024-09-13T15:08:52.170141+0000","last_unstale":"2024-09-13T15:16:59.674660+0000","last_undegraded":"2024-09-13T15:16:59.674660+0000","last_fullsized":"2024-09-13T15:16:59.674660+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:20.674380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1089,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.667360+0000","last_change":"2024-09-13T14:38:59.366029+0000","last_active":"2024-09-13T15:16:58.667360+0000","last_peered":"2024-09-13T15:16:58.667360+0000","last_clean":"2024-09-13T15:16:58.667360+0000","last_became_active":"2024-09-13T14:38:59.363472+0000","last_became_peered":"2024-09-13T14:38:59.363472+0000","last_unstale":"2024-09-13T15:16:58.667360+0000","last_undegraded":"2024-09-13T15:16:58.667360+0000","last_fullsized":"2024-09-13T15:16:58.667360+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:43:52.929075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1228,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.673459+0000","last_change":"2024-09-13T15:08:52.214302+0000","last_active":"2024-09-13T15:16:58.673459+0000","last_peered":"2024-09-13T15:16:58.673459+0000","last_clean":"2024-09-13T15:16:58.673459+0000","last_became_active":"2024-09-13T15:08:52.194638+0000","last_became_peered":"2024-09-13T15:08:52.194638+0000","last_unstale":"2024-09-13T15:16:58.673459+0000","last_undegraded":"2024-09-13T15:16:58.673459+0000","last_fullsized":"2024-09-13T15:16:58.673459+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T14:15:37.340490+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:57.236719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":265,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.688225+0000","last_change":"2024-09-13T15:05:34.078805+0000","last_active":"2024-09-13T15:16:59.688225+0000","last_peered":"2024-09-13T15:16:59.688225+0000","last_clean":"2024-09-13T15:16:59.688225+0000","last_became_active":"2024-09-13T15:05:34.078180+0000","last_became_peered":"2024-09-13T15:05:34.078180+0000","last_unstale":"2024-09-13T15:16:59.688225+0000","last_undegraded":"2024-09-13T15:16:59.688225+0000","last_fullsized":"2024-09-13T15:16:59.688225+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:47.240906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1405,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674792+0000","last_change":"2024-09-13T15:05:34.076149+0000","last_active":"2024-09-13T15:16:59.674792+0000","last_peered":"2024-09-13T15:16:59.674792+0000","last_clean":"2024-09-13T15:16:59.674792+0000","last_became_active":"2024-09-13T15:05:34.075623+0000","last_became_peered":"2024-09-13T15:05:34.075623+0000","last_unstale":"2024-09-13T15:16:59.674792+0000","last_undegraded":"2024-09-13T15:16:59.674792+0000","last_fullsized":"2024-09-13T15:16:59.674792+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:23.608262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":379,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.690244+0000","last_change":"2024-09-13T15:08:52.189786+0000","last_active":"2024-09-13T15:16:59.690244+0000","last_peered":"2024-09-13T15:16:59.690244+0000","last_clean":"2024-09-13T15:16:59.690244+0000","last_became_active":"2024-09-13T15:08:52.188997+0000","last_became_peered":"2024-09-13T15:08:52.188997+0000","last_unstale":"2024-09-13T15:16:59.690244+0000","last_undegraded":"2024-09-13T15:16:59.690244+0000","last_fullsized":"2024-09-13T15:16:59.690244+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:08.932986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":351,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689288+0000","last_change":"2024-09-13T14:47:46.328936+0000","last_active":"2024-09-13T15:16:59.689288+0000","last_peered":"2024-09-13T15:16:59.689288+0000","last_clean":"2024-09-13T15:16:59.689288+0000","last_became_active":"2024-09-13T14:47:46.328133+0000","last_became_peered":"2024-09-13T14:47:46.328133+0000","last_unstale":"2024-09-13T15:16:59.689288+0000","last_undegraded":"2024-09-13T15:16:59.689288+0000","last_fullsized":"2024-09-13T15:16:59.689288+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:32:21.654588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":365,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689921+0000","last_change":"2024-09-13T14:52:43.597595+0000","last_active":"2024-09-13T15:16:59.689921+0000","last_peered":"2024-09-13T15:16:59.689921+0000","last_clean":"2024-09-13T15:16:59.689921+0000","last_became_active":"2024-09-13T14:52:43.596592+0000","last_became_peered":"2024-09-13T14:52:43.596592+0000","last_unstale":"2024-09-13T15:16:59.689921+0000","last_undegraded":"2024-09-13T15:16:59.689921+0000","last_fullsized":"2024-09-13T15:16:59.689921+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:56.532603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":51,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.670057+0000","last_change":"2024-09-13T15:08:52.182709+0000","last_active":"2024-09-13T15:16:58.670057+0000","last_peered":"2024-09-13T15:16:58.670057+0000","last_clean":"2024-09-13T15:16:58.670057+0000","last_became_active":"2024-09-13T15:08:52.181906+0000","last_became_peered":"2024-09-13T15:08:52.181906+0000","last_unstale":"2024-09-13T15:16:58.670057+0000","last_undegraded":"2024-09-13T15:16:58.670057+0000","last_fullsized":"2024-09-13T15:16:58.670057+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:36.317662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":376,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689528+0000","last_change":"2024-09-13T15:08:52.182885+0000","last_active":"2024-09-13T15:16:59.689528+0000","last_peered":"2024-09-13T15:16:59.689528+0000","last_clean":"2024-09-13T15:16:59.689528+0000","last_became_active":"2024-09-13T15:08:52.180608+0000","last_became_peered":"2024-09-13T15:08:52.180608+0000","last_unstale":"2024-09-13T15:16:59.689528+0000","last_undegraded":"2024-09-13T15:16:59.689528+0000","last_fullsized":"2024-09-13T15:16:59.689528+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:12.517462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1389,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674596+0000","last_change":"2024-09-13T15:05:34.085458+0000","last_active":"2024-09-13T15:16:59.674596+0000","last_peered":"2024-09-13T15:16:59.674596+0000","last_clean":"2024-09-13T15:16:59.674596+0000","last_became_active":"2024-09-13T15:05:34.084748+0000","last_became_peered":"2024-09-13T15:05:34.084748+0000","last_unstale":"2024-09-13T15:16:59.674596+0000","last_undegraded":"2024-09-13T15:16:59.674596+0000","last_fullsized":"2024-09-13T15:16:59.674596+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:21.898959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1679,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.671167+0000","last_change":"2024-09-13T14:52:54.185117+0000","last_active":"2024-09-13T15:16:59.671167+0000","last_peered":"2024-09-13T15:16:59.671167+0000","last_clean":"2024-09-13T15:16:59.671167+0000","last_became_active":"2024-09-13T14:52:54.184712+0000","last_became_peered":"2024-09-13T14:52:54.184712+0000","last_unstale":"2024-09-13T15:16:59.671167+0000","last_undegraded":"2024-09-13T15:16:59.671167+0000","last_fullsized":"2024-09-13T15:16:59.671167+0000","mapping_epoch":1007,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1008,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:32.654401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1113,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.665831+0000","last_change":"2024-09-13T15:08:52.175569+0000","last_active":"2024-09-13T15:16:58.665831+0000","last_peered":"2024-09-13T15:16:58.665831+0000","last_clean":"2024-09-13T15:16:58.665831+0000","last_became_active":"2024-09-13T15:08:52.174891+0000","last_became_peered":"2024-09-13T15:08:52.174891+0000","last_unstale":"2024-09-13T15:16:58.665831+0000","last_undegraded":"2024-09-13T15:16:58.665831+0000","last_fullsized":"2024-09-13T15:16:58.665831+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.933115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":51,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.670261+0000","last_change":"2024-09-13T15:08:52.188074+0000","last_active":"2024-09-13T15:16:58.670261+0000","last_peered":"2024-09-13T15:16:58.670261+0000","last_clean":"2024-09-13T15:16:58.670261+0000","last_became_active":"2024-09-13T15:08:52.186649+0000","last_became_peered":"2024-09-13T15:08:52.186649+0000","last_unstale":"2024-09-13T15:16:58.670261+0000","last_undegraded":"2024-09-13T15:16:58.670261+0000","last_fullsized":"2024-09-13T15:16:58.670261+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:23.478250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":1092,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.676766+0000","last_change":"2024-09-13T15:08:52.208388+0000","last_active":"2024-09-13T15:16:58.676766+0000","last_peered":"2024-09-13T15:16:58.676766+0000","last_clean":"2024-09-13T15:16:58.676766+0000","last_became_active":"2024-09-13T15:08:52.192253+0000","last_became_peered":"2024-09-13T15:08:52.192253+0000","last_unstale":"2024-09-13T15:16:58.676766+0000","last_undegraded":"2024-09-13T15:16:58.676766+0000","last_fullsized":"2024-09-13T15:16:58.676766+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:07.508774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":376,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.689435+0000","last_change":"2024-09-13T15:05:34.072278+0000","last_active":"2024-09-13T15:16:59.689435+0000","last_peered":"2024-09-13T15:16:59.689435+0000","last_clean":"2024-09-13T15:16:59.689435+0000","last_became_active":"2024-09-13T15:05:34.071723+0000","last_became_peered":"2024-09-13T15:05:34.071723+0000","last_unstale":"2024-09-13T15:16:59.689435+0000","last_undegraded":"2024-09-13T15:16:59.689435+0000","last_fullsized":"2024-09-13T15:16:59.689435+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:02.130890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1155,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.674037+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T15:16:58.674037+0000","last_peered":"2024-09-13T15:16:58.674037+0000","last_clean":"2024-09-13T15:16:58.674037+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T15:16:58.674037+0000","last_undegraded":"2024-09-13T15:16:58.674037+0000","last_fullsized":"2024-09-13T15:16:58.674037+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1078,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.665793+0000","last_change":"2024-09-13T15:08:52.176830+0000","last_active":"2024-09-13T15:16:58.665793+0000","last_peered":"2024-09-13T15:16:58.665793+0000","last_clean":"2024-09-13T15:16:58.665793+0000","last_became_active":"2024-09-13T15:08:52.176303+0000","last_became_peered":"2024-09-13T15:08:52.176303+0000","last_unstale":"2024-09-13T15:16:58.665793+0000","last_undegraded":"2024-09-13T15:16:58.665793+0000","last_fullsized":"2024-09-13T15:16:58.665793+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:59.553257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1126,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.673938+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T15:16:58.673938+0000","last_peered":"2024-09-13T15:16:58.673938+0000","last_clean":"2024-09-13T15:16:58.673938+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T15:16:58.673938+0000","last_undegraded":"2024-09-13T15:16:58.673938+0000","last_fullsized":"2024-09-13T15:16:58.673938+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":61,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.669877+0000","last_change":"2024-09-13T15:08:52.200695+0000","last_active":"2024-09-13T15:16:58.669877+0000","last_peered":"2024-09-13T15:16:58.669877+0000","last_clean":"2024-09-13T15:16:58.669877+0000","last_became_active":"2024-09-13T15:08:52.183473+0000","last_became_peered":"2024-09-13T15:08:52.183473+0000","last_unstale":"2024-09-13T15:16:58.669877+0000","last_undegraded":"2024-09-13T15:16:58.669877+0000","last_fullsized":"2024-09-13T15:16:58.669877+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:42.810003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1402,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.675024+0000","last_change":"2024-09-13T15:05:34.074069+0000","last_active":"2024-09-13T15:16:59.675024+0000","last_peered":"2024-09-13T15:16:59.675024+0000","last_clean":"2024-09-13T15:16:59.675024+0000","last_became_active":"2024-09-13T15:05:34.073282+0000","last_became_peered":"2024-09-13T15:05:34.073282+0000","last_unstale":"2024-09-13T15:16:59.675024+0000","last_undegraded":"2024-09-13T15:16:59.675024+0000","last_fullsized":"2024-09-13T15:16:59.675024+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:14.307582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1119,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.673422+0000","last_change":"2024-09-13T15:08:52.192368+0000","last_active":"2024-09-13T15:16:58.673422+0000","last_peered":"2024-09-13T15:16:58.673422+0000","last_clean":"2024-09-13T15:16:58.673422+0000","last_became_active":"2024-09-13T15:08:52.191668+0000","last_became_peered":"2024-09-13T15:08:52.191668+0000","last_unstale":"2024-09-13T15:16:58.673422+0000","last_undegraded":"2024-09-13T15:16:58.673422+0000","last_fullsized":"2024-09-13T15:16:58.673422+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:27.410222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1642,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674241+0000","last_change":"2024-09-13T15:05:34.087636+0000","last_active":"2024-09-13T15:16:59.674241+0000","last_peered":"2024-09-13T15:16:59.674241+0000","last_clean":"2024-09-13T15:16:59.674241+0000","last_became_active":"2024-09-13T15:05:34.087255+0000","last_became_peered":"2024-09-13T15:05:34.087255+0000","last_unstale":"2024-09-13T15:16:59.674241+0000","last_undegraded":"2024-09-13T15:16:59.674241+0000","last_fullsized":"2024-09-13T15:16:59.674241+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:36.313581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1643,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.670274+0000","last_change":"2024-09-13T15:05:34.089162+0000","last_active":"2024-09-13T15:16:59.670274+0000","last_peered":"2024-09-13T15:16:59.670274+0000","last_clean":"2024-09-13T15:16:59.670274+0000","last_became_active":"2024-09-13T15:05:34.088727+0000","last_became_peered":"2024-09-13T15:05:34.088727+0000","last_unstale":"2024-09-13T15:16:59.670274+0000","last_undegraded":"2024-09-13T15:16:59.670274+0000","last_fullsized":"2024-09-13T15:16:59.670274+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:29.209082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1092,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.665907+0000","last_change":"2024-09-13T15:05:34.073363+0000","last_active":"2024-09-13T15:16:58.665907+0000","last_peered":"2024-09-13T15:16:58.665907+0000","last_clean":"2024-09-13T15:16:58.665907+0000","last_became_active":"2024-09-13T15:05:34.072906+0000","last_became_peered":"2024-09-13T15:05:34.072906+0000","last_unstale":"2024-09-13T15:16:58.665907+0000","last_undegraded":"2024-09-13T15:16:58.665907+0000","last_fullsized":"2024-09-13T15:16:58.665907+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:27.966654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1132,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-13T15:16:58.673725+0000","last_change":"2024-09-13T14:47:46.343052+0000","last_active":"2024-09-13T15:16:58.673725+0000","last_peered":"2024-09-13T15:16:58.673725+0000","last_clean":"2024-09-13T15:16:58.673725+0000","last_became_active":"2024-09-13T14:47:46.326567+0000","last_became_peered":"2024-09-13T14:47:46.326567+0000","last_unstale":"2024-09-13T15:16:58.673725+0000","last_undegraded":"2024-09-13T15:16:58.673725+0000","last_fullsized":"2024-09-13T15:16:58.673725+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:17.022870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1414,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-13T15:16:59.674947+0000","last_change":"2024-09-13T15:05:34.075192+0000","last_active":"2024-09-13T15:16:59.674947+0000","last_peered":"2024-09-13T15:16:59.674947+0000","last_clean":"2024-09-13T15:16:59.674947+0000","last_became_active":"2024-09-13T15:05:34.073315+0000","last_became_peered":"2024-09-13T15:05:34.073315+0000","last_unstale":"2024-09-13T15:16:59.674947+0000","last_undegraded":"2024-09-13T15:16:59.674947+0000","last_fullsized":"2024-09-13T15:16:59.674947+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:38.699775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":46,"num_pg":26,"stat_sum":{"num_bytes":5652545560,"num_objects":86252,"num_object_clones":0,"num_object_copies":258756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8943,"num_objects_misplaced":4911,"num_objects_unfound":0,"num_objects_dirty":86252,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":118570,"num_write_kb":6554241,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6205,"num_bytes_recovered":406650880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17458016256,"data_stored":17458004040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173969,"ondisk_log_size":173969,"up":78,"acting":73,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":26,"num_read_kb":28,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":16374,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398194,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2761228,"kb_used_data":1860996,"kb_used_omap":1,"kb_used_meta":900222,"kb_avail":90983924,"statfs":{"total":95995035648,"available":93167538176,"internally_reserved":0,"allocated":1905659904,"data_stored":1901008463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":921827672},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594429,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3422916,"kb_used_data":2530940,"kb_used_omap":1,"kb_used_meta":891966,"kb_avail":90322236,"statfs":{"total":95995035648,"available":92489969664,"internally_reserved":0,"allocated":2591682560,"data_stored":2587039722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913373528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790663,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4017140,"kb_used_data":3135212,"kb_used_omap":1,"kb_used_meta":881918,"kb_avail":89728012,"statfs":{"total":95995035648,"available":91881484288,"internally_reserved":0,"allocated":3210457088,"data_stored":3205825338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":903084376},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980832192,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1456448,"kb_used_data":767992,"kb_used_omap":76,"kb_used_meta":688371,"kb_avail":92288704,"statfs":{"total":95995035648,"available":94503632896,"internally_reserved":0,"allocated":786423808,"data_stored":781784029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":704892736},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1066,"seq":4578435137677,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2179308,"kb_used_data":1845668,"kb_used_omap":121,"kb_used_meta":333446,"kb_avail":91565844,"statfs":{"total":95995035648,"available":93763424256,"internally_reserved":0,"allocated":1889964032,"data_stored":1885339941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124093,"internal_metadata":341449539},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023373,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2978992,"kb_used_data":2146920,"kb_used_omap":71,"kb_used_meta":831864,"kb_avail":90766160,"statfs":{"total":95995035648,"available":92944547840,"internally_reserved":0,"allocated":2198446080,"data_stored":2193818415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":851829408},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004503,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2819632,"kb_used_data":2104360,"kb_used_omap":66,"kb_used_meta":715197,"kb_avail":90925520,"statfs":{"total":95995035648,"available":93107732480,"internally_reserved":0,"allocated":2154864640,"data_stored":2150236406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":732361898},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195695,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3627504,"kb_used_data":2776552,"kb_used_omap":56,"kb_used_meta":850695,"kb_avail":90117648,"statfs":{"total":95995035648,"available":92280471552,"internally_reserved":0,"allocated":2843189248,"data_stored":2838559439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":871111823},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,2,2,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2829651968,"data_stored":2829647896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2142572544,"data_stored":2142568472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2185822208,"data_stored":2185822208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1874722816,"data_stored":1874722816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":771620864,"data_stored":771620864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3190423552,"data_stored":3190423552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2570387456,"data_stored":2570387456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1892814848,"data_stored":1892810776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:17:28.508 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T15:17:28.508 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 11 2024-09-13T15:17:28.600 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:28.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:28 smithi177 bash[68594]: cluster 2024-09-13T15:17:27.334150+0000 osd.5 (osd.5) 19 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:28.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:28 smithi177 bash[68594]: cluster 2024-09-13T15:17:27.334150+0000 osd.5 (osd.5) 19 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:28.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:28 smithi099 bash[84576]: cluster 2024-09-13T15:17:27.334150+0000 osd.5 (osd.5) 19 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:28.621 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:28 smithi099 bash[84576]: cluster 2024-09-13T15:17:27.334150+0000 osd.5 (osd.5) 19 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:28.621 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:28 smithi099 bash[87147]: cluster 2024-09-13T15:17:27.334150+0000 osd.5 (osd.5) 19 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:28.621 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:28 smithi099 bash[87147]: cluster 2024-09-13T15:17:27.334150+0000 osd.5 (osd.5) 19 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:28.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:28.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:28.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:28.789 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:28.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:28.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:28.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:28.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:28.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:29.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:29.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:29.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:29.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:29.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:29.184 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:29.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:29.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:29.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:29.330 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:29.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:29.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:29.434 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:29.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:29.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:29.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:29.568 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:29.596 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:29.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:29.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:29.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:29 smithi177 bash[68594]: cluster 2024-09-13T15:17:29.208913+0000 mgr.y (mgr.26252) 1545 : cluster [DBG] pgmap v2154: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8274/230154 objects degraded (3.595%); 1200/230154 objects misplaced (0.521%); 15 MiB/s, 239 objects/s recovering 2024-09-13T15:17:29.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:29 smithi177 bash[68594]: cluster 2024-09-13T15:17:29.208913+0000 mgr.y (mgr.26252) 1545 : cluster [DBG] pgmap v2154: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8274/230154 objects degraded (3.595%); 1200/230154 objects misplaced (0.521%); 15 MiB/s, 239 objects/s recovering 2024-09-13T15:17:29.700 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:29.725 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:29.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:29.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:29 smithi099 bash[84576]: cluster 2024-09-13T15:17:29.208913+0000 mgr.y (mgr.26252) 1545 : cluster [DBG] pgmap v2154: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8274/230154 objects degraded (3.595%); 1200/230154 objects misplaced (0.521%); 15 MiB/s, 239 objects/s recovering 2024-09-13T15:17:29.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:29 smithi099 bash[84576]: cluster 2024-09-13T15:17:29.208913+0000 mgr.y (mgr.26252) 1545 : cluster [DBG] pgmap v2154: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8274/230154 objects degraded (3.595%); 1200/230154 objects misplaced (0.521%); 15 MiB/s, 239 objects/s recovering 2024-09-13T15:17:29.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:29 smithi099 bash[87147]: cluster 2024-09-13T15:17:29.208913+0000 mgr.y (mgr.26252) 1545 : cluster [DBG] pgmap v2154: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8274/230154 objects degraded (3.595%); 1200/230154 objects misplaced (0.521%); 15 MiB/s, 239 objects/s recovering 2024-09-13T15:17:29.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:29 smithi099 bash[87147]: cluster 2024-09-13T15:17:29.208913+0000 mgr.y (mgr.26252) 1545 : cluster [DBG] pgmap v2154: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 8274/230154 objects degraded (3.595%); 1200/230154 objects misplaced (0.521%); 15 MiB/s, 239 objects/s recovering 2024-09-13T15:17:29.829 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:29.858 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:29.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:29.963 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:29.989 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:29.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:29.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:30.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:30.119 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:30.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:30.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:30.226 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:30.250 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:30.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:30.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:30.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:30.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:17:30 smithi177 bash[61439]: ts=2024-09-13T15:17:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:17:30.377 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:30.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:30.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:30.401 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:17:30.401 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:17:30.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:30.511 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:30.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:30.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:30.538 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:17:30.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:17:30.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:30.638 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:30.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:30.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:30.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:30.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:30.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:30.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:30.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:30.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:30.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:31.010 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:31.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:31.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:31.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:31.145 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:31.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:31.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:31.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:31.278 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:31.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:31.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:31.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:31.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:31.441 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:31.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:31.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:31.548 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:31.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:31.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:31.677 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:31.698 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:31.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:31.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:31.753 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:17:31.801 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:31.835 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:31.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:31.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:31.938 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:31.970 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:31.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:32.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:32.108 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:32.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:32.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:32.212 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:32.240 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:32.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:32.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:32.377 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:32.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:32.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:32.482 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:32.491 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[84576]: cluster 2024-09-13T15:17:31.209522+0000 mgr.y (mgr.26252) 1546 : cluster [DBG] pgmap v2155: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 8273/226473 objects degraded (3.653%); 1200/226473 objects misplaced (0.530%) 2024-09-13T15:17:32.491 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[84576]: cluster 2024-09-13T15:17:31.209522+0000 mgr.y (mgr.26252) 1546 : cluster [DBG] pgmap v2155: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 8273/226473 objects degraded (3.653%); 1200/226473 objects misplaced (0.530%) 2024-09-13T15:17:32.491 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[84576]: cluster 2024-09-13T15:17:31.375536+0000 osd.5 (osd.5) 20 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:32.491 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[84576]: cluster 2024-09-13T15:17:31.375536+0000 osd.5 (osd.5) 20 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:32.491 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[84576]: cluster 2024-09-13T15:17:31.661120+0000 mon.a (mon.0) 2711 : cluster [WRN] Health check update: Degraded data redundancy: 8273/226473 objects degraded (3.653%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:32.491 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[84576]: cluster 2024-09-13T15:17:31.661120+0000 mon.a (mon.0) 2711 : cluster [WRN] Health check update: Degraded data redundancy: 8273/226473 objects degraded (3.653%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:32.492 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[87147]: cluster 2024-09-13T15:17:31.209522+0000 mgr.y (mgr.26252) 1546 : cluster [DBG] pgmap v2155: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 8273/226473 objects degraded (3.653%); 1200/226473 objects misplaced (0.530%) 2024-09-13T15:17:32.492 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[87147]: cluster 2024-09-13T15:17:31.209522+0000 mgr.y (mgr.26252) 1546 : cluster [DBG] pgmap v2155: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 8273/226473 objects degraded (3.653%); 1200/226473 objects misplaced (0.530%) 2024-09-13T15:17:32.492 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[87147]: cluster 2024-09-13T15:17:31.375536+0000 osd.5 (osd.5) 20 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:32.492 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[87147]: cluster 2024-09-13T15:17:31.375536+0000 osd.5 (osd.5) 20 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:32.492 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[87147]: cluster 2024-09-13T15:17:31.661120+0000 mon.a (mon.0) 2711 : cluster [WRN] Health check update: Degraded data redundancy: 8273/226473 objects degraded (3.653%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:32.492 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:32 smithi099 bash[87147]: cluster 2024-09-13T15:17:31.661120+0000 mon.a (mon.0) 2711 : cluster [WRN] Health check update: Degraded data redundancy: 8273/226473 objects degraded (3.653%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:32.511 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:32.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:32.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:32.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:32 smithi177 bash[68594]: cluster 2024-09-13T15:17:31.209522+0000 mgr.y (mgr.26252) 1546 : cluster [DBG] pgmap v2155: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 8273/226473 objects degraded (3.653%); 1200/226473 objects misplaced (0.530%) 2024-09-13T15:17:32.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:32 smithi177 bash[68594]: cluster 2024-09-13T15:17:31.209522+0000 mgr.y (mgr.26252) 1546 : cluster [DBG] pgmap v2155: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 8273/226473 objects degraded (3.653%); 1200/226473 objects misplaced (0.530%) 2024-09-13T15:17:32.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:32 smithi177 bash[68594]: cluster 2024-09-13T15:17:31.375536+0000 osd.5 (osd.5) 20 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:32.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:32 smithi177 bash[68594]: cluster 2024-09-13T15:17:31.375536+0000 osd.5 (osd.5) 20 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:32.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:32 smithi177 bash[68594]: cluster 2024-09-13T15:17:31.661120+0000 mon.a (mon.0) 2711 : cluster [WRN] Health check update: Degraded data redundancy: 8273/226473 objects degraded (3.653%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:32.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:32 smithi177 bash[68594]: cluster 2024-09-13T15:17:31.661120+0000 mon.a (mon.0) 2711 : cluster [WRN] Health check update: Degraded data redundancy: 8273/226473 objects degraded (3.653%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:32.616 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:32.656 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:32.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:32.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:32.760 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:32.787 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:32.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:32.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:32.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:32.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:32.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:33.027 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:33.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:33.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:33.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:33.183 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:33.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:33.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:33.286 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:33.297 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:33 smithi099 bash[84576]: cluster 2024-09-13T15:17:32.362729+0000 osd.5 (osd.5) 21 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:33.298 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:33 smithi099 bash[84576]: cluster 2024-09-13T15:17:32.362729+0000 osd.5 (osd.5) 21 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:33.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:33 smithi099 bash[87147]: cluster 2024-09-13T15:17:32.362729+0000 osd.5 (osd.5) 21 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:33.298 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:33 smithi099 bash[87147]: cluster 2024-09-13T15:17:32.362729+0000 osd.5 (osd.5) 21 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:33.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:33.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:33.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:33.354 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:17:33.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:33.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:33.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:33.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:33.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:33.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:33 smithi177 bash[68594]: cluster 2024-09-13T15:17:32.362729+0000 osd.5 (osd.5) 21 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:33.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:33 smithi177 bash[68594]: cluster 2024-09-13T15:17:32.362729+0000 osd.5 (osd.5) 21 : cluster [DBG] 46.16 scrub starts 2024-09-13T15:17:33.581 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:33.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:33.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:33.691 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:33.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:33.724 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:33.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:33.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:33.852 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:33.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:33.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:33.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:33.987 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:33.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:33.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:34.090 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:34.107 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:34.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:34.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:34.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:34.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:34.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:34.352 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:34 smithi177 bash[68594]: cluster 2024-09-13T15:17:33.210193+0000 mgr.y (mgr.26252) 1547 : cluster [DBG] pgmap v2156: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 8273/224166 objects degraded (3.691%); 1200/224166 objects misplaced (0.535%) 2024-09-13T15:17:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:34 smithi177 bash[68594]: cluster 2024-09-13T15:17:33.210193+0000 mgr.y (mgr.26252) 1547 : cluster [DBG] pgmap v2156: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 8273/224166 objects degraded (3.691%); 1200/224166 objects misplaced (0.535%) 2024-09-13T15:17:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:34 smithi177 bash[68594]: audit 2024-09-13T15:17:33.317973+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:34.361 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:34 smithi177 bash[68594]: audit 2024-09-13T15:17:33.317973+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:34.378 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:34.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:34.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:34.504 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:34.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:34.522 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:34 smithi099 bash[84576]: cluster 2024-09-13T15:17:33.210193+0000 mgr.y (mgr.26252) 1547 : cluster [DBG] pgmap v2156: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 8273/224166 objects degraded (3.691%); 1200/224166 objects misplaced (0.535%) 2024-09-13T15:17:34.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:34 smithi099 bash[84576]: cluster 2024-09-13T15:17:33.210193+0000 mgr.y (mgr.26252) 1547 : cluster [DBG] pgmap v2156: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 8273/224166 objects degraded (3.691%); 1200/224166 objects misplaced (0.535%) 2024-09-13T15:17:34.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:34 smithi099 bash[84576]: audit 2024-09-13T15:17:33.317973+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:34.523 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:34 smithi099 bash[84576]: audit 2024-09-13T15:17:33.317973+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:34.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:34 smithi099 bash[87147]: cluster 2024-09-13T15:17:33.210193+0000 mgr.y (mgr.26252) 1547 : cluster [DBG] pgmap v2156: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 8273/224166 objects degraded (3.691%); 1200/224166 objects misplaced (0.535%) 2024-09-13T15:17:34.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:34 smithi099 bash[87147]: cluster 2024-09-13T15:17:33.210193+0000 mgr.y (mgr.26252) 1547 : cluster [DBG] pgmap v2156: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 8273/224166 objects degraded (3.691%); 1200/224166 objects misplaced (0.535%) 2024-09-13T15:17:34.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:34 smithi099 bash[87147]: audit 2024-09-13T15:17:33.317973+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:34.523 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:34 smithi099 bash[87147]: audit 2024-09-13T15:17:33.317973+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:34.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:34.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:34.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:34.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:34.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:34.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:34.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:34.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:34.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:34.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:34.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:34.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:35.016 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:35.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:35.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:35.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:35.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:35.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:35.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:35.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:35.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:35.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:35.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:35.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:35.389 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:17:35.391 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:17:35.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:35.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:35.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:35.518 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:35.536 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:35.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:35.595 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:35 smithi099 bash[84576]: cluster 2024-09-13T15:17:34.355347+0000 osd.5 (osd.5) 22 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:35.595 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:35 smithi099 bash[84576]: cluster 2024-09-13T15:17:34.355347+0000 osd.5 (osd.5) 22 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:35.595 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:35 smithi099 bash[84576]: cluster 2024-09-13T15:17:35.210994+0000 mgr.y (mgr.26252) 1548 : cluster [DBG] pgmap v2157: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/213423 objects degraded (3.731%); 524/213423 objects misplaced (0.246%) 2024-09-13T15:17:35.595 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:35 smithi099 bash[84576]: cluster 2024-09-13T15:17:35.210994+0000 mgr.y (mgr.26252) 1548 : cluster [DBG] pgmap v2157: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/213423 objects degraded (3.731%); 524/213423 objects misplaced (0.246%) 2024-09-13T15:17:35.596 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:35 smithi099 bash[87147]: cluster 2024-09-13T15:17:34.355347+0000 osd.5 (osd.5) 22 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:35.596 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:35 smithi099 bash[87147]: cluster 2024-09-13T15:17:34.355347+0000 osd.5 (osd.5) 22 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:35.596 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:35 smithi099 bash[87147]: cluster 2024-09-13T15:17:35.210994+0000 mgr.y (mgr.26252) 1548 : cluster [DBG] pgmap v2157: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/213423 objects degraded (3.731%); 524/213423 objects misplaced (0.246%) 2024-09-13T15:17:35.596 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:35 smithi099 bash[87147]: cluster 2024-09-13T15:17:35.210994+0000 mgr.y (mgr.26252) 1548 : cluster [DBG] pgmap v2157: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/213423 objects degraded (3.731%); 524/213423 objects misplaced (0.246%) 2024-09-13T15:17:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:35 smithi177 bash[68594]: cluster 2024-09-13T15:17:34.355347+0000 osd.5 (osd.5) 22 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:35 smithi177 bash[68594]: cluster 2024-09-13T15:17:34.355347+0000 osd.5 (osd.5) 22 : cluster [DBG] 46.6 scrub starts 2024-09-13T15:17:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:35 smithi177 bash[68594]: cluster 2024-09-13T15:17:35.210994+0000 mgr.y (mgr.26252) 1548 : cluster [DBG] pgmap v2157: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/213423 objects degraded (3.731%); 524/213423 objects misplaced (0.246%) 2024-09-13T15:17:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:35 smithi177 bash[68594]: cluster 2024-09-13T15:17:35.210994+0000 mgr.y (mgr.26252) 1548 : cluster [DBG] pgmap v2157: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/213423 objects degraded (3.731%); 524/213423 objects misplaced (0.246%) 2024-09-13T15:17:35.638 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:35.679 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:35.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:35.787 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:35.812 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:35.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:35.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:35.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:35.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:35.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:35.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:36.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:36.085 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:36.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:36.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:36.189 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:36.214 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:36.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:36.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:36.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:36.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:36.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:36.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:36.448 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:36.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:36.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:36.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:36.565 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:17:36.569 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:36.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:36.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:36.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:36.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:36.731 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:36.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:36.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:36 smithi099 bash[84576]: cluster 2024-09-13T15:17:36.661944+0000 mon.a (mon.0) 2713 : cluster [WRN] Health check update: Degraded data redundancy: 7963/213423 objects degraded (3.731%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:36.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:36 smithi099 bash[84576]: cluster 2024-09-13T15:17:36.661944+0000 mon.a (mon.0) 2713 : cluster [WRN] Health check update: Degraded data redundancy: 7963/213423 objects degraded (3.731%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:36.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:36 smithi099 bash[87147]: cluster 2024-09-13T15:17:36.661944+0000 mon.a (mon.0) 2713 : cluster [WRN] Health check update: Degraded data redundancy: 7963/213423 objects degraded (3.731%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:36.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:36 smithi099 bash[87147]: cluster 2024-09-13T15:17:36.661944+0000 mon.a (mon.0) 2713 : cluster [WRN] Health check update: Degraded data redundancy: 7963/213423 objects degraded (3.731%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:36.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:36.858 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:36.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:36.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:36.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:36 smithi177 bash[68594]: cluster 2024-09-13T15:17:36.661944+0000 mon.a (mon.0) 2713 : cluster [WRN] Health check update: Degraded data redundancy: 7963/213423 objects degraded (3.731%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:36.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:36 smithi177 bash[68594]: cluster 2024-09-13T15:17:36.661944+0000 mon.a (mon.0) 2713 : cluster [WRN] Health check update: Degraded data redundancy: 7963/213423 objects degraded (3.731%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:36.961 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:36.992 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:36.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:37.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:37.107 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:17:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:17:37] "GET /metrics HTTP/1.1" 200 36785 "" "Prometheus/2.51.0" 2024-09-13T15:17:37.123 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:37.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:37.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:37.227 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:37.254 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:37.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:37.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:37.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:37.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:37.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:37.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:37.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:37.527 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:37.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:37.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:37.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:37.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:37.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:37.756 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:37.781 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:37.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:37.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:37.909 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:37.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:37.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:38.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:38.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:37 smithi099 bash[84576]: cluster 2024-09-13T15:17:37.211672+0000 mgr.y (mgr.26252) 1549 : cluster [DBG] pgmap v2158: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/211698 objects degraded (3.761%); 524/211698 objects misplaced (0.248%) 2024-09-13T15:17:38.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:37 smithi099 bash[84576]: cluster 2024-09-13T15:17:37.211672+0000 mgr.y (mgr.26252) 1549 : cluster [DBG] pgmap v2158: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/211698 objects degraded (3.761%); 524/211698 objects misplaced (0.248%) 2024-09-13T15:17:38.023 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:37 smithi099 bash[87147]: cluster 2024-09-13T15:17:37.211672+0000 mgr.y (mgr.26252) 1549 : cluster [DBG] pgmap v2158: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/211698 objects degraded (3.761%); 524/211698 objects misplaced (0.248%) 2024-09-13T15:17:38.023 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:37 smithi099 bash[87147]: cluster 2024-09-13T15:17:37.211672+0000 mgr.y (mgr.26252) 1549 : cluster [DBG] pgmap v2158: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/211698 objects degraded (3.761%); 524/211698 objects misplaced (0.248%) 2024-09-13T15:17:38.035 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:38.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:38.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:38.138 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:38.158 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:38.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:38.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:38.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:38.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:38.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:38.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:38.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:38 smithi177 bash[68594]: cluster 2024-09-13T15:17:37.211672+0000 mgr.y (mgr.26252) 1549 : cluster [DBG] pgmap v2158: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/211698 objects degraded (3.761%); 524/211698 objects misplaced (0.248%) 2024-09-13T15:17:38.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:38 smithi177 bash[68594]: cluster 2024-09-13T15:17:37.211672+0000 mgr.y (mgr.26252) 1549 : cluster [DBG] pgmap v2158: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7963/211698 objects degraded (3.761%); 524/211698 objects misplaced (0.248%) 2024-09-13T15:17:38.388 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:38.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:38.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:38.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:38.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:38.555 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:38.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:38.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:38.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:38.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:38.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:38.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:38.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:38.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:38.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:38.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:38.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:38.920 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:38 smithi177 bash[68594]: cluster 2024-09-13T15:17:37.737318+0000 mon.a (mon.0) 2714 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2024-09-13T15:17:38.920 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:38 smithi177 bash[68594]: cluster 2024-09-13T15:17:37.737318+0000 mon.a (mon.0) 2714 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2024-09-13T15:17:38.941 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:38.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:38.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:39.046 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:39.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:39.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:39.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:39.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:38 smithi099 bash[84576]: cluster 2024-09-13T15:17:37.737318+0000 mon.a (mon.0) 2714 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2024-09-13T15:17:39.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:38 smithi099 bash[84576]: cluster 2024-09-13T15:17:37.737318+0000 mon.a (mon.0) 2714 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2024-09-13T15:17:39.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:38 smithi099 bash[87147]: cluster 2024-09-13T15:17:37.737318+0000 mon.a (mon.0) 2714 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2024-09-13T15:17:39.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:38 smithi099 bash[87147]: cluster 2024-09-13T15:17:37.737318+0000 mon.a (mon.0) 2714 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2024-09-13T15:17:39.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:39.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:39.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:39.304 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:39.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:39.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:39.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:39.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:39.448 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:39.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:39.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:39.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:39.576 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:39.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:39.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:39.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:39.705 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:39.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:39.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:39.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:39.835 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:39.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:39.940 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:39.950 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:39 smithi177 bash[68594]: cluster 2024-09-13T15:17:38.752962+0000 mon.a (mon.0) 2715 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2024-09-13T15:17:39.950 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:39 smithi177 bash[68594]: cluster 2024-09-13T15:17:38.752962+0000 mon.a (mon.0) 2715 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2024-09-13T15:17:39.950 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:39 smithi177 bash[68594]: cluster 2024-09-13T15:17:39.212258+0000 mgr.y (mgr.26252) 1550 : cluster [DBG] pgmap v2161: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 5459/203169 objects degraded (2.687%); 2019/203169 objects misplaced (0.994%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:39.950 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:39 smithi177 bash[68594]: cluster 2024-09-13T15:17:39.212258+0000 mgr.y (mgr.26252) 1550 : cluster [DBG] pgmap v2161: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 5459/203169 objects degraded (2.687%); 2019/203169 objects misplaced (0.994%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:39.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:39.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:40.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:39 smithi099 bash[84576]: cluster 2024-09-13T15:17:38.752962+0000 mon.a (mon.0) 2715 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2024-09-13T15:17:40.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:39 smithi099 bash[84576]: cluster 2024-09-13T15:17:38.752962+0000 mon.a (mon.0) 2715 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2024-09-13T15:17:40.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:39 smithi099 bash[84576]: cluster 2024-09-13T15:17:39.212258+0000 mgr.y (mgr.26252) 1550 : cluster [DBG] pgmap v2161: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 5459/203169 objects degraded (2.687%); 2019/203169 objects misplaced (0.994%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:40.039 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:39 smithi099 bash[84576]: cluster 2024-09-13T15:17:39.212258+0000 mgr.y (mgr.26252) 1550 : cluster [DBG] pgmap v2161: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 5459/203169 objects degraded (2.687%); 2019/203169 objects misplaced (0.994%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:40.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:39 smithi099 bash[87147]: cluster 2024-09-13T15:17:38.752962+0000 mon.a (mon.0) 2715 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2024-09-13T15:17:40.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:39 smithi099 bash[87147]: cluster 2024-09-13T15:17:38.752962+0000 mon.a (mon.0) 2715 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2024-09-13T15:17:40.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:39 smithi099 bash[87147]: cluster 2024-09-13T15:17:39.212258+0000 mgr.y (mgr.26252) 1550 : cluster [DBG] pgmap v2161: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 5459/203169 objects degraded (2.687%); 2019/203169 objects misplaced (0.994%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:40.040 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:39 smithi099 bash[87147]: cluster 2024-09-13T15:17:39.212258+0000 mgr.y (mgr.26252) 1550 : cluster [DBG] pgmap v2161: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 970 op/s; 5459/203169 objects degraded (2.687%); 2019/203169 objects misplaced (0.994%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:40.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:40.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:40.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:40.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:40.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:40.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:40.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:40.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:40.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:40.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:40.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:40.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:40.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:40.486 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:17:40 smithi177 bash[61439]: ts=2024-09-13T15:17:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:17:40.508 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:40.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:40.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:40.648 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:40.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:40.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:40.752 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:40.782 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:40.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:40.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:40.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:40.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:40.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:40.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:41.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:40 smithi177 bash[68594]: cluster 2024-09-13T15:17:39.310788+0000 osd.6 (osd.6) 15 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:40 smithi177 bash[68594]: cluster 2024-09-13T15:17:39.310788+0000 osd.6 (osd.6) 15 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:41.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:41.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:41.151 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:41.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:41.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:41.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:41.280 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:41.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:41.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:41.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:41.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:41.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:41.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:41.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:41.542 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:41.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:41.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:41.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[84576]: cluster 2024-09-13T15:17:39.310788+0000 osd.6 (osd.6) 15 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:41.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[84576]: cluster 2024-09-13T15:17:39.310788+0000 osd.6 (osd.6) 15 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:41.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[87147]: cluster 2024-09-13T15:17:39.310788+0000 osd.6 (osd.6) 15 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:41.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[87147]: cluster 2024-09-13T15:17:39.310788+0000 osd.6 (osd.6) 15 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:41.675 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:41.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:41.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:41.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:41.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:41.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:41.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:41.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:41.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:41.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:41.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:42.071 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[84576]: cluster 2024-09-13T15:17:41.212925+0000 mgr.y (mgr.26252) 1551 : cluster [DBG] pgmap v2162: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 977 op/s; 5459/200703 objects degraded (2.720%); 2019/200703 objects misplaced (1.006%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[84576]: cluster 2024-09-13T15:17:41.212925+0000 mgr.y (mgr.26252) 1551 : cluster [DBG] pgmap v2162: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 977 op/s; 5459/200703 objects degraded (2.720%); 2019/200703 objects misplaced (1.006%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[84576]: cluster 2024-09-13T15:17:41.662737+0000 mon.a (mon.0) 2716 : cluster [WRN] Health check update: Degraded data redundancy: 5459/203169 objects degraded (2.687%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:42.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[84576]: cluster 2024-09-13T15:17:41.662737+0000 mon.a (mon.0) 2716 : cluster [WRN] Health check update: Degraded data redundancy: 5459/203169 objects degraded (2.687%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[87147]: cluster 2024-09-13T15:17:41.212925+0000 mgr.y (mgr.26252) 1551 : cluster [DBG] pgmap v2162: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 977 op/s; 5459/200703 objects degraded (2.720%); 2019/200703 objects misplaced (1.006%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[87147]: cluster 2024-09-13T15:17:41.212925+0000 mgr.y (mgr.26252) 1551 : cluster [DBG] pgmap v2162: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 977 op/s; 5459/200703 objects degraded (2.720%); 2019/200703 objects misplaced (1.006%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[87147]: cluster 2024-09-13T15:17:41.662737+0000 mon.a (mon.0) 2716 : cluster [WRN] Health check update: Degraded data redundancy: 5459/203169 objects degraded (2.687%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:42.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:41 smithi099 bash[87147]: cluster 2024-09-13T15:17:41.662737+0000 mon.a (mon.0) 2716 : cluster [WRN] Health check update: Degraded data redundancy: 5459/203169 objects degraded (2.687%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:42.079 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:41 smithi177 bash[68594]: cluster 2024-09-13T15:17:41.212925+0000 mgr.y (mgr.26252) 1551 : cluster [DBG] pgmap v2162: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 977 op/s; 5459/200703 objects degraded (2.720%); 2019/200703 objects misplaced (1.006%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:42.079 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:41 smithi177 bash[68594]: cluster 2024-09-13T15:17:41.212925+0000 mgr.y (mgr.26252) 1551 : cluster [DBG] pgmap v2162: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 977 op/s; 5459/200703 objects degraded (2.720%); 2019/200703 objects misplaced (1.006%); 7.7 MiB/s, 122 objects/s recovering 2024-09-13T15:17:42.079 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:41 smithi177 bash[68594]: cluster 2024-09-13T15:17:41.662737+0000 mon.a (mon.0) 2716 : cluster [WRN] Health check update: Degraded data redundancy: 5459/203169 objects degraded (2.687%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:42.079 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:41 smithi177 bash[68594]: cluster 2024-09-13T15:17:41.662737+0000 mon.a (mon.0) 2716 : cluster [WRN] Health check update: Degraded data redundancy: 5459/203169 objects degraded (2.687%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:42.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:42.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:42.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:42.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:42.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:42.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:42.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:42.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:42.360 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:42.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:42.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:42.463 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:42.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:42.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:42.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:42.617 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:42.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:42.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:42.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:42.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:42.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:42.766 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:17:42.766 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:17:42.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:42.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:42.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:42.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:42.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:43.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:43.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:43.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:43.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:43.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:43.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:43.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:43.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:43.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:43.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:43.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:43.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:43.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:43.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:43.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:43.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:43.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:43.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:43.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:43.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:43.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:43.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:43.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:43.778 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:43.808 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:43.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:43.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:43.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:43.946 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:43.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:43.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:43.979 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:17:44.052 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:44.092 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:44.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:44.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:44.196 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:44.238 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:44.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:44.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:44.341 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:44.372 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:44.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:44.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:44.476 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:44.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:44.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:44.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:44.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:44 smithi099 bash[84576]: cluster 2024-09-13T15:17:43.213478+0000 mgr.y (mgr.26252) 1552 : cluster [DBG] pgmap v2163: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5459/199206 objects degraded (2.740%); 2019/199206 objects misplaced (1.014%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T15:17:44.573 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:44 smithi099 bash[84576]: cluster 2024-09-13T15:17:43.213478+0000 mgr.y (mgr.26252) 1552 : cluster [DBG] pgmap v2163: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5459/199206 objects degraded (2.740%); 2019/199206 objects misplaced (1.014%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T15:17:44.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:44 smithi099 bash[87147]: cluster 2024-09-13T15:17:43.213478+0000 mgr.y (mgr.26252) 1552 : cluster [DBG] pgmap v2163: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5459/199206 objects degraded (2.740%); 2019/199206 objects misplaced (1.014%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T15:17:44.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:44 smithi099 bash[87147]: cluster 2024-09-13T15:17:43.213478+0000 mgr.y (mgr.26252) 1552 : cluster [DBG] pgmap v2163: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5459/199206 objects degraded (2.740%); 2019/199206 objects misplaced (1.014%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T15:17:44.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:44.611 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:44 smithi177 bash[68594]: cluster 2024-09-13T15:17:43.213478+0000 mgr.y (mgr.26252) 1552 : cluster [DBG] pgmap v2163: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5459/199206 objects degraded (2.740%); 2019/199206 objects misplaced (1.014%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T15:17:44.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:44 smithi177 bash[68594]: cluster 2024-09-13T15:17:43.213478+0000 mgr.y (mgr.26252) 1552 : cluster [DBG] pgmap v2163: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 5459/199206 objects degraded (2.740%); 2019/199206 objects misplaced (1.014%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T15:17:44.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:44.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:44.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:44.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:44.759 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:44.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:44.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:44.863 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:44.891 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:44.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:44.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:45.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:45.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:45.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:45.144 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:45.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:45.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:45.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:45.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:45.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:45.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:45.385 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:45.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:45.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:45.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:45.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:45.530 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:45.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:45.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:45.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:45 smithi177 bash[68594]: cluster 2024-09-13T15:17:43.316919+0000 osd.6 (osd.6) 16 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:17:45.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:45 smithi177 bash[68594]: cluster 2024-09-13T15:17:43.316919+0000 osd.6 (osd.6) 16 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:17:45.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:45 smithi177 bash[68594]: cluster 2024-09-13T15:17:43.978542+0000 osd.6 (osd.6) 17 : cluster [DBG] 46.12 scrub ok 2024-09-13T15:17:45.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:45 smithi177 bash[68594]: cluster 2024-09-13T15:17:43.978542+0000 osd.6 (osd.6) 17 : cluster [DBG] 46.12 scrub ok 2024-09-13T15:17:45.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:45 smithi177 bash[68594]: cluster 2024-09-13T15:17:45.214211+0000 mgr.y (mgr.26252) 1553 : cluster [DBG] pgmap v2164: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 5167/190191 objects degraded (2.717%); 1896/190191 objects misplaced (0.997%) 2024-09-13T15:17:45.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:45 smithi177 bash[68594]: cluster 2024-09-13T15:17:45.214211+0000 mgr.y (mgr.26252) 1553 : cluster [DBG] pgmap v2164: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 5167/190191 objects degraded (2.717%); 1896/190191 objects misplaced (0.997%) 2024-09-13T15:17:45.633 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:45.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:45.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:45.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:45.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:45.794 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:45.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:45.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:45.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[84576]: cluster 2024-09-13T15:17:43.316919+0000 osd.6 (osd.6) 16 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:17:45.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[84576]: cluster 2024-09-13T15:17:43.316919+0000 osd.6 (osd.6) 16 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:17:45.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[84576]: cluster 2024-09-13T15:17:43.978542+0000 osd.6 (osd.6) 17 : cluster [DBG] 46.12 scrub ok 2024-09-13T15:17:45.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[84576]: cluster 2024-09-13T15:17:43.978542+0000 osd.6 (osd.6) 17 : cluster [DBG] 46.12 scrub ok 2024-09-13T15:17:45.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[84576]: cluster 2024-09-13T15:17:45.214211+0000 mgr.y (mgr.26252) 1553 : cluster [DBG] pgmap v2164: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 5167/190191 objects degraded (2.717%); 1896/190191 objects misplaced (0.997%) 2024-09-13T15:17:45.911 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[84576]: cluster 2024-09-13T15:17:45.214211+0000 mgr.y (mgr.26252) 1553 : cluster [DBG] pgmap v2164: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 5167/190191 objects degraded (2.717%); 1896/190191 objects misplaced (0.997%) 2024-09-13T15:17:45.911 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[87147]: cluster 2024-09-13T15:17:43.316919+0000 osd.6 (osd.6) 16 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:17:45.911 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[87147]: cluster 2024-09-13T15:17:43.316919+0000 osd.6 (osd.6) 16 : cluster [DBG] 46.12 scrub starts 2024-09-13T15:17:45.911 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[87147]: cluster 2024-09-13T15:17:43.978542+0000 osd.6 (osd.6) 17 : cluster [DBG] 46.12 scrub ok 2024-09-13T15:17:45.912 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[87147]: cluster 2024-09-13T15:17:43.978542+0000 osd.6 (osd.6) 17 : cluster [DBG] 46.12 scrub ok 2024-09-13T15:17:45.912 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[87147]: cluster 2024-09-13T15:17:45.214211+0000 mgr.y (mgr.26252) 1553 : cluster [DBG] pgmap v2164: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 5167/190191 objects degraded (2.717%); 1896/190191 objects misplaced (0.997%) 2024-09-13T15:17:45.912 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:45 smithi099 bash[87147]: cluster 2024-09-13T15:17:45.214211+0000 mgr.y (mgr.26252) 1553 : cluster [DBG] pgmap v2164: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 5167/190191 objects degraded (2.717%); 1896/190191 objects misplaced (0.997%) 2024-09-13T15:17:45.946 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:45.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:45.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:46.051 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:46.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:46.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:46.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:46.179 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:46.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:46.212 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:46.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:46.313 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:46.354 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:46.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:46.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:46.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:46.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:46.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:46.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:46.635 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:46.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:46.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:46.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:46.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:46.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:46.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:46.885 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:46 smithi177 bash[68594]: cluster 2024-09-13T15:17:46.663525+0000 mon.a (mon.0) 2717 : cluster [WRN] Health check update: Degraded data redundancy: 5167/190191 objects degraded (2.717%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:46.885 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:46 smithi177 bash[68594]: cluster 2024-09-13T15:17:46.663525+0000 mon.a (mon.0) 2717 : cluster [WRN] Health check update: Degraded data redundancy: 5167/190191 objects degraded (2.717%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:46.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:46.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:46.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:47.009 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:47.028 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:47.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:47.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:47.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:46 smithi099 bash[84576]: cluster 2024-09-13T15:17:46.663525+0000 mon.a (mon.0) 2717 : cluster [WRN] Health check update: Degraded data redundancy: 5167/190191 objects degraded (2.717%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:47.050 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:46 smithi099 bash[84576]: cluster 2024-09-13T15:17:46.663525+0000 mon.a (mon.0) 2717 : cluster [WRN] Health check update: Degraded data redundancy: 5167/190191 objects degraded (2.717%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:47.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:46 smithi099 bash[87147]: cluster 2024-09-13T15:17:46.663525+0000 mon.a (mon.0) 2717 : cluster [WRN] Health check update: Degraded data redundancy: 5167/190191 objects degraded (2.717%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:47.051 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:46 smithi099 bash[87147]: cluster 2024-09-13T15:17:46.663525+0000 mon.a (mon.0) 2717 : cluster [WRN] Health check update: Degraded data redundancy: 5167/190191 objects degraded (2.717%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:47.131 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:47.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:47.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:47.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:47.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:47.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:47.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:47.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:17:47] "GET /metrics HTTP/1.1" 200 36785 "" "Prometheus/2.51.0" 2024-09-13T15:17:47.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:47.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:47.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:47.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:47.510 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:47.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:47.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:47.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:47.643 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:47.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:47.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:47.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:47.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:47.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:47.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:47.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:47.919 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:47 smithi177 bash[68594]: cluster 2024-09-13T15:17:46.363731+0000 osd.6 (osd.6) 18 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:17:47.919 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:47 smithi177 bash[68594]: cluster 2024-09-13T15:17:46.363731+0000 osd.6 (osd.6) 18 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:17:47.919 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:47 smithi177 bash[68594]: cluster 2024-09-13T15:17:46.975969+0000 osd.6 (osd.6) 19 : cluster [DBG] 46.10 scrub ok 2024-09-13T15:17:47.919 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:47 smithi177 bash[68594]: cluster 2024-09-13T15:17:46.975969+0000 osd.6 (osd.6) 19 : cluster [DBG] 46.10 scrub ok 2024-09-13T15:17:47.919 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:47 smithi177 bash[68594]: cluster 2024-09-13T15:17:47.214899+0000 mgr.y (mgr.26252) 1554 : cluster [DBG] pgmap v2165: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 357 op/s; 5167/188463 objects degraded (2.742%); 1896/188463 objects misplaced (1.006%) 2024-09-13T15:17:47.919 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:47 smithi177 bash[68594]: cluster 2024-09-13T15:17:47.214899+0000 mgr.y (mgr.26252) 1554 : cluster [DBG] pgmap v2165: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 357 op/s; 5167/188463 objects degraded (2.742%); 1896/188463 objects misplaced (1.006%) 2024-09-13T15:17:47.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:47.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:48.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:48.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:48.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:48.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[84576]: cluster 2024-09-13T15:17:46.363731+0000 osd.6 (osd.6) 18 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:17:48.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[84576]: cluster 2024-09-13T15:17:46.363731+0000 osd.6 (osd.6) 18 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:17:48.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[84576]: cluster 2024-09-13T15:17:46.975969+0000 osd.6 (osd.6) 19 : cluster [DBG] 46.10 scrub ok 2024-09-13T15:17:48.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[84576]: cluster 2024-09-13T15:17:46.975969+0000 osd.6 (osd.6) 19 : cluster [DBG] 46.10 scrub ok 2024-09-13T15:17:48.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[84576]: cluster 2024-09-13T15:17:47.214899+0000 mgr.y (mgr.26252) 1554 : cluster [DBG] pgmap v2165: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 357 op/s; 5167/188463 objects degraded (2.742%); 1896/188463 objects misplaced (1.006%) 2024-09-13T15:17:48.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[84576]: cluster 2024-09-13T15:17:47.214899+0000 mgr.y (mgr.26252) 1554 : cluster [DBG] pgmap v2165: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 357 op/s; 5167/188463 objects degraded (2.742%); 1896/188463 objects misplaced (1.006%) 2024-09-13T15:17:48.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[87147]: cluster 2024-09-13T15:17:46.363731+0000 osd.6 (osd.6) 18 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:17:48.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[87147]: cluster 2024-09-13T15:17:46.363731+0000 osd.6 (osd.6) 18 : cluster [DBG] 46.10 scrub starts 2024-09-13T15:17:48.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[87147]: cluster 2024-09-13T15:17:46.975969+0000 osd.6 (osd.6) 19 : cluster [DBG] 46.10 scrub ok 2024-09-13T15:17:48.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[87147]: cluster 2024-09-13T15:17:46.975969+0000 osd.6 (osd.6) 19 : cluster [DBG] 46.10 scrub ok 2024-09-13T15:17:48.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[87147]: cluster 2024-09-13T15:17:47.214899+0000 mgr.y (mgr.26252) 1554 : cluster [DBG] pgmap v2165: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 357 op/s; 5167/188463 objects degraded (2.742%); 1896/188463 objects misplaced (1.006%) 2024-09-13T15:17:48.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:47 smithi099 bash[87147]: cluster 2024-09-13T15:17:47.214899+0000 mgr.y (mgr.26252) 1554 : cluster [DBG] pgmap v2165: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 357 op/s; 5167/188463 objects degraded (2.742%); 1896/188463 objects misplaced (1.006%) 2024-09-13T15:17:48.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:48.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:48.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:48.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:48.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:48.332 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:48.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:48.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:48.435 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:48.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:48.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:48.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:48.561 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:48.587 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:48.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:48.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:48.691 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:48.717 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:48.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:48.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:48.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:48.842 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:17:48.842 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:48.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:48.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:48.946 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:48.958 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:48 smithi177 bash[68594]: audit 2024-09-13T15:17:48.320102+0000 mon.a (mon.0) 2718 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:48.958 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:48 smithi177 bash[68594]: audit 2024-09-13T15:17:48.320102+0000 mon.a (mon.0) 2718 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:48.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:48.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:49.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:48 smithi099 bash[84576]: audit 2024-09-13T15:17:48.320102+0000 mon.a (mon.0) 2718 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:49.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:48 smithi099 bash[84576]: audit 2024-09-13T15:17:48.320102+0000 mon.a (mon.0) 2718 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:49.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:48 smithi099 bash[87147]: audit 2024-09-13T15:17:48.320102+0000 mon.a (mon.0) 2718 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:49.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:48 smithi099 bash[87147]: audit 2024-09-13T15:17:48.320102+0000 mon.a (mon.0) 2718 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:17:49.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:49.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:49.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:49.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:49.219 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:49.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:49.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:49.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:49.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:49.375 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:49.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:49.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:49.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:49.502 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:49.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:49.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:49.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:49.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:49.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:49.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:49.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:49.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:49.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:49.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:49.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:49.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:49.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:49.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:50.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:50.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:50.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:50.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:49 smithi099 bash[87147]: cluster 2024-09-13T15:17:48.290378+0000 osd.6 (osd.6) 20 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:50.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:49 smithi099 bash[87147]: cluster 2024-09-13T15:17:48.290378+0000 osd.6 (osd.6) 20 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:50.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:49 smithi099 bash[87147]: cluster 2024-09-13T15:17:49.215511+0000 mgr.y (mgr.26252) 1555 : cluster [DBG] pgmap v2166: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 5167/182676 objects degraded (2.829%); 1896/182676 objects misplaced (1.038%) 2024-09-13T15:17:50.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:49 smithi099 bash[87147]: cluster 2024-09-13T15:17:49.215511+0000 mgr.y (mgr.26252) 1555 : cluster [DBG] pgmap v2166: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 5167/182676 objects degraded (2.829%); 1896/182676 objects misplaced (1.038%) 2024-09-13T15:17:50.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:49 smithi099 bash[84576]: cluster 2024-09-13T15:17:48.290378+0000 osd.6 (osd.6) 20 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:50.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:49 smithi099 bash[84576]: cluster 2024-09-13T15:17:48.290378+0000 osd.6 (osd.6) 20 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:50.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:49 smithi099 bash[84576]: cluster 2024-09-13T15:17:49.215511+0000 mgr.y (mgr.26252) 1555 : cluster [DBG] pgmap v2166: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 5167/182676 objects degraded (2.829%); 1896/182676 objects misplaced (1.038%) 2024-09-13T15:17:50.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:49 smithi099 bash[84576]: cluster 2024-09-13T15:17:49.215511+0000 mgr.y (mgr.26252) 1555 : cluster [DBG] pgmap v2166: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 5167/182676 objects degraded (2.829%); 1896/182676 objects misplaced (1.038%) 2024-09-13T15:17:50.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:50.139 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:50.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:50.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:50.242 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:50.268 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:50.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:50.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:50.370 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:50.390 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:49 smithi177 bash[68594]: cluster 2024-09-13T15:17:48.290378+0000 osd.6 (osd.6) 20 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:50.390 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:49 smithi177 bash[68594]: cluster 2024-09-13T15:17:48.290378+0000 osd.6 (osd.6) 20 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:50.390 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:49 smithi177 bash[68594]: cluster 2024-09-13T15:17:49.215511+0000 mgr.y (mgr.26252) 1555 : cluster [DBG] pgmap v2166: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 5167/182676 objects degraded (2.829%); 1896/182676 objects misplaced (1.038%) 2024-09-13T15:17:50.390 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:49 smithi177 bash[68594]: cluster 2024-09-13T15:17:49.215511+0000 mgr.y (mgr.26252) 1555 : cluster [DBG] pgmap v2166: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 5167/182676 objects degraded (2.829%); 1896/182676 objects misplaced (1.038%) 2024-09-13T15:17:50.396 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:50.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:50.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:50.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:50.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:50.533 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:50.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:50.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:50.643 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:17:50 smithi177 bash[61439]: ts=2024-09-13T15:17:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:17:50.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:50.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:50.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:50.760 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:50.794 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:50.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:50.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:50.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:50.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:50 smithi177 bash[68594]: cluster 2024-09-13T15:17:49.278681+0000 osd.6 (osd.6) 21 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:50.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:50 smithi177 bash[68594]: cluster 2024-09-13T15:17:49.278681+0000 osd.6 (osd.6) 21 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:50.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:50 smithi177 bash[68594]: cluster 2024-09-13T15:17:50.246874+0000 osd.6 (osd.6) 22 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:50.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:50 smithi177 bash[68594]: cluster 2024-09-13T15:17:50.246874+0000 osd.6 (osd.6) 22 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:50.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:50.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:50.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:51.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:51.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:51.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:51.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:51.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:51.162 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:50 smithi099 bash[84576]: cluster 2024-09-13T15:17:49.278681+0000 osd.6 (osd.6) 21 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:51.162 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:50 smithi099 bash[84576]: cluster 2024-09-13T15:17:49.278681+0000 osd.6 (osd.6) 21 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:51.162 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:50 smithi099 bash[84576]: cluster 2024-09-13T15:17:50.246874+0000 osd.6 (osd.6) 22 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:51.162 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:50 smithi099 bash[84576]: cluster 2024-09-13T15:17:50.246874+0000 osd.6 (osd.6) 22 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:51.162 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:50 smithi099 bash[87147]: cluster 2024-09-13T15:17:49.278681+0000 osd.6 (osd.6) 21 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:51.162 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:50 smithi099 bash[87147]: cluster 2024-09-13T15:17:49.278681+0000 osd.6 (osd.6) 21 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:51.162 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:50 smithi099 bash[87147]: cluster 2024-09-13T15:17:50.246874+0000 osd.6 (osd.6) 22 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:51.162 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:50 smithi099 bash[87147]: cluster 2024-09-13T15:17:50.246874+0000 osd.6 (osd.6) 22 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:51.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:51.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:51.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:51.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:51.327 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:51.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:51.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:51.434 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:51.467 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:51.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:51.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:51.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:51.614 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:51.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:51.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:51.720 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:51.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:51.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:51.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:51.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:51 smithi177 bash[68594]: cluster 2024-09-13T15:17:51.216176+0000 mgr.y (mgr.26252) 1556 : cluster [DBG] pgmap v2167: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/176733 objects degraded (2.795%); 1723/176733 objects misplaced (0.975%) 2024-09-13T15:17:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:51 smithi177 bash[68594]: cluster 2024-09-13T15:17:51.216176+0000 mgr.y (mgr.26252) 1556 : cluster [DBG] pgmap v2167: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/176733 objects degraded (2.795%); 1723/176733 objects misplaced (0.975%) 2024-09-13T15:17:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:51 smithi177 bash[68594]: cluster 2024-09-13T15:17:51.664605+0000 mon.a (mon.0) 2719 : cluster [WRN] Health check update: Degraded data redundancy: 5167/182676 objects degraded (2.829%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:51 smithi177 bash[68594]: cluster 2024-09-13T15:17:51.664605+0000 mon.a (mon.0) 2719 : cluster [WRN] Health check update: Degraded data redundancy: 5167/182676 objects degraded (2.829%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:51.872 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:51.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:51.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:51.977 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:52.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:52.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:52.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:52.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:51 smithi099 bash[84576]: cluster 2024-09-13T15:17:51.216176+0000 mgr.y (mgr.26252) 1556 : cluster [DBG] pgmap v2167: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/176733 objects degraded (2.795%); 1723/176733 objects misplaced (0.975%) 2024-09-13T15:17:52.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:51 smithi099 bash[84576]: cluster 2024-09-13T15:17:51.216176+0000 mgr.y (mgr.26252) 1556 : cluster [DBG] pgmap v2167: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/176733 objects degraded (2.795%); 1723/176733 objects misplaced (0.975%) 2024-09-13T15:17:52.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:51 smithi099 bash[84576]: cluster 2024-09-13T15:17:51.664605+0000 mon.a (mon.0) 2719 : cluster [WRN] Health check update: Degraded data redundancy: 5167/182676 objects degraded (2.829%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:52.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:51 smithi099 bash[84576]: cluster 2024-09-13T15:17:51.664605+0000 mon.a (mon.0) 2719 : cluster [WRN] Health check update: Degraded data redundancy: 5167/182676 objects degraded (2.829%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:52.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:51 smithi099 bash[87147]: cluster 2024-09-13T15:17:51.216176+0000 mgr.y (mgr.26252) 1556 : cluster [DBG] pgmap v2167: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/176733 objects degraded (2.795%); 1723/176733 objects misplaced (0.975%) 2024-09-13T15:17:52.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:51 smithi099 bash[87147]: cluster 2024-09-13T15:17:51.216176+0000 mgr.y (mgr.26252) 1556 : cluster [DBG] pgmap v2167: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/176733 objects degraded (2.795%); 1723/176733 objects misplaced (0.975%) 2024-09-13T15:17:52.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:51 smithi099 bash[87147]: cluster 2024-09-13T15:17:51.664605+0000 mon.a (mon.0) 2719 : cluster [WRN] Health check update: Degraded data redundancy: 5167/182676 objects degraded (2.829%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:52.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:51 smithi099 bash[87147]: cluster 2024-09-13T15:17:51.664605+0000 mon.a (mon.0) 2719 : cluster [WRN] Health check update: Degraded data redundancy: 5167/182676 objects degraded (2.829%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:52.103 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:52.129 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:52.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:52.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:52.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:52.260 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:52.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:52.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:52.363 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:52.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:52.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:52.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:52.493 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:52.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:52.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:52.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:52.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:52.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:52.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:52.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:52.782 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:52.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:52.889 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:52.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:52.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:52.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:53.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:53.065 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:53.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:53.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:52 smithi177 bash[68594]: cluster 2024-09-13T15:17:52.252039+0000 osd.6 (osd.6) 23 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:53.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:52 smithi177 bash[68594]: cluster 2024-09-13T15:17:52.252039+0000 osd.6 (osd.6) 23 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:53.167 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:52 smithi099 bash[84576]: cluster 2024-09-13T15:17:52.252039+0000 osd.6 (osd.6) 23 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:53.167 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:52 smithi099 bash[84576]: cluster 2024-09-13T15:17:52.252039+0000 osd.6 (osd.6) 23 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:53.168 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:52 smithi099 bash[87147]: cluster 2024-09-13T15:17:52.252039+0000 osd.6 (osd.6) 23 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:53.168 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:52 smithi099 bash[87147]: cluster 2024-09-13T15:17:52.252039+0000 osd.6 (osd.6) 23 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:53.170 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:53.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:53.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:53.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:53.298 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:53.321 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:53.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:53.424 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:53.443 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:53.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:53.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:53.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:53.576 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:53.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:53.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:53.679 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:53.703 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:53.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:53.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:53.808 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:53.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:53.842 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:53.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:53.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:53.945 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:53 smithi099 bash[84576]: cluster 2024-09-13T15:17:53.216698+0000 mgr.y (mgr.26252) 1557 : cluster [DBG] pgmap v2168: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/174993 objects degraded (2.822%); 1723/174993 objects misplaced (0.985%) 2024-09-13T15:17:53.945 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:53 smithi099 bash[84576]: cluster 2024-09-13T15:17:53.216698+0000 mgr.y (mgr.26252) 1557 : cluster [DBG] pgmap v2168: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/174993 objects degraded (2.822%); 1723/174993 objects misplaced (0.985%) 2024-09-13T15:17:53.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:53.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:53.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:54.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:53 smithi099 bash[87147]: cluster 2024-09-13T15:17:53.216698+0000 mgr.y (mgr.26252) 1557 : cluster [DBG] pgmap v2168: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/174993 objects degraded (2.822%); 1723/174993 objects misplaced (0.985%) 2024-09-13T15:17:54.074 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:53 smithi099 bash[87147]: cluster 2024-09-13T15:17:53.216698+0000 mgr.y (mgr.26252) 1557 : cluster [DBG] pgmap v2168: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/174993 objects degraded (2.822%); 1723/174993 objects misplaced (0.985%) 2024-09-13T15:17:54.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:54.084 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:53 smithi177 bash[68594]: cluster 2024-09-13T15:17:53.216698+0000 mgr.y (mgr.26252) 1557 : cluster [DBG] pgmap v2168: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/174993 objects degraded (2.822%); 1723/174993 objects misplaced (0.985%) 2024-09-13T15:17:54.084 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:53 smithi177 bash[68594]: cluster 2024-09-13T15:17:53.216698+0000 mgr.y (mgr.26252) 1557 : cluster [DBG] pgmap v2168: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 4939/174993 objects degraded (2.822%); 1723/174993 objects misplaced (0.985%) 2024-09-13T15:17:54.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:54.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:54.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:54.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:54.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:54.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:54.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:54.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:54.372 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:54.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:54.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:54.504 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:54.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:54.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:54.608 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:54.638 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:54.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:54.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:54.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:54.770 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:54.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:54.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:54.873 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:54.903 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:54.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:54.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:55.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:55.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:55.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:55.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:55.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:55.167 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:55.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:55.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:55.270 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:55.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:55.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:55.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:55.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:55.435 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:55.436 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2024-09-13T15:17:55.436 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:17:55.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:55.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:55.540 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:55.560 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:55.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:55.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:55.664 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:55.699 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:55.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:55.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:55.708 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:17:55.708 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:17:55.757 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:17:55.758 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:17:55.803 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:17:55.803 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:17:55.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:55.839 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:55.841 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:17:55.841 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:17:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:55.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:55.875 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:17:55.875 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:17:55.908 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:17:55.908 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:17:55.942 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:17:55.943 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:17:55.944 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:55.973 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:55.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:55.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:55.978 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:17:55.978 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:17:56.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:56.116 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:56.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:56.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:56.219 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:56.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:56.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:56.283 INFO:teuthology.orchestra.run.smithi099.stderr:set pool 1 pg_num to 11 2024-09-13T15:17:56.297 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:17:56.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: cluster 2024-09-13T15:17:54.306760+0000 osd.6 (osd.6) 24 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: cluster 2024-09-13T15:17:54.306760+0000 osd.6 (osd.6) 24 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: cluster 2024-09-13T15:17:55.217532+0000 mgr.y (mgr.26252) 1558 : cluster [DBG] pgmap v2169: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 4731/168186 objects degraded (2.813%); 1470/168186 objects misplaced (0.874%) 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: cluster 2024-09-13T15:17:55.217532+0000 mgr.y (mgr.26252) 1558 : cluster [DBG] pgmap v2169: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 4731/168186 objects degraded (2.813%); 1470/168186 objects misplaced (0.874%) 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: audit 2024-09-13T15:17:55.664985+0000 mon.a (mon.0) 2720 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: audit 2024-09-13T15:17:55.664985+0000 mon.a (mon.0) 2720 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: audit 2024-09-13T15:17:55.675962+0000 mon.a (mon.0) 2721 : audit [DBG] from='client.? 172.21.15.99:0/3835668325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: audit 2024-09-13T15:17:55.675962+0000 mon.a (mon.0) 2721 : audit [DBG] from='client.? 172.21.15.99:0/3835668325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: audit 2024-09-13T15:17:55.958039+0000 mon.c (mon.2) 121 : audit [INF] from='client.? 172.21.15.99:0/884172862' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: audit 2024-09-13T15:17:55.958039+0000 mon.c (mon.2) 121 : audit [INF] from='client.? 172.21.15.99:0/884172862' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: audit 2024-09-13T15:17:55.959142+0000 mon.a (mon.0) 2722 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.372 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[87147]: audit 2024-09-13T15:17:55.959142+0000 mon.a (mon.0) 2722 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.372 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: cluster 2024-09-13T15:17:54.306760+0000 osd.6 (osd.6) 24 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:56.372 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: cluster 2024-09-13T15:17:54.306760+0000 osd.6 (osd.6) 24 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:56.372 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: cluster 2024-09-13T15:17:55.217532+0000 mgr.y (mgr.26252) 1558 : cluster [DBG] pgmap v2169: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 4731/168186 objects degraded (2.813%); 1470/168186 objects misplaced (0.874%) 2024-09-13T15:17:56.372 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: cluster 2024-09-13T15:17:55.217532+0000 mgr.y (mgr.26252) 1558 : cluster [DBG] pgmap v2169: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 4731/168186 objects degraded (2.813%); 1470/168186 objects misplaced (0.874%) 2024-09-13T15:17:56.372 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: audit 2024-09-13T15:17:55.664985+0000 mon.a (mon.0) 2720 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T15:17:56.372 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: audit 2024-09-13T15:17:55.664985+0000 mon.a (mon.0) 2720 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T15:17:56.372 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: audit 2024-09-13T15:17:55.675962+0000 mon.a (mon.0) 2721 : audit [DBG] from='client.? 172.21.15.99:0/3835668325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:56.372 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: audit 2024-09-13T15:17:55.675962+0000 mon.a (mon.0) 2721 : audit [DBG] from='client.? 172.21.15.99:0/3835668325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:56.372 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: audit 2024-09-13T15:17:55.958039+0000 mon.c (mon.2) 121 : audit [INF] from='client.? 172.21.15.99:0/884172862' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.373 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: audit 2024-09-13T15:17:55.958039+0000 mon.c (mon.2) 121 : audit [INF] from='client.? 172.21.15.99:0/884172862' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.373 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: audit 2024-09-13T15:17:55.959142+0000 mon.a (mon.0) 2722 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.373 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:56 smithi099 bash[84576]: audit 2024-09-13T15:17:55.959142+0000 mon.a (mon.0) 2722 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:56.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:56.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:56.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:56.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:56.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:56.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:56.607 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: cluster 2024-09-13T15:17:54.306760+0000 osd.6 (osd.6) 24 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: cluster 2024-09-13T15:17:54.306760+0000 osd.6 (osd.6) 24 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:17:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: cluster 2024-09-13T15:17:55.217532+0000 mgr.y (mgr.26252) 1558 : cluster [DBG] pgmap v2169: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 4731/168186 objects degraded (2.813%); 1470/168186 objects misplaced (0.874%) 2024-09-13T15:17:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: cluster 2024-09-13T15:17:55.217532+0000 mgr.y (mgr.26252) 1558 : cluster [DBG] pgmap v2169: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 4731/168186 objects degraded (2.813%); 1470/168186 objects misplaced (0.874%) 2024-09-13T15:17:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: audit 2024-09-13T15:17:55.664985+0000 mon.a (mon.0) 2720 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T15:17:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: audit 2024-09-13T15:17:55.664985+0000 mon.a (mon.0) 2720 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T15:17:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: audit 2024-09-13T15:17:55.675962+0000 mon.a (mon.0) 2721 : audit [DBG] from='client.? 172.21.15.99:0/3835668325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:56.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: audit 2024-09-13T15:17:55.675962+0000 mon.a (mon.0) 2721 : audit [DBG] from='client.? 172.21.15.99:0/3835668325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:17:56.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: audit 2024-09-13T15:17:55.958039+0000 mon.c (mon.2) 121 : audit [INF] from='client.? 172.21.15.99:0/884172862' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: audit 2024-09-13T15:17:55.958039+0000 mon.c (mon.2) 121 : audit [INF] from='client.? 172.21.15.99:0/884172862' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: audit 2024-09-13T15:17:55.959142+0000 mon.a (mon.0) 2722 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:56 smithi177 bash[68594]: audit 2024-09-13T15:17:55.959142+0000 mon.a (mon.0) 2722 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:17:56.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:17:56.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:56.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:56.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:56.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:56.764 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:56.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:56.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:56.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:56.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:17:56.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:56.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:17:56.999 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:57.023 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:57.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:57.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:57.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:57.156 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:57.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:57.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:57.267 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:17:57] "GET /metrics HTTP/1.1" 200 36782 "" "Prometheus/2.51.0" 2024-09-13T15:17:57.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:57.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:57.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:57.304 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: cluster 2024-09-13T15:17:56.273215+0000 mon.a (mon.0) 2723 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:17:57.304 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: cluster 2024-09-13T15:17:56.273215+0000 mon.a (mon.0) 2723 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:17:57.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: audit 2024-09-13T15:17:56.283088+0000 mon.a (mon.0) 2724 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T15:17:57.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: audit 2024-09-13T15:17:56.283088+0000 mon.a (mon.0) 2724 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T15:17:57.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: audit 2024-09-13T15:17:56.283351+0000 mon.a (mon.0) 2725 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:17:57.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: audit 2024-09-13T15:17:56.283351+0000 mon.a (mon.0) 2725 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:17:57.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: cluster 2024-09-13T15:17:56.289514+0000 mon.a (mon.0) 2726 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2024-09-13T15:17:57.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: cluster 2024-09-13T15:17:56.289514+0000 mon.a (mon.0) 2726 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2024-09-13T15:17:57.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: cluster 2024-09-13T15:17:56.665196+0000 mon.a (mon.0) 2727 : cluster [WRN] Health check update: Degraded data redundancy: 4731/168186 objects degraded (2.813%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:57.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: cluster 2024-09-13T15:17:56.665196+0000 mon.a (mon.0) 2727 : cluster [WRN] Health check update: Degraded data redundancy: 4731/168186 objects degraded (2.813%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:57.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: audit 2024-09-13T15:17:57.218615+0000 mon.a (mon.0) 2728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T15:17:57.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:57.420 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:57.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:57.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:57.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: cluster 2024-09-13T15:17:56.273215+0000 mon.a (mon.0) 2723 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: cluster 2024-09-13T15:17:56.273215+0000 mon.a (mon.0) 2723 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: audit 2024-09-13T15:17:56.283088+0000 mon.a (mon.0) 2724 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: audit 2024-09-13T15:17:56.283088+0000 mon.a (mon.0) 2724 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: audit 2024-09-13T15:17:56.283351+0000 mon.a (mon.0) 2725 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: audit 2024-09-13T15:17:56.283351+0000 mon.a (mon.0) 2725 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: cluster 2024-09-13T15:17:56.289514+0000 mon.a (mon.0) 2726 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: cluster 2024-09-13T15:17:56.289514+0000 mon.a (mon.0) 2726 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: cluster 2024-09-13T15:17:56.665196+0000 mon.a (mon.0) 2727 : cluster [WRN] Health check update: Degraded data redundancy: 4731/168186 objects degraded (2.813%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: cluster 2024-09-13T15:17:56.665196+0000 mon.a (mon.0) 2727 : cluster [WRN] Health check update: Degraded data redundancy: 4731/168186 objects degraded (2.813%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: audit 2024-09-13T15:17:57.218615+0000 mon.a (mon.0) 2728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T15:17:57.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:57 smithi177 bash[68594]: audit 2024-09-13T15:17:57.218615+0000 mon.a (mon.0) 2728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T15:17:57.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:57.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:57.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:57.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[84576]: audit 2024-09-13T15:17:57.218615+0000 mon.a (mon.0) 2728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: cluster 2024-09-13T15:17:56.273215+0000 mon.a (mon.0) 2723 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: cluster 2024-09-13T15:17:56.273215+0000 mon.a (mon.0) 2723 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: audit 2024-09-13T15:17:56.283088+0000 mon.a (mon.0) 2724 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: audit 2024-09-13T15:17:56.283088+0000 mon.a (mon.0) 2724 : audit [INF] from='client.? 172.21.15.99:0/3345105923' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: audit 2024-09-13T15:17:56.283351+0000 mon.a (mon.0) 2725 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: audit 2024-09-13T15:17:56.283351+0000 mon.a (mon.0) 2725 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: cluster 2024-09-13T15:17:56.289514+0000 mon.a (mon.0) 2726 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: cluster 2024-09-13T15:17:56.289514+0000 mon.a (mon.0) 2726 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: cluster 2024-09-13T15:17:56.665196+0000 mon.a (mon.0) 2727 : cluster [WRN] Health check update: Degraded data redundancy: 4731/168186 objects degraded (2.813%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: cluster 2024-09-13T15:17:56.665196+0000 mon.a (mon.0) 2727 : cluster [WRN] Health check update: Degraded data redundancy: 4731/168186 objects degraded (2.813%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:17:57.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: audit 2024-09-13T15:17:57.218615+0000 mon.a (mon.0) 2728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T15:17:57.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:57 smithi099 bash[87147]: audit 2024-09-13T15:17:57.218615+0000 mon.a (mon.0) 2728 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T15:17:57.653 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:57.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:57.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:57.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:57.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:57.820 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:57.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:57.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:57.925 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:57.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:57.956 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:57.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:58.057 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:58.080 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:58.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:58.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:58.185 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:58.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:58.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:58.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:58.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:58.336 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:58.340 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:58 smithi177 bash[68594]: cluster 2024-09-13T15:17:57.218068+0000 mgr.y (mgr.26252) 1559 : cluster [DBG] pgmap v2171: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 300 op/s; 4731/165696 objects degraded (2.855%); 1470/165696 objects misplaced (0.887%) 2024-09-13T15:17:58.340 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:58 smithi177 bash[68594]: cluster 2024-09-13T15:17:57.218068+0000 mgr.y (mgr.26252) 1559 : cluster [DBG] pgmap v2171: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 300 op/s; 4731/165696 objects degraded (2.855%); 1470/165696 objects misplaced (0.887%) 2024-09-13T15:17:58.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:58 smithi177 bash[68594]: audit 2024-09-13T15:17:57.299523+0000 mon.a (mon.0) 2729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T15:17:58.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:58 smithi177 bash[68594]: audit 2024-09-13T15:17:57.299523+0000 mon.a (mon.0) 2729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T15:17:58.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:58 smithi177 bash[68594]: cluster 2024-09-13T15:17:57.304127+0000 mon.a (mon.0) 2730 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2024-09-13T15:17:58.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:58 smithi177 bash[68594]: cluster 2024-09-13T15:17:57.304127+0000 mon.a (mon.0) 2730 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2024-09-13T15:17:58.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:58.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:58.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:58.465 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:58.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:58.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:58.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[84576]: cluster 2024-09-13T15:17:57.218068+0000 mgr.y (mgr.26252) 1559 : cluster [DBG] pgmap v2171: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 300 op/s; 4731/165696 objects degraded (2.855%); 1470/165696 objects misplaced (0.887%) 2024-09-13T15:17:58.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[84576]: cluster 2024-09-13T15:17:57.218068+0000 mgr.y (mgr.26252) 1559 : cluster [DBG] pgmap v2171: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 300 op/s; 4731/165696 objects degraded (2.855%); 1470/165696 objects misplaced (0.887%) 2024-09-13T15:17:58.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[84576]: audit 2024-09-13T15:17:57.299523+0000 mon.a (mon.0) 2729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T15:17:58.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[84576]: audit 2024-09-13T15:17:57.299523+0000 mon.a (mon.0) 2729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T15:17:58.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[84576]: cluster 2024-09-13T15:17:57.304127+0000 mon.a (mon.0) 2730 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2024-09-13T15:17:58.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[84576]: cluster 2024-09-13T15:17:57.304127+0000 mon.a (mon.0) 2730 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2024-09-13T15:17:58.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[87147]: cluster 2024-09-13T15:17:57.218068+0000 mgr.y (mgr.26252) 1559 : cluster [DBG] pgmap v2171: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 300 op/s; 4731/165696 objects degraded (2.855%); 1470/165696 objects misplaced (0.887%) 2024-09-13T15:17:58.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[87147]: cluster 2024-09-13T15:17:57.218068+0000 mgr.y (mgr.26252) 1559 : cluster [DBG] pgmap v2171: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 300 op/s; 4731/165696 objects degraded (2.855%); 1470/165696 objects misplaced (0.887%) 2024-09-13T15:17:58.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[87147]: audit 2024-09-13T15:17:57.299523+0000 mon.a (mon.0) 2729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T15:17:58.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[87147]: audit 2024-09-13T15:17:57.299523+0000 mon.a (mon.0) 2729 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T15:17:58.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[87147]: cluster 2024-09-13T15:17:57.304127+0000 mon.a (mon.0) 2730 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2024-09-13T15:17:58.577 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:58 smithi099 bash[87147]: cluster 2024-09-13T15:17:57.304127+0000 mon.a (mon.0) 2730 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2024-09-13T15:17:58.602 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:58.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:58.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:58.705 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:58.730 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:17:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:58.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:17:58.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:58.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:58.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:58.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:58.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:58.998 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:59.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:59.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:59.103 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:59.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:59.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:59.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:17:59.241 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:59.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:17:59.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:59.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:17:59.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:59.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:17:59.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:59.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:17:59.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:59.529 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:17:59.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:59.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:17:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:59 smithi099 bash[84576]: cluster 2024-09-13T15:17:58.307638+0000 mon.a (mon.0) 2731 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2024-09-13T15:17:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:59 smithi099 bash[84576]: cluster 2024-09-13T15:17:58.307638+0000 mon.a (mon.0) 2731 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2024-09-13T15:17:59.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:59 smithi099 bash[84576]: cluster 2024-09-13T15:17:59.218920+0000 mgr.y (mgr.26252) 1560 : cluster [DBG] pgmap v2174: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4731/159168 objects degraded (2.972%); 1470/159168 objects misplaced (0.924%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T15:17:59.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:17:59 smithi099 bash[84576]: cluster 2024-09-13T15:17:59.218920+0000 mgr.y (mgr.26252) 1560 : cluster [DBG] pgmap v2174: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4731/159168 objects degraded (2.972%); 1470/159168 objects misplaced (0.924%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T15:17:59.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:59 smithi099 bash[87147]: cluster 2024-09-13T15:17:58.307638+0000 mon.a (mon.0) 2731 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2024-09-13T15:17:59.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:59 smithi099 bash[87147]: cluster 2024-09-13T15:17:58.307638+0000 mon.a (mon.0) 2731 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2024-09-13T15:17:59.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:59 smithi099 bash[87147]: cluster 2024-09-13T15:17:59.218920+0000 mgr.y (mgr.26252) 1560 : cluster [DBG] pgmap v2174: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4731/159168 objects degraded (2.972%); 1470/159168 objects misplaced (0.924%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T15:17:59.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:17:59 smithi099 bash[87147]: cluster 2024-09-13T15:17:59.218920+0000 mgr.y (mgr.26252) 1560 : cluster [DBG] pgmap v2174: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4731/159168 objects degraded (2.972%); 1470/159168 objects misplaced (0.924%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T15:17:59.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:59 smithi177 bash[68594]: cluster 2024-09-13T15:17:58.307638+0000 mon.a (mon.0) 2731 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2024-09-13T15:17:59.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:59 smithi177 bash[68594]: cluster 2024-09-13T15:17:58.307638+0000 mon.a (mon.0) 2731 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2024-09-13T15:17:59.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:59 smithi177 bash[68594]: cluster 2024-09-13T15:17:59.218920+0000 mgr.y (mgr.26252) 1560 : cluster [DBG] pgmap v2174: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4731/159168 objects degraded (2.972%); 1470/159168 objects misplaced (0.924%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T15:17:59.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:17:59 smithi177 bash[68594]: cluster 2024-09-13T15:17:59.218920+0000 mgr.y (mgr.26252) 1560 : cluster [DBG] pgmap v2174: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4731/159168 objects degraded (2.972%); 1470/159168 objects misplaced (0.924%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T15:17:59.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:59.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:17:59.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:59.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:17:59.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:59.788 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:17:59.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:59.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:17:59.893 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:59.917 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:17:59.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:17:59.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:00.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:00.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:00.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:00.152 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:00.176 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:00.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:00.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:00.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:00.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:00.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:00.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:00.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:00.443 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:00.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:00.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:00.546 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:00.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:00 smithi177 bash[68594]: cluster 2024-09-13T15:17:58.727180+0000 osd.7 (osd.7) 11 : cluster [DBG] 1.0 scrub starts 2024-09-13T15:18:00.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:00 smithi177 bash[68594]: cluster 2024-09-13T15:17:58.727180+0000 osd.7 (osd.7) 11 : cluster [DBG] 1.0 scrub starts 2024-09-13T15:18:00.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:00 smithi177 bash[68594]: cluster 2024-09-13T15:17:58.733674+0000 osd.7 (osd.7) 12 : cluster [DBG] 1.0 scrub ok 2024-09-13T15:18:00.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:00 smithi177 bash[68594]: cluster 2024-09-13T15:17:58.733674+0000 osd.7 (osd.7) 12 : cluster [DBG] 1.0 scrub ok 2024-09-13T15:18:00.556 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:18:00 smithi177 bash[61439]: ts=2024-09-13T15:18:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:18:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:00 smithi099 bash[84576]: cluster 2024-09-13T15:17:58.727180+0000 osd.7 (osd.7) 11 : cluster [DBG] 1.0 scrub starts 2024-09-13T15:18:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:00 smithi099 bash[84576]: cluster 2024-09-13T15:17:58.727180+0000 osd.7 (osd.7) 11 : cluster [DBG] 1.0 scrub starts 2024-09-13T15:18:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:00 smithi099 bash[84576]: cluster 2024-09-13T15:17:58.733674+0000 osd.7 (osd.7) 12 : cluster [DBG] 1.0 scrub ok 2024-09-13T15:18:00.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:00 smithi099 bash[84576]: cluster 2024-09-13T15:17:58.733674+0000 osd.7 (osd.7) 12 : cluster [DBG] 1.0 scrub ok 2024-09-13T15:18:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:00 smithi099 bash[87147]: cluster 2024-09-13T15:17:58.727180+0000 osd.7 (osd.7) 11 : cluster [DBG] 1.0 scrub starts 2024-09-13T15:18:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:00 smithi099 bash[87147]: cluster 2024-09-13T15:17:58.727180+0000 osd.7 (osd.7) 11 : cluster [DBG] 1.0 scrub starts 2024-09-13T15:18:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:00 smithi099 bash[87147]: cluster 2024-09-13T15:17:58.733674+0000 osd.7 (osd.7) 12 : cluster [DBG] 1.0 scrub ok 2024-09-13T15:18:00.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:00 smithi099 bash[87147]: cluster 2024-09-13T15:17:58.733674+0000 osd.7 (osd.7) 12 : cluster [DBG] 1.0 scrub ok 2024-09-13T15:18:00.575 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:00.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:00.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:00.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:00.703 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:00.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:00.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:00.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:00.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:00.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:00.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:00.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:00.962 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:00.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:01.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:01.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:01.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:01.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:01.195 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:01.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:01.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:01.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:01.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:01 smithi177 bash[68594]: cluster 2024-09-13T15:18:01.219612+0000 mgr.y (mgr.26252) 1561 : cluster [DBG] pgmap v2175: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 4423/152019 objects degraded (2.910%); 1320/152019 objects misplaced (0.868%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:18:01.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:01 smithi177 bash[68594]: cluster 2024-09-13T15:18:01.219612+0000 mgr.y (mgr.26252) 1561 : cluster [DBG] pgmap v2175: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 4423/152019 objects degraded (2.910%); 1320/152019 objects misplaced (0.868%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:18:01.369 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:01.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:01.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:01.442 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:18:01.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:01.503 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:01.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:01.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:01.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:01 smithi099 bash[84576]: cluster 2024-09-13T15:18:01.219612+0000 mgr.y (mgr.26252) 1561 : cluster [DBG] pgmap v2175: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 4423/152019 objects degraded (2.910%); 1320/152019 objects misplaced (0.868%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:18:01.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:01 smithi099 bash[84576]: cluster 2024-09-13T15:18:01.219612+0000 mgr.y (mgr.26252) 1561 : cluster [DBG] pgmap v2175: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 4423/152019 objects degraded (2.910%); 1320/152019 objects misplaced (0.868%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:18:01.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:01 smithi099 bash[87147]: cluster 2024-09-13T15:18:01.219612+0000 mgr.y (mgr.26252) 1561 : cluster [DBG] pgmap v2175: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 4423/152019 objects degraded (2.910%); 1320/152019 objects misplaced (0.868%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:18:01.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:01 smithi099 bash[87147]: cluster 2024-09-13T15:18:01.219612+0000 mgr.y (mgr.26252) 1561 : cluster [DBG] pgmap v2175: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 4423/152019 objects degraded (2.910%); 1320/152019 objects misplaced (0.868%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:18:01.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:01.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:01.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:01.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:01.766 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:01.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:01.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:01.868 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:01.893 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:01.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:01.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:01.998 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:02.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:02.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:02.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:02.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:02.167 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:02.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:02.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:02.273 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:02.300 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:02.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:02.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:02.403 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:02.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:02.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:02.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:02.533 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:02.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:02.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:02.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:02.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:02 smithi099 bash[84576]: cluster 2024-09-13T15:18:01.666011+0000 mon.a (mon.0) 2732 : cluster [WRN] Health check update: Degraded data redundancy: 4423/152019 objects degraded (2.910%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:02.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:02 smithi099 bash[84576]: cluster 2024-09-13T15:18:01.666011+0000 mon.a (mon.0) 2732 : cluster [WRN] Health check update: Degraded data redundancy: 4423/152019 objects degraded (2.910%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:02.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:02 smithi099 bash[84576]: cluster 2024-09-13T15:18:02.180954+0000 osd.6 (osd.6) 25 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:02.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:02 smithi099 bash[84576]: cluster 2024-09-13T15:18:02.180954+0000 osd.6 (osd.6) 25 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:02.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:02 smithi099 bash[87147]: cluster 2024-09-13T15:18:01.666011+0000 mon.a (mon.0) 2732 : cluster [WRN] Health check update: Degraded data redundancy: 4423/152019 objects degraded (2.910%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:02.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:02 smithi099 bash[87147]: cluster 2024-09-13T15:18:01.666011+0000 mon.a (mon.0) 2732 : cluster [WRN] Health check update: Degraded data redundancy: 4423/152019 objects degraded (2.910%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:02.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:02 smithi099 bash[87147]: cluster 2024-09-13T15:18:02.180954+0000 osd.6 (osd.6) 25 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:02.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:02 smithi099 bash[87147]: cluster 2024-09-13T15:18:02.180954+0000 osd.6 (osd.6) 25 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:02.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:02 smithi177 bash[68594]: cluster 2024-09-13T15:18:01.666011+0000 mon.a (mon.0) 2732 : cluster [WRN] Health check update: Degraded data redundancy: 4423/152019 objects degraded (2.910%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:02.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:02 smithi177 bash[68594]: cluster 2024-09-13T15:18:01.666011+0000 mon.a (mon.0) 2732 : cluster [WRN] Health check update: Degraded data redundancy: 4423/152019 objects degraded (2.910%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:02.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:02 smithi177 bash[68594]: cluster 2024-09-13T15:18:02.180954+0000 osd.6 (osd.6) 25 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:02.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:02 smithi177 bash[68594]: cluster 2024-09-13T15:18:02.180954+0000 osd.6 (osd.6) 25 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:02.670 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:02.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:02.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:02.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:02.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:02.815 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:02.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:02.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:02.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:02.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:02.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:02.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:03.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:03.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:03.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:03.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:03.181 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:03.211 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:03.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:03.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:03.318 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:03.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:03.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:03.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:03.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:03.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:03 smithi177 bash[68594]: cluster 2024-09-13T15:18:01.756202+0000 osd.7 (osd.7) 13 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:18:03.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:03 smithi177 bash[68594]: cluster 2024-09-13T15:18:01.756202+0000 osd.7 (osd.7) 13 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:18:03.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:03 smithi177 bash[68594]: cluster 2024-09-13T15:18:01.758749+0000 osd.7 (osd.7) 14 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:18:03.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:03 smithi177 bash[68594]: cluster 2024-09-13T15:18:01.758749+0000 osd.7 (osd.7) 14 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:18:03.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:03 smithi177 bash[68594]: cluster 2024-09-13T15:18:03.220213+0000 mgr.y (mgr.26252) 1562 : cluster [DBG] pgmap v2176: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 4423/150105 objects degraded (2.947%); 1320/150105 objects misplaced (0.879%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:18:03.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:03 smithi177 bash[68594]: cluster 2024-09-13T15:18:03.220213+0000 mgr.y (mgr.26252) 1562 : cluster [DBG] pgmap v2176: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 4423/150105 objects degraded (2.947%); 1320/150105 objects misplaced (0.879%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:18:03.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:03.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:03.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:03.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:03.598 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[84576]: cluster 2024-09-13T15:18:01.756202+0000 osd.7 (osd.7) 13 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:18:03.598 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[84576]: cluster 2024-09-13T15:18:01.756202+0000 osd.7 (osd.7) 13 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:18:03.598 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[84576]: cluster 2024-09-13T15:18:01.758749+0000 osd.7 (osd.7) 14 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:18:03.599 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[84576]: cluster 2024-09-13T15:18:01.758749+0000 osd.7 (osd.7) 14 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:18:03.599 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[84576]: cluster 2024-09-13T15:18:03.220213+0000 mgr.y (mgr.26252) 1562 : cluster [DBG] pgmap v2176: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 4423/150105 objects degraded (2.947%); 1320/150105 objects misplaced (0.879%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:18:03.599 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[84576]: cluster 2024-09-13T15:18:03.220213+0000 mgr.y (mgr.26252) 1562 : cluster [DBG] pgmap v2176: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 4423/150105 objects degraded (2.947%); 1320/150105 objects misplaced (0.879%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:18:03.599 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[87147]: cluster 2024-09-13T15:18:01.756202+0000 osd.7 (osd.7) 13 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:18:03.599 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[87147]: cluster 2024-09-13T15:18:01.756202+0000 osd.7 (osd.7) 13 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:18:03.599 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[87147]: cluster 2024-09-13T15:18:01.758749+0000 osd.7 (osd.7) 14 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:18:03.599 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[87147]: cluster 2024-09-13T15:18:01.758749+0000 osd.7 (osd.7) 14 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:18:03.599 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[87147]: cluster 2024-09-13T15:18:03.220213+0000 mgr.y (mgr.26252) 1562 : cluster [DBG] pgmap v2176: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 4423/150105 objects degraded (2.947%); 1320/150105 objects misplaced (0.879%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:18:03.599 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:03 smithi099 bash[87147]: cluster 2024-09-13T15:18:03.220213+0000 mgr.y (mgr.26252) 1562 : cluster [DBG] pgmap v2176: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 4423/150105 objects degraded (2.947%); 1320/150105 objects misplaced (0.879%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T15:18:03.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:03.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:03.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:03.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:03.743 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:03.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:03.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:03.848 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:03.878 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:03.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:03.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:03.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:04.011 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:04.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:04.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:04.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:04.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:04.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:04.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:04.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:04.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:04.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:04.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:04.376 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:04.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:04.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:04.512 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:04.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:04 smithi177 bash[68594]: cluster 2024-09-13T15:18:02.989359+0000 osd.7 (osd.7) 15 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:18:04.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:04 smithi177 bash[68594]: cluster 2024-09-13T15:18:02.989359+0000 osd.7 (osd.7) 15 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:18:04.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:04 smithi177 bash[68594]: cluster 2024-09-13T15:18:02.996196+0000 osd.7 (osd.7) 16 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:18:04.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:04 smithi177 bash[68594]: cluster 2024-09-13T15:18:02.996196+0000 osd.7 (osd.7) 16 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:18:04.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:04 smithi177 bash[68594]: audit 2024-09-13T15:18:03.322687+0000 mon.a (mon.0) 2733 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:04.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:04 smithi177 bash[68594]: audit 2024-09-13T15:18:03.322687+0000 mon.a (mon.0) 2733 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:04.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:04 smithi177 bash[68594]: audit 2024-09-13T15:18:03.328097+0000 mon.a (mon.0) 2734 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:04.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:04 smithi177 bash[68594]: audit 2024-09-13T15:18:03.328097+0000 mon.a (mon.0) 2734 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:04.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:04.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:04.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:04.625 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:18:04.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:18:04.635 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[84576]: cluster 2024-09-13T15:18:02.989359+0000 osd.7 (osd.7) 15 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:18:04.635 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[84576]: cluster 2024-09-13T15:18:02.989359+0000 osd.7 (osd.7) 15 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:18:04.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[84576]: cluster 2024-09-13T15:18:02.996196+0000 osd.7 (osd.7) 16 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:18:04.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[84576]: cluster 2024-09-13T15:18:02.996196+0000 osd.7 (osd.7) 16 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:18:04.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[84576]: audit 2024-09-13T15:18:03.322687+0000 mon.a (mon.0) 2733 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:04.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[84576]: audit 2024-09-13T15:18:03.322687+0000 mon.a (mon.0) 2733 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:04.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[84576]: audit 2024-09-13T15:18:03.328097+0000 mon.a (mon.0) 2734 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:04.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[84576]: audit 2024-09-13T15:18:03.328097+0000 mon.a (mon.0) 2734 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:04.636 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[87147]: cluster 2024-09-13T15:18:02.989359+0000 osd.7 (osd.7) 15 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:18:04.636 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[87147]: cluster 2024-09-13T15:18:02.989359+0000 osd.7 (osd.7) 15 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:18:04.636 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[87147]: cluster 2024-09-13T15:18:02.996196+0000 osd.7 (osd.7) 16 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:18:04.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[87147]: cluster 2024-09-13T15:18:02.996196+0000 osd.7 (osd.7) 16 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:18:04.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[87147]: audit 2024-09-13T15:18:03.322687+0000 mon.a (mon.0) 2733 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:04.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[87147]: audit 2024-09-13T15:18:03.322687+0000 mon.a (mon.0) 2733 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:04.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[87147]: audit 2024-09-13T15:18:03.328097+0000 mon.a (mon.0) 2734 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:04.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:04 smithi099 bash[87147]: audit 2024-09-13T15:18:03.328097+0000 mon.a (mon.0) 2734 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:04.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:04.665 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:04.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:04.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:04.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:04.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:04.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:04.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:04.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:04.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:04.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:05.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:05.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:05.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:05.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:05.157 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:05.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:05.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:05.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:05.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:05.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:05.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:05.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:05.455 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:05.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:05.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:05.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:05.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[84576]: cluster 2024-09-13T15:18:04.966797+0000 osd.7 (osd.7) 17 : cluster [DBG] 1.9 scrub starts 2024-09-13T15:18:05.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[84576]: cluster 2024-09-13T15:18:04.966797+0000 osd.7 (osd.7) 17 : cluster [DBG] 1.9 scrub starts 2024-09-13T15:18:05.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[84576]: cluster 2024-09-13T15:18:04.969988+0000 osd.7 (osd.7) 18 : cluster [DBG] 1.9 scrub ok 2024-09-13T15:18:05.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[84576]: cluster 2024-09-13T15:18:04.969988+0000 osd.7 (osd.7) 18 : cluster [DBG] 1.9 scrub ok 2024-09-13T15:18:05.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[84576]: cluster 2024-09-13T15:18:05.157873+0000 osd.6 (osd.6) 26 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:05.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[84576]: cluster 2024-09-13T15:18:05.157873+0000 osd.6 (osd.6) 26 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:05.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[84576]: cluster 2024-09-13T15:18:05.220928+0000 mgr.y (mgr.26252) 1563 : cluster [DBG] pgmap v2177: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 4163/142092 objects degraded (2.930%); 1123/142092 objects misplaced (0.790%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:05.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[84576]: cluster 2024-09-13T15:18:05.220928+0000 mgr.y (mgr.26252) 1563 : cluster [DBG] pgmap v2177: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 4163/142092 objects degraded (2.930%); 1123/142092 objects misplaced (0.790%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:05.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[87147]: cluster 2024-09-13T15:18:04.966797+0000 osd.7 (osd.7) 17 : cluster [DBG] 1.9 scrub starts 2024-09-13T15:18:05.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[87147]: cluster 2024-09-13T15:18:04.966797+0000 osd.7 (osd.7) 17 : cluster [DBG] 1.9 scrub starts 2024-09-13T15:18:05.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[87147]: cluster 2024-09-13T15:18:04.969988+0000 osd.7 (osd.7) 18 : cluster [DBG] 1.9 scrub ok 2024-09-13T15:18:05.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[87147]: cluster 2024-09-13T15:18:04.969988+0000 osd.7 (osd.7) 18 : cluster [DBG] 1.9 scrub ok 2024-09-13T15:18:05.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[87147]: cluster 2024-09-13T15:18:05.157873+0000 osd.6 (osd.6) 26 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:05.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[87147]: cluster 2024-09-13T15:18:05.157873+0000 osd.6 (osd.6) 26 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:05.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[87147]: cluster 2024-09-13T15:18:05.220928+0000 mgr.y (mgr.26252) 1563 : cluster [DBG] pgmap v2177: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 4163/142092 objects degraded (2.930%); 1123/142092 objects misplaced (0.790%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:05.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:05 smithi099 bash[87147]: cluster 2024-09-13T15:18:05.220928+0000 mgr.y (mgr.26252) 1563 : cluster [DBG] pgmap v2177: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 4163/142092 objects degraded (2.930%); 1123/142092 objects misplaced (0.790%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:05.582 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:05.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:05.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:05 smithi177 bash[68594]: cluster 2024-09-13T15:18:04.966797+0000 osd.7 (osd.7) 17 : cluster [DBG] 1.9 scrub starts 2024-09-13T15:18:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:05 smithi177 bash[68594]: cluster 2024-09-13T15:18:04.966797+0000 osd.7 (osd.7) 17 : cluster [DBG] 1.9 scrub starts 2024-09-13T15:18:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:05 smithi177 bash[68594]: cluster 2024-09-13T15:18:04.969988+0000 osd.7 (osd.7) 18 : cluster [DBG] 1.9 scrub ok 2024-09-13T15:18:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:05 smithi177 bash[68594]: cluster 2024-09-13T15:18:04.969988+0000 osd.7 (osd.7) 18 : cluster [DBG] 1.9 scrub ok 2024-09-13T15:18:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:05 smithi177 bash[68594]: cluster 2024-09-13T15:18:05.157873+0000 osd.6 (osd.6) 26 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:05 smithi177 bash[68594]: cluster 2024-09-13T15:18:05.157873+0000 osd.6 (osd.6) 26 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:05 smithi177 bash[68594]: cluster 2024-09-13T15:18:05.220928+0000 mgr.y (mgr.26252) 1563 : cluster [DBG] pgmap v2177: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 4163/142092 objects degraded (2.930%); 1123/142092 objects misplaced (0.790%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:05 smithi177 bash[68594]: cluster 2024-09-13T15:18:05.220928+0000 mgr.y (mgr.26252) 1563 : cluster [DBG] pgmap v2177: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 4163/142092 objects degraded (2.930%); 1123/142092 objects misplaced (0.790%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:05.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:05.721 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:05.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:05.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:05.825 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:05.859 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:05.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:05.961 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:05.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:05.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:05.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:06.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:06.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:06.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:06.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:06.241 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:06.260 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:06.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:06.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:06.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:06.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:06.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:06.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:06.495 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:06.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:06.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:06.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:06.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:06.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:06.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:06.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:06.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:06.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:06.884 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:06.895 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:06 smithi177 bash[68594]: cluster 2024-09-13T15:18:06.666605+0000 mon.a (mon.0) 2735 : cluster [WRN] Health check update: Degraded data redundancy: 4163/142092 objects degraded (2.930%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:06.895 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:06 smithi177 bash[68594]: cluster 2024-09-13T15:18:06.666605+0000 mon.a (mon.0) 2735 : cluster [WRN] Health check update: Degraded data redundancy: 4163/142092 objects degraded (2.930%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:06.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:06.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:06.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:07.012 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:07.041 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:07.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:07.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:07.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:06 smithi099 bash[84576]: cluster 2024-09-13T15:18:06.666605+0000 mon.a (mon.0) 2735 : cluster [WRN] Health check update: Degraded data redundancy: 4163/142092 objects degraded (2.930%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:07.053 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:06 smithi099 bash[84576]: cluster 2024-09-13T15:18:06.666605+0000 mon.a (mon.0) 2735 : cluster [WRN] Health check update: Degraded data redundancy: 4163/142092 objects degraded (2.930%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:06 smithi099 bash[87147]: cluster 2024-09-13T15:18:06.666605+0000 mon.a (mon.0) 2735 : cluster [WRN] Health check update: Degraded data redundancy: 4163/142092 objects degraded (2.930%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:07.053 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:06 smithi099 bash[87147]: cluster 2024-09-13T15:18:06.666605+0000 mon.a (mon.0) 2735 : cluster [WRN] Health check update: Degraded data redundancy: 4163/142092 objects degraded (2.930%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:07.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:07.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:07.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:07.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:07.278 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:07.308 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:07.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:07.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:07.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:18:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:18:07] "GET /metrics HTTP/1.1" 200 36802 "" "Prometheus/2.51.0" 2024-09-13T15:18:07.415 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:07.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:07.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:07.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:07.540 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:07.563 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:07.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:07.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:07.615 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 5, 7, 4, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:18:07.615 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:18:07.615 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 2, 3, 5, 7, 4, 1, 6] 2024-09-13T15:18:07.616 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd out 2 2024-09-13T15:18:07.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:07.710 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:07.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:07.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:07.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:07 smithi099 bash[84576]: cluster 2024-09-13T15:18:07.221607+0000 mgr.y (mgr.26252) 1564 : cluster [DBG] pgmap v2178: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4163/139938 objects degraded (2.975%); 1123/139938 objects misplaced (0.802%); 3.9 MiB/s, 63 objects/s recovering 2024-09-13T15:18:07.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:07 smithi099 bash[84576]: cluster 2024-09-13T15:18:07.221607+0000 mgr.y (mgr.26252) 1564 : cluster [DBG] pgmap v2178: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4163/139938 objects degraded (2.975%); 1123/139938 objects misplaced (0.802%); 3.9 MiB/s, 63 objects/s recovering 2024-09-13T15:18:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:07 smithi099 bash[87147]: cluster 2024-09-13T15:18:07.221607+0000 mgr.y (mgr.26252) 1564 : cluster [DBG] pgmap v2178: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4163/139938 objects degraded (2.975%); 1123/139938 objects misplaced (0.802%); 3.9 MiB/s, 63 objects/s recovering 2024-09-13T15:18:07.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:07 smithi099 bash[87147]: cluster 2024-09-13T15:18:07.221607+0000 mgr.y (mgr.26252) 1564 : cluster [DBG] pgmap v2178: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4163/139938 objects degraded (2.975%); 1123/139938 objects misplaced (0.802%); 3.9 MiB/s, 63 objects/s recovering 2024-09-13T15:18:07.840 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:07.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:07.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:07.948 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:07.993 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:08.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:08.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:08.101 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:08.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:07 smithi177 bash[68594]: cluster 2024-09-13T15:18:07.221607+0000 mgr.y (mgr.26252) 1564 : cluster [DBG] pgmap v2178: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4163/139938 objects degraded (2.975%); 1123/139938 objects misplaced (0.802%); 3.9 MiB/s, 63 objects/s recovering 2024-09-13T15:18:08.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:07 smithi177 bash[68594]: cluster 2024-09-13T15:18:07.221607+0000 mgr.y (mgr.26252) 1564 : cluster [DBG] pgmap v2178: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4163/139938 objects degraded (2.975%); 1123/139938 objects misplaced (0.802%); 3.9 MiB/s, 63 objects/s recovering 2024-09-13T15:18:08.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:08.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:08.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:08.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:08.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:08.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:08.360 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:18:08.360 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T15:18:08.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T15:18:08.360 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:18:08.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.360 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213425 44.a 44:5900890a:::benchmark_data_smithi099_188632_object106221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.108465+0000", 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 594.24336392700002, 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.60089703299999997, 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213425 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108465+0000", 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108465+0000", 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108464+0000", 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108471+0000", 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.204e-06 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108486+0000", 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4640999999999999e-05 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108492+0000", 2024-09-13T15:18:08.363 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5339999999999996e-06 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108536+0000", 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3627000000000001e-05 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108664+0000", 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012790699999999999 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.364 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.108784+0000", 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120684 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.115159+0000", 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063750070000000002 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.115169+0000", 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0076e-05 2024-09-13T15:18:08.365 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709261+0000", 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.59409184900000001 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709318+0000", 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6325e-05 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709362+0000", 2024-09-13T15:18:08.366 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4634e-05 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213436 44.a 44:5fef46cf:::benchmark_data_smithi099_188632_object106232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.138114+0000", 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 594.21371554699999, 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.57135144800000004, 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:18:08.367 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:18:08.368 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213436 2024-09-13T15:18:08.368 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.368 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.368 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.368 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.368 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138114+0000", 2024-09-13T15:18:08.368 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.368 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.368 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138114+0000", 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138112+0000", 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138119+0000", 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3990000000000001e-06 2024-09-13T15:18:08.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138131+0000", 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2343e-05 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138136+0000", 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3250000000000001e-06 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138168+0000", 2024-09-13T15:18:08.370 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2678000000000001e-05 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138258+0000", 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9660999999999994e-05 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138333+0000", 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4777000000000006e-05 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.371 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.143497+0000", 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051641050000000004 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.148562+0000", 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050652279999999997 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709391+0000", 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.56082900400000002 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.372 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709428+0000", 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6557000000000003e-05 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709465+0000", 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7490999999999997e-05 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.373 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213437 44.a 44:5b63eef5:::benchmark_data_smithi099_188632_object106233:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.138145+0000", 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 594.21368438000002, 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.57140967799999998, 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213437 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.374 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138145+0000", 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138145+0000", 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138144+0000", 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.375 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138146+0000", 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7370000000000001e-06 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138150+0000", 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6349999999999999e-06 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.138152+0000", 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2189999999999998e-06 2024-09-13T15:18:08.376 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141238+0000", 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030859199999999998 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141369+0000", 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000131359 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141505+0000", 2024-09-13T15:18:08.377 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013533600000000001 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.145190+0000", 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036849109999999999 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.148772+0000", 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035820600000000002 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.378 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709497+0000", 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.56072498299999995 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709522+0000", 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5346000000000001e-05 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709554+0000", 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2471999999999997e-05 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.379 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213439 44.a 44:5b71bed8:::benchmark_data_smithi099_188632_object106235:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.141274+0000", 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 594.210554925, 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.56835986699999996, 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213439 2024-09-13T15:18:08.380 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141274+0000", 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141274+0000", 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.381 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141273+0000", 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141278+0000", 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3630000000000004e-06 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141288+0000", 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0655e-05 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.141294+0000", 2024-09-13T15:18:08.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7309999999999998e-06 2024-09-13T15:18:08.383 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.383 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.383 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.143516+0000", 2024-09-13T15:18:08.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002222183 2024-09-13T15:18:08.383 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.383 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.383 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.143586+0000", 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9492000000000005e-05 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.143651+0000", 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4654999999999997e-05 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.145296+0000", 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001645758 2024-09-13T15:18:08.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.153894+0000", 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0085979769999999997 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709576+0000", 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.55568201299999997 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709600+0000", 2024-09-13T15:18:08.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3839e-05 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.709634+0000", 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3964000000000002e-05 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45764.0:213444 44.a 44:56de2f64:::benchmark_data_smithi099_188632_object106240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1073)", 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:08:14.159431+0000", 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 594.19239822099996, 2024-09-13T15:18:08.386 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.55524475500000003, 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45764", 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/542481954", 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 213444 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159431+0000", 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.387 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159431+0000", 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159429+0000", 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159437+0000", 2024-09-13T15:18:08.388 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.447e-06 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159452+0000", 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5247e-05 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159458+0000", 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0929999999999999e-06 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.389 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159494+0000", 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6439999999999997e-05 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159586+0000", 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2042999999999999e-05 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.159662+0000", 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5417e-05 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.390 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.162711+0000", 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00304931 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.165941+0000", 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032297720000000001 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.714579+0000", 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.54863759599999995 2024-09-13T15:18:08.391 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.714633+0000", 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4481999999999998e-05 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:08:14.714676+0000", 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2654000000000002e-05 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.392 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36057.0:37141 45.e 45:70df77e4:::benchmark_data_smithi099_210648_object37140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1088)", 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:10:06.208684+0000", 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 482.14314531999997, 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.499635261, 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36057", 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:18:08.393 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 37141 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.208684+0000", 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.208684+0000", 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.394 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.208683+0000", 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.208733+0000", 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0358999999999997e-05 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.208744+0000", 2024-09-13T15:18:08.395 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1206000000000001e-05 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.208749+0000", 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.284e-06 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.208783+0000", 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4813e-05 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.396 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.208893+0000", 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109831 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.209003+0000", 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010963000000000001 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.211521+0000", 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025184040000000001 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.397 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.212348+0000", 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00082643699999999996 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.708041+0000", 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49569363 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.708212+0000", 2024-09-13T15:18:08.398 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000171163 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.708319+0000", 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010661 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36057.0:37152 45.e 45:71a79d95:::benchmark_data_smithi099_210648_object37151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1088)", 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:10:06.215157+0000", 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 482.13667194700002, 2024-09-13T15:18:08.399 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49343962800000002, 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36057", 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 37152 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.215157+0000", 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.400 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.215157+0000", 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.215156+0000", 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.215230+0000", 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4114000000000002e-05 2024-09-13T15:18:08.401 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.215245+0000", 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4946000000000001e-05 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.215251+0000", 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8780000000000003e-06 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.215298+0000", 2024-09-13T15:18:08.402 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7037999999999999e-05 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.215425+0000", 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000127031 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.215569+0000", 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014385900000000001 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.403 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.217330+0000", 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00176155 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.217818+0000", 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000487529 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.708417+0000", 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49059883300000001 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.404 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.708518+0000", 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000101424 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.708597+0000", 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8835e-05 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.405 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36057.0:37163 45.e 45:7c84eee1:::benchmark_data_smithi099_210648_object37162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1088)", 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:10:06.221520+0000", 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 482.13030921299998, 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.495581728, 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36057", 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 37163 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.406 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.221520+0000", 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.221520+0000", 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.221519+0000", 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.407 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.221574+0000", 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5373e-05 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.221584+0000", 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6879999999999994e-06 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.221588+0000", 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2289999999999997e-06 2024-09-13T15:18:08.408 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.221630+0000", 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1627e-05 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.221728+0000", 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7977000000000001e-05 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:18:08.409 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.221823+0000", 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5433000000000006e-05 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.222900+0000", 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001077286 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.223205+0000", 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00030455100000000002 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.410 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.716842+0000", 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49363739200000001 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.716978+0000", 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013549699999999999 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:10:06.717102+0000", 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012373399999999999 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.411 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36057.0:154943 45.e 45:7bc32636:::benchmark_data_smithi099_210648_object53210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1092)", 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:12:28.867280+0000", 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 339.48454913799998, 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.65986736800000001, 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36057", 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 154943 2024-09-13T15:18:08.412 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.867280+0000", 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.867280+0000", 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.867278+0000", 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.867285+0000", 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6889999999999998e-06 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.867300+0000", 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4938e-05 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.414 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.867306+0000", 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7860000000000002e-06 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.867358+0000", 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2194999999999997e-05 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.415 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.867476+0000", 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011758 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.867585+0000", 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109864 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.872922+0000", 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005337049 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.416 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:28.873261+0000", 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000338146 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:29.527016+0000", 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.65375554800000002 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:29.527081+0000", 2024-09-13T15:18:08.417 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4836000000000001e-05 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:29.527147+0000", 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6395000000000004e-05 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36057.0:164185 45.e 45:7d6e013a:::benchmark_data_smithi099_210648_object62452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1092)", 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:12:44.143495+0000", 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 324.208333917, 2024-09-13T15:18:08.418 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.478834549, 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36057", 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/3195549015", 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 164185 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.143495+0000", 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.419 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.143495+0000", 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.143494+0000", 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.143500+0000", 2024-09-13T15:18:08.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6159999999999998e-06 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.143515+0000", 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4217e-05 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.143524+0000", 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3519999999999996e-06 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.421 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.143584+0000", 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0476000000000002e-05 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.143739+0000", 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015434800000000001 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.143840+0000", 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000101474 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.422 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.148568+0000", 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0047278889999999999 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.150041+0000", 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014725910000000001 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.621867+0000", 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.47182633800000001 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.423 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.622167+0000", 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000300153 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:12:44.622330+0000", 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016259900000000001 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.424 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:53806 46.4 46:278b9455:::benchmark_data_smithi099_233884_object53805:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:50.790196+0000", 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 137.56163281799999, 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49713887099999998, 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 53806 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.425 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790196+0000", 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790196+0000", 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790195+0000", 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790234+0000", 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9099e-05 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790249+0000", 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5415000000000001e-05 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790256+0000", 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3280000000000002e-06 2024-09-13T15:18:08.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790308+0000", 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2525000000000003e-05 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790440+0000", 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013153899999999999 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790550+0000", 2024-09-13T15:18:08.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110309 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.792375+0000", 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018250899999999999 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.793986+0000", 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00161035 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.287210+0000", 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49322427800000002 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.287290+0000", 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9741999999999996e-05 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.287335+0000", 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5661000000000002e-05 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.430 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:53807 46.a 46:56308894:::benchmark_data_smithi099_233884_object53806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:50.792892+0000", 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 137.55893679799999, 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49443502700000003, 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:18:08.431 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 53807 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.792892+0000", 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.792892+0000", 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.432 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.792891+0000", 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.792930+0000", 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9240999999999999e-05 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.792945+0000", 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5149e-05 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.433 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.434 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.434 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.792952+0000", 2024-09-13T15:18:08.434 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2899999999999999e-06 2024-09-13T15:18:08.434 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.434 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.434 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.434 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.793003+0000", 2024-09-13T15:18:08.434 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1369999999999998e-05 2024-09-13T15:18:08.435 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.435 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.435 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.435 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.793133+0000", 2024-09-13T15:18:08.435 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013032699999999999 2024-09-13T15:18:08.435 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.435 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.435 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.793268+0000", 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013420399999999999 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.795060+0000", 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001792096 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.795444+0000", 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00038453800000000002 2024-09-13T15:18:08.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.287168+0000", 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49172335499999997 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.287270+0000", 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010276 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.287327+0000", 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7012000000000002e-05 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:53818 46.4 46:2783c1b4:::benchmark_data_smithi099_233884_object53817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:50.799624+0000", 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 137.55220509099999, 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49978747800000001, 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.438 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 53818 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799624+0000", 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799624+0000", 2024-09-13T15:18:08.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799622+0000", 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799668+0000", 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5992000000000003e-05 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799684+0000", 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5778e-05 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799691+0000", 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6150000000000002e-06 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799741+0000", 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9752000000000001e-05 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.441 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799871+0000", 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013068199999999999 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.800012+0000", 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014106700000000001 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.802376+0000", 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023633299999999999 2024-09-13T15:18:08.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.804500+0000", 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021246860000000002 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.299299+0000", 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49479895400000001 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.443 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.299357+0000", 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7238000000000003e-05 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.299412+0000", 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4997000000000002e-05 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:56928 46.4 46:200211df:::benchmark_data_smithi099_233884_object56927:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:18:08.444 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:53.548758+0000", 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 134.80307148700001, 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.46825767099999999, 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 56928 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.548758+0000", 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.548758+0000", 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.548756+0000", 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.548826+0000", 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0072000000000003e-05 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.548840+0000", 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3315999999999999e-05 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.548845+0000", 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9410000000000004e-06 2024-09-13T15:18:08.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.548898+0000", 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2305000000000002e-05 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.548979+0000", 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1624000000000003e-05 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:18:08.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.549058+0000", 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8801999999999995e-05 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.550986+0000", 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019279340000000001 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:53.554039+0000", 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030527029999999999 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.449 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:54.016916+0000", 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.46287702600000002 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:54.016971+0000", 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4681e-05 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:54.017015+0000", 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4777000000000002e-05 2024-09-13T15:18:08.450 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89299 46.4 46:24ece472:::benchmark_data_smithi099_233884_object89298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.721651+0000", 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 96.630178567000002, 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.00749683, 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.451 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89299 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721651+0000", 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721651+0000", 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721649+0000", 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721697+0000", 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7987e-05 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721705+0000", 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8619999999999992e-06 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721710+0000", 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7389999999999999e-06 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721807+0000", 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6679000000000001e-05 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721817+0000", 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0407e-05 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930714+0000", 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.20889722199999999 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930724+0000", 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1300000000000007e-06 2024-09-13T15:18:08.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.456 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.456 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.456 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.729055+0000", 2024-09-13T15:18:08.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.79833184800000001 2024-09-13T15:18:08.456 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.456 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.456 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.456 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.729147+0000", 2024-09-13T15:18:08.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1933000000000002e-05 2024-09-13T15:18:08.456 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89306 46.14 46:29c5ea78:::benchmark_data_smithi099_233884_object89305:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.721727+0000", 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 96.630102238999996, 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1242219280000001, 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:18:08.457 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89306 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721727+0000", 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721727+0000", 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721726+0000", 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721765+0000", 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8800000000000001e-05 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721774+0000", 2024-09-13T15:18:08.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3260000000000002e-06 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721780+0000", 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5010000000000004e-06 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930834+0000", 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.20905432500000001 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:18:08.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930840+0000", 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4720000000000003e-06 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.845819+0000", 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91497909300000002 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.845949+0000", 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012993400000000001 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.461 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89321 46.4 46:24190b25:::benchmark_data_smithi099_233884_object89320:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.819689+0000", 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 96.532139822999994, 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.016089185, 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89321 2024-09-13T15:18:08.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819689+0000", 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819689+0000", 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819686+0000", 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999967 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819746+0000", 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0609000000000002e-05 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819787+0000", 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0629e-05 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819800+0000", 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2469999999999999e-05 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930861+0000", 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.111061036 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930867+0000", 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4110000000000001e-06 2024-09-13T15:18:08.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.729174+0000", 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.79830749499999998 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.729330+0000", 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015536 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.729996+0000", 2024-09-13T15:18:08.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00066596500000000002 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.733110+0000", 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031144440000000001 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.743335+0000", 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010225237 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.835687+0000", 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.092351741000000001 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.835738+0000", 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0723999999999997e-05 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.835778+0000", 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0571999999999997e-05 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.468 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:107356 46.4 46:27540f99:::benchmark_data_smithi099_233884_object4943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:57.940638+0000", 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 70.411191602000002, 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68978922200000004, 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 107356 2024-09-13T15:18:08.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940638+0000", 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940638+0000", 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940636+0000", 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940645+0000", 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2750000000000001e-06 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940664+0000", 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8668000000000002e-05 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940671+0000", 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7510000000000008e-06 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940727+0000", 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6079999999999998e-05 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940882+0000", 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015453299999999999 2024-09-13T15:18:08.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940988+0000", 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000106291 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.942812+0000", 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018234010000000001 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.967954+0000", 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025142178000000001 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630310+0000", 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.66235603499999995 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630380+0000", 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0083e-05 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630427+0000", 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6864999999999999e-05 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:107364 46.a 46:5497e79b:::benchmark_data_smithi099_233884_object4951:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:57.968424+0000", 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 70.383405401999994, 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.66201485000000004, 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.475 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 107364 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968424+0000", 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968424+0000", 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968422+0000", 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968431+0000", 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8980000000000002e-06 2024-09-13T15:18:08.477 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968446+0000", 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4873e-05 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968452+0000", 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4679999999999997e-06 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968508+0000", 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5238999999999997e-05 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968604+0000", 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6761000000000004e-05 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968670+0000", 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5514999999999999e-05 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972366+0000", 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036965930000000002 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.973672+0000", 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013058600000000001 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630369+0000", 2024-09-13T15:18:08.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.65669628599999996 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630404+0000", 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5707999999999998e-05 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630439+0000", 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4303e-05 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.481 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:107368 46.4 46:220f6261:::benchmark_data_smithi099_233884_object4955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:57.972723+0000", 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 70.379105757999994, 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.65779028399999995, 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 107368 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.482 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972723+0000", 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972723+0000", 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:18:08.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972722+0000", 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972728+0000", 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.631e-06 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972739+0000", 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0876000000000001e-05 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972745+0000", 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1480000000000003e-06 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972796+0000", 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1777999999999998e-05 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972912+0000", 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000115529 2024-09-13T15:18:08.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972984+0000", 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2489000000000003e-05 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.974819+0000", 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018349550000000001 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.486 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.975128+0000", 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00030878200000000001 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630464+0000", 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.65533615300000003 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630493+0000", 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8561999999999999e-05 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:18:08.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:18:08.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630514+0000", 2024-09-13T15:18:08.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0755e-05 2024-09-13T15:18:08.488 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.488 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.488 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.488 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:18:08.488 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:18:08.488 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:18:08.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:08.520 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:08.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:08.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:08.627 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:08.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:08.668 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:08.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:08.769 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:08.787 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:08.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:08.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:08.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:08.917 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:08.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:08.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:09.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:09.057 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:09.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:09.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:09.164 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:09.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:09.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:09.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:09.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:09.312 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:09.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:09.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:09.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:09 smithi099 bash[84576]: cluster 2024-09-13T15:18:08.010901+0000 osd.7 (osd.7) 19 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:18:09.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:09 smithi099 bash[84576]: cluster 2024-09-13T15:18:08.010901+0000 osd.7 (osd.7) 19 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:18:09.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:09 smithi099 bash[84576]: cluster 2024-09-13T15:18:08.013453+0000 osd.7 (osd.7) 20 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:18:09.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:09 smithi099 bash[84576]: cluster 2024-09-13T15:18:08.013453+0000 osd.7 (osd.7) 20 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:18:09.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:09 smithi099 bash[87147]: cluster 2024-09-13T15:18:08.010901+0000 osd.7 (osd.7) 19 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:18:09.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:09 smithi099 bash[87147]: cluster 2024-09-13T15:18:08.010901+0000 osd.7 (osd.7) 19 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:18:09.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:09 smithi099 bash[87147]: cluster 2024-09-13T15:18:08.013453+0000 osd.7 (osd.7) 20 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:18:09.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:09 smithi099 bash[87147]: cluster 2024-09-13T15:18:08.013453+0000 osd.7 (osd.7) 20 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:18:09.420 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:09.446 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:09.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:09.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:18:09.453 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:18:09.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:09.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:09.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:09.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:09.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:09 smithi177 bash[68594]: cluster 2024-09-13T15:18:08.010901+0000 osd.7 (osd.7) 19 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:18:09.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:09 smithi177 bash[68594]: cluster 2024-09-13T15:18:08.010901+0000 osd.7 (osd.7) 19 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:18:09.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:09 smithi177 bash[68594]: cluster 2024-09-13T15:18:08.013453+0000 osd.7 (osd.7) 20 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:18:09.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:09 smithi177 bash[68594]: cluster 2024-09-13T15:18:08.013453+0000 osd.7 (osd.7) 20 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:18:09.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:18:09.682 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:09.752 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:09.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:09.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:09.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:09.885 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:09.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:09.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:10.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:10.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:10.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:10.128 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:10.164 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:10.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:10.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:10.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:10.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:10.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:10.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:10.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[84576]: cluster 2024-09-13T15:18:09.012701+0000 osd.7 (osd.7) 21 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:18:10.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[84576]: cluster 2024-09-13T15:18:09.012701+0000 osd.7 (osd.7) 21 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:18:10.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[84576]: cluster 2024-09-13T15:18:09.014019+0000 osd.7 (osd.7) 22 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:18:10.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[84576]: cluster 2024-09-13T15:18:09.014019+0000 osd.7 (osd.7) 22 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:18:10.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[84576]: cluster 2024-09-13T15:18:09.120149+0000 osd.6 (osd.6) 27 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:10.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[84576]: cluster 2024-09-13T15:18:09.120149+0000 osd.6 (osd.6) 27 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:10.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[84576]: cluster 2024-09-13T15:18:09.222295+0000 mgr.y (mgr.26252) 1565 : cluster [DBG] pgmap v2179: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 4163/132990 objects degraded (3.130%); 1123/132990 objects misplaced (0.844%); 3.5 MiB/s, 57 objects/s recovering 2024-09-13T15:18:10.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[84576]: cluster 2024-09-13T15:18:09.222295+0000 mgr.y (mgr.26252) 1565 : cluster [DBG] pgmap v2179: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 4163/132990 objects degraded (3.130%); 1123/132990 objects misplaced (0.844%); 3.5 MiB/s, 57 objects/s recovering 2024-09-13T15:18:10.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[87147]: cluster 2024-09-13T15:18:09.012701+0000 osd.7 (osd.7) 21 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:18:10.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[87147]: cluster 2024-09-13T15:18:09.012701+0000 osd.7 (osd.7) 21 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:18:10.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[87147]: cluster 2024-09-13T15:18:09.014019+0000 osd.7 (osd.7) 22 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:18:10.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[87147]: cluster 2024-09-13T15:18:09.014019+0000 osd.7 (osd.7) 22 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:18:10.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[87147]: cluster 2024-09-13T15:18:09.120149+0000 osd.6 (osd.6) 27 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:10.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[87147]: cluster 2024-09-13T15:18:09.120149+0000 osd.6 (osd.6) 27 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:10.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[87147]: cluster 2024-09-13T15:18:09.222295+0000 mgr.y (mgr.26252) 1565 : cluster [DBG] pgmap v2179: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 4163/132990 objects degraded (3.130%); 1123/132990 objects misplaced (0.844%); 3.5 MiB/s, 57 objects/s recovering 2024-09-13T15:18:10.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:10 smithi099 bash[87147]: cluster 2024-09-13T15:18:09.222295+0000 mgr.y (mgr.26252) 1565 : cluster [DBG] pgmap v2179: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 4163/132990 objects degraded (3.130%); 1123/132990 objects misplaced (0.844%); 3.5 MiB/s, 57 objects/s recovering 2024-09-13T15:18:10.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:18:10 smithi177 bash[61439]: ts=2024-09-13T15:18:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:18:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:10 smithi177 bash[68594]: cluster 2024-09-13T15:18:09.012701+0000 osd.7 (osd.7) 21 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:18:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:10 smithi177 bash[68594]: cluster 2024-09-13T15:18:09.012701+0000 osd.7 (osd.7) 21 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:18:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:10 smithi177 bash[68594]: cluster 2024-09-13T15:18:09.014019+0000 osd.7 (osd.7) 22 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:18:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:10 smithi177 bash[68594]: cluster 2024-09-13T15:18:09.014019+0000 osd.7 (osd.7) 22 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:18:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:10 smithi177 bash[68594]: cluster 2024-09-13T15:18:09.120149+0000 osd.6 (osd.6) 27 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:10.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:10 smithi177 bash[68594]: cluster 2024-09-13T15:18:09.120149+0000 osd.6 (osd.6) 27 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:10.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:10 smithi177 bash[68594]: cluster 2024-09-13T15:18:09.222295+0000 mgr.y (mgr.26252) 1565 : cluster [DBG] pgmap v2179: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 4163/132990 objects degraded (3.130%); 1123/132990 objects misplaced (0.844%); 3.5 MiB/s, 57 objects/s recovering 2024-09-13T15:18:10.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:10 smithi177 bash[68594]: cluster 2024-09-13T15:18:09.222295+0000 mgr.y (mgr.26252) 1565 : cluster [DBG] pgmap v2179: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 4163/132990 objects degraded (3.130%); 1123/132990 objects misplaced (0.844%); 3.5 MiB/s, 57 objects/s recovering 2024-09-13T15:18:10.398 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:10.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:10.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:10.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:10.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:10.558 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:10.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:10.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:10.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:10.687 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:10.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:10.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:10.791 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:10.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:10.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:10.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:10.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:10.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:10.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:10.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:11.065 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:11.091 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:11.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:11.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:11.195 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:11.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:11.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:11.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:11.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:11.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:11 smithi099 bash[84576]: cluster 2024-09-13T15:18:10.988485+0000 osd.7 (osd.7) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:18:11.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:11 smithi099 bash[84576]: cluster 2024-09-13T15:18:10.988485+0000 osd.7 (osd.7) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:18:11.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:11 smithi099 bash[87147]: cluster 2024-09-13T15:18:10.988485+0000 osd.7 (osd.7) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:18:11.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:11 smithi099 bash[87147]: cluster 2024-09-13T15:18:10.988485+0000 osd.7 (osd.7) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:18:11.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:11.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:11.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:11 smithi177 bash[68594]: cluster 2024-09-13T15:18:10.988485+0000 osd.7 (osd.7) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:18:11.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:11 smithi177 bash[68594]: cluster 2024-09-13T15:18:10.988485+0000 osd.7 (osd.7) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:18:11.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:11.489 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:18:11.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:11.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:11.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:11.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:11.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:11.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:11.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:11.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:11.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:11.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:11.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:11.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:11.880 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:11.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:11.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:11.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:12.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:12.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:12.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:12.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:12.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[84576]: cluster 2024-09-13T15:18:10.990621+0000 osd.7 (osd.7) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:18:12.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[84576]: cluster 2024-09-13T15:18:10.990621+0000 osd.7 (osd.7) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:18:12.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[84576]: cluster 2024-09-13T15:18:11.223088+0000 mgr.y (mgr.26252) 1566 : cluster [DBG] pgmap v2180: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 3897/125193 objects degraded (3.113%); 943/125193 objects misplaced (0.753%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:12.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[84576]: cluster 2024-09-13T15:18:11.223088+0000 mgr.y (mgr.26252) 1566 : cluster [DBG] pgmap v2180: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 3897/125193 objects degraded (3.113%); 943/125193 objects misplaced (0.753%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:12.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[84576]: cluster 2024-09-13T15:18:11.667503+0000 mon.a (mon.0) 2736 : cluster [WRN] Health check update: Degraded data redundancy: 4163/132990 objects degraded (3.130%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:12.121 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[84576]: cluster 2024-09-13T15:18:11.667503+0000 mon.a (mon.0) 2736 : cluster [WRN] Health check update: Degraded data redundancy: 4163/132990 objects degraded (3.130%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:12.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[87147]: cluster 2024-09-13T15:18:10.990621+0000 osd.7 (osd.7) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:18:12.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[87147]: cluster 2024-09-13T15:18:10.990621+0000 osd.7 (osd.7) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:18:12.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[87147]: cluster 2024-09-13T15:18:11.223088+0000 mgr.y (mgr.26252) 1566 : cluster [DBG] pgmap v2180: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 3897/125193 objects degraded (3.113%); 943/125193 objects misplaced (0.753%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:12.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[87147]: cluster 2024-09-13T15:18:11.223088+0000 mgr.y (mgr.26252) 1566 : cluster [DBG] pgmap v2180: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 3897/125193 objects degraded (3.113%); 943/125193 objects misplaced (0.753%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:12.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[87147]: cluster 2024-09-13T15:18:11.667503+0000 mon.a (mon.0) 2736 : cluster [WRN] Health check update: Degraded data redundancy: 4163/132990 objects degraded (3.130%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:12.122 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:12 smithi099 bash[87147]: cluster 2024-09-13T15:18:11.667503+0000 mon.a (mon.0) 2736 : cluster [WRN] Health check update: Degraded data redundancy: 4163/132990 objects degraded (3.130%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:12.149 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:12.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:12.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:12.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:12.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:12 smithi177 bash[68594]: cluster 2024-09-13T15:18:10.990621+0000 osd.7 (osd.7) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:18:12.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:12 smithi177 bash[68594]: cluster 2024-09-13T15:18:10.990621+0000 osd.7 (osd.7) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:18:12.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:12 smithi177 bash[68594]: cluster 2024-09-13T15:18:11.223088+0000 mgr.y (mgr.26252) 1566 : cluster [DBG] pgmap v2180: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 3897/125193 objects degraded (3.113%); 943/125193 objects misplaced (0.753%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:12.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:12 smithi177 bash[68594]: cluster 2024-09-13T15:18:11.223088+0000 mgr.y (mgr.26252) 1566 : cluster [DBG] pgmap v2180: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 3897/125193 objects degraded (3.113%); 943/125193 objects misplaced (0.753%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T15:18:12.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:12 smithi177 bash[68594]: cluster 2024-09-13T15:18:11.667503+0000 mon.a (mon.0) 2736 : cluster [WRN] Health check update: Degraded data redundancy: 4163/132990 objects degraded (3.130%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:12.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:12 smithi177 bash[68594]: cluster 2024-09-13T15:18:11.667503+0000 mon.a (mon.0) 2736 : cluster [WRN] Health check update: Degraded data redundancy: 4163/132990 objects degraded (3.130%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:12.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:12.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:12.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:12.386 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:12.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:12.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:12.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:12.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:12.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:12.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:12.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:12.639 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:12.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:12.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:12.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:12.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:12.799 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:12.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:12.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:12.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:12.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:12.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:12.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:13.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:13.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:13.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:13.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:13.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:13.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:13.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:13.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:13.282 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:13.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:13.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:13.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:13.420 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:13.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:13.448 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:13.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:13.531 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:18:13.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:13.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:13.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:13.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:13.679 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:13.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:13.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:13.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:13.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:13.843 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:13.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:13.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:13.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:13.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:13.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:13.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:14.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:14.099 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:14.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:14.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:14.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:14.223 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:14.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:14.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:14.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:14.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:14.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:14.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:14.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:14 smithi177 bash[68594]: cluster 2024-09-13T15:18:13.223693+0000 mgr.y (mgr.26252) 1567 : cluster [DBG] pgmap v2181: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 3897/123120 objects degraded (3.165%); 943/123120 objects misplaced (0.766%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:14.460 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:14 smithi177 bash[68594]: cluster 2024-09-13T15:18:13.223693+0000 mgr.y (mgr.26252) 1567 : cluster [DBG] pgmap v2181: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 3897/123120 objects degraded (3.165%); 943/123120 objects misplaced (0.766%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:14.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:14.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:14.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:14 smithi099 bash[84576]: cluster 2024-09-13T15:18:13.223693+0000 mgr.y (mgr.26252) 1567 : cluster [DBG] pgmap v2181: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 3897/123120 objects degraded (3.165%); 943/123120 objects misplaced (0.766%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:14.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:14 smithi099 bash[84576]: cluster 2024-09-13T15:18:13.223693+0000 mgr.y (mgr.26252) 1567 : cluster [DBG] pgmap v2181: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 3897/123120 objects degraded (3.165%); 943/123120 objects misplaced (0.766%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:14 smithi099 bash[87147]: cluster 2024-09-13T15:18:13.223693+0000 mgr.y (mgr.26252) 1567 : cluster [DBG] pgmap v2181: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 3897/123120 objects degraded (3.165%); 943/123120 objects misplaced (0.766%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:14 smithi099 bash[87147]: cluster 2024-09-13T15:18:13.223693+0000 mgr.y (mgr.26252) 1567 : cluster [DBG] pgmap v2181: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 3897/123120 objects degraded (3.165%); 943/123120 objects misplaced (0.766%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:14.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:14.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:14.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:14.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:14.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:14.745 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:14.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:14.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:14.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:14.876 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:14.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:14.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:14.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:15.006 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:15.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:15.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:15.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:15.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:15.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:15.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:15.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:15.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:15.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:15.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:15.401 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:15.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:15.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:15.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:15.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:15.545 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:15 smithi099 bash[84576]: cluster 2024-09-13T15:18:14.123411+0000 osd.7 (osd.7) 25 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:18:15.545 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:15 smithi099 bash[84576]: cluster 2024-09-13T15:18:14.123411+0000 osd.7 (osd.7) 25 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:18:15.545 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:15 smithi099 bash[84576]: cluster 2024-09-13T15:18:14.125126+0000 osd.7 (osd.7) 26 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:18:15.545 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:15 smithi099 bash[84576]: cluster 2024-09-13T15:18:14.125126+0000 osd.7 (osd.7) 26 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:18:15.546 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:15 smithi099 bash[87147]: cluster 2024-09-13T15:18:14.123411+0000 osd.7 (osd.7) 25 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:18:15.546 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:15 smithi099 bash[87147]: cluster 2024-09-13T15:18:14.123411+0000 osd.7 (osd.7) 25 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:18:15.546 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:15 smithi099 bash[87147]: cluster 2024-09-13T15:18:14.125126+0000 osd.7 (osd.7) 26 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:18:15.546 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:15 smithi099 bash[87147]: cluster 2024-09-13T15:18:14.125126+0000 osd.7 (osd.7) 26 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:18:15.571 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:15.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:15.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:15 smithi177 bash[68594]: cluster 2024-09-13T15:18:14.123411+0000 osd.7 (osd.7) 25 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:18:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:15 smithi177 bash[68594]: cluster 2024-09-13T15:18:14.123411+0000 osd.7 (osd.7) 25 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:18:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:15 smithi177 bash[68594]: cluster 2024-09-13T15:18:14.125126+0000 osd.7 (osd.7) 26 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:18:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:15 smithi177 bash[68594]: cluster 2024-09-13T15:18:14.125126+0000 osd.7 (osd.7) 26 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:18:15.675 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:15.713 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:15.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:15.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:15.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:15.849 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:15.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:15.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:15.988 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:15.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:15.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:16.091 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:16.111 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:16.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:16.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:16.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:16.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:16.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:16.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:16.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:16.383 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:16.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:16.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:16.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:16.505 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:16.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:16.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:16.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:16 smithi099 bash[84576]: cluster 2024-09-13T15:18:15.224396+0000 mgr.y (mgr.26252) 1568 : cluster [DBG] pgmap v2182: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 3637/115725 objects degraded (3.143%); 767/115725 objects misplaced (0.663%); 4.8 MiB/s, 78 objects/s recovering 2024-09-13T15:18:16.589 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:16 smithi099 bash[84576]: cluster 2024-09-13T15:18:15.224396+0000 mgr.y (mgr.26252) 1568 : cluster [DBG] pgmap v2182: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 3637/115725 objects degraded (3.143%); 767/115725 objects misplaced (0.663%); 4.8 MiB/s, 78 objects/s recovering 2024-09-13T15:18:16.589 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:16 smithi099 bash[87147]: cluster 2024-09-13T15:18:15.224396+0000 mgr.y (mgr.26252) 1568 : cluster [DBG] pgmap v2182: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 3637/115725 objects degraded (3.143%); 767/115725 objects misplaced (0.663%); 4.8 MiB/s, 78 objects/s recovering 2024-09-13T15:18:16.590 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:16 smithi099 bash[87147]: cluster 2024-09-13T15:18:15.224396+0000 mgr.y (mgr.26252) 1568 : cluster [DBG] pgmap v2182: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 3637/115725 objects degraded (3.143%); 767/115725 objects misplaced (0.663%); 4.8 MiB/s, 78 objects/s recovering 2024-09-13T15:18:16.608 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:16.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:16 smithi177 bash[68594]: cluster 2024-09-13T15:18:15.224396+0000 mgr.y (mgr.26252) 1568 : cluster [DBG] pgmap v2182: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 3637/115725 objects degraded (3.143%); 767/115725 objects misplaced (0.663%); 4.8 MiB/s, 78 objects/s recovering 2024-09-13T15:18:16.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:16 smithi177 bash[68594]: cluster 2024-09-13T15:18:15.224396+0000 mgr.y (mgr.26252) 1568 : cluster [DBG] pgmap v2182: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 3637/115725 objects degraded (3.143%); 767/115725 objects misplaced (0.663%); 4.8 MiB/s, 78 objects/s recovering 2024-09-13T15:18:16.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:16.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:16.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:16.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:16.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:16.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:16.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:16.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:16.920 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:16.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:16.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:17.026 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:17.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:17.055 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:17.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:17.156 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:17.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:17.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:17.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:17.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:17.316 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:18:17] "GET /metrics HTTP/1.1" 200 36802 "" "Prometheus/2.51.0" 2024-09-13T15:18:17.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:17.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:17.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:17.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:17.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:17.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:17.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:17.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[84576]: cluster 2024-09-13T15:18:16.201193+0000 osd.7 (osd.7) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:18:17.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[84576]: cluster 2024-09-13T15:18:16.201193+0000 osd.7 (osd.7) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:18:17.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[84576]: cluster 2024-09-13T15:18:16.213351+0000 osd.7 (osd.7) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:18:17.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[84576]: cluster 2024-09-13T15:18:16.213351+0000 osd.7 (osd.7) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:18:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[84576]: cluster 2024-09-13T15:18:16.668485+0000 mon.a (mon.0) 2737 : cluster [WRN] Health check update: Degraded data redundancy: 3637/115725 objects degraded (3.143%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[84576]: cluster 2024-09-13T15:18:16.668485+0000 mon.a (mon.0) 2737 : cluster [WRN] Health check update: Degraded data redundancy: 3637/115725 objects degraded (3.143%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[84576]: cluster 2024-09-13T15:18:17.225029+0000 mgr.y (mgr.26252) 1569 : cluster [DBG] pgmap v2183: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3637/113748 objects degraded (3.197%); 767/113748 objects misplaced (0.674%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:18:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[84576]: cluster 2024-09-13T15:18:17.225029+0000 mgr.y (mgr.26252) 1569 : cluster [DBG] pgmap v2183: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3637/113748 objects degraded (3.197%); 767/113748 objects misplaced (0.674%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:18:17.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[87147]: cluster 2024-09-13T15:18:16.201193+0000 osd.7 (osd.7) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:18:17.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[87147]: cluster 2024-09-13T15:18:16.201193+0000 osd.7 (osd.7) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:18:17.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[87147]: cluster 2024-09-13T15:18:16.213351+0000 osd.7 (osd.7) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:18:17.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[87147]: cluster 2024-09-13T15:18:16.213351+0000 osd.7 (osd.7) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:18:17.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[87147]: cluster 2024-09-13T15:18:16.668485+0000 mon.a (mon.0) 2737 : cluster [WRN] Health check update: Degraded data redundancy: 3637/115725 objects degraded (3.143%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:17.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[87147]: cluster 2024-09-13T15:18:16.668485+0000 mon.a (mon.0) 2737 : cluster [WRN] Health check update: Degraded data redundancy: 3637/115725 objects degraded (3.143%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:17.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[87147]: cluster 2024-09-13T15:18:17.225029+0000 mgr.y (mgr.26252) 1569 : cluster [DBG] pgmap v2183: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3637/113748 objects degraded (3.197%); 767/113748 objects misplaced (0.674%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:18:17.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:17 smithi099 bash[87147]: cluster 2024-09-13T15:18:17.225029+0000 mgr.y (mgr.26252) 1569 : cluster [DBG] pgmap v2183: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3637/113748 objects degraded (3.197%); 767/113748 objects misplaced (0.674%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:18:17.600 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:17.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:17.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:17.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:17 smithi177 bash[68594]: cluster 2024-09-13T15:18:16.201193+0000 osd.7 (osd.7) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:18:17.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:17 smithi177 bash[68594]: cluster 2024-09-13T15:18:16.201193+0000 osd.7 (osd.7) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:18:17.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:17 smithi177 bash[68594]: cluster 2024-09-13T15:18:16.213351+0000 osd.7 (osd.7) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:18:17.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:17 smithi177 bash[68594]: cluster 2024-09-13T15:18:16.213351+0000 osd.7 (osd.7) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:18:17.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:17 smithi177 bash[68594]: cluster 2024-09-13T15:18:16.668485+0000 mon.a (mon.0) 2737 : cluster [WRN] Health check update: Degraded data redundancy: 3637/115725 objects degraded (3.143%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:17.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:17 smithi177 bash[68594]: cluster 2024-09-13T15:18:16.668485+0000 mon.a (mon.0) 2737 : cluster [WRN] Health check update: Degraded data redundancy: 3637/115725 objects degraded (3.143%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:17.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:17 smithi177 bash[68594]: cluster 2024-09-13T15:18:17.225029+0000 mgr.y (mgr.26252) 1569 : cluster [DBG] pgmap v2183: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3637/113748 objects degraded (3.197%); 767/113748 objects misplaced (0.674%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:18:17.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:17 smithi177 bash[68594]: cluster 2024-09-13T15:18:17.225029+0000 mgr.y (mgr.26252) 1569 : cluster [DBG] pgmap v2183: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3637/113748 objects degraded (3.197%); 767/113748 objects misplaced (0.674%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T15:18:17.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:17.738 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:17.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:17.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:17.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:17.870 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:17.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:17.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:17.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:18.012 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:18.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:18.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:18.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:18.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:18.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:18.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:18.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:18.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:18.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:18.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:18.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:18.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:18.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:18.535 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:18.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:18.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[84576]: cluster 2024-09-13T15:18:17.203806+0000 osd.7 (osd.7) 29 : cluster [DBG] 1.2 scrub starts 2024-09-13T15:18:18.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[84576]: cluster 2024-09-13T15:18:17.203806+0000 osd.7 (osd.7) 29 : cluster [DBG] 1.2 scrub starts 2024-09-13T15:18:18.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[84576]: cluster 2024-09-13T15:18:17.216941+0000 osd.7 (osd.7) 30 : cluster [DBG] 1.2 scrub ok 2024-09-13T15:18:18.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[84576]: cluster 2024-09-13T15:18:17.216941+0000 osd.7 (osd.7) 30 : cluster [DBG] 1.2 scrub ok 2024-09-13T15:18:18.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[84576]: audit 2024-09-13T15:18:17.602463+0000 mon.a (mon.0) 2738 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:18:18.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[84576]: audit 2024-09-13T15:18:17.602463+0000 mon.a (mon.0) 2738 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:18:18.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:18.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:18.579 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[87147]: cluster 2024-09-13T15:18:17.203806+0000 osd.7 (osd.7) 29 : cluster [DBG] 1.2 scrub starts 2024-09-13T15:18:18.579 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[87147]: cluster 2024-09-13T15:18:17.203806+0000 osd.7 (osd.7) 29 : cluster [DBG] 1.2 scrub starts 2024-09-13T15:18:18.579 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[87147]: cluster 2024-09-13T15:18:17.216941+0000 osd.7 (osd.7) 30 : cluster [DBG] 1.2 scrub ok 2024-09-13T15:18:18.579 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[87147]: cluster 2024-09-13T15:18:17.216941+0000 osd.7 (osd.7) 30 : cluster [DBG] 1.2 scrub ok 2024-09-13T15:18:18.579 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[87147]: audit 2024-09-13T15:18:17.602463+0000 mon.a (mon.0) 2738 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:18:18.579 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:18 smithi099 bash[87147]: audit 2024-09-13T15:18:17.602463+0000 mon.a (mon.0) 2738 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:18:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:18 smithi177 bash[68594]: cluster 2024-09-13T15:18:17.203806+0000 osd.7 (osd.7) 29 : cluster [DBG] 1.2 scrub starts 2024-09-13T15:18:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:18 smithi177 bash[68594]: cluster 2024-09-13T15:18:17.203806+0000 osd.7 (osd.7) 29 : cluster [DBG] 1.2 scrub starts 2024-09-13T15:18:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:18 smithi177 bash[68594]: cluster 2024-09-13T15:18:17.216941+0000 osd.7 (osd.7) 30 : cluster [DBG] 1.2 scrub ok 2024-09-13T15:18:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:18 smithi177 bash[68594]: cluster 2024-09-13T15:18:17.216941+0000 osd.7 (osd.7) 30 : cluster [DBG] 1.2 scrub ok 2024-09-13T15:18:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:18 smithi177 bash[68594]: audit 2024-09-13T15:18:17.602463+0000 mon.a (mon.0) 2738 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:18:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:18 smithi177 bash[68594]: audit 2024-09-13T15:18:17.602463+0000 mon.a (mon.0) 2738 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:18:18.677 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:18.695 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:18.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:18.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:18.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:18.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:18.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:18.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:18.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:18.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:19.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:19.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:19.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:19.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:19.201 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:19.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:19.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:19.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:19.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:19.364 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:19.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:19.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:19.368 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[84576]: audit 2024-09-13T15:18:18.339149+0000 mon.a (mon.0) 2739 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:19.368 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[84576]: audit 2024-09-13T15:18:18.339149+0000 mon.a (mon.0) 2739 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:19.368 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[84576]: cluster 2024-09-13T15:18:19.202830+0000 osd.6 (osd.6) 28 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:19.368 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[84576]: cluster 2024-09-13T15:18:19.202830+0000 osd.6 (osd.6) 28 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:19.368 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[84576]: cluster 2024-09-13T15:18:19.225792+0000 mgr.y (mgr.26252) 1570 : cluster [DBG] pgmap v2184: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 3637/107988 objects degraded (3.368%); 767/107988 objects misplaced (0.710%); 3.1 MiB/s, 51 objects/s recovering 2024-09-13T15:18:19.368 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[84576]: cluster 2024-09-13T15:18:19.225792+0000 mgr.y (mgr.26252) 1570 : cluster [DBG] pgmap v2184: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 3637/107988 objects degraded (3.368%); 767/107988 objects misplaced (0.710%); 3.1 MiB/s, 51 objects/s recovering 2024-09-13T15:18:19.368 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[87147]: audit 2024-09-13T15:18:18.339149+0000 mon.a (mon.0) 2739 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:19.369 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[87147]: audit 2024-09-13T15:18:18.339149+0000 mon.a (mon.0) 2739 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:19.369 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[87147]: cluster 2024-09-13T15:18:19.202830+0000 osd.6 (osd.6) 28 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:19.369 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[87147]: cluster 2024-09-13T15:18:19.202830+0000 osd.6 (osd.6) 28 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:19.369 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[87147]: cluster 2024-09-13T15:18:19.225792+0000 mgr.y (mgr.26252) 1570 : cluster [DBG] pgmap v2184: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 3637/107988 objects degraded (3.368%); 767/107988 objects misplaced (0.710%); 3.1 MiB/s, 51 objects/s recovering 2024-09-13T15:18:19.369 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:19 smithi099 bash[87147]: cluster 2024-09-13T15:18:19.225792+0000 mgr.y (mgr.26252) 1570 : cluster [DBG] pgmap v2184: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 3637/107988 objects degraded (3.368%); 767/107988 objects misplaced (0.710%); 3.1 MiB/s, 51 objects/s recovering 2024-09-13T15:18:19.469 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:19.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:19.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:19.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:19.599 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:19.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:19 smithi177 bash[68594]: audit 2024-09-13T15:18:18.339149+0000 mon.a (mon.0) 2739 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:19.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:19 smithi177 bash[68594]: audit 2024-09-13T15:18:18.339149+0000 mon.a (mon.0) 2739 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:19.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:19 smithi177 bash[68594]: cluster 2024-09-13T15:18:19.202830+0000 osd.6 (osd.6) 28 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:19.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:19 smithi177 bash[68594]: cluster 2024-09-13T15:18:19.202830+0000 osd.6 (osd.6) 28 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:19.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:19 smithi177 bash[68594]: cluster 2024-09-13T15:18:19.225792+0000 mgr.y (mgr.26252) 1570 : cluster [DBG] pgmap v2184: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 3637/107988 objects degraded (3.368%); 767/107988 objects misplaced (0.710%); 3.1 MiB/s, 51 objects/s recovering 2024-09-13T15:18:19.610 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:19 smithi177 bash[68594]: cluster 2024-09-13T15:18:19.225792+0000 mgr.y (mgr.26252) 1570 : cluster [DBG] pgmap v2184: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 3637/107988 objects degraded (3.368%); 767/107988 objects misplaced (0.710%); 3.1 MiB/s, 51 objects/s recovering 2024-09-13T15:18:19.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:19.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:19.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:19.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:19.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:19.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:19.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:19.860 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:19.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:19.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:19.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:19.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:20.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:20.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:20.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:20.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:20.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:20.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:20.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:20.254 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:20.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:20.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:20.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:20.410 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:20.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:20.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:20.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:20.547 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:20.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:20.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:20.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:18:20 smithi177 bash[61439]: ts=2024-09-13T15:18:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:18:20.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:20.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:20.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:20.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:20.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:20.811 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:20.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:20.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:20.919 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:20.948 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:20.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:20.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:21.054 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:21.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:21.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:21.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:21.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:21.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:21.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:21.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:21.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:21.342 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:21.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:21.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:21.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:21.468 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:21.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:21.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:21.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:21.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:21.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:21.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:21.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:21.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:21.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:21.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:21.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:21.852 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:21.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:21.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:21.956 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:21.983 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:21.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:21.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:22.086 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:22.122 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:22.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:22.227 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:22.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:22.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:22.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:22.363 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:22.384 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:22.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:22.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:22.515 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:22.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:22.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:22 smithi177 bash[68594]: cluster 2024-09-13T15:18:21.226746+0000 mgr.y (mgr.26252) 1571 : cluster [DBG] pgmap v2185: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3353/100989 objects degraded (3.320%); 565/100989 objects misplaced (0.559%); 4.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:22 smithi177 bash[68594]: cluster 2024-09-13T15:18:21.226746+0000 mgr.y (mgr.26252) 1571 : cluster [DBG] pgmap v2185: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3353/100989 objects degraded (3.320%); 565/100989 objects misplaced (0.559%); 4.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:22 smithi177 bash[68594]: cluster 2024-09-13T15:18:21.284416+0000 osd.7 (osd.7) 31 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:18:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:22 smithi177 bash[68594]: cluster 2024-09-13T15:18:21.284416+0000 osd.7 (osd.7) 31 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:18:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:22 smithi177 bash[68594]: cluster 2024-09-13T15:18:21.290006+0000 osd.7 (osd.7) 32 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:18:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:22 smithi177 bash[68594]: cluster 2024-09-13T15:18:21.290006+0000 osd.7 (osd.7) 32 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:18:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:22 smithi177 bash[68594]: cluster 2024-09-13T15:18:21.669349+0000 mon.a (mon.0) 2740 : cluster [WRN] Health check update: Degraded data redundancy: 3353/100989 objects degraded (3.320%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:22 smithi177 bash[68594]: cluster 2024-09-13T15:18:21.669349+0000 mon.a (mon.0) 2740 : cluster [WRN] Health check update: Degraded data redundancy: 3353/100989 objects degraded (3.320%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:22.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:22.636 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:22.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:22.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:22.739 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:22.770 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:22.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:22.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:22.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[84576]: cluster 2024-09-13T15:18:21.226746+0000 mgr.y (mgr.26252) 1571 : cluster [DBG] pgmap v2185: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3353/100989 objects degraded (3.320%); 565/100989 objects misplaced (0.559%); 4.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:22.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[84576]: cluster 2024-09-13T15:18:21.226746+0000 mgr.y (mgr.26252) 1571 : cluster [DBG] pgmap v2185: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3353/100989 objects degraded (3.320%); 565/100989 objects misplaced (0.559%); 4.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:22.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[84576]: cluster 2024-09-13T15:18:21.284416+0000 osd.7 (osd.7) 31 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:18:22.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[84576]: cluster 2024-09-13T15:18:21.284416+0000 osd.7 (osd.7) 31 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:18:22.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[84576]: cluster 2024-09-13T15:18:21.290006+0000 osd.7 (osd.7) 32 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:18:22.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[84576]: cluster 2024-09-13T15:18:21.290006+0000 osd.7 (osd.7) 32 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:18:22.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[84576]: cluster 2024-09-13T15:18:21.669349+0000 mon.a (mon.0) 2740 : cluster [WRN] Health check update: Degraded data redundancy: 3353/100989 objects degraded (3.320%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:22.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[84576]: cluster 2024-09-13T15:18:21.669349+0000 mon.a (mon.0) 2740 : cluster [WRN] Health check update: Degraded data redundancy: 3353/100989 objects degraded (3.320%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:22.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[87147]: cluster 2024-09-13T15:18:21.226746+0000 mgr.y (mgr.26252) 1571 : cluster [DBG] pgmap v2185: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3353/100989 objects degraded (3.320%); 565/100989 objects misplaced (0.559%); 4.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:22.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[87147]: cluster 2024-09-13T15:18:21.226746+0000 mgr.y (mgr.26252) 1571 : cluster [DBG] pgmap v2185: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 3353/100989 objects degraded (3.320%); 565/100989 objects misplaced (0.559%); 4.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:22.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[87147]: cluster 2024-09-13T15:18:21.284416+0000 osd.7 (osd.7) 31 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:18:22.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[87147]: cluster 2024-09-13T15:18:21.284416+0000 osd.7 (osd.7) 31 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:18:22.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[87147]: cluster 2024-09-13T15:18:21.290006+0000 osd.7 (osd.7) 32 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:18:22.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[87147]: cluster 2024-09-13T15:18:21.290006+0000 osd.7 (osd.7) 32 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:18:22.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[87147]: cluster 2024-09-13T15:18:21.669349+0000 mon.a (mon.0) 2740 : cluster [WRN] Health check update: Degraded data redundancy: 3353/100989 objects degraded (3.320%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:22.828 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:22 smithi099 bash[87147]: cluster 2024-09-13T15:18:21.669349+0000 mon.a (mon.0) 2740 : cluster [WRN] Health check update: Degraded data redundancy: 3353/100989 objects degraded (3.320%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:22.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:22.903 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:22.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:22.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:23.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:23.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:23.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:23.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:23.152 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:23.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:23.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:23.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:23.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:23.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:23.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:23.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:23.433 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:23.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:23.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:23.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:23.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:23.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:23 smithi099 bash[84576]: cluster 2024-09-13T15:18:23.227260+0000 mgr.y (mgr.26252) 1572 : cluster [DBG] pgmap v2186: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 3353/99177 objects degraded (3.381%); 565/99177 objects misplaced (0.570%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:23.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:23 smithi099 bash[84576]: cluster 2024-09-13T15:18:23.227260+0000 mgr.y (mgr.26252) 1572 : cluster [DBG] pgmap v2186: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 3353/99177 objects degraded (3.381%); 565/99177 objects misplaced (0.570%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:23.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:23 smithi099 bash[84576]: cluster 2024-09-13T15:18:23.237172+0000 osd.6 (osd.6) 29 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:23.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:23 smithi099 bash[84576]: cluster 2024-09-13T15:18:23.237172+0000 osd.6 (osd.6) 29 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:23.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:23 smithi099 bash[87147]: cluster 2024-09-13T15:18:23.227260+0000 mgr.y (mgr.26252) 1572 : cluster [DBG] pgmap v2186: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 3353/99177 objects degraded (3.381%); 565/99177 objects misplaced (0.570%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:23.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:23 smithi099 bash[87147]: cluster 2024-09-13T15:18:23.227260+0000 mgr.y (mgr.26252) 1572 : cluster [DBG] pgmap v2186: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 3353/99177 objects degraded (3.381%); 565/99177 objects misplaced (0.570%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:23.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:23 smithi099 bash[87147]: cluster 2024-09-13T15:18:23.237172+0000 osd.6 (osd.6) 29 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:23.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:23 smithi099 bash[87147]: cluster 2024-09-13T15:18:23.237172+0000 osd.6 (osd.6) 29 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:23.594 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:23 smithi177 bash[68594]: cluster 2024-09-13T15:18:23.227260+0000 mgr.y (mgr.26252) 1572 : cluster [DBG] pgmap v2186: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 3353/99177 objects degraded (3.381%); 565/99177 objects misplaced (0.570%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:23.594 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:23 smithi177 bash[68594]: cluster 2024-09-13T15:18:23.227260+0000 mgr.y (mgr.26252) 1572 : cluster [DBG] pgmap v2186: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 3353/99177 objects degraded (3.381%); 565/99177 objects misplaced (0.570%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T15:18:23.594 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:23 smithi177 bash[68594]: cluster 2024-09-13T15:18:23.237172+0000 osd.6 (osd.6) 29 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:23.595 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:23 smithi177 bash[68594]: cluster 2024-09-13T15:18:23.237172+0000 osd.6 (osd.6) 29 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:23.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:23.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:23.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:23.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:23.745 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:23.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:23.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:23.848 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:23.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:23.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:23.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:23.984 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:24.017 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:24.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:24.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:24.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:24.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:24.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:24.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:24.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:24.264 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:24.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:24.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:24.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:24.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:24.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:24.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:24.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:24.524 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:24.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:24.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:24.628 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:24.667 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:24.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:24.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:24.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:24.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:24.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:24.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:24.939 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:24.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:24.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:25.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:25.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:25.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:25.168 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:25.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:25.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:25.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:25.294 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:25.332 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:25.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:25.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:25.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:25.462 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:25.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:25.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:25.565 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:25.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:25.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:25.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:25 smithi177 bash[68594]: audit 2024-09-13T15:18:24.251372+0000 mon.a (mon.0) 2741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:25 smithi177 bash[68594]: audit 2024-09-13T15:18:24.251372+0000 mon.a (mon.0) 2741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:25 smithi177 bash[68594]: cluster 2024-09-13T15:18:24.254437+0000 osd.6 (osd.6) 30 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:25 smithi177 bash[68594]: cluster 2024-09-13T15:18:24.254437+0000 osd.6 (osd.6) 30 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:25 smithi177 bash[68594]: audit 2024-09-13T15:18:24.262037+0000 mon.a (mon.0) 2742 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:25 smithi177 bash[68594]: audit 2024-09-13T15:18:24.262037+0000 mon.a (mon.0) 2742 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:25.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[84576]: audit 2024-09-13T15:18:24.251372+0000 mon.a (mon.0) 2741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[84576]: audit 2024-09-13T15:18:24.251372+0000 mon.a (mon.0) 2741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[84576]: cluster 2024-09-13T15:18:24.254437+0000 osd.6 (osd.6) 30 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:25.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[84576]: cluster 2024-09-13T15:18:24.254437+0000 osd.6 (osd.6) 30 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:25.728 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[84576]: audit 2024-09-13T15:18:24.262037+0000 mon.a (mon.0) 2742 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.729 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[84576]: audit 2024-09-13T15:18:24.262037+0000 mon.a (mon.0) 2742 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.729 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[87147]: audit 2024-09-13T15:18:24.251372+0000 mon.a (mon.0) 2741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.729 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[87147]: audit 2024-09-13T15:18:24.251372+0000 mon.a (mon.0) 2741 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.729 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[87147]: cluster 2024-09-13T15:18:24.254437+0000 osd.6 (osd.6) 30 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:25.729 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[87147]: cluster 2024-09-13T15:18:24.254437+0000 osd.6 (osd.6) 30 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:25.729 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[87147]: audit 2024-09-13T15:18:24.262037+0000 mon.a (mon.0) 2742 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.729 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:25 smithi099 bash[87147]: audit 2024-09-13T15:18:24.262037+0000 mon.a (mon.0) 2742 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:25.736 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:25.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:25.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:25.839 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:25.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:25.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:25.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:26.002 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:26.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:26.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:26.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:26.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:26.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:26.242 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:26.278 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:26.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:26.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:26.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:26.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:26 smithi177 bash[68594]: cluster 2024-09-13T15:18:25.228360+0000 mgr.y (mgr.26252) 1573 : cluster [DBG] pgmap v2187: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 3091/91377 objects degraded (3.383%); 374/91377 objects misplaced (0.409%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:18:26.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:26 smithi177 bash[68594]: cluster 2024-09-13T15:18:25.228360+0000 mgr.y (mgr.26252) 1573 : cluster [DBG] pgmap v2187: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 3091/91377 objects degraded (3.383%); 374/91377 objects misplaced (0.409%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:18:26.423 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:26.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:26.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:26.527 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:26.538 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:26 smithi099 bash[84576]: cluster 2024-09-13T15:18:25.228360+0000 mgr.y (mgr.26252) 1573 : cluster [DBG] pgmap v2187: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 3091/91377 objects degraded (3.383%); 374/91377 objects misplaced (0.409%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:18:26.538 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:26 smithi099 bash[84576]: cluster 2024-09-13T15:18:25.228360+0000 mgr.y (mgr.26252) 1573 : cluster [DBG] pgmap v2187: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 3091/91377 objects degraded (3.383%); 374/91377 objects misplaced (0.409%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:18:26.538 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:26 smithi099 bash[87147]: cluster 2024-09-13T15:18:25.228360+0000 mgr.y (mgr.26252) 1573 : cluster [DBG] pgmap v2187: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 3091/91377 objects degraded (3.383%); 374/91377 objects misplaced (0.409%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:18:26.539 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:26 smithi099 bash[87147]: cluster 2024-09-13T15:18:25.228360+0000 mgr.y (mgr.26252) 1573 : cluster [DBG] pgmap v2187: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 3091/91377 objects degraded (3.383%); 374/91377 objects misplaced (0.409%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T15:18:26.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:26.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:26.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:26.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:26.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:26.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:26.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:26.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:26.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:26.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:26.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:26.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:26.937 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:26.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:26.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:27.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:27.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:27.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:27.166 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:27.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:27.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:27.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:27.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:27.307 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:27 smithi177 bash[68594]: cluster 2024-09-13T15:18:26.287171+0000 osd.6 (osd.6) 31 : cluster [DBG] 46.1 deep-scrub starts 2024-09-13T15:18:27.307 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:27 smithi177 bash[68594]: cluster 2024-09-13T15:18:26.287171+0000 osd.6 (osd.6) 31 : cluster [DBG] 46.1 deep-scrub starts 2024-09-13T15:18:27.307 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:27 smithi177 bash[68594]: cluster 2024-09-13T15:18:26.670189+0000 mon.a (mon.0) 2743 : cluster [WRN] Health check update: Degraded data redundancy: 3091/91377 objects degraded (3.383%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:27.307 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:27 smithi177 bash[68594]: cluster 2024-09-13T15:18:26.670189+0000 mon.a (mon.0) 2743 : cluster [WRN] Health check update: Degraded data redundancy: 3091/91377 objects degraded (3.383%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:27.338 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:27.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:27.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:27.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:27.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:27.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:27.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:27.574 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:18:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:18:27] "GET /metrics HTTP/1.1" 200 36795 "" "Prometheus/2.51.0" 2024-09-13T15:18:27.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:27 smithi099 bash[87147]: cluster 2024-09-13T15:18:26.287171+0000 osd.6 (osd.6) 31 : cluster [DBG] 46.1 deep-scrub starts 2024-09-13T15:18:27.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:27 smithi099 bash[87147]: cluster 2024-09-13T15:18:26.287171+0000 osd.6 (osd.6) 31 : cluster [DBG] 46.1 deep-scrub starts 2024-09-13T15:18:27.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:27 smithi099 bash[87147]: cluster 2024-09-13T15:18:26.670189+0000 mon.a (mon.0) 2743 : cluster [WRN] Health check update: Degraded data redundancy: 3091/91377 objects degraded (3.383%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:27.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:27 smithi099 bash[87147]: cluster 2024-09-13T15:18:26.670189+0000 mon.a (mon.0) 2743 : cluster [WRN] Health check update: Degraded data redundancy: 3091/91377 objects degraded (3.383%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:27.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:27 smithi099 bash[84576]: cluster 2024-09-13T15:18:26.287171+0000 osd.6 (osd.6) 31 : cluster [DBG] 46.1 deep-scrub starts 2024-09-13T15:18:27.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:27 smithi099 bash[84576]: cluster 2024-09-13T15:18:26.287171+0000 osd.6 (osd.6) 31 : cluster [DBG] 46.1 deep-scrub starts 2024-09-13T15:18:27.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:27 smithi099 bash[84576]: cluster 2024-09-13T15:18:26.670189+0000 mon.a (mon.0) 2743 : cluster [WRN] Health check update: Degraded data redundancy: 3091/91377 objects degraded (3.383%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:27.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:27 smithi099 bash[84576]: cluster 2024-09-13T15:18:26.670189+0000 mon.a (mon.0) 2743 : cluster [WRN] Health check update: Degraded data redundancy: 3091/91377 objects degraded (3.383%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:27.584 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:27.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:27.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:27.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:27.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:27.749 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:27.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:27.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:27.853 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:27.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:27.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:27.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:28.009 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:28.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:28.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:28.140 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:28.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:28.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:28.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:28.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:28.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:28.374 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:28.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:28.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:28.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:28.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:28.510 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:28 smithi099 bash[84576]: cluster 2024-09-13T15:18:27.229799+0000 mgr.y (mgr.26252) 1574 : cluster [DBG] pgmap v2188: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3091/89382 objects degraded (3.458%); 374/89382 objects misplaced (0.418%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:28.511 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:28 smithi099 bash[84576]: cluster 2024-09-13T15:18:27.229799+0000 mgr.y (mgr.26252) 1574 : cluster [DBG] pgmap v2188: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3091/89382 objects degraded (3.458%); 374/89382 objects misplaced (0.418%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:28.512 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:28 smithi099 bash[87147]: cluster 2024-09-13T15:18:27.229799+0000 mgr.y (mgr.26252) 1574 : cluster [DBG] pgmap v2188: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3091/89382 objects degraded (3.458%); 374/89382 objects misplaced (0.418%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:28.512 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:28 smithi099 bash[87147]: cluster 2024-09-13T15:18:27.229799+0000 mgr.y (mgr.26252) 1574 : cluster [DBG] pgmap v2188: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3091/89382 objects degraded (3.458%); 374/89382 objects misplaced (0.418%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:28.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:28.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:28.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:28.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:28 smithi177 bash[68594]: cluster 2024-09-13T15:18:27.229799+0000 mgr.y (mgr.26252) 1574 : cluster [DBG] pgmap v2188: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3091/89382 objects degraded (3.458%); 374/89382 objects misplaced (0.418%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:28.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:28 smithi177 bash[68594]: cluster 2024-09-13T15:18:27.229799+0000 mgr.y (mgr.26252) 1574 : cluster [DBG] pgmap v2188: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 3091/89382 objects degraded (3.458%); 374/89382 objects misplaced (0.418%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:28.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:28.659 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:28.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:28.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:28.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:28.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:28.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:28.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:28.896 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:28.927 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:28.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:28.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:29.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:29.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:29.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:29.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:29.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:29.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:29.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:29.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:29.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:29.332 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:29.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:29.439 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:29.449 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:29 smithi177 bash[68594]: cluster 2024-09-13T15:18:27.332625+0000 osd.6 (osd.6) 32 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:29.449 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:29 smithi177 bash[68594]: cluster 2024-09-13T15:18:27.332625+0000 osd.6 (osd.6) 32 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:29.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:29.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:29.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:29.573 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:29.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:29 smithi099 bash[84576]: cluster 2024-09-13T15:18:27.332625+0000 osd.6 (osd.6) 32 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:29.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:29 smithi099 bash[84576]: cluster 2024-09-13T15:18:27.332625+0000 osd.6 (osd.6) 32 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:29 smithi099 bash[87147]: cluster 2024-09-13T15:18:27.332625+0000 osd.6 (osd.6) 32 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:29.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:29 smithi099 bash[87147]: cluster 2024-09-13T15:18:27.332625+0000 osd.6 (osd.6) 32 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:29.593 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:29.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:29.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:29.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:29.733 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:29.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:29.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:29.836 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:29.864 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:29.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:29.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:29.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:29.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:29.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:29.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:30.097 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:30.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:30.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:30.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:30.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:30.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:30.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:30 smithi177 bash[68594]: cluster 2024-09-13T15:18:29.230849+0000 mgr.y (mgr.26252) 1575 : cluster [DBG] pgmap v2189: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3091/84303 objects degraded (3.667%); 374/84303 objects misplaced (0.444%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:30.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:30 smithi177 bash[68594]: cluster 2024-09-13T15:18:29.230849+0000 mgr.y (mgr.26252) 1575 : cluster [DBG] pgmap v2189: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3091/84303 objects degraded (3.667%); 374/84303 objects misplaced (0.444%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:30.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:18:30 smithi177 bash[61439]: ts=2024-09-13T15:18:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:18:30.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:30.385 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:30.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:30.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:30.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:30.511 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:30.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:30.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:30 smithi099 bash[84576]: cluster 2024-09-13T15:18:29.230849+0000 mgr.y (mgr.26252) 1575 : cluster [DBG] pgmap v2189: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3091/84303 objects degraded (3.667%); 374/84303 objects misplaced (0.444%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:30 smithi099 bash[84576]: cluster 2024-09-13T15:18:29.230849+0000 mgr.y (mgr.26252) 1575 : cluster [DBG] pgmap v2189: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3091/84303 objects degraded (3.667%); 374/84303 objects misplaced (0.444%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:30 smithi099 bash[87147]: cluster 2024-09-13T15:18:29.230849+0000 mgr.y (mgr.26252) 1575 : cluster [DBG] pgmap v2189: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3091/84303 objects degraded (3.667%); 374/84303 objects misplaced (0.444%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:30 smithi099 bash[87147]: cluster 2024-09-13T15:18:29.230849+0000 mgr.y (mgr.26252) 1575 : cluster [DBG] pgmap v2189: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3091/84303 objects degraded (3.667%); 374/84303 objects misplaced (0.444%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:30.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:30.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:30.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:30.743 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:30.770 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:30.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:30.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:30.874 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:30.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:30.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:30.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:30.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:31.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:31.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:31.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:31.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:31.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:31.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:31.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:31.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:31.272 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:31.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:31.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:31.375 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:31.401 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:31.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:31 smithi099 bash[84576]: cluster 2024-09-13T15:18:29.413185+0000 osd.6 (osd.6) 33 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:31.404 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:31 smithi099 bash[84576]: cluster 2024-09-13T15:18:29.413185+0000 osd.6 (osd.6) 33 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:31.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:31 smithi099 bash[87147]: cluster 2024-09-13T15:18:29.413185+0000 osd.6 (osd.6) 33 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:31.405 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:31 smithi099 bash[87147]: cluster 2024-09-13T15:18:29.413185+0000 osd.6 (osd.6) 33 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:31.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:31.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:31.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:31.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:31 smithi177 bash[68594]: cluster 2024-09-13T15:18:29.413185+0000 osd.6 (osd.6) 33 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:31.514 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:31 smithi177 bash[68594]: cluster 2024-09-13T15:18:29.413185+0000 osd.6 (osd.6) 33 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:31.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:31.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:31.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:31.634 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:31.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:31.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:31.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:31.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:31.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:31.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:31.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:31.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:31.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:31.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:31.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:32.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:32.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:32.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:32.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:32.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:32.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:32.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:32.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:32.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:32.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:32.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:32.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:32.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:32.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:32.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:32.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:32.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:32.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:32 smithi177 bash[68594]: cluster 2024-09-13T15:18:31.231717+0000 mgr.y (mgr.26252) 1576 : cluster [DBG] pgmap v2190: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 2845/77874 objects degraded (3.653%); 157/77874 objects misplaced (0.202%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T15:18:32.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:32 smithi177 bash[68594]: cluster 2024-09-13T15:18:31.231717+0000 mgr.y (mgr.26252) 1576 : cluster [DBG] pgmap v2190: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 2845/77874 objects degraded (3.653%); 157/77874 objects misplaced (0.202%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T15:18:32.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:32 smithi177 bash[68594]: cluster 2024-09-13T15:18:31.671772+0000 mon.a (mon.0) 2744 : cluster [WRN] Health check update: Degraded data redundancy: 2845/77874 objects degraded (3.653%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:32.563 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:32 smithi177 bash[68594]: cluster 2024-09-13T15:18:31.671772+0000 mon.a (mon.0) 2744 : cluster [WRN] Health check update: Degraded data redundancy: 2845/77874 objects degraded (3.653%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:32.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:32 smithi099 bash[84576]: cluster 2024-09-13T15:18:31.231717+0000 mgr.y (mgr.26252) 1576 : cluster [DBG] pgmap v2190: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 2845/77874 objects degraded (3.653%); 157/77874 objects misplaced (0.202%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T15:18:32.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:32 smithi099 bash[84576]: cluster 2024-09-13T15:18:31.231717+0000 mgr.y (mgr.26252) 1576 : cluster [DBG] pgmap v2190: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 2845/77874 objects degraded (3.653%); 157/77874 objects misplaced (0.202%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T15:18:32.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:32 smithi099 bash[84576]: cluster 2024-09-13T15:18:31.671772+0000 mon.a (mon.0) 2744 : cluster [WRN] Health check update: Degraded data redundancy: 2845/77874 objects degraded (3.653%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:32.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:32 smithi099 bash[84576]: cluster 2024-09-13T15:18:31.671772+0000 mon.a (mon.0) 2744 : cluster [WRN] Health check update: Degraded data redundancy: 2845/77874 objects degraded (3.653%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:32.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:32 smithi099 bash[87147]: cluster 2024-09-13T15:18:31.231717+0000 mgr.y (mgr.26252) 1576 : cluster [DBG] pgmap v2190: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 2845/77874 objects degraded (3.653%); 157/77874 objects misplaced (0.202%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T15:18:32.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:32 smithi099 bash[87147]: cluster 2024-09-13T15:18:31.231717+0000 mgr.y (mgr.26252) 1576 : cluster [DBG] pgmap v2190: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 2845/77874 objects degraded (3.653%); 157/77874 objects misplaced (0.202%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T15:18:32.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:32 smithi099 bash[87147]: cluster 2024-09-13T15:18:31.671772+0000 mon.a (mon.0) 2744 : cluster [WRN] Health check update: Degraded data redundancy: 2845/77874 objects degraded (3.653%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:32.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:32 smithi099 bash[87147]: cluster 2024-09-13T15:18:31.671772+0000 mon.a (mon.0) 2744 : cluster [WRN] Health check update: Degraded data redundancy: 2845/77874 objects degraded (3.653%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:32.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:32.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:32.691 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:32.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:32.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:32.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:32.826 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:32.851 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:32.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:32.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:32.957 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:32.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:32.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:33.078 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:33.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:33.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:33.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:33.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:33.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:33.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:33.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:33.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:33.372 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:33.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:33.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:33.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:33.509 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:33.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:33.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:33.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:33.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:33.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:33.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:33.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:33.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:33.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:33.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:33.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:33.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:33.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:33.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:33.996 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:34.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:34.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:34.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:34.123 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:34.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:34.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:34.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:34.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:34.275 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:34.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:34.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:34.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:34.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:34.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:34.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:34.508 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:34.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:34.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:34 smithi177 bash[68594]: cluster 2024-09-13T15:18:33.232373+0000 mgr.y (mgr.26252) 1577 : cluster [DBG] pgmap v2191: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2845/76005 objects degraded (3.743%); 157/76005 objects misplaced (0.207%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:34.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:34 smithi177 bash[68594]: cluster 2024-09-13T15:18:33.232373+0000 mgr.y (mgr.26252) 1577 : cluster [DBG] pgmap v2191: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2845/76005 objects degraded (3.743%); 157/76005 objects misplaced (0.207%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:34.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:34 smithi177 bash[68594]: audit 2024-09-13T15:18:33.322454+0000 mon.a (mon.0) 2745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:34.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:34 smithi177 bash[68594]: audit 2024-09-13T15:18:33.322454+0000 mon.a (mon.0) 2745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:34.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:34 smithi177 bash[68594]: audit 2024-09-13T15:18:33.653689+0000 mon.a (mon.0) 2746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:34 smithi177 bash[68594]: audit 2024-09-13T15:18:33.653689+0000 mon.a (mon.0) 2746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:34 smithi177 bash[68594]: audit 2024-09-13T15:18:33.665070+0000 mon.a (mon.0) 2747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:34 smithi177 bash[68594]: audit 2024-09-13T15:18:33.665070+0000 mon.a (mon.0) 2747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:34.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:34.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[84576]: cluster 2024-09-13T15:18:33.232373+0000 mgr.y (mgr.26252) 1577 : cluster [DBG] pgmap v2191: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2845/76005 objects degraded (3.743%); 157/76005 objects misplaced (0.207%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:34.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[84576]: cluster 2024-09-13T15:18:33.232373+0000 mgr.y (mgr.26252) 1577 : cluster [DBG] pgmap v2191: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2845/76005 objects degraded (3.743%); 157/76005 objects misplaced (0.207%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:34.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[84576]: audit 2024-09-13T15:18:33.322454+0000 mon.a (mon.0) 2745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:34.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[84576]: audit 2024-09-13T15:18:33.322454+0000 mon.a (mon.0) 2745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:34.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[84576]: audit 2024-09-13T15:18:33.653689+0000 mon.a (mon.0) 2746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[84576]: audit 2024-09-13T15:18:33.653689+0000 mon.a (mon.0) 2746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[84576]: audit 2024-09-13T15:18:33.665070+0000 mon.a (mon.0) 2747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[84576]: audit 2024-09-13T15:18:33.665070+0000 mon.a (mon.0) 2747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[87147]: cluster 2024-09-13T15:18:33.232373+0000 mgr.y (mgr.26252) 1577 : cluster [DBG] pgmap v2191: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2845/76005 objects degraded (3.743%); 157/76005 objects misplaced (0.207%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:34.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[87147]: cluster 2024-09-13T15:18:33.232373+0000 mgr.y (mgr.26252) 1577 : cluster [DBG] pgmap v2191: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2845/76005 objects degraded (3.743%); 157/76005 objects misplaced (0.207%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T15:18:34.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[87147]: audit 2024-09-13T15:18:33.322454+0000 mon.a (mon.0) 2745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:34.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[87147]: audit 2024-09-13T15:18:33.322454+0000 mon.a (mon.0) 2745 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:34.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[87147]: audit 2024-09-13T15:18:33.653689+0000 mon.a (mon.0) 2746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[87147]: audit 2024-09-13T15:18:33.653689+0000 mon.a (mon.0) 2746 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[87147]: audit 2024-09-13T15:18:33.665070+0000 mon.a (mon.0) 2747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:34 smithi099 bash[87147]: audit 2024-09-13T15:18:33.665070+0000 mon.a (mon.0) 2747 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:34.643 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:34.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:34.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:34.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:34.785 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:34.806 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:34.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:34.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:34.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:34.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:34.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:34.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:35.038 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:35.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:35.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:35.179 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:35.214 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:35.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:35.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:35.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:35.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[84576]: cluster 2024-09-13T15:18:33.355537+0000 osd.6 (osd.6) 34 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:35.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[84576]: cluster 2024-09-13T15:18:33.355537+0000 osd.6 (osd.6) 34 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:35.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[84576]: audit 2024-09-13T15:18:35.045576+0000 mon.a (mon.0) 2748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:18:35.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[84576]: audit 2024-09-13T15:18:35.045576+0000 mon.a (mon.0) 2748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:18:35.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[84576]: audit 2024-09-13T15:18:35.047041+0000 mon.a (mon.0) 2749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:18:35.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[84576]: audit 2024-09-13T15:18:35.047041+0000 mon.a (mon.0) 2749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:18:35.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[84576]: audit 2024-09-13T15:18:35.054751+0000 mon.a (mon.0) 2750 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:35.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[84576]: audit 2024-09-13T15:18:35.054751+0000 mon.a (mon.0) 2750 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:35.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[87147]: cluster 2024-09-13T15:18:33.355537+0000 osd.6 (osd.6) 34 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:35.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[87147]: cluster 2024-09-13T15:18:33.355537+0000 osd.6 (osd.6) 34 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:35.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[87147]: audit 2024-09-13T15:18:35.045576+0000 mon.a (mon.0) 2748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:18:35.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[87147]: audit 2024-09-13T15:18:35.045576+0000 mon.a (mon.0) 2748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:18:35.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[87147]: audit 2024-09-13T15:18:35.047041+0000 mon.a (mon.0) 2749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:18:35.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[87147]: audit 2024-09-13T15:18:35.047041+0000 mon.a (mon.0) 2749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:18:35.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[87147]: audit 2024-09-13T15:18:35.054751+0000 mon.a (mon.0) 2750 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:35.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:35 smithi099 bash[87147]: audit 2024-09-13T15:18:35.054751+0000 mon.a (mon.0) 2750 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:35.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:35.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:35.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:35.462 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:35.492 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:35.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:35.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:35 smithi177 bash[68594]: cluster 2024-09-13T15:18:33.355537+0000 osd.6 (osd.6) 34 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:35 smithi177 bash[68594]: cluster 2024-09-13T15:18:33.355537+0000 osd.6 (osd.6) 34 : cluster [DBG] 46.1 scrub starts 2024-09-13T15:18:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:35 smithi177 bash[68594]: audit 2024-09-13T15:18:35.045576+0000 mon.a (mon.0) 2748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:18:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:35 smithi177 bash[68594]: audit 2024-09-13T15:18:35.045576+0000 mon.a (mon.0) 2748 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:18:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:35 smithi177 bash[68594]: audit 2024-09-13T15:18:35.047041+0000 mon.a (mon.0) 2749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:18:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:35 smithi177 bash[68594]: audit 2024-09-13T15:18:35.047041+0000 mon.a (mon.0) 2749 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:18:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:35 smithi177 bash[68594]: audit 2024-09-13T15:18:35.054751+0000 mon.a (mon.0) 2750 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:35.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:35 smithi177 bash[68594]: audit 2024-09-13T15:18:35.054751+0000 mon.a (mon.0) 2750 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:35.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:35.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:35.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:35.703 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:18:35.703 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:18:35.703 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2024-09-13T15:18:35.703 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:18:35.736 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:35.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:35.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:35.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:35.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:35.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:35.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:35.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:36.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:36.042 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:36.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:36.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:36.148 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:36.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:36.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:36.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:36.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:36.307 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:36.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:36.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:36.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:36.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:36.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:36.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:36.540 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:36.551 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:36 smithi099 bash[84576]: cluster 2024-09-13T15:18:35.233293+0000 mgr.y (mgr.26252) 1578 : cluster [DBG] pgmap v2192: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2563/68634 objects degraded (3.734%); 4.9 MiB/s, 92 objects/s recovering 2024-09-13T15:18:36.551 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:36 smithi099 bash[84576]: cluster 2024-09-13T15:18:35.233293+0000 mgr.y (mgr.26252) 1578 : cluster [DBG] pgmap v2192: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2563/68634 objects degraded (3.734%); 4.9 MiB/s, 92 objects/s recovering 2024-09-13T15:18:36.551 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:36 smithi099 bash[87147]: cluster 2024-09-13T15:18:35.233293+0000 mgr.y (mgr.26252) 1578 : cluster [DBG] pgmap v2192: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2563/68634 objects degraded (3.734%); 4.9 MiB/s, 92 objects/s recovering 2024-09-13T15:18:36.551 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:36 smithi099 bash[87147]: cluster 2024-09-13T15:18:35.233293+0000 mgr.y (mgr.26252) 1578 : cluster [DBG] pgmap v2192: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2563/68634 objects degraded (3.734%); 4.9 MiB/s, 92 objects/s recovering 2024-09-13T15:18:36.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:36.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:36.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:36 smithi177 bash[68594]: cluster 2024-09-13T15:18:35.233293+0000 mgr.y (mgr.26252) 1578 : cluster [DBG] pgmap v2192: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2563/68634 objects degraded (3.734%); 4.9 MiB/s, 92 objects/s recovering 2024-09-13T15:18:36.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:36 smithi177 bash[68594]: cluster 2024-09-13T15:18:35.233293+0000 mgr.y (mgr.26252) 1578 : cluster [DBG] pgmap v2192: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2563/68634 objects degraded (3.734%); 4.9 MiB/s, 92 objects/s recovering 2024-09-13T15:18:36.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:36.691 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:36.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:36.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:36.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:36.814 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:36.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:36.917 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:36.942 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:36.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:37.045 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:37.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:37.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:37.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:37.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:37.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:37.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:37.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:37.306 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:18:37] "GET /metrics HTTP/1.1" 200 36787 "" "Prometheus/2.51.0" 2024-09-13T15:18:37.321 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is set 2024-09-13T15:18:37.323 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:37.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:37.350 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:37.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:37.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:37 smithi177 bash[68594]: audit 2024-09-13T15:18:36.474091+0000 mon.c (mon.2) 122 : audit [INF] from='client.? 172.21.15.99:0/3666531830' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:37 smithi177 bash[68594]: audit 2024-09-13T15:18:36.474091+0000 mon.c (mon.2) 122 : audit [INF] from='client.? 172.21.15.99:0/3666531830' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:37 smithi177 bash[68594]: audit 2024-09-13T15:18:36.476033+0000 mon.a (mon.0) 2751 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:37 smithi177 bash[68594]: audit 2024-09-13T15:18:36.476033+0000 mon.a (mon.0) 2751 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:37 smithi177 bash[68594]: cluster 2024-09-13T15:18:36.672539+0000 mon.a (mon.0) 2752 : cluster [WRN] Health check update: Degraded data redundancy: 2563/68634 objects degraded (3.734%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:37.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:37 smithi177 bash[68594]: cluster 2024-09-13T15:18:36.672539+0000 mon.a (mon.0) 2752 : cluster [WRN] Health check update: Degraded data redundancy: 2563/68634 objects degraded (3.734%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:37.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:37 smithi177 bash[68594]: cluster 2024-09-13T15:18:37.234218+0000 mgr.y (mgr.26252) 1579 : cluster [DBG] pgmap v2193: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2563/66432 objects degraded (3.858%); 3.2 MiB/s, 65 objects/s recovering 2024-09-13T15:18:37.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:37 smithi177 bash[68594]: cluster 2024-09-13T15:18:37.234218+0000 mgr.y (mgr.26252) 1579 : cluster [DBG] pgmap v2193: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2563/66432 objects degraded (3.858%); 3.2 MiB/s, 65 objects/s recovering 2024-09-13T15:18:37.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:37.469 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:37.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:37.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:37.594 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:37.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:37.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:37.602 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[84576]: audit 2024-09-13T15:18:36.474091+0000 mon.c (mon.2) 122 : audit [INF] from='client.? 172.21.15.99:0/3666531830' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.602 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[84576]: audit 2024-09-13T15:18:36.474091+0000 mon.c (mon.2) 122 : audit [INF] from='client.? 172.21.15.99:0/3666531830' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.602 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[84576]: audit 2024-09-13T15:18:36.476033+0000 mon.a (mon.0) 2751 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.602 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[84576]: audit 2024-09-13T15:18:36.476033+0000 mon.a (mon.0) 2751 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.602 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[84576]: cluster 2024-09-13T15:18:36.672539+0000 mon.a (mon.0) 2752 : cluster [WRN] Health check update: Degraded data redundancy: 2563/68634 objects degraded (3.734%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:37.602 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[84576]: cluster 2024-09-13T15:18:36.672539+0000 mon.a (mon.0) 2752 : cluster [WRN] Health check update: Degraded data redundancy: 2563/68634 objects degraded (3.734%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:37.602 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[84576]: cluster 2024-09-13T15:18:37.234218+0000 mgr.y (mgr.26252) 1579 : cluster [DBG] pgmap v2193: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2563/66432 objects degraded (3.858%); 3.2 MiB/s, 65 objects/s recovering 2024-09-13T15:18:37.602 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[84576]: cluster 2024-09-13T15:18:37.234218+0000 mgr.y (mgr.26252) 1579 : cluster [DBG] pgmap v2193: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2563/66432 objects degraded (3.858%); 3.2 MiB/s, 65 objects/s recovering 2024-09-13T15:18:37.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[87147]: audit 2024-09-13T15:18:36.474091+0000 mon.c (mon.2) 122 : audit [INF] from='client.? 172.21.15.99:0/3666531830' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[87147]: audit 2024-09-13T15:18:36.474091+0000 mon.c (mon.2) 122 : audit [INF] from='client.? 172.21.15.99:0/3666531830' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[87147]: audit 2024-09-13T15:18:36.476033+0000 mon.a (mon.0) 2751 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[87147]: audit 2024-09-13T15:18:36.476033+0000 mon.a (mon.0) 2751 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:18:37.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[87147]: cluster 2024-09-13T15:18:36.672539+0000 mon.a (mon.0) 2752 : cluster [WRN] Health check update: Degraded data redundancy: 2563/68634 objects degraded (3.734%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:37.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[87147]: cluster 2024-09-13T15:18:36.672539+0000 mon.a (mon.0) 2752 : cluster [WRN] Health check update: Degraded data redundancy: 2563/68634 objects degraded (3.734%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:37.603 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[87147]: cluster 2024-09-13T15:18:37.234218+0000 mgr.y (mgr.26252) 1579 : cluster [DBG] pgmap v2193: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2563/66432 objects degraded (3.858%); 3.2 MiB/s, 65 objects/s recovering 2024-09-13T15:18:37.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:37 smithi099 bash[87147]: cluster 2024-09-13T15:18:37.234218+0000 mgr.y (mgr.26252) 1579 : cluster [DBG] pgmap v2193: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2563/66432 objects degraded (3.858%); 3.2 MiB/s, 65 objects/s recovering 2024-09-13T15:18:37.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:37.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:37.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:37.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:37.824 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:37.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:37.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:37.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:37.961 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:18:37.961 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:18:37.965 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:37.988 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:37.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:37.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:38.017 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:18:38.017 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:18:38.050 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:18:38.051 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:18:38.067 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:18:38.067 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:18:38.084 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:18:38.084 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:18:38.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:38.107 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:18:38.107 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:18:38.116 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:38.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:38.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:38.151 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:18:38.152 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:18:38.181 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:18:38.182 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:18:38.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:38.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:38.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:38.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:38.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:38.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:38 smithi177 bash[68594]: cluster 2024-09-13T15:18:37.290797+0000 mon.a (mon.0) 2753 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:18:38.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:38 smithi177 bash[68594]: cluster 2024-09-13T15:18:37.290797+0000 mon.a (mon.0) 2753 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:18:38.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:38 smithi177 bash[68594]: audit 2024-09-13T15:18:37.302416+0000 mon.a (mon.0) 2754 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:18:38.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:38 smithi177 bash[68594]: audit 2024-09-13T15:18:37.302416+0000 mon.a (mon.0) 2754 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:18:38.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:38 smithi177 bash[68594]: cluster 2024-09-13T15:18:37.323929+0000 mon.a (mon.0) 2755 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2024-09-13T15:18:38.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:38 smithi177 bash[68594]: cluster 2024-09-13T15:18:37.323929+0000 mon.a (mon.0) 2755 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2024-09-13T15:18:38.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:38 smithi177 bash[68594]: audit 2024-09-13T15:18:37.924409+0000 mon.a (mon.0) 2756 : audit [DBG] from='client.? 172.21.15.99:0/2445719765' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:18:38.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:38 smithi177 bash[68594]: audit 2024-09-13T15:18:37.924409+0000 mon.a (mon.0) 2756 : audit [DBG] from='client.? 172.21.15.99:0/2445719765' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:18:38.381 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:38.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:38.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:38.484 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:38.503 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[87147]: cluster 2024-09-13T15:18:37.290797+0000 mon.a (mon.0) 2753 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:18:38.503 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[87147]: cluster 2024-09-13T15:18:37.290797+0000 mon.a (mon.0) 2753 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:18:38.503 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[87147]: audit 2024-09-13T15:18:37.302416+0000 mon.a (mon.0) 2754 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:18:38.503 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[87147]: audit 2024-09-13T15:18:37.302416+0000 mon.a (mon.0) 2754 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:18:38.503 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[87147]: cluster 2024-09-13T15:18:37.323929+0000 mon.a (mon.0) 2755 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2024-09-13T15:18:38.503 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[87147]: cluster 2024-09-13T15:18:37.323929+0000 mon.a (mon.0) 2755 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2024-09-13T15:18:38.503 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[87147]: audit 2024-09-13T15:18:37.924409+0000 mon.a (mon.0) 2756 : audit [DBG] from='client.? 172.21.15.99:0/2445719765' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:18:38.503 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[87147]: audit 2024-09-13T15:18:37.924409+0000 mon.a (mon.0) 2756 : audit [DBG] from='client.? 172.21.15.99:0/2445719765' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:18:38.507 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:38.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:38.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:38.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:38.632 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:38.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:38.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:38.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:38.773 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:38.778 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[84576]: cluster 2024-09-13T15:18:37.290797+0000 mon.a (mon.0) 2753 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:18:38.778 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[84576]: cluster 2024-09-13T15:18:37.290797+0000 mon.a (mon.0) 2753 : cluster [WRN] Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:18:38.778 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[84576]: audit 2024-09-13T15:18:37.302416+0000 mon.a (mon.0) 2754 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:18:38.778 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[84576]: audit 2024-09-13T15:18:37.302416+0000 mon.a (mon.0) 2754 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:18:38.779 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[84576]: cluster 2024-09-13T15:18:37.323929+0000 mon.a (mon.0) 2755 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2024-09-13T15:18:38.779 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[84576]: cluster 2024-09-13T15:18:37.323929+0000 mon.a (mon.0) 2755 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2024-09-13T15:18:38.779 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[84576]: audit 2024-09-13T15:18:37.924409+0000 mon.a (mon.0) 2756 : audit [DBG] from='client.? 172.21.15.99:0/2445719765' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:18:38.779 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:38 smithi099 bash[84576]: audit 2024-09-13T15:18:37.924409+0000 mon.a (mon.0) 2756 : audit [DBG] from='client.? 172.21.15.99:0/2445719765' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:18:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:38.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:38.880 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:38.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:38.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:38.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:39.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:39.035 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:39.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:39.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:39.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:39.161 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:39.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:39.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:39.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:39.293 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:39.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:39.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:39.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:39.427 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:39.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:39.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:39.533 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:39.566 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:39.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:39.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:39.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:39 smithi099 bash[84576]: cluster 2024-09-13T15:18:39.234917+0000 mgr.y (mgr.26252) 1580 : cluster [DBG] pgmap v2195: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 2563/61551 objects degraded (4.164%); 3.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:39.604 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:39 smithi099 bash[84576]: cluster 2024-09-13T15:18:39.234917+0000 mgr.y (mgr.26252) 1580 : cluster [DBG] pgmap v2195: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 2563/61551 objects degraded (4.164%); 3.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:39.604 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:39 smithi099 bash[87147]: cluster 2024-09-13T15:18:39.234917+0000 mgr.y (mgr.26252) 1580 : cluster [DBG] pgmap v2195: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 2563/61551 objects degraded (4.164%); 3.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:39.605 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:39 smithi099 bash[87147]: cluster 2024-09-13T15:18:39.234917+0000 mgr.y (mgr.26252) 1580 : cluster [DBG] pgmap v2195: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 2563/61551 objects degraded (4.164%); 3.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:39.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:39 smithi177 bash[68594]: cluster 2024-09-13T15:18:39.234917+0000 mgr.y (mgr.26252) 1580 : cluster [DBG] pgmap v2195: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 2563/61551 objects degraded (4.164%); 3.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:39.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:39 smithi177 bash[68594]: cluster 2024-09-13T15:18:39.234917+0000 mgr.y (mgr.26252) 1580 : cluster [DBG] pgmap v2195: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 2563/61551 objects degraded (4.164%); 3.9 MiB/s, 79 objects/s recovering 2024-09-13T15:18:39.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:39.698 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:39.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:39.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:39.803 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:39.831 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:39.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:39.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:39.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:39.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:39.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:39.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:40.067 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:40.103 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:40.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:40.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:40.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:40.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:40.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:40.353 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:40.385 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:40.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:40.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:40.500 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:18:40 smithi177 bash[61439]: ts=2024-09-13T15:18:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:18:40.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:40.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:40.567 INFO:teuthology.orchestra.run.smithi099.stderr:marked out osd.2. 2024-09-13T15:18:40.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:40.628 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:40 smithi177 bash[68594]: audit 2024-09-13T15:18:40.514271+0000 mon.a (mon.0) 2757 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:18:40.628 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:40 smithi177 bash[68594]: audit 2024-09-13T15:18:40.514271+0000 mon.a (mon.0) 2757 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:18:40.629 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:40 smithi177 bash[68594]: cluster 2024-09-13T15:18:40.514330+0000 mon.a (mon.0) 2758 : cluster [INF] Client client.admin marked osd.2 out, while it was still marked up 2024-09-13T15:18:40.629 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:40 smithi177 bash[68594]: cluster 2024-09-13T15:18:40.514330+0000 mon.a (mon.0) 2758 : cluster [INF] Client client.admin marked osd.2 out, while it was still marked up 2024-09-13T15:18:40.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:40.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:40.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:40.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:40.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:40.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:40.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:40.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:40.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:40.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:40.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:40.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:40 smithi099 bash[84576]: audit 2024-09-13T15:18:40.514271+0000 mon.a (mon.0) 2757 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:18:40.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:40 smithi099 bash[84576]: debug 2024-09-13T15:18:40.510+0000 7f0688bd2640 -1 mon.a@0(leader).osd e1131 definitely_dead 0 2024-09-13T15:18:40.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:40 smithi099 bash[84576]: audit 2024-09-13T15:18:40.514271+0000 mon.a (mon.0) 2757 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:18:40.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:40 smithi099 bash[84576]: cluster 2024-09-13T15:18:40.514330+0000 mon.a (mon.0) 2758 : cluster [INF] Client client.admin marked osd.2 out, while it was still marked up 2024-09-13T15:18:40.950 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:40 smithi099 bash[84576]: cluster 2024-09-13T15:18:40.514330+0000 mon.a (mon.0) 2758 : cluster [INF] Client client.admin marked osd.2 out, while it was still marked up 2024-09-13T15:18:40.950 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:40 smithi099 bash[87147]: audit 2024-09-13T15:18:40.514271+0000 mon.a (mon.0) 2757 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:18:40.950 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:40 smithi099 bash[87147]: audit 2024-09-13T15:18:40.514271+0000 mon.a (mon.0) 2757 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:18:40.951 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:40 smithi099 bash[87147]: cluster 2024-09-13T15:18:40.514330+0000 mon.a (mon.0) 2758 : cluster [INF] Client client.admin marked osd.2 out, while it was still marked up 2024-09-13T15:18:40.951 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:40 smithi099 bash[87147]: cluster 2024-09-13T15:18:40.514330+0000 mon.a (mon.0) 2758 : cluster [INF] Client client.admin marked osd.2 out, while it was still marked up 2024-09-13T15:18:40.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:41.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:41.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:41.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:41.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:41.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:41.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:41.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:41.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:41.285 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:41.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:41.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:41.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:41.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:41.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:41.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:41.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:41.545 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:41.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:41.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:41.648 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:41.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:41.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:41.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:41.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:41.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:41.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:41.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:41 smithi177 bash[68594]: audit 2024-09-13T15:18:40.566612+0000 mon.a (mon.0) 2759 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-13T15:18:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:41 smithi177 bash[68594]: audit 2024-09-13T15:18:40.566612+0000 mon.a (mon.0) 2759 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-13T15:18:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:41 smithi177 bash[68594]: cluster 2024-09-13T15:18:40.572648+0000 mon.a (mon.0) 2760 : cluster [DBG] osdmap e1132: 8 total, 8 up, 7 in 2024-09-13T15:18:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:41 smithi177 bash[68594]: cluster 2024-09-13T15:18:40.572648+0000 mon.a (mon.0) 2760 : cluster [DBG] osdmap e1132: 8 total, 8 up, 7 in 2024-09-13T15:18:41.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:41 smithi177 bash[68594]: cluster 2024-09-13T15:18:41.235706+0000 mgr.y (mgr.26252) 1581 : cluster [DBG] pgmap v2197: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 828 op/s; 2068/56124 objects degraded (3.685%); 3.0 MiB/s, 106 objects/s recovering 2024-09-13T15:18:41.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:41 smithi177 bash[68594]: cluster 2024-09-13T15:18:41.235706+0000 mgr.y (mgr.26252) 1581 : cluster [DBG] pgmap v2197: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 828 op/s; 2068/56124 objects degraded (3.685%); 3.0 MiB/s, 106 objects/s recovering 2024-09-13T15:18:41.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:41.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[84576]: audit 2024-09-13T15:18:40.566612+0000 mon.a (mon.0) 2759 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-13T15:18:41.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[84576]: audit 2024-09-13T15:18:40.566612+0000 mon.a (mon.0) 2759 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-13T15:18:41.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[84576]: cluster 2024-09-13T15:18:40.572648+0000 mon.a (mon.0) 2760 : cluster [DBG] osdmap e1132: 8 total, 8 up, 7 in 2024-09-13T15:18:41.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[84576]: cluster 2024-09-13T15:18:40.572648+0000 mon.a (mon.0) 2760 : cluster [DBG] osdmap e1132: 8 total, 8 up, 7 in 2024-09-13T15:18:41.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[84576]: cluster 2024-09-13T15:18:41.235706+0000 mgr.y (mgr.26252) 1581 : cluster [DBG] pgmap v2197: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 828 op/s; 2068/56124 objects degraded (3.685%); 3.0 MiB/s, 106 objects/s recovering 2024-09-13T15:18:41.909 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[84576]: cluster 2024-09-13T15:18:41.235706+0000 mgr.y (mgr.26252) 1581 : cluster [DBG] pgmap v2197: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 828 op/s; 2068/56124 objects degraded (3.685%); 3.0 MiB/s, 106 objects/s recovering 2024-09-13T15:18:41.909 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[87147]: audit 2024-09-13T15:18:40.566612+0000 mon.a (mon.0) 2759 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-13T15:18:41.910 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[87147]: audit 2024-09-13T15:18:40.566612+0000 mon.a (mon.0) 2759 : audit [INF] from='client.? 172.21.15.99:0/68422451' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-13T15:18:41.910 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[87147]: cluster 2024-09-13T15:18:40.572648+0000 mon.a (mon.0) 2760 : cluster [DBG] osdmap e1132: 8 total, 8 up, 7 in 2024-09-13T15:18:41.910 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[87147]: cluster 2024-09-13T15:18:40.572648+0000 mon.a (mon.0) 2760 : cluster [DBG] osdmap e1132: 8 total, 8 up, 7 in 2024-09-13T15:18:41.910 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[87147]: cluster 2024-09-13T15:18:41.235706+0000 mgr.y (mgr.26252) 1581 : cluster [DBG] pgmap v2197: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 828 op/s; 2068/56124 objects degraded (3.685%); 3.0 MiB/s, 106 objects/s recovering 2024-09-13T15:18:41.910 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:41 smithi099 bash[87147]: cluster 2024-09-13T15:18:41.235706+0000 mgr.y (mgr.26252) 1581 : cluster [DBG] pgmap v2197: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 828 op/s; 2068/56124 objects degraded (3.685%); 3.0 MiB/s, 106 objects/s recovering 2024-09-13T15:18:41.923 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:41.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:41.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:42.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:42.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:42.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:42.156 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:42.186 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:42.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:42.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:42.289 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:42.313 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:42.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:42.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:42.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:42.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:42.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:42.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:42.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:42.582 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:42.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:42.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:42.686 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:42.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:42 smithi177 bash[68594]: cluster 2024-09-13T15:18:41.578333+0000 mon.a (mon.0) 2761 : cluster [DBG] osdmap e1133: 8 total, 8 up, 7 in 2024-09-13T15:18:42.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:42 smithi177 bash[68594]: cluster 2024-09-13T15:18:41.578333+0000 mon.a (mon.0) 2761 : cluster [DBG] osdmap e1133: 8 total, 8 up, 7 in 2024-09-13T15:18:42.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:42 smithi177 bash[68594]: cluster 2024-09-13T15:18:41.673217+0000 mon.a (mon.0) 2762 : cluster [WRN] Health check update: Degraded data redundancy: 2068/56124 objects degraded (3.685%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:42.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:42 smithi177 bash[68594]: cluster 2024-09-13T15:18:41.673217+0000 mon.a (mon.0) 2762 : cluster [WRN] Health check update: Degraded data redundancy: 2068/56124 objects degraded (3.685%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:42.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:42 smithi177 bash[68594]: cluster 2024-09-13T15:18:42.576075+0000 mon.a (mon.0) 2763 : cluster [DBG] osdmap e1134: 8 total, 8 up, 7 in 2024-09-13T15:18:42.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:42 smithi177 bash[68594]: cluster 2024-09-13T15:18:42.576075+0000 mon.a (mon.0) 2763 : cluster [DBG] osdmap e1134: 8 total, 8 up, 7 in 2024-09-13T15:18:42.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:42.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:42.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:42.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[87147]: cluster 2024-09-13T15:18:41.578333+0000 mon.a (mon.0) 2761 : cluster [DBG] osdmap e1133: 8 total, 8 up, 7 in 2024-09-13T15:18:42.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[87147]: cluster 2024-09-13T15:18:41.578333+0000 mon.a (mon.0) 2761 : cluster [DBG] osdmap e1133: 8 total, 8 up, 7 in 2024-09-13T15:18:42.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[87147]: cluster 2024-09-13T15:18:41.673217+0000 mon.a (mon.0) 2762 : cluster [WRN] Health check update: Degraded data redundancy: 2068/56124 objects degraded (3.685%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:42.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[87147]: cluster 2024-09-13T15:18:41.673217+0000 mon.a (mon.0) 2762 : cluster [WRN] Health check update: Degraded data redundancy: 2068/56124 objects degraded (3.685%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:42.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[87147]: cluster 2024-09-13T15:18:42.576075+0000 mon.a (mon.0) 2763 : cluster [DBG] osdmap e1134: 8 total, 8 up, 7 in 2024-09-13T15:18:42.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[87147]: cluster 2024-09-13T15:18:42.576075+0000 mon.a (mon.0) 2763 : cluster [DBG] osdmap e1134: 8 total, 8 up, 7 in 2024-09-13T15:18:42.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[84576]: cluster 2024-09-13T15:18:41.578333+0000 mon.a (mon.0) 2761 : cluster [DBG] osdmap e1133: 8 total, 8 up, 7 in 2024-09-13T15:18:42.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[84576]: cluster 2024-09-13T15:18:41.578333+0000 mon.a (mon.0) 2761 : cluster [DBG] osdmap e1133: 8 total, 8 up, 7 in 2024-09-13T15:18:42.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[84576]: cluster 2024-09-13T15:18:41.673217+0000 mon.a (mon.0) 2762 : cluster [WRN] Health check update: Degraded data redundancy: 2068/56124 objects degraded (3.685%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:42.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[84576]: cluster 2024-09-13T15:18:41.673217+0000 mon.a (mon.0) 2762 : cluster [WRN] Health check update: Degraded data redundancy: 2068/56124 objects degraded (3.685%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:18:42.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[84576]: cluster 2024-09-13T15:18:42.576075+0000 mon.a (mon.0) 2763 : cluster [DBG] osdmap e1134: 8 total, 8 up, 7 in 2024-09-13T15:18:42.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:42 smithi099 bash[84576]: cluster 2024-09-13T15:18:42.576075+0000 mon.a (mon.0) 2763 : cluster [DBG] osdmap e1134: 8 total, 8 up, 7 in 2024-09-13T15:18:42.838 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:42.872 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:42.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:42.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:42.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:43.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:43.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:43.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:43.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:43.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:43.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:43.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:43.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:43.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:43.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:43.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:43.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:43.418 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:43.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:43.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:43.521 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:43.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:43.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:43.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:43.661 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:43.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[84576]: cluster 2024-09-13T15:18:42.579585+0000 osd.7 (osd.7) 33 : cluster [DBG] 46.a starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11798 2024-09-13T15:18:43.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[84576]: cluster 2024-09-13T15:18:42.579585+0000 osd.7 (osd.7) 33 : cluster [DBG] 46.a starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11798 2024-09-13T15:18:43.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[84576]: cluster 2024-09-13T15:18:42.581993+0000 osd.0 (osd.0) 58 : cluster [DBG] 46.b starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11771 2024-09-13T15:18:43.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[84576]: cluster 2024-09-13T15:18:42.581993+0000 osd.0 (osd.0) 58 : cluster [DBG] 46.b starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11771 2024-09-13T15:18:43.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[84576]: cluster 2024-09-13T15:18:42.584950+0000 osd.7 (osd.7) 34 : cluster [DBG] 46.e starting backfill to osd.3 from (0'0,0'0] MAX to 1131'11644 2024-09-13T15:18:43.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[84576]: cluster 2024-09-13T15:18:42.584950+0000 osd.7 (osd.7) 34 : cluster [DBG] 46.e starting backfill to osd.3 from (0'0,0'0] MAX to 1131'11644 2024-09-13T15:18:43.670 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[84576]: cluster 2024-09-13T15:18:43.236209+0000 mgr.y (mgr.26252) 1582 : cluster [DBG] pgmap v2200: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 2068/56124 objects degraded (3.685%); 1.0 MiB/s, 67 objects/s recovering 2024-09-13T15:18:43.671 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[84576]: cluster 2024-09-13T15:18:43.236209+0000 mgr.y (mgr.26252) 1582 : cluster [DBG] pgmap v2200: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 2068/56124 objects degraded (3.685%); 1.0 MiB/s, 67 objects/s recovering 2024-09-13T15:18:43.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[87147]: cluster 2024-09-13T15:18:42.579585+0000 osd.7 (osd.7) 33 : cluster [DBG] 46.a starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11798 2024-09-13T15:18:43.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[87147]: cluster 2024-09-13T15:18:42.579585+0000 osd.7 (osd.7) 33 : cluster [DBG] 46.a starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11798 2024-09-13T15:18:43.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[87147]: cluster 2024-09-13T15:18:42.581993+0000 osd.0 (osd.0) 58 : cluster [DBG] 46.b starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11771 2024-09-13T15:18:43.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[87147]: cluster 2024-09-13T15:18:42.581993+0000 osd.0 (osd.0) 58 : cluster [DBG] 46.b starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11771 2024-09-13T15:18:43.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[87147]: cluster 2024-09-13T15:18:42.584950+0000 osd.7 (osd.7) 34 : cluster [DBG] 46.e starting backfill to osd.3 from (0'0,0'0] MAX to 1131'11644 2024-09-13T15:18:43.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[87147]: cluster 2024-09-13T15:18:42.584950+0000 osd.7 (osd.7) 34 : cluster [DBG] 46.e starting backfill to osd.3 from (0'0,0'0] MAX to 1131'11644 2024-09-13T15:18:43.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[87147]: cluster 2024-09-13T15:18:43.236209+0000 mgr.y (mgr.26252) 1582 : cluster [DBG] pgmap v2200: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 2068/56124 objects degraded (3.685%); 1.0 MiB/s, 67 objects/s recovering 2024-09-13T15:18:43.671 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:43 smithi099 bash[87147]: cluster 2024-09-13T15:18:43.236209+0000 mgr.y (mgr.26252) 1582 : cluster [DBG] pgmap v2200: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 2068/56124 objects degraded (3.685%); 1.0 MiB/s, 67 objects/s recovering 2024-09-13T15:18:43.688 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:43.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:43.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:43.791 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:43.802 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:43 smithi177 bash[68594]: cluster 2024-09-13T15:18:42.579585+0000 osd.7 (osd.7) 33 : cluster [DBG] 46.a starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11798 2024-09-13T15:18:43.802 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:43 smithi177 bash[68594]: cluster 2024-09-13T15:18:42.579585+0000 osd.7 (osd.7) 33 : cluster [DBG] 46.a starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11798 2024-09-13T15:18:43.802 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:43 smithi177 bash[68594]: cluster 2024-09-13T15:18:42.581993+0000 osd.0 (osd.0) 58 : cluster [DBG] 46.b starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11771 2024-09-13T15:18:43.802 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:43 smithi177 bash[68594]: cluster 2024-09-13T15:18:42.581993+0000 osd.0 (osd.0) 58 : cluster [DBG] 46.b starting backfill to osd.4 from (0'0,0'0] MAX to 1131'11771 2024-09-13T15:18:43.802 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:43 smithi177 bash[68594]: cluster 2024-09-13T15:18:42.584950+0000 osd.7 (osd.7) 34 : cluster [DBG] 46.e starting backfill to osd.3 from (0'0,0'0] MAX to 1131'11644 2024-09-13T15:18:43.802 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:43 smithi177 bash[68594]: cluster 2024-09-13T15:18:42.584950+0000 osd.7 (osd.7) 34 : cluster [DBG] 46.e starting backfill to osd.3 from (0'0,0'0] MAX to 1131'11644 2024-09-13T15:18:43.802 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:43 smithi177 bash[68594]: cluster 2024-09-13T15:18:43.236209+0000 mgr.y (mgr.26252) 1582 : cluster [DBG] pgmap v2200: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 2068/56124 objects degraded (3.685%); 1.0 MiB/s, 67 objects/s recovering 2024-09-13T15:18:43.802 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:43 smithi177 bash[68594]: cluster 2024-09-13T15:18:43.236209+0000 mgr.y (mgr.26252) 1582 : cluster [DBG] pgmap v2200: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 2068/56124 objects degraded (3.685%); 1.0 MiB/s, 67 objects/s recovering 2024-09-13T15:18:43.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:43.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:43.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:43.921 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:43.939 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:43.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:43.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:44.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:44.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:44.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:44.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:44.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:44.208 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:44.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:44.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:44.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:44.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:44.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:44.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:44.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:44.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:44.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:44.591 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:44.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:44.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:44.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:44.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:44.760 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:44.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:44.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:44.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:44.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:44.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:44.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:45.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:45.028 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:45.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:45.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:45.134 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:45.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:45.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:45.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:45.238 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T15:18:45.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:45.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:45.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:45.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:45.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:45.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:45.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:45.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:45 smithi099 bash[84576]: cluster 2024-09-13T15:18:45.236973+0000 mgr.y (mgr.26252) 1583 : cluster [DBG] pgmap v2201: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 102 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 6483/50862 objects degraded (12.746%); 4197/50862 objects misplaced (8.252%); 5.1 MiB/s, 1 keys/s, 142 objects/s recovering 2024-09-13T15:18:45.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:45 smithi099 bash[84576]: cluster 2024-09-13T15:18:45.236973+0000 mgr.y (mgr.26252) 1583 : cluster [DBG] pgmap v2201: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 102 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 6483/50862 objects degraded (12.746%); 4197/50862 objects misplaced (8.252%); 5.1 MiB/s, 1 keys/s, 142 objects/s recovering 2024-09-13T15:18:45.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:45 smithi099 bash[87147]: cluster 2024-09-13T15:18:45.236973+0000 mgr.y (mgr.26252) 1583 : cluster [DBG] pgmap v2201: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 102 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 6483/50862 objects degraded (12.746%); 4197/50862 objects misplaced (8.252%); 5.1 MiB/s, 1 keys/s, 142 objects/s recovering 2024-09-13T15:18:45.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:45 smithi099 bash[87147]: cluster 2024-09-13T15:18:45.236973+0000 mgr.y (mgr.26252) 1583 : cluster [DBG] pgmap v2201: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 102 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 6483/50862 objects degraded (12.746%); 4197/50862 objects misplaced (8.252%); 5.1 MiB/s, 1 keys/s, 142 objects/s recovering 2024-09-13T15:18:45.577 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:45.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:45.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:45 smithi177 bash[68594]: cluster 2024-09-13T15:18:45.236973+0000 mgr.y (mgr.26252) 1583 : cluster [DBG] pgmap v2201: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 102 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 6483/50862 objects degraded (12.746%); 4197/50862 objects misplaced (8.252%); 5.1 MiB/s, 1 keys/s, 142 objects/s recovering 2024-09-13T15:18:45.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:45 smithi177 bash[68594]: cluster 2024-09-13T15:18:45.236973+0000 mgr.y (mgr.26252) 1583 : cluster [DBG] pgmap v2201: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 102 active+clean; 1.0 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 6483/50862 objects degraded (12.746%); 4197/50862 objects misplaced (8.252%); 5.1 MiB/s, 1 keys/s, 142 objects/s recovering 2024-09-13T15:18:45.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:45.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:45.718 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:45.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:45.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:45.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:45.848 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:45.878 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:45.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:45.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:45.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:46.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:46.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:46.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:46.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:46.143 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:46.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:46.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:46.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:46.274 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:46.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:46.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:46.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:46.414 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:46.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:46.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:46.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:46.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:46.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:46.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:46.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:46.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:46.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:46.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:46.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:46.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:46 smithi099 bash[84576]: cluster 2024-09-13T15:18:46.675688+0000 mon.a (mon.0) 2764 : cluster [WRN] Health check update: Degraded data redundancy: 6483/50862 objects degraded (12.746%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:46.790 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:46 smithi099 bash[84576]: cluster 2024-09-13T15:18:46.675688+0000 mon.a (mon.0) 2764 : cluster [WRN] Health check update: Degraded data redundancy: 6483/50862 objects degraded (12.746%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:46.790 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:46 smithi099 bash[87147]: cluster 2024-09-13T15:18:46.675688+0000 mon.a (mon.0) 2764 : cluster [WRN] Health check update: Degraded data redundancy: 6483/50862 objects degraded (12.746%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:46.790 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:46 smithi099 bash[87147]: cluster 2024-09-13T15:18:46.675688+0000 mon.a (mon.0) 2764 : cluster [WRN] Health check update: Degraded data redundancy: 6483/50862 objects degraded (12.746%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:46.815 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:46.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:46.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:46.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:46.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:46.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:46.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:47.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:47.063 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:46 smithi177 bash[68594]: cluster 2024-09-13T15:18:46.675688+0000 mon.a (mon.0) 2764 : cluster [WRN] Health check update: Degraded data redundancy: 6483/50862 objects degraded (12.746%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:47.063 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:46 smithi177 bash[68594]: cluster 2024-09-13T15:18:46.675688+0000 mon.a (mon.0) 2764 : cluster [WRN] Health check update: Degraded data redundancy: 6483/50862 objects degraded (12.746%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:47.097 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:47.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:47.203 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:47.241 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:47.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:47.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:47.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:18:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:18:47] "GET /metrics HTTP/1.1" 200 36787 "" "Prometheus/2.51.0" 2024-09-13T15:18:47.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:18:47.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:18:47.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:47.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:47.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:47.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:47.478 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:47.531 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:47.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:47.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:47.637 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:47.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:47.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:47.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:47.774 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:47.807 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:47.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:47.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:47.911 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:47.921 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:47 smithi099 bash[84576]: cluster 2024-09-13T15:18:47.237949+0000 mgr.y (mgr.26252) 1584 : cluster [DBG] pgmap v2202: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 1016 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 12851/48840 objects degraded (26.312%); 4197/48840 objects misplaced (8.593%); 3.7 MiB/s, 1 keys/s, 69 objects/s recovering 2024-09-13T15:18:47.922 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:47 smithi099 bash[84576]: cluster 2024-09-13T15:18:47.237949+0000 mgr.y (mgr.26252) 1584 : cluster [DBG] pgmap v2202: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 1016 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 12851/48840 objects degraded (26.312%); 4197/48840 objects misplaced (8.593%); 3.7 MiB/s, 1 keys/s, 69 objects/s recovering 2024-09-13T15:18:47.922 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:47 smithi099 bash[87147]: cluster 2024-09-13T15:18:47.237949+0000 mgr.y (mgr.26252) 1584 : cluster [DBG] pgmap v2202: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 1016 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 12851/48840 objects degraded (26.312%); 4197/48840 objects misplaced (8.593%); 3.7 MiB/s, 1 keys/s, 69 objects/s recovering 2024-09-13T15:18:47.922 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:47 smithi099 bash[87147]: cluster 2024-09-13T15:18:47.237949+0000 mgr.y (mgr.26252) 1584 : cluster [DBG] pgmap v2202: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 1016 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 12851/48840 objects degraded (26.312%); 4197/48840 objects misplaced (8.593%); 3.7 MiB/s, 1 keys/s, 69 objects/s recovering 2024-09-13T15:18:47.934 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:47.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:47.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:48.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:48.049 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:47 smithi177 bash[68594]: cluster 2024-09-13T15:18:47.237949+0000 mgr.y (mgr.26252) 1584 : cluster [DBG] pgmap v2202: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 1016 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 12851/48840 objects degraded (26.312%); 4197/48840 objects misplaced (8.593%); 3.7 MiB/s, 1 keys/s, 69 objects/s recovering 2024-09-13T15:18:48.050 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:47 smithi177 bash[68594]: cluster 2024-09-13T15:18:47.237949+0000 mgr.y (mgr.26252) 1584 : cluster [DBG] pgmap v2202: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 1016 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 12851/48840 objects degraded (26.312%); 4197/48840 objects misplaced (8.593%); 3.7 MiB/s, 1 keys/s, 69 objects/s recovering 2024-09-13T15:18:48.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:48.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:48.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:48.184 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:48.213 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:48.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:48.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:48.317 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:48.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:48.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:48.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:48.491 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:48.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:48.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:48.597 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:48.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:48.626 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:48.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:48.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:48.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:48.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:48.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:48.865 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:48.885 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:48.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:48.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:48.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:49.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:49.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:49.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:49.165 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:49.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:49.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:49.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:49.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:49.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:49.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:49.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:49.431 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:49.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:49.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:49.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:49.567 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:49.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:49.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:49 smithi177 bash[68594]: audit 2024-09-13T15:18:48.331507+0000 mon.a (mon.0) 2765 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:49 smithi177 bash[68594]: audit 2024-09-13T15:18:48.331507+0000 mon.a (mon.0) 2765 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:49 smithi177 bash[68594]: audit 2024-09-13T15:18:48.334500+0000 mon.a (mon.0) 2766 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:49 smithi177 bash[68594]: audit 2024-09-13T15:18:48.334500+0000 mon.a (mon.0) 2766 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:49.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:49 smithi177 bash[68594]: cluster 2024-09-13T15:18:49.238588+0000 mgr.y (mgr.26252) 1585 : cluster [DBG] pgmap v2203: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 908 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 12851/43617 objects degraded (29.463%); 3346/43617 objects misplaced (7.671%); 5.2 MiB/s, 1 keys/s, 91 objects/s recovering 2024-09-13T15:18:49.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:49 smithi177 bash[68594]: cluster 2024-09-13T15:18:49.238588+0000 mgr.y (mgr.26252) 1585 : cluster [DBG] pgmap v2203: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 908 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 12851/43617 objects degraded (29.463%); 3346/43617 objects misplaced (7.671%); 5.2 MiB/s, 1 keys/s, 91 objects/s recovering 2024-09-13T15:18:49.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:49.680 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[84576]: audit 2024-09-13T15:18:48.331507+0000 mon.a (mon.0) 2765 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:49.681 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[84576]: audit 2024-09-13T15:18:48.331507+0000 mon.a (mon.0) 2765 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:49.681 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[84576]: audit 2024-09-13T15:18:48.334500+0000 mon.a (mon.0) 2766 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:49.681 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[84576]: audit 2024-09-13T15:18:48.334500+0000 mon.a (mon.0) 2766 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:49.681 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[84576]: cluster 2024-09-13T15:18:49.238588+0000 mgr.y (mgr.26252) 1585 : cluster [DBG] pgmap v2203: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 908 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 12851/43617 objects degraded (29.463%); 3346/43617 objects misplaced (7.671%); 5.2 MiB/s, 1 keys/s, 91 objects/s recovering 2024-09-13T15:18:49.681 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[84576]: cluster 2024-09-13T15:18:49.238588+0000 mgr.y (mgr.26252) 1585 : cluster [DBG] pgmap v2203: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 908 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 12851/43617 objects degraded (29.463%); 3346/43617 objects misplaced (7.671%); 5.2 MiB/s, 1 keys/s, 91 objects/s recovering 2024-09-13T15:18:49.681 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[87147]: audit 2024-09-13T15:18:48.331507+0000 mon.a (mon.0) 2765 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:49.681 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[87147]: audit 2024-09-13T15:18:48.331507+0000 mon.a (mon.0) 2765 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:18:49.681 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[87147]: audit 2024-09-13T15:18:48.334500+0000 mon.a (mon.0) 2766 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:49.681 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[87147]: audit 2024-09-13T15:18:48.334500+0000 mon.a (mon.0) 2766 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:18:49.682 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[87147]: cluster 2024-09-13T15:18:49.238588+0000 mgr.y (mgr.26252) 1585 : cluster [DBG] pgmap v2203: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 908 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 12851/43617 objects degraded (29.463%); 3346/43617 objects misplaced (7.671%); 5.2 MiB/s, 1 keys/s, 91 objects/s recovering 2024-09-13T15:18:49.682 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:49 smithi099 bash[87147]: cluster 2024-09-13T15:18:49.238588+0000 mgr.y (mgr.26252) 1585 : cluster [DBG] pgmap v2203: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 908 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 12851/43617 objects degraded (29.463%); 3346/43617 objects misplaced (7.671%); 5.2 MiB/s, 1 keys/s, 91 objects/s recovering 2024-09-13T15:18:49.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:49.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:49.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:49.800 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:49.824 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:49.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:49.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:49.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:49.958 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:49.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:49.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:50.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:50.100 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:50.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:50.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:50.128 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:18:50.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:50.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:50.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:50.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:50.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:50.352 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:18:50 smithi177 bash[61439]: ts=2024-09-13T15:18:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:18:50.357 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 4, 1, 6] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:18:50.357 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:18:50.357 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-09-13T15:18:50.357 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:18:50.357 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:18:50.357 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-09-13T15:18:50.372 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:50.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:50.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:50.482 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:50.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:50.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:50.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:50.614 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:50.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:50.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:50.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:50.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:50.792 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:50.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:50.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:50.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:50.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:50.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:50.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:51.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:51.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:51.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:51.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:51.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:51.197 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:51.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:51.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:51.304 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:51.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:51.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:51.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:51.439 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:51.449 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:51 smithi177 bash[68594]: cluster 2024-09-13T15:18:51.239252+0000 mgr.y (mgr.26252) 1586 : cluster [DBG] pgmap v2204: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 6.0 MiB/s, 1 keys/s, 102 objects/s recovering 2024-09-13T15:18:51.449 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:51 smithi177 bash[68594]: cluster 2024-09-13T15:18:51.239252+0000 mgr.y (mgr.26252) 1586 : cluster [DBG] pgmap v2204: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 6.0 MiB/s, 1 keys/s, 102 objects/s recovering 2024-09-13T15:18:51.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:51.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:51 smithi099 bash[84576]: cluster 2024-09-13T15:18:51.239252+0000 mgr.y (mgr.26252) 1586 : cluster [DBG] pgmap v2204: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 6.0 MiB/s, 1 keys/s, 102 objects/s recovering 2024-09-13T15:18:51.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:51 smithi099 bash[84576]: cluster 2024-09-13T15:18:51.239252+0000 mgr.y (mgr.26252) 1586 : cluster [DBG] pgmap v2204: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 6.0 MiB/s, 1 keys/s, 102 objects/s recovering 2024-09-13T15:18:51.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:51 smithi099 bash[87147]: cluster 2024-09-13T15:18:51.239252+0000 mgr.y (mgr.26252) 1586 : cluster [DBG] pgmap v2204: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 6.0 MiB/s, 1 keys/s, 102 objects/s recovering 2024-09-13T15:18:51.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:51 smithi099 bash[87147]: cluster 2024-09-13T15:18:51.239252+0000 mgr.y (mgr.26252) 1586 : cluster [DBG] pgmap v2204: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 6.0 MiB/s, 1 keys/s, 102 objects/s recovering 2024-09-13T15:18:51.579 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:51.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:51.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:51.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:51.723 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:51.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:51.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:51.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:51.865 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:51.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:51.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:51.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:51.986 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:52.011 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:52.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:52.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:52.114 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:52.140 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:52.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:52.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:52.154 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:18:52.155 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:18:52.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:52.268 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:52.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:52.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:52.372 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:52.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:52.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:52.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:52.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:52.521 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:52.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:52.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:52 smithi177 bash[68594]: cluster 2024-09-13T15:18:51.676381+0000 mon.a (mon.0) 2767 : cluster [WRN] Health check update: Degraded data redundancy: 12851/35340 objects degraded (36.364%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:52.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:52 smithi177 bash[68594]: cluster 2024-09-13T15:18:51.676381+0000 mon.a (mon.0) 2767 : cluster [WRN] Health check update: Degraded data redundancy: 12851/35340 objects degraded (36.364%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:52.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:52.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:52.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:52.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:52.783 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:52.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:52.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:52.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:52 smithi099 bash[84576]: cluster 2024-09-13T15:18:51.676381+0000 mon.a (mon.0) 2767 : cluster [WRN] Health check update: Degraded data redundancy: 12851/35340 objects degraded (36.364%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:52.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:52 smithi099 bash[84576]: cluster 2024-09-13T15:18:51.676381+0000 mon.a (mon.0) 2767 : cluster [WRN] Health check update: Degraded data redundancy: 12851/35340 objects degraded (36.364%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:52.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:52 smithi099 bash[87147]: cluster 2024-09-13T15:18:51.676381+0000 mon.a (mon.0) 2767 : cluster [WRN] Health check update: Degraded data redundancy: 12851/35340 objects degraded (36.364%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:52.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:52 smithi099 bash[87147]: cluster 2024-09-13T15:18:51.676381+0000 mon.a (mon.0) 2767 : cluster [WRN] Health check update: Degraded data redundancy: 12851/35340 objects degraded (36.364%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:18:52.887 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:52.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:52.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:53.011 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:53.034 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:53.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:53.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:53.137 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:53.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:53.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:53.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:53.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:53.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:53.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:53.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:53.397 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:53 smithi177 bash[68594]: cluster 2024-09-13T15:18:52.360015+0000 mon.a (mon.0) 2768 : cluster [DBG] osdmap e1135: 8 total, 8 up, 7 in 2024-09-13T15:18:53.397 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:53 smithi177 bash[68594]: cluster 2024-09-13T15:18:52.360015+0000 mon.a (mon.0) 2768 : cluster [DBG] osdmap e1135: 8 total, 8 up, 7 in 2024-09-13T15:18:53.398 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:53 smithi177 bash[68594]: cluster 2024-09-13T15:18:53.240180+0000 mgr.y (mgr.26252) 1587 : cluster [DBG] pgmap v2206: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 5.8 MiB/s, 0 keys/s, 98 objects/s recovering 2024-09-13T15:18:53.398 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:53 smithi177 bash[68594]: cluster 2024-09-13T15:18:53.240180+0000 mgr.y (mgr.26252) 1587 : cluster [DBG] pgmap v2206: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 5.8 MiB/s, 0 keys/s, 98 objects/s recovering 2024-09-13T15:18:53.408 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:53.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:53.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:53.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:53.548 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:53 smithi099 bash[84576]: cluster 2024-09-13T15:18:52.360015+0000 mon.a (mon.0) 2768 : cluster [DBG] osdmap e1135: 8 total, 8 up, 7 in 2024-09-13T15:18:53.548 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:53 smithi099 bash[84576]: cluster 2024-09-13T15:18:52.360015+0000 mon.a (mon.0) 2768 : cluster [DBG] osdmap e1135: 8 total, 8 up, 7 in 2024-09-13T15:18:53.548 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:53 smithi099 bash[84576]: cluster 2024-09-13T15:18:53.240180+0000 mgr.y (mgr.26252) 1587 : cluster [DBG] pgmap v2206: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 5.8 MiB/s, 0 keys/s, 98 objects/s recovering 2024-09-13T15:18:53.548 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:53 smithi099 bash[84576]: cluster 2024-09-13T15:18:53.240180+0000 mgr.y (mgr.26252) 1587 : cluster [DBG] pgmap v2206: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 5.8 MiB/s, 0 keys/s, 98 objects/s recovering 2024-09-13T15:18:53.549 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:53 smithi099 bash[87147]: cluster 2024-09-13T15:18:52.360015+0000 mon.a (mon.0) 2768 : cluster [DBG] osdmap e1135: 8 total, 8 up, 7 in 2024-09-13T15:18:53.549 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:53 smithi099 bash[87147]: cluster 2024-09-13T15:18:52.360015+0000 mon.a (mon.0) 2768 : cluster [DBG] osdmap e1135: 8 total, 8 up, 7 in 2024-09-13T15:18:53.549 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:53 smithi099 bash[87147]: cluster 2024-09-13T15:18:53.240180+0000 mgr.y (mgr.26252) 1587 : cluster [DBG] pgmap v2206: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 5.8 MiB/s, 0 keys/s, 98 objects/s recovering 2024-09-13T15:18:53.549 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:53 smithi099 bash[87147]: cluster 2024-09-13T15:18:53.240180+0000 mgr.y (mgr.26252) 1587 : cluster [DBG] pgmap v2206: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 100 active+clean; 735 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 12851/35340 objects degraded (36.364%); 2545/35340 objects misplaced (7.201%); 5.8 MiB/s, 0 keys/s, 98 objects/s recovering 2024-09-13T15:18:53.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:53.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:53.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:53.680 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:53.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:53.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:53.787 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:53.814 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:53.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:53.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:53.952 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:53.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:53.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:54.059 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:54.086 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:54.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:54.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:54.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:54.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:54.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:54.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:54.324 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:54.368 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:54.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:54.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:54.476 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:54.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:54.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:54.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:54 smithi177 bash[68594]: cluster 2024-09-13T15:18:53.372205+0000 mon.a (mon.0) 2769 : cluster [DBG] osdmap e1136: 8 total, 8 up, 7 in 2024-09-13T15:18:54.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:54 smithi177 bash[68594]: cluster 2024-09-13T15:18:53.372205+0000 mon.a (mon.0) 2769 : cluster [DBG] osdmap e1136: 8 total, 8 up, 7 in 2024-09-13T15:18:54.624 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:54.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:54.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:54.688 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:54 smithi099 bash[84576]: cluster 2024-09-13T15:18:53.372205+0000 mon.a (mon.0) 2769 : cluster [DBG] osdmap e1136: 8 total, 8 up, 7 in 2024-09-13T15:18:54.688 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:54 smithi099 bash[84576]: cluster 2024-09-13T15:18:53.372205+0000 mon.a (mon.0) 2769 : cluster [DBG] osdmap e1136: 8 total, 8 up, 7 in 2024-09-13T15:18:54.688 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:54 smithi099 bash[87147]: cluster 2024-09-13T15:18:53.372205+0000 mon.a (mon.0) 2769 : cluster [DBG] osdmap e1136: 8 total, 8 up, 7 in 2024-09-13T15:18:54.688 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:54 smithi099 bash[87147]: cluster 2024-09-13T15:18:53.372205+0000 mon.a (mon.0) 2769 : cluster [DBG] osdmap e1136: 8 total, 8 up, 7 in 2024-09-13T15:18:54.754 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:54.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:54.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:54.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:54.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:54.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:54.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:54.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:55.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:55.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:55.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:55.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:55.186 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:55.224 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:18:55.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:55.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:55.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:55.338 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:55.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:55.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:55.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:55.500 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:55 smithi177 bash[68594]: cluster 2024-09-13T15:18:55.241003+0000 mgr.y (mgr.26252) 1588 : cluster [DBG] pgmap v2208: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 582 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 12851/27999 objects degraded (45.898%); 1695/27999 objects misplaced (6.054%); 5.9 MiB/s, 144 objects/s recovering 2024-09-13T15:18:55.500 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:55 smithi177 bash[68594]: cluster 2024-09-13T15:18:55.241003+0000 mgr.y (mgr.26252) 1588 : cluster [DBG] pgmap v2208: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 582 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 12851/27999 objects degraded (45.898%); 1695/27999 objects misplaced (6.054%); 5.9 MiB/s, 144 objects/s recovering 2024-09-13T15:18:55.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:55.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:55.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:55.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:55.632 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:55 smithi099 bash[84576]: cluster 2024-09-13T15:18:55.241003+0000 mgr.y (mgr.26252) 1588 : cluster [DBG] pgmap v2208: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 582 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 12851/27999 objects degraded (45.898%); 1695/27999 objects misplaced (6.054%); 5.9 MiB/s, 144 objects/s recovering 2024-09-13T15:18:55.632 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:55 smithi099 bash[84576]: cluster 2024-09-13T15:18:55.241003+0000 mgr.y (mgr.26252) 1588 : cluster [DBG] pgmap v2208: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 582 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 12851/27999 objects degraded (45.898%); 1695/27999 objects misplaced (6.054%); 5.9 MiB/s, 144 objects/s recovering 2024-09-13T15:18:55.632 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:55 smithi099 bash[87147]: cluster 2024-09-13T15:18:55.241003+0000 mgr.y (mgr.26252) 1588 : cluster [DBG] pgmap v2208: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 582 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 12851/27999 objects degraded (45.898%); 1695/27999 objects misplaced (6.054%); 5.9 MiB/s, 144 objects/s recovering 2024-09-13T15:18:55.632 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:55 smithi099 bash[87147]: cluster 2024-09-13T15:18:55.241003+0000 mgr.y (mgr.26252) 1588 : cluster [DBG] pgmap v2208: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 582 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 12851/27999 objects degraded (45.898%); 1695/27999 objects misplaced (6.054%); 5.9 MiB/s, 144 objects/s recovering 2024-09-13T15:18:55.652 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:55.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:55.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:55.756 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:55.787 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:55.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:55.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:55.895 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:55.927 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:55.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:55.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:56.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:56.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:56.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:56.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:56.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:56.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:56.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:56.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:56.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:56.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:56.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:56.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:56.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:56.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:56.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:56.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:56.581 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:56.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:56.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:56.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:56.725 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:56.748 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:56.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:56.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:56.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:56.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:56.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:56.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:56.979 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:56.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:56 smithi177 bash[68594]: cluster 2024-09-13T15:18:56.677170+0000 mon.a (mon.0) 2770 : cluster [WRN] Health check update: Degraded data redundancy: 12851/27999 objects degraded (45.898%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:18:56.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:56 smithi177 bash[68594]: cluster 2024-09-13T15:18:56.677170+0000 mon.a (mon.0) 2770 : cluster [WRN] Health check update: Degraded data redundancy: 12851/27999 objects degraded (45.898%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:18:57.012 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:57.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:57.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:57.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:57.157 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:56 smithi099 bash[84576]: cluster 2024-09-13T15:18:56.677170+0000 mon.a (mon.0) 2770 : cluster [WRN] Health check update: Degraded data redundancy: 12851/27999 objects degraded (45.898%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:18:57.157 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:56 smithi099 bash[84576]: cluster 2024-09-13T15:18:56.677170+0000 mon.a (mon.0) 2770 : cluster [WRN] Health check update: Degraded data redundancy: 12851/27999 objects degraded (45.898%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:18:57.157 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:56 smithi099 bash[87147]: cluster 2024-09-13T15:18:56.677170+0000 mon.a (mon.0) 2770 : cluster [WRN] Health check update: Degraded data redundancy: 12851/27999 objects degraded (45.898%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:18:57.158 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:56 smithi099 bash[87147]: cluster 2024-09-13T15:18:56.677170+0000 mon.a (mon.0) 2770 : cluster [WRN] Health check update: Degraded data redundancy: 12851/27999 objects degraded (45.898%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:18:57.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:57.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:57.260 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:57.292 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:57.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:57.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:57.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:18:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:57.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:18:57.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:57.529 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:18:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:18:57] "GET /metrics HTTP/1.1" 200 36778 "" "Prometheus/2.51.0" 2024-09-13T15:18:57.542 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:57.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:57.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:57.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:57.674 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:57.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:57.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:57.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:57.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:57.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:57.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:57.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:57 smithi099 bash[84576]: cluster 2024-09-13T15:18:57.241679+0000 mgr.y (mgr.26252) 1589 : cluster [DBG] pgmap v2209: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 532 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 12851/25575 objects degraded (50.248%); 1695/25575 objects misplaced (6.628%); 3.8 MiB/s, 110 objects/s recovering 2024-09-13T15:18:57.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:57 smithi099 bash[84576]: cluster 2024-09-13T15:18:57.241679+0000 mgr.y (mgr.26252) 1589 : cluster [DBG] pgmap v2209: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 532 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 12851/25575 objects degraded (50.248%); 1695/25575 objects misplaced (6.628%); 3.8 MiB/s, 110 objects/s recovering 2024-09-13T15:18:57.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:57 smithi099 bash[87147]: cluster 2024-09-13T15:18:57.241679+0000 mgr.y (mgr.26252) 1589 : cluster [DBG] pgmap v2209: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 532 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 12851/25575 objects degraded (50.248%); 1695/25575 objects misplaced (6.628%); 3.8 MiB/s, 110 objects/s recovering 2024-09-13T15:18:57.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:57 smithi099 bash[87147]: cluster 2024-09-13T15:18:57.241679+0000 mgr.y (mgr.26252) 1589 : cluster [DBG] pgmap v2209: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 532 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 12851/25575 objects degraded (50.248%); 1695/25575 objects misplaced (6.628%); 3.8 MiB/s, 110 objects/s recovering 2024-09-13T15:18:57.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:57.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:57.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:57.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:58.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:58.050 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:57 smithi177 bash[68594]: cluster 2024-09-13T15:18:57.241679+0000 mgr.y (mgr.26252) 1589 : cluster [DBG] pgmap v2209: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 532 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 12851/25575 objects degraded (50.248%); 1695/25575 objects misplaced (6.628%); 3.8 MiB/s, 110 objects/s recovering 2024-09-13T15:18:58.050 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:57 smithi177 bash[68594]: cluster 2024-09-13T15:18:57.241679+0000 mgr.y (mgr.26252) 1589 : cluster [DBG] pgmap v2209: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 532 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 12851/25575 objects degraded (50.248%); 1695/25575 objects misplaced (6.628%); 3.8 MiB/s, 110 objects/s recovering 2024-09-13T15:18:58.081 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:18:58.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:58.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:18:58.185 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:58.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:58.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:58.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:58.323 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:58.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:58.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:58.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:58.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:58.496 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:58.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:58.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:58.600 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:58.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:18:58.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:58.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:18:58.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:58.759 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:58.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:58.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:58.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:58.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:58.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:59.011 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:59.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:59.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:59.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:59.135 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:59.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:59.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:59.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:59.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:59.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:59.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:18:59.408 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:59.454 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:18:59.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:59.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:18:59.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:59.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:59 smithi099 bash[84576]: cluster 2024-09-13T15:18:59.242250+0000 mgr.y (mgr.26252) 1590 : cluster [DBG] pgmap v2210: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 422 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 12851/20292 objects degraded (63.330%); 1134/20292 objects misplaced (5.588%); 1.8 MiB/s, 125 objects/s recovering 2024-09-13T15:18:59.567 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:18:59 smithi099 bash[84576]: cluster 2024-09-13T15:18:59.242250+0000 mgr.y (mgr.26252) 1590 : cluster [DBG] pgmap v2210: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 422 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 12851/20292 objects degraded (63.330%); 1134/20292 objects misplaced (5.588%); 1.8 MiB/s, 125 objects/s recovering 2024-09-13T15:18:59.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:59 smithi099 bash[87147]: cluster 2024-09-13T15:18:59.242250+0000 mgr.y (mgr.26252) 1590 : cluster [DBG] pgmap v2210: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 422 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 12851/20292 objects degraded (63.330%); 1134/20292 objects misplaced (5.588%); 1.8 MiB/s, 125 objects/s recovering 2024-09-13T15:18:59.568 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:18:59 smithi099 bash[87147]: cluster 2024-09-13T15:18:59.242250+0000 mgr.y (mgr.26252) 1590 : cluster [DBG] pgmap v2210: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 422 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 12851/20292 objects degraded (63.330%); 1134/20292 objects misplaced (5.588%); 1.8 MiB/s, 125 objects/s recovering 2024-09-13T15:18:59.591 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:18:59.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:59.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:18:59.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:59 smithi177 bash[68594]: cluster 2024-09-13T15:18:59.242250+0000 mgr.y (mgr.26252) 1590 : cluster [DBG] pgmap v2210: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 422 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 12851/20292 objects degraded (63.330%); 1134/20292 objects misplaced (5.588%); 1.8 MiB/s, 125 objects/s recovering 2024-09-13T15:18:59.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:18:59 smithi177 bash[68594]: cluster 2024-09-13T15:18:59.242250+0000 mgr.y (mgr.26252) 1590 : cluster [DBG] pgmap v2210: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 422 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 12851/20292 objects degraded (63.330%); 1134/20292 objects misplaced (5.588%); 1.8 MiB/s, 125 objects/s recovering 2024-09-13T15:18:59.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:59.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:18:59.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:59.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:18:59.834 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:59.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:18:59.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:59.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:18:59.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:59.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:18:59.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:18:59.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:00.096 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:00.118 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:00.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:00.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:00.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:00.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:00.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:00.357 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:19:00 smithi177 bash[61439]: ts=2024-09-13T15:19:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:19:00.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:00.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:00.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:00.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:00.512 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:00.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:00.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:00.639 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:00.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:00.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:00.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:00.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:00.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:00.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:00.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:00.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:00.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:01.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:01.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:01.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:01.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:01.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:01.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:01.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:01.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:01.273 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:01.303 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:01.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:01.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:01.408 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:01.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:01.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:01.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:01.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:01.548 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:01 smithi099 bash[84576]: cluster 2024-09-13T15:19:01.242857+0000 mgr.y (mgr.26252) 1591 : cluster [DBG] pgmap v2211: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 841 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.6 MiB/s, 113 objects/s recovering 2024-09-13T15:19:01.549 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:01 smithi099 bash[84576]: cluster 2024-09-13T15:19:01.242857+0000 mgr.y (mgr.26252) 1591 : cluster [DBG] pgmap v2211: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 841 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.6 MiB/s, 113 objects/s recovering 2024-09-13T15:19:01.549 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:01 smithi099 bash[87147]: cluster 2024-09-13T15:19:01.242857+0000 mgr.y (mgr.26252) 1591 : cluster [DBG] pgmap v2211: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 841 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.6 MiB/s, 113 objects/s recovering 2024-09-13T15:19:01.549 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:01 smithi099 bash[87147]: cluster 2024-09-13T15:19:01.242857+0000 mgr.y (mgr.26252) 1591 : cluster [DBG] pgmap v2211: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 841 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.6 MiB/s, 113 objects/s recovering 2024-09-13T15:19:01.561 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:01.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:01.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:01.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:01 smithi177 bash[68594]: cluster 2024-09-13T15:19:01.242857+0000 mgr.y (mgr.26252) 1591 : cluster [DBG] pgmap v2211: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 841 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.6 MiB/s, 113 objects/s recovering 2024-09-13T15:19:01.616 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:01 smithi177 bash[68594]: cluster 2024-09-13T15:19:01.242857+0000 mgr.y (mgr.26252) 1591 : cluster [DBG] pgmap v2211: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 841 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.6 MiB/s, 113 objects/s recovering 2024-09-13T15:19:01.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:01.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:01.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:01.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:01.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:01.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:01.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:01.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:01.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:01.941 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:01.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:01.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:02.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:02.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:02.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:02.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:02.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:02.207 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:02.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:02.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:02.311 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:02.331 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:02.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:02.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:02.434 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:02.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:02.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:02.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:02.564 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:02.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:02 smithi099 bash[84576]: cluster 2024-09-13T15:19:01.680580+0000 mon.a (mon.0) 2771 : cluster [WRN] Health check update: Degraded data redundancy: 12851/12891 objects degraded (99.690%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:02.571 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:02 smithi099 bash[84576]: cluster 2024-09-13T15:19:01.680580+0000 mon.a (mon.0) 2771 : cluster [WRN] Health check update: Degraded data redundancy: 12851/12891 objects degraded (99.690%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:02.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:02 smithi099 bash[87147]: cluster 2024-09-13T15:19:01.680580+0000 mon.a (mon.0) 2771 : cluster [WRN] Health check update: Degraded data redundancy: 12851/12891 objects degraded (99.690%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:02.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:02 smithi099 bash[87147]: cluster 2024-09-13T15:19:01.680580+0000 mon.a (mon.0) 2771 : cluster [WRN] Health check update: Degraded data redundancy: 12851/12891 objects degraded (99.690%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:02.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:02.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:02.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:02.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:02 smithi177 bash[68594]: cluster 2024-09-13T15:19:01.680580+0000 mon.a (mon.0) 2771 : cluster [WRN] Health check update: Degraded data redundancy: 12851/12891 objects degraded (99.690%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:02.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:02 smithi177 bash[68594]: cluster 2024-09-13T15:19:01.680580+0000 mon.a (mon.0) 2771 : cluster [WRN] Health check update: Degraded data redundancy: 12851/12891 objects degraded (99.690%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:02.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:02.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:02.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:02.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:02.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:02.833 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:02.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:02.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:02.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:02.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:02.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:03.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:03.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:03.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:03.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:03.205 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:03.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:03.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:03.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:03.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:03.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:03.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:03.470 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:03.479 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:03 smithi177 bash[68594]: cluster 2024-09-13T15:19:03.243475+0000 mgr.y (mgr.26252) 1592 : cluster [DBG] pgmap v2212: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.4 MiB/s, 100 objects/s recovering 2024-09-13T15:19:03.479 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:03 smithi177 bash[68594]: cluster 2024-09-13T15:19:03.243475+0000 mgr.y (mgr.26252) 1592 : cluster [DBG] pgmap v2212: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.4 MiB/s, 100 objects/s recovering 2024-09-13T15:19:03.480 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:03 smithi177 bash[68594]: audit 2024-09-13T15:19:03.323099+0000 mon.a (mon.0) 2772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:03.480 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:03 smithi177 bash[68594]: audit 2024-09-13T15:19:03.323099+0000 mon.a (mon.0) 2772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:03.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:03.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:03.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:03 smithi099 bash[84576]: cluster 2024-09-13T15:19:03.243475+0000 mgr.y (mgr.26252) 1592 : cluster [DBG] pgmap v2212: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.4 MiB/s, 100 objects/s recovering 2024-09-13T15:19:03.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:03 smithi099 bash[84576]: cluster 2024-09-13T15:19:03.243475+0000 mgr.y (mgr.26252) 1592 : cluster [DBG] pgmap v2212: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.4 MiB/s, 100 objects/s recovering 2024-09-13T15:19:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:03 smithi099 bash[84576]: audit 2024-09-13T15:19:03.323099+0000 mon.a (mon.0) 2772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:03 smithi099 bash[84576]: audit 2024-09-13T15:19:03.323099+0000 mon.a (mon.0) 2772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:03 smithi099 bash[87147]: cluster 2024-09-13T15:19:03.243475+0000 mgr.y (mgr.26252) 1592 : cluster [DBG] pgmap v2212: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.4 MiB/s, 100 objects/s recovering 2024-09-13T15:19:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:03 smithi099 bash[87147]: cluster 2024-09-13T15:19:03.243475+0000 mgr.y (mgr.26252) 1592 : cluster [DBG] pgmap v2212: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 267 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 12851/12891 objects degraded (99.690%); 871/12891 objects misplaced (6.757%); 1.4 MiB/s, 100 objects/s recovering 2024-09-13T15:19:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:03 smithi099 bash[87147]: audit 2024-09-13T15:19:03.323099+0000 mon.a (mon.0) 2772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:03.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:03 smithi099 bash[87147]: audit 2024-09-13T15:19:03.323099+0000 mon.a (mon.0) 2772 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:03.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:03.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:03.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:03.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:03.783 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:03.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:03.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:03.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:03.920 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:03.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:03.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:04.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:04.052 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:04.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:04.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:04.161 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:04.195 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:04.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:04.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:04.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:04.320 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:04.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:04.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:04.422 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:04.445 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:04.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:04.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:04.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:04.574 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:04.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:04.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:04.677 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:04.704 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:04.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:04.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:04.807 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:04.834 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:04.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:04.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:04.937 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:04.961 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:04.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:04.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:05.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:05.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:05.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:05.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:05.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:05.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:05.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:05.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:05.352 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:05.378 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:05.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:05.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:05.482 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:05.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:05.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:05.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:05.588 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Removed 102410 objects 2024-09-13T15:19:05.588 INFO:tasks.radosbench.radosbench.0.smithi099.stdout:Clean up completed and total clean up time :134.507 2024-09-13T15:19:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:05 smithi177 bash[68594]: cluster 2024-09-13T15:19:05.244215+0000 mgr.y (mgr.26252) 1593 : cluster [DBG] pgmap v2213: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 88 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 12851/4293 objects degraded (299.348%); 140/4293 objects misplaced (3.261%); 1.2 MiB/s, 128 objects/s recovering 2024-09-13T15:19:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:05 smithi177 bash[68594]: cluster 2024-09-13T15:19:05.244215+0000 mgr.y (mgr.26252) 1593 : cluster [DBG] pgmap v2213: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 88 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 12851/4293 objects degraded (299.348%); 140/4293 objects misplaced (3.261%); 1.2 MiB/s, 128 objects/s recovering 2024-09-13T15:19:05.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:05.630 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:05 smithi099 bash[84576]: cluster 2024-09-13T15:19:05.244215+0000 mgr.y (mgr.26252) 1593 : cluster [DBG] pgmap v2213: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 88 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 12851/4293 objects degraded (299.348%); 140/4293 objects misplaced (3.261%); 1.2 MiB/s, 128 objects/s recovering 2024-09-13T15:19:05.631 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:05 smithi099 bash[84576]: cluster 2024-09-13T15:19:05.244215+0000 mgr.y (mgr.26252) 1593 : cluster [DBG] pgmap v2213: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 88 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 12851/4293 objects degraded (299.348%); 140/4293 objects misplaced (3.261%); 1.2 MiB/s, 128 objects/s recovering 2024-09-13T15:19:05.631 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:05 smithi099 bash[87147]: cluster 2024-09-13T15:19:05.244215+0000 mgr.y (mgr.26252) 1593 : cluster [DBG] pgmap v2213: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 88 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 12851/4293 objects degraded (299.348%); 140/4293 objects misplaced (3.261%); 1.2 MiB/s, 128 objects/s recovering 2024-09-13T15:19:05.631 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:05 smithi099 bash[87147]: cluster 2024-09-13T15:19:05.244215+0000 mgr.y (mgr.26252) 1593 : cluster [DBG] pgmap v2213: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 88 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 12851/4293 objects degraded (299.348%); 140/4293 objects misplaced (3.261%); 1.2 MiB/s, 128 objects/s recovering 2024-09-13T15:19:05.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:05.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:05.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:05.743 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:05.768 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:05.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:05.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:05.770 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-09-13T15:19:05.770 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-09-13T15:19:05.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:05.891 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:05.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:05.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:05.994 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:06.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:06.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:06.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:06.123 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:06.146 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:06.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:06.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:06.278 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:06.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:06.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:06.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:06.408 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:06.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:06.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:06.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:06.536 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:06.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:06.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:06.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:06.667 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:06.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:06.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:06.770 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:06.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:06.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:06.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:06.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:06 smithi099 bash[87147]: cluster 2024-09-13T15:19:06.681458+0000 mon.a (mon.0) 2773 : cluster [WRN] Health check update: Degraded data redundancy: 12851/4293 objects degraded (299.348%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:06.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:06 smithi099 bash[87147]: cluster 2024-09-13T15:19:06.681458+0000 mon.a (mon.0) 2773 : cluster [WRN] Health check update: Degraded data redundancy: 12851/4293 objects degraded (299.348%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:06.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:06 smithi099 bash[84576]: cluster 2024-09-13T15:19:06.681458+0000 mon.a (mon.0) 2773 : cluster [WRN] Health check update: Degraded data redundancy: 12851/4293 objects degraded (299.348%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:06.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:06 smithi099 bash[84576]: cluster 2024-09-13T15:19:06.681458+0000 mon.a (mon.0) 2773 : cluster [WRN] Health check update: Degraded data redundancy: 12851/4293 objects degraded (299.348%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:06.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:06.908 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:06 smithi177 bash[68594]: cluster 2024-09-13T15:19:06.681458+0000 mon.a (mon.0) 2773 : cluster [WRN] Health check update: Degraded data redundancy: 12851/4293 objects degraded (299.348%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:06.909 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:06 smithi177 bash[68594]: cluster 2024-09-13T15:19:06.681458+0000 mon.a (mon.0) 2773 : cluster [WRN] Health check update: Degraded data redundancy: 12851/4293 objects degraded (299.348%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:06.921 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:06.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:06.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:07.024 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:07.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:07.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:07.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:07.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:07.175 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:07.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:07.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:07.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:07.288 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:19:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:19:07] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-13T15:19:07.303 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:07.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:07.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:07.406 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:07.433 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:07.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:07.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:07.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:07.563 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:07.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:07.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:07.615 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:19:07.615 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "" 2024-09-13T15:19:07.615 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:19:07.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:07.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:07.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:07.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:07.797 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:07.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:07 smithi099 bash[84576]: cluster 2024-09-13T15:19:07.245379+0000 mgr.y (mgr.26252) 1594 : cluster [DBG] pgmap v2214: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 37 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 12851/1842 objects degraded (697.666%); 140/1842 objects misplaced (7.600%); 0 B/s, 74 objects/s recovering 2024-09-13T15:19:07.804 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:07 smithi099 bash[84576]: cluster 2024-09-13T15:19:07.245379+0000 mgr.y (mgr.26252) 1594 : cluster [DBG] pgmap v2214: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 37 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 12851/1842 objects degraded (697.666%); 140/1842 objects misplaced (7.600%); 0 B/s, 74 objects/s recovering 2024-09-13T15:19:07.820 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:07.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:07.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:07.923 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:07.950 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:07.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:07.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:08.053 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:08.064 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:07 smithi099 bash[87147]: cluster 2024-09-13T15:19:07.245379+0000 mgr.y (mgr.26252) 1594 : cluster [DBG] pgmap v2214: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 37 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 12851/1842 objects degraded (697.666%); 140/1842 objects misplaced (7.600%); 0 B/s, 74 objects/s recovering 2024-09-13T15:19:08.064 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:07 smithi099 bash[87147]: cluster 2024-09-13T15:19:07.245379+0000 mgr.y (mgr.26252) 1594 : cluster [DBG] pgmap v2214: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 37 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 12851/1842 objects degraded (697.666%); 140/1842 objects misplaced (7.600%); 0 B/s, 74 objects/s recovering 2024-09-13T15:19:08.079 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:08.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:08.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:08.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:07 smithi177 bash[68594]: cluster 2024-09-13T15:19:07.245379+0000 mgr.y (mgr.26252) 1594 : cluster [DBG] pgmap v2214: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 37 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 12851/1842 objects degraded (697.666%); 140/1842 objects misplaced (7.600%); 0 B/s, 74 objects/s recovering 2024-09-13T15:19:08.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:07 smithi177 bash[68594]: cluster 2024-09-13T15:19:07.245379+0000 mgr.y (mgr.26252) 1594 : cluster [DBG] pgmap v2214: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 37 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 12851/1842 objects degraded (697.666%); 140/1842 objects misplaced (7.600%); 0 B/s, 74 objects/s recovering 2024-09-13T15:19:08.182 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:08.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:08.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:08.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:08.311 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:08.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:08.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:08.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:08.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:08.465 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:08.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:08.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:08.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:08.599 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:08.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:08.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:08.726 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:08.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:08.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:08.829 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:08.855 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:08.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:08.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:08.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:08.983 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:08.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:08.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:09.086 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:09.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:09.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:09.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:09.182 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:19:09.182 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:19:09.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:09.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:09.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:09.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:09.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:09.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:09.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:09.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:09.508 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:09.517 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:09 smithi177 bash[68594]: cluster 2024-09-13T15:19:09.246327+0000 mgr.y (mgr.26252) 1595 : cluster [DBG] pgmap v2215: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 17 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 12851/852 objects degraded (1508.333%); 31/852 objects misplaced (3.638%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:09.517 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:09 smithi177 bash[68594]: cluster 2024-09-13T15:19:09.246327+0000 mgr.y (mgr.26252) 1595 : cluster [DBG] pgmap v2215: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 17 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 12851/852 objects degraded (1508.333%); 31/852 objects misplaced (3.638%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:09.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:09.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:09.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:09.586 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:09 smithi099 bash[87147]: cluster 2024-09-13T15:19:09.246327+0000 mgr.y (mgr.26252) 1595 : cluster [DBG] pgmap v2215: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 17 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 12851/852 objects degraded (1508.333%); 31/852 objects misplaced (3.638%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:09.586 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:09 smithi099 bash[87147]: cluster 2024-09-13T15:19:09.246327+0000 mgr.y (mgr.26252) 1595 : cluster [DBG] pgmap v2215: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 17 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 12851/852 objects degraded (1508.333%); 31/852 objects misplaced (3.638%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:09.608 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:09 smithi099 bash[84576]: cluster 2024-09-13T15:19:09.246327+0000 mgr.y (mgr.26252) 1595 : cluster [DBG] pgmap v2215: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 17 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 12851/852 objects degraded (1508.333%); 31/852 objects misplaced (3.638%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:09.608 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:09 smithi099 bash[84576]: cluster 2024-09-13T15:19:09.246327+0000 mgr.y (mgr.26252) 1595 : cluster [DBG] pgmap v2215: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 17 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 12851/852 objects degraded (1508.333%); 31/852 objects misplaced (3.638%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:09.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:09.660 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:09.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:09.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:09.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:09.789 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:09.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:09.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:09.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:09.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:09.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:09.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:10.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:10.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:10.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:10.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:10.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:10.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:10.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:10.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:10.279 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:10.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:10.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:10.362 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is set 2024-09-13T15:19:10.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:10.414 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:10 smithi177 bash[68594]: audit 2024-09-13T15:19:10.221845+0000 mon.a (mon.0) 2774 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:19:10.414 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:10 smithi177 bash[68594]: audit 2024-09-13T15:19:10.221845+0000 mon.a (mon.0) 2774 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:19:10.414 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:19:10 smithi177 bash[61439]: ts=2024-09-13T15:19:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:19:10.429 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:10.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:10.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:10.533 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:10.552 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:10.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:10.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:10.655 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:10.680 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:10.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:10.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:10.726 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:19:10.726 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:19:10.743 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:19:10.744 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:19:10.761 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:19:10.761 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:19:10.778 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:19:10.779 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:19:10.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:10.800 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:19:10.800 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:19:10.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:10.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:10.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:10.823 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:19:10.823 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:19:10.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:10 smithi099 bash[87147]: audit 2024-09-13T15:19:10.221845+0000 mon.a (mon.0) 2774 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:19:10.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:10 smithi099 bash[87147]: audit 2024-09-13T15:19:10.221845+0000 mon.a (mon.0) 2774 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:19:10.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:10 smithi099 bash[84576]: audit 2024-09-13T15:19:10.221845+0000 mon.a (mon.0) 2774 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:19:10.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:10 smithi099 bash[84576]: audit 2024-09-13T15:19:10.221845+0000 mon.a (mon.0) 2774 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:19:10.840 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:19:10.840 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:19:10.859 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:19:10.860 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:19:10.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:10.941 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:10.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:10.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:11.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:11.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:11.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:11.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:11.174 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:11.197 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:11.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:11.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:11.300 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:11.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:11.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:11.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:11.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:11.458 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:11.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:11.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:11.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:11.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:11.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:11.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:11 smithi177 bash[68594]: cluster 2024-09-13T15:19:10.351171+0000 mon.a (mon.0) 2775 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:19:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:11 smithi177 bash[68594]: cluster 2024-09-13T15:19:10.351171+0000 mon.a (mon.0) 2775 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:19:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:11 smithi177 bash[68594]: audit 2024-09-13T15:19:10.362007+0000 mon.a (mon.0) 2776 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:19:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:11 smithi177 bash[68594]: audit 2024-09-13T15:19:10.362007+0000 mon.a (mon.0) 2776 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:19:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:11 smithi177 bash[68594]: cluster 2024-09-13T15:19:10.371146+0000 mon.a (mon.0) 2777 : cluster [DBG] osdmap e1137: 8 total, 8 up, 7 in 2024-09-13T15:19:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:11 smithi177 bash[68594]: cluster 2024-09-13T15:19:10.371146+0000 mon.a (mon.0) 2777 : cluster [DBG] osdmap e1137: 8 total, 8 up, 7 in 2024-09-13T15:19:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:11 smithi177 bash[68594]: audit 2024-09-13T15:19:10.707226+0000 mon.a (mon.0) 2778 : audit [DBG] from='client.? 172.21.15.99:0/362739941' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:19:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:11 smithi177 bash[68594]: audit 2024-09-13T15:19:10.707226+0000 mon.a (mon.0) 2778 : audit [DBG] from='client.? 172.21.15.99:0/362739941' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:19:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:11 smithi177 bash[68594]: cluster 2024-09-13T15:19:11.246915+0000 mgr.y (mgr.26252) 1596 : cluster [DBG] pgmap v2217: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 141 objects/s recovering 2024-09-13T15:19:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:11 smithi177 bash[68594]: cluster 2024-09-13T15:19:11.246915+0000 mgr.y (mgr.26252) 1596 : cluster [DBG] pgmap v2217: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 141 objects/s recovering 2024-09-13T15:19:11.688 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:11.714 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:11.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:11.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:11.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[87147]: cluster 2024-09-13T15:19:10.351171+0000 mon.a (mon.0) 2775 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:19:11.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[87147]: cluster 2024-09-13T15:19:10.351171+0000 mon.a (mon.0) 2775 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:19:11.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[87147]: audit 2024-09-13T15:19:10.362007+0000 mon.a (mon.0) 2776 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:19:11.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[87147]: audit 2024-09-13T15:19:10.362007+0000 mon.a (mon.0) 2776 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:19:11.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[87147]: cluster 2024-09-13T15:19:10.371146+0000 mon.a (mon.0) 2777 : cluster [DBG] osdmap e1137: 8 total, 8 up, 7 in 2024-09-13T15:19:11.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[87147]: cluster 2024-09-13T15:19:10.371146+0000 mon.a (mon.0) 2777 : cluster [DBG] osdmap e1137: 8 total, 8 up, 7 in 2024-09-13T15:19:11.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[87147]: audit 2024-09-13T15:19:10.707226+0000 mon.a (mon.0) 2778 : audit [DBG] from='client.? 172.21.15.99:0/362739941' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:19:11.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[87147]: audit 2024-09-13T15:19:10.707226+0000 mon.a (mon.0) 2778 : audit [DBG] from='client.? 172.21.15.99:0/362739941' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:19:11.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[87147]: cluster 2024-09-13T15:19:11.246915+0000 mgr.y (mgr.26252) 1596 : cluster [DBG] pgmap v2217: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 141 objects/s recovering 2024-09-13T15:19:11.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[87147]: cluster 2024-09-13T15:19:11.246915+0000 mgr.y (mgr.26252) 1596 : cluster [DBG] pgmap v2217: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 141 objects/s recovering 2024-09-13T15:19:11.827 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[84576]: cluster 2024-09-13T15:19:10.351171+0000 mon.a (mon.0) 2775 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:19:11.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[84576]: cluster 2024-09-13T15:19:10.351171+0000 mon.a (mon.0) 2775 : cluster [WRN] Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:19:11.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[84576]: audit 2024-09-13T15:19:10.362007+0000 mon.a (mon.0) 2776 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:19:11.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[84576]: audit 2024-09-13T15:19:10.362007+0000 mon.a (mon.0) 2776 : audit [INF] from='client.? 172.21.15.99:0/4146111071' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:19:11.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[84576]: cluster 2024-09-13T15:19:10.371146+0000 mon.a (mon.0) 2777 : cluster [DBG] osdmap e1137: 8 total, 8 up, 7 in 2024-09-13T15:19:11.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[84576]: cluster 2024-09-13T15:19:10.371146+0000 mon.a (mon.0) 2777 : cluster [DBG] osdmap e1137: 8 total, 8 up, 7 in 2024-09-13T15:19:11.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[84576]: audit 2024-09-13T15:19:10.707226+0000 mon.a (mon.0) 2778 : audit [DBG] from='client.? 172.21.15.99:0/362739941' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:19:11.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[84576]: audit 2024-09-13T15:19:10.707226+0000 mon.a (mon.0) 2778 : audit [DBG] from='client.? 172.21.15.99:0/362739941' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:19:11.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[84576]: cluster 2024-09-13T15:19:11.246915+0000 mgr.y (mgr.26252) 1596 : cluster [DBG] pgmap v2217: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 141 objects/s recovering 2024-09-13T15:19:11.829 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:11 smithi099 bash[84576]: cluster 2024-09-13T15:19:11.246915+0000 mgr.y (mgr.26252) 1596 : cluster [DBG] pgmap v2217: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 429 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 141 objects/s recovering 2024-09-13T15:19:11.842 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:11.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:11.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:11.947 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:11.969 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:11.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:12.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:12.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:12.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:12.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:12.230 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:12.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:12.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:12.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:12.361 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:12.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:12.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:12.464 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:12.488 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:12.490 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:12 smithi099 bash[84576]: cluster 2024-09-13T15:19:11.375771+0000 mon.a (mon.0) 2779 : cluster [DBG] osdmap e1138: 8 total, 8 up, 7 in 2024-09-13T15:19:12.490 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:12 smithi099 bash[84576]: cluster 2024-09-13T15:19:11.375771+0000 mon.a (mon.0) 2779 : cluster [DBG] osdmap e1138: 8 total, 8 up, 7 in 2024-09-13T15:19:12.490 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:12 smithi099 bash[84576]: cluster 2024-09-13T15:19:11.682549+0000 mon.a (mon.0) 2780 : cluster [WRN] Health check update: Degraded data redundancy: 12851/120 objects degraded (10709.167%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:12.490 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:12 smithi099 bash[84576]: cluster 2024-09-13T15:19:11.682549+0000 mon.a (mon.0) 2780 : cluster [WRN] Health check update: Degraded data redundancy: 12851/120 objects degraded (10709.167%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:12.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:12 smithi099 bash[87147]: cluster 2024-09-13T15:19:11.375771+0000 mon.a (mon.0) 2779 : cluster [DBG] osdmap e1138: 8 total, 8 up, 7 in 2024-09-13T15:19:12.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:12 smithi099 bash[87147]: cluster 2024-09-13T15:19:11.375771+0000 mon.a (mon.0) 2779 : cluster [DBG] osdmap e1138: 8 total, 8 up, 7 in 2024-09-13T15:19:12.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:12 smithi099 bash[87147]: cluster 2024-09-13T15:19:11.682549+0000 mon.a (mon.0) 2780 : cluster [WRN] Health check update: Degraded data redundancy: 12851/120 objects degraded (10709.167%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:12.490 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:12 smithi099 bash[87147]: cluster 2024-09-13T15:19:11.682549+0000 mon.a (mon.0) 2780 : cluster [WRN] Health check update: Degraded data redundancy: 12851/120 objects degraded (10709.167%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:12.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:12.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:12.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:12.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:12.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:12.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:12.720 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:12.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:12.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:12.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:12.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:12.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:12 smithi177 bash[68594]: cluster 2024-09-13T15:19:11.375771+0000 mon.a (mon.0) 2779 : cluster [DBG] osdmap e1138: 8 total, 8 up, 7 in 2024-09-13T15:19:12.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:12 smithi177 bash[68594]: cluster 2024-09-13T15:19:11.375771+0000 mon.a (mon.0) 2779 : cluster [DBG] osdmap e1138: 8 total, 8 up, 7 in 2024-09-13T15:19:12.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:12 smithi177 bash[68594]: cluster 2024-09-13T15:19:11.682549+0000 mon.a (mon.0) 2780 : cluster [WRN] Health check update: Degraded data redundancy: 12851/120 objects degraded (10709.167%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:12.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:12 smithi177 bash[68594]: cluster 2024-09-13T15:19:11.682549+0000 mon.a (mon.0) 2780 : cluster [WRN] Health check update: Degraded data redundancy: 12851/120 objects degraded (10709.167%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:12.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:12.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:12.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:12.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:13.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:13.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:13.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:13.103 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:13.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:13.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:13.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:13.233 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:13.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:13.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:13.362 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:13.388 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:13.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:13.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:13.491 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:13.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:13 smithi177 bash[68594]: cluster 2024-09-13T15:19:12.373598+0000 mon.a (mon.0) 2781 : cluster [DBG] osdmap e1139: 8 total, 8 up, 7 in 2024-09-13T15:19:13.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:13 smithi177 bash[68594]: cluster 2024-09-13T15:19:12.373598+0000 mon.a (mon.0) 2781 : cluster [DBG] osdmap e1139: 8 total, 8 up, 7 in 2024-09-13T15:19:13.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:13 smithi177 bash[68594]: cluster 2024-09-13T15:19:13.247682+0000 mgr.y (mgr.26252) 1597 : cluster [DBG] pgmap v2220: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 100 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:13.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:13 smithi177 bash[68594]: cluster 2024-09-13T15:19:13.247682+0000 mgr.y (mgr.26252) 1597 : cluster [DBG] pgmap v2220: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 100 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:13.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:13.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:13.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:13.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:13.645 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:13.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:13.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:13.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:13.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:13.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:13.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:13.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:13 smithi099 bash[87147]: cluster 2024-09-13T15:19:12.373598+0000 mon.a (mon.0) 2781 : cluster [DBG] osdmap e1139: 8 total, 8 up, 7 in 2024-09-13T15:19:13.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:13 smithi099 bash[87147]: cluster 2024-09-13T15:19:12.373598+0000 mon.a (mon.0) 2781 : cluster [DBG] osdmap e1139: 8 total, 8 up, 7 in 2024-09-13T15:19:13.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:13 smithi099 bash[87147]: cluster 2024-09-13T15:19:13.247682+0000 mgr.y (mgr.26252) 1597 : cluster [DBG] pgmap v2220: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 100 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:13.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:13 smithi099 bash[87147]: cluster 2024-09-13T15:19:13.247682+0000 mgr.y (mgr.26252) 1597 : cluster [DBG] pgmap v2220: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 100 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:13 smithi099 bash[84576]: cluster 2024-09-13T15:19:12.373598+0000 mon.a (mon.0) 2781 : cluster [DBG] osdmap e1139: 8 total, 8 up, 7 in 2024-09-13T15:19:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:13 smithi099 bash[84576]: cluster 2024-09-13T15:19:12.373598+0000 mon.a (mon.0) 2781 : cluster [DBG] osdmap e1139: 8 total, 8 up, 7 in 2024-09-13T15:19:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:13 smithi099 bash[84576]: cluster 2024-09-13T15:19:13.247682+0000 mgr.y (mgr.26252) 1597 : cluster [DBG] pgmap v2220: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 100 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:13 smithi099 bash[84576]: cluster 2024-09-13T15:19:13.247682+0000 mgr.y (mgr.26252) 1597 : cluster [DBG] pgmap v2220: 109 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 101 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 100 op/s; 12851/120 objects degraded (10709.167%); 0 B/s, 149 objects/s recovering 2024-09-13T15:19:13.877 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:13.900 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:13.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:14.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:14.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:14.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:14.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:14.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:14.150 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:14.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:14.210 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 4, 1, 6] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:19:14.210 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:19:14.211 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-09-13T15:19:14.211 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:19:14.211 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:19:14.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-09-13T15:19:14.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:14.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:14.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:14.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:14.408 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:14.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:14.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:14.510 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:14.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:14.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:14.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:14.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:14.660 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:14.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:14.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:14.764 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:14.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:14.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:14.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T15:19:14.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:14.940 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:14.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:14.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:15.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:15.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:15.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:15.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:15.171 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:15.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:15.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:15.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:15.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:15.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:15.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:15.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:15.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:15.442 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:15 smithi177 bash[68594]: cluster 2024-09-13T15:19:15.248798+0000 mgr.y (mgr.26252) 1598 : cluster [DBG] pgmap v2221: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 40 op/s; 10221/120 objects degraded (8517.500%); 0 B/s, 189 objects/s recovering 2024-09-13T15:19:15.442 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:15 smithi177 bash[68594]: cluster 2024-09-13T15:19:15.248798+0000 mgr.y (mgr.26252) 1598 : cluster [DBG] pgmap v2221: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 40 op/s; 10221/120 objects degraded (8517.500%); 0 B/s, 189 objects/s recovering 2024-09-13T15:19:15.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:15.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:15.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:15.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:15.584 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:15.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:15.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:15.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:15.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:15.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:15.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:15.842 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:15.847 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:15 smithi099 bash[84576]: cluster 2024-09-13T15:19:15.248798+0000 mgr.y (mgr.26252) 1598 : cluster [DBG] pgmap v2221: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 40 op/s; 10221/120 objects degraded (8517.500%); 0 B/s, 189 objects/s recovering 2024-09-13T15:19:15.847 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:15 smithi099 bash[84576]: cluster 2024-09-13T15:19:15.248798+0000 mgr.y (mgr.26252) 1598 : cluster [DBG] pgmap v2221: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 40 op/s; 10221/120 objects degraded (8517.500%); 0 B/s, 189 objects/s recovering 2024-09-13T15:19:15.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:15 smithi099 bash[87147]: cluster 2024-09-13T15:19:15.248798+0000 mgr.y (mgr.26252) 1598 : cluster [DBG] pgmap v2221: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 40 op/s; 10221/120 objects degraded (8517.500%); 0 B/s, 189 objects/s recovering 2024-09-13T15:19:15.847 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:15 smithi099 bash[87147]: cluster 2024-09-13T15:19:15.248798+0000 mgr.y (mgr.26252) 1598 : cluster [DBG] pgmap v2221: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 40 op/s; 10221/120 objects degraded (8517.500%); 0 B/s, 189 objects/s recovering 2024-09-13T15:19:15.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:15.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:15.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:15.973 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:15.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:15.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:16.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:16.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:16.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:16.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:16.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:16.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:16.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:16.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:16.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:16.354 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:16.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:16.457 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:16.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:16.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:16.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:16.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:16.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:16.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:16.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:16.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:16.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:16.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:16.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:16.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:16.872 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:16.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:16.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:16.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:16.984 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:16 smithi099 bash[84576]: cluster 2024-09-13T15:19:16.683223+0000 mon.a (mon.0) 2782 : cluster [WRN] Health check update: Degraded data redundancy: 10221/120 objects degraded (8517.500%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:16.984 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:16 smithi099 bash[84576]: cluster 2024-09-13T15:19:16.683223+0000 mon.a (mon.0) 2782 : cluster [WRN] Health check update: Degraded data redundancy: 10221/120 objects degraded (8517.500%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:16.984 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:16 smithi099 bash[87147]: cluster 2024-09-13T15:19:16.683223+0000 mon.a (mon.0) 2782 : cluster [WRN] Health check update: Degraded data redundancy: 10221/120 objects degraded (8517.500%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:16.984 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:16 smithi099 bash[87147]: cluster 2024-09-13T15:19:16.683223+0000 mon.a (mon.0) 2782 : cluster [WRN] Health check update: Degraded data redundancy: 10221/120 objects degraded (8517.500%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:16.997 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:16.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:16.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:17.100 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:16 smithi177 bash[68594]: cluster 2024-09-13T15:19:16.683223+0000 mon.a (mon.0) 2782 : cluster [WRN] Health check update: Degraded data redundancy: 10221/120 objects degraded (8517.500%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:17.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:16 smithi177 bash[68594]: cluster 2024-09-13T15:19:16.683223+0000 mon.a (mon.0) 2782 : cluster [WRN] Health check update: Degraded data redundancy: 10221/120 objects degraded (8517.500%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:17.120 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:17.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:17.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:17.244 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:17.245 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:19:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:19:17] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-13T15:19:17.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:17.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:17.347 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:17.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:17.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:17.476 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:17.503 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:17.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:17.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:17.606 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:17.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:17.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:17.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:17.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:17.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:17.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:17.864 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:17.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:17 smithi177 bash[68594]: cluster 2024-09-13T15:19:17.249442+0000 mgr.y (mgr.26252) 1599 : cluster [DBG] pgmap v2222: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 7572/120 objects degraded (6310.000%); 0 B/s, 258 objects/s recovering 2024-09-13T15:19:17.875 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:17 smithi177 bash[68594]: cluster 2024-09-13T15:19:17.249442+0000 mgr.y (mgr.26252) 1599 : cluster [DBG] pgmap v2222: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 7572/120 objects degraded (6310.000%); 0 B/s, 258 objects/s recovering 2024-09-13T15:19:17.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:17.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:17.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:17.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:18.004 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:17 smithi099 bash[84576]: cluster 2024-09-13T15:19:17.249442+0000 mgr.y (mgr.26252) 1599 : cluster [DBG] pgmap v2222: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 7572/120 objects degraded (6310.000%); 0 B/s, 258 objects/s recovering 2024-09-13T15:19:18.004 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:17 smithi099 bash[84576]: cluster 2024-09-13T15:19:17.249442+0000 mgr.y (mgr.26252) 1599 : cluster [DBG] pgmap v2222: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 7572/120 objects degraded (6310.000%); 0 B/s, 258 objects/s recovering 2024-09-13T15:19:18.005 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:17 smithi099 bash[87147]: cluster 2024-09-13T15:19:17.249442+0000 mgr.y (mgr.26252) 1599 : cluster [DBG] pgmap v2222: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 7572/120 objects degraded (6310.000%); 0 B/s, 258 objects/s recovering 2024-09-13T15:19:18.005 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:17 smithi099 bash[87147]: cluster 2024-09-13T15:19:17.249442+0000 mgr.y (mgr.26252) 1599 : cluster [DBG] pgmap v2222: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 7572/120 objects degraded (6310.000%); 0 B/s, 258 objects/s recovering 2024-09-13T15:19:18.019 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:18.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:18.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:18.122 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:18.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:18.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:18.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:18.277 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:18.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:18.380 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:18.404 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:18.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:18.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:18.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:18.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:18.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:18.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:18.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:18.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:18.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:18.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:18.760 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:18.771 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:18 smithi099 bash[87147]: audit 2024-09-13T15:19:18.323193+0000 mon.a (mon.0) 2783 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:18.771 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:18 smithi099 bash[87147]: audit 2024-09-13T15:19:18.323193+0000 mon.a (mon.0) 2783 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:18.771 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:18 smithi099 bash[84576]: audit 2024-09-13T15:19:18.323193+0000 mon.a (mon.0) 2783 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:18.771 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:18 smithi099 bash[84576]: audit 2024-09-13T15:19:18.323193+0000 mon.a (mon.0) 2783 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:18.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:18.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:18.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:18.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:18.912 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:18.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:18.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:19.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:19.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:19.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:19.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:19.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:18 smithi177 bash[68594]: audit 2024-09-13T15:19:18.323193+0000 mon.a (mon.0) 2783 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:19.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:18 smithi177 bash[68594]: audit 2024-09-13T15:19:18.323193+0000 mon.a (mon.0) 2783 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:19.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:19.171 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:19.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:19.274 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:19.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:19.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:19.403 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:19.427 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:19.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:19.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:19.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:19.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:19.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:19.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:19.685 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:19.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:19.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:19.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:19.799 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:19 smithi177 bash[68594]: cluster 2024-09-13T15:19:19.250454+0000 mgr.y (mgr.26252) 1600 : cluster [DBG] pgmap v2223: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 221 objects/s recovering 2024-09-13T15:19:19.799 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:19 smithi177 bash[68594]: cluster 2024-09-13T15:19:19.250454+0000 mgr.y (mgr.26252) 1600 : cluster [DBG] pgmap v2223: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 221 objects/s recovering 2024-09-13T15:19:19.813 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:19.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:19.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:19.916 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:19.947 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:19.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:19.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:20.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:20.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:19 smithi099 bash[84576]: cluster 2024-09-13T15:19:19.250454+0000 mgr.y (mgr.26252) 1600 : cluster [DBG] pgmap v2223: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 221 objects/s recovering 2024-09-13T15:19:20.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:19 smithi099 bash[84576]: cluster 2024-09-13T15:19:19.250454+0000 mgr.y (mgr.26252) 1600 : cluster [DBG] pgmap v2223: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 221 objects/s recovering 2024-09-13T15:19:20.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:19 smithi099 bash[87147]: cluster 2024-09-13T15:19:19.250454+0000 mgr.y (mgr.26252) 1600 : cluster [DBG] pgmap v2223: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 221 objects/s recovering 2024-09-13T15:19:20.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:19 smithi099 bash[87147]: cluster 2024-09-13T15:19:19.250454+0000 mgr.y (mgr.26252) 1600 : cluster [DBG] pgmap v2223: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 221 objects/s recovering 2024-09-13T15:19:20.079 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:20.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:20.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:20.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:20.211 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:20.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:20.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:20.315 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:20.341 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:20.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:20.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:20.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:19:20 smithi177 bash[61439]: ts=2024-09-13T15:19:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:19:20.443 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:20.469 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:20.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:20.572 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:20.593 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:20.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:20.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:20.696 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:20.725 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:20.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:20.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:20.828 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:20.854 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:20.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:20.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:20.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:20.982 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:20.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:20.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:21.085 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:21.111 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:21.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:21.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:21.242 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:21.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:21.345 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:21.352 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:21 smithi099 bash[84576]: cluster 2024-09-13T15:19:21.251642+0000 mgr.y (mgr.26252) 1601 : cluster [DBG] pgmap v2224: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 351 objects/s recovering 2024-09-13T15:19:21.353 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:21 smithi099 bash[84576]: cluster 2024-09-13T15:19:21.251642+0000 mgr.y (mgr.26252) 1601 : cluster [DBG] pgmap v2224: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 351 objects/s recovering 2024-09-13T15:19:21.353 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:21 smithi099 bash[87147]: cluster 2024-09-13T15:19:21.251642+0000 mgr.y (mgr.26252) 1601 : cluster [DBG] pgmap v2224: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 351 objects/s recovering 2024-09-13T15:19:21.375 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:21.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:21.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:21.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:21.506 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:21.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:21.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:21.610 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:21.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:21 smithi177 bash[68594]: cluster 2024-09-13T15:19:21.251642+0000 mgr.y (mgr.26252) 1601 : cluster [DBG] pgmap v2224: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 351 objects/s recovering 2024-09-13T15:19:21.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:21 smithi177 bash[68594]: cluster 2024-09-13T15:19:21.251642+0000 mgr.y (mgr.26252) 1601 : cluster [DBG] pgmap v2224: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 351 objects/s recovering 2024-09-13T15:19:21.637 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:21.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:21.740 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:21.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:21.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:21.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:21.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:21 smithi099 bash[87147]: cluster 2024-09-13T15:19:21.251642+0000 mgr.y (mgr.26252) 1601 : cluster [DBG] pgmap v2224: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 351 objects/s recovering 2024-09-13T15:19:21.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:21.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:21.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:21.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:21.998 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:22.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:22.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:22.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:22.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:22.150 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:22.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:22.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:22.253 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:22.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:22.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:22.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:22.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:22.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:22.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:22.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:22.515 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:22.527 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:22 smithi099 bash[87147]: cluster 2024-09-13T15:19:21.684079+0000 mon.a (mon.0) 2784 : cluster [WRN] Health check update: Degraded data redundancy: 7572/120 objects degraded (6310.000%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:22.527 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:22 smithi099 bash[87147]: cluster 2024-09-13T15:19:21.684079+0000 mon.a (mon.0) 2784 : cluster [WRN] Health check update: Degraded data redundancy: 7572/120 objects degraded (6310.000%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:22.527 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:22 smithi099 bash[84576]: cluster 2024-09-13T15:19:21.684079+0000 mon.a (mon.0) 2784 : cluster [WRN] Health check update: Degraded data redundancy: 7572/120 objects degraded (6310.000%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:22.527 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:22 smithi099 bash[84576]: cluster 2024-09-13T15:19:21.684079+0000 mon.a (mon.0) 2784 : cluster [WRN] Health check update: Degraded data redundancy: 7572/120 objects degraded (6310.000%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:22.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:22.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:22.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:22 smithi177 bash[68594]: cluster 2024-09-13T15:19:21.684079+0000 mon.a (mon.0) 2784 : cluster [WRN] Health check update: Degraded data redundancy: 7572/120 objects degraded (6310.000%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:22.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:22 smithi177 bash[68594]: cluster 2024-09-13T15:19:21.684079+0000 mon.a (mon.0) 2784 : cluster [WRN] Health check update: Degraded data redundancy: 7572/120 objects degraded (6310.000%), 4 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:22.646 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:22.670 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:22.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:22.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:22.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:22.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:22.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:22.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:22.922 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:22.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:22.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:23.025 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:23.051 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:23.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:23.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:23.155 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:23.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:23.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:23.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:23.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:23.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:23.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:23.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:23.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:23.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:23.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:23.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:23.542 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:23.571 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:23.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:23.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:23.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:23 smithi099 bash[84576]: cluster 2024-09-13T15:19:23.252344+0000 mgr.y (mgr.26252) 1602 : cluster [DBG] pgmap v2225: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:23.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:23 smithi099 bash[84576]: cluster 2024-09-13T15:19:23.252344+0000 mgr.y (mgr.26252) 1602 : cluster [DBG] pgmap v2225: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:23.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:23 smithi099 bash[87147]: cluster 2024-09-13T15:19:23.252344+0000 mgr.y (mgr.26252) 1602 : cluster [DBG] pgmap v2225: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:23.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:23 smithi099 bash[87147]: cluster 2024-09-13T15:19:23.252344+0000 mgr.y (mgr.26252) 1602 : cluster [DBG] pgmap v2225: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:23.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:23.686 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:23 smithi177 bash[68594]: cluster 2024-09-13T15:19:23.252344+0000 mgr.y (mgr.26252) 1602 : cluster [DBG] pgmap v2225: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:23.686 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:23 smithi177 bash[68594]: cluster 2024-09-13T15:19:23.252344+0000 mgr.y (mgr.26252) 1602 : cluster [DBG] pgmap v2225: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:23.702 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:23.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:23.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:23.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:23.828 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:23.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:23.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:23.931 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:23.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:23.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:23.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:24.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:24.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:24.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:24.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:24.187 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:24.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:24.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:24.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:24.218 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:19:24.315 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:24.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:24.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:24.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:24.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:24.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:24.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:24.566 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:24.592 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:24.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:24.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:24.696 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:24.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:24.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:24.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:24.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:24.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:24.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:24.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:24.948 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:24.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:24.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:24.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:25.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:25.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:25.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:25.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:25.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:25.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:25.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:25.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:25.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:25.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:25.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:25.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:25.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:25.484 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:25.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:25.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:25.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:25.599 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:25 smithi177 bash[68594]: cluster 2024-09-13T15:19:25.253607+0000 mgr.y (mgr.26252) 1603 : cluster [DBG] pgmap v2226: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 347 objects/s recovering 2024-09-13T15:19:25.599 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:25 smithi177 bash[68594]: cluster 2024-09-13T15:19:25.253607+0000 mgr.y (mgr.26252) 1603 : cluster [DBG] pgmap v2226: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 347 objects/s recovering 2024-09-13T15:19:25.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:25.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:25.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:25.715 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:19:25.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:25.724 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:25 smithi099 bash[84576]: cluster 2024-09-13T15:19:25.253607+0000 mgr.y (mgr.26252) 1603 : cluster [DBG] pgmap v2226: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 347 objects/s recovering 2024-09-13T15:19:25.724 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:25 smithi099 bash[84576]: cluster 2024-09-13T15:19:25.253607+0000 mgr.y (mgr.26252) 1603 : cluster [DBG] pgmap v2226: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 347 objects/s recovering 2024-09-13T15:19:25.724 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:25 smithi099 bash[87147]: cluster 2024-09-13T15:19:25.253607+0000 mgr.y (mgr.26252) 1603 : cluster [DBG] pgmap v2226: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 347 objects/s recovering 2024-09-13T15:19:25.724 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:25 smithi099 bash[87147]: cluster 2024-09-13T15:19:25.253607+0000 mgr.y (mgr.26252) 1603 : cluster [DBG] pgmap v2226: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 347 objects/s recovering 2024-09-13T15:19:25.746 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:25.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:25.758 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:19:25.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:25.876 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:25.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:25.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:25.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:26.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:26.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:26.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:26.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:26.137 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:26.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:26.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:26.240 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:26.261 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:26.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:26.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:26.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:26.393 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:26.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:26.497 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:26.521 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:26.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:26.624 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:26.651 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:26.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:26.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:26.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:26.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:26.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:26.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:26.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:26.904 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:26.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:26.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:27.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:27.034 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:27.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:27.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:27.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:27.160 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:27.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:27.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:27.290 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:27.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:27.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:27.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:19:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:19:27] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-13T15:19:27.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:27.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:27.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:27.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:27.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:27.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:27 smithi177 bash[68594]: cluster 2024-09-13T15:19:27.254546+0000 mgr.y (mgr.26252) 1604 : cluster [DBG] pgmap v2227: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 333 objects/s recovering 2024-09-13T15:19:27.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:27 smithi177 bash[68594]: cluster 2024-09-13T15:19:27.254546+0000 mgr.y (mgr.26252) 1604 : cluster [DBG] pgmap v2227: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 333 objects/s recovering 2024-09-13T15:19:27.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:27.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:27.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:27 smithi099 bash[84576]: cluster 2024-09-13T15:19:27.254546+0000 mgr.y (mgr.26252) 1604 : cluster [DBG] pgmap v2227: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 333 objects/s recovering 2024-09-13T15:19:27.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:27 smithi099 bash[84576]: cluster 2024-09-13T15:19:27.254546+0000 mgr.y (mgr.26252) 1604 : cluster [DBG] pgmap v2227: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 333 objects/s recovering 2024-09-13T15:19:27.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:27 smithi099 bash[87147]: cluster 2024-09-13T15:19:27.254546+0000 mgr.y (mgr.26252) 1604 : cluster [DBG] pgmap v2227: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 333 objects/s recovering 2024-09-13T15:19:27.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:27 smithi099 bash[87147]: cluster 2024-09-13T15:19:27.254546+0000 mgr.y (mgr.26252) 1604 : cluster [DBG] pgmap v2227: 109 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 333 objects/s recovering 2024-09-13T15:19:27.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:27.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:27.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:27.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:27.779 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:27.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:27.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:27.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:27.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:27.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:27.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:28.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:28.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:28.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:28.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:28.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:28.193 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:28.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:28.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:28.250 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:19:28.253 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:19:28.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:28.325 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:28.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:28.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:28.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:28.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:28.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:28.556 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:28.577 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:28.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:28.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:28.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:28.704 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:28.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:28.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:28.807 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:28.834 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:28.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:28.938 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:28.962 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:28.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:28.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:29.065 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:29.089 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:29.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:29.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:29.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:29.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:29.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:29.324 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:29.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:29.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:29.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:29.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:29.467 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:29 smithi099 bash[84576]: cluster 2024-09-13T15:19:28.358660+0000 mon.a (mon.0) 2785 : cluster [DBG] osdmap e1140: 8 total, 8 up, 7 in 2024-09-13T15:19:29.467 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:29 smithi099 bash[84576]: cluster 2024-09-13T15:19:28.358660+0000 mon.a (mon.0) 2785 : cluster [DBG] osdmap e1140: 8 total, 8 up, 7 in 2024-09-13T15:19:29.467 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:29 smithi099 bash[84576]: cluster 2024-09-13T15:19:29.255203+0000 mgr.y (mgr.26252) 1605 : cluster [DBG] pgmap v2229: 109 pgs: 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 336 objects/s recovering 2024-09-13T15:19:29.468 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:29 smithi099 bash[84576]: cluster 2024-09-13T15:19:29.255203+0000 mgr.y (mgr.26252) 1605 : cluster [DBG] pgmap v2229: 109 pgs: 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 336 objects/s recovering 2024-09-13T15:19:29.468 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:29 smithi099 bash[87147]: cluster 2024-09-13T15:19:28.358660+0000 mon.a (mon.0) 2785 : cluster [DBG] osdmap e1140: 8 total, 8 up, 7 in 2024-09-13T15:19:29.468 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:29 smithi099 bash[87147]: cluster 2024-09-13T15:19:28.358660+0000 mon.a (mon.0) 2785 : cluster [DBG] osdmap e1140: 8 total, 8 up, 7 in 2024-09-13T15:19:29.468 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:29 smithi099 bash[87147]: cluster 2024-09-13T15:19:29.255203+0000 mgr.y (mgr.26252) 1605 : cluster [DBG] pgmap v2229: 109 pgs: 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 336 objects/s recovering 2024-09-13T15:19:29.468 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:29 smithi099 bash[87147]: cluster 2024-09-13T15:19:29.255203+0000 mgr.y (mgr.26252) 1605 : cluster [DBG] pgmap v2229: 109 pgs: 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 336 objects/s recovering 2024-09-13T15:19:29.482 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:29.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:29.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:29.585 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:29.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:29.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:29.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:29.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:29 smithi177 bash[68594]: cluster 2024-09-13T15:19:28.358660+0000 mon.a (mon.0) 2785 : cluster [DBG] osdmap e1140: 8 total, 8 up, 7 in 2024-09-13T15:19:29.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:29 smithi177 bash[68594]: cluster 2024-09-13T15:19:28.358660+0000 mon.a (mon.0) 2785 : cluster [DBG] osdmap e1140: 8 total, 8 up, 7 in 2024-09-13T15:19:29.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:29 smithi177 bash[68594]: cluster 2024-09-13T15:19:29.255203+0000 mgr.y (mgr.26252) 1605 : cluster [DBG] pgmap v2229: 109 pgs: 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 336 objects/s recovering 2024-09-13T15:19:29.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:29 smithi177 bash[68594]: cluster 2024-09-13T15:19:29.255203+0000 mgr.y (mgr.26252) 1605 : cluster [DBG] pgmap v2229: 109 pgs: 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 7572/120 objects degraded (6310.000%); 0 B/s, 336 objects/s recovering 2024-09-13T15:19:29.713 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:29.738 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:29.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:29.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:29.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:29.864 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:29.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:29.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:29.967 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:29.994 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:29.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:29.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:30.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:30.124 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:30.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:30.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:30.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:30.254 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:30.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:30.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:30.274 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:19:30.274 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:19:30.357 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:30.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:19:30 smithi177 bash[61439]: ts=2024-09-13T15:19:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:19:30.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:30.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:30.491 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:30.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:30 smithi177 bash[68594]: cluster 2024-09-13T15:19:29.371085+0000 mon.a (mon.0) 2786 : cluster [DBG] osdmap e1141: 8 total, 8 up, 7 in 2024-09-13T15:19:30.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:30 smithi177 bash[68594]: cluster 2024-09-13T15:19:29.371085+0000 mon.a (mon.0) 2786 : cluster [DBG] osdmap e1141: 8 total, 8 up, 7 in 2024-09-13T15:19:30.519 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:30.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:30.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:30.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:30 smithi099 bash[84576]: cluster 2024-09-13T15:19:29.371085+0000 mon.a (mon.0) 2786 : cluster [DBG] osdmap e1141: 8 total, 8 up, 7 in 2024-09-13T15:19:30.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:30 smithi099 bash[84576]: cluster 2024-09-13T15:19:29.371085+0000 mon.a (mon.0) 2786 : cluster [DBG] osdmap e1141: 8 total, 8 up, 7 in 2024-09-13T15:19:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:30 smithi099 bash[87147]: cluster 2024-09-13T15:19:29.371085+0000 mon.a (mon.0) 2786 : cluster [DBG] osdmap e1141: 8 total, 8 up, 7 in 2024-09-13T15:19:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:30 smithi099 bash[87147]: cluster 2024-09-13T15:19:29.371085+0000 mon.a (mon.0) 2786 : cluster [DBG] osdmap e1141: 8 total, 8 up, 7 in 2024-09-13T15:19:30.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:30.649 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:30.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:30.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:30.752 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:30.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:30.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:30.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:30.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:30.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:30.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:30.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:31.009 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:31.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:31.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:31.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:31.140 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:31.169 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:31.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:31.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:31.271 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:31.297 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:31.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:31.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:31.400 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:31.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:31.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:31.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:31.531 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:31.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:31.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:31.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:31.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:31 smithi099 bash[84576]: cluster 2024-09-13T15:19:30.378172+0000 mon.a (mon.0) 2787 : cluster [DBG] osdmap e1142: 8 total, 8 up, 7 in 2024-09-13T15:19:31.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:31 smithi099 bash[84576]: cluster 2024-09-13T15:19:30.378172+0000 mon.a (mon.0) 2787 : cluster [DBG] osdmap e1142: 8 total, 8 up, 7 in 2024-09-13T15:19:31.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:31 smithi099 bash[84576]: cluster 2024-09-13T15:19:31.255845+0000 mgr.y (mgr.26252) 1606 : cluster [DBG] pgmap v2232: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 528 objects/s recovering 2024-09-13T15:19:31.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:31 smithi099 bash[84576]: cluster 2024-09-13T15:19:31.255845+0000 mgr.y (mgr.26252) 1606 : cluster [DBG] pgmap v2232: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 528 objects/s recovering 2024-09-13T15:19:31.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:31 smithi099 bash[87147]: cluster 2024-09-13T15:19:30.378172+0000 mon.a (mon.0) 2787 : cluster [DBG] osdmap e1142: 8 total, 8 up, 7 in 2024-09-13T15:19:31.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:31 smithi099 bash[87147]: cluster 2024-09-13T15:19:30.378172+0000 mon.a (mon.0) 2787 : cluster [DBG] osdmap e1142: 8 total, 8 up, 7 in 2024-09-13T15:19:31.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:31 smithi099 bash[87147]: cluster 2024-09-13T15:19:31.255845+0000 mgr.y (mgr.26252) 1606 : cluster [DBG] pgmap v2232: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 528 objects/s recovering 2024-09-13T15:19:31.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:31 smithi099 bash[87147]: cluster 2024-09-13T15:19:31.255845+0000 mgr.y (mgr.26252) 1606 : cluster [DBG] pgmap v2232: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 528 objects/s recovering 2024-09-13T15:19:31.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:31.673 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:31 smithi177 bash[68594]: cluster 2024-09-13T15:19:30.378172+0000 mon.a (mon.0) 2787 : cluster [DBG] osdmap e1142: 8 total, 8 up, 7 in 2024-09-13T15:19:31.673 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:31 smithi177 bash[68594]: cluster 2024-09-13T15:19:30.378172+0000 mon.a (mon.0) 2787 : cluster [DBG] osdmap e1142: 8 total, 8 up, 7 in 2024-09-13T15:19:31.673 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:31 smithi177 bash[68594]: cluster 2024-09-13T15:19:31.255845+0000 mgr.y (mgr.26252) 1606 : cluster [DBG] pgmap v2232: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 528 objects/s recovering 2024-09-13T15:19:31.673 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:31 smithi177 bash[68594]: cluster 2024-09-13T15:19:31.255845+0000 mgr.y (mgr.26252) 1606 : cluster [DBG] pgmap v2232: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 528 objects/s recovering 2024-09-13T15:19:31.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:31.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:31.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:31.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:31.817 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:31.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:31.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:31.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:31.946 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:31.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:31.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:32.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:32.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:32.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:32.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:32.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:32.207 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:32.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:32.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:32.310 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:32.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:32.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:32.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:32.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:32.451 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:32 smithi099 bash[84576]: cluster 2024-09-13T15:19:30.585505+0000 osd.0 (osd.0) 59 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:32.451 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:32 smithi099 bash[84576]: cluster 2024-09-13T15:19:30.585505+0000 osd.0 (osd.0) 59 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:32.451 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:32 smithi099 bash[84576]: cluster 2024-09-13T15:19:31.372707+0000 mon.a (mon.0) 2788 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:32.451 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:32 smithi099 bash[84576]: cluster 2024-09-13T15:19:31.372707+0000 mon.a (mon.0) 2788 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:32.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:32 smithi099 bash[87147]: cluster 2024-09-13T15:19:30.585505+0000 osd.0 (osd.0) 59 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:32.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:32 smithi099 bash[87147]: cluster 2024-09-13T15:19:30.585505+0000 osd.0 (osd.0) 59 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:32.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:32 smithi099 bash[87147]: cluster 2024-09-13T15:19:31.372707+0000 mon.a (mon.0) 2788 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:32.452 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:32 smithi099 bash[87147]: cluster 2024-09-13T15:19:31.372707+0000 mon.a (mon.0) 2788 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:32.470 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:32.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:32.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:32.573 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:32.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:32.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:32.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:32 smithi177 bash[68594]: cluster 2024-09-13T15:19:30.585505+0000 osd.0 (osd.0) 59 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:32.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:32 smithi177 bash[68594]: cluster 2024-09-13T15:19:30.585505+0000 osd.0 (osd.0) 59 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:32.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:32 smithi177 bash[68594]: cluster 2024-09-13T15:19:31.372707+0000 mon.a (mon.0) 2788 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:32.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:32 smithi177 bash[68594]: cluster 2024-09-13T15:19:31.372707+0000 mon.a (mon.0) 2788 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:19:32.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:32.723 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:32.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:32.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:32.826 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:32.855 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:32.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:32.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:32.959 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:32.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:32.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:32.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:33.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:33.118 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:33.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:33.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:33.221 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:33.244 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:33.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:33.347 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:33.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:33.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:33.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:33.478 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:33.503 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:33.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:33.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:33.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:33.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:33 smithi177 bash[68594]: cluster 2024-09-13T15:19:33.256317+0000 mgr.y (mgr.26252) 1607 : cluster [DBG] pgmap v2233: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:33.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:33 smithi177 bash[68594]: cluster 2024-09-13T15:19:33.256317+0000 mgr.y (mgr.26252) 1607 : cluster [DBG] pgmap v2233: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:33.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:33 smithi177 bash[68594]: audit 2024-09-13T15:19:33.323597+0000 mon.a (mon.0) 2789 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:33.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:33 smithi177 bash[68594]: audit 2024-09-13T15:19:33.323597+0000 mon.a (mon.0) 2789 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:33.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:33.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:33.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:33.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:33.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:33.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:33.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:33.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:33 smithi099 bash[84576]: cluster 2024-09-13T15:19:33.256317+0000 mgr.y (mgr.26252) 1607 : cluster [DBG] pgmap v2233: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:33.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:33 smithi099 bash[84576]: cluster 2024-09-13T15:19:33.256317+0000 mgr.y (mgr.26252) 1607 : cluster [DBG] pgmap v2233: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:33.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:33 smithi099 bash[84576]: audit 2024-09-13T15:19:33.323597+0000 mon.a (mon.0) 2789 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:33.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:33 smithi099 bash[84576]: audit 2024-09-13T15:19:33.323597+0000 mon.a (mon.0) 2789 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:33.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:33 smithi099 bash[87147]: cluster 2024-09-13T15:19:33.256317+0000 mgr.y (mgr.26252) 1607 : cluster [DBG] pgmap v2233: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:33.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:33 smithi099 bash[87147]: cluster 2024-09-13T15:19:33.256317+0000 mgr.y (mgr.26252) 1607 : cluster [DBG] pgmap v2233: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:33.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:33 smithi099 bash[87147]: audit 2024-09-13T15:19:33.323597+0000 mon.a (mon.0) 2789 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:33.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:33 smithi099 bash[87147]: audit 2024-09-13T15:19:33.323597+0000 mon.a (mon.0) 2789 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:33.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:33.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:33.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:33.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:33.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:34.014 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:34.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:34.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:34.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:34.147 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:34.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:34.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:34.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:34.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:34.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:34.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:34.376 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:34.398 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:34.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:34.502 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:34.529 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:34.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:34.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:34.632 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:34.657 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:34.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:34.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:34.761 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:34.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:34.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:34.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:34.887 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:34.915 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:34.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:34.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:35.018 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:35.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:35.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:35.147 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:35.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:35.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:35.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:35.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:35.297 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:35.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:35.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:35.400 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:35.420 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:35.421 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:35 smithi099 bash[84576]: audit 2024-09-13T15:19:35.179844+0000 mon.a (mon.0) 2790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:19:35.421 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:35 smithi099 bash[84576]: audit 2024-09-13T15:19:35.179844+0000 mon.a (mon.0) 2790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:19:35.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:35 smithi099 bash[87147]: audit 2024-09-13T15:19:35.179844+0000 mon.a (mon.0) 2790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:19:35.422 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:35 smithi099 bash[87147]: audit 2024-09-13T15:19:35.179844+0000 mon.a (mon.0) 2790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:19:35.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:35.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:35.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:35.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:35 smithi177 bash[68594]: audit 2024-09-13T15:19:35.179844+0000 mon.a (mon.0) 2790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:19:35.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:35 smithi177 bash[68594]: audit 2024-09-13T15:19:35.179844+0000 mon.a (mon.0) 2790 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:19:35.548 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:35.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:35.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:35.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:35.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:35.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:35.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:35.802 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:35.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:35.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:35.906 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:35.932 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:35.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:36.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:36.059 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:36.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:36.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:36.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:36.180 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:36.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:36.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:36.283 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:36.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:36.306 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:36 smithi099 bash[84576]: cluster 2024-09-13T15:19:35.256993+0000 mgr.y (mgr.26252) 1608 : cluster [DBG] pgmap v2234: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:36.306 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:36 smithi099 bash[84576]: cluster 2024-09-13T15:19:35.256993+0000 mgr.y (mgr.26252) 1608 : cluster [DBG] pgmap v2234: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:36.306 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:36 smithi099 bash[87147]: cluster 2024-09-13T15:19:35.256993+0000 mgr.y (mgr.26252) 1608 : cluster [DBG] pgmap v2234: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:36.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:36 smithi099 bash[87147]: cluster 2024-09-13T15:19:35.256993+0000 mgr.y (mgr.26252) 1608 : cluster [DBG] pgmap v2234: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:36.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:36.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:36.408 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:36.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:36.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:36.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:36.540 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:36.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:36 smithi177 bash[68594]: cluster 2024-09-13T15:19:35.256993+0000 mgr.y (mgr.26252) 1608 : cluster [DBG] pgmap v2234: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:36.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:36 smithi177 bash[68594]: cluster 2024-09-13T15:19:35.256993+0000 mgr.y (mgr.26252) 1608 : cluster [DBG] pgmap v2234: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 319 objects/s recovering 2024-09-13T15:19:36.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:36.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:36.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:36.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:36.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:36.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:36.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:36.819 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:36.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:36.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:36.922 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:36.950 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:36.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:36.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:37.053 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:37.082 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:37.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:37.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:37.185 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:37.212 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:37.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:37.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:37.316 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:37.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:37 smithi099 bash[84576]: cluster 2024-09-13T15:19:36.385464+0000 osd.0 (osd.0) 60 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:37.329 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:37 smithi099 bash[84576]: cluster 2024-09-13T15:19:36.385464+0000 osd.0 (osd.0) 60 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:37.329 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:19:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:19:37] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-13T15:19:37.330 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:37 smithi099 bash[87147]: cluster 2024-09-13T15:19:36.385464+0000 osd.0 (osd.0) 60 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:37.330 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:37 smithi099 bash[87147]: cluster 2024-09-13T15:19:36.385464+0000 osd.0 (osd.0) 60 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:37.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:37.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:37.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:37.473 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:37.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:37.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:37.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:37.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:37 smithi177 bash[68594]: cluster 2024-09-13T15:19:36.385464+0000 osd.0 (osd.0) 60 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:37.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:37 smithi177 bash[68594]: cluster 2024-09-13T15:19:36.385464+0000 osd.0 (osd.0) 60 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:37.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:37.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:37.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:37.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:37.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:37.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:37.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:37.840 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:37.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:37.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:37.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:37.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:37.998 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:38.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:38.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:38.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:38.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:38.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:38.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:38.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:38.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:38.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:38.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:38.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:38.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:38.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:38.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:38.506 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:38 smithi099 bash[84576]: cluster 2024-09-13T15:19:37.257564+0000 mgr.y (mgr.26252) 1609 : cluster [DBG] pgmap v2235: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:38.506 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:38 smithi099 bash[84576]: cluster 2024-09-13T15:19:37.257564+0000 mgr.y (mgr.26252) 1609 : cluster [DBG] pgmap v2235: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:38.507 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:38 smithi099 bash[87147]: cluster 2024-09-13T15:19:37.257564+0000 mgr.y (mgr.26252) 1609 : cluster [DBG] pgmap v2235: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:38.507 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:38 smithi099 bash[87147]: cluster 2024-09-13T15:19:37.257564+0000 mgr.y (mgr.26252) 1609 : cluster [DBG] pgmap v2235: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:38.521 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:38.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:38.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:38.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:38 smithi177 bash[68594]: cluster 2024-09-13T15:19:37.257564+0000 mgr.y (mgr.26252) 1609 : cluster [DBG] pgmap v2235: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:38.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:38 smithi177 bash[68594]: cluster 2024-09-13T15:19:37.257564+0000 mgr.y (mgr.26252) 1609 : cluster [DBG] pgmap v2235: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 366 objects/s recovering 2024-09-13T15:19:38.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:38.655 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:38.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:38.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:38.758 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:38.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:38.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:38.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:38.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:38.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:38.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:38.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:39.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:39.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:39.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:39.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:39.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:39.178 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:39.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:39.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:39.281 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:39.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:39.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:39.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:39.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:39.420 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:39 smithi099 bash[84576]: cluster 2024-09-13T15:19:39.258725+0000 mgr.y (mgr.26252) 1610 : cluster [DBG] pgmap v2236: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 296 objects/s recovering 2024-09-13T15:19:39.421 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:39 smithi099 bash[84576]: cluster 2024-09-13T15:19:39.258725+0000 mgr.y (mgr.26252) 1610 : cluster [DBG] pgmap v2236: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 296 objects/s recovering 2024-09-13T15:19:39.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:39 smithi099 bash[87147]: cluster 2024-09-13T15:19:39.258725+0000 mgr.y (mgr.26252) 1610 : cluster [DBG] pgmap v2236: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 296 objects/s recovering 2024-09-13T15:19:39.421 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:39 smithi099 bash[87147]: cluster 2024-09-13T15:19:39.258725+0000 mgr.y (mgr.26252) 1610 : cluster [DBG] pgmap v2236: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 296 objects/s recovering 2024-09-13T15:19:39.434 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:39.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:39.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:39.537 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:39.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:39 smithi177 bash[68594]: cluster 2024-09-13T15:19:39.258725+0000 mgr.y (mgr.26252) 1610 : cluster [DBG] pgmap v2236: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 296 objects/s recovering 2024-09-13T15:19:39.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:39 smithi177 bash[68594]: cluster 2024-09-13T15:19:39.258725+0000 mgr.y (mgr.26252) 1610 : cluster [DBG] pgmap v2236: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 296 objects/s recovering 2024-09-13T15:19:39.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:39.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:39.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:39.670 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:39.698 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:39.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:39.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:39.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:39.828 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:39.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:39.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:39.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:39.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:39.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:39.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:40.057 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:40.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:40.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:40.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:40.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:40.217 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:40.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:40.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:40.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:40.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:40.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:40.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:40.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:19:40 smithi177 bash[61439]: ts=2024-09-13T15:19:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:19:40.449 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:40.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:40.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:40.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:40.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:40.602 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:40.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:40.705 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:40.731 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:40.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:40.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:40.834 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:40.853 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:40.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:40.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:40.956 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:40.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:40.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:40.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:41.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:41.107 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:41.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:41.210 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:41.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:41.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:41.333 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:41.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:41.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:41.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:41.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:41.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:41.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:41.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:41.582 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:41.612 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:41.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:41.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:41.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:41.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:41.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:41.836 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:41.858 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:41.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:41.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:41.961 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:41.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:41.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:41.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:42.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:42.114 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:42.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:42.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:42.217 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:42.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:42.246 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:42 smithi177 bash[68594]: audit 2024-09-13T15:19:41.200132+0000 mon.a (mon.0) 2791 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.246 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:42 smithi177 bash[68594]: audit 2024-09-13T15:19:41.200132+0000 mon.a (mon.0) 2791 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.246 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:42 smithi177 bash[68594]: audit 2024-09-13T15:19:41.206297+0000 mon.a (mon.0) 2792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.246 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:42 smithi177 bash[68594]: audit 2024-09-13T15:19:41.206297+0000 mon.a (mon.0) 2792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.246 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:42 smithi177 bash[68594]: cluster 2024-09-13T15:19:41.259723+0000 mgr.y (mgr.26252) 1611 : cluster [DBG] pgmap v2237: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 265 objects/s recovering 2024-09-13T15:19:42.246 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:42 smithi177 bash[68594]: cluster 2024-09-13T15:19:41.259723+0000 mgr.y (mgr.26252) 1611 : cluster [DBG] pgmap v2237: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 265 objects/s recovering 2024-09-13T15:19:42.246 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:42 smithi177 bash[68594]: cluster 2024-09-13T15:19:41.418218+0000 osd.0 (osd.0) 61 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:42.247 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:42 smithi177 bash[68594]: cluster 2024-09-13T15:19:41.418218+0000 osd.0 (osd.0) 61 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:42.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:42.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:42.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[84576]: audit 2024-09-13T15:19:41.200132+0000 mon.a (mon.0) 2791 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[84576]: audit 2024-09-13T15:19:41.200132+0000 mon.a (mon.0) 2791 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[84576]: audit 2024-09-13T15:19:41.206297+0000 mon.a (mon.0) 2792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[84576]: audit 2024-09-13T15:19:41.206297+0000 mon.a (mon.0) 2792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[84576]: cluster 2024-09-13T15:19:41.259723+0000 mgr.y (mgr.26252) 1611 : cluster [DBG] pgmap v2237: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 265 objects/s recovering 2024-09-13T15:19:42.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[84576]: cluster 2024-09-13T15:19:41.259723+0000 mgr.y (mgr.26252) 1611 : cluster [DBG] pgmap v2237: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 265 objects/s recovering 2024-09-13T15:19:42.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[84576]: cluster 2024-09-13T15:19:41.418218+0000 osd.0 (osd.0) 61 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:42.358 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[84576]: cluster 2024-09-13T15:19:41.418218+0000 osd.0 (osd.0) 61 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:42.359 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[87147]: audit 2024-09-13T15:19:41.200132+0000 mon.a (mon.0) 2791 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.359 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[87147]: audit 2024-09-13T15:19:41.200132+0000 mon.a (mon.0) 2791 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.359 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[87147]: audit 2024-09-13T15:19:41.206297+0000 mon.a (mon.0) 2792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.359 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[87147]: audit 2024-09-13T15:19:41.206297+0000 mon.a (mon.0) 2792 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:42.359 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[87147]: cluster 2024-09-13T15:19:41.259723+0000 mgr.y (mgr.26252) 1611 : cluster [DBG] pgmap v2237: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 265 objects/s recovering 2024-09-13T15:19:42.359 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[87147]: cluster 2024-09-13T15:19:41.259723+0000 mgr.y (mgr.26252) 1611 : cluster [DBG] pgmap v2237: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 265 objects/s recovering 2024-09-13T15:19:42.359 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[87147]: cluster 2024-09-13T15:19:41.418218+0000 osd.0 (osd.0) 61 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:42.359 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:42 smithi099 bash[87147]: cluster 2024-09-13T15:19:41.418218+0000 osd.0 (osd.0) 61 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:42.372 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:42.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:42.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:42.475 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:42.496 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:42.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:42.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:42.599 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:42.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:42.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:42.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:42.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:42.745 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:42.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:42.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:42.848 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:42.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:42.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:42.973 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:42.993 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:42.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:42.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:43.096 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:43.123 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:43.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:43.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:43.230 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:43.257 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:43.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:43.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:43.361 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:43.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:43 smithi177 bash[68594]: cluster 2024-09-13T15:19:43.260546+0000 mgr.y (mgr.26252) 1612 : cluster [DBG] pgmap v2238: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 140 objects/s recovering 2024-09-13T15:19:43.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:43 smithi177 bash[68594]: cluster 2024-09-13T15:19:43.260546+0000 mgr.y (mgr.26252) 1612 : cluster [DBG] pgmap v2238: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 140 objects/s recovering 2024-09-13T15:19:43.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:43 smithi177 bash[68594]: cluster 2024-09-13T15:19:43.311638+0000 mon.a (mon.0) 2793 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:19:43.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:43 smithi177 bash[68594]: cluster 2024-09-13T15:19:43.311638+0000 mon.a (mon.0) 2793 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:19:43.388 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:43.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:43.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:43.492 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:43.519 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:43.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:43.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:43.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:43.647 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:43.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:43.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:43.720 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:43 smithi099 bash[87147]: cluster 2024-09-13T15:19:43.260546+0000 mgr.y (mgr.26252) 1612 : cluster [DBG] pgmap v2238: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 140 objects/s recovering 2024-09-13T15:19:43.720 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:43 smithi099 bash[87147]: cluster 2024-09-13T15:19:43.260546+0000 mgr.y (mgr.26252) 1612 : cluster [DBG] pgmap v2238: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 140 objects/s recovering 2024-09-13T15:19:43.721 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:43 smithi099 bash[87147]: cluster 2024-09-13T15:19:43.311638+0000 mon.a (mon.0) 2793 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:19:43.721 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:43 smithi099 bash[87147]: cluster 2024-09-13T15:19:43.311638+0000 mon.a (mon.0) 2793 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:19:43.721 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:43 smithi099 bash[84576]: cluster 2024-09-13T15:19:43.260546+0000 mgr.y (mgr.26252) 1612 : cluster [DBG] pgmap v2238: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 140 objects/s recovering 2024-09-13T15:19:43.721 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:43 smithi099 bash[84576]: cluster 2024-09-13T15:19:43.260546+0000 mgr.y (mgr.26252) 1612 : cluster [DBG] pgmap v2238: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 140 objects/s recovering 2024-09-13T15:19:43.721 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:43 smithi099 bash[84576]: cluster 2024-09-13T15:19:43.311638+0000 mon.a (mon.0) 2793 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:19:43.721 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:43 smithi099 bash[84576]: cluster 2024-09-13T15:19:43.311638+0000 mon.a (mon.0) 2793 : cluster [WRN] Health check update: Degraded data redundancy: 3810/120 objects degraded (3175.000%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T15:19:43.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:43.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:43.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:43.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:43.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:43.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:43.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:43.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:44.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:44.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:44.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:44.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:44.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:44.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:44.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:44.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:44.269 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:44.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:44.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:44.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:44.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:44.428 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:44.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:44.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:44.532 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:44.560 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:44.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:44.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:44.663 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:44.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:44.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:44.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:44.793 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:44.821 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:44.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:44.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:44.924 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:44.951 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:44.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:44.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:45.053 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:45.081 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:45.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:45.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:45.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:45.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:45.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:45.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:45.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:45.335 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:45.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:45.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:45.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:45.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:45.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:45.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:45.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:45.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:45.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:45.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:45 smithi177 bash[68594]: cluster 2024-09-13T15:19:44.534167+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e1143: 8 total, 8 up, 7 in 2024-09-13T15:19:45.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:45 smithi177 bash[68594]: cluster 2024-09-13T15:19:44.534167+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e1143: 8 total, 8 up, 7 in 2024-09-13T15:19:45.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:45 smithi177 bash[68594]: cluster 2024-09-13T15:19:45.261122+0000 mgr.y (mgr.26252) 1613 : cluster [DBG] pgmap v2240: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 168 objects/s recovering 2024-09-13T15:19:45.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:45 smithi177 bash[68594]: cluster 2024-09-13T15:19:45.261122+0000 mgr.y (mgr.26252) 1613 : cluster [DBG] pgmap v2240: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 168 objects/s recovering 2024-09-13T15:19:45.686 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:45.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:45.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:45.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:45.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:45.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:45.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:45.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:45.947 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:45.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:45.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:45.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:46.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:45 smithi099 bash[84576]: cluster 2024-09-13T15:19:44.534167+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e1143: 8 total, 8 up, 7 in 2024-09-13T15:19:46.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:45 smithi099 bash[84576]: cluster 2024-09-13T15:19:44.534167+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e1143: 8 total, 8 up, 7 in 2024-09-13T15:19:46.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:45 smithi099 bash[84576]: cluster 2024-09-13T15:19:45.261122+0000 mgr.y (mgr.26252) 1613 : cluster [DBG] pgmap v2240: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 168 objects/s recovering 2024-09-13T15:19:46.023 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:45 smithi099 bash[84576]: cluster 2024-09-13T15:19:45.261122+0000 mgr.y (mgr.26252) 1613 : cluster [DBG] pgmap v2240: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 168 objects/s recovering 2024-09-13T15:19:46.023 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:45 smithi099 bash[87147]: cluster 2024-09-13T15:19:44.534167+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e1143: 8 total, 8 up, 7 in 2024-09-13T15:19:46.023 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:45 smithi099 bash[87147]: cluster 2024-09-13T15:19:44.534167+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e1143: 8 total, 8 up, 7 in 2024-09-13T15:19:46.024 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:45 smithi099 bash[87147]: cluster 2024-09-13T15:19:45.261122+0000 mgr.y (mgr.26252) 1613 : cluster [DBG] pgmap v2240: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 168 objects/s recovering 2024-09-13T15:19:46.024 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:45 smithi099 bash[87147]: cluster 2024-09-13T15:19:45.261122+0000 mgr.y (mgr.26252) 1613 : cluster [DBG] pgmap v2240: 109 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 3810/120 objects degraded (3175.000%); 0 B/s, 168 objects/s recovering 2024-09-13T15:19:46.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:46.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:46.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:46.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:46.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:46.227 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:46.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:46.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:46.330 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:46.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:46.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:46.454 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:46.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:46.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:46.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:46.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:46.614 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:46.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:46.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: cluster 2024-09-13T15:19:45.541634+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e1144: 8 total, 8 up, 7 in 2024-09-13T15:19:46.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: cluster 2024-09-13T15:19:45.541634+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e1144: 8 total, 8 up, 7 in 2024-09-13T15:19:46.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: audit 2024-09-13T15:19:45.712399+0000 mon.a (mon.0) 2796 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: audit 2024-09-13T15:19:45.712399+0000 mon.a (mon.0) 2796 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: audit 2024-09-13T15:19:45.718838+0000 mon.a (mon.0) 2797 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: audit 2024-09-13T15:19:45.718838+0000 mon.a (mon.0) 2797 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: audit 2024-09-13T15:19:46.473465+0000 mon.a (mon.0) 2798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: audit 2024-09-13T15:19:46.473465+0000 mon.a (mon.0) 2798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: audit 2024-09-13T15:19:46.474326+0000 mon.a (mon.0) 2799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: audit 2024-09-13T15:19:46.474326+0000 mon.a (mon.0) 2799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: audit 2024-09-13T15:19:46.481713+0000 mon.a (mon.0) 2800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[84576]: audit 2024-09-13T15:19:46.481713+0000 mon.a (mon.0) 2800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: cluster 2024-09-13T15:19:45.541634+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e1144: 8 total, 8 up, 7 in 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: cluster 2024-09-13T15:19:45.541634+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e1144: 8 total, 8 up, 7 in 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: audit 2024-09-13T15:19:45.712399+0000 mon.a (mon.0) 2796 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: audit 2024-09-13T15:19:45.712399+0000 mon.a (mon.0) 2796 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: audit 2024-09-13T15:19:45.718838+0000 mon.a (mon.0) 2797 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.638 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: audit 2024-09-13T15:19:45.718838+0000 mon.a (mon.0) 2797 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.638 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: audit 2024-09-13T15:19:46.473465+0000 mon.a (mon.0) 2798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:19:46.638 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: audit 2024-09-13T15:19:46.473465+0000 mon.a (mon.0) 2798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:19:46.638 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: audit 2024-09-13T15:19:46.474326+0000 mon.a (mon.0) 2799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:19:46.638 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: audit 2024-09-13T15:19:46.474326+0000 mon.a (mon.0) 2799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:19:46.638 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: audit 2024-09-13T15:19:46.481713+0000 mon.a (mon.0) 2800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.638 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:46 smithi099 bash[87147]: audit 2024-09-13T15:19:46.481713+0000 mon.a (mon.0) 2800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:46.732 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:46.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:46.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:46.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:46.859 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:46.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:46.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: cluster 2024-09-13T15:19:45.541634+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e1144: 8 total, 8 up, 7 in 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: cluster 2024-09-13T15:19:45.541634+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e1144: 8 total, 8 up, 7 in 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: audit 2024-09-13T15:19:45.712399+0000 mon.a (mon.0) 2796 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: audit 2024-09-13T15:19:45.712399+0000 mon.a (mon.0) 2796 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: audit 2024-09-13T15:19:45.718838+0000 mon.a (mon.0) 2797 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: audit 2024-09-13T15:19:45.718838+0000 mon.a (mon.0) 2797 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: audit 2024-09-13T15:19:46.473465+0000 mon.a (mon.0) 2798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: audit 2024-09-13T15:19:46.473465+0000 mon.a (mon.0) 2798 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: audit 2024-09-13T15:19:46.474326+0000 mon.a (mon.0) 2799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: audit 2024-09-13T15:19:46.474326+0000 mon.a (mon.0) 2799 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: audit 2024-09-13T15:19:46.481713+0000 mon.a (mon.0) 2800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:46 smithi177 bash[68594]: audit 2024-09-13T15:19:46.481713+0000 mon.a (mon.0) 2800 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:19:46.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:46.986 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:46.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:46.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:47.089 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:47.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:47.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:47.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:47.212 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:47.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:47.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:47.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:47.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:19:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:19:47] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-13T15:19:47.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:47.370 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:47.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:47.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:47.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:47.495 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:47.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:47.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:47.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:47.649 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:47.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:47.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:47 smithi099 bash[84576]: cluster 2024-09-13T15:19:47.250681+0000 osd.0 (osd.0) 62 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:47.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:47 smithi099 bash[84576]: cluster 2024-09-13T15:19:47.250681+0000 osd.0 (osd.0) 62 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:47.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:47 smithi099 bash[84576]: cluster 2024-09-13T15:19:47.261941+0000 mgr.y (mgr.26252) 1614 : cluster [DBG] pgmap v2242: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 264 objects/s recovering 2024-09-13T15:19:47.737 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:47 smithi099 bash[84576]: cluster 2024-09-13T15:19:47.261941+0000 mgr.y (mgr.26252) 1614 : cluster [DBG] pgmap v2242: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 264 objects/s recovering 2024-09-13T15:19:47.737 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:47 smithi099 bash[87147]: cluster 2024-09-13T15:19:47.250681+0000 osd.0 (osd.0) 62 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:47.737 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:47 smithi099 bash[87147]: cluster 2024-09-13T15:19:47.250681+0000 osd.0 (osd.0) 62 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:47.738 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:47 smithi099 bash[87147]: cluster 2024-09-13T15:19:47.261941+0000 mgr.y (mgr.26252) 1614 : cluster [DBG] pgmap v2242: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 264 objects/s recovering 2024-09-13T15:19:47.738 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:47 smithi099 bash[87147]: cluster 2024-09-13T15:19:47.261941+0000 mgr.y (mgr.26252) 1614 : cluster [DBG] pgmap v2242: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 264 objects/s recovering 2024-09-13T15:19:47.752 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:47.773 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:47.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:47.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:47.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:47 smithi177 bash[68594]: cluster 2024-09-13T15:19:47.250681+0000 osd.0 (osd.0) 62 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:47.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:47 smithi177 bash[68594]: cluster 2024-09-13T15:19:47.250681+0000 osd.0 (osd.0) 62 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:47.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:47 smithi177 bash[68594]: cluster 2024-09-13T15:19:47.261941+0000 mgr.y (mgr.26252) 1614 : cluster [DBG] pgmap v2242: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 264 objects/s recovering 2024-09-13T15:19:47.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:47 smithi177 bash[68594]: cluster 2024-09-13T15:19:47.261941+0000 mgr.y (mgr.26252) 1614 : cluster [DBG] pgmap v2242: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 103 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 264 objects/s recovering 2024-09-13T15:19:47.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:47.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:47.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:47.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:48.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:48.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:48.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:48.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:48.138 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:48.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:48.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:48.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:48.269 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:48.295 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:48.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:48.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:48.408 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:48.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:48.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:48.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:48.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:48.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:48.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:48.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:48.669 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:48.680 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:48 smithi177 bash[68594]: audit 2024-09-13T15:19:48.323790+0000 mon.a (mon.0) 2801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:48.680 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:48 smithi177 bash[68594]: audit 2024-09-13T15:19:48.323790+0000 mon.a (mon.0) 2801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:48.696 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:48.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:48.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:48.799 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:48.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:48 smithi099 bash[84576]: audit 2024-09-13T15:19:48.323790+0000 mon.a (mon.0) 2801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:48.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:48 smithi099 bash[84576]: audit 2024-09-13T15:19:48.323790+0000 mon.a (mon.0) 2801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:48.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:48 smithi099 bash[87147]: audit 2024-09-13T15:19:48.323790+0000 mon.a (mon.0) 2801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:48.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:48 smithi099 bash[87147]: audit 2024-09-13T15:19:48.323790+0000 mon.a (mon.0) 2801 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:19:48.826 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:48.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:48.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:48.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:49.104 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:49.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:49.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:49.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:49.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:49.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:49.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:49.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:49.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:49.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:49.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:49.467 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:49.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:49.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:49.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:49.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:49.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:49.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:49.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:49.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:49.755 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:49.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:49.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:49.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:49 smithi099 bash[84576]: cluster 2024-09-13T15:19:49.262914+0000 mgr.y (mgr.26252) 1615 : cluster [DBG] pgmap v2243: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 144 objects/s recovering 2024-09-13T15:19:49.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:49 smithi099 bash[84576]: cluster 2024-09-13T15:19:49.262914+0000 mgr.y (mgr.26252) 1615 : cluster [DBG] pgmap v2243: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 144 objects/s recovering 2024-09-13T15:19:49.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:49 smithi099 bash[87147]: cluster 2024-09-13T15:19:49.262914+0000 mgr.y (mgr.26252) 1615 : cluster [DBG] pgmap v2243: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 144 objects/s recovering 2024-09-13T15:19:49.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:49 smithi099 bash[87147]: cluster 2024-09-13T15:19:49.262914+0000 mgr.y (mgr.26252) 1615 : cluster [DBG] pgmap v2243: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 144 objects/s recovering 2024-09-13T15:19:49.858 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:49.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:49 smithi177 bash[68594]: cluster 2024-09-13T15:19:49.262914+0000 mgr.y (mgr.26252) 1615 : cluster [DBG] pgmap v2243: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 144 objects/s recovering 2024-09-13T15:19:49.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:49 smithi177 bash[68594]: cluster 2024-09-13T15:19:49.262914+0000 mgr.y (mgr.26252) 1615 : cluster [DBG] pgmap v2243: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 3279/120 objects degraded (2732.500%); 0 B/s, 144 objects/s recovering 2024-09-13T15:19:49.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:49.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:49.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:49.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:50.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:50.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:50.121 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:50.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:50.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:50.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:50.277 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:50.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:50.380 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:50.391 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:19:50 smithi177 bash[61439]: ts=2024-09-13T15:19:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:19:50.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:50.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:50.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:50.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:50.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:50.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:50.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:50.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:50.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:50.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:50.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:50.770 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:50.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:50.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:50.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:50.901 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:50.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:50.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:50.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:51.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:51.058 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:51.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:51.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:51.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:51.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:51.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:51.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:51.290 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:51.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:51.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:51.421 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:51.432 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:51 smithi099 bash[84576]: cluster 2024-09-13T15:19:51.264205+0000 mgr.y (mgr.26252) 1616 : cluster [DBG] pgmap v2244: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 210 objects/s recovering 2024-09-13T15:19:51.432 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:51 smithi099 bash[84576]: cluster 2024-09-13T15:19:51.264205+0000 mgr.y (mgr.26252) 1616 : cluster [DBG] pgmap v2244: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 210 objects/s recovering 2024-09-13T15:19:51.432 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:51 smithi099 bash[84576]: cluster 2024-09-13T15:19:51.315072+0000 mon.a (mon.0) 2802 : cluster [WRN] Health check update: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:19:51.432 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:51 smithi099 bash[84576]: cluster 2024-09-13T15:19:51.315072+0000 mon.a (mon.0) 2802 : cluster [WRN] Health check update: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:19:51.433 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:51 smithi099 bash[87147]: cluster 2024-09-13T15:19:51.264205+0000 mgr.y (mgr.26252) 1616 : cluster [DBG] pgmap v2244: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 210 objects/s recovering 2024-09-13T15:19:51.433 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:51 smithi099 bash[87147]: cluster 2024-09-13T15:19:51.264205+0000 mgr.y (mgr.26252) 1616 : cluster [DBG] pgmap v2244: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 210 objects/s recovering 2024-09-13T15:19:51.433 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:51 smithi099 bash[87147]: cluster 2024-09-13T15:19:51.315072+0000 mon.a (mon.0) 2802 : cluster [WRN] Health check update: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:19:51.433 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:51 smithi099 bash[87147]: cluster 2024-09-13T15:19:51.315072+0000 mon.a (mon.0) 2802 : cluster [WRN] Health check update: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:19:51.445 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:51.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:51.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:51.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:51.576 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:51.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:51.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:51.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:51 smithi177 bash[68594]: cluster 2024-09-13T15:19:51.264205+0000 mgr.y (mgr.26252) 1616 : cluster [DBG] pgmap v2244: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 210 objects/s recovering 2024-09-13T15:19:51.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:51 smithi177 bash[68594]: cluster 2024-09-13T15:19:51.264205+0000 mgr.y (mgr.26252) 1616 : cluster [DBG] pgmap v2244: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 210 objects/s recovering 2024-09-13T15:19:51.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:51 smithi177 bash[68594]: cluster 2024-09-13T15:19:51.315072+0000 mon.a (mon.0) 2802 : cluster [WRN] Health check update: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:19:51.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:51 smithi177 bash[68594]: cluster 2024-09-13T15:19:51.315072+0000 mon.a (mon.0) 2802 : cluster [WRN] Health check update: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:19:51.679 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:51.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:51.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:51.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:51.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:51.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:51.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:51.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:51.939 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:51.965 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:51.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:51.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:52.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:52.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:52.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:52.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:52.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:52.225 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:52.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:52.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:52.328 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:52.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:52.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:52.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:52.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:52.484 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:52.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:52.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:52.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:52.615 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:52.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:52.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:52.719 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:52.746 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:52.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:52.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:52.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:52.880 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:52.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:52.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:52.984 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:53.012 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:53.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:53.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:53.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:53.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:53.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:53.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:53.272 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:53.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:53.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:53.375 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:53.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:53 smithi177 bash[68594]: cluster 2024-09-13T15:19:53.265007+0000 mgr.y (mgr.26252) 1617 : cluster [DBG] pgmap v2245: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 192 objects/s recovering 2024-09-13T15:19:53.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:53 smithi177 bash[68594]: cluster 2024-09-13T15:19:53.265007+0000 mgr.y (mgr.26252) 1617 : cluster [DBG] pgmap v2245: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 192 objects/s recovering 2024-09-13T15:19:53.402 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:53.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:53.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:53.505 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:53.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:53.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:53.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:53.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:53.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:53.677 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:53 smithi099 bash[84576]: cluster 2024-09-13T15:19:53.265007+0000 mgr.y (mgr.26252) 1617 : cluster [DBG] pgmap v2245: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 192 objects/s recovering 2024-09-13T15:19:53.677 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:53 smithi099 bash[84576]: cluster 2024-09-13T15:19:53.265007+0000 mgr.y (mgr.26252) 1617 : cluster [DBG] pgmap v2245: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 192 objects/s recovering 2024-09-13T15:19:53.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:53.793 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:53.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:53.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:53.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:53.922 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:53.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:53.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:53.960 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:53 smithi099 bash[87147]: cluster 2024-09-13T15:19:53.265007+0000 mgr.y (mgr.26252) 1617 : cluster [DBG] pgmap v2245: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 192 objects/s recovering 2024-09-13T15:19:53.960 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:53 smithi099 bash[87147]: cluster 2024-09-13T15:19:53.265007+0000 mgr.y (mgr.26252) 1617 : cluster [DBG] pgmap v2245: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 192 objects/s recovering 2024-09-13T15:19:54.027 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:54.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:54.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:54.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:54.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:54.185 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:54.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:54.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:54.289 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:54.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:54.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:54.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:54.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:54.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:54.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:54.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:54.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:54.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:54 smithi177 bash[68594]: cluster 2024-09-13T15:19:53.156260+0000 osd.0 (osd.0) 63 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:54.565 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:54 smithi177 bash[68594]: cluster 2024-09-13T15:19:53.156260+0000 osd.0 (osd.0) 63 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:54.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:54 smithi099 bash[84576]: cluster 2024-09-13T15:19:53.156260+0000 osd.0 (osd.0) 63 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:54.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:54 smithi099 bash[84576]: cluster 2024-09-13T15:19:53.156260+0000 osd.0 (osd.0) 63 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:54.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:54 smithi099 bash[87147]: cluster 2024-09-13T15:19:53.156260+0000 osd.0 (osd.0) 63 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:54.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:54 smithi099 bash[87147]: cluster 2024-09-13T15:19:53.156260+0000 osd.0 (osd.0) 63 : cluster [DBG] 46.13 scrub starts 2024-09-13T15:19:54.582 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:54.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:54.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:54.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:54.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:54.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:54.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:54.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:54.838 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:54.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:54.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:54.970 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:54.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:55.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:55.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:55.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:55.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:55.205 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:55.226 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:55.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:55.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:55.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:55.354 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:55.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:55.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:55.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:55.468 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:55 smithi099 bash[84576]: cluster 2024-09-13T15:19:55.265698+0000 mgr.y (mgr.26252) 1618 : cluster [DBG] pgmap v2246: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 219 objects/s recovering 2024-09-13T15:19:55.468 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:55 smithi099 bash[84576]: cluster 2024-09-13T15:19:55.265698+0000 mgr.y (mgr.26252) 1618 : cluster [DBG] pgmap v2246: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 219 objects/s recovering 2024-09-13T15:19:55.469 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:55 smithi099 bash[87147]: cluster 2024-09-13T15:19:55.265698+0000 mgr.y (mgr.26252) 1618 : cluster [DBG] pgmap v2246: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 219 objects/s recovering 2024-09-13T15:19:55.469 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:55 smithi099 bash[87147]: cluster 2024-09-13T15:19:55.265698+0000 mgr.y (mgr.26252) 1618 : cluster [DBG] pgmap v2246: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 219 objects/s recovering 2024-09-13T15:19:55.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:55.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:55.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:55.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:55.613 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:55.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:55.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:55.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:55.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:55.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:55.844 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:55.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:55 smithi177 bash[68594]: cluster 2024-09-13T15:19:55.265698+0000 mgr.y (mgr.26252) 1618 : cluster [DBG] pgmap v2246: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 219 objects/s recovering 2024-09-13T15:19:55.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:55 smithi177 bash[68594]: cluster 2024-09-13T15:19:55.265698+0000 mgr.y (mgr.26252) 1618 : cluster [DBG] pgmap v2246: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.4 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 564/120 objects degraded (470.000%); 0 B/s, 219 objects/s recovering 2024-09-13T15:19:55.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:55.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:55.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:55.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:56.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:56.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:56.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:56.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:56.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:56.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:56.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:56.232 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:56.260 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:56.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:56.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:56.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:56.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:56.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:56.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:56.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:56.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:56.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:56.595 INFO:teuthology.orchestra.run.smithi099.stderr:pool 'unique_pool_5' removed 2024-09-13T15:19:56.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:56.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:56.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:56.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:56.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:56.815 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:56 smithi099 bash[84576]: audit 2024-09-13T15:19:56.542173+0000 mon.a (mon.0) 2803 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:19:56.815 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:56 smithi099 bash[84576]: audit 2024-09-13T15:19:56.542173+0000 mon.a (mon.0) 2803 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:19:56.815 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:56 smithi099 bash[87147]: audit 2024-09-13T15:19:56.542173+0000 mon.a (mon.0) 2803 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:19:56.815 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:56 smithi099 bash[87147]: audit 2024-09-13T15:19:56.542173+0000 mon.a (mon.0) 2803 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:19:56.827 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:56.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:56.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:56 smithi177 bash[68594]: audit 2024-09-13T15:19:56.542173+0000 mon.a (mon.0) 2803 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:19:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:56 smithi177 bash[68594]: audit 2024-09-13T15:19:56.542173+0000 mon.a (mon.0) 2803 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:19:56.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:56.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:56.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:56.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:57.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:57.074 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:19:57] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-13T15:19:57.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:57.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:57.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:57.192 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:57.236 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:57.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:57.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:57.366 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:19:57.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:57.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:19:57.469 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:57.492 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:57.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:57.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:57.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:57.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:57.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:57.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:57.737 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:57.748 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[84576]: audit 2024-09-13T15:19:56.594611+0000 mon.a (mon.0) 2804 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:19:57.748 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[84576]: audit 2024-09-13T15:19:56.594611+0000 mon.a (mon.0) 2804 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:19:57.748 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[84576]: cluster 2024-09-13T15:19:56.598289+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e1145: 8 total, 8 up, 7 in 2024-09-13T15:19:57.748 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[84576]: cluster 2024-09-13T15:19:56.598289+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e1145: 8 total, 8 up, 7 in 2024-09-13T15:19:57.748 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[84576]: cluster 2024-09-13T15:19:57.266425+0000 mgr.y (mgr.26252) 1619 : cluster [DBG] pgmap v2248: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[84576]: cluster 2024-09-13T15:19:57.266425+0000 mgr.y (mgr.26252) 1619 : cluster [DBG] pgmap v2248: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[84576]: audit 2024-09-13T15:19:57.267680+0000 mon.a (mon.0) 2806 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[84576]: audit 2024-09-13T15:19:57.267680+0000 mon.a (mon.0) 2806 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[87147]: audit 2024-09-13T15:19:56.594611+0000 mon.a (mon.0) 2804 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[87147]: audit 2024-09-13T15:19:56.594611+0000 mon.a (mon.0) 2804 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[87147]: cluster 2024-09-13T15:19:56.598289+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e1145: 8 total, 8 up, 7 in 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[87147]: cluster 2024-09-13T15:19:56.598289+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e1145: 8 total, 8 up, 7 in 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[87147]: cluster 2024-09-13T15:19:57.266425+0000 mgr.y (mgr.26252) 1619 : cluster [DBG] pgmap v2248: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[87147]: cluster 2024-09-13T15:19:57.266425+0000 mgr.y (mgr.26252) 1619 : cluster [DBG] pgmap v2248: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[87147]: audit 2024-09-13T15:19:57.267680+0000 mon.a (mon.0) 2806 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T15:19:57.749 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:57 smithi099 bash[87147]: audit 2024-09-13T15:19:57.267680+0000 mon.a (mon.0) 2806 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T15:19:57.763 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:57.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:57.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:57.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:57 smithi177 bash[68594]: audit 2024-09-13T15:19:56.594611+0000 mon.a (mon.0) 2804 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:19:57.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:57 smithi177 bash[68594]: audit 2024-09-13T15:19:56.594611+0000 mon.a (mon.0) 2804 : audit [INF] from='client.? 172.21.15.99:0/2175188192' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:19:57.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:57 smithi177 bash[68594]: cluster 2024-09-13T15:19:56.598289+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e1145: 8 total, 8 up, 7 in 2024-09-13T15:19:57.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:57 smithi177 bash[68594]: cluster 2024-09-13T15:19:56.598289+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e1145: 8 total, 8 up, 7 in 2024-09-13T15:19:57.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:57 smithi177 bash[68594]: cluster 2024-09-13T15:19:57.266425+0000 mgr.y (mgr.26252) 1619 : cluster [DBG] pgmap v2248: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:19:57.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:57 smithi177 bash[68594]: cluster 2024-09-13T15:19:57.266425+0000 mgr.y (mgr.26252) 1619 : cluster [DBG] pgmap v2248: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:19:57.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:57 smithi177 bash[68594]: audit 2024-09-13T15:19:57.267680+0000 mon.a (mon.0) 2806 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T15:19:57.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:57 smithi177 bash[68594]: audit 2024-09-13T15:19:57.267680+0000 mon.a (mon.0) 2806 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T15:19:57.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:57.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:57.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:57.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:57.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:58.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:58.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:58.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:58.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:58.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:58.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:58.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:58.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:58.287 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:19:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:58.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:19:58.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:58.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:19:58.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:58.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:19:58.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:58.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:19:58.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:58.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:19:58.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:58.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:58.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:58.779 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:58.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:58.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:58.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:58.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:58.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:58.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:58.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:58.949 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[84576]: cluster 2024-09-13T15:19:57.604815+0000 mon.a (mon.0) 2807 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:19:58.949 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[84576]: cluster 2024-09-13T15:19:57.604815+0000 mon.a (mon.0) 2807 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:19:58.949 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[84576]: audit 2024-09-13T15:19:57.615045+0000 mon.a (mon.0) 2808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T15:19:58.949 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[84576]: audit 2024-09-13T15:19:57.615045+0000 mon.a (mon.0) 2808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T15:19:58.949 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[84576]: cluster 2024-09-13T15:19:57.625281+0000 mon.a (mon.0) 2809 : cluster [DBG] osdmap e1146: 8 total, 8 up, 7 in 2024-09-13T15:19:58.949 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[84576]: cluster 2024-09-13T15:19:57.625281+0000 mon.a (mon.0) 2809 : cluster [DBG] osdmap e1146: 8 total, 8 up, 7 in 2024-09-13T15:19:59.040 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:59.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:59.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:59.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:59.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:58 smithi177 bash[68594]: cluster 2024-09-13T15:19:57.604815+0000 mon.a (mon.0) 2807 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:19:59.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:58 smithi177 bash[68594]: cluster 2024-09-13T15:19:57.604815+0000 mon.a (mon.0) 2807 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:19:59.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:58 smithi177 bash[68594]: audit 2024-09-13T15:19:57.615045+0000 mon.a (mon.0) 2808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T15:19:59.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:58 smithi177 bash[68594]: audit 2024-09-13T15:19:57.615045+0000 mon.a (mon.0) 2808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T15:19:59.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:58 smithi177 bash[68594]: cluster 2024-09-13T15:19:57.625281+0000 mon.a (mon.0) 2809 : cluster [DBG] osdmap e1146: 8 total, 8 up, 7 in 2024-09-13T15:19:59.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:58 smithi177 bash[68594]: cluster 2024-09-13T15:19:57.625281+0000 mon.a (mon.0) 2809 : cluster [DBG] osdmap e1146: 8 total, 8 up, 7 in 2024-09-13T15:19:59.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:59.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:19:59.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:59.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:19:59.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:59.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[87147]: cluster 2024-09-13T15:19:57.604815+0000 mon.a (mon.0) 2807 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:19:59.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[87147]: cluster 2024-09-13T15:19:57.604815+0000 mon.a (mon.0) 2807 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 564/120 objects degraded (470.000%), 1 pg degraded, 1 pg undersized) 2024-09-13T15:19:59.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[87147]: audit 2024-09-13T15:19:57.615045+0000 mon.a (mon.0) 2808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T15:19:59.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[87147]: audit 2024-09-13T15:19:57.615045+0000 mon.a (mon.0) 2808 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T15:19:59.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[87147]: cluster 2024-09-13T15:19:57.625281+0000 mon.a (mon.0) 2809 : cluster [DBG] osdmap e1146: 8 total, 8 up, 7 in 2024-09-13T15:19:59.312 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:58 smithi099 bash[87147]: cluster 2024-09-13T15:19:57.625281+0000 mon.a (mon.0) 2809 : cluster [DBG] osdmap e1146: 8 total, 8 up, 7 in 2024-09-13T15:19:59.327 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:59.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:59.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:59.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:59.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:19:59.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:59.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:19:59.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:59.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:59.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:59.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:59.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:59.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:19:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:59.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:19:59.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[84576]: cluster 2024-09-13T15:19:58.626394+0000 mon.a (mon.0) 2810 : cluster [DBG] osdmap e1147: 8 total, 8 up, 7 in 2024-09-13T15:19:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[84576]: cluster 2024-09-13T15:19:58.626394+0000 mon.a (mon.0) 2810 : cluster [DBG] osdmap e1147: 8 total, 8 up, 7 in 2024-09-13T15:19:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[84576]: cluster 2024-09-13T15:19:58.667709+0000 osd.3 (osd.3) 28 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:19:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[84576]: cluster 2024-09-13T15:19:58.667709+0000 osd.3 (osd.3) 28 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:19:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[84576]: cluster 2024-09-13T15:19:59.267334+0000 mgr.y (mgr.26252) 1620 : cluster [DBG] pgmap v2251: 83 pgs: 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 23 KiB/s rd, 9 op/s 2024-09-13T15:19:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[84576]: cluster 2024-09-13T15:19:59.267334+0000 mgr.y (mgr.26252) 1620 : cluster [DBG] pgmap v2251: 83 pgs: 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 23 KiB/s rd, 9 op/s 2024-09-13T15:19:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[84576]: cluster 2024-09-13T15:19:59.355872+0000 osd.4 (osd.4) 33 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:19:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[84576]: cluster 2024-09-13T15:19:59.355872+0000 osd.4 (osd.4) 33 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:19:59.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[87147]: cluster 2024-09-13T15:19:58.626394+0000 mon.a (mon.0) 2810 : cluster [DBG] osdmap e1147: 8 total, 8 up, 7 in 2024-09-13T15:19:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[87147]: cluster 2024-09-13T15:19:58.626394+0000 mon.a (mon.0) 2810 : cluster [DBG] osdmap e1147: 8 total, 8 up, 7 in 2024-09-13T15:19:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[87147]: cluster 2024-09-13T15:19:58.667709+0000 osd.3 (osd.3) 28 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:19:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[87147]: cluster 2024-09-13T15:19:58.667709+0000 osd.3 (osd.3) 28 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:19:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[87147]: cluster 2024-09-13T15:19:59.267334+0000 mgr.y (mgr.26252) 1620 : cluster [DBG] pgmap v2251: 83 pgs: 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 23 KiB/s rd, 9 op/s 2024-09-13T15:19:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[87147]: cluster 2024-09-13T15:19:59.267334+0000 mgr.y (mgr.26252) 1620 : cluster [DBG] pgmap v2251: 83 pgs: 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 23 KiB/s rd, 9 op/s 2024-09-13T15:19:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[87147]: cluster 2024-09-13T15:19:59.355872+0000 osd.4 (osd.4) 33 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:19:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:19:59 smithi099 bash[87147]: cluster 2024-09-13T15:19:59.355872+0000 osd.4 (osd.4) 33 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:19:59.846 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:59.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:59.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:19:59.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:59.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:19:59.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:19:59.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:00.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:00.091 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:59 smithi177 bash[68594]: cluster 2024-09-13T15:19:58.626394+0000 mon.a (mon.0) 2810 : cluster [DBG] osdmap e1147: 8 total, 8 up, 7 in 2024-09-13T15:20:00.091 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:59 smithi177 bash[68594]: cluster 2024-09-13T15:19:58.626394+0000 mon.a (mon.0) 2810 : cluster [DBG] osdmap e1147: 8 total, 8 up, 7 in 2024-09-13T15:20:00.091 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:59 smithi177 bash[68594]: cluster 2024-09-13T15:19:58.667709+0000 osd.3 (osd.3) 28 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:00.091 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:59 smithi177 bash[68594]: cluster 2024-09-13T15:19:58.667709+0000 osd.3 (osd.3) 28 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:00.091 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:59 smithi177 bash[68594]: cluster 2024-09-13T15:19:59.267334+0000 mgr.y (mgr.26252) 1620 : cluster [DBG] pgmap v2251: 83 pgs: 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 23 KiB/s rd, 9 op/s 2024-09-13T15:20:00.091 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:59 smithi177 bash[68594]: cluster 2024-09-13T15:19:59.267334+0000 mgr.y (mgr.26252) 1620 : cluster [DBG] pgmap v2251: 83 pgs: 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 23 KiB/s rd, 9 op/s 2024-09-13T15:20:00.091 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:59 smithi177 bash[68594]: cluster 2024-09-13T15:19:59.355872+0000 osd.4 (osd.4) 33 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:00.091 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:19:59 smithi177 bash[68594]: cluster 2024-09-13T15:19:59.355872+0000 osd.4 (osd.4) 33 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:00.107 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:00.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:00.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:00.210 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:00.238 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:00.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:00.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:00.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:00.352 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:20:00 smithi177 bash[61439]: ts=2024-09-13T15:20:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:20:00.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:00.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:00.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:00.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:00.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:00.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:00.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:00.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:00.631 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:00.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:00.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:00.734 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:00.763 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:00.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:00.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:00.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:00.894 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:00.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:00.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:00.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:01.022 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:01.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:01.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:00 smithi099 bash[84576]: cluster 2024-09-13T15:19:59.236688+0000 osd.1 (osd.1) 48 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:01.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:00 smithi099 bash[84576]: cluster 2024-09-13T15:19:59.236688+0000 osd.1 (osd.1) 48 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:01.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:00 smithi099 bash[84576]: cluster 2024-09-13T15:20:00.000154+0000 mon.a (mon.0) 2811 : cluster [WRN] overall HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:20:01.087 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:00 smithi099 bash[84576]: cluster 2024-09-13T15:20:00.000154+0000 mon.a (mon.0) 2811 : cluster [WRN] overall HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:20:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:00 smithi099 bash[87147]: cluster 2024-09-13T15:19:59.236688+0000 osd.1 (osd.1) 48 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:01.087 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:00 smithi099 bash[87147]: cluster 2024-09-13T15:19:59.236688+0000 osd.1 (osd.1) 48 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:01.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:00 smithi099 bash[87147]: cluster 2024-09-13T15:20:00.000154+0000 mon.a (mon.0) 2811 : cluster [WRN] overall HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:20:01.088 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:00 smithi099 bash[87147]: cluster 2024-09-13T15:20:00.000154+0000 mon.a (mon.0) 2811 : cluster [WRN] overall HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:20:01.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:00 smithi177 bash[68594]: cluster 2024-09-13T15:19:59.236688+0000 osd.1 (osd.1) 48 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:01.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:00 smithi177 bash[68594]: cluster 2024-09-13T15:19:59.236688+0000 osd.1 (osd.1) 48 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:01.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:00 smithi177 bash[68594]: cluster 2024-09-13T15:20:00.000154+0000 mon.a (mon.0) 2811 : cluster [WRN] overall HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:20:01.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:00 smithi177 bash[68594]: cluster 2024-09-13T15:20:00.000154+0000 mon.a (mon.0) 2811 : cluster [WRN] overall HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-13T15:20:01.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:01.149 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:01.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:01.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:01.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:01.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:01.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:01.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:01.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:01.411 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:01.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:01.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:01.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:01.543 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:01.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:01.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:01.647 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:01.675 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:01.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:01.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:01.778 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:01.806 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:01.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:01.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:01.909 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:01.921 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:01 smithi099 bash[84576]: cluster 2024-09-13T15:20:01.268460+0000 mgr.y (mgr.26252) 1621 : cluster [DBG] pgmap v2252: 83 pgs: 83 active+clean; 372 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 79 KiB/s rd, 31 op/s 2024-09-13T15:20:01.921 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:01 smithi099 bash[84576]: cluster 2024-09-13T15:20:01.268460+0000 mgr.y (mgr.26252) 1621 : cluster [DBG] pgmap v2252: 83 pgs: 83 active+clean; 372 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 79 KiB/s rd, 31 op/s 2024-09-13T15:20:01.921 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:01 smithi099 bash[87147]: cluster 2024-09-13T15:20:01.268460+0000 mgr.y (mgr.26252) 1621 : cluster [DBG] pgmap v2252: 83 pgs: 83 active+clean; 372 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 79 KiB/s rd, 31 op/s 2024-09-13T15:20:01.922 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:01 smithi099 bash[87147]: cluster 2024-09-13T15:20:01.268460+0000 mgr.y (mgr.26252) 1621 : cluster [DBG] pgmap v2252: 83 pgs: 83 active+clean; 372 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 79 KiB/s rd, 31 op/s 2024-09-13T15:20:01.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:01.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:01.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:02.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:02.051 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:01 smithi177 bash[68594]: cluster 2024-09-13T15:20:01.268460+0000 mgr.y (mgr.26252) 1621 : cluster [DBG] pgmap v2252: 83 pgs: 83 active+clean; 372 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 79 KiB/s rd, 31 op/s 2024-09-13T15:20:02.052 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:01 smithi177 bash[68594]: cluster 2024-09-13T15:20:01.268460+0000 mgr.y (mgr.26252) 1621 : cluster [DBG] pgmap v2252: 83 pgs: 83 active+clean; 372 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 79 KiB/s rd, 31 op/s 2024-09-13T15:20:02.066 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:02.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:02.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:02.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:02.197 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:02.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:02.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:02.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:02.330 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:02.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:02.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:02.433 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:02.461 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:02.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:02.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:02.564 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:02.592 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:02.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:20:02.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:02.722 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:02.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:02.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:02.825 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:02.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:02.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:02.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:02.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:02.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:02.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:02.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:03.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:03.103 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:03.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:03.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:03.161 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T15:20:03.161 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-13T15:20:03.161 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-13T15:20:03.162 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T15:20:03.162 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T15:20:03.163 DEBUG:teuthology.orchestra.run.smithi099:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T15:20:03.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:03.208 INFO:teuthology.orchestra.run.smithi099.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T15:20:03.208 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T15:20:03.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:03.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:03.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:03.257 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T15:20:03.257 DEBUG:teuthology.orchestra.run.smithi099:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T15:20:03.302 INFO:tasks.workunit:timeout=3h 2024-09-13T15:20:03.303 INFO:tasks.workunit:cleanup=True 2024-09-13T15:20:03.303 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T15:20:03.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:03.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:03.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:03.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:03.459 INFO:tasks.workunit.client.0.smithi099.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T15:20:03.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:03.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:03.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:03.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[84576]: cluster 2024-09-13T15:20:03.269093+0000 mgr.y (mgr.26252) 1622 : cluster [DBG] pgmap v2253: 83 pgs: 83 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 82 KiB/s rd, 177 KiB/s wr, 48 op/s; 165 KiB/s, 0 objects/s recovering 2024-09-13T15:20:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[84576]: cluster 2024-09-13T15:20:03.269093+0000 mgr.y (mgr.26252) 1622 : cluster [DBG] pgmap v2253: 83 pgs: 83 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 82 KiB/s rd, 177 KiB/s wr, 48 op/s; 165 KiB/s, 0 objects/s recovering 2024-09-13T15:20:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[84576]: audit 2024-09-13T15:20:03.330559+0000 mon.a (mon.0) 2812 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[84576]: audit 2024-09-13T15:20:03.330559+0000 mon.a (mon.0) 2812 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[84576]: audit 2024-09-13T15:20:03.332306+0000 mon.a (mon.0) 2813 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:03.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[84576]: audit 2024-09-13T15:20:03.332306+0000 mon.a (mon.0) 2813 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:03.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[87147]: cluster 2024-09-13T15:20:03.269093+0000 mgr.y (mgr.26252) 1622 : cluster [DBG] pgmap v2253: 83 pgs: 83 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 82 KiB/s rd, 177 KiB/s wr, 48 op/s; 165 KiB/s, 0 objects/s recovering 2024-09-13T15:20:03.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[87147]: cluster 2024-09-13T15:20:03.269093+0000 mgr.y (mgr.26252) 1622 : cluster [DBG] pgmap v2253: 83 pgs: 83 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 82 KiB/s rd, 177 KiB/s wr, 48 op/s; 165 KiB/s, 0 objects/s recovering 2024-09-13T15:20:03.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[87147]: audit 2024-09-13T15:20:03.330559+0000 mon.a (mon.0) 2812 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:03.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[87147]: audit 2024-09-13T15:20:03.330559+0000 mon.a (mon.0) 2812 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:03.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[87147]: audit 2024-09-13T15:20:03.332306+0000 mon.a (mon.0) 2813 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:03.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:03 smithi099 bash[87147]: audit 2024-09-13T15:20:03.332306+0000 mon.a (mon.0) 2813 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:03.608 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:03 smithi177 bash[68594]: cluster 2024-09-13T15:20:03.269093+0000 mgr.y (mgr.26252) 1622 : cluster [DBG] pgmap v2253: 83 pgs: 83 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 82 KiB/s rd, 177 KiB/s wr, 48 op/s; 165 KiB/s, 0 objects/s recovering 2024-09-13T15:20:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:03 smithi177 bash[68594]: cluster 2024-09-13T15:20:03.269093+0000 mgr.y (mgr.26252) 1622 : cluster [DBG] pgmap v2253: 83 pgs: 83 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 82 KiB/s rd, 177 KiB/s wr, 48 op/s; 165 KiB/s, 0 objects/s recovering 2024-09-13T15:20:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:03 smithi177 bash[68594]: audit 2024-09-13T15:20:03.330559+0000 mon.a (mon.0) 2812 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:03 smithi177 bash[68594]: audit 2024-09-13T15:20:03.330559+0000 mon.a (mon.0) 2812 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:03 smithi177 bash[68594]: audit 2024-09-13T15:20:03.332306+0000 mon.a (mon.0) 2813 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:03 smithi177 bash[68594]: audit 2024-09-13T15:20:03.332306+0000 mon.a (mon.0) 2813 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:03.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:03.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:03.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:03.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:03.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:03.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:03.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:03.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:03.898 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:03.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:03.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:04.001 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:04.029 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:04.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:04.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:04.131 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:04.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:04.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:04.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:04.262 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:04.283 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:04.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:04.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:04.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:04.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:04.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:04.431 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:20:04.431 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:20:04.449 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:20:04.449 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:20:04.466 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:20:04.466 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:20:04.484 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:20:04.485 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:20:04.505 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:20:04.505 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:20:04.512 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:04.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:04 smithi099 bash[84576]: audit 2024-09-13T15:20:04.412698+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.99:0/60172004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:04.520 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:04 smithi099 bash[84576]: audit 2024-09-13T15:20:04.412698+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.99:0/60172004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:04.520 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:04 smithi099 bash[87147]: audit 2024-09-13T15:20:04.412698+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.99:0/60172004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:04.520 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:04 smithi099 bash[87147]: audit 2024-09-13T15:20:04.412698+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.99:0/60172004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:04.525 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:20:04.525 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:20:04.531 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:04.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:04.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:04.547 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:20:04.547 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:20:04.566 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:20:04.567 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:20:04.635 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:04.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:04.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:04.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:04.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:04.786 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:04.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:04.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:04.789 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:20:04.789 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:20:04.826 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:20:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T15:20:04.827 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:20:04.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:04 smithi177 bash[68594]: audit 2024-09-13T15:20:04.412698+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.99:0/60172004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:04.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:04 smithi177 bash[68594]: audit 2024-09-13T15:20:04.412698+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.99:0/60172004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:04.890 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:04.908 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:04.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:04.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:05.011 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:05.032 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:05.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:05.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:05.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:05.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:05.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:05.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:05.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:05.281 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:05.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:05.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:05.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:05.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:05.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:05.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:20:05.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:05.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:05.546 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: cluster 2024-09-13T15:20:03.718723+0000 osd.3 (osd.3) 29 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:05.546 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: cluster 2024-09-13T15:20:03.718723+0000 osd.3 (osd.3) 29 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:05.546 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: cluster 2024-09-13T15:20:04.465681+0000 osd.4 (osd.4) 34 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:05.547 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: cluster 2024-09-13T15:20:04.465681+0000 osd.4 (osd.4) 34 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:05.547 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: audit 2024-09-13T15:20:04.724012+0000 mon.a (mon.0) 2814 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:05.547 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: audit 2024-09-13T15:20:04.724012+0000 mon.a (mon.0) 2814 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:05.547 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: audit 2024-09-13T15:20:04.981248+0000 mon.a (mon.0) 2815 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:05.547 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: audit 2024-09-13T15:20:04.981248+0000 mon.a (mon.0) 2815 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:05.547 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: cluster 2024-09-13T15:20:05.164801+0000 osd.1 (osd.1) 49 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:05.547 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: cluster 2024-09-13T15:20:05.164801+0000 osd.1 (osd.1) 49 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:05.547 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: cluster 2024-09-13T15:20:05.269996+0000 mgr.y (mgr.26252) 1623 : cluster [DBG] pgmap v2254: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 68 KiB/s rd, 148 KiB/s wr, 40 op/s; 138 KiB/s, 0 objects/s recovering 2024-09-13T15:20:05.547 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[84576]: cluster 2024-09-13T15:20:05.269996+0000 mgr.y (mgr.26252) 1623 : cluster [DBG] pgmap v2254: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 68 KiB/s rd, 148 KiB/s wr, 40 op/s; 138 KiB/s, 0 objects/s recovering 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: cluster 2024-09-13T15:20:03.718723+0000 osd.3 (osd.3) 29 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: cluster 2024-09-13T15:20:03.718723+0000 osd.3 (osd.3) 29 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: cluster 2024-09-13T15:20:04.465681+0000 osd.4 (osd.4) 34 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: cluster 2024-09-13T15:20:04.465681+0000 osd.4 (osd.4) 34 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: audit 2024-09-13T15:20:04.724012+0000 mon.a (mon.0) 2814 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: audit 2024-09-13T15:20:04.724012+0000 mon.a (mon.0) 2814 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: audit 2024-09-13T15:20:04.981248+0000 mon.a (mon.0) 2815 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: audit 2024-09-13T15:20:04.981248+0000 mon.a (mon.0) 2815 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: cluster 2024-09-13T15:20:05.164801+0000 osd.1 (osd.1) 49 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: cluster 2024-09-13T15:20:05.164801+0000 osd.1 (osd.1) 49 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: cluster 2024-09-13T15:20:05.269996+0000 mgr.y (mgr.26252) 1623 : cluster [DBG] pgmap v2254: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 68 KiB/s rd, 148 KiB/s wr, 40 op/s; 138 KiB/s, 0 objects/s recovering 2024-09-13T15:20:05.548 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:05 smithi099 bash[87147]: cluster 2024-09-13T15:20:05.269996+0000 mgr.y (mgr.26252) 1623 : cluster [DBG] pgmap v2254: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 68 KiB/s rd, 148 KiB/s wr, 40 op/s; 138 KiB/s, 0 objects/s recovering 2024-09-13T15:20:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:05.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:05.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:05.661 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: cluster 2024-09-13T15:20:03.718723+0000 osd.3 (osd.3) 29 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:05.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: cluster 2024-09-13T15:20:03.718723+0000 osd.3 (osd.3) 29 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:05.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: cluster 2024-09-13T15:20:04.465681+0000 osd.4 (osd.4) 34 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:05.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: cluster 2024-09-13T15:20:04.465681+0000 osd.4 (osd.4) 34 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:05.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: audit 2024-09-13T15:20:04.724012+0000 mon.a (mon.0) 2814 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:05.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: audit 2024-09-13T15:20:04.724012+0000 mon.a (mon.0) 2814 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:05.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: audit 2024-09-13T15:20:04.981248+0000 mon.a (mon.0) 2815 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:05.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: audit 2024-09-13T15:20:04.981248+0000 mon.a (mon.0) 2815 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:05.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: cluster 2024-09-13T15:20:05.164801+0000 osd.1 (osd.1) 49 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:05.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: cluster 2024-09-13T15:20:05.164801+0000 osd.1 (osd.1) 49 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:05.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: cluster 2024-09-13T15:20:05.269996+0000 mgr.y (mgr.26252) 1623 : cluster [DBG] pgmap v2254: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 68 KiB/s rd, 148 KiB/s wr, 40 op/s; 138 KiB/s, 0 objects/s recovering 2024-09-13T15:20:05.663 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:05 smithi177 bash[68594]: cluster 2024-09-13T15:20:05.269996+0000 mgr.y (mgr.26252) 1623 : cluster [DBG] pgmap v2254: 83 pgs: 1 active+clean+scrubbing, 82 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 68 KiB/s rd, 148 KiB/s wr, 40 op/s; 138 KiB/s, 0 objects/s recovering 2024-09-13T15:20:05.677 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:05.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:05.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:05.734 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:20:05.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:05.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:05.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:05.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:05.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:05.939 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:05.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:05.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:06.043 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:06.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:06.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:06.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:06.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:06.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:06.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:06.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:06.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:06.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:06.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:20:06.436 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:06.464 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:06.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:06.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:06.568 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:06.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:06.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:06.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:06.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:06.709 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[87147]: cluster 2024-09-13T15:20:05.722294+0000 mon.a (mon.0) 2816 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:20:06.709 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[87147]: cluster 2024-09-13T15:20:05.722294+0000 mon.a (mon.0) 2816 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:20:06.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[87147]: audit 2024-09-13T15:20:05.733565+0000 mon.a (mon.0) 2817 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:06.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[87147]: audit 2024-09-13T15:20:05.733565+0000 mon.a (mon.0) 2817 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:06.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[87147]: cluster 2024-09-13T15:20:05.743257+0000 mon.a (mon.0) 2818 : cluster [DBG] osdmap e1148: 8 total, 8 up, 7 in 2024-09-13T15:20:06.710 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[87147]: cluster 2024-09-13T15:20:05.743257+0000 mon.a (mon.0) 2818 : cluster [DBG] osdmap e1148: 8 total, 8 up, 7 in 2024-09-13T15:20:06.710 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[84576]: cluster 2024-09-13T15:20:05.722294+0000 mon.a (mon.0) 2816 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:20:06.710 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[84576]: cluster 2024-09-13T15:20:05.722294+0000 mon.a (mon.0) 2816 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:20:06.710 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[84576]: audit 2024-09-13T15:20:05.733565+0000 mon.a (mon.0) 2817 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:06.710 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[84576]: audit 2024-09-13T15:20:05.733565+0000 mon.a (mon.0) 2817 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:06.710 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[84576]: cluster 2024-09-13T15:20:05.743257+0000 mon.a (mon.0) 2818 : cluster [DBG] osdmap e1148: 8 total, 8 up, 7 in 2024-09-13T15:20:06.710 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:06 smithi099 bash[84576]: cluster 2024-09-13T15:20:05.743257+0000 mon.a (mon.0) 2818 : cluster [DBG] osdmap e1148: 8 total, 8 up, 7 in 2024-09-13T15:20:06.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:06.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:06.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:06.833 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:06.857 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:06.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:06.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:06 smithi177 bash[68594]: cluster 2024-09-13T15:20:05.722294+0000 mon.a (mon.0) 2816 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:20:06.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:06 smithi177 bash[68594]: cluster 2024-09-13T15:20:05.722294+0000 mon.a (mon.0) 2816 : cluster [WRN] Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:20:06.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:06 smithi177 bash[68594]: audit 2024-09-13T15:20:05.733565+0000 mon.a (mon.0) 2817 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:06.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:06 smithi177 bash[68594]: audit 2024-09-13T15:20:05.733565+0000 mon.a (mon.0) 2817 : audit [INF] from='client.? 172.21.15.99:0/2864721552' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:06.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:06 smithi177 bash[68594]: cluster 2024-09-13T15:20:05.743257+0000 mon.a (mon.0) 2818 : cluster [DBG] osdmap e1148: 8 total, 8 up, 7 in 2024-09-13T15:20:06.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:06 smithi177 bash[68594]: cluster 2024-09-13T15:20:05.743257+0000 mon.a (mon.0) 2818 : cluster [DBG] osdmap e1148: 8 total, 8 up, 7 in 2024-09-13T15:20:06.960 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:06.985 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:06.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:07.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:07.114 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:07.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:07.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:07.217 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:07.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:07.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:07.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:07.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:07.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:07.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:07.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:07.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:07.494 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:20:07] "GET /metrics HTTP/1.1" 200 34375 "" "Prometheus/2.51.0" 2024-09-13T15:20:07.506 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:07.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:07.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:07.609 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:07.637 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:07.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:07.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:07.741 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:07.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[87147]: cluster 2024-09-13T15:20:06.721980+0000 osd.5 (osd.5) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:20:07.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[87147]: cluster 2024-09-13T15:20:06.721980+0000 osd.5 (osd.5) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:20:07.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[87147]: cluster 2024-09-13T15:20:06.725817+0000 osd.5 (osd.5) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:20:07.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[87147]: cluster 2024-09-13T15:20:06.725817+0000 osd.5 (osd.5) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:20:07.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[87147]: cluster 2024-09-13T15:20:07.270745+0000 mgr.y (mgr.26252) 1624 : cluster [DBG] pgmap v2256: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 63 KiB/s rd, 137 KiB/s wr, 37 op/s; 127 KiB/s, 0 objects/s recovering 2024-09-13T15:20:07.751 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[87147]: cluster 2024-09-13T15:20:07.270745+0000 mgr.y (mgr.26252) 1624 : cluster [DBG] pgmap v2256: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 63 KiB/s rd, 137 KiB/s wr, 37 op/s; 127 KiB/s, 0 objects/s recovering 2024-09-13T15:20:07.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[84576]: cluster 2024-09-13T15:20:06.721980+0000 osd.5 (osd.5) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:20:07.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[84576]: cluster 2024-09-13T15:20:06.721980+0000 osd.5 (osd.5) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:20:07.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[84576]: cluster 2024-09-13T15:20:06.725817+0000 osd.5 (osd.5) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:20:07.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[84576]: cluster 2024-09-13T15:20:06.725817+0000 osd.5 (osd.5) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:20:07.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[84576]: cluster 2024-09-13T15:20:07.270745+0000 mgr.y (mgr.26252) 1624 : cluster [DBG] pgmap v2256: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 63 KiB/s rd, 137 KiB/s wr, 37 op/s; 127 KiB/s, 0 objects/s recovering 2024-09-13T15:20:07.752 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:07 smithi099 bash[84576]: cluster 2024-09-13T15:20:07.270745+0000 mgr.y (mgr.26252) 1624 : cluster [DBG] pgmap v2256: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 63 KiB/s rd, 137 KiB/s wr, 37 op/s; 127 KiB/s, 0 objects/s recovering 2024-09-13T15:20:07.765 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:07.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:07.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:07 smithi177 bash[68594]: cluster 2024-09-13T15:20:06.721980+0000 osd.5 (osd.5) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:20:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:07 smithi177 bash[68594]: cluster 2024-09-13T15:20:06.721980+0000 osd.5 (osd.5) 23 : cluster [DBG] 1.1 scrub starts 2024-09-13T15:20:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:07 smithi177 bash[68594]: cluster 2024-09-13T15:20:06.725817+0000 osd.5 (osd.5) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:20:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:07 smithi177 bash[68594]: cluster 2024-09-13T15:20:06.725817+0000 osd.5 (osd.5) 24 : cluster [DBG] 1.1 scrub ok 2024-09-13T15:20:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:07 smithi177 bash[68594]: cluster 2024-09-13T15:20:07.270745+0000 mgr.y (mgr.26252) 1624 : cluster [DBG] pgmap v2256: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 63 KiB/s rd, 137 KiB/s wr, 37 op/s; 127 KiB/s, 0 objects/s recovering 2024-09-13T15:20:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:07 smithi177 bash[68594]: cluster 2024-09-13T15:20:07.270745+0000 mgr.y (mgr.26252) 1624 : cluster [DBG] pgmap v2256: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 63 KiB/s rd, 137 KiB/s wr, 37 op/s; 127 KiB/s, 0 objects/s recovering 2024-09-13T15:20:07.869 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:07.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:07.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:07.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:07.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:08.022 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:08.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:08.126 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:08.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:08.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:08.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:08.287 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:08.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:08.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:08.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:08.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:08.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:08.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:08.518 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:08.548 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:08.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:08.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:08.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:08.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:08.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:08.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:08.779 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:08.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:08.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:08.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:08.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:08.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:08.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:08.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:09.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:09.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:09.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:09.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:09.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:09.182 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:08 smithi099 bash[87147]: cluster 2024-09-13T15:20:08.701665+0000 osd.5 (osd.5) 25 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:20:09.182 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:08 smithi099 bash[87147]: cluster 2024-09-13T15:20:08.701665+0000 osd.5 (osd.5) 25 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:20:09.182 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:08 smithi099 bash[87147]: cluster 2024-09-13T15:20:08.705937+0000 osd.5 (osd.5) 26 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:20:09.182 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:08 smithi099 bash[87147]: cluster 2024-09-13T15:20:08.705937+0000 osd.5 (osd.5) 26 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:20:09.182 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:08 smithi099 bash[84576]: cluster 2024-09-13T15:20:08.701665+0000 osd.5 (osd.5) 25 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:20:09.182 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:08 smithi099 bash[84576]: cluster 2024-09-13T15:20:08.701665+0000 osd.5 (osd.5) 25 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:20:09.183 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:08 smithi099 bash[84576]: cluster 2024-09-13T15:20:08.705937+0000 osd.5 (osd.5) 26 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:20:09.183 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:08 smithi099 bash[84576]: cluster 2024-09-13T15:20:08.705937+0000 osd.5 (osd.5) 26 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:20:09.198 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:09.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:09.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:09.302 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:09.314 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:08 smithi177 bash[68594]: cluster 2024-09-13T15:20:08.701665+0000 osd.5 (osd.5) 25 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:20:09.314 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:08 smithi177 bash[68594]: cluster 2024-09-13T15:20:08.701665+0000 osd.5 (osd.5) 25 : cluster [DBG] 1.8 scrub starts 2024-09-13T15:20:09.314 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:08 smithi177 bash[68594]: cluster 2024-09-13T15:20:08.705937+0000 osd.5 (osd.5) 26 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:20:09.314 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:08 smithi177 bash[68594]: cluster 2024-09-13T15:20:08.705937+0000 osd.5 (osd.5) 26 : cluster [DBG] 1.8 scrub ok 2024-09-13T15:20:09.330 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:09.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:09.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:09.433 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:09.458 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:09.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:09.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:09.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:09.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:09.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:09.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:09.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:09.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:09.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:09.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:09.836 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:09.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:09.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:09.938 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:09.958 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:09.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.271324+0000 mgr.y (mgr.26252) 1625 : cluster [DBG] pgmap v2257: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 41 KiB/s rd, 89 KiB/s wr, 24 op/s; 89 KiB/s, 0 objects/s recovering 2024-09-13T15:20:09.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.271324+0000 mgr.y (mgr.26252) 1625 : cluster [DBG] pgmap v2257: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 41 KiB/s rd, 89 KiB/s wr, 24 op/s; 89 KiB/s, 0 objects/s recovering 2024-09-13T15:20:09.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.657518+0000 osd.5 (osd.5) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:20:09.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.657518+0000 osd.5 (osd.5) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:20:09.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.661068+0000 osd.5 (osd.5) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:20:09.959 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.661068+0000 osd.5 (osd.5) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:20:09.960 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.271324+0000 mgr.y (mgr.26252) 1625 : cluster [DBG] pgmap v2257: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 41 KiB/s rd, 89 KiB/s wr, 24 op/s; 89 KiB/s, 0 objects/s recovering 2024-09-13T15:20:09.960 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.271324+0000 mgr.y (mgr.26252) 1625 : cluster [DBG] pgmap v2257: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 41 KiB/s rd, 89 KiB/s wr, 24 op/s; 89 KiB/s, 0 objects/s recovering 2024-09-13T15:20:09.960 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.657518+0000 osd.5 (osd.5) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:20:09.960 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.657518+0000 osd.5 (osd.5) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:20:09.960 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.661068+0000 osd.5 (osd.5) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:20:09.960 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:09 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.661068+0000 osd.5 (osd.5) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:20:09.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:09.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:10.032 DEBUG:teuthology.orchestra.run.smithi099:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T15:20:10.062 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:10.077 INFO:tasks.workunit.client.0.smithi099.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T15:20:10.080 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T15:20:10.080 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T15:20:10.095 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:10.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:10.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:10.188 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T15:20:10.194 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T15:20:10.197 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:10.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:09 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.271324+0000 mgr.y (mgr.26252) 1625 : cluster [DBG] pgmap v2257: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 41 KiB/s rd, 89 KiB/s wr, 24 op/s; 89 KiB/s, 0 objects/s recovering 2024-09-13T15:20:10.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:09 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.271324+0000 mgr.y (mgr.26252) 1625 : cluster [DBG] pgmap v2257: 83 pgs: 2 active+clean+scrubbing, 81 active+clean; 425 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 41 KiB/s rd, 89 KiB/s wr, 24 op/s; 89 KiB/s, 0 objects/s recovering 2024-09-13T15:20:10.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:09 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.657518+0000 osd.5 (osd.5) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:20:10.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:09 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.657518+0000 osd.5 (osd.5) 27 : cluster [DBG] 1.a scrub starts 2024-09-13T15:20:10.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:09 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.661068+0000 osd.5 (osd.5) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:20:10.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:09 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.661068+0000 osd.5 (osd.5) 28 : cluster [DBG] 1.a scrub ok 2024-09-13T15:20:10.220 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:10.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:10.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:10.250 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T15:20:10.290 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T15:20:10.291 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T15:20:10.292 INFO:tasks.workunit.client.0.smithi099.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T15:20:10.323 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:10.330 INFO:tasks.workunit.client.0.smithi099.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T15:20:10.333 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-09-13T15:20:10.333 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T15:20:10.341 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:10.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:10.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:10.354 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-09-13T15:20:10.355 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-13T15:20:10.355 DEBUG:teuthology.orchestra.run.smithi099:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-13T15:20:10.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:10.460 INFO:tasks.workunit.client.0.smithi099.stdout:Running main() from gmock_main.cc 2024-09-13T15:20:10.460 INFO:tasks.workunit.client.0.smithi099.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-13T15:20:10.460 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] Global test environment set-up. 2024-09-13T15:20:10.460 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 70 tests from TestClsRbd 2024-09-13T15:20:10.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:10.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:10.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:10.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:10.575 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:20:10.590 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:10.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:10.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:10.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[61439]: ts=2024-09-13T15:20:10.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:20:10.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:10.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:10.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:10.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:10.819 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:10.839 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:10.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:10.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:10.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:10.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:10.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:10.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:11.071 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:11.083 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.520979+0000 osd.4 (osd.4) 35 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:11.083 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.520979+0000 osd.4 (osd.4) 35 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.523736+0000 osd.4 (osd.4) 36 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.523736+0000 osd.4 (osd.4) 36 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.713095+0000 osd.3 (osd.3) 30 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.713095+0000 osd.3 (osd.3) 30 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.715236+0000 osd.3 (osd.3) 31 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:09.715236+0000 osd.3 (osd.3) 31 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:10.179696+0000 osd.1 (osd.1) 50 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:10.179696+0000 osd.1 (osd.1) 50 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:10.181557+0000 osd.1 (osd.1) 51 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:10.181557+0000 osd.1 (osd.1) 51 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:10.558322+0000 osd.6 (osd.6) 35 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:10.558322+0000 osd.6 (osd.6) 35 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:20:11.084 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:10.562413+0000 osd.6 (osd.6) 36 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[84576]: cluster 2024-09-13T15:20:10.562413+0000 osd.6 (osd.6) 36 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.520979+0000 osd.4 (osd.4) 35 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.520979+0000 osd.4 (osd.4) 35 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.523736+0000 osd.4 (osd.4) 36 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.523736+0000 osd.4 (osd.4) 36 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.713095+0000 osd.3 (osd.3) 30 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.713095+0000 osd.3 (osd.3) 30 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.715236+0000 osd.3 (osd.3) 31 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:09.715236+0000 osd.3 (osd.3) 31 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:10.179696+0000 osd.1 (osd.1) 50 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:11.085 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:10.179696+0000 osd.1 (osd.1) 50 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:11.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:10.181557+0000 osd.1 (osd.1) 51 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:20:11.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:10.181557+0000 osd.1 (osd.1) 51 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:20:11.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:10.558322+0000 osd.6 (osd.6) 35 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:20:11.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:10.558322+0000 osd.6 (osd.6) 35 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:20:11.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:10.562413+0000 osd.6 (osd.6) 36 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:20:11.086 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:10 smithi099 bash[87147]: cluster 2024-09-13T15:20:10.562413+0000 osd.6 (osd.6) 36 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:20:11.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:11.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:11.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:11.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.520979+0000 osd.4 (osd.4) 35 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.520979+0000 osd.4 (osd.4) 35 : cluster [DBG] 1.5 scrub starts 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.523736+0000 osd.4 (osd.4) 36 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.523736+0000 osd.4 (osd.4) 36 : cluster [DBG] 1.5 scrub ok 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.713095+0000 osd.3 (osd.3) 30 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.713095+0000 osd.3 (osd.3) 30 : cluster [DBG] 1.4 scrub starts 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.715236+0000 osd.3 (osd.3) 31 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:09.715236+0000 osd.3 (osd.3) 31 : cluster [DBG] 1.4 scrub ok 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:10.179696+0000 osd.1 (osd.1) 50 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:10.179696+0000 osd.1 (osd.1) 50 : cluster [DBG] 1.3 scrub starts 2024-09-13T15:20:11.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:10.181557+0000 osd.1 (osd.1) 51 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:20:11.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:10.181557+0000 osd.1 (osd.1) 51 : cluster [DBG] 1.3 scrub ok 2024-09-13T15:20:11.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:10.558322+0000 osd.6 (osd.6) 35 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:20:11.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:10.558322+0000 osd.6 (osd.6) 35 : cluster [DBG] 1.6 scrub starts 2024-09-13T15:20:11.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:10.562413+0000 osd.6 (osd.6) 36 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:20:11.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:10 smithi177 bash[68594]: cluster 2024-09-13T15:20:10.562413+0000 osd.6 (osd.6) 36 : cluster [DBG] 1.6 scrub ok 2024-09-13T15:20:11.230 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:11.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:11.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:11.360 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:11.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:11.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:11.463 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:11.488 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:11.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:11.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:11.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:11.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:11.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:11.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:11.721 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:11.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:11.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:11.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:11.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:11.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:11.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:11.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:11.937 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-13T15:20:11.970 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.get_all_features (33 ms) 2024-09-13T15:20:11.970 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.copyup 2024-09-13T15:20:11.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:12.005 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:12.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:12.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:12.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:12.134 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:12.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:12.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:12.189 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.copyup (218 ms) 2024-09-13T15:20:12.189 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-13T15:20:12.233 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.sparse_copyup (45 ms) 2024-09-13T15:20:12.233 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-13T15:20:12.236 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:12.247 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.get_and_set_id (14 ms) 2024-09-13T15:20:12.247 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-13T15:20:12.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[84576]: cluster 2024-09-13T15:20:10.923922+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e1149: 8 total, 8 up, 7 in 2024-09-13T15:20:12.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[84576]: cluster 2024-09-13T15:20:10.923922+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e1149: 8 total, 8 up, 7 in 2024-09-13T15:20:12.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[84576]: audit 2024-09-13T15:20:10.930941+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.99:0/2207394421' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[84576]: audit 2024-09-13T15:20:10.930941+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.99:0/2207394421' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[84576]: audit 2024-09-13T15:20:10.934959+0000 mon.a (mon.0) 2820 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.249 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[84576]: audit 2024-09-13T15:20:10.934959+0000 mon.a (mon.0) 2820 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.250 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[84576]: cluster 2024-09-13T15:20:11.272040+0000 mgr.y (mgr.26252) 1626 : cluster [DBG] pgmap v2259: 115 pgs: 32 unknown, 83 active+clean; 739 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:12.251 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[84576]: cluster 2024-09-13T15:20:11.272040+0000 mgr.y (mgr.26252) 1626 : cluster [DBG] pgmap v2259: 115 pgs: 32 unknown, 83 active+clean; 739 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:12.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[87147]: cluster 2024-09-13T15:20:10.923922+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e1149: 8 total, 8 up, 7 in 2024-09-13T15:20:12.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[87147]: cluster 2024-09-13T15:20:10.923922+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e1149: 8 total, 8 up, 7 in 2024-09-13T15:20:12.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[87147]: audit 2024-09-13T15:20:10.930941+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.99:0/2207394421' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[87147]: audit 2024-09-13T15:20:10.930941+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.99:0/2207394421' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[87147]: audit 2024-09-13T15:20:10.934959+0000 mon.a (mon.0) 2820 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[87147]: audit 2024-09-13T15:20:10.934959+0000 mon.a (mon.0) 2820 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[87147]: cluster 2024-09-13T15:20:11.272040+0000 mgr.y (mgr.26252) 1626 : cluster [DBG] pgmap v2259: 115 pgs: 32 unknown, 83 active+clean; 739 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:12.251 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:11 smithi099 bash[87147]: cluster 2024-09-13T15:20:11.272040+0000 mgr.y (mgr.26252) 1626 : cluster [DBG] pgmap v2259: 115 pgs: 32 unknown, 83 active+clean; 739 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:12.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:20:12.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:12.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:12.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:12.273 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.add_remove_child (26 ms) 2024-09-13T15:20:12.273 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-13T15:20:12.315 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.directory_methods (42 ms) 2024-09-13T15:20:12.315 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.create 2024-09-13T15:20:12.340 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.create (25 ms) 2024-09-13T15:20:12.340 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.get_features 2024-09-13T15:20:12.344 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.get_features (4 ms) 2024-09-13T15:20:12.344 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-13T15:20:12.349 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.get_object_prefix (5 ms) 2024-09-13T15:20:12.349 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-13T15:20:12.351 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-09-13T15:20:12.351 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-13T15:20:12.353 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-09-13T15:20:12.353 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-13T15:20:12.356 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.get_modify_timestamp (4 ms) 2024-09-13T15:20:12.357 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-13T15:20:12.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:11 smithi177 bash[68594]: cluster 2024-09-13T15:20:10.923922+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e1149: 8 total, 8 up, 7 in 2024-09-13T15:20:12.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:11 smithi177 bash[68594]: cluster 2024-09-13T15:20:10.923922+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e1149: 8 total, 8 up, 7 in 2024-09-13T15:20:12.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:11 smithi177 bash[68594]: audit 2024-09-13T15:20:10.930941+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.99:0/2207394421' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:11 smithi177 bash[68594]: audit 2024-09-13T15:20:10.930941+0000 mon.c (mon.2) 124 : audit [INF] from='client.? 172.21.15.99:0/2207394421' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:11 smithi177 bash[68594]: audit 2024-09-13T15:20:10.934959+0000 mon.a (mon.0) 2820 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:11 smithi177 bash[68594]: audit 2024-09-13T15:20:10.934959+0000 mon.a (mon.0) 2820 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:20:12.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:11 smithi177 bash[68594]: cluster 2024-09-13T15:20:11.272040+0000 mgr.y (mgr.26252) 1626 : cluster [DBG] pgmap v2259: 115 pgs: 32 unknown, 83 active+clean; 739 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:12.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:11 smithi177 bash[68594]: cluster 2024-09-13T15:20:11.272040+0000 mgr.y (mgr.26252) 1626 : cluster [DBG] pgmap v2259: 115 pgs: 32 unknown, 83 active+clean; 739 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:12.365 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:12.366 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.get_data_pool (8 ms) 2024-09-13T15:20:12.366 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.get_size 2024-09-13T15:20:12.375 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.get_size (10 ms) 2024-09-13T15:20:12.375 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.set_size 2024-09-13T15:20:12.383 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.set_size (8 ms) 2024-09-13T15:20:12.383 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-13T15:20:12.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:12.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:12.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:12.407 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.protection_status (24 ms) 2024-09-13T15:20:12.407 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-13T15:20:12.439 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.snapshot_limits (32 ms) 2024-09-13T15:20:12.439 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-13T15:20:12.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:12.504 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.parents_v1 (64 ms) 2024-09-13T15:20:12.504 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-13T15:20:12.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:12.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:12.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:12.576 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.parents_v2 (73 ms) 2024-09-13T15:20:12.576 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-13T15:20:12.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:12.636 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.snapshots (60 ms) 2024-09-13T15:20:12.636 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-13T15:20:12.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:12.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:12.645 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.snapid_race (9 ms) 2024-09-13T15:20:12.645 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-13T15:20:12.663 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.stripingv2 (18 ms) 2024-09-13T15:20:12.663 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-13T15:20:12.667 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2024-09-13T15:20:12.667 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-13T15:20:12.679 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.object_map_resize (12 ms) 2024-09-13T15:20:12.680 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-13T15:20:12.692 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.object_map_update (13 ms) 2024-09-13T15:20:12.692 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-13T15:20:12.692 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2024-09-13T15:20:12.692 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-13T15:20:12.705 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.object_map_snap_add (13 ms) 2024-09-13T15:20:12.706 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-13T15:20:12.721 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.object_map_snap_remove (15 ms) 2024-09-13T15:20:12.722 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.flags 2024-09-13T15:20:12.733 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.flags (12 ms) 2024-09-13T15:20:12.734 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.metadata 2024-09-13T15:20:12.746 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:12.770 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:12.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:12.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:12.801 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.metadata (67 ms) 2024-09-13T15:20:12.801 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.set_features 2024-09-13T15:20:12.820 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.set_features (18 ms) 2024-09-13T15:20:12.820 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.mirror 2024-09-13T15:20:12.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:12.891 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.mirror (72 ms) 2024-09-13T15:20:12.891 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-13T15:20:12.902 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:12.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:12.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:12.916 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.mirror_image (25 ms) 2024-09-13T15:20:12.917 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-13T15:20:13.005 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:13.030 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:13.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:13.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:13.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[84576]: cluster 2024-09-13T15:20:11.920832+0000 mon.a (mon.0) 2821 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:20:13.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[84576]: cluster 2024-09-13T15:20:11.920832+0000 mon.a (mon.0) 2821 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:20:13.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[84576]: audit 2024-09-13T15:20:11.928710+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:20:13.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[84576]: audit 2024-09-13T15:20:11.928710+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:20:13.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[84576]: cluster 2024-09-13T15:20:11.934560+0000 mon.a (mon.0) 2823 : cluster [DBG] osdmap e1150: 8 total, 8 up, 7 in 2024-09-13T15:20:13.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[84576]: cluster 2024-09-13T15:20:11.934560+0000 mon.a (mon.0) 2823 : cluster [DBG] osdmap e1150: 8 total, 8 up, 7 in 2024-09-13T15:20:13.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[87147]: cluster 2024-09-13T15:20:11.920832+0000 mon.a (mon.0) 2821 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:20:13.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[87147]: cluster 2024-09-13T15:20:11.920832+0000 mon.a (mon.0) 2821 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:20:13.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[87147]: audit 2024-09-13T15:20:11.928710+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:20:13.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[87147]: audit 2024-09-13T15:20:11.928710+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:20:13.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[87147]: cluster 2024-09-13T15:20:11.934560+0000 mon.a (mon.0) 2823 : cluster [DBG] osdmap e1150: 8 total, 8 up, 7 in 2024-09-13T15:20:13.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:12 smithi099 bash[87147]: cluster 2024-09-13T15:20:11.934560+0000 mon.a (mon.0) 2823 : cluster [DBG] osdmap e1150: 8 total, 8 up, 7 in 2024-09-13T15:20:13.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:13.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:13.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:13.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:13.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:13.274 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:12 smithi177 bash[68594]: cluster 2024-09-13T15:20:11.920832+0000 mon.a (mon.0) 2821 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:20:13.274 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:12 smithi177 bash[68594]: cluster 2024-09-13T15:20:11.920832+0000 mon.a (mon.0) 2821 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:20:13.274 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:12 smithi177 bash[68594]: audit 2024-09-13T15:20:11.928710+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:20:13.274 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:12 smithi177 bash[68594]: audit 2024-09-13T15:20:11.928710+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi099-249932-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:20:13.275 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:12 smithi177 bash[68594]: cluster 2024-09-13T15:20:11.934560+0000 mon.a (mon.0) 2823 : cluster [DBG] osdmap e1150: 8 total, 8 up, 7 in 2024-09-13T15:20:13.275 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:12 smithi177 bash[68594]: cluster 2024-09-13T15:20:11.934560+0000 mon.a (mon.0) 2823 : cluster [DBG] osdmap e1150: 8 total, 8 up, 7 in 2024-09-13T15:20:13.290 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:13.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:13.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:13.394 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:13.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:13.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:13.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:13.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:13.551 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:13.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:13.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:13.683 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:13.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:13.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:13.787 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:13.813 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:13.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:13.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:13.917 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:13.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:13.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:13.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:14.065 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:14.076 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:13 smithi177 bash[68594]: cluster 2024-09-13T15:20:12.948117+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e1151: 8 total, 8 up, 7 in 2024-09-13T15:20:14.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:13 smithi177 bash[68594]: cluster 2024-09-13T15:20:12.948117+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e1151: 8 total, 8 up, 7 in 2024-09-13T15:20:14.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:13 smithi177 bash[68594]: cluster 2024-09-13T15:20:13.272655+0000 mgr.y (mgr.26252) 1627 : cluster [DBG] pgmap v2262: 115 pgs: 5 creating+peering, 27 unknown, 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:14.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:13 smithi177 bash[68594]: cluster 2024-09-13T15:20:13.272655+0000 mgr.y (mgr.26252) 1627 : cluster [DBG] pgmap v2262: 115 pgs: 5 creating+peering, 27 unknown, 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:14.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:14.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:14.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:20:14.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:14.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:14.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:14.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:13 smithi099 bash[87147]: cluster 2024-09-13T15:20:12.948117+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e1151: 8 total, 8 up, 7 in 2024-09-13T15:20:14.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:13 smithi099 bash[87147]: cluster 2024-09-13T15:20:12.948117+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e1151: 8 total, 8 up, 7 in 2024-09-13T15:20:14.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:13 smithi099 bash[87147]: cluster 2024-09-13T15:20:13.272655+0000 mgr.y (mgr.26252) 1627 : cluster [DBG] pgmap v2262: 115 pgs: 5 creating+peering, 27 unknown, 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:14.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:13 smithi099 bash[87147]: cluster 2024-09-13T15:20:13.272655+0000 mgr.y (mgr.26252) 1627 : cluster [DBG] pgmap v2262: 115 pgs: 5 creating+peering, 27 unknown, 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:14.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:13 smithi099 bash[84576]: cluster 2024-09-13T15:20:12.948117+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e1151: 8 total, 8 up, 7 in 2024-09-13T15:20:14.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:13 smithi099 bash[84576]: cluster 2024-09-13T15:20:12.948117+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e1151: 8 total, 8 up, 7 in 2024-09-13T15:20:14.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:13 smithi099 bash[84576]: cluster 2024-09-13T15:20:13.272655+0000 mgr.y (mgr.26252) 1627 : cluster [DBG] pgmap v2262: 115 pgs: 5 creating+peering, 27 unknown, 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:14.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:13 smithi099 bash[84576]: cluster 2024-09-13T15:20:13.272655+0000 mgr.y (mgr.26252) 1627 : cluster [DBG] pgmap v2262: 115 pgs: 5 creating+peering, 27 unknown, 83 active+clean; 687 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:14.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:14.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:14.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:14.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:14.454 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:14.476 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:14.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:14.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:14.604 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:14.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:14.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:14.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:14.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:14.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:14.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:14.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:14.861 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:14.862 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:20:14 smithi177 bash[60145]: logger=cleanup t=2024-09-13T15:20:14.585346383Z level=info msg="Completed cleanup jobs" duration=143.006226ms 2024-09-13T15:20:14.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:14.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:14.931 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:20:14.932 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:20:14.964 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:14.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:14.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:14.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:15.094 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:15.103 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[84576]: cluster 2024-09-13T15:20:13.942378+0000 mon.a (mon.0) 2825 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:20:15.103 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[84576]: cluster 2024-09-13T15:20:13.942378+0000 mon.a (mon.0) 2825 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:20:15.103 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[84576]: cluster 2024-09-13T15:20:14.513789+0000 osd.6 (osd.6) 37 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:20:15.103 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[84576]: cluster 2024-09-13T15:20:14.513789+0000 osd.6 (osd.6) 37 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:20:15.103 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[84576]: cluster 2024-09-13T15:20:14.516161+0000 osd.6 (osd.6) 38 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:20:15.104 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[84576]: cluster 2024-09-13T15:20:14.516161+0000 osd.6 (osd.6) 38 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:20:15.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[87147]: cluster 2024-09-13T15:20:13.942378+0000 mon.a (mon.0) 2825 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:20:15.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[87147]: cluster 2024-09-13T15:20:13.942378+0000 mon.a (mon.0) 2825 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:20:15.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[87147]: cluster 2024-09-13T15:20:14.513789+0000 osd.6 (osd.6) 37 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:20:15.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[87147]: cluster 2024-09-13T15:20:14.513789+0000 osd.6 (osd.6) 37 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:20:15.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[87147]: cluster 2024-09-13T15:20:14.516161+0000 osd.6 (osd.6) 38 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:20:15.104 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:14 smithi099 bash[87147]: cluster 2024-09-13T15:20:14.516161+0000 osd.6 (osd.6) 38 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:20:15.117 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:15.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:15.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:15.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:15.238 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 5, 7, 4, 1, 6] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-13T15:20:15.238 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:20:15.238 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T15:20:15.238 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:20:15.242 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:15.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:15.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:15.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:14 smithi177 bash[68594]: cluster 2024-09-13T15:20:13.942378+0000 mon.a (mon.0) 2825 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:20:15.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:14 smithi177 bash[68594]: cluster 2024-09-13T15:20:13.942378+0000 mon.a (mon.0) 2825 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:20:15.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:14 smithi177 bash[68594]: cluster 2024-09-13T15:20:14.513789+0000 osd.6 (osd.6) 37 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:20:15.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:14 smithi177 bash[68594]: cluster 2024-09-13T15:20:14.513789+0000 osd.6 (osd.6) 37 : cluster [DBG] 1.7 scrub starts 2024-09-13T15:20:15.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:14 smithi177 bash[68594]: cluster 2024-09-13T15:20:14.516161+0000 osd.6 (osd.6) 38 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:20:15.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:14 smithi177 bash[68594]: cluster 2024-09-13T15:20:14.516161+0000 osd.6 (osd.6) 38 : cluster [DBG] 1.7 scrub ok 2024-09-13T15:20:15.368 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:15.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:15.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:15.471 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:15.491 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:15.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:15.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:15.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:15.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:15.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:15.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:15.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:15.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:15.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:15.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:15.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:15.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:15.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:15.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:15.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:16.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:16.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:16.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:16.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:16.124 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:15 smithi177 bash[68594]: cluster 2024-09-13T15:20:15.273239+0000 mgr.y (mgr.26252) 1628 : cluster [DBG] pgmap v2263: 115 pgs: 5 creating+peering, 11 unknown, 99 active+clean; 634 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 734 KiB/s wr, 560 op/s 2024-09-13T15:20:16.124 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:15 smithi177 bash[68594]: cluster 2024-09-13T15:20:15.273239+0000 mgr.y (mgr.26252) 1628 : cluster [DBG] pgmap v2263: 115 pgs: 5 creating+peering, 11 unknown, 99 active+clean; 634 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 734 KiB/s wr, 560 op/s 2024-09-13T15:20:16.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:16.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:16.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:16.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:16.268 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:16.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:16.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:16.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:15 smithi099 bash[84576]: cluster 2024-09-13T15:20:15.273239+0000 mgr.y (mgr.26252) 1628 : cluster [DBG] pgmap v2263: 115 pgs: 5 creating+peering, 11 unknown, 99 active+clean; 634 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 734 KiB/s wr, 560 op/s 2024-09-13T15:20:16.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:16 smithi099 bash[84576]: cluster 2024-09-13T15:20:15.273239+0000 mgr.y (mgr.26252) 1628 : cluster [DBG] pgmap v2263: 115 pgs: 5 creating+peering, 11 unknown, 99 active+clean; 634 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 734 KiB/s wr, 560 op/s 2024-09-13T15:20:16.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:16 smithi099 bash[87147]: cluster 2024-09-13T15:20:15.273239+0000 mgr.y (mgr.26252) 1628 : cluster [DBG] pgmap v2263: 115 pgs: 5 creating+peering, 11 unknown, 99 active+clean; 634 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 734 KiB/s wr, 560 op/s 2024-09-13T15:20:16.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:16 smithi099 bash[87147]: cluster 2024-09-13T15:20:15.273239+0000 mgr.y (mgr.26252) 1628 : cluster [DBG] pgmap v2263: 115 pgs: 5 creating+peering, 11 unknown, 99 active+clean; 634 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 734 KiB/s wr, 560 op/s 2024-09-13T15:20:16.371 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:16.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:16.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:16.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:16.501 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:16.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:16.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:16.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:16.631 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:16.655 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:16.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:16.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:16.759 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:16.787 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:16.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:16.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:20:16.890 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:16.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:16.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:16.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:16.959 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:20:17.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:17.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:17.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:17.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:17.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:17.169 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:17.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:17.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:17.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:17.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:17.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:20:17.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:20:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:20:17] "GET /metrics HTTP/1.1" 200 34375 "" "Prometheus/2.51.0" 2024-09-13T15:20:17.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:17.424 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:17.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:17.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:17.444 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.mirror_image_status (4528 ms) 2024-09-13T15:20:17.444 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-13T15:20:17.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:17.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:17.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:17.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:17.580 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.mirror_image_map (136 ms) 2024-09-13T15:20:17.580 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-13T15:20:17.603 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.mirror_instances (24 ms) 2024-09-13T15:20:17.604 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-13T15:20:17.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:17 smithi177 bash[68594]: cluster 2024-09-13T15:20:17.274248+0000 mgr.y (mgr.26252) 1629 : cluster [DBG] pgmap v2264: 115 pgs: 115 active+clean; 635 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 822 KiB/s wr, 620 op/s 2024-09-13T15:20:17.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:17 smithi177 bash[68594]: cluster 2024-09-13T15:20:17.274248+0000 mgr.y (mgr.26252) 1629 : cluster [DBG] pgmap v2264: 115 pgs: 115 active+clean; 635 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 822 KiB/s wr, 620 op/s 2024-09-13T15:20:17.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:17.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:17.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:17.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:17.724 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.mirror_snapshot (119 ms) 2024-09-13T15:20:17.724 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-13T15:20:17.728 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_dir_list (4 ms) 2024-09-13T15:20:17.728 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-13T15:20:17.732 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_dir_add (4 ms) 2024-09-13T15:20:17.732 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-13T15:20:17.737 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.dir_add_already_existing (5 ms) 2024-09-13T15:20:17.737 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-13T15:20:17.748 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_dir_rename (11 ms) 2024-09-13T15:20:17.748 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-13T15:20:17.754 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_dir_remove (6 ms) 2024-09-13T15:20:17.754 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-13T15:20:17.765 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (10 ms) 2024-09-13T15:20:17.765 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-13T15:20:17.770 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_image_add (5 ms) 2024-09-13T15:20:17.770 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-13T15:20:17.777 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_image_remove (7 ms) 2024-09-13T15:20:17.778 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-13T15:20:17.784 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:17.786 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_image_list (8 ms) 2024-09-13T15:20:17.786 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-13T15:20:17.796 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_image_clean (11 ms) 2024-09-13T15:20:17.796 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-13T15:20:17.802 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.image_group_add (6 ms) 2024-09-13T15:20:17.802 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-13T15:20:17.808 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.image_group_remove (6 ms) 2024-09-13T15:20:17.808 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-13T15:20:17.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:17.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:17.814 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.image_group_get (5 ms) 2024-09-13T15:20:17.814 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-13T15:20:17.817 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (4 ms) 2024-09-13T15:20:17.817 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-13T15:20:17.820 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-13T15:20:17.820 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-13T15:20:17.823 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (4 ms) 2024-09-13T15:20:17.823 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-13T15:20:17.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:17 smithi099 bash[84576]: cluster 2024-09-13T15:20:17.274248+0000 mgr.y (mgr.26252) 1629 : cluster [DBG] pgmap v2264: 115 pgs: 115 active+clean; 635 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 822 KiB/s wr, 620 op/s 2024-09-13T15:20:17.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:17 smithi099 bash[84576]: cluster 2024-09-13T15:20:17.274248+0000 mgr.y (mgr.26252) 1629 : cluster [DBG] pgmap v2264: 115 pgs: 115 active+clean; 635 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 822 KiB/s wr, 620 op/s 2024-09-13T15:20:17.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:17 smithi099 bash[87147]: cluster 2024-09-13T15:20:17.274248+0000 mgr.y (mgr.26252) 1629 : cluster [DBG] pgmap v2264: 115 pgs: 115 active+clean; 635 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 822 KiB/s wr, 620 op/s 2024-09-13T15:20:17.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:17 smithi099 bash[87147]: cluster 2024-09-13T15:20:17.274248+0000 mgr.y (mgr.26252) 1629 : cluster [DBG] pgmap v2264: 115 pgs: 115 active+clean; 635 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 822 KiB/s wr, 620 op/s 2024-09-13T15:20:17.828 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (5 ms) 2024-09-13T15:20:17.828 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-13T15:20:17.832 INFO:tasks.workunit.client.0.smithi099.stdout:./src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-13T15:20:17.832 INFO:tasks.workunit.client.0.smithi099.stdout:Expected equality of these values: 2024-09-13T15:20:17.832 INFO:tasks.workunit.client.0.smithi099.stdout: 1U 2024-09-13T15:20:17.832 INFO:tasks.workunit.client.0.smithi099.stdout: Which is: 1 2024-09-13T15:20:17.832 INFO:tasks.workunit.client.0.smithi099.stdout: keys.size() 2024-09-13T15:20:17.832 INFO:tasks.workunit.client.0.smithi099.stdout: Which is: 3 2024-09-13T15:20:17.832 INFO:tasks.workunit.client.0.smithi099.stdout:[ FAILED ] TestClsRbd.group_snap_set (4 ms) 2024-09-13T15:20:17.833 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-13T15:20:17.840 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_snap_list (8 ms) 2024-09-13T15:20:17.841 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-13T15:20:17.884 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (44 ms) 2024-09-13T15:20:17.884 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-13T15:20:17.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:17.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:17.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:17.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:18.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:18.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:18.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:18.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:18.172 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:18.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:18.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:18.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:18.260 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (376 ms) 2024-09-13T15:20:18.260 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-13T15:20:18.266 INFO:tasks.workunit.client.0.smithi099.stdout:./src/test/cls_rbd/test_cls_rbd.cc:2792: Failure 2024-09-13T15:20:18.266 INFO:tasks.workunit.client.0.smithi099.stdout:Expected equality of these values: 2024-09-13T15:20:18.266 INFO:tasks.workunit.client.0.smithi099.stdout: 1U 2024-09-13T15:20:18.266 INFO:tasks.workunit.client.0.smithi099.stdout: Which is: 1 2024-09-13T15:20:18.266 INFO:tasks.workunit.client.0.smithi099.stdout: keys.size() 2024-09-13T15:20:18.266 INFO:tasks.workunit.client.0.smithi099.stdout: Which is: 3 2024-09-13T15:20:18.266 INFO:tasks.workunit.client.0.smithi099.stdout:[ FAILED ] TestClsRbd.group_snap_remove (6 ms) 2024-09-13T15:20:18.266 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-13T15:20:18.271 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (5 ms) 2024-09-13T15:20:18.271 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-13T15:20:18.295 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.trash_methods (24 ms) 2024-09-13T15:20:18.296 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.op_features 2024-09-13T15:20:18.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:18.309 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.op_features (14 ms) 2024-09-13T15:20:18.310 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-13T15:20:18.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:18.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:18.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:18.383 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.clone_parent (73 ms) 2024-09-13T15:20:18.383 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-13T15:20:18.400 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.clone_parent_ns (18 ms) 2024-09-13T15:20:18.400 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-13T15:20:18.429 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.clone_child (29 ms) 2024-09-13T15:20:18.429 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-13T15:20:18.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:18.445 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.namespace_methods (16 ms) 2024-09-13T15:20:18.445 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.migration 2024-09-13T15:20:18.458 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:18.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:18.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:18.471 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.migration (26 ms) 2024-09-13T15:20:18.471 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-13T15:20:18.495 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.migration_v1 (24 ms) 2024-09-13T15:20:18.495 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-13T15:20:18.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:18.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:18 smithi099 bash[84576]: audit 2024-09-13T15:20:18.324828+0000 mon.a (mon.0) 2826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:18.572 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:18 smithi099 bash[84576]: audit 2024-09-13T15:20:18.324828+0000 mon.a (mon.0) 2826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:18.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:18 smithi099 bash[87147]: audit 2024-09-13T15:20:18.324828+0000 mon.a (mon.0) 2826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:18.572 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:18 smithi099 bash[87147]: audit 2024-09-13T15:20:18.324828+0000 mon.a (mon.0) 2826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:18.587 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:18.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:18.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:18 smithi177 bash[68594]: audit 2024-09-13T15:20:18.324828+0000 mon.a (mon.0) 2826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:18 smithi177 bash[68594]: audit 2024-09-13T15:20:18.324828+0000 mon.a (mon.0) 2826 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:18.691 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:18.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:20:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:18.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:20:18.822 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:18.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:18.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:18.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:18.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:18.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:18.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:18.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:19.082 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:19.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:19.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:19.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:19.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:19.242 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:19.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:19.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:19.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:19.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:19.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:19.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:19.478 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:19.505 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:19.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:19.608 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:19 smithi177 bash[68594]: cluster 2024-09-13T15:20:19.275129+0000 mgr.y (mgr.26252) 1630 : cluster [DBG] pgmap v2265: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.4 MiB/s rd, 659 KiB/s wr, 582 op/s 2024-09-13T15:20:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:19 smithi177 bash[68594]: cluster 2024-09-13T15:20:19.275129+0000 mgr.y (mgr.26252) 1630 : cluster [DBG] pgmap v2265: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.4 MiB/s rd, 659 KiB/s wr, 582 op/s 2024-09-13T15:20:19.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:19 smithi099 bash[84576]: cluster 2024-09-13T15:20:19.275129+0000 mgr.y (mgr.26252) 1630 : cluster [DBG] pgmap v2265: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.4 MiB/s rd, 659 KiB/s wr, 582 op/s 2024-09-13T15:20:19.620 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:19 smithi099 bash[84576]: cluster 2024-09-13T15:20:19.275129+0000 mgr.y (mgr.26252) 1630 : cluster [DBG] pgmap v2265: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.4 MiB/s rd, 659 KiB/s wr, 582 op/s 2024-09-13T15:20:19.621 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:19 smithi099 bash[87147]: cluster 2024-09-13T15:20:19.275129+0000 mgr.y (mgr.26252) 1630 : cluster [DBG] pgmap v2265: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.4 MiB/s rd, 659 KiB/s wr, 582 op/s 2024-09-13T15:20:19.621 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:19 smithi099 bash[87147]: cluster 2024-09-13T15:20:19.275129+0000 mgr.y (mgr.26252) 1630 : cluster [DBG] pgmap v2265: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.4 MiB/s rd, 659 KiB/s wr, 582 op/s 2024-09-13T15:20:19.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:19.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:19.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:19.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:19.767 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:19.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:19.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:19.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:19.898 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:19.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:19.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:20:20.001 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:20.002 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:20:20.026 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:20:20.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:20.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:20:20.129 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:20.156 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:20.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:20.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:20.259 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:20.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:20.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:20.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:20.389 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:20.389 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1894 ms) 2024-09-13T15:20:20.390 INFO:tasks.workunit.client.0.smithi099.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-13T15:20:20.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:20.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:20.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:20.445 INFO:tasks.workunit.client.0.smithi099.stdout:[ OK ] TestClsRbd.sparsify (57 ms) 2024-09-13T15:20:20.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:20.535 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:20:20 smithi177 bash[61439]: ts=2024-09-13T15:20:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:20:20.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:20 smithi177 bash[68594]: cluster 2024-09-13T15:20:19.376487+0000 mon.a (mon.0) 2827 : cluster [DBG] osdmap e1152: 8 total, 8 up, 7 in 2024-09-13T15:20:20.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:20 smithi177 bash[68594]: cluster 2024-09-13T15:20:19.376487+0000 mon.a (mon.0) 2827 : cluster [DBG] osdmap e1152: 8 total, 8 up, 7 in 2024-09-13T15:20:20.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:20.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:20.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:20.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:20.666 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:20 smithi099 bash[84576]: cluster 2024-09-13T15:20:19.376487+0000 mon.a (mon.0) 2827 : cluster [DBG] osdmap e1152: 8 total, 8 up, 7 in 2024-09-13T15:20:20.666 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:20 smithi099 bash[84576]: cluster 2024-09-13T15:20:19.376487+0000 mon.a (mon.0) 2827 : cluster [DBG] osdmap e1152: 8 total, 8 up, 7 in 2024-09-13T15:20:20.666 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:20 smithi099 bash[87147]: cluster 2024-09-13T15:20:19.376487+0000 mon.a (mon.0) 2827 : cluster [DBG] osdmap e1152: 8 total, 8 up, 7 in 2024-09-13T15:20:20.667 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:20 smithi099 bash[87147]: cluster 2024-09-13T15:20:19.376487+0000 mon.a (mon.0) 2827 : cluster [DBG] osdmap e1152: 8 total, 8 up, 7 in 2024-09-13T15:20:20.682 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:20.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:20.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:20.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:20.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:20.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:20.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:20.919 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:20.946 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:20:20.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:20.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:20:21.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:21.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:20:21.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:21.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:20:21.180 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:21.208 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:21.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:21.311 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:21.339 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:20:21.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:21.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:20:21.405 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] 70 tests from TestClsRbd (8509 ms total) 2024-09-13T15:20:21.405 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T15:20:21.405 INFO:tasks.workunit.client.0.smithi099.stdout:[----------] Global test environment tear-down 2024-09-13T15:20:21.405 INFO:tasks.workunit.client.0.smithi099.stdout:[==========] 70 tests from 1 test suite ran. (10945 ms total) 2024-09-13T15:20:21.405 INFO:tasks.workunit.client.0.smithi099.stdout:[ PASSED ] 68 tests. 2024-09-13T15:20:21.406 INFO:tasks.workunit.client.0.smithi099.stdout:[ FAILED ] 2 tests, listed below: 2024-09-13T15:20:21.406 INFO:tasks.workunit.client.0.smithi099.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-13T15:20:21.406 INFO:tasks.workunit.client.0.smithi099.stdout:[ FAILED ] TestClsRbd.group_snap_remove 2024-09-13T15:20:21.406 INFO:tasks.workunit.client.0.smithi099.stdout: 2024-09-13T15:20:21.406 INFO:tasks.workunit.client.0.smithi099.stdout: 2 FAILED TESTS 2024-09-13T15:20:21.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:21.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:21.464 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-09-13T15:20:21.464 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T15:20:21.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:21.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:20:21.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:20:21.573 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:21.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:21 smithi099 bash[84576]: cluster 2024-09-13T15:20:20.387351+0000 mon.a (mon.0) 2828 : cluster [DBG] osdmap e1153: 8 total, 8 up, 7 in 2024-09-13T15:20:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:21 smithi099 bash[84576]: cluster 2024-09-13T15:20:20.387351+0000 mon.a (mon.0) 2828 : cluster [DBG] osdmap e1153: 8 total, 8 up, 7 in 2024-09-13T15:20:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:21 smithi099 bash[84576]: cluster 2024-09-13T15:20:21.276112+0000 mgr.y (mgr.26252) 1631 : cluster [DBG] pgmap v2268: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.3 MiB/s rd, 1.0 MiB/s wr, 1.83k op/s 2024-09-13T15:20:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:21 smithi099 bash[84576]: cluster 2024-09-13T15:20:21.276112+0000 mgr.y (mgr.26252) 1631 : cluster [DBG] pgmap v2268: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.3 MiB/s rd, 1.0 MiB/s wr, 1.83k op/s 2024-09-13T15:20:21.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:21 smithi099 bash[87147]: cluster 2024-09-13T15:20:20.387351+0000 mon.a (mon.0) 2828 : cluster [DBG] osdmap e1153: 8 total, 8 up, 7 in 2024-09-13T15:20:21.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:21 smithi099 bash[87147]: cluster 2024-09-13T15:20:20.387351+0000 mon.a (mon.0) 2828 : cluster [DBG] osdmap e1153: 8 total, 8 up, 7 in 2024-09-13T15:20:21.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:21 smithi099 bash[87147]: cluster 2024-09-13T15:20:21.276112+0000 mgr.y (mgr.26252) 1631 : cluster [DBG] pgmap v2268: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.3 MiB/s rd, 1.0 MiB/s wr, 1.83k op/s 2024-09-13T15:20:21.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:21 smithi099 bash[87147]: cluster 2024-09-13T15:20:21.276112+0000 mgr.y (mgr.26252) 1631 : cluster [DBG] pgmap v2268: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.3 MiB/s rd, 1.0 MiB/s wr, 1.83k op/s 2024-09-13T15:20:21.599 INFO:teuthology.orchestra.run.smithi177.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:20:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:21.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:20:21.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:21.713 INFO:tasks.thrashosds:joining thrashosds 2024-09-13T15:20:21.726 INFO:teuthology.orchestra.run.smithi099.stderr:Error response from daemon: Cannot kill container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:20:21.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:20:21.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi099 with status 1: 'sudo docker kill -s 1 ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:20:21.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:21 smithi177 bash[68594]: cluster 2024-09-13T15:20:20.387351+0000 mon.a (mon.0) 2828 : cluster [DBG] osdmap e1153: 8 total, 8 up, 7 in 2024-09-13T15:20:21.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:21 smithi177 bash[68594]: cluster 2024-09-13T15:20:20.387351+0000 mon.a (mon.0) 2828 : cluster [DBG] osdmap e1153: 8 total, 8 up, 7 in 2024-09-13T15:20:21.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:21 smithi177 bash[68594]: cluster 2024-09-13T15:20:21.276112+0000 mgr.y (mgr.26252) 1631 : cluster [DBG] pgmap v2268: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.3 MiB/s rd, 1.0 MiB/s wr, 1.83k op/s 2024-09-13T15:20:21.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:21 smithi177 bash[68594]: cluster 2024-09-13T15:20:21.276112+0000 mgr.y (mgr.26252) 1631 : cluster [DBG] pgmap v2268: 115 pgs: 115 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.3 MiB/s rd, 1.0 MiB/s wr, 1.83k op/s 2024-09-13T15:20:22.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:22 smithi099 bash[84576]: cluster 2024-09-13T15:20:21.397585+0000 mon.a (mon.0) 2829 : cluster [DBG] osdmap e1154: 8 total, 8 up, 7 in 2024-09-13T15:20:22.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:22 smithi099 bash[84576]: cluster 2024-09-13T15:20:21.397585+0000 mon.a (mon.0) 2829 : cluster [DBG] osdmap e1154: 8 total, 8 up, 7 in 2024-09-13T15:20:22.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:22 smithi099 bash[87147]: cluster 2024-09-13T15:20:21.397585+0000 mon.a (mon.0) 2829 : cluster [DBG] osdmap e1154: 8 total, 8 up, 7 in 2024-09-13T15:20:22.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:22 smithi099 bash[87147]: cluster 2024-09-13T15:20:21.397585+0000 mon.a (mon.0) 2829 : cluster [DBG] osdmap e1154: 8 total, 8 up, 7 in 2024-09-13T15:20:22.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:22 smithi177 bash[68594]: cluster 2024-09-13T15:20:21.397585+0000 mon.a (mon.0) 2829 : cluster [DBG] osdmap e1154: 8 total, 8 up, 7 in 2024-09-13T15:20:22.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:22 smithi177 bash[68594]: cluster 2024-09-13T15:20:21.397585+0000 mon.a (mon.0) 2829 : cluster [DBG] osdmap e1154: 8 total, 8 up, 7 in 2024-09-13T15:20:23.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:23 smithi099 bash[84576]: cluster 2024-09-13T15:20:23.276865+0000 mgr.y (mgr.26252) 1632 : cluster [DBG] pgmap v2270: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:23.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:23 smithi099 bash[84576]: cluster 2024-09-13T15:20:23.276865+0000 mgr.y (mgr.26252) 1632 : cluster [DBG] pgmap v2270: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:23.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:23 smithi099 bash[87147]: cluster 2024-09-13T15:20:23.276865+0000 mgr.y (mgr.26252) 1632 : cluster [DBG] pgmap v2270: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:23.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:23 smithi099 bash[87147]: cluster 2024-09-13T15:20:23.276865+0000 mgr.y (mgr.26252) 1632 : cluster [DBG] pgmap v2270: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:23.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:23 smithi177 bash[68594]: cluster 2024-09-13T15:20:23.276865+0000 mgr.y (mgr.26252) 1632 : cluster [DBG] pgmap v2270: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:23.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:23 smithi177 bash[68594]: cluster 2024-09-13T15:20:23.276865+0000 mgr.y (mgr.26252) 1632 : cluster [DBG] pgmap v2270: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:26.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:25 smithi177 bash[68594]: audit 2024-09-13T15:20:24.744502+0000 mon.a (mon.0) 2830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:26.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:25 smithi177 bash[68594]: audit 2024-09-13T15:20:24.744502+0000 mon.a (mon.0) 2830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:26.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:25 smithi177 bash[68594]: cluster 2024-09-13T15:20:25.277780+0000 mgr.y (mgr.26252) 1633 : cluster [DBG] pgmap v2271: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:26.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:25 smithi177 bash[68594]: cluster 2024-09-13T15:20:25.277780+0000 mgr.y (mgr.26252) 1633 : cluster [DBG] pgmap v2271: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:26.169 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:25 smithi099 bash[84576]: audit 2024-09-13T15:20:24.744502+0000 mon.a (mon.0) 2830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:26.169 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:25 smithi099 bash[84576]: audit 2024-09-13T15:20:24.744502+0000 mon.a (mon.0) 2830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:26.169 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:25 smithi099 bash[84576]: cluster 2024-09-13T15:20:25.277780+0000 mgr.y (mgr.26252) 1633 : cluster [DBG] pgmap v2271: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:26.169 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:25 smithi099 bash[84576]: cluster 2024-09-13T15:20:25.277780+0000 mgr.y (mgr.26252) 1633 : cluster [DBG] pgmap v2271: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:26.170 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:25 smithi099 bash[87147]: audit 2024-09-13T15:20:24.744502+0000 mon.a (mon.0) 2830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:26.170 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:25 smithi099 bash[87147]: audit 2024-09-13T15:20:24.744502+0000 mon.a (mon.0) 2830 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:26.170 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:25 smithi099 bash[87147]: cluster 2024-09-13T15:20:25.277780+0000 mgr.y (mgr.26252) 1633 : cluster [DBG] pgmap v2271: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:26.170 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:25 smithi099 bash[87147]: cluster 2024-09-13T15:20:25.277780+0000 mgr.y (mgr.26252) 1633 : cluster [DBG] pgmap v2271: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:27.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:20:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:20:27] "GET /metrics HTTP/1.1" 200 36777 "" "Prometheus/2.51.0" 2024-09-13T15:20:27.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:27 smithi177 bash[68594]: cluster 2024-09-13T15:20:27.278761+0000 mgr.y (mgr.26252) 1634 : cluster [DBG] pgmap v2272: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:27.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:27 smithi177 bash[68594]: cluster 2024-09-13T15:20:27.278761+0000 mgr.y (mgr.26252) 1634 : cluster [DBG] pgmap v2272: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:27.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:27 smithi099 bash[84576]: cluster 2024-09-13T15:20:27.278761+0000 mgr.y (mgr.26252) 1634 : cluster [DBG] pgmap v2272: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:27.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:27 smithi099 bash[84576]: cluster 2024-09-13T15:20:27.278761+0000 mgr.y (mgr.26252) 1634 : cluster [DBG] pgmap v2272: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:27.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:27 smithi099 bash[87147]: cluster 2024-09-13T15:20:27.278761+0000 mgr.y (mgr.26252) 1634 : cluster [DBG] pgmap v2272: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:27.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:27 smithi099 bash[87147]: cluster 2024-09-13T15:20:27.278761+0000 mgr.y (mgr.26252) 1634 : cluster [DBG] pgmap v2272: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:29.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:29 smithi177 bash[68594]: cluster 2024-09-13T15:20:29.279571+0000 mgr.y (mgr.26252) 1635 : cluster [DBG] pgmap v2273: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:29.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:29 smithi177 bash[68594]: cluster 2024-09-13T15:20:29.279571+0000 mgr.y (mgr.26252) 1635 : cluster [DBG] pgmap v2273: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:29.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:29 smithi099 bash[84576]: cluster 2024-09-13T15:20:29.279571+0000 mgr.y (mgr.26252) 1635 : cluster [DBG] pgmap v2273: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:29.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:29 smithi099 bash[84576]: cluster 2024-09-13T15:20:29.279571+0000 mgr.y (mgr.26252) 1635 : cluster [DBG] pgmap v2273: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:29.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:29 smithi099 bash[87147]: cluster 2024-09-13T15:20:29.279571+0000 mgr.y (mgr.26252) 1635 : cluster [DBG] pgmap v2273: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:29.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:29 smithi099 bash[87147]: cluster 2024-09-13T15:20:29.279571+0000 mgr.y (mgr.26252) 1635 : cluster [DBG] pgmap v2273: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:30.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:20:30 smithi177 bash[61439]: ts=2024-09-13T15:20:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:20:31.695 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2024-09-13T15:20:31.696 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:20:31.699 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:31 smithi099 bash[84576]: cluster 2024-09-13T15:20:31.280553+0000 mgr.y (mgr.26252) 1636 : cluster [DBG] pgmap v2274: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:31.699 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:31 smithi099 bash[84576]: cluster 2024-09-13T15:20:31.280553+0000 mgr.y (mgr.26252) 1636 : cluster [DBG] pgmap v2274: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:31.700 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:31 smithi099 bash[87147]: cluster 2024-09-13T15:20:31.280553+0000 mgr.y (mgr.26252) 1636 : cluster [DBG] pgmap v2274: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:31.700 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:31 smithi099 bash[87147]: cluster 2024-09-13T15:20:31.280553+0000 mgr.y (mgr.26252) 1636 : cluster [DBG] pgmap v2274: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:31.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:31 smithi177 bash[68594]: cluster 2024-09-13T15:20:31.280553+0000 mgr.y (mgr.26252) 1636 : cluster [DBG] pgmap v2274: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:31.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:31 smithi177 bash[68594]: cluster 2024-09-13T15:20:31.280553+0000 mgr.y (mgr.26252) 1636 : cluster [DBG] pgmap v2274: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:32.375 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:20:32.422 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:32 smithi099 bash[84576]: audit 2024-09-13T15:20:32.026646+0000 mon.c (mon.2) 125 : audit [INF] from='client.? 172.21.15.99:0/829588129' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.423 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:32 smithi099 bash[84576]: audit 2024-09-13T15:20:32.026646+0000 mon.c (mon.2) 125 : audit [INF] from='client.? 172.21.15.99:0/829588129' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.423 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:32 smithi099 bash[84576]: audit 2024-09-13T15:20:32.027363+0000 mon.a (mon.0) 2831 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.423 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:32 smithi099 bash[84576]: audit 2024-09-13T15:20:32.027363+0000 mon.a (mon.0) 2831 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.423 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:32 smithi099 bash[87147]: audit 2024-09-13T15:20:32.026646+0000 mon.c (mon.2) 125 : audit [INF] from='client.? 172.21.15.99:0/829588129' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.423 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:32 smithi099 bash[87147]: audit 2024-09-13T15:20:32.026646+0000 mon.c (mon.2) 125 : audit [INF] from='client.? 172.21.15.99:0/829588129' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.423 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:32 smithi099 bash[87147]: audit 2024-09-13T15:20:32.027363+0000 mon.a (mon.0) 2831 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.423 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:32 smithi099 bash[87147]: audit 2024-09-13T15:20:32.027363+0000 mon.a (mon.0) 2831 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:32 smithi177 bash[68594]: audit 2024-09-13T15:20:32.026646+0000 mon.c (mon.2) 125 : audit [INF] from='client.? 172.21.15.99:0/829588129' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:32 smithi177 bash[68594]: audit 2024-09-13T15:20:32.026646+0000 mon.c (mon.2) 125 : audit [INF] from='client.? 172.21.15.99:0/829588129' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:32 smithi177 bash[68594]: audit 2024-09-13T15:20:32.027363+0000 mon.a (mon.0) 2831 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:32.865 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:32 smithi177 bash[68594]: audit 2024-09-13T15:20:32.027363+0000 mon.a (mon.0) 2831 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:20:33.593 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T15:20:33.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: cluster 2024-09-13T15:20:32.363300+0000 mon.a (mon.0) 2832 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:20:33.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: cluster 2024-09-13T15:20:32.363300+0000 mon.a (mon.0) 2832 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:20:33.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: cluster 2024-09-13T15:20:32.363350+0000 mon.a (mon.0) 2833 : cluster [INF] Cluster is now healthy 2024-09-13T15:20:33.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: cluster 2024-09-13T15:20:32.363350+0000 mon.a (mon.0) 2833 : cluster [INF] Cluster is now healthy 2024-09-13T15:20:33.675 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: audit 2024-09-13T15:20:32.365837+0000 mon.a (mon.0) 2834 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: audit 2024-09-13T15:20:32.365837+0000 mon.a (mon.0) 2834 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: cluster 2024-09-13T15:20:32.373578+0000 mon.a (mon.0) 2835 : cluster [DBG] osdmap e1155: 8 total, 8 up, 7 in 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: cluster 2024-09-13T15:20:32.373578+0000 mon.a (mon.0) 2835 : cluster [DBG] osdmap e1155: 8 total, 8 up, 7 in 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: cluster 2024-09-13T15:20:33.280970+0000 mgr.y (mgr.26252) 1637 : cluster [DBG] pgmap v2276: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: cluster 2024-09-13T15:20:33.280970+0000 mgr.y (mgr.26252) 1637 : cluster [DBG] pgmap v2276: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: audit 2024-09-13T15:20:33.333599+0000 mon.a (mon.0) 2836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: audit 2024-09-13T15:20:33.333599+0000 mon.a (mon.0) 2836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: audit 2024-09-13T15:20:33.335664+0000 mon.a (mon.0) 2837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[84576]: audit 2024-09-13T15:20:33.335664+0000 mon.a (mon.0) 2837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: cluster 2024-09-13T15:20:32.363300+0000 mon.a (mon.0) 2832 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:20:33.676 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: cluster 2024-09-13T15:20:32.363300+0000 mon.a (mon.0) 2832 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: cluster 2024-09-13T15:20:32.363350+0000 mon.a (mon.0) 2833 : cluster [INF] Cluster is now healthy 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: cluster 2024-09-13T15:20:32.363350+0000 mon.a (mon.0) 2833 : cluster [INF] Cluster is now healthy 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: audit 2024-09-13T15:20:32.365837+0000 mon.a (mon.0) 2834 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: audit 2024-09-13T15:20:32.365837+0000 mon.a (mon.0) 2834 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: cluster 2024-09-13T15:20:32.373578+0000 mon.a (mon.0) 2835 : cluster [DBG] osdmap e1155: 8 total, 8 up, 7 in 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: cluster 2024-09-13T15:20:32.373578+0000 mon.a (mon.0) 2835 : cluster [DBG] osdmap e1155: 8 total, 8 up, 7 in 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: cluster 2024-09-13T15:20:33.280970+0000 mgr.y (mgr.26252) 1637 : cluster [DBG] pgmap v2276: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: cluster 2024-09-13T15:20:33.280970+0000 mgr.y (mgr.26252) 1637 : cluster [DBG] pgmap v2276: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: audit 2024-09-13T15:20:33.333599+0000 mon.a (mon.0) 2836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: audit 2024-09-13T15:20:33.333599+0000 mon.a (mon.0) 2836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: audit 2024-09-13T15:20:33.335664+0000 mon.a (mon.0) 2837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:33.677 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:33 smithi099 bash[87147]: audit 2024-09-13T15:20:33.335664+0000 mon.a (mon.0) 2837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:33.679 INFO:teuthology.orchestra.run.smithi099.stdout:osd.0: {} 2024-09-13T15:20:33.679 INFO:teuthology.orchestra.run.smithi099.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:20:33.697 INFO:teuthology.orchestra.run.smithi099.stdout:osd.1: {} 2024-09-13T15:20:33.697 INFO:teuthology.orchestra.run.smithi099.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:20:33.716 INFO:teuthology.orchestra.run.smithi099.stdout:osd.2: {} 2024-09-13T15:20:33.716 INFO:teuthology.orchestra.run.smithi099.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:20:33.739 INFO:teuthology.orchestra.run.smithi099.stdout:osd.3: {} 2024-09-13T15:20:33.739 INFO:teuthology.orchestra.run.smithi099.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:20:33.760 INFO:teuthology.orchestra.run.smithi099.stdout:osd.4: {} 2024-09-13T15:20:33.761 INFO:teuthology.orchestra.run.smithi099.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:20:33.793 INFO:teuthology.orchestra.run.smithi099.stdout:osd.5: {} 2024-09-13T15:20:33.793 INFO:teuthology.orchestra.run.smithi099.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:20:33.817 INFO:teuthology.orchestra.run.smithi099.stdout:osd.6: {} 2024-09-13T15:20:33.817 INFO:teuthology.orchestra.run.smithi099.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:20:33.836 INFO:teuthology.orchestra.run.smithi099.stdout:osd.7: {} 2024-09-13T15:20:33.836 INFO:teuthology.orchestra.run.smithi099.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:20:33.848 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T15:20:33.849 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T15:20:33.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: cluster 2024-09-13T15:20:32.363300+0000 mon.a (mon.0) 2832 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:20:33.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: cluster 2024-09-13T15:20:32.363300+0000 mon.a (mon.0) 2832 : cluster [INF] Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:20:33.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: cluster 2024-09-13T15:20:32.363350+0000 mon.a (mon.0) 2833 : cluster [INF] Cluster is now healthy 2024-09-13T15:20:33.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: cluster 2024-09-13T15:20:32.363350+0000 mon.a (mon.0) 2833 : cluster [INF] Cluster is now healthy 2024-09-13T15:20:33.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: audit 2024-09-13T15:20:32.365837+0000 mon.a (mon.0) 2834 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:20:33.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: audit 2024-09-13T15:20:32.365837+0000 mon.a (mon.0) 2834 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:20:33.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: cluster 2024-09-13T15:20:32.373578+0000 mon.a (mon.0) 2835 : cluster [DBG] osdmap e1155: 8 total, 8 up, 7 in 2024-09-13T15:20:33.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: cluster 2024-09-13T15:20:32.373578+0000 mon.a (mon.0) 2835 : cluster [DBG] osdmap e1155: 8 total, 8 up, 7 in 2024-09-13T15:20:33.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: cluster 2024-09-13T15:20:33.280970+0000 mgr.y (mgr.26252) 1637 : cluster [DBG] pgmap v2276: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:33.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: cluster 2024-09-13T15:20:33.280970+0000 mgr.y (mgr.26252) 1637 : cluster [DBG] pgmap v2276: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:33.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: audit 2024-09-13T15:20:33.333599+0000 mon.a (mon.0) 2836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:33.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: audit 2024-09-13T15:20:33.333599+0000 mon.a (mon.0) 2836 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:33.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: audit 2024-09-13T15:20:33.335664+0000 mon.a (mon.0) 2837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:33.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:33 smithi177 bash[68594]: audit 2024-09-13T15:20:33.335664+0000 mon.a (mon.0) 2837 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:34.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:34 smithi099 bash[84576]: audit 2024-09-13T15:20:33.646700+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.99:0/3594738450' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:34.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:34 smithi099 bash[84576]: audit 2024-09-13T15:20:33.646700+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.99:0/3594738450' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:34.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:34 smithi099 bash[84576]: audit 2024-09-13T15:20:33.842059+0000 mgr.y (mgr.26252) 1638 : audit [DBG] from='client.46235 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:20:34.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:34 smithi099 bash[84576]: audit 2024-09-13T15:20:33.842059+0000 mgr.y (mgr.26252) 1638 : audit [DBG] from='client.46235 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:20:34.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:34 smithi099 bash[87147]: audit 2024-09-13T15:20:33.646700+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.99:0/3594738450' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:34.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:34 smithi099 bash[87147]: audit 2024-09-13T15:20:33.646700+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.99:0/3594738450' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:34.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:34 smithi099 bash[87147]: audit 2024-09-13T15:20:33.842059+0000 mgr.y (mgr.26252) 1638 : audit [DBG] from='client.46235 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:20:34.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:34 smithi099 bash[87147]: audit 2024-09-13T15:20:33.842059+0000 mgr.y (mgr.26252) 1638 : audit [DBG] from='client.46235 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:20:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:34 smithi177 bash[68594]: audit 2024-09-13T15:20:33.646700+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.99:0/3594738450' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:34 smithi177 bash[68594]: audit 2024-09-13T15:20:33.646700+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.99:0/3594738450' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:20:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:34 smithi177 bash[68594]: audit 2024-09-13T15:20:33.842059+0000 mgr.y (mgr.26252) 1638 : audit [DBG] from='client.46235 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:20:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:34 smithi177 bash[68594]: audit 2024-09-13T15:20:33.842059+0000 mgr.y (mgr.26252) 1638 : audit [DBG] from='client.46235 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:20:35.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:35 smithi099 bash[84576]: cluster 2024-09-13T15:20:35.281756+0000 mgr.y (mgr.26252) 1639 : cluster [DBG] pgmap v2277: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:35.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:35 smithi099 bash[84576]: cluster 2024-09-13T15:20:35.281756+0000 mgr.y (mgr.26252) 1639 : cluster [DBG] pgmap v2277: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:35 smithi099 bash[87147]: cluster 2024-09-13T15:20:35.281756+0000 mgr.y (mgr.26252) 1639 : cluster [DBG] pgmap v2277: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:35.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:35 smithi099 bash[87147]: cluster 2024-09-13T15:20:35.281756+0000 mgr.y (mgr.26252) 1639 : cluster [DBG] pgmap v2277: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:35 smithi177 bash[68594]: cluster 2024-09-13T15:20:35.281756+0000 mgr.y (mgr.26252) 1639 : cluster [DBG] pgmap v2277: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:35 smithi177 bash[68594]: cluster 2024-09-13T15:20:35.281756+0000 mgr.y (mgr.26252) 1639 : cluster [DBG] pgmap v2277: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:36.868 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":2276,"stamp":"2024-09-13T15:20:33.280760+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":26,"num_read_kb":28,"num_write":63,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":17054,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1164,"ondisk_log_size":1164,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":250,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5693200,"kb_used_data":175308,"kb_used_omap":326,"kb_used_meta":5517369,"kb_avail":650522864,"statfs":{"total":671965249536,"available":666135412736,"internally_reserved":0,"allocated":179515392,"data_stored":145374312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":333884,"internal_metadata":5649786820},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004353"},"pg_stats":[{"pgid":"3.1f","version":"65'5","reported_seq":182,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.594526+0000","last_change":"2024-09-13T15:18:41.587924+0000","last_active":"2024-09-13T15:20:20.594526+0000","last_peered":"2024-09-13T15:20:20.594526+0000","last_clean":"2024-09-13T15:20:20.594526+0000","last_became_active":"2024-09-13T15:18:41.583046+0000","last_became_peered":"2024-09-13T15:18:41.583046+0000","last_unstale":"2024-09-13T15:20:20.594526+0000","last_undegraded":"2024-09-13T15:20:20.594526+0000","last_fullsized":"2024-09-13T15:20:20.594526+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:24:43.110515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2280,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":443,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.822819+0000","last_change":"2024-09-13T15:08:52.180022+0000","last_active":"2024-09-13T15:20:21.822819+0000","last_peered":"2024-09-13T15:20:21.822819+0000","last_clean":"2024-09-13T15:20:21.822819+0000","last_became_active":"2024-09-13T15:08:52.179546+0000","last_became_peered":"2024-09-13T15:08:52.179546+0000","last_unstale":"2024-09-13T15:20:21.822819+0000","last_undegraded":"2024-09-13T15:20:21.822819+0000","last_fullsized":"2024-09-13T15:20:21.822819+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:15:47.462895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1445,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.398107+0000","last_change":"2024-09-13T15:18:41.605980+0000","last_active":"2024-09-13T15:20:21.398107+0000","last_peered":"2024-09-13T15:20:21.398107+0000","last_clean":"2024-09-13T15:20:21.398107+0000","last_became_active":"2024-09-13T15:18:41.605097+0000","last_became_peered":"2024-09-13T15:18:41.605097+0000","last_unstale":"2024-09-13T15:20:21.398107+0000","last_undegraded":"2024-09-13T15:20:21.398107+0000","last_fullsized":"2024-09-13T15:20:21.398107+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:14.494693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":54,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.396418+0000","last_change":"2024-09-13T15:18:41.590277+0000","last_active":"2024-09-13T15:20:21.396418+0000","last_peered":"2024-09-13T15:20:21.396418+0000","last_clean":"2024-09-13T15:20:21.396418+0000","last_became_active":"2024-09-13T15:18:41.589816+0000","last_became_peered":"2024-09-13T15:18:41.589816+0000","last_unstale":"2024-09-13T15:20:21.396418+0000","last_undegraded":"2024-09-13T15:20:21.396418+0000","last_fullsized":"2024-09-13T15:20:21.396418+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:58.640365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1167,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.594390+0000","last_change":"2024-09-13T15:08:52.205951+0000","last_active":"2024-09-13T15:20:20.594390+0000","last_peered":"2024-09-13T15:20:20.594390+0000","last_clean":"2024-09-13T15:20:20.594390+0000","last_became_active":"2024-09-13T15:08:52.183627+0000","last_became_peered":"2024-09-13T15:08:52.183627+0000","last_unstale":"2024-09-13T15:20:20.594390+0000","last_undegraded":"2024-09-13T15:20:20.594390+0000","last_fullsized":"2024-09-13T15:20:20.594390+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:14:14.282531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":426,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.822624+0000","last_change":"2024-09-13T15:05:34.086551+0000","last_active":"2024-09-13T15:20:21.822624+0000","last_peered":"2024-09-13T15:20:21.822624+0000","last_clean":"2024-09-13T15:20:21.822624+0000","last_became_active":"2024-09-13T15:05:34.086077+0000","last_became_peered":"2024-09-13T15:05:34.086077+0000","last_unstale":"2024-09-13T15:20:21.822624+0000","last_undegraded":"2024-09-13T15:20:21.822624+0000","last_fullsized":"2024-09-13T15:20:21.822624+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:33:47.917777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1184,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.594177+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T15:20:20.594177+0000","last_peered":"2024-09-13T15:20:20.594177+0000","last_clean":"2024-09-13T15:20:20.594177+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T15:20:20.594177+0000","last_undegraded":"2024-09-13T15:20:20.594177+0000","last_fullsized":"2024-09-13T15:20:20.594177+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":440,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.823681+0000","last_change":"2024-09-13T15:05:34.075607+0000","last_active":"2024-09-13T15:20:21.823681+0000","last_peered":"2024-09-13T15:20:21.823681+0000","last_clean":"2024-09-13T15:20:21.823681+0000","last_became_active":"2024-09-13T15:05:34.075079+0000","last_became_peered":"2024-09-13T15:05:34.075079+0000","last_unstale":"2024-09-13T15:20:21.823681+0000","last_undegraded":"2024-09-13T15:20:21.823681+0000","last_fullsized":"2024-09-13T15:20:21.823681+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:05.384206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":1483,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.823390+0000","last_change":"2024-09-13T15:07:56.108198+0000","last_active":"2024-09-13T15:20:21.823390+0000","last_peered":"2024-09-13T15:20:21.823390+0000","last_clean":"2024-09-13T15:20:21.823390+0000","last_became_active":"2024-09-13T15:07:56.107824+0000","last_became_peered":"2024-09-13T15:07:56.107824+0000","last_unstale":"2024-09-13T15:20:21.823390+0000","last_undegraded":"2024-09-13T15:20:21.823390+0000","last_fullsized":"2024-09-13T15:20:21.823390+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:19.128127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1496,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.916543+0000","last_change":"2024-09-13T15:18:41.610547+0000","last_active":"2024-09-13T15:20:21.916543+0000","last_peered":"2024-09-13T15:20:21.916543+0000","last_clean":"2024-09-13T15:20:21.916543+0000","last_became_active":"2024-09-13T15:18:41.609623+0000","last_became_peered":"2024-09-13T15:18:41.609623+0000","last_unstale":"2024-09-13T15:20:21.916543+0000","last_undegraded":"2024-09-13T15:20:21.916543+0000","last_fullsized":"2024-09-13T15:20:21.916543+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:58:45.459089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":436,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.822244+0000","last_change":"2024-09-13T15:18:41.603581+0000","last_active":"2024-09-13T15:20:21.822244+0000","last_peered":"2024-09-13T15:20:21.822244+0000","last_clean":"2024-09-13T15:20:21.822244+0000","last_became_active":"2024-09-13T15:18:41.603125+0000","last_became_peered":"2024-09-13T15:18:41.603125+0000","last_unstale":"2024-09-13T15:20:21.822244+0000","last_undegraded":"2024-09-13T15:20:21.822244+0000","last_fullsized":"2024-09-13T15:20:21.822244+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:23.611676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1492,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.396334+0000","last_change":"2024-09-13T15:05:34.070717+0000","last_active":"2024-09-13T15:20:21.396334+0000","last_peered":"2024-09-13T15:20:21.396334+0000","last_clean":"2024-09-13T15:20:21.396334+0000","last_became_active":"2024-09-13T15:05:34.069929+0000","last_became_peered":"2024-09-13T15:05:34.069929+0000","last_unstale":"2024-09-13T15:20:21.396334+0000","last_undegraded":"2024-09-13T15:20:21.396334+0000","last_fullsized":"2024-09-13T15:20:21.396334+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:44:19.208697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":330,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.912433+0000","last_change":"2024-09-13T15:05:34.071133+0000","last_active":"2024-09-13T15:20:21.912433+0000","last_peered":"2024-09-13T15:20:21.912433+0000","last_clean":"2024-09-13T15:20:21.912433+0000","last_became_active":"2024-09-13T15:05:34.070519+0000","last_became_peered":"2024-09-13T15:05:34.070519+0000","last_unstale":"2024-09-13T15:20:21.912433+0000","last_undegraded":"2024-09-13T15:20:21.912433+0000","last_fullsized":"2024-09-13T15:20:21.912433+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:02.617403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1715,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.920918+0000","last_change":"2024-09-13T15:18:41.619980+0000","last_active":"2024-09-13T15:20:21.920918+0000","last_peered":"2024-09-13T15:20:21.920918+0000","last_clean":"2024-09-13T15:20:21.920918+0000","last_became_active":"2024-09-13T15:18:41.619445+0000","last_became_peered":"2024-09-13T15:18:41.619445+0000","last_unstale":"2024-09-13T15:20:21.920918+0000","last_undegraded":"2024-09-13T15:20:21.920918+0000","last_fullsized":"2024-09-13T15:20:21.920918+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:48.792190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1500,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.397868+0000","last_change":"2024-09-13T15:18:41.600829+0000","last_active":"2024-09-13T15:20:21.397868+0000","last_peered":"2024-09-13T15:20:21.397868+0000","last_clean":"2024-09-13T15:20:21.397868+0000","last_became_active":"2024-09-13T15:18:41.600440+0000","last_became_peered":"2024-09-13T15:18:41.600440+0000","last_unstale":"2024-09-13T15:20:21.397868+0000","last_undegraded":"2024-09-13T15:20:21.397868+0000","last_fullsized":"2024-09-13T15:20:21.397868+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:20:20.912079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":93,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.467574+0000","last_change":"2024-09-13T15:18:41.606623+0000","last_active":"2024-09-13T15:20:20.467574+0000","last_peered":"2024-09-13T15:20:20.467574+0000","last_clean":"2024-09-13T15:20:20.467574+0000","last_became_active":"2024-09-13T15:18:41.606144+0000","last_became_peered":"2024-09-13T15:18:41.606144+0000","last_unstale":"2024-09-13T15:20:20.467574+0000","last_undegraded":"2024-09-13T15:20:20.467574+0000","last_fullsized":"2024-09-13T15:20:20.467574+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:12.837936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1480,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.395485+0000","last_change":"2024-09-13T15:08:52.170737+0000","last_active":"2024-09-13T15:20:21.395485+0000","last_peered":"2024-09-13T15:20:21.395485+0000","last_clean":"2024-09-13T15:20:21.395485+0000","last_became_active":"2024-09-13T15:08:52.170217+0000","last_became_peered":"2024-09-13T15:08:52.170217+0000","last_unstale":"2024-09-13T15:20:21.395485+0000","last_undegraded":"2024-09-13T15:20:21.395485+0000","last_fullsized":"2024-09-13T15:20:21.395485+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:17.688936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1708,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.923832+0000","last_change":"2024-09-13T15:05:34.087376+0000","last_active":"2024-09-13T15:20:21.923832+0000","last_peered":"2024-09-13T15:20:21.923832+0000","last_clean":"2024-09-13T15:20:21.923832+0000","last_became_active":"2024-09-13T15:05:34.087046+0000","last_became_peered":"2024-09-13T15:05:34.087046+0000","last_unstale":"2024-09-13T15:20:21.923832+0000","last_undegraded":"2024-09-13T15:20:21.923832+0000","last_fullsized":"2024-09-13T15:20:21.923832+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:47.801431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1420,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.396178+0000","last_change":"2024-09-13T15:08:52.174279+0000","last_active":"2024-09-13T15:20:21.396178+0000","last_peered":"2024-09-13T15:20:21.396178+0000","last_clean":"2024-09-13T15:20:21.396178+0000","last_became_active":"2024-09-13T15:08:52.173877+0000","last_became_peered":"2024-09-13T15:08:52.173877+0000","last_unstale":"2024-09-13T15:20:21.396178+0000","last_undegraded":"2024-09-13T15:20:21.396178+0000","last_fullsized":"2024-09-13T15:20:21.396178+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:16.648765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1128,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.594468+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T15:20:20.594468+0000","last_peered":"2024-09-13T15:20:20.594468+0000","last_clean":"2024-09-13T15:20:20.594468+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T15:20:20.594468+0000","last_undegraded":"2024-09-13T15:20:20.594468+0000","last_fullsized":"2024-09-13T15:20:20.594468+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1182,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.086292+0000","last_change":"2024-09-13T15:05:34.089020+0000","last_active":"2024-09-13T15:20:21.086292+0000","last_peered":"2024-09-13T15:20:21.086292+0000","last_clean":"2024-09-13T15:20:21.086292+0000","last_became_active":"2024-09-13T15:05:34.088440+0000","last_became_peered":"2024-09-13T15:05:34.088440+0000","last_unstale":"2024-09-13T15:20:21.086292+0000","last_undegraded":"2024-09-13T15:20:21.086292+0000","last_fullsized":"2024-09-13T15:20:21.086292+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:24.826746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.913642+0000","last_change":"2024-09-13T15:18:41.615054+0000","last_active":"2024-09-13T15:20:21.913642+0000","last_peered":"2024-09-13T15:20:21.913642+0000","last_clean":"2024-09-13T15:20:21.913642+0000","last_became_active":"2024-09-13T15:18:41.614645+0000","last_became_peered":"2024-09-13T15:18:41.614645+0000","last_unstale":"2024-09-13T15:20:21.913642+0000","last_undegraded":"2024-09-13T15:20:21.913642+0000","last_fullsized":"2024-09-13T15:20:21.913642+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:20:29.018149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":459,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.824173+0000","last_change":"2024-09-13T15:08:52.203370+0000","last_active":"2024-09-13T15:20:21.824173+0000","last_peered":"2024-09-13T15:20:21.824173+0000","last_clean":"2024-09-13T15:20:21.824173+0000","last_became_active":"2024-09-13T15:08:52.179887+0000","last_became_peered":"2024-09-13T15:08:52.179887+0000","last_unstale":"2024-09-13T15:20:21.824173+0000","last_undegraded":"2024-09-13T15:20:21.824173+0000","last_fullsized":"2024-09-13T15:20:21.824173+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:57.293764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":444,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.824013+0000","last_change":"2024-09-13T15:08:52.180881+0000","last_active":"2024-09-13T15:20:21.824013+0000","last_peered":"2024-09-13T15:20:21.824013+0000","last_clean":"2024-09-13T15:20:21.824013+0000","last_became_active":"2024-09-13T15:08:52.180000+0000","last_became_peered":"2024-09-13T15:08:52.180000+0000","last_unstale":"2024-09-13T15:20:21.824013+0000","last_undegraded":"2024-09-13T15:20:21.824013+0000","last_fullsized":"2024-09-13T15:20:21.824013+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:32.622289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"346'100","reported_seq":45,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.398974+0000","last_change":"2024-09-13T15:20:09.715321+0000","last_active":"2024-09-13T15:20:21.398974+0000","last_peered":"2024-09-13T15:20:21.398974+0000","last_clean":"2024-09-13T15:20:21.398974+0000","last_became_active":"2024-09-13T15:19:58.641197+0000","last_became_peered":"2024-09-13T15:19:58.641197+0000","last_unstale":"2024-09-13T15:20:21.398974+0000","last_undegraded":"2024-09-13T15:20:21.398974+0000","last_fullsized":"2024-09-13T15:20:21.398974+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:09.715239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:09.715239+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:44:08.142181+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":431,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.822643+0000","last_change":"2024-09-13T15:08:52.188050+0000","last_active":"2024-09-13T15:20:21.822643+0000","last_peered":"2024-09-13T15:20:21.822643+0000","last_clean":"2024-09-13T15:20:21.822643+0000","last_became_active":"2024-09-13T15:08:52.187522+0000","last_became_peered":"2024-09-13T15:08:52.187522+0000","last_unstale":"2024-09-13T15:20:21.822643+0000","last_undegraded":"2024-09-13T15:20:21.822643+0000","last_fullsized":"2024-09-13T15:20:21.822643+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:29.314964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":93,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.467383+0000","last_change":"2024-09-13T15:18:41.585816+0000","last_active":"2024-09-13T15:20:20.467383+0000","last_peered":"2024-09-13T15:20:20.467383+0000","last_clean":"2024-09-13T15:20:20.467383+0000","last_became_active":"2024-09-13T15:18:41.585439+0000","last_became_peered":"2024-09-13T15:18:41.585439+0000","last_unstale":"2024-09-13T15:20:20.467383+0000","last_undegraded":"2024-09-13T15:20:20.467383+0000","last_fullsized":"2024-09-13T15:20:20.467383+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:23:09.780622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1759,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.921316+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T15:20:21.921316+0000","last_peered":"2024-09-13T15:20:21.921316+0000","last_clean":"2024-09-13T15:20:21.921316+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T15:20:21.921316+0000","last_undegraded":"2024-09-13T15:20:21.921316+0000","last_fullsized":"2024-09-13T15:20:21.921316+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"346'100","reported_seq":40,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.826405+0000","last_change":"2024-09-13T15:20:09.523890+0000","last_active":"2024-09-13T15:20:21.826405+0000","last_peered":"2024-09-13T15:20:21.826405+0000","last_clean":"2024-09-13T15:20:21.826405+0000","last_became_active":"2024-09-13T15:19:58.640534+0000","last_became_peered":"2024-09-13T15:19:58.640534+0000","last_unstale":"2024-09-13T15:20:21.826405+0000","last_undegraded":"2024-09-13T15:20:21.826405+0000","last_fullsized":"2024-09-13T15:20:21.826405+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:09.523750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:09.523750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:41.578547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1414,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.395610+0000","last_change":"2024-09-13T15:07:56.109509+0000","last_active":"2024-09-13T15:20:21.395610+0000","last_peered":"2024-09-13T15:20:21.395610+0000","last_clean":"2024-09-13T15:20:21.395610+0000","last_became_active":"2024-09-13T15:07:56.108034+0000","last_became_peered":"2024-09-13T15:07:56.108034+0000","last_unstale":"2024-09-13T15:20:21.395610+0000","last_undegraded":"2024-09-13T15:20:21.395610+0000","last_fullsized":"2024-09-13T15:20:21.395610+0000","mapping_epoch":1072,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1073,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:57:02.300327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":334,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.914957+0000","last_change":"2024-09-13T15:08:52.186362+0000","last_active":"2024-09-13T15:20:21.914957+0000","last_peered":"2024-09-13T15:20:21.914957+0000","last_clean":"2024-09-13T15:20:21.914957+0000","last_became_active":"2024-09-13T15:08:52.185731+0000","last_became_peered":"2024-09-13T15:08:52.185731+0000","last_unstale":"2024-09-13T15:20:21.914957+0000","last_undegraded":"2024-09-13T15:20:21.914957+0000","last_fullsized":"2024-09-13T15:20:21.914957+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:26.408919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1409,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.398463+0000","last_change":"2024-09-13T15:05:34.085917+0000","last_active":"2024-09-13T15:20:21.398463+0000","last_peered":"2024-09-13T15:20:21.398463+0000","last_clean":"2024-09-13T15:20:21.398463+0000","last_became_active":"2024-09-13T15:05:34.085156+0000","last_became_peered":"2024-09-13T15:05:34.085156+0000","last_unstale":"2024-09-13T15:20:21.398463+0000","last_undegraded":"2024-09-13T15:20:21.398463+0000","last_fullsized":"2024-09-13T15:20:21.398463+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:33.482732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"346'100","reported_seq":1202,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.467175+0000","last_change":"2024-09-13T15:20:10.562576+0000","last_active":"2024-09-13T15:20:20.467175+0000","last_peered":"2024-09-13T15:20:20.467175+0000","last_clean":"2024-09-13T15:20:20.467175+0000","last_became_active":"2024-09-13T15:19:58.639558+0000","last_became_peered":"2024-09-13T15:19:58.639558+0000","last_unstale":"2024-09-13T15:20:20.467175+0000","last_undegraded":"2024-09-13T15:20:20.467175+0000","last_fullsized":"2024-09-13T15:20:20.467175+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:10.562423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:10.562423+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:54.624883+0000","scrub_duration":0.0039412570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1689,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.919481+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T15:20:21.919481+0000","last_peered":"2024-09-13T15:20:21.919481+0000","last_clean":"2024-09-13T15:20:21.919481+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T15:20:21.919481+0000","last_undegraded":"2024-09-13T15:20:21.919481+0000","last_fullsized":"2024-09-13T15:20:21.919481+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1160,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.086409+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T15:20:21.086409+0000","last_peered":"2024-09-13T15:20:21.086409+0000","last_clean":"2024-09-13T15:20:21.086409+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T15:20:21.086409+0000","last_undegraded":"2024-09-13T15:20:21.086409+0000","last_fullsized":"2024-09-13T15:20:21.086409+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":328,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.917009+0000","last_change":"2024-09-13T15:18:41.618085+0000","last_active":"2024-09-13T15:20:21.917009+0000","last_peered":"2024-09-13T15:20:21.917009+0000","last_clean":"2024-09-13T15:20:21.917009+0000","last_became_active":"2024-09-13T15:18:41.617330+0000","last_became_peered":"2024-09-13T15:18:41.617330+0000","last_unstale":"2024-09-13T15:20:21.917009+0000","last_undegraded":"2024-09-13T15:20:21.917009+0000","last_fullsized":"2024-09-13T15:20:21.917009+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:13.166628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"346'100","reported_seq":1202,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.466986+0000","last_change":"2024-09-13T15:20:14.516263+0000","last_active":"2024-09-13T15:20:20.466986+0000","last_peered":"2024-09-13T15:20:20.466986+0000","last_clean":"2024-09-13T15:20:20.466986+0000","last_became_active":"2024-09-13T15:19:58.638300+0000","last_became_peered":"2024-09-13T15:19:58.638300+0000","last_unstale":"2024-09-13T15:20:20.466986+0000","last_undegraded":"2024-09-13T15:20:20.466986+0000","last_fullsized":"2024-09-13T15:20:20.466986+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:14.516167+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:14.516167+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:46:37.405030+0000","scrub_duration":0.0023068670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":321,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.915882+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T15:20:21.915882+0000","last_peered":"2024-09-13T15:20:21.915882+0000","last_clean":"2024-09-13T15:20:21.915882+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T15:20:21.915882+0000","last_undegraded":"2024-09-13T15:20:21.915882+0000","last_fullsized":"2024-09-13T15:20:21.915882+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":321,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.917531+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T15:20:21.917531+0000","last_peered":"2024-09-13T15:20:21.917531+0000","last_clean":"2024-09-13T15:20:21.917531+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T15:20:21.917531+0000","last_undegraded":"2024-09-13T15:20:21.917531+0000","last_fullsized":"2024-09-13T15:20:21.917531+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1152,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.594441+0000","last_change":"2024-09-13T15:18:41.583703+0000","last_active":"2024-09-13T15:20:20.594441+0000","last_peered":"2024-09-13T15:20:20.594441+0000","last_clean":"2024-09-13T15:20:20.594441+0000","last_became_active":"2024-09-13T15:18:41.583281+0000","last_became_peered":"2024-09-13T15:18:41.583281+0000","last_unstale":"2024-09-13T15:20:20.594441+0000","last_undegraded":"2024-09-13T15:20:20.594441+0000","last_fullsized":"2024-09-13T15:20:20.594441+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:43:07.415232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"346'100","reported_seq":27,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.593622+0000","last_change":"2024-09-13T15:20:06.725986+0000","last_active":"2024-09-13T15:20:20.593622+0000","last_peered":"2024-09-13T15:20:20.593622+0000","last_clean":"2024-09-13T15:20:20.593622+0000","last_became_active":"2024-09-13T15:19:58.636497+0000","last_became_peered":"2024-09-13T15:19:58.636497+0000","last_unstale":"2024-09-13T15:20:20.593622+0000","last_undegraded":"2024-09-13T15:20:20.593622+0000","last_fullsized":"2024-09-13T15:20:20.593622+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:06.725832+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:06.725832+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:59:37.442334+0000","scrub_duration":0.0036607340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":339,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.917280+0000","last_change":"2024-09-13T15:18:41.609972+0000","last_active":"2024-09-13T15:20:21.917280+0000","last_peered":"2024-09-13T15:20:21.917280+0000","last_clean":"2024-09-13T15:20:21.917280+0000","last_became_active":"2024-09-13T15:18:41.608900+0000","last_became_peered":"2024-09-13T15:18:41.608900+0000","last_unstale":"2024-09-13T15:20:21.917280+0000","last_undegraded":"2024-09-13T15:20:21.917280+0000","last_fullsized":"2024-09-13T15:20:21.917280+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:55:59.182788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":431,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.824141+0000","last_change":"2024-09-13T15:08:52.188254+0000","last_active":"2024-09-13T15:20:21.824141+0000","last_peered":"2024-09-13T15:20:21.824141+0000","last_clean":"2024-09-13T15:20:21.824141+0000","last_became_active":"2024-09-13T15:08:52.187535+0000","last_became_peered":"2024-09-13T15:08:52.187535+0000","last_unstale":"2024-09-13T15:20:21.824141+0000","last_undegraded":"2024-09-13T15:20:21.824141+0000","last_fullsized":"2024-09-13T15:20:21.824141+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:32.024558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1181,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.593818+0000","last_change":"2024-09-13T15:08:52.209457+0000","last_active":"2024-09-13T15:20:20.593818+0000","last_peered":"2024-09-13T15:20:20.593818+0000","last_clean":"2024-09-13T15:20:20.593818+0000","last_became_active":"2024-09-13T15:08:52.171595+0000","last_became_peered":"2024-09-13T15:08:52.171595+0000","last_unstale":"2024-09-13T15:20:20.593818+0000","last_undegraded":"2024-09-13T15:20:20.593818+0000","last_fullsized":"2024-09-13T15:20:20.593818+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:05:42.960465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"346'100","reported_seq":1290,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.085899+0000","last_change":"2024-09-13T15:19:58.639631+0000","last_active":"2024-09-13T15:20:21.085899+0000","last_peered":"2024-09-13T15:20:21.085899+0000","last_clean":"2024-09-13T15:20:21.085899+0000","last_became_active":"2024-09-13T15:19:58.639023+0000","last_became_peered":"2024-09-13T15:19:58.639023+0000","last_unstale":"2024-09-13T15:20:21.085899+0000","last_undegraded":"2024-09-13T15:20:21.085899+0000","last_fullsized":"2024-09-13T15:20:21.085899+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:18:17.216949+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:18:17.216949+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:10.007605+0000","scrub_duration":0.011279501000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":320,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.919133+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T15:20:21.919133+0000","last_peered":"2024-09-13T15:20:21.919133+0000","last_clean":"2024-09-13T15:20:21.919133+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T15:20:21.919133+0000","last_undegraded":"2024-09-13T15:20:21.919133+0000","last_fullsized":"2024-09-13T15:20:21.919133+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":342,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.913020+0000","last_change":"2024-09-13T15:18:41.616236+0000","last_active":"2024-09-13T15:20:21.913020+0000","last_peered":"2024-09-13T15:20:21.913020+0000","last_clean":"2024-09-13T15:20:21.913020+0000","last_became_active":"2024-09-13T15:18:41.615476+0000","last_became_peered":"2024-09-13T15:18:41.615476+0000","last_unstale":"2024-09-13T15:20:21.913020+0000","last_undegraded":"2024-09-13T15:20:21.913020+0000","last_fullsized":"2024-09-13T15:20:21.913020+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:20:43.278358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1472,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.396552+0000","last_change":"2024-09-13T15:18:41.607919+0000","last_active":"2024-09-13T15:20:21.396552+0000","last_peered":"2024-09-13T15:20:21.396552+0000","last_clean":"2024-09-13T15:20:21.396552+0000","last_became_active":"2024-09-13T15:18:41.607427+0000","last_became_peered":"2024-09-13T15:18:41.607427+0000","last_unstale":"2024-09-13T15:20:21.396552+0000","last_undegraded":"2024-09-13T15:20:21.396552+0000","last_fullsized":"2024-09-13T15:20:21.396552+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:26:18.980987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"346'100","reported_seq":40,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.914327+0000","last_change":"2024-09-13T15:20:10.181623+0000","last_active":"2024-09-13T15:20:21.914327+0000","last_peered":"2024-09-13T15:20:21.914327+0000","last_clean":"2024-09-13T15:20:21.914327+0000","last_became_active":"2024-09-13T15:19:58.637274+0000","last_became_peered":"2024-09-13T15:19:58.637274+0000","last_unstale":"2024-09-13T15:20:21.914327+0000","last_undegraded":"2024-09-13T15:20:21.914327+0000","last_fullsized":"2024-09-13T15:20:21.914327+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:10.181561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:10.181561+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:40:36.308605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1592,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.914509+0000","last_change":"2024-09-13T15:18:41.599202+0000","last_active":"2024-09-13T15:20:21.914509+0000","last_peered":"2024-09-13T15:20:21.914509+0000","last_clean":"2024-09-13T15:20:21.914509+0000","last_became_active":"2024-09-13T15:18:41.598755+0000","last_became_peered":"2024-09-13T15:18:41.598755+0000","last_unstale":"2024-09-13T15:20:21.914509+0000","last_undegraded":"2024-09-13T15:20:21.914509+0000","last_fullsized":"2024-09-13T15:20:21.914509+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:26.486207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1736,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.915808+0000","last_change":"2024-09-13T15:05:34.087817+0000","last_active":"2024-09-13T15:20:21.915808+0000","last_peered":"2024-09-13T15:20:21.915808+0000","last_clean":"2024-09-13T15:20:21.915808+0000","last_became_active":"2024-09-13T15:05:34.087450+0000","last_became_peered":"2024-09-13T15:05:34.087450+0000","last_unstale":"2024-09-13T15:20:21.915808+0000","last_undegraded":"2024-09-13T15:20:21.915808+0000","last_fullsized":"2024-09-13T15:20:21.915808+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:47.661935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1160,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.085851+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T15:20:21.085851+0000","last_peered":"2024-09-13T15:20:21.085851+0000","last_clean":"2024-09-13T15:20:21.085851+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T15:20:21.085851+0000","last_undegraded":"2024-09-13T15:20:21.085851+0000","last_fullsized":"2024-09-13T15:20:21.085851+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":83,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.467510+0000","last_change":"2024-09-13T15:08:52.188028+0000","last_active":"2024-09-13T15:20:20.467510+0000","last_peered":"2024-09-13T15:20:20.467510+0000","last_clean":"2024-09-13T15:20:20.467510+0000","last_became_active":"2024-09-13T15:08:52.187424+0000","last_became_peered":"2024-09-13T15:08:52.187424+0000","last_unstale":"2024-09-13T15:20:20.467510+0000","last_undegraded":"2024-09-13T15:20:20.467510+0000","last_fullsized":"2024-09-13T15:20:20.467510+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:31.790876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1423,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.396364+0000","last_change":"2024-09-13T15:08:52.170510+0000","last_active":"2024-09-13T15:20:21.396364+0000","last_peered":"2024-09-13T15:20:21.396364+0000","last_clean":"2024-09-13T15:20:21.396364+0000","last_became_active":"2024-09-13T15:08:52.170141+0000","last_became_peered":"2024-09-13T15:08:52.170141+0000","last_unstale":"2024-09-13T15:20:21.396364+0000","last_undegraded":"2024-09-13T15:20:21.396364+0000","last_fullsized":"2024-09-13T15:20:21.396364+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:20.674380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1135,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.593875+0000","last_change":"2024-09-13T15:18:41.609616+0000","last_active":"2024-09-13T15:20:20.593875+0000","last_peered":"2024-09-13T15:20:20.593875+0000","last_clean":"2024-09-13T15:20:20.593875+0000","last_became_active":"2024-09-13T15:18:41.582656+0000","last_became_peered":"2024-09-13T15:18:41.582656+0000","last_unstale":"2024-09-13T15:20:20.593875+0000","last_undegraded":"2024-09-13T15:20:20.593875+0000","last_fullsized":"2024-09-13T15:20:20.593875+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:33.253167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1281,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.085864+0000","last_change":"2024-09-13T15:17:58.733763+0000","last_active":"2024-09-13T15:20:21.085864+0000","last_peered":"2024-09-13T15:20:21.085864+0000","last_clean":"2024-09-13T15:20:21.085864+0000","last_became_active":"2024-09-13T15:17:58.326073+0000","last_became_peered":"2024-09-13T15:17:58.326073+0000","last_unstale":"2024-09-13T15:20:21.085864+0000","last_undegraded":"2024-09-13T15:20:21.085864+0000","last_fullsized":"2024-09-13T15:20:21.085864+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:17:58.733680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:17:58.733680+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:09:09.783840+0000","scrub_duration":0.006408459,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"346'100","reported_seq":27,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.593746+0000","last_change":"2024-09-13T15:20:09.661232+0000","last_active":"2024-09-13T15:20:20.593746+0000","last_peered":"2024-09-13T15:20:20.593746+0000","last_clean":"2024-09-13T15:20:20.593746+0000","last_became_active":"2024-09-13T15:19:58.640467+0000","last_became_peered":"2024-09-13T15:19:58.640467+0000","last_unstale":"2024-09-13T15:20:20.593746+0000","last_undegraded":"2024-09-13T15:20:20.593746+0000","last_fullsized":"2024-09-13T15:20:20.593746+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:09.661080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:09.661080+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:23:21.977630+0000","scrub_duration":0.0033469020000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":331,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.923624+0000","last_change":"2024-09-13T15:05:34.078805+0000","last_active":"2024-09-13T15:20:21.923624+0000","last_peered":"2024-09-13T15:20:21.923624+0000","last_clean":"2024-09-13T15:20:21.923624+0000","last_became_active":"2024-09-13T15:05:34.078180+0000","last_became_peered":"2024-09-13T15:05:34.078180+0000","last_unstale":"2024-09-13T15:20:21.923624+0000","last_undegraded":"2024-09-13T15:20:21.923624+0000","last_fullsized":"2024-09-13T15:20:21.923624+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:47.240906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1470,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.396097+0000","last_change":"2024-09-13T15:05:34.076149+0000","last_active":"2024-09-13T15:20:21.396097+0000","last_peered":"2024-09-13T15:20:21.396097+0000","last_clean":"2024-09-13T15:20:21.396097+0000","last_became_active":"2024-09-13T15:05:34.075623+0000","last_became_peered":"2024-09-13T15:05:34.075623+0000","last_unstale":"2024-09-13T15:20:21.396097+0000","last_undegraded":"2024-09-13T15:20:21.396097+0000","last_fullsized":"2024-09-13T15:20:21.396097+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:23.608262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":444,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.827261+0000","last_change":"2024-09-13T15:08:52.189786+0000","last_active":"2024-09-13T15:20:21.827261+0000","last_peered":"2024-09-13T15:20:21.827261+0000","last_clean":"2024-09-13T15:20:21.827261+0000","last_became_active":"2024-09-13T15:08:52.188997+0000","last_became_peered":"2024-09-13T15:08:52.188997+0000","last_unstale":"2024-09-13T15:20:21.827261+0000","last_undegraded":"2024-09-13T15:20:21.827261+0000","last_fullsized":"2024-09-13T15:20:21.827261+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:08.932986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":424,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.822367+0000","last_change":"2024-09-13T15:18:41.593847+0000","last_active":"2024-09-13T15:20:21.822367+0000","last_peered":"2024-09-13T15:20:21.822367+0000","last_clean":"2024-09-13T15:20:21.822367+0000","last_became_active":"2024-09-13T15:18:41.593448+0000","last_became_peered":"2024-09-13T15:18:41.593448+0000","last_unstale":"2024-09-13T15:20:21.822367+0000","last_undegraded":"2024-09-13T15:20:21.822367+0000","last_fullsized":"2024-09-13T15:20:21.822367+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:31.693036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"346'100","reported_seq":32,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.593827+0000","last_change":"2024-09-13T15:20:08.706099+0000","last_active":"2024-09-13T15:20:20.593827+0000","last_peered":"2024-09-13T15:20:20.593827+0000","last_clean":"2024-09-13T15:20:20.593827+0000","last_became_active":"2024-09-13T15:19:58.637184+0000","last_became_peered":"2024-09-13T15:19:58.637184+0000","last_unstale":"2024-09-13T15:20:20.593827+0000","last_undegraded":"2024-09-13T15:20:20.593827+0000","last_fullsized":"2024-09-13T15:20:20.593827+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:08.705948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:08.705948+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:20:00.634507+0000","scrub_duration":0.0041351900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":437,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.827236+0000","last_change":"2024-09-13T15:18:41.598169+0000","last_active":"2024-09-13T15:20:21.827236+0000","last_peered":"2024-09-13T15:20:21.827236+0000","last_clean":"2024-09-13T15:20:21.827236+0000","last_became_active":"2024-09-13T15:18:41.597452+0000","last_became_peered":"2024-09-13T15:18:41.597452+0000","last_unstale":"2024-09-13T15:20:21.827236+0000","last_undegraded":"2024-09-13T15:20:21.827236+0000","last_fullsized":"2024-09-13T15:20:21.827236+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:09:37.791932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":83,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.467911+0000","last_change":"2024-09-13T15:08:52.182709+0000","last_active":"2024-09-13T15:20:20.467911+0000","last_peered":"2024-09-13T15:20:20.467911+0000","last_clean":"2024-09-13T15:20:20.467911+0000","last_became_active":"2024-09-13T15:08:52.181906+0000","last_became_peered":"2024-09-13T15:08:52.181906+0000","last_unstale":"2024-09-13T15:20:20.467911+0000","last_undegraded":"2024-09-13T15:20:20.467911+0000","last_fullsized":"2024-09-13T15:20:20.467911+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:36.317662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"346'100","reported_seq":1290,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.085796+0000","last_change":"2024-09-13T15:19:58.639586+0000","last_active":"2024-09-13T15:20:21.085796+0000","last_peered":"2024-09-13T15:20:21.085796+0000","last_clean":"2024-09-13T15:20:21.085796+0000","last_became_active":"2024-09-13T15:19:58.638975+0000","last_became_peered":"2024-09-13T15:19:58.638975+0000","last_unstale":"2024-09-13T15:20:21.085796+0000","last_undegraded":"2024-09-13T15:20:21.085796+0000","last_fullsized":"2024-09-13T15:20:21.085796+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:18:04.969997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:18:04.969997+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:51.382815+0000","scrub_duration":0.0031202019999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":441,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.824151+0000","last_change":"2024-09-13T15:08:52.182885+0000","last_active":"2024-09-13T15:20:21.824151+0000","last_peered":"2024-09-13T15:20:21.824151+0000","last_clean":"2024-09-13T15:20:21.824151+0000","last_became_active":"2024-09-13T15:08:52.180608+0000","last_became_peered":"2024-09-13T15:08:52.180608+0000","last_unstale":"2024-09-13T15:20:21.824151+0000","last_undegraded":"2024-09-13T15:20:21.824151+0000","last_fullsized":"2024-09-13T15:20:21.824151+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:12.517462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1454,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.397405+0000","last_change":"2024-09-13T15:05:34.085458+0000","last_active":"2024-09-13T15:20:21.397405+0000","last_peered":"2024-09-13T15:20:21.397405+0000","last_clean":"2024-09-13T15:20:21.397405+0000","last_became_active":"2024-09-13T15:05:34.084748+0000","last_became_peered":"2024-09-13T15:05:34.084748+0000","last_unstale":"2024-09-13T15:20:21.397405+0000","last_undegraded":"2024-09-13T15:20:21.397405+0000","last_fullsized":"2024-09-13T15:20:21.397405+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:21.898959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1751,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.919760+0000","last_change":"2024-09-13T15:18:41.638220+0000","last_active":"2024-09-13T15:20:21.919760+0000","last_peered":"2024-09-13T15:20:21.919760+0000","last_clean":"2024-09-13T15:20:21.919760+0000","last_became_active":"2024-09-13T15:18:41.637822+0000","last_became_peered":"2024-09-13T15:18:41.637822+0000","last_unstale":"2024-09-13T15:20:21.919760+0000","last_undegraded":"2024-09-13T15:20:21.919760+0000","last_fullsized":"2024-09-13T15:20:21.919760+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:59:28.421185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1145,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.594052+0000","last_change":"2024-09-13T15:08:52.175569+0000","last_active":"2024-09-13T15:20:20.594052+0000","last_peered":"2024-09-13T15:20:20.594052+0000","last_clean":"2024-09-13T15:20:20.594052+0000","last_became_active":"2024-09-13T15:08:52.174891+0000","last_became_peered":"2024-09-13T15:08:52.174891+0000","last_unstale":"2024-09-13T15:20:20.594052+0000","last_undegraded":"2024-09-13T15:20:20.594052+0000","last_fullsized":"2024-09-13T15:20:20.594052+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.933115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.467971+0000","last_change":"2024-09-13T15:08:52.188074+0000","last_active":"2024-09-13T15:20:20.467971+0000","last_peered":"2024-09-13T15:20:20.467971+0000","last_clean":"2024-09-13T15:20:20.467971+0000","last_became_active":"2024-09-13T15:08:52.186649+0000","last_became_peered":"2024-09-13T15:08:52.186649+0000","last_unstale":"2024-09-13T15:20:20.467971+0000","last_undegraded":"2024-09-13T15:20:20.467971+0000","last_fullsized":"2024-09-13T15:20:20.467971+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:23.478250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":1124,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.086425+0000","last_change":"2024-09-13T15:08:52.208388+0000","last_active":"2024-09-13T15:20:21.086425+0000","last_peered":"2024-09-13T15:20:21.086425+0000","last_clean":"2024-09-13T15:20:21.086425+0000","last_became_active":"2024-09-13T15:08:52.192253+0000","last_became_peered":"2024-09-13T15:08:52.192253+0000","last_unstale":"2024-09-13T15:20:21.086425+0000","last_undegraded":"2024-09-13T15:20:21.086425+0000","last_fullsized":"2024-09-13T15:20:21.086425+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:07.508774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":442,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.822904+0000","last_change":"2024-09-13T15:05:34.072278+0000","last_active":"2024-09-13T15:20:21.822904+0000","last_peered":"2024-09-13T15:20:21.822904+0000","last_clean":"2024-09-13T15:20:21.822904+0000","last_became_active":"2024-09-13T15:05:34.071723+0000","last_became_peered":"2024-09-13T15:05:34.071723+0000","last_unstale":"2024-09-13T15:20:21.822904+0000","last_undegraded":"2024-09-13T15:20:21.822904+0000","last_fullsized":"2024-09-13T15:20:21.822904+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:02.130890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1187,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.086673+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T15:20:21.086673+0000","last_peered":"2024-09-13T15:20:21.086673+0000","last_clean":"2024-09-13T15:20:21.086673+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T15:20:21.086673+0000","last_undegraded":"2024-09-13T15:20:21.086673+0000","last_fullsized":"2024-09-13T15:20:21.086673+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1110,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.594178+0000","last_change":"2024-09-13T15:08:52.176830+0000","last_active":"2024-09-13T15:20:20.594178+0000","last_peered":"2024-09-13T15:20:20.594178+0000","last_clean":"2024-09-13T15:20:20.594178+0000","last_became_active":"2024-09-13T15:08:52.176303+0000","last_became_peered":"2024-09-13T15:08:52.176303+0000","last_unstale":"2024-09-13T15:20:20.594178+0000","last_undegraded":"2024-09-13T15:20:20.594178+0000","last_fullsized":"2024-09-13T15:20:20.594178+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:59.553257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1158,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.086644+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T15:20:21.086644+0000","last_peered":"2024-09-13T15:20:21.086644+0000","last_clean":"2024-09-13T15:20:21.086644+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T15:20:21.086644+0000","last_undegraded":"2024-09-13T15:20:21.086644+0000","last_fullsized":"2024-09-13T15:20:21.086644+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":93,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.466404+0000","last_change":"2024-09-13T15:08:52.200695+0000","last_active":"2024-09-13T15:20:20.466404+0000","last_peered":"2024-09-13T15:20:20.466404+0000","last_clean":"2024-09-13T15:20:20.466404+0000","last_became_active":"2024-09-13T15:08:52.183473+0000","last_became_peered":"2024-09-13T15:08:52.183473+0000","last_unstale":"2024-09-13T15:20:20.466404+0000","last_undegraded":"2024-09-13T15:20:20.466404+0000","last_fullsized":"2024-09-13T15:20:20.466404+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:42.810003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1468,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.398325+0000","last_change":"2024-09-13T15:05:34.074069+0000","last_active":"2024-09-13T15:20:21.398325+0000","last_peered":"2024-09-13T15:20:21.398325+0000","last_clean":"2024-09-13T15:20:21.398325+0000","last_became_active":"2024-09-13T15:05:34.073282+0000","last_became_peered":"2024-09-13T15:05:34.073282+0000","last_unstale":"2024-09-13T15:20:21.398325+0000","last_undegraded":"2024-09-13T15:20:21.398325+0000","last_fullsized":"2024-09-13T15:20:21.398325+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:14.307582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1151,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.085900+0000","last_change":"2024-09-13T15:08:52.192368+0000","last_active":"2024-09-13T15:20:21.085900+0000","last_peered":"2024-09-13T15:20:21.085900+0000","last_clean":"2024-09-13T15:20:21.085900+0000","last_became_active":"2024-09-13T15:08:52.191668+0000","last_became_peered":"2024-09-13T15:08:52.191668+0000","last_unstale":"2024-09-13T15:20:21.085900+0000","last_undegraded":"2024-09-13T15:20:21.085900+0000","last_fullsized":"2024-09-13T15:20:21.085900+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:27.410222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1708,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.921054+0000","last_change":"2024-09-13T15:05:34.087636+0000","last_active":"2024-09-13T15:20:21.921054+0000","last_peered":"2024-09-13T15:20:21.921054+0000","last_clean":"2024-09-13T15:20:21.921054+0000","last_became_active":"2024-09-13T15:05:34.087255+0000","last_became_peered":"2024-09-13T15:05:34.087255+0000","last_unstale":"2024-09-13T15:20:21.921054+0000","last_undegraded":"2024-09-13T15:20:21.921054+0000","last_fullsized":"2024-09-13T15:20:21.921054+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:36.313581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1708,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.918321+0000","last_change":"2024-09-13T15:05:34.089162+0000","last_active":"2024-09-13T15:20:21.918321+0000","last_peered":"2024-09-13T15:20:21.918321+0000","last_clean":"2024-09-13T15:20:21.918321+0000","last_became_active":"2024-09-13T15:05:34.088727+0000","last_became_peered":"2024-09-13T15:05:34.088727+0000","last_unstale":"2024-09-13T15:20:21.918321+0000","last_undegraded":"2024-09-13T15:20:21.918321+0000","last_fullsized":"2024-09-13T15:20:21.918321+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:29.209082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1124,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:20.594338+0000","last_change":"2024-09-13T15:05:34.073363+0000","last_active":"2024-09-13T15:20:20.594338+0000","last_peered":"2024-09-13T15:20:20.594338+0000","last_clean":"2024-09-13T15:20:20.594338+0000","last_became_active":"2024-09-13T15:05:34.072906+0000","last_became_peered":"2024-09-13T15:05:34.072906+0000","last_unstale":"2024-09-13T15:20:20.594338+0000","last_undegraded":"2024-09-13T15:20:20.594338+0000","last_fullsized":"2024-09-13T15:20:20.594338+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:27.966654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1180,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.086355+0000","last_change":"2024-09-13T15:18:41.591934+0000","last_active":"2024-09-13T15:20:21.086355+0000","last_peered":"2024-09-13T15:20:21.086355+0000","last_clean":"2024-09-13T15:20:21.086355+0000","last_became_active":"2024-09-13T15:18:41.582782+0000","last_became_peered":"2024-09-13T15:18:41.582782+0000","last_unstale":"2024-09-13T15:20:21.086355+0000","last_undegraded":"2024-09-13T15:20:21.086355+0000","last_fullsized":"2024-09-13T15:20:21.086355+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:18.757781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1479,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-09-13T15:20:21.401648+0000","last_change":"2024-09-13T15:05:34.075192+0000","last_active":"2024-09-13T15:20:21.401648+0000","last_peered":"2024-09-13T15:20:21.401648+0000","last_clean":"2024-09-13T15:20:21.401648+0000","last_became_active":"2024-09-13T15:05:34.073315+0000","last_became_peered":"2024-09-13T15:05:34.073315+0000","last_unstale":"2024-09-13T15:20:21.401648+0000","last_undegraded":"2024-09-13T15:20:21.401648+0000","last_fullsized":"2024-09-13T15:20:21.401648+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:38.699775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":26,"num_read_kb":28,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":16978,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398233,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":994764,"kb_used_data":98692,"kb_used_omap":1,"kb_used_meta":896062,"kb_avail":92750388,"statfs":{"total":95995035648,"available":94976397312,"internally_reserved":0,"allocated":101060608,"data_stored":96174253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917567832},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594468,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900632,"kb_used_data":12808,"kb_used_omap":1,"kb_used_meta":887806,"kb_avail":92844520,"statfs":{"total":95995035648,"available":95072788480,"internally_reserved":0,"allocated":13115392,"data_stored":8225954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":909113688},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790702,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903368,"kb_used_data":13112,"kb_used_omap":1,"kb_used_meta":890238,"kb_avail":92841784,"statfs":{"total":95995035648,"available":95069986816,"internally_reserved":0,"allocated":13426688,"data_stored":8548848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":911604056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980832231,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":746832,"kb_used_data":12548,"kb_used_omap":76,"kb_used_meta":734195,"kb_avail":92998320,"statfs":{"total":95995035648,"available":95230279680,"internally_reserved":0,"allocated":12849152,"data_stored":7963309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":751816512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1066,"seq":4578435137717,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":386992,"kb_used_data":13104,"kb_used_omap":121,"kb_used_meta":373702,"kb_avail":93358160,"statfs":{"total":95995035648,"available":95598755840,"internally_reserved":0,"allocated":13418496,"data_stored":8548315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124093,"internal_metadata":382671683},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":2886218023413,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004542,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":793840,"kb_used_data":12524,"kb_used_omap":66,"kb_used_meta":781181,"kb_avail":92951312,"statfs":{"total":95995035648,"available":95182143488,"internally_reserved":0,"allocated":12824576,"data_stored":7957876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":799929514},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195734,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":966772,"kb_used_data":12520,"kb_used_omap":56,"kb_used_meta":954183,"kb_avail":92778380,"statfs":{"total":95995035648,"available":95005061120,"internally_reserved":0,"allocated":12820480,"data_stored":7955757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":977083535},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:20:36.873 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-13T15:20:37.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:20:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:20:37] "GET /metrics HTTP/1.1" 200 34344 "" "Prometheus/2.51.0" 2024-09-13T15:20:37.612 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:20:37.623 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:37 smithi099 bash[87147]: cluster 2024-09-13T15:20:37.282768+0000 mgr.y (mgr.26252) 1640 : cluster [DBG] pgmap v2278: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:37.623 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:37 smithi099 bash[87147]: cluster 2024-09-13T15:20:37.282768+0000 mgr.y (mgr.26252) 1640 : cluster [DBG] pgmap v2278: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:37.623 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:37 smithi099 bash[84576]: cluster 2024-09-13T15:20:37.282768+0000 mgr.y (mgr.26252) 1640 : cluster [DBG] pgmap v2278: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:37.624 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:37 smithi099 bash[84576]: cluster 2024-09-13T15:20:37.282768+0000 mgr.y (mgr.26252) 1640 : cluster [DBG] pgmap v2278: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:37.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:37 smithi177 bash[68594]: cluster 2024-09-13T15:20:37.282768+0000 mgr.y (mgr.26252) 1640 : cluster [DBG] pgmap v2278: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:37.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:37 smithi177 bash[68594]: cluster 2024-09-13T15:20:37.282768+0000 mgr.y (mgr.26252) 1640 : cluster [DBG] pgmap v2278: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:39.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:39 smithi099 bash[84576]: cluster 2024-09-13T15:20:39.283646+0000 mgr.y (mgr.26252) 1641 : cluster [DBG] pgmap v2279: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:39.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:39 smithi099 bash[84576]: cluster 2024-09-13T15:20:39.283646+0000 mgr.y (mgr.26252) 1641 : cluster [DBG] pgmap v2279: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:39.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:39 smithi099 bash[87147]: cluster 2024-09-13T15:20:39.283646+0000 mgr.y (mgr.26252) 1641 : cluster [DBG] pgmap v2279: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:39.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:39 smithi099 bash[87147]: cluster 2024-09-13T15:20:39.283646+0000 mgr.y (mgr.26252) 1641 : cluster [DBG] pgmap v2279: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:39 smithi177 bash[68594]: cluster 2024-09-13T15:20:39.283646+0000 mgr.y (mgr.26252) 1641 : cluster [DBG] pgmap v2279: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:39 smithi177 bash[68594]: cluster 2024-09-13T15:20:39.283646+0000 mgr.y (mgr.26252) 1641 : cluster [DBG] pgmap v2279: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:40.401 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:20:40.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:20:40 smithi177 bash[61439]: ts=2024-09-13T15:20:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:20:41.417 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:20:41.418 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:20:41.771 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:41 smithi099 bash[84576]: cluster 2024-09-13T15:20:41.284550+0000 mgr.y (mgr.26252) 1642 : cluster [DBG] pgmap v2280: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:41.771 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:41 smithi099 bash[84576]: cluster 2024-09-13T15:20:41.284550+0000 mgr.y (mgr.26252) 1642 : cluster [DBG] pgmap v2280: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:41.771 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:41 smithi099 bash[87147]: cluster 2024-09-13T15:20:41.284550+0000 mgr.y (mgr.26252) 1642 : cluster [DBG] pgmap v2280: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:41.771 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:41 smithi099 bash[87147]: cluster 2024-09-13T15:20:41.284550+0000 mgr.y (mgr.26252) 1642 : cluster [DBG] pgmap v2280: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:41 smithi177 bash[68594]: cluster 2024-09-13T15:20:41.284550+0000 mgr.y (mgr.26252) 1642 : cluster [DBG] pgmap v2280: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:41 smithi177 bash[68594]: cluster 2024-09-13T15:20:41.284550+0000 mgr.y (mgr.26252) 1642 : cluster [DBG] pgmap v2280: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:43.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:43 smithi099 bash[84576]: cluster 2024-09-13T15:20:43.285367+0000 mgr.y (mgr.26252) 1643 : cluster [DBG] pgmap v2281: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:43.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:43 smithi099 bash[84576]: cluster 2024-09-13T15:20:43.285367+0000 mgr.y (mgr.26252) 1643 : cluster [DBG] pgmap v2281: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:43.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:43 smithi099 bash[87147]: cluster 2024-09-13T15:20:43.285367+0000 mgr.y (mgr.26252) 1643 : cluster [DBG] pgmap v2281: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:43.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:43 smithi099 bash[87147]: cluster 2024-09-13T15:20:43.285367+0000 mgr.y (mgr.26252) 1643 : cluster [DBG] pgmap v2281: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:43 smithi177 bash[68594]: cluster 2024-09-13T15:20:43.285367+0000 mgr.y (mgr.26252) 1643 : cluster [DBG] pgmap v2281: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:43 smithi177 bash[68594]: cluster 2024-09-13T15:20:43.285367+0000 mgr.y (mgr.26252) 1643 : cluster [DBG] pgmap v2281: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:45.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:45 smithi099 bash[84576]: cluster 2024-09-13T15:20:45.286287+0000 mgr.y (mgr.26252) 1644 : cluster [DBG] pgmap v2282: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:45.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:45 smithi099 bash[84576]: cluster 2024-09-13T15:20:45.286287+0000 mgr.y (mgr.26252) 1644 : cluster [DBG] pgmap v2282: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:45.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:45 smithi099 bash[87147]: cluster 2024-09-13T15:20:45.286287+0000 mgr.y (mgr.26252) 1644 : cluster [DBG] pgmap v2282: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:45.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:45 smithi099 bash[87147]: cluster 2024-09-13T15:20:45.286287+0000 mgr.y (mgr.26252) 1644 : cluster [DBG] pgmap v2282: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:45.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:45 smithi177 bash[68594]: cluster 2024-09-13T15:20:45.286287+0000 mgr.y (mgr.26252) 1644 : cluster [DBG] pgmap v2282: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:45.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:45 smithi177 bash[68594]: cluster 2024-09-13T15:20:45.286287+0000 mgr.y (mgr.26252) 1644 : cluster [DBG] pgmap v2282: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:46.867 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:46 smithi099 bash[84576]: audit 2024-09-13T15:20:46.559516+0000 mon.a (mon.0) 2838 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:20:46.867 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:46 smithi099 bash[84576]: audit 2024-09-13T15:20:46.559516+0000 mon.a (mon.0) 2838 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:20:46.867 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:46 smithi099 bash[87147]: audit 2024-09-13T15:20:46.559516+0000 mon.a (mon.0) 2838 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:20:46.867 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:46 smithi099 bash[87147]: audit 2024-09-13T15:20:46.559516+0000 mon.a (mon.0) 2838 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:20:47.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:46 smithi177 bash[68594]: audit 2024-09-13T15:20:46.559516+0000 mon.a (mon.0) 2838 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:20:47.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:46 smithi177 bash[68594]: audit 2024-09-13T15:20:46.559516+0000 mon.a (mon.0) 2838 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:20:47.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:20:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:20:47] "GET /metrics HTTP/1.1" 200 34344 "" "Prometheus/2.51.0" 2024-09-13T15:20:47.953 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:47 smithi099 bash[84576]: cluster 2024-09-13T15:20:47.287280+0000 mgr.y (mgr.26252) 1645 : cluster [DBG] pgmap v2283: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:47.954 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:47 smithi099 bash[84576]: cluster 2024-09-13T15:20:47.287280+0000 mgr.y (mgr.26252) 1645 : cluster [DBG] pgmap v2283: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:47.954 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:47 smithi099 bash[87147]: cluster 2024-09-13T15:20:47.287280+0000 mgr.y (mgr.26252) 1645 : cluster [DBG] pgmap v2283: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:47.954 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:47 smithi099 bash[87147]: cluster 2024-09-13T15:20:47.287280+0000 mgr.y (mgr.26252) 1645 : cluster [DBG] pgmap v2283: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:48.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:47 smithi177 bash[68594]: cluster 2024-09-13T15:20:47.287280+0000 mgr.y (mgr.26252) 1645 : cluster [DBG] pgmap v2283: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:48.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:47 smithi177 bash[68594]: cluster 2024-09-13T15:20:47.287280+0000 mgr.y (mgr.26252) 1645 : cluster [DBG] pgmap v2283: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:48.886 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:48 smithi099 bash[84576]: audit 2024-09-13T15:20:48.326011+0000 mon.a (mon.0) 2839 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:48.886 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:48 smithi099 bash[84576]: audit 2024-09-13T15:20:48.326011+0000 mon.a (mon.0) 2839 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:48.886 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:48 smithi099 bash[87147]: audit 2024-09-13T15:20:48.326011+0000 mon.a (mon.0) 2839 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:48.886 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:48 smithi099 bash[87147]: audit 2024-09-13T15:20:48.326011+0000 mon.a (mon.0) 2839 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:49.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:48 smithi177 bash[68594]: audit 2024-09-13T15:20:48.326011+0000 mon.a (mon.0) 2839 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:49.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:48 smithi177 bash[68594]: audit 2024-09-13T15:20:48.326011+0000 mon.a (mon.0) 2839 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:20:49.884 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:49 smithi099 bash[84576]: cluster 2024-09-13T15:20:49.288089+0000 mgr.y (mgr.26252) 1646 : cluster [DBG] pgmap v2284: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:49.884 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:49 smithi099 bash[84576]: cluster 2024-09-13T15:20:49.288089+0000 mgr.y (mgr.26252) 1646 : cluster [DBG] pgmap v2284: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:49.884 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:49 smithi099 bash[87147]: cluster 2024-09-13T15:20:49.288089+0000 mgr.y (mgr.26252) 1646 : cluster [DBG] pgmap v2284: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:49.885 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:49 smithi099 bash[87147]: cluster 2024-09-13T15:20:49.288089+0000 mgr.y (mgr.26252) 1646 : cluster [DBG] pgmap v2284: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:50.015 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:20:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:20:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:20:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:20:50.016 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:20:50.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:49 smithi177 bash[68594]: cluster 2024-09-13T15:20:49.288089+0000 mgr.y (mgr.26252) 1646 : cluster [DBG] pgmap v2284: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:50.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:49 smithi177 bash[68594]: cluster 2024-09-13T15:20:49.288089+0000 mgr.y (mgr.26252) 1646 : cluster [DBG] pgmap v2284: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:50.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:20:50 smithi177 bash[61439]: ts=2024-09-13T15:20:50.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:20:51.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:51 smithi099 bash[84576]: cluster 2024-09-13T15:20:51.289066+0000 mgr.y (mgr.26252) 1647 : cluster [DBG] pgmap v2285: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:51.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:51 smithi099 bash[84576]: cluster 2024-09-13T15:20:51.289066+0000 mgr.y (mgr.26252) 1647 : cluster [DBG] pgmap v2285: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:51.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:51 smithi099 bash[87147]: cluster 2024-09-13T15:20:51.289066+0000 mgr.y (mgr.26252) 1647 : cluster [DBG] pgmap v2285: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:51.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:51 smithi099 bash[87147]: cluster 2024-09-13T15:20:51.289066+0000 mgr.y (mgr.26252) 1647 : cluster [DBG] pgmap v2285: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:51 smithi177 bash[68594]: cluster 2024-09-13T15:20:51.289066+0000 mgr.y (mgr.26252) 1647 : cluster [DBG] pgmap v2285: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:51 smithi177 bash[68594]: cluster 2024-09-13T15:20:51.289066+0000 mgr.y (mgr.26252) 1647 : cluster [DBG] pgmap v2285: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:52.375 INFO:teuthology.orchestra.run.smithi099.stderr:noscrub is unset 2024-09-13T15:20:52.421 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T15:20:52.721 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:52 smithi099 bash[84576]: audit 2024-09-13T15:20:51.755723+0000 mon.a (mon.0) 2840 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:52.721 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:52 smithi099 bash[84576]: audit 2024-09-13T15:20:51.755723+0000 mon.a (mon.0) 2840 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:52.721 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:52 smithi099 bash[87147]: audit 2024-09-13T15:20:51.755723+0000 mon.a (mon.0) 2840 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:52.722 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:52 smithi099 bash[87147]: audit 2024-09-13T15:20:51.755723+0000 mon.a (mon.0) 2840 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:52.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:52 smithi177 bash[68594]: audit 2024-09-13T15:20:51.755723+0000 mon.a (mon.0) 2840 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:52.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:52 smithi177 bash[68594]: audit 2024-09-13T15:20:51.755723+0000 mon.a (mon.0) 2840 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:20:53.600 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:20:53.616 INFO:teuthology.orchestra.run.smithi099.stderr:set pool 1 pgp_num to 11 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: audit 2024-09-13T15:20:52.374544+0000 mon.a (mon.0) 2841 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: audit 2024-09-13T15:20:52.374544+0000 mon.a (mon.0) 2841 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: cluster 2024-09-13T15:20:52.381395+0000 mon.a (mon.0) 2842 : cluster [DBG] osdmap e1156: 8 total, 8 up, 7 in 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: cluster 2024-09-13T15:20:52.381395+0000 mon.a (mon.0) 2842 : cluster [DBG] osdmap e1156: 8 total, 8 up, 7 in 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: audit 2024-09-13T15:20:52.594550+0000 mon.a (mon.0) 2843 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: audit 2024-09-13T15:20:52.594550+0000 mon.a (mon.0) 2843 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: audit 2024-09-13T15:20:52.601626+0000 mon.a (mon.0) 2844 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: audit 2024-09-13T15:20:52.601626+0000 mon.a (mon.0) 2844 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: audit 2024-09-13T15:20:52.820543+0000 mon.a (mon.0) 2845 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: audit 2024-09-13T15:20:52.820543+0000 mon.a (mon.0) 2845 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: cluster 2024-09-13T15:20:53.289702+0000 mgr.y (mgr.26252) 1648 : cluster [DBG] pgmap v2287: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:53.636 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[84576]: cluster 2024-09-13T15:20:53.289702+0000 mgr.y (mgr.26252) 1648 : cluster [DBG] pgmap v2287: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: audit 2024-09-13T15:20:52.374544+0000 mon.a (mon.0) 2841 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: audit 2024-09-13T15:20:52.374544+0000 mon.a (mon.0) 2841 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: cluster 2024-09-13T15:20:52.381395+0000 mon.a (mon.0) 2842 : cluster [DBG] osdmap e1156: 8 total, 8 up, 7 in 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: cluster 2024-09-13T15:20:52.381395+0000 mon.a (mon.0) 2842 : cluster [DBG] osdmap e1156: 8 total, 8 up, 7 in 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: audit 2024-09-13T15:20:52.594550+0000 mon.a (mon.0) 2843 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: audit 2024-09-13T15:20:52.594550+0000 mon.a (mon.0) 2843 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: audit 2024-09-13T15:20:52.601626+0000 mon.a (mon.0) 2844 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: audit 2024-09-13T15:20:52.601626+0000 mon.a (mon.0) 2844 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: audit 2024-09-13T15:20:52.820543+0000 mon.a (mon.0) 2845 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: audit 2024-09-13T15:20:52.820543+0000 mon.a (mon.0) 2845 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: cluster 2024-09-13T15:20:53.289702+0000 mgr.y (mgr.26252) 1648 : cluster [DBG] pgmap v2287: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:53.637 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:53 smithi099 bash[87147]: cluster 2024-09-13T15:20:53.289702+0000 mgr.y (mgr.26252) 1648 : cluster [DBG] pgmap v2287: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: audit 2024-09-13T15:20:52.374544+0000 mon.a (mon.0) 2841 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: audit 2024-09-13T15:20:52.374544+0000 mon.a (mon.0) 2841 : audit [INF] from='client.? 172.21.15.99:0/186067238' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: cluster 2024-09-13T15:20:52.381395+0000 mon.a (mon.0) 2842 : cluster [DBG] osdmap e1156: 8 total, 8 up, 7 in 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: cluster 2024-09-13T15:20:52.381395+0000 mon.a (mon.0) 2842 : cluster [DBG] osdmap e1156: 8 total, 8 up, 7 in 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: audit 2024-09-13T15:20:52.594550+0000 mon.a (mon.0) 2843 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: audit 2024-09-13T15:20:52.594550+0000 mon.a (mon.0) 2843 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: audit 2024-09-13T15:20:52.601626+0000 mon.a (mon.0) 2844 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: audit 2024-09-13T15:20:52.601626+0000 mon.a (mon.0) 2844 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: audit 2024-09-13T15:20:52.820543+0000 mon.a (mon.0) 2845 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: audit 2024-09-13T15:20:52.820543+0000 mon.a (mon.0) 2845 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: cluster 2024-09-13T15:20:53.289702+0000 mgr.y (mgr.26252) 1648 : cluster [DBG] pgmap v2287: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:53 smithi177 bash[68594]: cluster 2024-09-13T15:20:53.289702+0000 mgr.y (mgr.26252) 1648 : cluster [DBG] pgmap v2287: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:54.739 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: audit 2024-09-13T15:20:53.425478+0000 mon.a (mon.0) 2846 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-09-13T15:20:54.739 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: audit 2024-09-13T15:20:53.425478+0000 mon.a (mon.0) 2846 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-09-13T15:20:54.739 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: audit 2024-09-13T15:20:53.426150+0000 mon.a (mon.0) 2847 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-09-13T15:20:54.739 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: audit 2024-09-13T15:20:53.426150+0000 mon.a (mon.0) 2847 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-09-13T15:20:54.739 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: audit 2024-09-13T15:20:53.616085+0000 mon.a (mon.0) 2848 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T15:20:54.739 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: audit 2024-09-13T15:20:53.616085+0000 mon.a (mon.0) 2848 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T15:20:54.739 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: audit 2024-09-13T15:20:53.616172+0000 mon.a (mon.0) 2849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-13T15:20:54.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: audit 2024-09-13T15:20:53.616172+0000 mon.a (mon.0) 2849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-13T15:20:54.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: audit 2024-09-13T15:20:53.616253+0000 mon.a (mon.0) 2850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2024-09-13T15:20:54.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: audit 2024-09-13T15:20:53.616253+0000 mon.a (mon.0) 2850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2024-09-13T15:20:54.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: cluster 2024-09-13T15:20:53.621220+0000 mon.a (mon.0) 2851 : cluster [DBG] osdmap e1157: 8 total, 8 up, 7 in 2024-09-13T15:20:54.740 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[84576]: cluster 2024-09-13T15:20:53.621220+0000 mon.a (mon.0) 2851 : cluster [DBG] osdmap e1157: 8 total, 8 up, 7 in 2024-09-13T15:20:54.740 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: audit 2024-09-13T15:20:53.425478+0000 mon.a (mon.0) 2846 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-09-13T15:20:54.740 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: audit 2024-09-13T15:20:53.425478+0000 mon.a (mon.0) 2846 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-09-13T15:20:54.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: audit 2024-09-13T15:20:53.426150+0000 mon.a (mon.0) 2847 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-09-13T15:20:54.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: audit 2024-09-13T15:20:53.426150+0000 mon.a (mon.0) 2847 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-09-13T15:20:54.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: audit 2024-09-13T15:20:53.616085+0000 mon.a (mon.0) 2848 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T15:20:54.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: audit 2024-09-13T15:20:53.616085+0000 mon.a (mon.0) 2848 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T15:20:54.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: audit 2024-09-13T15:20:53.616172+0000 mon.a (mon.0) 2849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-13T15:20:54.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: audit 2024-09-13T15:20:53.616172+0000 mon.a (mon.0) 2849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-13T15:20:54.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: audit 2024-09-13T15:20:53.616253+0000 mon.a (mon.0) 2850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2024-09-13T15:20:54.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: audit 2024-09-13T15:20:53.616253+0000 mon.a (mon.0) 2850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2024-09-13T15:20:54.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: cluster 2024-09-13T15:20:53.621220+0000 mon.a (mon.0) 2851 : cluster [DBG] osdmap e1157: 8 total, 8 up, 7 in 2024-09-13T15:20:54.741 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:54 smithi099 bash[87147]: cluster 2024-09-13T15:20:53.621220+0000 mon.a (mon.0) 2851 : cluster [DBG] osdmap e1157: 8 total, 8 up, 7 in 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: audit 2024-09-13T15:20:53.425478+0000 mon.a (mon.0) 2846 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: audit 2024-09-13T15:20:53.425478+0000 mon.a (mon.0) 2846 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: audit 2024-09-13T15:20:53.426150+0000 mon.a (mon.0) 2847 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: audit 2024-09-13T15:20:53.426150+0000 mon.a (mon.0) 2847 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: audit 2024-09-13T15:20:53.616085+0000 mon.a (mon.0) 2848 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: audit 2024-09-13T15:20:53.616085+0000 mon.a (mon.0) 2848 : audit [INF] from='client.? 172.21.15.99:0/2803244716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: audit 2024-09-13T15:20:53.616172+0000 mon.a (mon.0) 2849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: audit 2024-09-13T15:20:53.616172+0000 mon.a (mon.0) 2849 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: audit 2024-09-13T15:20:53.616253+0000 mon.a (mon.0) 2850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: audit 2024-09-13T15:20:53.616253+0000 mon.a (mon.0) 2850 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: cluster 2024-09-13T15:20:53.621220+0000 mon.a (mon.0) 2851 : cluster [DBG] osdmap e1157: 8 total, 8 up, 7 in 2024-09-13T15:20:54.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:54 smithi177 bash[68594]: cluster 2024-09-13T15:20:53.621220+0000 mon.a (mon.0) 2851 : cluster [DBG] osdmap e1157: 8 total, 8 up, 7 in 2024-09-13T15:20:56.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:55 smithi099 bash[87147]: cluster 2024-09-13T15:20:54.627306+0000 mon.a (mon.0) 2852 : cluster [DBG] osdmap e1158: 8 total, 8 up, 7 in 2024-09-13T15:20:56.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:55 smithi099 bash[87147]: cluster 2024-09-13T15:20:54.627306+0000 mon.a (mon.0) 2852 : cluster [DBG] osdmap e1158: 8 total, 8 up, 7 in 2024-09-13T15:20:56.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:55 smithi099 bash[87147]: cluster 2024-09-13T15:20:55.290099+0000 mgr.y (mgr.26252) 1649 : cluster [DBG] pgmap v2290: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:56.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:55 smithi099 bash[87147]: cluster 2024-09-13T15:20:55.290099+0000 mgr.y (mgr.26252) 1649 : cluster [DBG] pgmap v2290: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:56.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:55 smithi099 bash[84576]: cluster 2024-09-13T15:20:54.627306+0000 mon.a (mon.0) 2852 : cluster [DBG] osdmap e1158: 8 total, 8 up, 7 in 2024-09-13T15:20:56.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:55 smithi099 bash[84576]: cluster 2024-09-13T15:20:54.627306+0000 mon.a (mon.0) 2852 : cluster [DBG] osdmap e1158: 8 total, 8 up, 7 in 2024-09-13T15:20:56.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:55 smithi099 bash[84576]: cluster 2024-09-13T15:20:55.290099+0000 mgr.y (mgr.26252) 1649 : cluster [DBG] pgmap v2290: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:56.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:55 smithi099 bash[84576]: cluster 2024-09-13T15:20:55.290099+0000 mgr.y (mgr.26252) 1649 : cluster [DBG] pgmap v2290: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:56.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:55 smithi177 bash[68594]: cluster 2024-09-13T15:20:54.627306+0000 mon.a (mon.0) 2852 : cluster [DBG] osdmap e1158: 8 total, 8 up, 7 in 2024-09-13T15:20:56.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:55 smithi177 bash[68594]: cluster 2024-09-13T15:20:54.627306+0000 mon.a (mon.0) 2852 : cluster [DBG] osdmap e1158: 8 total, 8 up, 7 in 2024-09-13T15:20:56.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:55 smithi177 bash[68594]: cluster 2024-09-13T15:20:55.290099+0000 mgr.y (mgr.26252) 1649 : cluster [DBG] pgmap v2290: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:56.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:55 smithi177 bash[68594]: cluster 2024-09-13T15:20:55.290099+0000 mgr.y (mgr.26252) 1649 : cluster [DBG] pgmap v2290: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:57.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:56 smithi177 bash[68594]: audit 2024-09-13T15:20:55.852561+0000 mon.a (mon.0) 2853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:56 smithi177 bash[68594]: audit 2024-09-13T15:20:55.852561+0000 mon.a (mon.0) 2853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:56 smithi177 bash[68594]: audit 2024-09-13T15:20:55.859721+0000 mon.a (mon.0) 2854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:56 smithi177 bash[68594]: audit 2024-09-13T15:20:55.859721+0000 mon.a (mon.0) 2854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:56 smithi177 bash[68594]: audit 2024-09-13T15:20:56.527219+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:20:57.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:56 smithi177 bash[68594]: audit 2024-09-13T15:20:56.527219+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:20:57.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:56 smithi177 bash[68594]: audit 2024-09-13T15:20:56.529008+0000 mon.a (mon.0) 2856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:20:57.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:56 smithi177 bash[68594]: audit 2024-09-13T15:20:56.529008+0000 mon.a (mon.0) 2856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:20:57.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:56 smithi177 bash[68594]: audit 2024-09-13T15:20:56.539501+0000 mon.a (mon.0) 2857 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:56 smithi177 bash[68594]: audit 2024-09-13T15:20:56.539501+0000 mon.a (mon.0) 2857 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.254 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:20:57.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:20:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:20:57] "GET /metrics HTTP/1.1" 200 34346 "" "Prometheus/2.51.0" 2024-09-13T15:20:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[84576]: audit 2024-09-13T15:20:55.852561+0000 mon.a (mon.0) 2853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[84576]: audit 2024-09-13T15:20:55.852561+0000 mon.a (mon.0) 2853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[84576]: audit 2024-09-13T15:20:55.859721+0000 mon.a (mon.0) 2854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[84576]: audit 2024-09-13T15:20:55.859721+0000 mon.a (mon.0) 2854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[84576]: audit 2024-09-13T15:20:56.527219+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:20:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[84576]: audit 2024-09-13T15:20:56.527219+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:20:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[84576]: audit 2024-09-13T15:20:56.529008+0000 mon.a (mon.0) 2856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:20:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[84576]: audit 2024-09-13T15:20:56.529008+0000 mon.a (mon.0) 2856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:20:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[84576]: audit 2024-09-13T15:20:56.539501+0000 mon.a (mon.0) 2857 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[84576]: audit 2024-09-13T15:20:56.539501+0000 mon.a (mon.0) 2857 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[87147]: audit 2024-09-13T15:20:55.852561+0000 mon.a (mon.0) 2853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[87147]: audit 2024-09-13T15:20:55.852561+0000 mon.a (mon.0) 2853 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[87147]: audit 2024-09-13T15:20:55.859721+0000 mon.a (mon.0) 2854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[87147]: audit 2024-09-13T15:20:55.859721+0000 mon.a (mon.0) 2854 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[87147]: audit 2024-09-13T15:20:56.527219+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:20:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[87147]: audit 2024-09-13T15:20:56.527219+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:20:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[87147]: audit 2024-09-13T15:20:56.529008+0000 mon.a (mon.0) 2856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:20:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[87147]: audit 2024-09-13T15:20:56.529008+0000 mon.a (mon.0) 2856 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:20:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[87147]: audit 2024-09-13T15:20:56.539501+0000 mon.a (mon.0) 2857 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:56 smithi099 bash[87147]: audit 2024-09-13T15:20:56.539501+0000 mon.a (mon.0) 2857 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:20:58.273 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:20:58.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:57 smithi099 bash[84576]: cluster 2024-09-13T15:20:57.290868+0000 mgr.y (mgr.26252) 1650 : cluster [DBG] pgmap v2291: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:58.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:57 smithi099 bash[84576]: cluster 2024-09-13T15:20:57.290868+0000 mgr.y (mgr.26252) 1650 : cluster [DBG] pgmap v2291: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:58.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:57 smithi099 bash[87147]: cluster 2024-09-13T15:20:57.290868+0000 mgr.y (mgr.26252) 1650 : cluster [DBG] pgmap v2291: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:58.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:57 smithi099 bash[87147]: cluster 2024-09-13T15:20:57.290868+0000 mgr.y (mgr.26252) 1650 : cluster [DBG] pgmap v2291: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:58.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:57 smithi177 bash[68594]: cluster 2024-09-13T15:20:57.290868+0000 mgr.y (mgr.26252) 1650 : cluster [DBG] pgmap v2291: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:58.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:57 smithi177 bash[68594]: cluster 2024-09-13T15:20:57.290868+0000 mgr.y (mgr.26252) 1650 : cluster [DBG] pgmap v2291: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:59.743 INFO:tasks.thrashosds.thrasher:inning osd 2024-09-13T15:20:59.743 INFO:tasks.thrashosds.thrasher:Adding osd 2 2024-09-13T15:20:59.744 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd in 2 2024-09-13T15:20:59.755 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:59 smithi099 bash[84576]: cluster 2024-09-13T15:20:59.291626+0000 mgr.y (mgr.26252) 1651 : cluster [DBG] pgmap v2292: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:59.755 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:20:59 smithi099 bash[84576]: cluster 2024-09-13T15:20:59.291626+0000 mgr.y (mgr.26252) 1651 : cluster [DBG] pgmap v2292: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:59.756 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:59 smithi099 bash[87147]: cluster 2024-09-13T15:20:59.291626+0000 mgr.y (mgr.26252) 1651 : cluster [DBG] pgmap v2292: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:59.756 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:20:59 smithi099 bash[87147]: cluster 2024-09-13T15:20:59.291626+0000 mgr.y (mgr.26252) 1651 : cluster [DBG] pgmap v2292: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:59.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:59 smithi177 bash[68594]: cluster 2024-09-13T15:20:59.291626+0000 mgr.y (mgr.26252) 1651 : cluster [DBG] pgmap v2292: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:20:59.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:20:59 smithi177 bash[68594]: cluster 2024-09-13T15:20:59.291626+0000 mgr.y (mgr.26252) 1651 : cluster [DBG] pgmap v2292: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:00.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:21:00 smithi177 bash[61439]: ts=2024-09-13T15:21:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:21:01.705 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:01 smithi099 bash[84576]: cluster 2024-09-13T15:21:01.292473+0000 mgr.y (mgr.26252) 1652 : cluster [DBG] pgmap v2293: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:01.706 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:01 smithi099 bash[84576]: cluster 2024-09-13T15:21:01.292473+0000 mgr.y (mgr.26252) 1652 : cluster [DBG] pgmap v2293: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:01.706 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:01 smithi099 bash[87147]: cluster 2024-09-13T15:21:01.292473+0000 mgr.y (mgr.26252) 1652 : cluster [DBG] pgmap v2293: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:01.706 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:01 smithi099 bash[87147]: cluster 2024-09-13T15:21:01.292473+0000 mgr.y (mgr.26252) 1652 : cluster [DBG] pgmap v2293: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:01.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:01 smithi177 bash[68594]: cluster 2024-09-13T15:21:01.292473+0000 mgr.y (mgr.26252) 1652 : cluster [DBG] pgmap v2293: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:01.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:01 smithi177 bash[68594]: cluster 2024-09-13T15:21:01.292473+0000 mgr.y (mgr.26252) 1652 : cluster [DBG] pgmap v2293: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:03.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:03 smithi099 bash[84576]: cluster 2024-09-13T15:21:03.293325+0000 mgr.y (mgr.26252) 1653 : cluster [DBG] pgmap v2294: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:03.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:03 smithi099 bash[84576]: cluster 2024-09-13T15:21:03.293325+0000 mgr.y (mgr.26252) 1653 : cluster [DBG] pgmap v2294: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:03.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:03 smithi099 bash[84576]: audit 2024-09-13T15:21:03.326286+0000 mon.a (mon.0) 2858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:03.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:03 smithi099 bash[84576]: audit 2024-09-13T15:21:03.326286+0000 mon.a (mon.0) 2858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:03.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:03 smithi099 bash[87147]: cluster 2024-09-13T15:21:03.293325+0000 mgr.y (mgr.26252) 1653 : cluster [DBG] pgmap v2294: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:03.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:03 smithi099 bash[87147]: cluster 2024-09-13T15:21:03.293325+0000 mgr.y (mgr.26252) 1653 : cluster [DBG] pgmap v2294: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:03.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:03 smithi099 bash[87147]: audit 2024-09-13T15:21:03.326286+0000 mon.a (mon.0) 2858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:03.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:03 smithi099 bash[87147]: audit 2024-09-13T15:21:03.326286+0000 mon.a (mon.0) 2858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:03 smithi177 bash[68594]: cluster 2024-09-13T15:21:03.293325+0000 mgr.y (mgr.26252) 1653 : cluster [DBG] pgmap v2294: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:03 smithi177 bash[68594]: cluster 2024-09-13T15:21:03.293325+0000 mgr.y (mgr.26252) 1653 : cluster [DBG] pgmap v2294: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:03 smithi177 bash[68594]: audit 2024-09-13T15:21:03.326286+0000 mon.a (mon.0) 2858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:03.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:03 smithi177 bash[68594]: audit 2024-09-13T15:21:03.326286+0000 mon.a (mon.0) 2858 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:04.749 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:21:04.749 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T15:21:04.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95378 46.10 46:091b65bc:::benchmark_data_smithi099_233884_object95377:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.758540+0000", 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98703655700001, 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.79229719, 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.750 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95378 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.758540+0000", 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.758540+0000", 2024-09-13T15:21:04.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.758542+0000", 2024-09-13T15:21:04.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9319999999999999e-06 2024-09-13T15:21:04.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.758602+0000", 2024-09-13T15:21:04.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0498999999999998e-05 2024-09-13T15:21:04.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.758622+0000", 2024-09-13T15:21:04.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9797e-05 2024-09-13T15:21:04.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.758633+0000", 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0635000000000001e-05 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.759717+0000", 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010847389999999999 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.759729+0000", 2024-09-13T15:21:04.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1253000000000001e-05 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.550733+0000", 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.791004834 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.550837+0000", 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000103501 2024-09-13T15:21:04.755 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95397 46.10 46:0ef09d9b:::benchmark_data_smithi099_233884_object95396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.758655+0000", 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.986921559, 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8184891059999999, 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.756 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95397 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.758655+0000", 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.758655+0000", 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.758655+0000", 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.61e-07 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.759799+0000", 2024-09-13T15:21:04.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011432899999999999 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.759814+0000", 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5115e-05 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.759822+0000", 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3529999999999995e-06 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760597+0000", 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00077450100000000003 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760604+0000", 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1300000000000003e-06 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.551968+0000", 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7913639269999999 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.552280+0000", 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00031272400000000001 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.552520+0000", 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000239619 2024-09-13T15:21:04.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.567361+0000", 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014840766 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.567787+0000", 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00042623599999999998 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.567962+0000", 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017472200000000001 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.575388+0000", 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0074265859999999998 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.575438+0000", 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9869e-05 2024-09-13T15:21:04.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.577144+0000", 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001705607 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95428 46.10 46:0987ec9e:::benchmark_data_smithi099_233884_object95427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760575+0000", 2024-09-13T15:21:04.764 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98500136299998, 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.819070116, 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95428 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760575+0000", 2024-09-13T15:21:04.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760575+0000", 2024-09-13T15:21:04.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760576+0000", 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2099999999999995e-07 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760613+0000", 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6888000000000003e-05 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760627+0000", 2024-09-13T15:21:04.768 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3725e-05 2024-09-13T15:21:04.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.768 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.768 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760633+0000", 2024-09-13T15:21:04.768 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3350000000000003e-06 2024-09-13T15:21:04.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.768 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.768 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760672+0000", 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9125000000000002e-05 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760676+0000", 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.783e-06 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.554719+0000", 2024-09-13T15:21:04.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.794042798 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.554865+0000", 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014629399999999999 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.555036+0000", 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017092999999999999 2024-09-13T15:21:04.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.567383+0000", 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012347197000000001 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.577483+0000", 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010099673 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.577537+0000", 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4447000000000002e-05 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.577597+0000", 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9521000000000002e-05 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.577689+0000", 2024-09-13T15:21:04.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2360000000000003e-05 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.579645+0000", 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019561190000000001 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95430 46.10 46:0d076c6d:::benchmark_data_smithi099_233884_object95429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.773 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760646+0000", 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98493082800002, 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8208594470000001, 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95430 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.774 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760646+0000", 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760646+0000", 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760646+0000", 2024-09-13T15:21:04.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7e-07 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760665+0000", 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9171999999999999e-05 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760670+0000", 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1780000000000002e-06 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760674+0000", 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6600000000000001e-06 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760710+0000", 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5633000000000003e-05 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760714+0000", 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3449999999999999e-06 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.556407+0000", 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.795692686 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.556623+0000", 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021617500000000001 2024-09-13T15:21:04.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.556793+0000", 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016984500000000001 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.567403+0000", 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010610286 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.579719+0000", 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012315688999999999 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.579789+0000", 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0314000000000005e-05 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.579834+0000", 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5268000000000001e-05 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.579879+0000", 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4536000000000002e-05 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.581505+0000", 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00162619 2024-09-13T15:21:04.781 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95439 46.10 46:092e59a6:::benchmark_data_smithi099_233884_object95438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760682+0000", 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98489415900002, 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8222381539999999, 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.782 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95439 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760682+0000", 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760682+0000", 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760683+0000", 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6199999999999999e-07 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760700+0000", 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7672e-05 2024-09-13T15:21:04.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760705+0000", 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6460000000000001e-06 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760708+0000", 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4680000000000001e-06 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760740+0000", 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2150999999999998e-05 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760744+0000", 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6179999999999999e-06 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.558132+0000", 2024-09-13T15:21:04.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.79738796 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.558316+0000", 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018357700000000001 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.558538+0000", 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000222499 2024-09-13T15:21:04.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.567423+0000", 2024-09-13T15:21:04.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.008884978 2024-09-13T15:21:04.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.581564+0000", 2024-09-13T15:21:04.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014140589 2024-09-13T15:21:04.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.581615+0000", 2024-09-13T15:21:04.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1588000000000001e-05 2024-09-13T15:21:04.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.581653+0000", 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7786e-05 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.581696+0000", 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2667000000000003e-05 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.582920+0000", 2024-09-13T15:21:04.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012245929999999999 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95467 46.10 46:094affad:::benchmark_data_smithi099_233884_object95466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760715+0000", 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98486125099998, 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.834202847, 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.791 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95467 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760715+0000", 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760715+0000", 2024-09-13T15:21:04.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760715+0000", 2024-09-13T15:21:04.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5400000000000002e-07 2024-09-13T15:21:04.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760733+0000", 2024-09-13T15:21:04.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7487999999999999e-05 2024-09-13T15:21:04.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760738+0000", 2024-09-13T15:21:04.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8189999999999999e-06 2024-09-13T15:21:04.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760741+0000", 2024-09-13T15:21:04.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2200000000000001e-06 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760768+0000", 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7109000000000001e-05 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760771+0000", 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3699999999999999e-06 2024-09-13T15:21:04.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.559851+0000", 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7990800249999999 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.560066+0000", 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000214873 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.560326+0000", 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000260132 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.567833+0000", 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075064830000000004 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.582981+0000", 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015148339 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.583021+0000", 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.994e-05 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.592850+0000", 2024-09-13T15:21:04.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0098287749999999997 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.592913+0000", 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2661000000000002e-05 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.594918+0000", 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002005359 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.802 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.803 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.803 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95500 46.10 46:0f89ca76:::benchmark_data_smithi099_233884_object95499:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.803 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760747+0000", 2024-09-13T15:21:04.803 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.984829198, 2024-09-13T15:21:04.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8355809939999999, 2024-09-13T15:21:04.803 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.803 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.803 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.804 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.804 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.804 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.804 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95500 2024-09-13T15:21:04.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.804 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760747+0000", 2024-09-13T15:21:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760747+0000", 2024-09-13T15:21:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760747+0000", 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7700000000000001e-07 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760768+0000", 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0429999999999999e-05 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760773+0000", 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2939999999999996e-06 2024-09-13T15:21:04.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760776+0000", 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2890000000000002e-06 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760809+0000", 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2628e-05 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760813+0000", 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5200000000000002e-06 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.561974+0000", 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8011614869999999 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.562165+0000", 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019091 2024-09-13T15:21:04.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.562313+0000", 2024-09-13T15:21:04.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014788600000000001 2024-09-13T15:21:04.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.567851+0000", 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055376690000000003 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.583056+0000", 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015205715999999999 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.589465+0000", 2024-09-13T15:21:04.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064084320000000004 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.595014+0000", 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055495559999999998 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.595080+0000", 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5659000000000003e-05 2024-09-13T15:21:04.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.596328+0000", 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001248241 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95553 46.10 46:0f44a814:::benchmark_data_smithi099_233884_object95552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.812 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760783+0000", 2024-09-13T15:21:04.813 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.984793669, 2024-09-13T15:21:04.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8368758889999999, 2024-09-13T15:21:04.813 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.813 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.813 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.813 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.813 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95553 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760783+0000", 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760783+0000", 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760783+0000", 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9900000000000002e-07 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760802+0000", 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8691000000000001e-05 2024-09-13T15:21:04.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760807+0000", 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2889999999999997e-06 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760810+0000", 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3830000000000001e-06 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760836+0000", 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6146e-05 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760840+0000", 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.349e-06 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.563805+0000", 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8029654989999999 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.564117+0000", 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000311952 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.564500+0000", 2024-09-13T15:21:04.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00038289800000000001 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.568005+0000", 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0035047339999999998 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.583090+0000", 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015085360000000001 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.589574+0000", 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064835589999999999 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.596500+0000", 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006925779 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.596560+0000", 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0822999999999997e-05 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.597659+0000", 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010981280000000001 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.822 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95558 46.10 46:089d8542:::benchmark_data_smithi099_233884_object95557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.822 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760816+0000", 2024-09-13T15:21:04.822 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98476033399999, 2024-09-13T15:21:04.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.838150395, 2024-09-13T15:21:04.822 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.822 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.822 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.822 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.822 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.822 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.823 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95558 2024-09-13T15:21:04.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.823 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.823 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760816+0000", 2024-09-13T15:21:04.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760816+0000", 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760816+0000", 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9400000000000001e-07 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760838+0000", 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1290000000000001e-05 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760842+0000", 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5379999999999996e-06 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760846+0000", 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6169999999999998e-06 2024-09-13T15:21:04.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760881+0000", 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5675e-05 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760885+0000", 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6890000000000002e-06 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.568200+0000", 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8073148699999999 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.568758+0000", 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00055761700000000005 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.569160+0000", 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00040234999999999999 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.577433+0000", 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0082730560000000009 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.583133+0000", 2024-09-13T15:21:04.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0056996950000000003 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.592735+0000", 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096027669999999999 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.597730+0000", 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049942700000000003 2024-09-13T15:21:04.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.597787+0000", 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6987999999999997e-05 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.598966+0000", 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011796790000000001 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.830 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.831 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95622 46.10 46:0bcf0ddd:::benchmark_data_smithi099_233884_object95621:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.831 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760852+0000", 2024-09-13T15:21:04.831 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98472464100001, 2024-09-13T15:21:04.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.853019808, 2024-09-13T15:21:04.831 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.831 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.831 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.831 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95622 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760852+0000", 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760852+0000", 2024-09-13T15:21:04.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760852+0000", 2024-09-13T15:21:04.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8799999999999998e-07 2024-09-13T15:21:04.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760870+0000", 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8091e-05 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760877+0000", 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9009999999999997e-06 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760880+0000", 2024-09-13T15:21:04.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3419999999999999e-06 2024-09-13T15:21:04.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760912+0000", 2024-09-13T15:21:04.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1668000000000002e-05 2024-09-13T15:21:04.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760915+0000", 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3409999999999999e-06 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.572949+0000", 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.81203388 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.573317+0000", 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00036809700000000001 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.573726+0000", 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000409023 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.577439+0000", 2024-09-13T15:21:04.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037131360000000001 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.583169+0000", 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057298799999999997 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.596396+0000", 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013226932 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.612039+0000", 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015642947000000001 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.612079+0000", 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0127999999999999e-05 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.613871+0000", 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001792154 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95664 46.10 46:0b7a22ed:::benchmark_data_smithi099_233884_object95663:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760886+0000", 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98469035400001, 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8553206200000001, 2024-09-13T15:21:04.840 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95664 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760886+0000", 2024-09-13T15:21:04.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760886+0000", 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760886+0000", 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.79e-07 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760905+0000", 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.838e-05 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760911+0000", 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4309999999999999e-06 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760914+0000", 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3179999999999998e-06 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760946+0000", 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1881000000000002e-05 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760950+0000", 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0890000000000002e-06 2024-09-13T15:21:04.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.583229+0000", 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8222789850000001 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.583514+0000", 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000284171 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.583677+0000", 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000163974 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.589372+0000", 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0056946820000000004 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.589427+0000", 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4679999999999998e-05 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.604636+0000", 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015209004999999999 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.614480+0000", 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0098439070000000007 2024-09-13T15:21:04.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.614546+0000", 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6024000000000003e-05 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.616207+0000", 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001660814 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.848 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95691 46.10 46:0d56cac2:::benchmark_data_smithi099_233884_object95690:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760921+0000", 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98465572100002, 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8569985120000001, 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.849 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95691 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760921+0000", 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760921+0000", 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760921+0000", 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0400000000000002e-07 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760938+0000", 2024-09-13T15:21:04.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7397e-05 2024-09-13T15:21:04.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760943+0000", 2024-09-13T15:21:04.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5950000000000001e-06 2024-09-13T15:21:04.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760946+0000", 2024-09-13T15:21:04.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1200000000000002e-06 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760974+0000", 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8407000000000001e-05 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760978+0000", 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3129999999999999e-06 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.585456+0000", 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8244777720000001 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.585642+0000", 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018696399999999999 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.585830+0000", 2024-09-13T15:21:04.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018731999999999999 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.589474+0000", 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0036443159999999999 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.589511+0000", 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6446999999999998e-05 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.604759+0000", 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015248566999999999 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.616277+0000", 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011517799 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.616346+0000", 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9003000000000003e-05 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.617919+0000", 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015731880000000001 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95876 46.10 46:0b21fc25:::benchmark_data_smithi099_233884_object95875:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760952+0000", 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98462476100002, 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.85871516, 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.858 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95876 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760952+0000", 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760952+0000", 2024-09-13T15:21:04.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760952+0000", 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7500000000000001e-07 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760969+0000", 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7272e-05 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760974+0000", 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5569999999999997e-06 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760977+0000", 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0800000000000002e-06 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761012+0000", 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5089000000000002e-05 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761016+0000", 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9530000000000001e-06 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.587545+0000", 2024-09-13T15:21:04.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.826529294 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.587717+0000", 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017238099999999999 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.587866+0000", 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000148675 2024-09-13T15:21:04.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.592635+0000", 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0047692619999999998 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.592702+0000", 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7090999999999999e-05 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.606145+0000", 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013442713 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.618006+0000", 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011860714 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.618085+0000", 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8949000000000006e-05 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.619667+0000", 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015818550000000001 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.866 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95891 46.10 46:0b06afec:::benchmark_data_smithi099_233884_object95890:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.760984+0000", 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98459252700002, 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8603818649999999, 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95891 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760984+0000", 2024-09-13T15:21:04.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760984+0000", 2024-09-13T15:21:04.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.760984+0000", 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8099999999999999e-07 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761002+0000", 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7479999999999999e-05 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761013+0000", 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1404e-05 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761016+0000", 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4680000000000001e-06 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761051+0000", 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4178999999999999e-05 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761054+0000", 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.427e-06 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.589613+0000", 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.828558699 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.589732+0000", 2024-09-13T15:21:04.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119008 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.589866+0000", 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000134678 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.592793+0000", 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029268689999999999 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.592812+0000", 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8597000000000002e-05 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.609324+0000", 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016511888999999998 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.619734+0000", 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010410617 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.619795+0000", 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0983999999999998e-05 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.621366+0000", 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001570285 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.874 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95920 46.10 46:0c706858:::benchmark_data_smithi099_233884_object95919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.761023+0000", 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98455323299999, 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.861784586, 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95920 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761023+0000", 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761023+0000", 2024-09-13T15:21:04.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761023+0000", 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9499999999999998e-07 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761042+0000", 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8391999999999999e-05 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761047+0000", 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8930000000000004e-06 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761050+0000", 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2490000000000001e-06 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761085+0000", 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4609e-05 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761089+0000", 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6260000000000003e-06 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.591153+0000", 2024-09-13T15:21:04.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8300637339999999 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.591302+0000", 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014861400000000001 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.591432+0000", 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013056400000000001 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.594940+0000", 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0035075570000000001 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.596448+0000", 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015080130000000001 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.609396+0000", 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012948339999999999 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.621521+0000", 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012125515 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.621594+0000", 2024-09-13T15:21:04.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2038000000000004e-05 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.622808+0000", 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012141470000000001 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:95921 46.10 46:0a181dfc:::benchmark_data_smithi099_233884_object95920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-13T15:21:04.883 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:38.761060+0000", 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.98451647799999, 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.86693205, 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95921 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761060+0000", 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761060+0000", 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761060+0000", 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8000000000000002e-07 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761083+0000", 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2861999999999999e-05 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761088+0000", 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4099999999999999e-06 2024-09-13T15:21:04.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761093+0000", 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0860000000000001e-06 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761126+0000", 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2855000000000003e-05 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:38.761130+0000", 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.41e-06 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.599029+0000", 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.837898998 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.599209+0000", 2024-09-13T15:21:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018001900000000001 2024-09-13T15:21:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,6", 2024-09-13T15:21:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.599400+0000", 2024-09-13T15:21:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019136899999999999 2024-09-13T15:21:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.604650+0000", 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052503070000000001 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.604695+0000", 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4122999999999998e-05 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.609553+0000", 2024-09-13T15:21:04.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048580990000000003 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.623019+0000", 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013466044999999999 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.625927+0000", 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029087169999999999 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:40.627992+0000", 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020644700000000001 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:107309 46.14 46:29a3c022:::benchmark_data_smithi099_233884_object4896:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1121)", 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:58.674336+0000", 2024-09-13T15:21:04.892 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 246.071240012, 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1853814620000001, 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 107309 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674336+0000", 2024-09-13T15:21:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674336+0000", 2024-09-13T15:21:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674337+0000", 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2599999999999998e-07 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674339+0000", 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3010000000000001e-06 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674345+0000", 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6239999999999999e-06 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674349+0000", 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3540000000000002e-06 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674935+0000", 2024-09-13T15:21:04.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00058623099999999999 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674942+0000", 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0149999999999998e-06 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.682381+0000", 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0074388869999999999 2024-09-13T15:21:04.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.682388+0000", 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6749999999999996e-06 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.840643+0000", 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.158254976 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.841332+0000", 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00068902999999999998 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.850502+0000", 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0091699139999999995 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.850580+0000", 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7860000000000003e-05 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.850677+0000", 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7386000000000006e-05 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.856911+0000", 2024-09-13T15:21:04.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062343620000000002 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.858396+0000", 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001485087 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.858464+0000", 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7630000000000001e-05 2024-09-13T15:21:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.858522+0000", 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7748999999999999e-05 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.859718+0000", 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011959550000000001 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:107375 46.14 46:286352fa:::benchmark_data_smithi099_233884_object4962:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1121)", 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:58.674354+0000", 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 246.071221968, 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.187553839, 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.903 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 107375 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674354+0000", 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674354+0000", 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674355+0000", 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1e-07 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674357+0000", 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.911e-06 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674362+0000", 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9629999999999997e-06 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674366+0000", 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1119999999999998e-06 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674975+0000", 2024-09-13T15:21:04.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00060965499999999996 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674982+0000", 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.268e-06 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.682411+0000", 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007429204 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.682417+0000", 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4810000000000003e-06 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.841386+0000", 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.158968486 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2024-09-13T15:21:04.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.842146+0000", 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00076043699999999998 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.854935+0000", 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012789277999999999 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.855049+0000", 2024-09-13T15:21:04.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0001131 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.855208+0000", 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0001596 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.858406+0000", 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031982680000000002 2024-09-13T15:21:04.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.860577+0000", 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021703260000000002 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.860643+0000", 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6421000000000007e-05 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.860698+0000", 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4384e-05 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.861908+0000", 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012106350000000001 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:107382 46.14 46:2af2724e:::benchmark_data_smithi099_233884_object4969:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1121)", 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:58.674370+0000", 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 246.07120586100001, 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2353294779999999, 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.913 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 107382 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674370+0000", 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674370+0000", 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674371+0000", 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2000000000000001e-07 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674373+0000", 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7880000000000001e-06 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674378+0000", 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.147e-06 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674382+0000", 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2350000000000001e-06 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.675016+0000", 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063379699999999996 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.675022+0000", 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3269999999999997e-06 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.682440+0000", 2024-09-13T15:21:04.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0074181890000000004 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.682446+0000", 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5450000000000003e-06 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.859773+0000", 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1773273740000001 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.860523+0000", 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00074930899999999998 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.878178+0000", 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017655219999999999 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.878336+0000", 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015793799999999999 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.878587+0000", 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000251432 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.886364+0000", 2024-09-13T15:21:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0077770740000000001 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.908034+0000", 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021670317000000001 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.908097+0000", 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2036e-05 2024-09-13T15:21:04.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.908163+0000", 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6392999999999999e-05 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.909700+0000", 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001537037 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.922 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:107406 46.14 46:29c03f82:::benchmark_data_smithi099_233884_object4993:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1121)", 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:58.674387+0000", 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 246.07118976500001, 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2363055000000001, 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:04.923 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 107406 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674387+0000", 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:04.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674387+0000", 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674387+0000", 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8099999999999999e-07 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674388+0000", 2024-09-13T15:21:04.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5659999999999999e-06 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674393+0000", 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7790000000000003e-06 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.674397+0000", 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0400000000000003e-06 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.675056+0000", 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00065840099999999997 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.675062+0000", 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2859999999999996e-06 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.682562+0000", 2024-09-13T15:21:04.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075002530000000001 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.682572+0000", 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0125999999999999e-05 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.861956+0000", 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.179384151 2024-09-13T15:21:04.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.862662+0000", 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00070519599999999995 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.882695+0000", 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020033604 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.882772+0000", 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6359000000000005e-05 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.882859+0000", 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7332999999999999e-05 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:04.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.889640+0000", 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0067813379999999996 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.909738+0000", 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020097568999999999 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.909769+0000", 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1143999999999999e-05 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:04.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.909804+0000", 2024-09-13T15:21:04.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5213000000000003e-05 2024-09-13T15:21:04.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:04.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:04.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:04.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:59.910692+0000", 2024-09-13T15:21:04.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00088786099999999999 2024-09-13T15:21:04.932 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.932 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.933 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.933 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:04.933 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:04.933 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:21:05.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:05 smithi099 bash[84576]: cluster 2024-09-13T15:21:05.294237+0000 mgr.y (mgr.26252) 1654 : cluster [DBG] pgmap v2295: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:05.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:05 smithi099 bash[84576]: cluster 2024-09-13T15:21:05.294237+0000 mgr.y (mgr.26252) 1654 : cluster [DBG] pgmap v2295: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:05.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:05 smithi099 bash[87147]: cluster 2024-09-13T15:21:05.294237+0000 mgr.y (mgr.26252) 1654 : cluster [DBG] pgmap v2295: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:05.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:05 smithi099 bash[87147]: cluster 2024-09-13T15:21:05.294237+0000 mgr.y (mgr.26252) 1654 : cluster [DBG] pgmap v2295: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:05.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:05 smithi177 bash[68594]: cluster 2024-09-13T15:21:05.294237+0000 mgr.y (mgr.26252) 1654 : cluster [DBG] pgmap v2295: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:05.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:05 smithi177 bash[68594]: cluster 2024-09-13T15:21:05.294237+0000 mgr.y (mgr.26252) 1654 : cluster [DBG] pgmap v2295: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:06.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T15:21:06.385 INFO:teuthology.orchestra.run.smithi099.stderr:nodeep-scrub is unset 2024-09-13T15:21:06.426 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:06 smithi099 bash[84576]: audit 2024-09-13T15:21:05.692514+0000 mon.c (mon.2) 127 : audit [INF] from='client.? 172.21.15.99:0/3146227083' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.426 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:06 smithi099 bash[84576]: audit 2024-09-13T15:21:05.692514+0000 mon.c (mon.2) 127 : audit [INF] from='client.? 172.21.15.99:0/3146227083' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.426 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:06 smithi099 bash[84576]: audit 2024-09-13T15:21:05.693052+0000 mon.a (mon.0) 2859 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.426 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:06 smithi099 bash[84576]: audit 2024-09-13T15:21:05.693052+0000 mon.a (mon.0) 2859 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.427 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:06 smithi099 bash[87147]: audit 2024-09-13T15:21:05.692514+0000 mon.c (mon.2) 127 : audit [INF] from='client.? 172.21.15.99:0/3146227083' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.427 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:06 smithi099 bash[87147]: audit 2024-09-13T15:21:05.692514+0000 mon.c (mon.2) 127 : audit [INF] from='client.? 172.21.15.99:0/3146227083' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.427 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:06 smithi099 bash[87147]: audit 2024-09-13T15:21:05.693052+0000 mon.a (mon.0) 2859 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.427 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:06 smithi099 bash[87147]: audit 2024-09-13T15:21:05.693052+0000 mon.a (mon.0) 2859 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:06 smithi177 bash[68594]: audit 2024-09-13T15:21:05.692514+0000 mon.c (mon.2) 127 : audit [INF] from='client.? 172.21.15.99:0/3146227083' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:06 smithi177 bash[68594]: audit 2024-09-13T15:21:05.692514+0000 mon.c (mon.2) 127 : audit [INF] from='client.? 172.21.15.99:0/3146227083' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:06 smithi177 bash[68594]: audit 2024-09-13T15:21:05.693052+0000 mon.a (mon.0) 2859 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:06.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:06 smithi177 bash[68594]: audit 2024-09-13T15:21:05.693052+0000 mon.a (mon.0) 2859 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:21:07.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:21:07] "GET /metrics HTTP/1.1" 200 34349 "" "Prometheus/2.51.0" 2024-09-13T15:21:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[84576]: audit 2024-09-13T15:21:06.377208+0000 mon.a (mon.0) 2860 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:21:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[84576]: audit 2024-09-13T15:21:06.377208+0000 mon.a (mon.0) 2860 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:21:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[84576]: cluster 2024-09-13T15:21:06.383931+0000 mon.a (mon.0) 2861 : cluster [DBG] osdmap e1159: 8 total, 8 up, 7 in 2024-09-13T15:21:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[84576]: cluster 2024-09-13T15:21:06.383931+0000 mon.a (mon.0) 2861 : cluster [DBG] osdmap e1159: 8 total, 8 up, 7 in 2024-09-13T15:21:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[84576]: cluster 2024-09-13T15:21:07.295146+0000 mgr.y (mgr.26252) 1655 : cluster [DBG] pgmap v2297: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:07.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[84576]: cluster 2024-09-13T15:21:07.295146+0000 mgr.y (mgr.26252) 1655 : cluster [DBG] pgmap v2297: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:07.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[87147]: audit 2024-09-13T15:21:06.377208+0000 mon.a (mon.0) 2860 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:21:07.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[87147]: audit 2024-09-13T15:21:06.377208+0000 mon.a (mon.0) 2860 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:21:07.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[87147]: cluster 2024-09-13T15:21:06.383931+0000 mon.a (mon.0) 2861 : cluster [DBG] osdmap e1159: 8 total, 8 up, 7 in 2024-09-13T15:21:07.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[87147]: cluster 2024-09-13T15:21:06.383931+0000 mon.a (mon.0) 2861 : cluster [DBG] osdmap e1159: 8 total, 8 up, 7 in 2024-09-13T15:21:07.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[87147]: cluster 2024-09-13T15:21:07.295146+0000 mgr.y (mgr.26252) 1655 : cluster [DBG] pgmap v2297: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:07.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:07 smithi099 bash[87147]: cluster 2024-09-13T15:21:07.295146+0000 mgr.y (mgr.26252) 1655 : cluster [DBG] pgmap v2297: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:07 smithi177 bash[68594]: audit 2024-09-13T15:21:06.377208+0000 mon.a (mon.0) 2860 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:21:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:07 smithi177 bash[68594]: audit 2024-09-13T15:21:06.377208+0000 mon.a (mon.0) 2860 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:21:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:07 smithi177 bash[68594]: cluster 2024-09-13T15:21:06.383931+0000 mon.a (mon.0) 2861 : cluster [DBG] osdmap e1159: 8 total, 8 up, 7 in 2024-09-13T15:21:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:07 smithi177 bash[68594]: cluster 2024-09-13T15:21:06.383931+0000 mon.a (mon.0) 2861 : cluster [DBG] osdmap e1159: 8 total, 8 up, 7 in 2024-09-13T15:21:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:07 smithi177 bash[68594]: cluster 2024-09-13T15:21:07.295146+0000 mgr.y (mgr.26252) 1655 : cluster [DBG] pgmap v2297: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:07 smithi177 bash[68594]: cluster 2024-09-13T15:21:07.295146+0000 mgr.y (mgr.26252) 1655 : cluster [DBG] pgmap v2297: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:07.878 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:21:09.699 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:09 smithi099 bash[84576]: cluster 2024-09-13T15:21:09.296023+0000 mgr.y (mgr.26252) 1656 : cluster [DBG] pgmap v2298: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:09.699 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:09 smithi099 bash[84576]: cluster 2024-09-13T15:21:09.296023+0000 mgr.y (mgr.26252) 1656 : cluster [DBG] pgmap v2298: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:09.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:09 smithi099 bash[87147]: cluster 2024-09-13T15:21:09.296023+0000 mgr.y (mgr.26252) 1656 : cluster [DBG] pgmap v2298: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:09.699 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:09 smithi099 bash[87147]: cluster 2024-09-13T15:21:09.296023+0000 mgr.y (mgr.26252) 1656 : cluster [DBG] pgmap v2298: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:09.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:09 smithi177 bash[68594]: cluster 2024-09-13T15:21:09.296023+0000 mgr.y (mgr.26252) 1656 : cluster [DBG] pgmap v2298: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:09.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:09 smithi177 bash[68594]: cluster 2024-09-13T15:21:09.296023+0000 mgr.y (mgr.26252) 1656 : cluster [DBG] pgmap v2298: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:10.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:21:10 smithi177 bash[61439]: ts=2024-09-13T15:21:10.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:21:11.104 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:21:11.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:11 smithi099 bash[84576]: cluster 2024-09-13T15:21:11.296889+0000 mgr.y (mgr.26252) 1657 : cluster [DBG] pgmap v2299: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:11.762 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:11 smithi099 bash[84576]: cluster 2024-09-13T15:21:11.296889+0000 mgr.y (mgr.26252) 1657 : cluster [DBG] pgmap v2299: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:11.762 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:11 smithi099 bash[87147]: cluster 2024-09-13T15:21:11.296889+0000 mgr.y (mgr.26252) 1657 : cluster [DBG] pgmap v2299: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:11.763 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:11 smithi099 bash[87147]: cluster 2024-09-13T15:21:11.296889+0000 mgr.y (mgr.26252) 1657 : cluster [DBG] pgmap v2299: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:11.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:11 smithi177 bash[68594]: cluster 2024-09-13T15:21:11.296889+0000 mgr.y (mgr.26252) 1657 : cluster [DBG] pgmap v2299: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:11.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:11 smithi177 bash[68594]: cluster 2024-09-13T15:21:11.296889+0000 mgr.y (mgr.26252) 1657 : cluster [DBG] pgmap v2299: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:13.389 INFO:teuthology.orchestra.run.smithi099.stderr:marked in osd.2. 2024-09-13T15:21:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[84576]: debug 2024-09-13T15:21:13.318+0000 7f0688bd2640 -1 mon.a@0(leader).osd e1159 definitely_dead 0 2024-09-13T15:21:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[84576]: cluster 2024-09-13T15:21:13.297711+0000 mgr.y (mgr.26252) 1658 : cluster [DBG] pgmap v2300: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[84576]: cluster 2024-09-13T15:21:13.297711+0000 mgr.y (mgr.26252) 1658 : cluster [DBG] pgmap v2300: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[84576]: audit 2024-09-13T15:21:13.323636+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.99:0/3971126154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[84576]: audit 2024-09-13T15:21:13.323636+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.99:0/3971126154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[84576]: audit 2024-09-13T15:21:13.324310+0000 mon.a (mon.0) 2862 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[84576]: audit 2024-09-13T15:21:13.324310+0000 mon.a (mon.0) 2862 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[87147]: cluster 2024-09-13T15:21:13.297711+0000 mgr.y (mgr.26252) 1658 : cluster [DBG] pgmap v2300: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:13.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[87147]: cluster 2024-09-13T15:21:13.297711+0000 mgr.y (mgr.26252) 1658 : cluster [DBG] pgmap v2300: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:13.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[87147]: audit 2024-09-13T15:21:13.323636+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.99:0/3971126154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[87147]: audit 2024-09-13T15:21:13.323636+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.99:0/3971126154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[87147]: audit 2024-09-13T15:21:13.324310+0000 mon.a (mon.0) 2862 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:13 smithi099 bash[87147]: audit 2024-09-13T15:21:13.324310+0000 mon.a (mon.0) 2862 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:13 smithi177 bash[68594]: cluster 2024-09-13T15:21:13.297711+0000 mgr.y (mgr.26252) 1658 : cluster [DBG] pgmap v2300: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:13.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:13 smithi177 bash[68594]: cluster 2024-09-13T15:21:13.297711+0000 mgr.y (mgr.26252) 1658 : cluster [DBG] pgmap v2300: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:13.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:13 smithi177 bash[68594]: audit 2024-09-13T15:21:13.323636+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.99:0/3971126154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:13 smithi177 bash[68594]: audit 2024-09-13T15:21:13.323636+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.99:0/3971126154' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:13 smithi177 bash[68594]: audit 2024-09-13T15:21:13.324310+0000 mon.a (mon.0) 2862 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:13.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:13 smithi177 bash[68594]: audit 2024-09-13T15:21:13.324310+0000 mon.a (mon.0) 2862 : audit [DBG] from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:21:14.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:14 smithi099 bash[84576]: audit 2024-09-13T15:21:13.378822+0000 mon.a (mon.0) 2863 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-13T15:21:14.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:14 smithi099 bash[84576]: audit 2024-09-13T15:21:13.378822+0000 mon.a (mon.0) 2863 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-13T15:21:14.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:14 smithi099 bash[84576]: cluster 2024-09-13T15:21:13.382209+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2024-09-13T15:21:14.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:14 smithi099 bash[84576]: cluster 2024-09-13T15:21:13.382209+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2024-09-13T15:21:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:14 smithi099 bash[87147]: audit 2024-09-13T15:21:13.378822+0000 mon.a (mon.0) 2863 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-13T15:21:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:14 smithi099 bash[87147]: audit 2024-09-13T15:21:13.378822+0000 mon.a (mon.0) 2863 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-13T15:21:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:14 smithi099 bash[87147]: cluster 2024-09-13T15:21:13.382209+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2024-09-13T15:21:14.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:14 smithi099 bash[87147]: cluster 2024-09-13T15:21:13.382209+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2024-09-13T15:21:14.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:14 smithi177 bash[68594]: audit 2024-09-13T15:21:13.378822+0000 mon.a (mon.0) 2863 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-13T15:21:14.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:14 smithi177 bash[68594]: audit 2024-09-13T15:21:13.378822+0000 mon.a (mon.0) 2863 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-13T15:21:14.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:14 smithi177 bash[68594]: cluster 2024-09-13T15:21:13.382209+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2024-09-13T15:21:14.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:14 smithi177 bash[68594]: cluster 2024-09-13T15:21:13.382209+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2024-09-13T15:21:15.259 INFO:tasks.thrashosds.thrasher:Added osd 2 2024-09-13T15:21:15.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-09-13T15:21:15.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:15 smithi099 bash[84576]: cluster 2024-09-13T15:21:14.398220+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2024-09-13T15:21:15.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:15 smithi099 bash[84576]: cluster 2024-09-13T15:21:14.398220+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2024-09-13T15:21:15.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:15 smithi099 bash[84576]: cluster 2024-09-13T15:21:15.298246+0000 mgr.y (mgr.26252) 1659 : cluster [DBG] pgmap v2303: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:15.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:15 smithi099 bash[84576]: cluster 2024-09-13T15:21:15.298246+0000 mgr.y (mgr.26252) 1659 : cluster [DBG] pgmap v2303: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:15.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:15 smithi099 bash[87147]: cluster 2024-09-13T15:21:14.398220+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2024-09-13T15:21:15.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:15 smithi099 bash[87147]: cluster 2024-09-13T15:21:14.398220+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2024-09-13T15:21:15.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:15 smithi099 bash[87147]: cluster 2024-09-13T15:21:15.298246+0000 mgr.y (mgr.26252) 1659 : cluster [DBG] pgmap v2303: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:15.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:15 smithi099 bash[87147]: cluster 2024-09-13T15:21:15.298246+0000 mgr.y (mgr.26252) 1659 : cluster [DBG] pgmap v2303: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:15.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:15 smithi177 bash[68594]: cluster 2024-09-13T15:21:14.398220+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2024-09-13T15:21:15.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:15 smithi177 bash[68594]: cluster 2024-09-13T15:21:14.398220+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2024-09-13T15:21:15.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:15 smithi177 bash[68594]: cluster 2024-09-13T15:21:15.298246+0000 mgr.y (mgr.26252) 1659 : cluster [DBG] pgmap v2303: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:15.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:15 smithi177 bash[68594]: cluster 2024-09-13T15:21:15.298246+0000 mgr.y (mgr.26252) 1659 : cluster [DBG] pgmap v2303: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2024-09-13T15:21:17.046 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:21:17.046 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:21:17.046 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2024-09-13T15:21:17.046 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:21:17.292 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:21:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:21:17] "GET /metrics HTTP/1.1" 200 34349 "" "Prometheus/2.51.0" 2024-09-13T15:21:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:17 smithi099 bash[84576]: cluster 2024-09-13T15:21:17.299242+0000 mgr.y (mgr.26252) 1660 : cluster [DBG] pgmap v2304: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-09-13T15:21:17.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:17 smithi099 bash[84576]: cluster 2024-09-13T15:21:17.299242+0000 mgr.y (mgr.26252) 1660 : cluster [DBG] pgmap v2304: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-09-13T15:21:17.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:17 smithi099 bash[87147]: cluster 2024-09-13T15:21:17.299242+0000 mgr.y (mgr.26252) 1660 : cluster [DBG] pgmap v2304: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-09-13T15:21:17.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:17 smithi099 bash[87147]: cluster 2024-09-13T15:21:17.299242+0000 mgr.y (mgr.26252) 1660 : cluster [DBG] pgmap v2304: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-09-13T15:21:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:17 smithi177 bash[68594]: cluster 2024-09-13T15:21:17.299242+0000 mgr.y (mgr.26252) 1660 : cluster [DBG] pgmap v2304: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-09-13T15:21:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:17 smithi177 bash[68594]: cluster 2024-09-13T15:21:17.299242+0000 mgr.y (mgr.26252) 1660 : cluster [DBG] pgmap v2304: 83 pgs: 3 peering, 80 active+clean; 582 KiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-09-13T15:21:17.981 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T15:21:18.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:18 smithi099 bash[84576]: audit 2024-09-13T15:21:18.326989+0000 mon.a (mon.0) 2866 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:18.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:18 smithi099 bash[84576]: audit 2024-09-13T15:21:18.326989+0000 mon.a (mon.0) 2866 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:18.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:18 smithi099 bash[87147]: audit 2024-09-13T15:21:18.326989+0000 mon.a (mon.0) 2866 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:18.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:18 smithi099 bash[87147]: audit 2024-09-13T15:21:18.326989+0000 mon.a (mon.0) 2866 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:18 smithi177 bash[68594]: audit 2024-09-13T15:21:18.326989+0000 mon.a (mon.0) 2866 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:18 smithi177 bash[68594]: audit 2024-09-13T15:21:18.326989+0000 mon.a (mon.0) 2866 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:19.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:19 smithi099 bash[84576]: cluster 2024-09-13T15:21:19.300125+0000 mgr.y (mgr.26252) 1661 : cluster [DBG] pgmap v2305: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:19.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:19 smithi099 bash[84576]: cluster 2024-09-13T15:21:19.300125+0000 mgr.y (mgr.26252) 1661 : cluster [DBG] pgmap v2305: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:19.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:19 smithi099 bash[87147]: cluster 2024-09-13T15:21:19.300125+0000 mgr.y (mgr.26252) 1661 : cluster [DBG] pgmap v2305: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:19.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:19 smithi099 bash[87147]: cluster 2024-09-13T15:21:19.300125+0000 mgr.y (mgr.26252) 1661 : cluster [DBG] pgmap v2305: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:19.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:19 smithi177 bash[68594]: cluster 2024-09-13T15:21:19.300125+0000 mgr.y (mgr.26252) 1661 : cluster [DBG] pgmap v2305: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:19.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:19 smithi177 bash[68594]: cluster 2024-09-13T15:21:19.300125+0000 mgr.y (mgr.26252) 1661 : cluster [DBG] pgmap v2305: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:20.510 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:21:20.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:21:20 smithi177 bash[61439]: ts=2024-09-13T15:21:20.216Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:21:21.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:21 smithi099 bash[84576]: cluster 2024-09-13T15:21:21.301024+0000 mgr.y (mgr.26252) 1662 : cluster [DBG] pgmap v2306: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:21.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:21 smithi099 bash[84576]: cluster 2024-09-13T15:21:21.301024+0000 mgr.y (mgr.26252) 1662 : cluster [DBG] pgmap v2306: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:21.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:21 smithi099 bash[87147]: cluster 2024-09-13T15:21:21.301024+0000 mgr.y (mgr.26252) 1662 : cluster [DBG] pgmap v2306: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:21.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:21 smithi099 bash[87147]: cluster 2024-09-13T15:21:21.301024+0000 mgr.y (mgr.26252) 1662 : cluster [DBG] pgmap v2306: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:21.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:21 smithi177 bash[68594]: cluster 2024-09-13T15:21:21.301024+0000 mgr.y (mgr.26252) 1662 : cluster [DBG] pgmap v2306: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:21.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:21 smithi177 bash[68594]: cluster 2024-09-13T15:21:21.301024+0000 mgr.y (mgr.26252) 1662 : cluster [DBG] pgmap v2306: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:22.557 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:21:23.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:23 smithi099 bash[84576]: cluster 2024-09-13T15:21:23.302019+0000 mgr.y (mgr.26252) 1663 : cluster [DBG] pgmap v2307: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:23.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:23 smithi099 bash[84576]: cluster 2024-09-13T15:21:23.302019+0000 mgr.y (mgr.26252) 1663 : cluster [DBG] pgmap v2307: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:23.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:23 smithi099 bash[87147]: cluster 2024-09-13T15:21:23.302019+0000 mgr.y (mgr.26252) 1663 : cluster [DBG] pgmap v2307: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:23.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:23 smithi099 bash[87147]: cluster 2024-09-13T15:21:23.302019+0000 mgr.y (mgr.26252) 1663 : cluster [DBG] pgmap v2307: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:23.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:23 smithi177 bash[68594]: cluster 2024-09-13T15:21:23.302019+0000 mgr.y (mgr.26252) 1663 : cluster [DBG] pgmap v2307: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:23.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:23 smithi177 bash[68594]: cluster 2024-09-13T15:21:23.302019+0000 mgr.y (mgr.26252) 1663 : cluster [DBG] pgmap v2307: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 26 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:21:25.818 INFO:teuthology.orchestra.run.smithi099.stdout:mon.a: {} 2024-09-13T15:21:25.819 INFO:teuthology.orchestra.run.smithi099.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T15:21:25.831 INFO:teuthology.orchestra.run.smithi099.stdout:mon.b: {} 2024-09-13T15:21:25.831 INFO:teuthology.orchestra.run.smithi099.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T15:21:25.841 INFO:teuthology.orchestra.run.smithi099.stdout:mon.c: {} 2024-09-13T15:21:25.841 INFO:teuthology.orchestra.run.smithi099.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T15:21:25.997 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:25 smithi099 bash[87147]: audit 2024-09-13T15:21:24.771670+0000 mon.a (mon.0) 2867 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:21:25.997 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:25 smithi099 bash[87147]: audit 2024-09-13T15:21:24.771670+0000 mon.a (mon.0) 2867 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:21:25.997 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:25 smithi099 bash[87147]: cluster 2024-09-13T15:21:25.302827+0000 mgr.y (mgr.26252) 1664 : cluster [DBG] pgmap v2308: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 24 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:25.997 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:25 smithi099 bash[87147]: cluster 2024-09-13T15:21:25.302827+0000 mgr.y (mgr.26252) 1664 : cluster [DBG] pgmap v2308: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 24 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:25.997 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:25 smithi099 bash[84576]: audit 2024-09-13T15:21:24.771670+0000 mon.a (mon.0) 2867 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:21:25.997 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:25 smithi099 bash[84576]: audit 2024-09-13T15:21:24.771670+0000 mon.a (mon.0) 2867 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:21:25.997 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:25 smithi099 bash[84576]: cluster 2024-09-13T15:21:25.302827+0000 mgr.y (mgr.26252) 1664 : cluster [DBG] pgmap v2308: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 24 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:25.998 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:25 smithi099 bash[84576]: cluster 2024-09-13T15:21:25.302827+0000 mgr.y (mgr.26252) 1664 : cluster [DBG] pgmap v2308: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 24 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:26.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:25 smithi177 bash[68594]: audit 2024-09-13T15:21:24.771670+0000 mon.a (mon.0) 2867 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:21:26.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:25 smithi177 bash[68594]: audit 2024-09-13T15:21:24.771670+0000 mon.a (mon.0) 2867 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:21:26.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:25 smithi177 bash[68594]: cluster 2024-09-13T15:21:25.302827+0000 mgr.y (mgr.26252) 1664 : cluster [DBG] pgmap v2308: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 24 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:26.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:25 smithi177 bash[68594]: cluster 2024-09-13T15:21:25.302827+0000 mgr.y (mgr.26252) 1664 : cluster [DBG] pgmap v2308: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 24 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:27.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:26 smithi099 bash[87147]: audit 2024-09-13T15:21:25.808265+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.99:0/585033455' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:21:27.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:26 smithi099 bash[87147]: audit 2024-09-13T15:21:25.808265+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.99:0/585033455' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:21:27.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:26 smithi099 bash[84576]: audit 2024-09-13T15:21:25.808265+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.99:0/585033455' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:21:27.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:26 smithi099 bash[84576]: audit 2024-09-13T15:21:25.808265+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.99:0/585033455' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:21:27.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:26 smithi177 bash[68594]: audit 2024-09-13T15:21:25.808265+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.99:0/585033455' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:21:27.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:26 smithi177 bash[68594]: audit 2024-09-13T15:21:25.808265+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.99:0/585033455' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:21:27.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:21:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:21:27] "GET /metrics HTTP/1.1" 200 34363 "" "Prometheus/2.51.0" 2024-09-13T15:21:28.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:27 smithi099 bash[84576]: cluster 2024-09-13T15:21:27.303985+0000 mgr.y (mgr.26252) 1665 : cluster [DBG] pgmap v2309: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 21 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:28.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:27 smithi099 bash[84576]: cluster 2024-09-13T15:21:27.303985+0000 mgr.y (mgr.26252) 1665 : cluster [DBG] pgmap v2309: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 21 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:28.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:27 smithi099 bash[87147]: cluster 2024-09-13T15:21:27.303985+0000 mgr.y (mgr.26252) 1665 : cluster [DBG] pgmap v2309: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 21 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:28.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:27 smithi099 bash[87147]: cluster 2024-09-13T15:21:27.303985+0000 mgr.y (mgr.26252) 1665 : cluster [DBG] pgmap v2309: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 21 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:28.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:27 smithi177 bash[68594]: cluster 2024-09-13T15:21:27.303985+0000 mgr.y (mgr.26252) 1665 : cluster [DBG] pgmap v2309: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 21 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:28.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:27 smithi177 bash[68594]: cluster 2024-09-13T15:21:27.303985+0000 mgr.y (mgr.26252) 1665 : cluster [DBG] pgmap v2309: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 21 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:28.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T15:21:29.414 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:21:29.414 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:21:29.414 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:21:29.414 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:21:29.414 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:21:29.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:29 smithi099 bash[84576]: cluster 2024-09-13T15:21:29.304754+0000 mgr.y (mgr.26252) 1666 : cluster [DBG] pgmap v2310: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:29.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:29 smithi099 bash[84576]: cluster 2024-09-13T15:21:29.304754+0000 mgr.y (mgr.26252) 1666 : cluster [DBG] pgmap v2310: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:29.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:29 smithi099 bash[87147]: cluster 2024-09-13T15:21:29.304754+0000 mgr.y (mgr.26252) 1666 : cluster [DBG] pgmap v2310: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:29.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:29 smithi099 bash[87147]: cluster 2024-09-13T15:21:29.304754+0000 mgr.y (mgr.26252) 1666 : cluster [DBG] pgmap v2310: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:29.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:29 smithi177 bash[68594]: cluster 2024-09-13T15:21:29.304754+0000 mgr.y (mgr.26252) 1666 : cluster [DBG] pgmap v2310: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:29.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:29 smithi177 bash[68594]: cluster 2024-09-13T15:21:29.304754+0000 mgr.y (mgr.26252) 1666 : cluster [DBG] pgmap v2310: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T15:21:30.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T15:21:30.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:21:30 smithi177 bash[61439]: ts=2024-09-13T15:21:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:21:31.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:31 smithi099 bash[84576]: cluster 2024-09-13T15:21:31.305570+0000 mgr.y (mgr.26252) 1667 : cluster [DBG] pgmap v2311: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:31.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:31 smithi099 bash[84576]: cluster 2024-09-13T15:21:31.305570+0000 mgr.y (mgr.26252) 1667 : cluster [DBG] pgmap v2311: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:31.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:31 smithi099 bash[87147]: cluster 2024-09-13T15:21:31.305570+0000 mgr.y (mgr.26252) 1667 : cluster [DBG] pgmap v2311: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:31.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:31 smithi099 bash[87147]: cluster 2024-09-13T15:21:31.305570+0000 mgr.y (mgr.26252) 1667 : cluster [DBG] pgmap v2311: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:31.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:31 smithi177 bash[68594]: cluster 2024-09-13T15:21:31.305570+0000 mgr.y (mgr.26252) 1667 : cluster [DBG] pgmap v2311: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:31.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:31 smithi177 bash[68594]: cluster 2024-09-13T15:21:31.305570+0000 mgr.y (mgr.26252) 1667 : cluster [DBG] pgmap v2311: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:32.958 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:21:33.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:33 smithi099 bash[84576]: cluster 2024-09-13T15:21:33.306542+0000 mgr.y (mgr.26252) 1668 : cluster [DBG] pgmap v2312: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:33.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:33 smithi099 bash[84576]: cluster 2024-09-13T15:21:33.306542+0000 mgr.y (mgr.26252) 1668 : cluster [DBG] pgmap v2312: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:33.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:33 smithi099 bash[84576]: audit 2024-09-13T15:21:33.327072+0000 mon.a (mon.0) 2868 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:33.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:33 smithi099 bash[84576]: audit 2024-09-13T15:21:33.327072+0000 mon.a (mon.0) 2868 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:33.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:33 smithi099 bash[87147]: cluster 2024-09-13T15:21:33.306542+0000 mgr.y (mgr.26252) 1668 : cluster [DBG] pgmap v2312: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:33.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:33 smithi099 bash[87147]: cluster 2024-09-13T15:21:33.306542+0000 mgr.y (mgr.26252) 1668 : cluster [DBG] pgmap v2312: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:33.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:33 smithi099 bash[87147]: audit 2024-09-13T15:21:33.327072+0000 mon.a (mon.0) 2868 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:33.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:33 smithi099 bash[87147]: audit 2024-09-13T15:21:33.327072+0000 mon.a (mon.0) 2868 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:33 smithi177 bash[68594]: cluster 2024-09-13T15:21:33.306542+0000 mgr.y (mgr.26252) 1668 : cluster [DBG] pgmap v2312: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:33 smithi177 bash[68594]: cluster 2024-09-13T15:21:33.306542+0000 mgr.y (mgr.26252) 1668 : cluster [DBG] pgmap v2312: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:33 smithi177 bash[68594]: audit 2024-09-13T15:21:33.327072+0000 mon.a (mon.0) 2868 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:33 smithi177 bash[68594]: audit 2024-09-13T15:21:33.327072+0000 mon.a (mon.0) 2868 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:35.011 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:21:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:35 smithi177 bash[68594]: cluster 2024-09-13T15:21:35.307372+0000 mgr.y (mgr.26252) 1669 : cluster [DBG] pgmap v2313: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:35.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:35 smithi177 bash[68594]: cluster 2024-09-13T15:21:35.307372+0000 mgr.y (mgr.26252) 1669 : cluster [DBG] pgmap v2313: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:35.905 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:35 smithi099 bash[87147]: cluster 2024-09-13T15:21:35.307372+0000 mgr.y (mgr.26252) 1669 : cluster [DBG] pgmap v2313: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:35.906 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:35 smithi099 bash[87147]: cluster 2024-09-13T15:21:35.307372+0000 mgr.y (mgr.26252) 1669 : cluster [DBG] pgmap v2313: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:35.906 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:35 smithi099 bash[84576]: cluster 2024-09-13T15:21:35.307372+0000 mgr.y (mgr.26252) 1669 : cluster [DBG] pgmap v2313: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:35.906 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:35 smithi099 bash[84576]: cluster 2024-09-13T15:21:35.307372+0000 mgr.y (mgr.26252) 1669 : cluster [DBG] pgmap v2313: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:37.074 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:21:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:21:37] "GET /metrics HTTP/1.1" 200 34344 "" "Prometheus/2.51.0" 2024-09-13T15:21:37.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:37 smithi177 bash[68594]: cluster 2024-09-13T15:21:37.308380+0000 mgr.y (mgr.26252) 1670 : cluster [DBG] pgmap v2314: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:37.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:37 smithi177 bash[68594]: cluster 2024-09-13T15:21:37.308380+0000 mgr.y (mgr.26252) 1670 : cluster [DBG] pgmap v2314: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:37.878 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:37 smithi099 bash[84576]: cluster 2024-09-13T15:21:37.308380+0000 mgr.y (mgr.26252) 1670 : cluster [DBG] pgmap v2314: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:37.878 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:37 smithi099 bash[84576]: cluster 2024-09-13T15:21:37.308380+0000 mgr.y (mgr.26252) 1670 : cluster [DBG] pgmap v2314: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:37.878 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:37 smithi099 bash[87147]: cluster 2024-09-13T15:21:37.308380+0000 mgr.y (mgr.26252) 1670 : cluster [DBG] pgmap v2314: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:37.879 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:37 smithi099 bash[87147]: cluster 2024-09-13T15:21:37.308380+0000 mgr.y (mgr.26252) 1670 : cluster [DBG] pgmap v2314: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:39.732 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:39 smithi099 bash[84576]: cluster 2024-09-13T15:21:39.309224+0000 mgr.y (mgr.26252) 1671 : cluster [DBG] pgmap v2315: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:39.733 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:39 smithi099 bash[84576]: cluster 2024-09-13T15:21:39.309224+0000 mgr.y (mgr.26252) 1671 : cluster [DBG] pgmap v2315: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:39.733 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:39 smithi099 bash[87147]: cluster 2024-09-13T15:21:39.309224+0000 mgr.y (mgr.26252) 1671 : cluster [DBG] pgmap v2315: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:39.733 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:39 smithi099 bash[87147]: cluster 2024-09-13T15:21:39.309224+0000 mgr.y (mgr.26252) 1671 : cluster [DBG] pgmap v2315: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:39 smithi177 bash[68594]: cluster 2024-09-13T15:21:39.309224+0000 mgr.y (mgr.26252) 1671 : cluster [DBG] pgmap v2315: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:39.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:39 smithi177 bash[68594]: cluster 2024-09-13T15:21:39.309224+0000 mgr.y (mgr.26252) 1671 : cluster [DBG] pgmap v2315: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:40.402 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T15:21:40.613 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:21:40 smithi177 bash[61439]: ts=2024-09-13T15:21:40.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:21:40.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:40 smithi177 bash[68594]: audit 2024-09-13T15:21:39.899508+0000 mon.c (mon.2) 130 : audit [INF] from='client.? 172.21.15.99:0/789699466' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:40 smithi177 bash[68594]: audit 2024-09-13T15:21:39.899508+0000 mon.c (mon.2) 130 : audit [INF] from='client.? 172.21.15.99:0/789699466' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:40 smithi177 bash[68594]: audit 2024-09-13T15:21:39.899952+0000 mon.a (mon.0) 2869 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:40 smithi177 bash[68594]: audit 2024-09-13T15:21:39.899952+0000 mon.a (mon.0) 2869 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:40 smithi099 bash[87147]: audit 2024-09-13T15:21:39.899508+0000 mon.c (mon.2) 130 : audit [INF] from='client.? 172.21.15.99:0/789699466' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:40 smithi099 bash[87147]: audit 2024-09-13T15:21:39.899508+0000 mon.c (mon.2) 130 : audit [INF] from='client.? 172.21.15.99:0/789699466' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:40 smithi099 bash[87147]: audit 2024-09-13T15:21:39.899952+0000 mon.a (mon.0) 2869 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:40 smithi099 bash[87147]: audit 2024-09-13T15:21:39.899952+0000 mon.a (mon.0) 2869 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:40 smithi099 bash[84576]: audit 2024-09-13T15:21:39.899508+0000 mon.c (mon.2) 130 : audit [INF] from='client.? 172.21.15.99:0/789699466' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:40 smithi099 bash[84576]: audit 2024-09-13T15:21:39.899508+0000 mon.c (mon.2) 130 : audit [INF] from='client.? 172.21.15.99:0/789699466' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:40 smithi099 bash[84576]: audit 2024-09-13T15:21:39.899952+0000 mon.a (mon.0) 2869 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:40.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:40 smithi099 bash[84576]: audit 2024-09-13T15:21:39.899952+0000 mon.a (mon.0) 2869 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:41.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[84576]: audit 2024-09-13T15:21:40.385350+0000 mon.a (mon.0) 2870 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:41.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[84576]: audit 2024-09-13T15:21:40.385350+0000 mon.a (mon.0) 2870 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:41.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[84576]: cluster 2024-09-13T15:21:40.393164+0000 mon.a (mon.0) 2871 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2024-09-13T15:21:41.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[84576]: cluster 2024-09-13T15:21:40.393164+0000 mon.a (mon.0) 2871 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2024-09-13T15:21:41.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[84576]: cluster 2024-09-13T15:21:41.310031+0000 mgr.y (mgr.26252) 1672 : cluster [DBG] pgmap v2317: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:41.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[84576]: cluster 2024-09-13T15:21:41.310031+0000 mgr.y (mgr.26252) 1672 : cluster [DBG] pgmap v2317: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:41.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[87147]: audit 2024-09-13T15:21:40.385350+0000 mon.a (mon.0) 2870 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:41.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[87147]: audit 2024-09-13T15:21:40.385350+0000 mon.a (mon.0) 2870 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:41.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[87147]: cluster 2024-09-13T15:21:40.393164+0000 mon.a (mon.0) 2871 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2024-09-13T15:21:41.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[87147]: cluster 2024-09-13T15:21:40.393164+0000 mon.a (mon.0) 2871 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2024-09-13T15:21:41.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[87147]: cluster 2024-09-13T15:21:41.310031+0000 mgr.y (mgr.26252) 1672 : cluster [DBG] pgmap v2317: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:41.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:41 smithi099 bash[87147]: cluster 2024-09-13T15:21:41.310031+0000 mgr.y (mgr.26252) 1672 : cluster [DBG] pgmap v2317: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:41 smithi177 bash[68594]: audit 2024-09-13T15:21:40.385350+0000 mon.a (mon.0) 2870 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:41.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:41 smithi177 bash[68594]: audit 2024-09-13T15:21:40.385350+0000 mon.a (mon.0) 2870 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:41.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:41 smithi177 bash[68594]: cluster 2024-09-13T15:21:40.393164+0000 mon.a (mon.0) 2871 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2024-09-13T15:21:41.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:41 smithi177 bash[68594]: cluster 2024-09-13T15:21:40.393164+0000 mon.a (mon.0) 2871 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2024-09-13T15:21:41.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:41 smithi177 bash[68594]: cluster 2024-09-13T15:21:41.310031+0000 mgr.y (mgr.26252) 1672 : cluster [DBG] pgmap v2317: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:41.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:41 smithi177 bash[68594]: cluster 2024-09-13T15:21:41.310031+0000 mgr.y (mgr.26252) 1672 : cluster [DBG] pgmap v2317: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:42.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 0 1 2024-09-13T15:21:42.516 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:21:42.516 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T15:21:42.516 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T15:21:42.516 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T15:21:42.516 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.516 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:27952 46.3 46:c4025a4b:::benchmark_data_smithi099_233884_object27951:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:21:42.516 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:15:32.106068+0000", 2024-09-13T15:21:42.516 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 370.40638677499999, 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85988564000000001, 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 27952 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.517 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.106068+0000", 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.106068+0000", 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.106070+0000", 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1550000000000001e-06 2024-09-13T15:21:42.518 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.106095+0000", 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4700999999999999e-05 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.106110+0000", 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5228000000000001e-05 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.519 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.106118+0000", 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5080000000000005e-06 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.106186+0000", 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7769999999999997e-05 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.106327+0000", 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014127399999999999 2024-09-13T15:21:42.520 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.106491+0000", 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000163714 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.108668+0000", 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021766899999999998 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.521 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.962552+0000", 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85388372899999998 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.965749+0000", 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031975240000000002 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.965854+0000", 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000105284 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.965953+0000", 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9062999999999997e-05 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:27963 46.8 46:1a3d25e7:::benchmark_data_smithi099_233884_object27962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:15:32.113517+0000", 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 370.39893745799998, 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.86556593999999998, 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.523 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 27963 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.113517+0000", 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.524 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.113517+0000", 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.113519+0000", 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4980000000000001e-06 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.113551+0000", 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.201e-05 2024-09-13T15:21:42.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.961899+0000", 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.84834805300000005 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.961938+0000", 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8832999999999999e-05 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.526 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.969534+0000", 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007596004 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.969794+0000", 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026050800000000001 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5", 2024-09-13T15:21:42.527 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.970030+0000", 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000235565 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.976859+0000", 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0068297870000000004 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.977067+0000", 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020736999999999999 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.528 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.978923+0000", 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018566100000000001 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.979009+0000", 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6063999999999995e-05 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.979083+0000", 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3638000000000002e-05 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.529 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:89292 46.13 46:c8c6fb1c:::benchmark_data_smithi099_233884_object89291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:31.713942+0000", 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 310.79851304800002, 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.05743411, 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T15:21:42.530 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 89292 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.713942+0000", 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.531 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.713942+0000", 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.713943+0000", 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.686e-06 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.713995+0000", 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1910999999999998e-05 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.532 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.714009+0000", 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3932e-05 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.714017+0000", 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8380000000000002e-06 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.991376+0000", 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27735870800000001 2024-09-13T15:21:42.533 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.991385+0000", 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8470000000000002e-06 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.771290+0000", 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.779905556 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.771376+0000", 2024-09-13T15:21:42.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5631999999999996e-05 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:89304 46.3 46:c25b49e8:::benchmark_data_smithi099_233884_object89303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:31.714035+0000", 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 310.79842006199999, 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.013787489, 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.535 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 89304 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.714035+0000", 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.714035+0000", 2024-09-13T15:21:42.536 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.714035+0000", 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9299999999999998e-07 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.714082+0000", 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6814999999999998e-05 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.537 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.714092+0000", 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7200000000000001e-06 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.714098+0000", 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4869999999999999e-06 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.714532+0000", 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00043351199999999999 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.538 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.714539+0000", 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4100000000000002e-06 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.991406+0000", 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27686693099999998 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.991411+0000", 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2249999999999999e-06 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.539 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.727768+0000", 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73635627299999995 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.727822+0000", 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4523e-05 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101362 46.11 46:884716f8:::benchmark_data_smithi099_233884_object101361:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.540 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:45.886195+0000", 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.62625936799998, 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9196567280000001, 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101362 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.541 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.886195+0000", 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.886195+0000", 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.886198+0000", 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4360000000000001e-06 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.886225+0000", 2024-09-13T15:21:42.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7639000000000001e-05 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.886245+0000", 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9913000000000001e-05 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.886254+0000", 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6039999999999993e-06 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.886340+0000", 2024-09-13T15:21:42.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6533999999999995e-05 2024-09-13T15:21:42.544 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.544 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.544 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.544 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.886354+0000", 2024-09-13T15:21:42.544 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3125999999999999e-05 2024-09-13T15:21:42.544 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.661650+0000", 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.77529600799999998 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.661666+0000", 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6829000000000001e-05 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.787584+0000", 2024-09-13T15:21:42.545 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.125917917 2024-09-13T15:21:42.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.546 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.546 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.787756+0000", 2024-09-13T15:21:42.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000171755 2024-09-13T15:21:42.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.546 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.787866+0000", 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011021100000000001 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.794472+0000", 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006606158 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.794521+0000", 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8915999999999997e-05 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.802923+0000", 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0084013679999999993 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.803080+0000", 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015689 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.805852+0000", 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027724239999999999 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101370 46.11 46:89bfc6f3:::benchmark_data_smithi099_233884_object101369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:45.893085+0000", 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.61937009100001, 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.914529833, 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.549 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101370 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.893085+0000", 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.893085+0000", 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.893086+0000", 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9290000000000001e-06 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.893111+0000", 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4117e-05 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.893129+0000", 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8593e-05 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.893138+0000", 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7110000000000001e-06 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.893207+0000", 2024-09-13T15:21:42.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9343000000000003e-05 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.893217+0000", 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.533e-06 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.661724+0000", 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.76850757999999997 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.553 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.554 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.554 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.661737+0000", 2024-09-13T15:21:42.554 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2612999999999999e-05 2024-09-13T15:21:42.554 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.555 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.555 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.555 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.790212+0000", 2024-09-13T15:21:42.555 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.128474757 2024-09-13T15:21:42.555 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.555 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.555 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.790440+0000", 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022779300000000001 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.790666+0000", 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000226741 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.556 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.794537+0000", 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0038706639999999998 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.794668+0000", 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013120499999999999 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.805950+0000", 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011281629 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.557 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.806020+0000", 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0158000000000002e-05 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.807614+0000", 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001594467 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101389 46.11 46:8b270a6e:::benchmark_data_smithi099_233884_object101388:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:45.911660+0000", 2024-09-13T15:21:42.558 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.60079465899997, 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.903763863, 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101389 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.911660+0000", 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.559 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.911660+0000", 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.911663+0000", 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.57e-06 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.911687+0000", 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4615000000000001e-05 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.911706+0000", 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8757999999999999e-05 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.911715+0000", 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2259999999999994e-06 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.561 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.911792+0000", 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6890000000000004e-05 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.911803+0000", 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1433e-05 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662221+0000", 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.75041715099999995 2024-09-13T15:21:42.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662235+0000", 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4659e-05 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.792573+0000", 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.130337645 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.792785+0000", 2024-09-13T15:21:42.563 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021253 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.792986+0000", 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000200841 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.799301+0000", 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0063144469999999999 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.564 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.799404+0000", 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010327000000000001 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.813321+0000", 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013917024 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.813392+0000", 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0863000000000002e-05 2024-09-13T15:21:42.565 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.815424+0000", 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020319410000000002 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101450 46.11 46:8f57d545:::benchmark_data_smithi099_233884_object101449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:45.960498+0000", 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.551957055, 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.862516372, 2024-09-13T15:21:42.566 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101450 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.960498+0000", 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.567 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.960498+0000", 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.960499+0000", 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.798e-06 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.960522+0000", 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2731999999999999e-05 2024-09-13T15:21:42.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.960600+0000", 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7954999999999995e-05 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.960608+0000", 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6310000000000003e-06 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.960675+0000", 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7606000000000003e-05 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.960685+0000", 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4429999999999992e-06 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662288+0000", 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70160281300000005 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.570 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662296+0000", 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2740000000000007e-06 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.794734+0000", 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1324380860000001 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.794877+0000", 2024-09-13T15:21:42.571 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014290799999999999 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.794988+0000", 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011107099999999999 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.799391+0000", 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044033020000000004 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.799408+0000", 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7085e-05 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.821013+0000", 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021604624999999999 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.821079+0000", 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5714000000000001e-05 2024-09-13T15:21:42.573 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.823014+0000", 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001935329 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101479 46.11 46:8f39789d:::benchmark_data_smithi099_233884_object101478:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:45.988672+0000", 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.52378295, 2024-09-13T15:21:42.574 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8360393149999998, 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101479 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.988672+0000", 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.988672+0000", 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.988673+0000", 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.697e-06 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.988697+0000", 2024-09-13T15:21:42.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3516999999999999e-05 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.988718+0000", 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1580999999999999e-05 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.988733+0000", 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4739000000000001e-05 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.577 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.988838+0000", 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104838 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:45.988847+0000", 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8629999999999997e-06 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662370+0000", 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67352327199999995 2024-09-13T15:21:42.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662377+0000", 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6619999999999999e-06 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.795851+0000", 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.13347401 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.796015+0000", 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000163869 2024-09-13T15:21:42.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.580 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.796197+0000", 2024-09-13T15:21:42.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018210899999999999 2024-09-13T15:21:42.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.802607+0000", 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064106049999999998 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.802724+0000", 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116693 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.823070+0000", 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020345772000000002 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.823139+0000", 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8627000000000005e-05 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.582 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.824711+0000", 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001572461 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101505 46.11 46:8b40a461:::benchmark_data_smithi099_233884_object101504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.022413+0000", 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.490041507, 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8040509230000001, 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.583 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101505 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.022413+0000", 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.022413+0000", 2024-09-13T15:21:42.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.586 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.022415+0000", 2024-09-13T15:21:42.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9089999999999998e-06 2024-09-13T15:21:42.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.022440+0000", 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4573999999999999e-05 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.022457+0000", 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7689999999999998e-05 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.022465+0000", 2024-09-13T15:21:42.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2039999999999997e-06 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.022534+0000", 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8878e-05 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.022543+0000", 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0510000000000002e-06 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662418+0000", 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.63987476899999995 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662427+0000", 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1099999999999992e-06 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.797308+0000", 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1348802629999999 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.797501+0000", 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019349699999999999 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.797718+0000", 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021644899999999999 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.802616+0000", 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048980330000000004 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.591 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.591 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.802805+0000", 2024-09-13T15:21:42.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018981099999999999 2024-09-13T15:21:42.591 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.591 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.591 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.591 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.824861+0000", 2024-09-13T15:21:42.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022055314999999999 2024-09-13T15:21:42.591 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.824915+0000", 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4370999999999998e-05 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.826464+0000", 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015489989999999999 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.592 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101532 46.11 46:898d280a:::benchmark_data_smithi099_233884_object101531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.044241+0000", 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.46821385200002, 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7837505579999999, 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.593 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101532 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.044241+0000", 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.044241+0000", 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.044242+0000", 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1000000000000001e-06 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.044274+0000", 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2394999999999997e-05 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.044289+0000", 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4949999999999999e-05 2024-09-13T15:21:42.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.044296+0000", 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9770000000000003e-06 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.044368+0000", 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1951000000000003e-05 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.044377+0000", 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3309999999999993e-06 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662464+0000", 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.61808684599999997 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662471+0000", 2024-09-13T15:21:42.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5660000000000003e-06 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.799456+0000", 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.136984751 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.799656+0000", 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020028600000000001 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.799884+0000", 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022836899999999999 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.802821+0000", 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029363700000000002 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.807689+0000", 2024-09-13T15:21:42.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048678690000000004 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.826517+0000", 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018828306999999999 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.826573+0000", 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6076999999999999e-05 2024-09-13T15:21:42.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.827991+0000", 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014184130000000001 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101557 46.11 46:8e7ef0ad:::benchmark_data_smithi099_233884_object101556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.067807+0000", 2024-09-13T15:21:42.601 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.44464804500001, 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.761543173, 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101557 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.067807+0000", 2024-09-13T15:21:42.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.067807+0000", 2024-09-13T15:21:42.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.067809+0000", 2024-09-13T15:21:42.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1239999999999999e-06 2024-09-13T15:21:42.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.067833+0000", 2024-09-13T15:21:42.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4039000000000001e-05 2024-09-13T15:21:42.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.067847+0000", 2024-09-13T15:21:42.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4599e-05 2024-09-13T15:21:42.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.067855+0000", 2024-09-13T15:21:42.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7919999999999993e-06 2024-09-13T15:21:42.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.067932+0000", 2024-09-13T15:21:42.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6376000000000002e-05 2024-09-13T15:21:42.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.067942+0000", 2024-09-13T15:21:42.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0441e-05 2024-09-13T15:21:42.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662502+0000", 2024-09-13T15:21:42.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59456010199999998 2024-09-13T15:21:42.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662510+0000", 2024-09-13T15:21:42.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4079999999999998e-06 2024-09-13T15:21:42.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.801509+0000", 2024-09-13T15:21:42.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.138998537 2024-09-13T15:21:42.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.801650+0000", 2024-09-13T15:21:42.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014145299999999999 2024-09-13T15:21:42.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.801762+0000", 2024-09-13T15:21:42.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111062 2024-09-13T15:21:42.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.807635+0000", 2024-09-13T15:21:42.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058738280000000002 2024-09-13T15:21:42.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.807769+0000", 2024-09-13T15:21:42.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013347599999999999 2024-09-13T15:21:42.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.828038+0000", 2024-09-13T15:21:42.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020268982000000001 2024-09-13T15:21:42.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.828095+0000", 2024-09-13T15:21:42.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7340999999999999e-05 2024-09-13T15:21:42.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.829350+0000", 2024-09-13T15:21:42.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012546129999999999 2024-09-13T15:21:42.616 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.616 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.616 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.617 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101577 46.11 46:8d57b67e:::benchmark_data_smithi099_233884_object101576:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.617 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.088874+0000", 2024-09-13T15:21:42.617 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.42358041300002, 2024-09-13T15:21:42.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7602929359999999, 2024-09-13T15:21:42.618 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.618 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.618 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.618 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.618 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.618 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.619 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101577 2024-09-13T15:21:42.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.619 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.088874+0000", 2024-09-13T15:21:42.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.088874+0000", 2024-09-13T15:21:42.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.088876+0000", 2024-09-13T15:21:42.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.455e-06 2024-09-13T15:21:42.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.088896+0000", 2024-09-13T15:21:42.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0472999999999999e-05 2024-09-13T15:21:42.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.088907+0000", 2024-09-13T15:21:42.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1277999999999999e-05 2024-09-13T15:21:42.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.088913+0000", 2024-09-13T15:21:42.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2279999999999998e-06 2024-09-13T15:21:42.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.088975+0000", 2024-09-13T15:21:42.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2708999999999999e-05 2024-09-13T15:21:42.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.626 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.088984+0000", 2024-09-13T15:21:42.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4100000000000008e-06 2024-09-13T15:21:42.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.626 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662579+0000", 2024-09-13T15:21:42.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.57359485099999996 2024-09-13T15:21:42.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662586+0000", 2024-09-13T15:21:42.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8180000000000004e-06 2024-09-13T15:21:42.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.807826+0000", 2024-09-13T15:21:42.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.145239729 2024-09-13T15:21:42.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.808003+0000", 2024-09-13T15:21:42.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017691300000000001 2024-09-13T15:21:42.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.808198+0000", 2024-09-13T15:21:42.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000194909 2024-09-13T15:21:42.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.813241+0000", 2024-09-13T15:21:42.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005043077 2024-09-13T15:21:42.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.815442+0000", 2024-09-13T15:21:42.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022013359999999999 2024-09-13T15:21:42.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.847290+0000", 2024-09-13T15:21:42.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.031847684000000001 2024-09-13T15:21:42.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.847356+0000", 2024-09-13T15:21:42.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6125999999999997e-05 2024-09-13T15:21:42.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.849167+0000", 2024-09-13T15:21:42.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00181094 2024-09-13T15:21:42.633 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.633 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.634 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.634 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101645 46.11 46:8e654261:::benchmark_data_smithi099_233884_object101644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.634 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.155590+0000", 2024-09-13T15:21:42.634 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.35686447900002, 2024-09-13T15:21:42.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.69506515, 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101645 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.635 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.155590+0000", 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.155590+0000", 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.155591+0000", 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.16e-07 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.155617+0000", 2024-09-13T15:21:42.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6139000000000002e-05 2024-09-13T15:21:42.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.155625+0000", 2024-09-13T15:21:42.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9489999999999996e-06 2024-09-13T15:21:42.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.155630+0000", 2024-09-13T15:21:42.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5709999999999999e-06 2024-09-13T15:21:42.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.155693+0000", 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3309e-05 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.155702+0000", 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2720000000000003e-06 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662621+0000", 2024-09-13T15:21:42.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.50691888399999996 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662628+0000", 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.024e-06 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.809749+0000", 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1471209250000001 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.809930+0000", 2024-09-13T15:21:42.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000180707 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.810112+0000", 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018193300000000001 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.815448+0000", 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0053361889999999999 2024-09-13T15:21:42.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.820902+0000", 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054541559999999999 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.849276+0000", 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028374124000000001 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.849315+0000", 2024-09-13T15:21:42.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8343000000000002e-05 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.850655+0000", 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001340709 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101678 46.11 46:8bc24ef3:::benchmark_data_smithi099_233884_object101677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.195285+0000", 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.31716930200002, 2024-09-13T15:21:42.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6567260350000002, 2024-09-13T15:21:42.647 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.647 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.647 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.647 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.647 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.647 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.647 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101678 2024-09-13T15:21:42.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.647 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.195285+0000", 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.195285+0000", 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.195287+0000", 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3430000000000001e-06 2024-09-13T15:21:42.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.195308+0000", 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1359e-05 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.195324+0000", 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6161e-05 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.195332+0000", 2024-09-13T15:21:42.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9289999999999998e-06 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.195419+0000", 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6395000000000002e-05 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.195429+0000", 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0556e-05 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662661+0000", 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.46723206900000003 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662668+0000", 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2030000000000003e-06 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.811560+0000", 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.148891917 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.811729+0000", 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016834700000000001 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.811898+0000", 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000169154 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.817310+0000", 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054123859999999999 2024-09-13T15:21:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.820966+0000", 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0036558770000000001 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.850746+0000", 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.029780152000000001 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.850821+0000", 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4734000000000006e-05 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.852011+0000", 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001190453 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101720 46.11 46:8d99e526:::benchmark_data_smithi099_233884_object101719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.336407+0000", 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.176047939, 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5167736700000001, 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101720 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.336407+0000", 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.336407+0000", 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.336408+0000", 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.387e-06 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.336479+0000", 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1242999999999995e-05 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.336499+0000", 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9964999999999999e-05 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.336510+0000", 2024-09-13T15:21:42.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0512e-05 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.336590+0000", 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9722999999999995e-05 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.336599+0000", 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1449999999999997e-06 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662700+0000", 2024-09-13T15:21:42.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.32610174200000003 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662707+0000", 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6549999999999998e-06 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.815488+0000", 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.152780514 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.659 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.815660+0000", 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017251899999999999 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.815832+0000", 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017188899999999999 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.820839+0000", 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0050072140000000003 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.824773+0000", 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003933598 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.852058+0000", 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.027285199999999999 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.852112+0000", 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3693000000000002e-05 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.853180+0000", 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001068671 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101735 46.11 46:8e8486cb:::benchmark_data_smithi099_233884_object101734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.490247+0000", 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.02220803, 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.364250006, 2024-09-13T15:21:42.662 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101735 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.490247+0000", 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.490247+0000", 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.490248+0000", 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.384e-06 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.490281+0000", 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2872e-05 2024-09-13T15:21:42.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.490300+0000", 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8899e-05 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.490313+0000", 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3499999999999999e-05 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.490459+0000", 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000146052 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.490485+0000", 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5361e-05 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662740+0000", 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.17225515799999999 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662747+0000", 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2710000000000003e-06 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.817342+0000", 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.154594557 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.817528+0000", 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018589499999999999 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.817712+0000", 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000184382 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.820916+0000", 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032043390000000001 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.824817+0000", 2024-09-13T15:21:42.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003900962 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.853234+0000", 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028416601 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.853293+0000", 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9304e-05 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.669 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.854497+0000", 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001203469 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101744 46.11 46:8fdbd90d:::benchmark_data_smithi099_233884_object101743:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.511346+0000", 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 296.00110903400002, 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.344623903, 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.670 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101744 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.511346+0000", 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.511346+0000", 2024-09-13T15:21:42.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.511348+0000", 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4030000000000001e-06 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.511398+0000", 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0182000000000002e-05 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.511416+0000", 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8133000000000001e-05 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.511426+0000", 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7359999999999996e-06 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.511555+0000", 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012853699999999999 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.511570+0000", 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5591e-05 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662769+0000", 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.15119907899999999 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.662776+0000", 2024-09-13T15:21:42.674 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.472e-06 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.819144+0000", 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.156367959 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.819287+0000", 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143263 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.819427+0000", 2024-09-13T15:21:42.675 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014034599999999999 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.824727+0000", 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052992650000000001 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.834530+0000", 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0098032540000000008 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.676 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.854636+0000", 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020105720000000001 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.854761+0000", 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012559 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.855969+0000", 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001208373 2024-09-13T15:21:42.677 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101826 46.11 46:894cd0a6:::benchmark_data_smithi099_233884_object101825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.715328+0000", 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 295.79712641200001, 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.144069244, 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.678 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101826 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.715328+0000", 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.715328+0000", 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.680 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.715331+0000", 2024-09-13T15:21:42.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.402e-06 2024-09-13T15:21:42.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.680 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.715358+0000", 2024-09-13T15:21:42.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.743e-05 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.715377+0000", 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9083e-05 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.715387+0000", 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0165e-05 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.681 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.715473+0000", 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5662999999999996e-05 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.715487+0000", 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3584000000000001e-05 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.829444+0000", 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1139572980000001 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.829622+0000", 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017856600000000001 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.829780+0000", 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015805300000000001 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.832991+0000", 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032103269999999998 2024-09-13T15:21:42.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.836545+0000", 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0035545860000000002 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.857817+0000", 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021272045 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.684 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.857854+0000", 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6437999999999999e-05 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.859397+0000", 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015436040000000001 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.685 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:101863 46.11 46:8fc96fab:::benchmark_data_smithi099_233884_object101862:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:46.822237+0000", 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 295.69021774700002, 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0457730430000001, 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 101863 2024-09-13T15:21:42.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.687 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:21:42.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:21:42.687 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.822237+0000", 2024-09-13T15:21:42.687 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:21:42.687 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.822237+0000", 2024-09-13T15:21:42.687 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:21:42.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.688 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:21:42.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.822239+0000", 2024-09-13T15:21:42.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3999999999999999e-06 2024-09-13T15:21:42.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.688 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:21:42.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.822269+0000", 2024-09-13T15:21:42.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0020000000000001e-05 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.822291+0000", 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2116999999999998e-05 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.822303+0000", 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1280999999999999e-05 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.689 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.822383+0000", 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0301000000000002e-05 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:46.822397+0000", 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3706999999999999e-05 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.830955+0000", 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.008558584 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.690 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.831087+0000", 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013125100000000001 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.831257+0000", 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017036899999999999 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:21:42.692 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.834107+0000", 2024-09-13T15:21:42.692 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028496680000000001 2024-09-13T15:21:42.692 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.692 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.693 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.847244+0000", 2024-09-13T15:21:42.693 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013136937 2024-09-13T15:21:42.693 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.693 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.693 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:21:42.693 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.866383+0000", 2024-09-13T15:21:42.694 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019139697000000001 2024-09-13T15:21:42.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.694 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.694 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:21:42.694 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.866448+0000", 2024-09-13T15:21:42.694 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5066000000000003e-05 2024-09-13T15:21:42.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:21:42.695 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:21:42.695 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:21:42.695 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:47.868010+0000", 2024-09-13T15:21:42.695 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001561645 2024-09-13T15:21:42.695 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.695 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.696 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.696 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:21:42.696 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:21:42.696 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:21:42.697 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:42 smithi099 bash[84576]: cluster 2024-09-13T15:21:41.400049+0000 mon.a (mon.0) 2872 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2024-09-13T15:21:42.697 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:42 smithi099 bash[84576]: cluster 2024-09-13T15:21:41.400049+0000 mon.a (mon.0) 2872 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2024-09-13T15:21:42.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:42 smithi099 bash[87147]: cluster 2024-09-13T15:21:41.400049+0000 mon.a (mon.0) 2872 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2024-09-13T15:21:42.698 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:42 smithi099 bash[87147]: cluster 2024-09-13T15:21:41.400049+0000 mon.a (mon.0) 2872 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2024-09-13T15:21:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:42 smithi177 bash[68594]: cluster 2024-09-13T15:21:41.400049+0000 mon.a (mon.0) 2872 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2024-09-13T15:21:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:42 smithi177 bash[68594]: cluster 2024-09-13T15:21:41.400049+0000 mon.a (mon.0) 2872 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2024-09-13T15:21:43.667 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:43 smithi099 bash[84576]: cluster 2024-09-13T15:21:43.311000+0000 mgr.y (mgr.26252) 1673 : cluster [DBG] pgmap v2319: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:43.667 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:43 smithi099 bash[84576]: cluster 2024-09-13T15:21:43.311000+0000 mgr.y (mgr.26252) 1673 : cluster [DBG] pgmap v2319: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:43.667 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:43 smithi099 bash[87147]: cluster 2024-09-13T15:21:43.311000+0000 mgr.y (mgr.26252) 1673 : cluster [DBG] pgmap v2319: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:43.667 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:43 smithi099 bash[87147]: cluster 2024-09-13T15:21:43.311000+0000 mgr.y (mgr.26252) 1673 : cluster [DBG] pgmap v2319: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:43.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T15:21:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:43 smithi177 bash[68594]: cluster 2024-09-13T15:21:43.311000+0000 mgr.y (mgr.26252) 1673 : cluster [DBG] pgmap v2319: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:43.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:43 smithi177 bash[68594]: cluster 2024-09-13T15:21:43.311000+0000 mgr.y (mgr.26252) 1673 : cluster [DBG] pgmap v2319: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:45.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:45 smithi099 bash[84576]: cluster 2024-09-13T15:21:45.311807+0000 mgr.y (mgr.26252) 1674 : cluster [DBG] pgmap v2320: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:45.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:45 smithi099 bash[84576]: cluster 2024-09-13T15:21:45.311807+0000 mgr.y (mgr.26252) 1674 : cluster [DBG] pgmap v2320: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:45.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:45 smithi099 bash[87147]: cluster 2024-09-13T15:21:45.311807+0000 mgr.y (mgr.26252) 1674 : cluster [DBG] pgmap v2320: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:45.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:45 smithi099 bash[87147]: cluster 2024-09-13T15:21:45.311807+0000 mgr.y (mgr.26252) 1674 : cluster [DBG] pgmap v2320: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:45.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:45 smithi177 bash[68594]: cluster 2024-09-13T15:21:45.311807+0000 mgr.y (mgr.26252) 1674 : cluster [DBG] pgmap v2320: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:45.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:45 smithi177 bash[68594]: cluster 2024-09-13T15:21:45.311807+0000 mgr.y (mgr.26252) 1674 : cluster [DBG] pgmap v2320: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:47.195 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:21:47.334 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:21:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:21:47] "GET /metrics HTTP/1.1" 200 34344 "" "Prometheus/2.51.0" 2024-09-13T15:21:47.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:47 smithi099 bash[84576]: cluster 2024-09-13T15:21:47.312829+0000 mgr.y (mgr.26252) 1675 : cluster [DBG] pgmap v2321: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:47.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:47 smithi099 bash[84576]: cluster 2024-09-13T15:21:47.312829+0000 mgr.y (mgr.26252) 1675 : cluster [DBG] pgmap v2321: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:47.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:47 smithi099 bash[87147]: cluster 2024-09-13T15:21:47.312829+0000 mgr.y (mgr.26252) 1675 : cluster [DBG] pgmap v2321: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:47.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:47 smithi099 bash[87147]: cluster 2024-09-13T15:21:47.312829+0000 mgr.y (mgr.26252) 1675 : cluster [DBG] pgmap v2321: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:47.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:47 smithi177 bash[68594]: cluster 2024-09-13T15:21:47.312829+0000 mgr.y (mgr.26252) 1675 : cluster [DBG] pgmap v2321: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:47.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:47 smithi177 bash[68594]: cluster 2024-09-13T15:21:47.312829+0000 mgr.y (mgr.26252) 1675 : cluster [DBG] pgmap v2321: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:48.216 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:21:48.673 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:48 smithi099 bash[84576]: audit 2024-09-13T15:21:48.327804+0000 mon.a (mon.0) 2873 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:48.673 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:48 smithi099 bash[84576]: audit 2024-09-13T15:21:48.327804+0000 mon.a (mon.0) 2873 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:48.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:48 smithi099 bash[87147]: audit 2024-09-13T15:21:48.327804+0000 mon.a (mon.0) 2873 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:48.673 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:48 smithi099 bash[87147]: audit 2024-09-13T15:21:48.327804+0000 mon.a (mon.0) 2873 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:48.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:48 smithi177 bash[68594]: audit 2024-09-13T15:21:48.327804+0000 mon.a (mon.0) 2873 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:48.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:48 smithi177 bash[68594]: audit 2024-09-13T15:21:48.327804+0000 mon.a (mon.0) 2873 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:21:49.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:49 smithi099 bash[84576]: cluster 2024-09-13T15:21:49.313707+0000 mgr.y (mgr.26252) 1676 : cluster [DBG] pgmap v2322: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:49.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:49 smithi099 bash[84576]: cluster 2024-09-13T15:21:49.313707+0000 mgr.y (mgr.26252) 1676 : cluster [DBG] pgmap v2322: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:49.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:49 smithi099 bash[87147]: cluster 2024-09-13T15:21:49.313707+0000 mgr.y (mgr.26252) 1676 : cluster [DBG] pgmap v2322: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:49.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:49 smithi099 bash[87147]: cluster 2024-09-13T15:21:49.313707+0000 mgr.y (mgr.26252) 1676 : cluster [DBG] pgmap v2322: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:49.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:49 smithi177 bash[68594]: cluster 2024-09-13T15:21:49.313707+0000 mgr.y (mgr.26252) 1676 : cluster [DBG] pgmap v2322: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:49.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:49 smithi177 bash[68594]: cluster 2024-09-13T15:21:49.313707+0000 mgr.y (mgr.26252) 1676 : cluster [DBG] pgmap v2322: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:50.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:21:50 smithi177 bash[61439]: ts=2024-09-13T15:21:50.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:21:51.792 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:51 smithi099 bash[84576]: cluster 2024-09-13T15:21:51.314423+0000 mgr.y (mgr.26252) 1677 : cluster [DBG] pgmap v2323: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:51.792 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:51 smithi099 bash[84576]: cluster 2024-09-13T15:21:51.314423+0000 mgr.y (mgr.26252) 1677 : cluster [DBG] pgmap v2323: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:51.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:51 smithi099 bash[87147]: cluster 2024-09-13T15:21:51.314423+0000 mgr.y (mgr.26252) 1677 : cluster [DBG] pgmap v2323: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:51.793 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:51 smithi099 bash[87147]: cluster 2024-09-13T15:21:51.314423+0000 mgr.y (mgr.26252) 1677 : cluster [DBG] pgmap v2323: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:51 smithi177 bash[68594]: cluster 2024-09-13T15:21:51.314423+0000 mgr.y (mgr.26252) 1677 : cluster [DBG] pgmap v2323: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:51.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:51 smithi177 bash[68594]: cluster 2024-09-13T15:21:51.314423+0000 mgr.y (mgr.26252) 1677 : cluster [DBG] pgmap v2323: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:53.580 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:53 smithi099 bash[84576]: cluster 2024-09-13T15:21:53.315436+0000 mgr.y (mgr.26252) 1678 : cluster [DBG] pgmap v2324: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:53.580 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:53 smithi099 bash[84576]: cluster 2024-09-13T15:21:53.315436+0000 mgr.y (mgr.26252) 1678 : cluster [DBG] pgmap v2324: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:53.580 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:53 smithi099 bash[87147]: cluster 2024-09-13T15:21:53.315436+0000 mgr.y (mgr.26252) 1678 : cluster [DBG] pgmap v2324: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:53.580 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:53 smithi099 bash[87147]: cluster 2024-09-13T15:21:53.315436+0000 mgr.y (mgr.26252) 1678 : cluster [DBG] pgmap v2324: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:53.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:53 smithi177 bash[68594]: cluster 2024-09-13T15:21:53.315436+0000 mgr.y (mgr.26252) 1678 : cluster [DBG] pgmap v2324: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:53.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:53 smithi177 bash[68594]: cluster 2024-09-13T15:21:53.315436+0000 mgr.y (mgr.26252) 1678 : cluster [DBG] pgmap v2324: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:54.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:54 smithi099 bash[84576]: audit 2024-09-13T15:21:53.639836+0000 mon.a (mon.0) 2874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]} : dispatch 2024-09-13T15:21:54.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:54 smithi099 bash[84576]: audit 2024-09-13T15:21:53.639836+0000 mon.a (mon.0) 2874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]} : dispatch 2024-09-13T15:21:54.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:54 smithi099 bash[87147]: audit 2024-09-13T15:21:53.639836+0000 mon.a (mon.0) 2874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]} : dispatch 2024-09-13T15:21:54.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:54 smithi099 bash[87147]: audit 2024-09-13T15:21:53.639836+0000 mon.a (mon.0) 2874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]} : dispatch 2024-09-13T15:21:54.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:54 smithi177 bash[68594]: audit 2024-09-13T15:21:53.639836+0000 mon.a (mon.0) 2874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]} : dispatch 2024-09-13T15:21:54.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:54 smithi177 bash[68594]: audit 2024-09-13T15:21:53.639836+0000 mon.a (mon.0) 2874 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]} : dispatch 2024-09-13T15:21:55.142 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:21:55.143 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:21:55.143 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2024-09-13T15:21:55.143 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:21:55.399 INFO:teuthology.orchestra.run.smithi099.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-13T15:21:55.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[84576]: audit 2024-09-13T15:21:54.385068+0000 mon.a (mon.0) 2875 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2024-09-13T15:21:55.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[84576]: audit 2024-09-13T15:21:54.385068+0000 mon.a (mon.0) 2875 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2024-09-13T15:21:55.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[84576]: cluster 2024-09-13T15:21:54.387843+0000 mon.a (mon.0) 2876 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2024-09-13T15:21:55.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[84576]: cluster 2024-09-13T15:21:54.387843+0000 mon.a (mon.0) 2876 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2024-09-13T15:21:55.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[84576]: audit 2024-09-13T15:21:54.667637+0000 mon.a (mon.0) 2877 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:55.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[84576]: audit 2024-09-13T15:21:54.667637+0000 mon.a (mon.0) 2877 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:55.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[84576]: cluster 2024-09-13T15:21:55.316108+0000 mgr.y (mgr.26252) 1679 : cluster [DBG] pgmap v2326: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:55.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[84576]: cluster 2024-09-13T15:21:55.316108+0000 mgr.y (mgr.26252) 1679 : cluster [DBG] pgmap v2326: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:55.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[87147]: audit 2024-09-13T15:21:54.385068+0000 mon.a (mon.0) 2875 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2024-09-13T15:21:55.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[87147]: audit 2024-09-13T15:21:54.385068+0000 mon.a (mon.0) 2875 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2024-09-13T15:21:55.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[87147]: cluster 2024-09-13T15:21:54.387843+0000 mon.a (mon.0) 2876 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2024-09-13T15:21:55.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[87147]: cluster 2024-09-13T15:21:54.387843+0000 mon.a (mon.0) 2876 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2024-09-13T15:21:55.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[87147]: audit 2024-09-13T15:21:54.667637+0000 mon.a (mon.0) 2877 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:55.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[87147]: audit 2024-09-13T15:21:54.667637+0000 mon.a (mon.0) 2877 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:55.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[87147]: cluster 2024-09-13T15:21:55.316108+0000 mgr.y (mgr.26252) 1679 : cluster [DBG] pgmap v2326: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:55.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:55 smithi099 bash[87147]: cluster 2024-09-13T15:21:55.316108+0000 mgr.y (mgr.26252) 1679 : cluster [DBG] pgmap v2326: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:55.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:55 smithi177 bash[68594]: audit 2024-09-13T15:21:54.385068+0000 mon.a (mon.0) 2875 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2024-09-13T15:21:55.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:55 smithi177 bash[68594]: audit 2024-09-13T15:21:54.385068+0000 mon.a (mon.0) 2875 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2024-09-13T15:21:55.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:55 smithi177 bash[68594]: cluster 2024-09-13T15:21:54.387843+0000 mon.a (mon.0) 2876 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2024-09-13T15:21:55.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:55 smithi177 bash[68594]: cluster 2024-09-13T15:21:54.387843+0000 mon.a (mon.0) 2876 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2024-09-13T15:21:55.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:55 smithi177 bash[68594]: audit 2024-09-13T15:21:54.667637+0000 mon.a (mon.0) 2877 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:55.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:55 smithi177 bash[68594]: audit 2024-09-13T15:21:54.667637+0000 mon.a (mon.0) 2877 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:21:55.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:55 smithi177 bash[68594]: cluster 2024-09-13T15:21:55.316108+0000 mgr.y (mgr.26252) 1679 : cluster [DBG] pgmap v2326: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:55.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:55 smithi177 bash[68594]: cluster 2024-09-13T15:21:55.316108+0000 mgr.y (mgr.26252) 1679 : cluster [DBG] pgmap v2326: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:56.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T15:21:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:56 smithi099 bash[87147]: audit 2024-09-13T15:21:55.398208+0000 mon.a (mon.0) 2878 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:56 smithi099 bash[87147]: audit 2024-09-13T15:21:55.398208+0000 mon.a (mon.0) 2878 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:56 smithi099 bash[87147]: cluster 2024-09-13T15:21:55.402879+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2024-09-13T15:21:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:56 smithi099 bash[87147]: cluster 2024-09-13T15:21:55.402879+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2024-09-13T15:21:56.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:56 smithi099 bash[84576]: audit 2024-09-13T15:21:55.398208+0000 mon.a (mon.0) 2878 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:56.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:56 smithi099 bash[84576]: audit 2024-09-13T15:21:55.398208+0000 mon.a (mon.0) 2878 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:56.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:56 smithi099 bash[84576]: cluster 2024-09-13T15:21:55.402879+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2024-09-13T15:21:56.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:56 smithi099 bash[84576]: cluster 2024-09-13T15:21:55.402879+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2024-09-13T15:21:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:56 smithi177 bash[68594]: audit 2024-09-13T15:21:55.398208+0000 mon.a (mon.0) 2878 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:56.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:56 smithi177 bash[68594]: audit 2024-09-13T15:21:55.398208+0000 mon.a (mon.0) 2878 : audit [INF] from='client.? 172.21.15.99:0/2399096275' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:21:56.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:56 smithi177 bash[68594]: cluster 2024-09-13T15:21:55.402879+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2024-09-13T15:21:56.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:56 smithi177 bash[68594]: cluster 2024-09-13T15:21:55.402879+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2024-09-13T15:21:57.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T15:21:57.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:21:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:21:57] "GET /metrics HTTP/1.1" 200 34330 "" "Prometheus/2.51.0" 2024-09-13T15:21:57.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:57 smithi099 bash[84576]: audit 2024-09-13T15:21:56.674242+0000 mon.a (mon.0) 2880 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:21:57.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:57 smithi099 bash[84576]: audit 2024-09-13T15:21:56.674242+0000 mon.a (mon.0) 2880 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:21:57.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:57 smithi099 bash[84576]: cluster 2024-09-13T15:21:57.316653+0000 mgr.y (mgr.26252) 1680 : cluster [DBG] pgmap v2328: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:57.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:57 smithi099 bash[84576]: cluster 2024-09-13T15:21:57.316653+0000 mgr.y (mgr.26252) 1680 : cluster [DBG] pgmap v2328: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:57.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:57 smithi099 bash[87147]: audit 2024-09-13T15:21:56.674242+0000 mon.a (mon.0) 2880 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:21:57.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:57 smithi099 bash[87147]: audit 2024-09-13T15:21:56.674242+0000 mon.a (mon.0) 2880 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:21:57.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:57 smithi099 bash[87147]: cluster 2024-09-13T15:21:57.316653+0000 mgr.y (mgr.26252) 1680 : cluster [DBG] pgmap v2328: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:57.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:57 smithi099 bash[87147]: cluster 2024-09-13T15:21:57.316653+0000 mgr.y (mgr.26252) 1680 : cluster [DBG] pgmap v2328: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:57.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:57 smithi177 bash[68594]: audit 2024-09-13T15:21:56.674242+0000 mon.a (mon.0) 2880 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:21:57.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:57 smithi177 bash[68594]: audit 2024-09-13T15:21:56.674242+0000 mon.a (mon.0) 2880 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:21:57.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:57 smithi177 bash[68594]: cluster 2024-09-13T15:21:57.316653+0000 mgr.y (mgr.26252) 1680 : cluster [DBG] pgmap v2328: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:57.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:57 smithi177 bash[68594]: cluster 2024-09-13T15:21:57.316653+0000 mgr.y (mgr.26252) 1680 : cluster [DBG] pgmap v2328: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:59 smithi099 bash[84576]: cluster 2024-09-13T15:21:59.317152+0000 mgr.y (mgr.26252) 1681 : cluster [DBG] pgmap v2329: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:21:59 smithi099 bash[84576]: cluster 2024-09-13T15:21:59.317152+0000 mgr.y (mgr.26252) 1681 : cluster [DBG] pgmap v2329: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:59.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:59 smithi099 bash[87147]: cluster 2024-09-13T15:21:59.317152+0000 mgr.y (mgr.26252) 1681 : cluster [DBG] pgmap v2329: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:59.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:21:59 smithi099 bash[87147]: cluster 2024-09-13T15:21:59.317152+0000 mgr.y (mgr.26252) 1681 : cluster [DBG] pgmap v2329: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:59.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:59 smithi177 bash[68594]: cluster 2024-09-13T15:21:59.317152+0000 mgr.y (mgr.26252) 1681 : cluster [DBG] pgmap v2329: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:21:59.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:21:59 smithi177 bash[68594]: cluster 2024-09-13T15:21:59.317152+0000 mgr.y (mgr.26252) 1681 : cluster [DBG] pgmap v2329: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:00.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:22:00 smithi177 bash[61439]: ts=2024-09-13T15:22:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:22:01.342 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:22:01.343 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:22:01.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:01 smithi099 bash[87147]: cluster 2024-09-13T15:22:01.317987+0000 mgr.y (mgr.26252) 1682 : cluster [DBG] pgmap v2330: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:01.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:01 smithi099 bash[87147]: cluster 2024-09-13T15:22:01.317987+0000 mgr.y (mgr.26252) 1682 : cluster [DBG] pgmap v2330: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:01.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:01 smithi099 bash[84576]: cluster 2024-09-13T15:22:01.317987+0000 mgr.y (mgr.26252) 1682 : cluster [DBG] pgmap v2330: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:01.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:01 smithi099 bash[84576]: cluster 2024-09-13T15:22:01.317987+0000 mgr.y (mgr.26252) 1682 : cluster [DBG] pgmap v2330: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:01.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:01 smithi177 bash[68594]: cluster 2024-09-13T15:22:01.317987+0000 mgr.y (mgr.26252) 1682 : cluster [DBG] pgmap v2330: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:01.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:01 smithi177 bash[68594]: cluster 2024-09-13T15:22:01.317987+0000 mgr.y (mgr.26252) 1682 : cluster [DBG] pgmap v2330: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[84576]: audit 2024-09-13T15:22:01.995570+0000 mon.a (mon.0) 2881 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[84576]: audit 2024-09-13T15:22:01.995570+0000 mon.a (mon.0) 2881 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[84576]: audit 2024-09-13T15:22:02.000782+0000 mon.a (mon.0) 2882 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[84576]: audit 2024-09-13T15:22:02.000782+0000 mon.a (mon.0) 2882 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[84576]: audit 2024-09-13T15:22:02.741009+0000 mon.a (mon.0) 2883 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[84576]: audit 2024-09-13T15:22:02.741009+0000 mon.a (mon.0) 2883 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[84576]: audit 2024-09-13T15:22:02.750364+0000 mon.a (mon.0) 2884 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[84576]: audit 2024-09-13T15:22:02.750364+0000 mon.a (mon.0) 2884 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[87147]: audit 2024-09-13T15:22:01.995570+0000 mon.a (mon.0) 2881 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[87147]: audit 2024-09-13T15:22:01.995570+0000 mon.a (mon.0) 2881 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[87147]: audit 2024-09-13T15:22:02.000782+0000 mon.a (mon.0) 2882 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[87147]: audit 2024-09-13T15:22:02.000782+0000 mon.a (mon.0) 2882 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[87147]: audit 2024-09-13T15:22:02.741009+0000 mon.a (mon.0) 2883 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[87147]: audit 2024-09-13T15:22:02.741009+0000 mon.a (mon.0) 2883 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[87147]: audit 2024-09-13T15:22:02.750364+0000 mon.a (mon.0) 2884 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:02 smithi099 bash[87147]: audit 2024-09-13T15:22:02.750364+0000 mon.a (mon.0) 2884 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:02 smithi177 bash[68594]: audit 2024-09-13T15:22:01.995570+0000 mon.a (mon.0) 2881 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:02 smithi177 bash[68594]: audit 2024-09-13T15:22:01.995570+0000 mon.a (mon.0) 2881 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:02 smithi177 bash[68594]: audit 2024-09-13T15:22:02.000782+0000 mon.a (mon.0) 2882 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:02 smithi177 bash[68594]: audit 2024-09-13T15:22:02.000782+0000 mon.a (mon.0) 2882 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:02 smithi177 bash[68594]: audit 2024-09-13T15:22:02.741009+0000 mon.a (mon.0) 2883 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:02 smithi177 bash[68594]: audit 2024-09-13T15:22:02.741009+0000 mon.a (mon.0) 2883 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:02 smithi177 bash[68594]: audit 2024-09-13T15:22:02.750364+0000 mon.a (mon.0) 2884 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:03.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:02 smithi177 bash[68594]: audit 2024-09-13T15:22:02.750364+0000 mon.a (mon.0) 2884 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[84576]: cluster 2024-09-13T15:22:03.318930+0000 mgr.y (mgr.26252) 1683 : cluster [DBG] pgmap v2331: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[84576]: cluster 2024-09-13T15:22:03.318930+0000 mgr.y (mgr.26252) 1683 : cluster [DBG] pgmap v2331: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[84576]: audit 2024-09-13T15:22:03.328027+0000 mon.a (mon.0) 2885 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[84576]: audit 2024-09-13T15:22:03.328027+0000 mon.a (mon.0) 2885 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[84576]: audit 2024-09-13T15:22:03.499618+0000 mon.a (mon.0) 2886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[84576]: audit 2024-09-13T15:22:03.499618+0000 mon.a (mon.0) 2886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[84576]: audit 2024-09-13T15:22:03.501089+0000 mon.a (mon.0) 2887 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[84576]: audit 2024-09-13T15:22:03.501089+0000 mon.a (mon.0) 2887 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[84576]: audit 2024-09-13T15:22:03.515904+0000 mon.a (mon.0) 2888 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[84576]: audit 2024-09-13T15:22:03.515904+0000 mon.a (mon.0) 2888 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[87147]: cluster 2024-09-13T15:22:03.318930+0000 mgr.y (mgr.26252) 1683 : cluster [DBG] pgmap v2331: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[87147]: cluster 2024-09-13T15:22:03.318930+0000 mgr.y (mgr.26252) 1683 : cluster [DBG] pgmap v2331: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[87147]: audit 2024-09-13T15:22:03.328027+0000 mon.a (mon.0) 2885 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[87147]: audit 2024-09-13T15:22:03.328027+0000 mon.a (mon.0) 2885 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[87147]: audit 2024-09-13T15:22:03.499618+0000 mon.a (mon.0) 2886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:22:04.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[87147]: audit 2024-09-13T15:22:03.499618+0000 mon.a (mon.0) 2886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:22:04.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[87147]: audit 2024-09-13T15:22:03.501089+0000 mon.a (mon.0) 2887 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:22:04.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[87147]: audit 2024-09-13T15:22:03.501089+0000 mon.a (mon.0) 2887 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:22:04.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[87147]: audit 2024-09-13T15:22:03.515904+0000 mon.a (mon.0) 2888 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:04.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:04 smithi099 bash[87147]: audit 2024-09-13T15:22:03.515904+0000 mon.a (mon.0) 2888 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:04 smithi177 bash[68594]: cluster 2024-09-13T15:22:03.318930+0000 mgr.y (mgr.26252) 1683 : cluster [DBG] pgmap v2331: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:04 smithi177 bash[68594]: cluster 2024-09-13T15:22:03.318930+0000 mgr.y (mgr.26252) 1683 : cluster [DBG] pgmap v2331: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:04 smithi177 bash[68594]: audit 2024-09-13T15:22:03.328027+0000 mon.a (mon.0) 2885 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:04 smithi177 bash[68594]: audit 2024-09-13T15:22:03.328027+0000 mon.a (mon.0) 2885 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:04 smithi177 bash[68594]: audit 2024-09-13T15:22:03.499618+0000 mon.a (mon.0) 2886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:22:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:04 smithi177 bash[68594]: audit 2024-09-13T15:22:03.499618+0000 mon.a (mon.0) 2886 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:22:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:04 smithi177 bash[68594]: audit 2024-09-13T15:22:03.501089+0000 mon.a (mon.0) 2887 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:22:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:04 smithi177 bash[68594]: audit 2024-09-13T15:22:03.501089+0000 mon.a (mon.0) 2887 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:22:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:04 smithi177 bash[68594]: audit 2024-09-13T15:22:03.515904+0000 mon.a (mon.0) 2888 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:04 smithi177 bash[68594]: audit 2024-09-13T15:22:03.515904+0000 mon.a (mon.0) 2888 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:22:05.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:05 smithi099 bash[84576]: cluster 2024-09-13T15:22:05.319795+0000 mgr.y (mgr.26252) 1684 : cluster [DBG] pgmap v2332: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:05.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:05 smithi099 bash[84576]: cluster 2024-09-13T15:22:05.319795+0000 mgr.y (mgr.26252) 1684 : cluster [DBG] pgmap v2332: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:05.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:05 smithi099 bash[87147]: cluster 2024-09-13T15:22:05.319795+0000 mgr.y (mgr.26252) 1684 : cluster [DBG] pgmap v2332: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:05.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:05 smithi099 bash[87147]: cluster 2024-09-13T15:22:05.319795+0000 mgr.y (mgr.26252) 1684 : cluster [DBG] pgmap v2332: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:05.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:05 smithi177 bash[68594]: cluster 2024-09-13T15:22:05.319795+0000 mgr.y (mgr.26252) 1684 : cluster [DBG] pgmap v2332: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:05.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:05 smithi177 bash[68594]: cluster 2024-09-13T15:22:05.319795+0000 mgr.y (mgr.26252) 1684 : cluster [DBG] pgmap v2332: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:07.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:22:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:22:07] "GET /metrics HTTP/1.1" 200 34331 "" "Prometheus/2.51.0" 2024-09-13T15:22:07.652 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:07 smithi099 bash[84576]: cluster 2024-09-13T15:22:07.320821+0000 mgr.y (mgr.26252) 1685 : cluster [DBG] pgmap v2333: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:07.653 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:07 smithi099 bash[84576]: cluster 2024-09-13T15:22:07.320821+0000 mgr.y (mgr.26252) 1685 : cluster [DBG] pgmap v2333: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:07.653 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:07 smithi099 bash[87147]: cluster 2024-09-13T15:22:07.320821+0000 mgr.y (mgr.26252) 1685 : cluster [DBG] pgmap v2333: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:07.653 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:07 smithi099 bash[87147]: cluster 2024-09-13T15:22:07.320821+0000 mgr.y (mgr.26252) 1685 : cluster [DBG] pgmap v2333: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:07.824 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:22:07.824 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:22:07.824 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:22:07.824 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:22:07.824 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:22:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:07 smithi177 bash[68594]: cluster 2024-09-13T15:22:07.320821+0000 mgr.y (mgr.26252) 1685 : cluster [DBG] pgmap v2333: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:07.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:07 smithi177 bash[68594]: cluster 2024-09-13T15:22:07.320821+0000 mgr.y (mgr.26252) 1685 : cluster [DBG] pgmap v2333: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:08.405 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T15:22:08.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:08 smithi099 bash[84576]: audit 2024-09-13T15:22:08.047857+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.99:0/3982132379' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:08 smithi099 bash[84576]: audit 2024-09-13T15:22:08.047857+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.99:0/3982132379' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:08 smithi099 bash[84576]: audit 2024-09-13T15:22:08.048588+0000 mon.a (mon.0) 2889 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:08 smithi099 bash[84576]: audit 2024-09-13T15:22:08.048588+0000 mon.a (mon.0) 2889 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:08 smithi099 bash[87147]: audit 2024-09-13T15:22:08.047857+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.99:0/3982132379' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:08 smithi099 bash[87147]: audit 2024-09-13T15:22:08.047857+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.99:0/3982132379' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:08 smithi099 bash[87147]: audit 2024-09-13T15:22:08.048588+0000 mon.a (mon.0) 2889 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:08 smithi099 bash[87147]: audit 2024-09-13T15:22:08.048588+0000 mon.a (mon.0) 2889 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:08 smithi177 bash[68594]: audit 2024-09-13T15:22:08.047857+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.99:0/3982132379' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:08 smithi177 bash[68594]: audit 2024-09-13T15:22:08.047857+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.99:0/3982132379' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:08 smithi177 bash[68594]: audit 2024-09-13T15:22:08.048588+0000 mon.a (mon.0) 2889 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:08.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:08 smithi177 bash[68594]: audit 2024-09-13T15:22:08.048588+0000 mon.a (mon.0) 2889 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:09.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T15:22:09.659 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[87147]: audit 2024-09-13T15:22:08.395374+0000 mon.a (mon.0) 2890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:09.659 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[87147]: audit 2024-09-13T15:22:08.395374+0000 mon.a (mon.0) 2890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:09.660 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[87147]: cluster 2024-09-13T15:22:08.401663+0000 mon.a (mon.0) 2891 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2024-09-13T15:22:09.660 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[87147]: cluster 2024-09-13T15:22:08.401663+0000 mon.a (mon.0) 2891 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2024-09-13T15:22:09.660 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[87147]: cluster 2024-09-13T15:22:09.321671+0000 mgr.y (mgr.26252) 1686 : cluster [DBG] pgmap v2335: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:09.660 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[87147]: cluster 2024-09-13T15:22:09.321671+0000 mgr.y (mgr.26252) 1686 : cluster [DBG] pgmap v2335: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:09.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[84576]: audit 2024-09-13T15:22:08.395374+0000 mon.a (mon.0) 2890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:09.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[84576]: audit 2024-09-13T15:22:08.395374+0000 mon.a (mon.0) 2890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:09.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[84576]: cluster 2024-09-13T15:22:08.401663+0000 mon.a (mon.0) 2891 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2024-09-13T15:22:09.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[84576]: cluster 2024-09-13T15:22:08.401663+0000 mon.a (mon.0) 2891 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2024-09-13T15:22:09.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[84576]: cluster 2024-09-13T15:22:09.321671+0000 mgr.y (mgr.26252) 1686 : cluster [DBG] pgmap v2335: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:09.661 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:09 smithi099 bash[84576]: cluster 2024-09-13T15:22:09.321671+0000 mgr.y (mgr.26252) 1686 : cluster [DBG] pgmap v2335: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:09.661 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 1 1 2024-09-13T15:22:09.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:09 smithi177 bash[68594]: audit 2024-09-13T15:22:08.395374+0000 mon.a (mon.0) 2890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:09.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:09 smithi177 bash[68594]: audit 2024-09-13T15:22:08.395374+0000 mon.a (mon.0) 2890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:09.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:09 smithi177 bash[68594]: cluster 2024-09-13T15:22:08.401663+0000 mon.a (mon.0) 2891 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2024-09-13T15:22:09.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:09 smithi177 bash[68594]: cluster 2024-09-13T15:22:08.401663+0000 mon.a (mon.0) 2891 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2024-09-13T15:22:09.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:09 smithi177 bash[68594]: cluster 2024-09-13T15:22:09.321671+0000 mgr.y (mgr.26252) 1686 : cluster [DBG] pgmap v2335: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:09.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:09 smithi177 bash[68594]: cluster 2024-09-13T15:22:09.321671+0000 mgr.y (mgr.26252) 1686 : cluster [DBG] pgmap v2335: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:10.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:22:10 smithi177 bash[61439]: ts=2024-09-13T15:22:10.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:22:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:11 smithi099 bash[84576]: cluster 2024-09-13T15:22:11.322536+0000 mgr.y (mgr.26252) 1687 : cluster [DBG] pgmap v2336: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:11.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:11 smithi099 bash[84576]: cluster 2024-09-13T15:22:11.322536+0000 mgr.y (mgr.26252) 1687 : cluster [DBG] pgmap v2336: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:11 smithi099 bash[87147]: cluster 2024-09-13T15:22:11.322536+0000 mgr.y (mgr.26252) 1687 : cluster [DBG] pgmap v2336: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:11 smithi099 bash[87147]: cluster 2024-09-13T15:22:11.322536+0000 mgr.y (mgr.26252) 1687 : cluster [DBG] pgmap v2336: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:11.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:11 smithi177 bash[68594]: cluster 2024-09-13T15:22:11.322536+0000 mgr.y (mgr.26252) 1687 : cluster [DBG] pgmap v2336: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:11.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:11 smithi177 bash[68594]: cluster 2024-09-13T15:22:11.322536+0000 mgr.y (mgr.26252) 1687 : cluster [DBG] pgmap v2336: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:13 smithi099 bash[84576]: cluster 2024-09-13T15:22:13.323298+0000 mgr.y (mgr.26252) 1688 : cluster [DBG] pgmap v2337: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:13.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:13 smithi099 bash[84576]: cluster 2024-09-13T15:22:13.323298+0000 mgr.y (mgr.26252) 1688 : cluster [DBG] pgmap v2337: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:13.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:13 smithi099 bash[87147]: cluster 2024-09-13T15:22:13.323298+0000 mgr.y (mgr.26252) 1688 : cluster [DBG] pgmap v2337: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:13.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:13 smithi099 bash[87147]: cluster 2024-09-13T15:22:13.323298+0000 mgr.y (mgr.26252) 1688 : cluster [DBG] pgmap v2337: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:13.837 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:22:13.837 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:22:13.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:13 smithi177 bash[68594]: cluster 2024-09-13T15:22:13.323298+0000 mgr.y (mgr.26252) 1688 : cluster [DBG] pgmap v2337: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:13.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:13 smithi177 bash[68594]: cluster 2024-09-13T15:22:13.323298+0000 mgr.y (mgr.26252) 1688 : cluster [DBG] pgmap v2337: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:15.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:15 smithi099 bash[84576]: cluster 2024-09-13T15:22:15.324094+0000 mgr.y (mgr.26252) 1689 : cluster [DBG] pgmap v2338: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:15.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:15 smithi099 bash[84576]: cluster 2024-09-13T15:22:15.324094+0000 mgr.y (mgr.26252) 1689 : cluster [DBG] pgmap v2338: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:15.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:15 smithi099 bash[87147]: cluster 2024-09-13T15:22:15.324094+0000 mgr.y (mgr.26252) 1689 : cluster [DBG] pgmap v2338: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:15.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:15 smithi099 bash[87147]: cluster 2024-09-13T15:22:15.324094+0000 mgr.y (mgr.26252) 1689 : cluster [DBG] pgmap v2338: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:15.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:15 smithi177 bash[68594]: cluster 2024-09-13T15:22:15.324094+0000 mgr.y (mgr.26252) 1689 : cluster [DBG] pgmap v2338: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:15.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:15 smithi177 bash[68594]: cluster 2024-09-13T15:22:15.324094+0000 mgr.y (mgr.26252) 1689 : cluster [DBG] pgmap v2338: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:17.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:22:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:22:17] "GET /metrics HTTP/1.1" 200 34331 "" "Prometheus/2.51.0" 2024-09-13T15:22:17.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:17 smithi099 bash[84576]: cluster 2024-09-13T15:22:17.325201+0000 mgr.y (mgr.26252) 1690 : cluster [DBG] pgmap v2339: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:17.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:17 smithi099 bash[84576]: cluster 2024-09-13T15:22:17.325201+0000 mgr.y (mgr.26252) 1690 : cluster [DBG] pgmap v2339: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:17.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:17 smithi099 bash[87147]: cluster 2024-09-13T15:22:17.325201+0000 mgr.y (mgr.26252) 1690 : cluster [DBG] pgmap v2339: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:17.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:17 smithi099 bash[87147]: cluster 2024-09-13T15:22:17.325201+0000 mgr.y (mgr.26252) 1690 : cluster [DBG] pgmap v2339: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:17 smithi177 bash[68594]: cluster 2024-09-13T15:22:17.325201+0000 mgr.y (mgr.26252) 1690 : cluster [DBG] pgmap v2339: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:17.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:17 smithi177 bash[68594]: cluster 2024-09-13T15:22:17.325201+0000 mgr.y (mgr.26252) 1690 : cluster [DBG] pgmap v2339: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:18.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:18 smithi099 bash[84576]: audit 2024-09-13T15:22:18.328565+0000 mon.a (mon.0) 2892 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:18.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:18 smithi099 bash[84576]: audit 2024-09-13T15:22:18.328565+0000 mon.a (mon.0) 2892 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:18.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:18 smithi099 bash[87147]: audit 2024-09-13T15:22:18.328565+0000 mon.a (mon.0) 2892 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:18.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:18 smithi099 bash[87147]: audit 2024-09-13T15:22:18.328565+0000 mon.a (mon.0) 2892 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:18 smithi177 bash[68594]: audit 2024-09-13T15:22:18.328565+0000 mon.a (mon.0) 2892 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:18.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:18 smithi177 bash[68594]: audit 2024-09-13T15:22:18.328565+0000 mon.a (mon.0) 2892 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:19.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:19 smithi099 bash[84576]: cluster 2024-09-13T15:22:19.326096+0000 mgr.y (mgr.26252) 1691 : cluster [DBG] pgmap v2340: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:19.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:19 smithi099 bash[84576]: cluster 2024-09-13T15:22:19.326096+0000 mgr.y (mgr.26252) 1691 : cluster [DBG] pgmap v2340: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:19.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:19 smithi099 bash[87147]: cluster 2024-09-13T15:22:19.326096+0000 mgr.y (mgr.26252) 1691 : cluster [DBG] pgmap v2340: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:19.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:19 smithi099 bash[87147]: cluster 2024-09-13T15:22:19.326096+0000 mgr.y (mgr.26252) 1691 : cluster [DBG] pgmap v2340: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:19.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:19 smithi177 bash[68594]: cluster 2024-09-13T15:22:19.326096+0000 mgr.y (mgr.26252) 1691 : cluster [DBG] pgmap v2340: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:19.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:19 smithi177 bash[68594]: cluster 2024-09-13T15:22:19.326096+0000 mgr.y (mgr.26252) 1691 : cluster [DBG] pgmap v2340: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:20.543 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:27971 46.0 46:07d2c86f:::benchmark_data_smithi099_233884_object27970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:15:32.115535+0000", 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 408.42402797300002, 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.86534235800000003, 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.544 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 27971 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.115535+0000", 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.115535+0000", 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.115536+0000", 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.083e-06 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.115553+0000", 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7249999999999999e-05 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.546 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.115563+0000", 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8279999999999998e-06 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.115568+0000", 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.254e-06 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.116089+0000", 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00052074499999999998 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.116178+0000", 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9184000000000005e-05 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.116256+0000", 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7678999999999999e-05 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117486+0000", 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012298319999999999 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.119023+0000", 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001537411 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.980665+0000", 2024-09-13T15:22:20.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.86164204300000002 2024-09-13T15:22:20.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.550 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.550 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.980784+0000", 2024-09-13T15:22:20.550 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011926699999999999 2024-09-13T15:22:20.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.550 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.550 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.980877+0000", 2024-09-13T15:22:20.550 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2781999999999994e-05 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:27972 46.7 46:ed3fc292:::benchmark_data_smithi099_233884_object27971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:15:32.117214+0000", 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 408.42234925899999, 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.86044770000000004, 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.551 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 27972 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117214+0000", 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117214+0000", 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.552 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117214+0000", 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5700000000000001e-07 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117234+0000", 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9493000000000001e-05 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.553 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117243+0000", 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5249999999999994e-06 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117250+0000", 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.63e-06 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117298+0000", 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7893999999999999e-05 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.554 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117421+0000", 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012296099999999999 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117607+0000", 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000186002 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.119790+0000", 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021831770000000001 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.119826+0000", 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5848e-05 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.977491+0000", 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85766533700000003 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.977586+0000", 2024-09-13T15:22:20.556 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4326000000000005e-05 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.977661+0000", 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5649999999999996e-05 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:27974 46.7 46:ea488627:::benchmark_data_smithi099_233884_object27973:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:15:32.117416+0000", 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 408.42214668399998, 2024-09-13T15:22:20.557 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.86053569399999996, 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 27974 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117416+0000", 2024-09-13T15:22:20.558 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117416+0000", 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117417+0000", 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7199999999999998e-07 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.559 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117439+0000", 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2345999999999999e-05 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117448+0000", 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8279999999999992e-06 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.117454+0000", 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0929999999999999e-06 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.560 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.118764+0000", 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013097429999999999 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.118848+0000", 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3721999999999996e-05 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.118932+0000", 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4386000000000001e-05 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.561 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.119899+0000", 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00096653799999999997 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.120475+0000", 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00057678799999999995 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.977798+0000", 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85732309600000001 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.977875+0000", 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6207999999999999e-05 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.977952+0000", 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7174000000000003e-05 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.563 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:27979 46.7 46:e5ee8cff:::benchmark_data_smithi099_233884_object27978:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:15:32.121266+0000", 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 408.41829699900001, 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85698749200000002, 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 27979 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.564 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.121266+0000", 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.121266+0000", 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.565 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.121267+0000", 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.159e-06 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.121288+0000", 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0763000000000001e-05 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.121299+0000", 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1447999999999999e-05 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.566 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.121306+0000", 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6760000000000001e-06 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.121348+0000", 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2326000000000001e-05 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.121467+0000", 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011891399999999999 2024-09-13T15:22:20.567 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.121588+0000", 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121253 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.122672+0000", 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010834589999999999 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.568 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.123186+0000", 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00051449000000000004 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.978079+0000", 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85489250800000005 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.978192+0000", 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113131 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.978253+0000", 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1365000000000006e-05 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:27988 46.7 46:ef7c0437:::benchmark_data_smithi099_233884_object27987:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:15:32.129703+0000", 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 408.40986012100001, 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.84876021499999998, 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.570 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 27988 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.129703+0000", 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.571 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.129703+0000", 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.129705+0000", 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9860000000000001e-06 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.129725+0000", 2024-09-13T15:22:20.572 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9953999999999999e-05 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.129739+0000", 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4725999999999999e-05 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.129746+0000", 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0500000000000003e-06 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.573 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.129797+0000", 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0299000000000001e-05 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.129931+0000", 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000133898 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.130026+0000", 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5508999999999996e-05 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.574 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.131415+0000", 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013884959999999999 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.131823+0000", 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00040830199999999998 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.978323+0000", 2024-09-13T15:22:20.575 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.84649982000000001 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.978404+0000", 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1624000000000003e-05 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:15:32.978463+0000", 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8551000000000002e-05 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.576 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:106878 46.13 46.d7064053 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:56.042854+0000", 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 324.496708463, 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62093835799999997, 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 106878 2024-09-13T15:22:20.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.042854+0000", 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.042854+0000", 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.042856+0000", 2024-09-13T15:22:20.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.561e-06 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.042861+0000", 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3109999999999996e-06 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.042875+0000", 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3594000000000001e-05 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.042885+0000", 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6290000000000005e-06 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.042942+0000", 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7726999999999998e-05 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.042950+0000", 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7610000000000007e-06 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.663783+0000", 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62083301199999996 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.663793+0000", 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7629999999999995e-06 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.581 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:106884 46.13 46.d8628ab3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:56.049685+0000", 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 324.48987776299998, 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.61441349099999998, 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 106884 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.582 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.049685+0000", 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.049685+0000", 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.049687+0000", 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4839999999999999e-06 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.583 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.049693+0000", 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1970000000000001e-06 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.049712+0000", 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9647999999999999e-05 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.049721+0000", 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9460000000000004e-06 2024-09-13T15:22:20.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.049784+0000", 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2983000000000003e-05 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.049793+0000", 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4950000000000008e-06 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.664090+0000", 2024-09-13T15:22:20.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.61429673699999998 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.664099+0000", 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0010000000000007e-06 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:106897 46.13 46.714ca413 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:56.066594+0000", 2024-09-13T15:22:20.586 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 324.47296860500001, 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59753202299999997, 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 106897 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066594+0000", 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066594+0000", 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066596+0000", 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.699e-06 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066602+0000", 2024-09-13T15:22:20.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8900000000000004e-06 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066614+0000", 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2563e-05 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066623+0000", 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7080000000000003e-06 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066681+0000", 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7411000000000003e-05 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066689+0000", 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0539999999999995e-06 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.664119+0000", 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59743068399999999 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.664126+0000", 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0140000000000001e-06 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:106902 46.13 46.3f6ee893 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:56.066713+0000", 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 324.47284965900002, 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59743845299999998, 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.591 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 106902 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066713+0000", 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.592 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066713+0000", 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066714+0000", 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.22e-07 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066715+0000", 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9199999999999998e-06 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066721+0000", 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0629999999999996e-06 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066725+0000", 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7849999999999999e-06 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066798+0000", 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2959000000000004e-05 2024-09-13T15:22:20.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.066804+0000", 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0920000000000003e-06 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.664145+0000", 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59734052699999995 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.664152+0000", 2024-09-13T15:22:20.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7850000000000003e-06 2024-09-13T15:22:20.596 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.596 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.596 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.596 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:106927 46.13 46.fd6344f3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-13T15:22:20.596 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:56.092020+0000", 2024-09-13T15:22:20.596 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 324.44754237900003, 2024-09-13T15:22:20.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.57215612699999996, 2024-09-13T15:22:20.596 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 106927 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.092020+0000", 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.092020+0000", 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.092022+0000", 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.232e-06 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.092027+0000", 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9559999999999996e-06 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.092038+0000", 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1542999999999999e-05 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.092045+0000", 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8349999999999998e-06 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.092100+0000", 2024-09-13T15:22:20.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5417999999999997e-05 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.092108+0000", 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3250000000000002e-06 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.664170+0000", 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.57206201400000001 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.664177+0000", 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8040000000000004e-06 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:107330 46.0 46:04c46fa6:::benchmark_data_smithi099_233884_object4917:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:57.912095+0000", 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 322.62746797300002, 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70563509499999999, 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.601 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 107330 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.912095+0000", 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.912095+0000", 2024-09-13T15:22:20.602 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.912096+0000", 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0419999999999999e-06 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.912100+0000", 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3189999999999996e-06 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.912112+0000", 2024-09-13T15:22:20.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1263999999999999e-05 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.912118+0000", 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8589999999999999e-06 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.912167+0000", 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8572999999999997e-05 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.912272+0000", 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010468599999999999 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.912366+0000", 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4853999999999997e-05 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.915443+0000", 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030769090000000001 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.915870+0000", 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000426413 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.617454+0000", 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70158408900000002 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.617626+0000", 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017212099999999999 2024-09-13T15:22:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.617730+0000", 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000103966 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:107353 46.0 46:04ab5e02:::benchmark_data_smithi099_233884_object4940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:57.936534+0000", 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 322.60302922599999, 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.68160973999999996, 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 107353 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936534+0000", 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936534+0000", 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936535+0000", 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.778e-06 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936541+0000", 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8590000000000001e-06 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936554+0000", 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2574e-05 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936562+0000", 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7689999999999996e-06 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936619+0000", 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7222000000000002e-05 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936710+0000", 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1612000000000003e-05 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936823+0000", 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011246 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.616749+0000", 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67992619799999998 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.617841+0000", 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010915930000000001 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.617971+0000", 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013036699999999999 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.618068+0000", 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6773000000000003e-05 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.618143+0000", 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5535000000000001e-05 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.46109.0:107350 46.13 e1120/1119 46:cf643666:::benchmark_data_smithi099_233884_object4937:head v 1120'6145, mlcod=1103'264)", 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:57.936717+0000", 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 322.60284581100001, 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.69674581400000002, 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936717+0000", 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936717+0000", 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936718+0000", 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4300000000000002e-07 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936724+0000", 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3450000000000002e-06 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936733+0000", 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2399999999999996e-06 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936740+0000", 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4500000000000001e-06 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936783+0000", 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2930999999999999e-05 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.936806+0000", 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3039000000000001e-05 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.633399+0000", 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.69659358599999999 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.633463+0000", 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3380000000000006e-05 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.46109.0:107355 46.b e1120/1099 46:d7b67437:::benchmark_data_smithi099_233884_object4942:head v 1120'6757, mlcod=1120'6753)", 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:57.938261+0000", 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 322.601301476, 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.69525058799999995, 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.938261+0000", 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.938261+0000", 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.938262+0000", 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6400000000000001e-07 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.938268+0000", 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2970000000000003e-06 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.616760+0000", 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67849241599999999 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.616779+0000", 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8814e-05 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.616949+0000", 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016986200000000001 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.616999+0000", 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0832999999999999e-05 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.633471+0000", 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016471237999999999 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.633512+0000", 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1264000000000003e-05 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.46109.0:107356 46.4 e1120/1108 46:27540f99:::benchmark_data_smithi099_233884_object4943:head v 1120'6239, mlcod=1120'6237)", 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:57.940542+0000", 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 322.59902047399999, 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.68892390100000001, 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.940542+0000", 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.940542+0000", 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.940543+0000", 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4000000000000001e-07 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:57.940551+0000", 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3310000000000004e-06 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.616802+0000", 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67625064700000004 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.616822+0000", 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0080000000000001e-05 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.616952+0000", 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012948699999999999 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.617034+0000", 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2546999999999994e-05 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.629372+0000", 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012337656000000001 2024-09-13T15:22:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:58.629466+0000", 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4413000000000006e-05 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:165249 46.0 46:04fe4bc2:::benchmark_data_smithi099_233884_object62836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:18:11.499348+0000", 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 249.04021451899999, 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.57179643999999996, 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 165249 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499348+0000", 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499348+0000", 2024-09-13T15:22:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499350+0000", 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4389999999999999e-06 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499354+0000", 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9369999999999997e-06 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499364+0000", 2024-09-13T15:22:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0275000000000001e-05 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499370+0000", 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6740000000000002e-06 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499423+0000", 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3111e-05 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499552+0000", 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012935799999999999 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499669+0000", 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116484 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.509979+0000", 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010310477 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.510034+0000", 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5127000000000003e-05 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.070904+0000", 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.56087016000000001 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.071037+0000", 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013272300000000001 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.071145+0000", 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000107675 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:165252 46.0 46:005a58e2:::benchmark_data_smithi099_233884_object62839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:18:11.499416+0000", 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 249.04014733299999, 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.57200617300000001, 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 165252 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499416+0000", 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499416+0000", 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499416+0000", 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.58e-07 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499418+0000", 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.035e-06 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499424+0000", 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8919999999999997e-06 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.499429+0000", 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7679999999999999e-06 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.503578+0000", 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041489910000000003 2024-09-13T15:22:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.503682+0000", 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010452 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.503779+0000", 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6898999999999995e-05 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.510016+0000", 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062370960000000001 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.510124+0000", 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000107675 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.071233+0000", 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.561109372 2024-09-13T15:22:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.071339+0000", 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000105849 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.071422+0000", 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2618000000000001e-05 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:165261 46.e 46:79240bd3:::benchmark_data_smithi099_233884_object62848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:18:11.502944+0000", 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 249.03661868399999, 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.57659886800000004, 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 165261 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.502944+0000", 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.502944+0000", 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.502945+0000", 2024-09-13T15:22:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6300000000000005e-07 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.502947+0000", 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.525e-06 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.502954+0000", 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3559999999999998e-06 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.502959+0000", 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9049999999999996e-06 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.505651+0000", 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026921409999999999 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.505763+0000", 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112576 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.505874+0000", 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110641 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.079126+0000", 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.57325196300000003 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.079147+0000", 2024-09-13T15:22:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0724999999999999e-05 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.079198+0000", 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1607999999999998e-05 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.079468+0000", 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026931500000000001 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.079543+0000", 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5549999999999993e-05 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:165266 46.7 46:e3bc7dc7:::benchmark_data_smithi099_233884_object62853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:18:11.509982+0000", 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 249.029580955, 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58369462699999997, 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 165266 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.509982+0000", 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.509982+0000", 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.509983+0000", 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.226e-06 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.509988+0000", 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6439999999999999e-06 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.509999+0000", 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1569000000000001e-05 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.510007+0000", 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4279999999999999e-06 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.510056+0000", 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9489e-05 2024-09-13T15:22:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.510170+0000", 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000114182 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.068665+0000", 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.55849431500000002 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.086119+0000", 2024-09-13T15:22:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017454561 2024-09-13T15:22:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:22:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.090032+0000", 2024-09-13T15:22:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039124520000000003 2024-09-13T15:22:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:22:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.093583+0000", 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003550812 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.093629+0000", 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6430000000000001e-05 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.093677+0000", 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7518999999999997e-05 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.46109.0:165261 46.e e1130/1099 ondisk, result = 0)", 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:18:11.511764+0000", 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 249.027799036, 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.56738400300000003, 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:22:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.511764+0000", 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.511764+0000", 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.511765+0000", 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.23e-07 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:11.511769+0000", 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3900000000000003e-06 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.068506+0000", 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.55673717099999998 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.068539+0000", 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2625e-05 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.079103+0000", 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010563944 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.079116+0000", 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2847000000000001e-05 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:18:12.079148+0000", 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2203000000000003e-05 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:22:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:22:20.656 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:22:20.656 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:22:20 smithi177 bash[61439]: ts=2024-09-13T15:22:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:22:20.745 INFO:teuthology.orchestra.run.smithi099.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-09-13T15:22:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:20 smithi099 bash[84576]: audit 2024-09-13T15:22:20.681446+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.99:0/3255605974' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:20 smithi099 bash[84576]: audit 2024-09-13T15:22:20.681446+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.99:0/3255605974' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:20 smithi099 bash[84576]: audit 2024-09-13T15:22:20.682101+0000 mon.a (mon.0) 2893 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:20 smithi099 bash[84576]: audit 2024-09-13T15:22:20.682101+0000 mon.a (mon.0) 2893 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:20 smithi099 bash[87147]: audit 2024-09-13T15:22:20.681446+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.99:0/3255605974' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:20 smithi099 bash[87147]: audit 2024-09-13T15:22:20.681446+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.99:0/3255605974' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:20 smithi099 bash[87147]: audit 2024-09-13T15:22:20.682101+0000 mon.a (mon.0) 2893 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:20 smithi099 bash[87147]: audit 2024-09-13T15:22:20.682101+0000 mon.a (mon.0) 2893 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:20 smithi177 bash[68594]: audit 2024-09-13T15:22:20.681446+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.99:0/3255605974' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:20 smithi177 bash[68594]: audit 2024-09-13T15:22:20.681446+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.99:0/3255605974' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:20 smithi177 bash[68594]: audit 2024-09-13T15:22:20.682101+0000 mon.a (mon.0) 2893 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:20 smithi177 bash[68594]: audit 2024-09-13T15:22:20.682101+0000 mon.a (mon.0) 2893 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:22:21.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[84576]: audit 2024-09-13T15:22:20.735168+0000 mon.a (mon.0) 2894 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:21.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[84576]: audit 2024-09-13T15:22:20.735168+0000 mon.a (mon.0) 2894 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:21.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[84576]: cluster 2024-09-13T15:22:20.743807+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2024-09-13T15:22:21.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[84576]: cluster 2024-09-13T15:22:20.743807+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2024-09-13T15:22:21.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[84576]: cluster 2024-09-13T15:22:21.326710+0000 mgr.y (mgr.26252) 1692 : cluster [DBG] pgmap v2342: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:21.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[84576]: cluster 2024-09-13T15:22:21.326710+0000 mgr.y (mgr.26252) 1692 : cluster [DBG] pgmap v2342: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:21.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[87147]: audit 2024-09-13T15:22:20.735168+0000 mon.a (mon.0) 2894 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:21.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[87147]: audit 2024-09-13T15:22:20.735168+0000 mon.a (mon.0) 2894 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:21.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[87147]: cluster 2024-09-13T15:22:20.743807+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2024-09-13T15:22:21.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[87147]: cluster 2024-09-13T15:22:20.743807+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2024-09-13T15:22:21.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[87147]: cluster 2024-09-13T15:22:21.326710+0000 mgr.y (mgr.26252) 1692 : cluster [DBG] pgmap v2342: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:21.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:21 smithi099 bash[87147]: cluster 2024-09-13T15:22:21.326710+0000 mgr.y (mgr.26252) 1692 : cluster [DBG] pgmap v2342: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:22.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:21 smithi177 bash[68594]: audit 2024-09-13T15:22:20.735168+0000 mon.a (mon.0) 2894 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:22.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:21 smithi177 bash[68594]: audit 2024-09-13T15:22:20.735168+0000 mon.a (mon.0) 2894 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:22:22.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:21 smithi177 bash[68594]: cluster 2024-09-13T15:22:20.743807+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2024-09-13T15:22:22.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:21 smithi177 bash[68594]: cluster 2024-09-13T15:22:20.743807+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2024-09-13T15:22:22.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:21 smithi177 bash[68594]: cluster 2024-09-13T15:22:21.326710+0000 mgr.y (mgr.26252) 1692 : cluster [DBG] pgmap v2342: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:22.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:21 smithi177 bash[68594]: cluster 2024-09-13T15:22:21.326710+0000 mgr.y (mgr.26252) 1692 : cluster [DBG] pgmap v2342: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:22.246 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T15:22:22.286 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T15:22:23.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:23 smithi099 bash[84576]: cluster 2024-09-13T15:22:23.327737+0000 mgr.y (mgr.26252) 1693 : cluster [DBG] pgmap v2343: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:23.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:23 smithi099 bash[84576]: cluster 2024-09-13T15:22:23.327737+0000 mgr.y (mgr.26252) 1693 : cluster [DBG] pgmap v2343: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:23.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:23 smithi099 bash[87147]: cluster 2024-09-13T15:22:23.327737+0000 mgr.y (mgr.26252) 1693 : cluster [DBG] pgmap v2343: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:23.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:23 smithi099 bash[87147]: cluster 2024-09-13T15:22:23.327737+0000 mgr.y (mgr.26252) 1693 : cluster [DBG] pgmap v2343: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:23.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:23 smithi177 bash[68594]: cluster 2024-09-13T15:22:23.327737+0000 mgr.y (mgr.26252) 1693 : cluster [DBG] pgmap v2343: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:23.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:23 smithi177 bash[68594]: cluster 2024-09-13T15:22:23.327737+0000 mgr.y (mgr.26252) 1693 : cluster [DBG] pgmap v2343: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:26.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:26 smithi099 bash[84576]: cluster 2024-09-13T15:22:25.328681+0000 mgr.y (mgr.26252) 1694 : cluster [DBG] pgmap v2344: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:26.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:26 smithi099 bash[84576]: cluster 2024-09-13T15:22:25.328681+0000 mgr.y (mgr.26252) 1694 : cluster [DBG] pgmap v2344: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:26.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:26 smithi099 bash[87147]: cluster 2024-09-13T15:22:25.328681+0000 mgr.y (mgr.26252) 1694 : cluster [DBG] pgmap v2344: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:26.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:26 smithi099 bash[87147]: cluster 2024-09-13T15:22:25.328681+0000 mgr.y (mgr.26252) 1694 : cluster [DBG] pgmap v2344: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:26.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:26 smithi177 bash[68594]: cluster 2024-09-13T15:22:25.328681+0000 mgr.y (mgr.26252) 1694 : cluster [DBG] pgmap v2344: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:26.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:26 smithi177 bash[68594]: cluster 2024-09-13T15:22:25.328681+0000 mgr.y (mgr.26252) 1694 : cluster [DBG] pgmap v2344: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:27.031 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:22:27.076 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:22:27.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:22:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:22:27] "GET /metrics HTTP/1.1" 200 34329 "" "Prometheus/2.51.0" 2024-09-13T15:22:28.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:28 smithi177 bash[68594]: cluster 2024-09-13T15:22:27.329748+0000 mgr.y (mgr.26252) 1695 : cluster [DBG] pgmap v2345: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:28.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:28 smithi177 bash[68594]: cluster 2024-09-13T15:22:27.329748+0000 mgr.y (mgr.26252) 1695 : cluster [DBG] pgmap v2345: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:29.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:28 smithi099 bash[87147]: cluster 2024-09-13T15:22:27.329748+0000 mgr.y (mgr.26252) 1695 : cluster [DBG] pgmap v2345: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:29.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:28 smithi099 bash[87147]: cluster 2024-09-13T15:22:27.329748+0000 mgr.y (mgr.26252) 1695 : cluster [DBG] pgmap v2345: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:29.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:28 smithi099 bash[84576]: cluster 2024-09-13T15:22:27.329748+0000 mgr.y (mgr.26252) 1695 : cluster [DBG] pgmap v2345: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:29.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:28 smithi099 bash[84576]: cluster 2024-09-13T15:22:27.329748+0000 mgr.y (mgr.26252) 1695 : cluster [DBG] pgmap v2345: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:30.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:22:30 smithi177 bash[61439]: ts=2024-09-13T15:22:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:22:30.836 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:22:30.836 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:22:30.836 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2024-09-13T15:22:30.836 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:22:31.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:30 smithi099 bash[84576]: cluster 2024-09-13T15:22:29.330835+0000 mgr.y (mgr.26252) 1696 : cluster [DBG] pgmap v2346: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:31.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:30 smithi099 bash[84576]: cluster 2024-09-13T15:22:29.330835+0000 mgr.y (mgr.26252) 1696 : cluster [DBG] pgmap v2346: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:30 smithi099 bash[87147]: cluster 2024-09-13T15:22:29.330835+0000 mgr.y (mgr.26252) 1696 : cluster [DBG] pgmap v2346: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:31.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:30 smithi099 bash[87147]: cluster 2024-09-13T15:22:29.330835+0000 mgr.y (mgr.26252) 1696 : cluster [DBG] pgmap v2346: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:31.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:30 smithi177 bash[68594]: cluster 2024-09-13T15:22:29.330835+0000 mgr.y (mgr.26252) 1696 : cluster [DBG] pgmap v2346: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:31.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:30 smithi177 bash[68594]: cluster 2024-09-13T15:22:29.330835+0000 mgr.y (mgr.26252) 1696 : cluster [DBG] pgmap v2346: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:31.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T15:22:31.754 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T15:22:32.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:31 smithi099 bash[84576]: audit 2024-09-13T15:22:31.498659+0000 mon.a (mon.0) 2896 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:32.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:31 smithi099 bash[84576]: audit 2024-09-13T15:22:31.498659+0000 mon.a (mon.0) 2896 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:32.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:31 smithi099 bash[87147]: audit 2024-09-13T15:22:31.498659+0000 mon.a (mon.0) 2896 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:32.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:31 smithi099 bash[87147]: audit 2024-09-13T15:22:31.498659+0000 mon.a (mon.0) 2896 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:32.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:31 smithi177 bash[68594]: audit 2024-09-13T15:22:31.498659+0000 mon.a (mon.0) 2896 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:32.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:31 smithi177 bash[68594]: audit 2024-09-13T15:22:31.498659+0000 mon.a (mon.0) 2896 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:32.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 2 1 2024-09-13T15:22:33.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[84576]: cluster 2024-09-13T15:22:31.331330+0000 mgr.y (mgr.26252) 1697 : cluster [DBG] pgmap v2347: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:33.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[84576]: cluster 2024-09-13T15:22:31.331330+0000 mgr.y (mgr.26252) 1697 : cluster [DBG] pgmap v2347: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:33.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[84576]: audit 2024-09-13T15:22:31.753612+0000 mon.a (mon.0) 2897 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:33.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[84576]: audit 2024-09-13T15:22:31.753612+0000 mon.a (mon.0) 2897 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:33.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[84576]: cluster 2024-09-13T15:22:31.761200+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2024-09-13T15:22:33.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[84576]: cluster 2024-09-13T15:22:31.761200+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2024-09-13T15:22:33.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[87147]: cluster 2024-09-13T15:22:31.331330+0000 mgr.y (mgr.26252) 1697 : cluster [DBG] pgmap v2347: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:33.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[87147]: cluster 2024-09-13T15:22:31.331330+0000 mgr.y (mgr.26252) 1697 : cluster [DBG] pgmap v2347: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:33.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[87147]: audit 2024-09-13T15:22:31.753612+0000 mon.a (mon.0) 2897 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:33.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[87147]: audit 2024-09-13T15:22:31.753612+0000 mon.a (mon.0) 2897 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:33.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[87147]: cluster 2024-09-13T15:22:31.761200+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2024-09-13T15:22:33.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:32 smithi099 bash[87147]: cluster 2024-09-13T15:22:31.761200+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2024-09-13T15:22:33.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:32 smithi177 bash[68594]: cluster 2024-09-13T15:22:31.331330+0000 mgr.y (mgr.26252) 1697 : cluster [DBG] pgmap v2347: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:33.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:32 smithi177 bash[68594]: cluster 2024-09-13T15:22:31.331330+0000 mgr.y (mgr.26252) 1697 : cluster [DBG] pgmap v2347: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:33.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:32 smithi177 bash[68594]: audit 2024-09-13T15:22:31.753612+0000 mon.a (mon.0) 2897 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:33.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:32 smithi177 bash[68594]: audit 2024-09-13T15:22:31.753612+0000 mon.a (mon.0) 2897 : audit [INF] from='client.? 172.21.15.99:0/140823954' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:33.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:32 smithi177 bash[68594]: cluster 2024-09-13T15:22:31.761200+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2024-09-13T15:22:33.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:32 smithi177 bash[68594]: cluster 2024-09-13T15:22:31.761200+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2024-09-13T15:22:34.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:33 smithi099 bash[84576]: audit 2024-09-13T15:22:33.328857+0000 mon.a (mon.0) 2899 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:34.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:33 smithi099 bash[84576]: audit 2024-09-13T15:22:33.328857+0000 mon.a (mon.0) 2899 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:34.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:33 smithi099 bash[87147]: audit 2024-09-13T15:22:33.328857+0000 mon.a (mon.0) 2899 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:34.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:33 smithi099 bash[87147]: audit 2024-09-13T15:22:33.328857+0000 mon.a (mon.0) 2899 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:34.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:33 smithi177 bash[68594]: audit 2024-09-13T15:22:33.328857+0000 mon.a (mon.0) 2899 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:34.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:33 smithi177 bash[68594]: audit 2024-09-13T15:22:33.328857+0000 mon.a (mon.0) 2899 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:35.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:34 smithi099 bash[84576]: cluster 2024-09-13T15:22:33.332050+0000 mgr.y (mgr.26252) 1698 : cluster [DBG] pgmap v2349: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:35.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:34 smithi099 bash[84576]: cluster 2024-09-13T15:22:33.332050+0000 mgr.y (mgr.26252) 1698 : cluster [DBG] pgmap v2349: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:35.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:34 smithi099 bash[87147]: cluster 2024-09-13T15:22:33.332050+0000 mgr.y (mgr.26252) 1698 : cluster [DBG] pgmap v2349: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:35.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:34 smithi099 bash[87147]: cluster 2024-09-13T15:22:33.332050+0000 mgr.y (mgr.26252) 1698 : cluster [DBG] pgmap v2349: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:35.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:34 smithi177 bash[68594]: cluster 2024-09-13T15:22:33.332050+0000 mgr.y (mgr.26252) 1698 : cluster [DBG] pgmap v2349: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:35.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:34 smithi177 bash[68594]: cluster 2024-09-13T15:22:33.332050+0000 mgr.y (mgr.26252) 1698 : cluster [DBG] pgmap v2349: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:36.468 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:22:36.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:36 smithi177 bash[68594]: cluster 2024-09-13T15:22:35.333209+0000 mgr.y (mgr.26252) 1699 : cluster [DBG] pgmap v2350: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:36.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:36 smithi177 bash[68594]: cluster 2024-09-13T15:22:35.333209+0000 mgr.y (mgr.26252) 1699 : cluster [DBG] pgmap v2350: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:37.051 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:36 smithi099 bash[84576]: cluster 2024-09-13T15:22:35.333209+0000 mgr.y (mgr.26252) 1699 : cluster [DBG] pgmap v2350: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:37.052 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:36 smithi099 bash[84576]: cluster 2024-09-13T15:22:35.333209+0000 mgr.y (mgr.26252) 1699 : cluster [DBG] pgmap v2350: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:37.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:36 smithi099 bash[87147]: cluster 2024-09-13T15:22:35.333209+0000 mgr.y (mgr.26252) 1699 : cluster [DBG] pgmap v2350: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:37.052 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:36 smithi099 bash[87147]: cluster 2024-09-13T15:22:35.333209+0000 mgr.y (mgr.26252) 1699 : cluster [DBG] pgmap v2350: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:37.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:22:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:22:37] "GET /metrics HTTP/1.1" 200 34332 "" "Prometheus/2.51.0" 2024-09-13T15:22:37.587 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:22:39.050 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:38 smithi177 bash[68594]: cluster 2024-09-13T15:22:37.334211+0000 mgr.y (mgr.26252) 1700 : cluster [DBG] pgmap v2351: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:39.050 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:38 smithi177 bash[68594]: cluster 2024-09-13T15:22:37.334211+0000 mgr.y (mgr.26252) 1700 : cluster [DBG] pgmap v2351: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:39.074 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:38 smithi099 bash[84576]: cluster 2024-09-13T15:22:37.334211+0000 mgr.y (mgr.26252) 1700 : cluster [DBG] pgmap v2351: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:39.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:38 smithi099 bash[84576]: cluster 2024-09-13T15:22:37.334211+0000 mgr.y (mgr.26252) 1700 : cluster [DBG] pgmap v2351: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:38 smithi099 bash[87147]: cluster 2024-09-13T15:22:37.334211+0000 mgr.y (mgr.26252) 1700 : cluster [DBG] pgmap v2351: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:39.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:38 smithi099 bash[87147]: cluster 2024-09-13T15:22:37.334211+0000 mgr.y (mgr.26252) 1700 : cluster [DBG] pgmap v2351: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:40.487 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:22:40.487 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:22:40.487 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:22:40.487 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:22:40.487 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:22:40.496 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:22:40 smithi177 bash[61439]: ts=2024-09-13T15:22:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:22:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:40 smithi099 bash[84576]: cluster 2024-09-13T15:22:39.335148+0000 mgr.y (mgr.26252) 1701 : cluster [DBG] pgmap v2352: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:40.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:40 smithi099 bash[84576]: cluster 2024-09-13T15:22:39.335148+0000 mgr.y (mgr.26252) 1701 : cluster [DBG] pgmap v2352: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:40 smithi099 bash[87147]: cluster 2024-09-13T15:22:39.335148+0000 mgr.y (mgr.26252) 1701 : cluster [DBG] pgmap v2352: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:40.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:40 smithi099 bash[87147]: cluster 2024-09-13T15:22:39.335148+0000 mgr.y (mgr.26252) 1701 : cluster [DBG] pgmap v2352: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:41.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:40 smithi177 bash[68594]: cluster 2024-09-13T15:22:39.335148+0000 mgr.y (mgr.26252) 1701 : cluster [DBG] pgmap v2352: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:41.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:40 smithi177 bash[68594]: cluster 2024-09-13T15:22:39.335148+0000 mgr.y (mgr.26252) 1701 : cluster [DBG] pgmap v2352: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:41.474 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:22:42.795 INFO:teuthology.orchestra.run.smithi099.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-09-13T15:22:43.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[84576]: cluster 2024-09-13T15:22:41.335869+0000 mgr.y (mgr.26252) 1702 : cluster [DBG] pgmap v2353: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:43.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[84576]: cluster 2024-09-13T15:22:41.335869+0000 mgr.y (mgr.26252) 1702 : cluster [DBG] pgmap v2353: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:43.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[84576]: audit 2024-09-13T15:22:42.273285+0000 mon.c (mon.2) 133 : audit [INF] from='client.? 172.21.15.99:0/328908844' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[84576]: audit 2024-09-13T15:22:42.273285+0000 mon.c (mon.2) 133 : audit [INF] from='client.? 172.21.15.99:0/328908844' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[84576]: audit 2024-09-13T15:22:42.273947+0000 mon.a (mon.0) 2900 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[84576]: audit 2024-09-13T15:22:42.273947+0000 mon.a (mon.0) 2900 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[87147]: cluster 2024-09-13T15:22:41.335869+0000 mgr.y (mgr.26252) 1702 : cluster [DBG] pgmap v2353: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:43.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[87147]: cluster 2024-09-13T15:22:41.335869+0000 mgr.y (mgr.26252) 1702 : cluster [DBG] pgmap v2353: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:43.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[87147]: audit 2024-09-13T15:22:42.273285+0000 mon.c (mon.2) 133 : audit [INF] from='client.? 172.21.15.99:0/328908844' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[87147]: audit 2024-09-13T15:22:42.273285+0000 mon.c (mon.2) 133 : audit [INF] from='client.? 172.21.15.99:0/328908844' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[87147]: audit 2024-09-13T15:22:42.273947+0000 mon.a (mon.0) 2900 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:42 smithi099 bash[87147]: audit 2024-09-13T15:22:42.273947+0000 mon.a (mon.0) 2900 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:42 smithi177 bash[68594]: cluster 2024-09-13T15:22:41.335869+0000 mgr.y (mgr.26252) 1702 : cluster [DBG] pgmap v2353: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:43.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:42 smithi177 bash[68594]: cluster 2024-09-13T15:22:41.335869+0000 mgr.y (mgr.26252) 1702 : cluster [DBG] pgmap v2353: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:43.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:42 smithi177 bash[68594]: audit 2024-09-13T15:22:42.273285+0000 mon.c (mon.2) 133 : audit [INF] from='client.? 172.21.15.99:0/328908844' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:42 smithi177 bash[68594]: audit 2024-09-13T15:22:42.273285+0000 mon.c (mon.2) 133 : audit [INF] from='client.? 172.21.15.99:0/328908844' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:42 smithi177 bash[68594]: audit 2024-09-13T15:22:42.273947+0000 mon.a (mon.0) 2900 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:42 smithi177 bash[68594]: audit 2024-09-13T15:22:42.273947+0000 mon.a (mon.0) 2900 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:22:43.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T15:22:44.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:43 smithi099 bash[84576]: audit 2024-09-13T15:22:42.783559+0000 mon.a (mon.0) 2901 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:44.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:43 smithi099 bash[84576]: audit 2024-09-13T15:22:42.783559+0000 mon.a (mon.0) 2901 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:44.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:43 smithi099 bash[84576]: cluster 2024-09-13T15:22:42.792631+0000 mon.a (mon.0) 2902 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2024-09-13T15:22:44.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:43 smithi099 bash[84576]: cluster 2024-09-13T15:22:42.792631+0000 mon.a (mon.0) 2902 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2024-09-13T15:22:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:43 smithi099 bash[87147]: audit 2024-09-13T15:22:42.783559+0000 mon.a (mon.0) 2901 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:43 smithi099 bash[87147]: audit 2024-09-13T15:22:42.783559+0000 mon.a (mon.0) 2901 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:43 smithi099 bash[87147]: cluster 2024-09-13T15:22:42.792631+0000 mon.a (mon.0) 2902 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2024-09-13T15:22:44.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:43 smithi099 bash[87147]: cluster 2024-09-13T15:22:42.792631+0000 mon.a (mon.0) 2902 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2024-09-13T15:22:44.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:43 smithi177 bash[68594]: audit 2024-09-13T15:22:42.783559+0000 mon.a (mon.0) 2901 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:44.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:43 smithi177 bash[68594]: audit 2024-09-13T15:22:42.783559+0000 mon.a (mon.0) 2901 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:22:44.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:43 smithi177 bash[68594]: cluster 2024-09-13T15:22:42.792631+0000 mon.a (mon.0) 2902 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2024-09-13T15:22:44.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:43 smithi177 bash[68594]: cluster 2024-09-13T15:22:42.792631+0000 mon.a (mon.0) 2902 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2024-09-13T15:22:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:44 smithi099 bash[87147]: cluster 2024-09-13T15:22:43.336511+0000 mgr.y (mgr.26252) 1703 : cluster [DBG] pgmap v2355: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:45.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:44 smithi099 bash[87147]: cluster 2024-09-13T15:22:43.336511+0000 mgr.y (mgr.26252) 1703 : cluster [DBG] pgmap v2355: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:45.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:44 smithi099 bash[84576]: cluster 2024-09-13T15:22:43.336511+0000 mgr.y (mgr.26252) 1703 : cluster [DBG] pgmap v2355: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:45.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:44 smithi099 bash[84576]: cluster 2024-09-13T15:22:43.336511+0000 mgr.y (mgr.26252) 1703 : cluster [DBG] pgmap v2355: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:45.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:44 smithi177 bash[68594]: cluster 2024-09-13T15:22:43.336511+0000 mgr.y (mgr.26252) 1703 : cluster [DBG] pgmap v2355: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:45.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:44 smithi177 bash[68594]: cluster 2024-09-13T15:22:43.336511+0000 mgr.y (mgr.26252) 1703 : cluster [DBG] pgmap v2355: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:46.275 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:22:47.075 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:22:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:22:47] "GET /metrics HTTP/1.1" 200 34332 "" "Prometheus/2.51.0" 2024-09-13T15:22:47.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:46 smithi099 bash[84576]: cluster 2024-09-13T15:22:45.337259+0000 mgr.y (mgr.26252) 1704 : cluster [DBG] pgmap v2356: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:47.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:46 smithi099 bash[84576]: cluster 2024-09-13T15:22:45.337259+0000 mgr.y (mgr.26252) 1704 : cluster [DBG] pgmap v2356: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:47.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:46 smithi099 bash[87147]: cluster 2024-09-13T15:22:45.337259+0000 mgr.y (mgr.26252) 1704 : cluster [DBG] pgmap v2356: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:47.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:46 smithi099 bash[87147]: cluster 2024-09-13T15:22:45.337259+0000 mgr.y (mgr.26252) 1704 : cluster [DBG] pgmap v2356: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:47.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:46 smithi177 bash[68594]: cluster 2024-09-13T15:22:45.337259+0000 mgr.y (mgr.26252) 1704 : cluster [DBG] pgmap v2356: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:47.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:46 smithi177 bash[68594]: cluster 2024-09-13T15:22:45.337259+0000 mgr.y (mgr.26252) 1704 : cluster [DBG] pgmap v2356: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:48.581 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:22:49.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:48 smithi099 bash[84576]: cluster 2024-09-13T15:22:47.338261+0000 mgr.y (mgr.26252) 1705 : cluster [DBG] pgmap v2357: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:49.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:48 smithi099 bash[84576]: cluster 2024-09-13T15:22:47.338261+0000 mgr.y (mgr.26252) 1705 : cluster [DBG] pgmap v2357: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:49.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:48 smithi099 bash[84576]: audit 2024-09-13T15:22:48.329273+0000 mon.a (mon.0) 2903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:49.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:48 smithi099 bash[84576]: audit 2024-09-13T15:22:48.329273+0000 mon.a (mon.0) 2903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:49.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:48 smithi099 bash[87147]: cluster 2024-09-13T15:22:47.338261+0000 mgr.y (mgr.26252) 1705 : cluster [DBG] pgmap v2357: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:49.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:48 smithi099 bash[87147]: cluster 2024-09-13T15:22:47.338261+0000 mgr.y (mgr.26252) 1705 : cluster [DBG] pgmap v2357: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:49.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:48 smithi099 bash[87147]: audit 2024-09-13T15:22:48.329273+0000 mon.a (mon.0) 2903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:49.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:48 smithi099 bash[87147]: audit 2024-09-13T15:22:48.329273+0000 mon.a (mon.0) 2903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:49.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:48 smithi177 bash[68594]: cluster 2024-09-13T15:22:47.338261+0000 mgr.y (mgr.26252) 1705 : cluster [DBG] pgmap v2357: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:49.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:48 smithi177 bash[68594]: cluster 2024-09-13T15:22:47.338261+0000 mgr.y (mgr.26252) 1705 : cluster [DBG] pgmap v2357: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:49.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:48 smithi177 bash[68594]: audit 2024-09-13T15:22:48.329273+0000 mon.a (mon.0) 2903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:49.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:48 smithi177 bash[68594]: audit 2024-09-13T15:22:48.329273+0000 mon.a (mon.0) 2903 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:22:50.230 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:30526 46.7 e1104/1099 46:e1d849e2:::benchmark_data_smithi099_233884_object30525:head v 1104'1911, mlcod=1104'1909)", 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:34.459450+0000", 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 435.76944575700003, 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022396363999999998, 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:34.459450+0000", 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:34.459450+0000", 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:34.459467+0000", 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6286999999999998e-05 2024-09-13T15:22:50.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:34.459550+0000", 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3126000000000003e-05 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:34.459571+0000", 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1228000000000001e-05 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:34.459583+0000", 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2051e-05 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:34.459659+0000", 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5872999999999997e-05 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:34.459685+0000", 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5996999999999999e-05 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:34.481810+0000", 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022124971 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:34.481847+0000", 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6831e-05 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:37481 46.7 e1104/1099 46:eedd78d0:::benchmark_data_smithi099_233884_object37480:head v 1104'2323, mlcod=1104'2321)", 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:38.962816+0000", 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 431.26607988900003, 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028214344999999998, 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.962816+0000", 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.962816+0000", 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.962818+0000", 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4330000000000001e-06 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.962860+0000", 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2630999999999999e-05 2024-09-13T15:22:50.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.962873+0000", 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2587e-05 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.962882+0000", 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3500000000000003e-06 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.962934+0000", 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2265000000000002e-05 2024-09-13T15:22:50.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.962949+0000", 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4883e-05 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.990975+0000", 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028025431 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.991030+0000", 2024-09-13T15:22:50.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5764999999999999e-05 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:37489 46.8 e1104/1099 46:129454cf:::benchmark_data_smithi099_233884_object37488:head v 1104'2385, mlcod=1104'2383)", 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:38.970434+0000", 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 431.25846171500001, 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022752613000000001, 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.970434+0000", 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.970434+0000", 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.970436+0000", 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.75e-06 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.970469+0000", 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2966000000000003e-05 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.987189+0000", 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016720406 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.987208+0000", 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9063999999999999e-05 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.987287+0000", 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8948000000000004e-05 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.987315+0000", 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7282000000000001e-05 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.993128+0000", 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058129410000000003 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:38.993187+0000", 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9256000000000003e-05 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:95500 46.10 e1112/1111 46:0f89ca76:::benchmark_data_smithi099_233884_object95499:head v 1112'5822, mlcod=0'0)", 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:40.563246+0000", 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 369.665650314, 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023410119, 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.244 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.563246+0000", 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.563246+0000", 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.563248+0000", 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.57e-06 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.563400+0000", 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015167300000000001 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.563437+0000", 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6705000000000002e-05 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.563458+0000", 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1855999999999999e-05 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.572156+0000", 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0086979940000000006 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.572181+0000", 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4281999999999999e-05 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.586600+0000", 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014419474999999999 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.586656+0000", 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5563999999999999e-05 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:95553 46.10 e1112/1111 46:0f44a814:::benchmark_data_smithi099_233884_object95552:head v 1112'5823, mlcod=0'0)", 2024-09-13T15:22:50.248 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:40.565415+0000", 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 369.66348092999999, 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021277444999999999, 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.565415+0000", 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.565415+0000", 2024-09-13T15:22:50.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.565418+0000", 2024-09-13T15:22:50.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6960000000000001e-06 2024-09-13T15:22:50.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.565551+0000", 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013337700000000001 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.565580+0000", 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9246000000000001e-05 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.565601+0000", 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0662e-05 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.575232+0000", 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0096314599999999997 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.575260+0000", 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7755000000000001e-05 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.586662+0000", 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011401923 2024-09-13T15:22:50.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.586692+0000", 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0326000000000001e-05 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:95558 46.10 e1112/1111 46:089d8542:::benchmark_data_smithi099_233884_object95557:head v 1112'5824, mlcod=0'0)", 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:40.570142+0000", 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 369.65875434700001, 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020235587999999999, 2024-09-13T15:22:50.253 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.570142+0000", 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.570142+0000", 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.570143+0000", 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.57e-06 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.570214+0000", 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0343000000000001e-05 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.570230+0000", 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6594000000000001e-05 2024-09-13T15:22:50.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.570243+0000", 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2804e-05 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.579263+0000", 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0090199579999999998 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.579297+0000", 2024-09-13T15:22:50.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4313000000000002e-05 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.590339+0000", 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011042151 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:40.590377+0000", 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7855000000000003e-05 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:102419 46.8 e1117/1108 46:10bc519e:::benchmark_data_smithi099_233884_object6:head v 1117'6025, mlcod=1115'6023)", 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:51.086654+0000", 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 359.14224225800001, 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023244151000000001, 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.086654+0000", 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.086654+0000", 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.086654+0000", 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7000000000000004e-07 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.086665+0000", 2024-09-13T15:22:50.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0536999999999999e-05 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.086709+0000", 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4181000000000003e-05 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.086731+0000", 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2172e-05 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.086892+0000", 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000160879 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.086929+0000", 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7041e-05 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.107469+0000", 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020539519999999999 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.109898+0000", 2024-09-13T15:22:50.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024291510000000001 2024-09-13T15:22:50.262 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.262 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.262 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.262 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:102416 46.10 e1117/1111 46:0e4f18f8:::benchmark_data_smithi099_233884_object3:head v 1117'6043, mlcod=1103'564)", 2024-09-13T15:22:50.262 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:51.090588+0000", 2024-09-13T15:22:50.262 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 359.13830840000003, 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.019406685999999999, 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.090588+0000", 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.090588+0000", 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.090588+0000", 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6600000000000005e-07 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.090599+0000", 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0702e-05 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.090617+0000", 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8105999999999999e-05 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.090630+0000", 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2428e-05 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.095704+0000", 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050738609999999998 2024-09-13T15:22:50.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.095742+0000", 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8532999999999999e-05 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.109960+0000", 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014217990999999999 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:51.109994+0000", 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4199000000000002e-05 2024-09-13T15:22:50.266 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:167038 46.8 e1130/1108 46:15694485:::benchmark_data_smithi099_233884_object64625:head v 1130'8082, mlcod=1130'8080)", 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:14.592412+0000", 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 275.63648358799998, 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015780701000000001, 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.592412+0000", 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.592412+0000", 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.592413+0000", 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8299999999999997e-07 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.592417+0000", 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8750000000000002e-06 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.592424+0000", 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3699999999999997e-06 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.592429+0000", 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7639999999999996e-06 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.592474+0000", 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4638999999999997e-05 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.592493+0000", 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9318e-05 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.608147+0000", 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015654214999999999 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.608193+0000", 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6037e-05 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:170018 46.18 e1130/1108 46:1de1449b:::benchmark_data_smithi099_233884_object67605:head v 1130'8152, mlcod=1130'8150)", 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:17.987191+0000", 2024-09-13T15:22:50.271 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 272.24170498500001, 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016224903999999998, 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:17.987191+0000", 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:17.987191+0000", 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:17.987193+0000", 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.821e-06 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:17.987204+0000", 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1565e-05 2024-09-13T15:22:50.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:17.987228+0000", 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3257000000000001e-05 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:17.987239+0000", 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1622e-05 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:17.987437+0000", 2024-09-13T15:22:50.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000197608 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:17.987489+0000", 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2463999999999998e-05 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:18.003379+0000", 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015889840999999998 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:18.003416+0000", 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6726e-05 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:178135 46.10 e1130/1126 46:0970ae99:::benchmark_data_smithi099_233884_object75722:head v 1130'8356, mlcod=1130'8351)", 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:28.521063+0000", 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 261.70783266199999, 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015820092000000001, 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.276 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:28.521063+0000", 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:28.521063+0000", 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:28.521065+0000", 2024-09-13T15:22:50.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9269999999999999e-06 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:28.521078+0000", 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2938999999999999e-05 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:28.521114+0000", 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6211999999999998e-05 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:28.521130+0000", 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5586999999999998e-05 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:28.521346+0000", 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00021619900000000001 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:28.521396+0000", 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9660999999999998e-05 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:28.536608+0000", 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015212343999999999 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:28.536883+0000", 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000275223 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:180190 46.18 e1130/1108 46:18457bc4:::benchmark_data_smithi099_233884_object77777:head v 1130'8491, mlcod=1130'8488)", 2024-09-13T15:22:50.280 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:31.167107+0000", 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 259.06178872800001, 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015013317999999999, 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.167107+0000", 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.167107+0000", 2024-09-13T15:22:50.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.167109+0000", 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.793e-06 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.167121+0000", 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1932e-05 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.167140+0000", 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9454999999999999e-05 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.167152+0000", 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2074e-05 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.167244+0000", 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1120999999999997e-05 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.167291+0000", 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7675e-05 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.182058+0000", 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014767071 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.182121+0000", 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2197000000000001e-05 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.284 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:180219 46.18 e1130/1108 46:18590470:::benchmark_data_smithi099_233884_object77806:head v 1130'8493, mlcod=1130'8489)", 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:31.175158+0000", 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 259.05373828699999, 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024687008, 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.175158+0000", 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.175158+0000", 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.175158+0000", 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1399999999999996e-07 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.175165+0000", 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2280000000000003e-06 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.175176+0000", 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0909e-05 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.175183+0000", 2024-09-13T15:22:50.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1400000000000002e-06 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.178501+0000", 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033179199999999998 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.178529+0000", 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7878e-05 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.199751+0000", 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021222449000000001 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.199845+0000", 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3670000000000003e-05 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:180221 46.18 e1130/1108 46:1e358606:::benchmark_data_smithi099_233884_object77808:head v 1130'8494, mlcod=1130'8489)", 2024-09-13T15:22:50.289 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:31.178217+0000", 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 259.05067889200001, 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021660634000000002, 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.178217+0000", 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.178217+0000", 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.178218+0000", 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.254e-06 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.178228+0000", 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6600000000000007e-06 2024-09-13T15:22:50.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.178244+0000", 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6154000000000002e-05 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.178253+0000", 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1349999999999998e-06 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.182160+0000", 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039067620000000003 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.182186+0000", 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6390999999999999e-05 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.199850+0000", 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017663647000000001 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.199878+0000", 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7631e-05 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:180222 46.18 e1130/1108 46:1b439717:::benchmark_data_smithi099_233884_object77809:head v 1130'8495, mlcod=1130'8489)", 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:31.181314+0000", 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 259.04758171499998, 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018596906, 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.294 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.181314+0000", 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.181314+0000", 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.181316+0000", 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.074e-06 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.181328+0000", 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1914e-05 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.181350+0000", 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2090999999999999e-05 2024-09-13T15:22:50.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.181360+0000", 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8579999999999995e-06 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.185497+0000", 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041368259999999997 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.185522+0000", 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4980000000000001e-05 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.199880+0000", 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014358123 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.199911+0000", 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1040000000000001e-05 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.298 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:180223 46.18 e1130/1108 46:1e1cd6ea:::benchmark_data_smithi099_233884_object77810:head v 1130'8496, mlcod=1130'8489)", 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:31.184340+0000", 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 259.044556242, 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015603605, 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.184340+0000", 2024-09-13T15:22:50.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.184340+0000", 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.184341+0000", 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.499e-06 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.184350+0000", 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2639999999999997e-06 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.184368+0000", 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7663e-05 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.184378+0000", 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3540000000000006e-06 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.189063+0000", 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046856550000000004 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.189091+0000", 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7339e-05 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.199913+0000", 2024-09-13T15:22:50.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010822589000000001 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:31.199943+0000", 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0241999999999999e-05 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(46.0 e1144,1133 46:0186659a:::benchmark_data_smithi099_233884_object22393:head,1123'6764)", 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:19:51.562255+0000", 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 178.66664083500001, 2024-09-13T15:22:50.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00048856599999999996, 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:51.562255+0000", 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:51.562255+0000", 2024-09-13T15:22:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:51.562257+0000", 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.266e-06 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:51.562269+0000", 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.137e-05 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:51.562303+0000", 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4598999999999999e-05 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:51.562327+0000", 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3903000000000001e-05 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:51.562535+0000", 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000207777 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:22:50.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:51.562572+0000", 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7236000000000001e-05 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:51.562744+0000", 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017141500000000001 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(46.0 e1144,1133 46:06269a9d:::benchmark_data_smithi099_233884_object44204:head,1127'7434)", 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:19:57.122296+0000", 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 173.10660029300001, 2024-09-13T15:22:50.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036832129999999999, 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: "continuous": false, 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:57.122296+0000", 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:57.122296+0000", 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:57.122298+0000", 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.04e-06 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:57.123750+0000", 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001451796 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:57.124048+0000", 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000298962 2024-09-13T15:22:50.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:57.124058+0000", 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9920000000000002e-06 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:57.125960+0000", 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019019169999999999 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:19:57.125979+0000", 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8505999999999999e-05 2024-09-13T15:22:50.310 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.311 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.311 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.311 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:22:50.311 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:22:50.311 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:22:50.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:22:50 smithi177 bash[61439]: ts=2024-09-13T15:22:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:22:51.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:50 smithi099 bash[84576]: cluster 2024-09-13T15:22:49.339239+0000 mgr.y (mgr.26252) 1706 : cluster [DBG] pgmap v2358: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:51.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:50 smithi099 bash[84576]: cluster 2024-09-13T15:22:49.339239+0000 mgr.y (mgr.26252) 1706 : cluster [DBG] pgmap v2358: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:51.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:50 smithi099 bash[87147]: cluster 2024-09-13T15:22:49.339239+0000 mgr.y (mgr.26252) 1706 : cluster [DBG] pgmap v2358: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:51.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:50 smithi099 bash[87147]: cluster 2024-09-13T15:22:49.339239+0000 mgr.y (mgr.26252) 1706 : cluster [DBG] pgmap v2358: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:51.084 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:50 smithi177 bash[68594]: cluster 2024-09-13T15:22:49.339239+0000 mgr.y (mgr.26252) 1706 : cluster [DBG] pgmap v2358: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:51.084 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:50 smithi177 bash[68594]: cluster 2024-09-13T15:22:49.339239+0000 mgr.y (mgr.26252) 1706 : cluster [DBG] pgmap v2358: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:51.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:22:53.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:52 smithi177 bash[68594]: cluster 2024-09-13T15:22:51.340096+0000 mgr.y (mgr.26252) 1707 : cluster [DBG] pgmap v2359: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:53.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:52 smithi177 bash[68594]: cluster 2024-09-13T15:22:51.340096+0000 mgr.y (mgr.26252) 1707 : cluster [DBG] pgmap v2359: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:53.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:52 smithi099 bash[84576]: cluster 2024-09-13T15:22:51.340096+0000 mgr.y (mgr.26252) 1707 : cluster [DBG] pgmap v2359: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:53.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:52 smithi099 bash[84576]: cluster 2024-09-13T15:22:51.340096+0000 mgr.y (mgr.26252) 1707 : cluster [DBG] pgmap v2359: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:53.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:52 smithi099 bash[87147]: cluster 2024-09-13T15:22:51.340096+0000 mgr.y (mgr.26252) 1707 : cluster [DBG] pgmap v2359: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:53.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:52 smithi099 bash[87147]: cluster 2024-09-13T15:22:51.340096+0000 mgr.y (mgr.26252) 1707 : cluster [DBG] pgmap v2359: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:53.839 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T15:22:54.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:53 smithi177 bash[68594]: audit 2024-09-13T15:22:53.246665+0000 mon.a (mon.0) 2904 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:22:54.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:53 smithi177 bash[68594]: audit 2024-09-13T15:22:53.246665+0000 mon.a (mon.0) 2904 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:22:54.153 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:53 smithi099 bash[84576]: audit 2024-09-13T15:22:53.246665+0000 mon.a (mon.0) 2904 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:22:54.153 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:53 smithi099 bash[84576]: audit 2024-09-13T15:22:53.246665+0000 mon.a (mon.0) 2904 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:22:54.154 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:53 smithi099 bash[87147]: audit 2024-09-13T15:22:53.246665+0000 mon.a (mon.0) 2904 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:22:54.154 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:53 smithi099 bash[87147]: audit 2024-09-13T15:22:53.246665+0000 mon.a (mon.0) 2904 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:22:54.840 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 4 1 2024-09-13T15:22:55.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:54 smithi177 bash[68594]: cluster 2024-09-13T15:22:53.340939+0000 mgr.y (mgr.26252) 1708 : cluster [DBG] pgmap v2360: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:55.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:54 smithi177 bash[68594]: cluster 2024-09-13T15:22:53.340939+0000 mgr.y (mgr.26252) 1708 : cluster [DBG] pgmap v2360: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:55.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:54 smithi177 bash[68594]: audit 2024-09-13T15:22:53.838632+0000 mon.a (mon.0) 2905 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:22:55.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:54 smithi177 bash[68594]: audit 2024-09-13T15:22:53.838632+0000 mon.a (mon.0) 2905 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:22:55.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:54 smithi177 bash[68594]: cluster 2024-09-13T15:22:53.843824+0000 mon.a (mon.0) 2906 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2024-09-13T15:22:55.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:54 smithi177 bash[68594]: cluster 2024-09-13T15:22:53.843824+0000 mon.a (mon.0) 2906 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2024-09-13T15:22:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[84576]: cluster 2024-09-13T15:22:53.340939+0000 mgr.y (mgr.26252) 1708 : cluster [DBG] pgmap v2360: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[84576]: cluster 2024-09-13T15:22:53.340939+0000 mgr.y (mgr.26252) 1708 : cluster [DBG] pgmap v2360: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[84576]: audit 2024-09-13T15:22:53.838632+0000 mon.a (mon.0) 2905 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:22:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[84576]: audit 2024-09-13T15:22:53.838632+0000 mon.a (mon.0) 2905 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:22:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[84576]: cluster 2024-09-13T15:22:53.843824+0000 mon.a (mon.0) 2906 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2024-09-13T15:22:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[84576]: cluster 2024-09-13T15:22:53.843824+0000 mon.a (mon.0) 2906 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2024-09-13T15:22:55.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[87147]: cluster 2024-09-13T15:22:53.340939+0000 mgr.y (mgr.26252) 1708 : cluster [DBG] pgmap v2360: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[87147]: cluster 2024-09-13T15:22:53.340939+0000 mgr.y (mgr.26252) 1708 : cluster [DBG] pgmap v2360: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[87147]: audit 2024-09-13T15:22:53.838632+0000 mon.a (mon.0) 2905 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:22:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[87147]: audit 2024-09-13T15:22:53.838632+0000 mon.a (mon.0) 2905 : audit [INF] from='client.? 172.21.15.99:0/3250226221' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:22:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[87147]: cluster 2024-09-13T15:22:53.843824+0000 mon.a (mon.0) 2906 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2024-09-13T15:22:55.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:54 smithi099 bash[87147]: cluster 2024-09-13T15:22:53.843824+0000 mon.a (mon.0) 2906 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2024-09-13T15:22:55.874 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:22:57.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:56 smithi177 bash[68594]: cluster 2024-09-13T15:22:55.341725+0000 mgr.y (mgr.26252) 1709 : cluster [DBG] pgmap v2362: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:57.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:56 smithi177 bash[68594]: cluster 2024-09-13T15:22:55.341725+0000 mgr.y (mgr.26252) 1709 : cluster [DBG] pgmap v2362: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:57.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:22:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:22:57] "GET /metrics HTTP/1.1" 200 34328 "" "Prometheus/2.51.0" 2024-09-13T15:22:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:56 smithi099 bash[84576]: cluster 2024-09-13T15:22:55.341725+0000 mgr.y (mgr.26252) 1709 : cluster [DBG] pgmap v2362: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:56 smithi099 bash[84576]: cluster 2024-09-13T15:22:55.341725+0000 mgr.y (mgr.26252) 1709 : cluster [DBG] pgmap v2362: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:56 smithi099 bash[87147]: cluster 2024-09-13T15:22:55.341725+0000 mgr.y (mgr.26252) 1709 : cluster [DBG] pgmap v2362: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:56 smithi099 bash[87147]: cluster 2024-09-13T15:22:55.341725+0000 mgr.y (mgr.26252) 1709 : cluster [DBG] pgmap v2362: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:59.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:58 smithi177 bash[68594]: cluster 2024-09-13T15:22:57.342828+0000 mgr.y (mgr.26252) 1710 : cluster [DBG] pgmap v2363: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:59.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:22:58 smithi177 bash[68594]: cluster 2024-09-13T15:22:57.342828+0000 mgr.y (mgr.26252) 1710 : cluster [DBG] pgmap v2363: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:59.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:58 smithi099 bash[84576]: cluster 2024-09-13T15:22:57.342828+0000 mgr.y (mgr.26252) 1710 : cluster [DBG] pgmap v2363: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:59.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:22:58 smithi099 bash[84576]: cluster 2024-09-13T15:22:57.342828+0000 mgr.y (mgr.26252) 1710 : cluster [DBG] pgmap v2363: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:59.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:58 smithi099 bash[87147]: cluster 2024-09-13T15:22:57.342828+0000 mgr.y (mgr.26252) 1710 : cluster [DBG] pgmap v2363: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:59.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:22:58 smithi099 bash[87147]: cluster 2024-09-13T15:22:57.342828+0000 mgr.y (mgr.26252) 1710 : cluster [DBG] pgmap v2363: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:22:59.619 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:22:59.619 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:22:59.619 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2024-09-13T15:22:59.620 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:22:59.625 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:23:00.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:23:00.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:23:00 smithi177 bash[61439]: ts=2024-09-13T15:23:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:23:01.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:00 smithi177 bash[68594]: cluster 2024-09-13T15:22:59.343753+0000 mgr.y (mgr.26252) 1711 : cluster [DBG] pgmap v2364: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:01.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:00 smithi177 bash[68594]: cluster 2024-09-13T15:22:59.343753+0000 mgr.y (mgr.26252) 1711 : cluster [DBG] pgmap v2364: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:01.146 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:00 smithi099 bash[84576]: cluster 2024-09-13T15:22:59.343753+0000 mgr.y (mgr.26252) 1711 : cluster [DBG] pgmap v2364: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:01.146 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:00 smithi099 bash[84576]: cluster 2024-09-13T15:22:59.343753+0000 mgr.y (mgr.26252) 1711 : cluster [DBG] pgmap v2364: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:01.146 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:00 smithi099 bash[87147]: cluster 2024-09-13T15:22:59.343753+0000 mgr.y (mgr.26252) 1711 : cluster [DBG] pgmap v2364: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:01.146 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:00 smithi099 bash[87147]: cluster 2024-09-13T15:22:59.343753+0000 mgr.y (mgr.26252) 1711 : cluster [DBG] pgmap v2364: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:03.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:02 smithi177 bash[68594]: cluster 2024-09-13T15:23:01.344594+0000 mgr.y (mgr.26252) 1712 : cluster [DBG] pgmap v2365: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:03.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:02 smithi177 bash[68594]: cluster 2024-09-13T15:23:01.344594+0000 mgr.y (mgr.26252) 1712 : cluster [DBG] pgmap v2365: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:02 smithi099 bash[84576]: cluster 2024-09-13T15:23:01.344594+0000 mgr.y (mgr.26252) 1712 : cluster [DBG] pgmap v2365: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:03.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:02 smithi099 bash[84576]: cluster 2024-09-13T15:23:01.344594+0000 mgr.y (mgr.26252) 1712 : cluster [DBG] pgmap v2365: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:03.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:02 smithi099 bash[87147]: cluster 2024-09-13T15:23:01.344594+0000 mgr.y (mgr.26252) 1712 : cluster [DBG] pgmap v2365: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:03.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:02 smithi099 bash[87147]: cluster 2024-09-13T15:23:01.344594+0000 mgr.y (mgr.26252) 1712 : cluster [DBG] pgmap v2365: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:03 smithi099 bash[84576]: audit 2024-09-13T15:23:03.329605+0000 mon.a (mon.0) 2907 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:03 smithi099 bash[84576]: audit 2024-09-13T15:23:03.329605+0000 mon.a (mon.0) 2907 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:03 smithi099 bash[84576]: audit 2024-09-13T15:23:03.605600+0000 mon.a (mon.0) 2908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:23:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:03 smithi099 bash[84576]: audit 2024-09-13T15:23:03.605600+0000 mon.a (mon.0) 2908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:23:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:03 smithi099 bash[87147]: audit 2024-09-13T15:23:03.329605+0000 mon.a (mon.0) 2907 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:03 smithi099 bash[87147]: audit 2024-09-13T15:23:03.329605+0000 mon.a (mon.0) 2907 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:03 smithi099 bash[87147]: audit 2024-09-13T15:23:03.605600+0000 mon.a (mon.0) 2908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:23:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:03 smithi099 bash[87147]: audit 2024-09-13T15:23:03.605600+0000 mon.a (mon.0) 2908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:23:04.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:03 smithi177 bash[68594]: audit 2024-09-13T15:23:03.329605+0000 mon.a (mon.0) 2907 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:04.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:03 smithi177 bash[68594]: audit 2024-09-13T15:23:03.329605+0000 mon.a (mon.0) 2907 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:04.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:03 smithi177 bash[68594]: audit 2024-09-13T15:23:03.605600+0000 mon.a (mon.0) 2908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:23:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:03 smithi177 bash[68594]: audit 2024-09-13T15:23:03.605600+0000 mon.a (mon.0) 2908 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:23:04.876 INFO:teuthology.orchestra.run.smithi099.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-13T15:23:05.200 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[84576]: cluster 2024-09-13T15:23:03.345448+0000 mgr.y (mgr.26252) 1713 : cluster [DBG] pgmap v2366: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:05.201 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[84576]: cluster 2024-09-13T15:23:03.345448+0000 mgr.y (mgr.26252) 1713 : cluster [DBG] pgmap v2366: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:05.201 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[84576]: audit 2024-09-13T15:23:03.880881+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.99:0/3844497515' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.201 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[84576]: audit 2024-09-13T15:23:03.880881+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.99:0/3844497515' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.201 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[84576]: audit 2024-09-13T15:23:03.881350+0000 mon.a (mon.0) 2909 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.201 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[84576]: audit 2024-09-13T15:23:03.881350+0000 mon.a (mon.0) 2909 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.201 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[87147]: cluster 2024-09-13T15:23:03.345448+0000 mgr.y (mgr.26252) 1713 : cluster [DBG] pgmap v2366: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:05.201 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[87147]: cluster 2024-09-13T15:23:03.345448+0000 mgr.y (mgr.26252) 1713 : cluster [DBG] pgmap v2366: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:05.201 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[87147]: audit 2024-09-13T15:23:03.880881+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.99:0/3844497515' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.201 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[87147]: audit 2024-09-13T15:23:03.880881+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.99:0/3844497515' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.201 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[87147]: audit 2024-09-13T15:23:03.881350+0000 mon.a (mon.0) 2909 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.202 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:04 smithi099 bash[87147]: audit 2024-09-13T15:23:03.881350+0000 mon.a (mon.0) 2909 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.202 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:23:05.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:04 smithi177 bash[68594]: cluster 2024-09-13T15:23:03.345448+0000 mgr.y (mgr.26252) 1713 : cluster [DBG] pgmap v2366: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:05.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:04 smithi177 bash[68594]: cluster 2024-09-13T15:23:03.345448+0000 mgr.y (mgr.26252) 1713 : cluster [DBG] pgmap v2366: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:05.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:04 smithi177 bash[68594]: audit 2024-09-13T15:23:03.880881+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.99:0/3844497515' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:04 smithi177 bash[68594]: audit 2024-09-13T15:23:03.880881+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.99:0/3844497515' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:04 smithi177 bash[68594]: audit 2024-09-13T15:23:03.881350+0000 mon.a (mon.0) 2909 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.303 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:04 smithi177 bash[68594]: audit 2024-09-13T15:23:03.881350+0000 mon.a (mon.0) 2909 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:23:05.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T15:23:06.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:05 smithi099 bash[84576]: audit 2024-09-13T15:23:04.867172+0000 mon.a (mon.0) 2910 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:23:06.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:05 smithi099 bash[84576]: audit 2024-09-13T15:23:04.867172+0000 mon.a (mon.0) 2910 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:23:06.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:05 smithi099 bash[84576]: cluster 2024-09-13T15:23:04.875298+0000 mon.a (mon.0) 2911 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2024-09-13T15:23:06.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:05 smithi099 bash[84576]: cluster 2024-09-13T15:23:04.875298+0000 mon.a (mon.0) 2911 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2024-09-13T15:23:06.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:05 smithi099 bash[87147]: audit 2024-09-13T15:23:04.867172+0000 mon.a (mon.0) 2910 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:23:06.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:05 smithi099 bash[87147]: audit 2024-09-13T15:23:04.867172+0000 mon.a (mon.0) 2910 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:23:06.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:05 smithi099 bash[87147]: cluster 2024-09-13T15:23:04.875298+0000 mon.a (mon.0) 2911 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2024-09-13T15:23:06.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:05 smithi099 bash[87147]: cluster 2024-09-13T15:23:04.875298+0000 mon.a (mon.0) 2911 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2024-09-13T15:23:06.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:05 smithi177 bash[68594]: audit 2024-09-13T15:23:04.867172+0000 mon.a (mon.0) 2910 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:23:06.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:05 smithi177 bash[68594]: audit 2024-09-13T15:23:04.867172+0000 mon.a (mon.0) 2910 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:23:06.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:05 smithi177 bash[68594]: cluster 2024-09-13T15:23:04.875298+0000 mon.a (mon.0) 2911 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2024-09-13T15:23:06.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:05 smithi177 bash[68594]: cluster 2024-09-13T15:23:04.875298+0000 mon.a (mon.0) 2911 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2024-09-13T15:23:07.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:06 smithi177 bash[68594]: cluster 2024-09-13T15:23:05.346098+0000 mgr.y (mgr.26252) 1714 : cluster [DBG] pgmap v2368: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:07.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:06 smithi177 bash[68594]: cluster 2024-09-13T15:23:05.346098+0000 mgr.y (mgr.26252) 1714 : cluster [DBG] pgmap v2368: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:07.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:23:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:23:07] "GET /metrics HTTP/1.1" 200 34332 "" "Prometheus/2.51.0" 2024-09-13T15:23:07.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:06 smithi099 bash[84576]: cluster 2024-09-13T15:23:05.346098+0000 mgr.y (mgr.26252) 1714 : cluster [DBG] pgmap v2368: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:07.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:06 smithi099 bash[84576]: cluster 2024-09-13T15:23:05.346098+0000 mgr.y (mgr.26252) 1714 : cluster [DBG] pgmap v2368: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:06 smithi099 bash[87147]: cluster 2024-09-13T15:23:05.346098+0000 mgr.y (mgr.26252) 1714 : cluster [DBG] pgmap v2368: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:06 smithi099 bash[87147]: cluster 2024-09-13T15:23:05.346098+0000 mgr.y (mgr.26252) 1714 : cluster [DBG] pgmap v2368: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:09.098 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:23:09.098 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:23:09.099 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:23:09.099 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:23:09.099 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:23:09.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:08 smithi099 bash[84576]: cluster 2024-09-13T15:23:07.347076+0000 mgr.y (mgr.26252) 1715 : cluster [DBG] pgmap v2369: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:09.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:08 smithi099 bash[84576]: cluster 2024-09-13T15:23:07.347076+0000 mgr.y (mgr.26252) 1715 : cluster [DBG] pgmap v2369: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:09.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:08 smithi099 bash[87147]: cluster 2024-09-13T15:23:07.347076+0000 mgr.y (mgr.26252) 1715 : cluster [DBG] pgmap v2369: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:09.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:08 smithi099 bash[87147]: cluster 2024-09-13T15:23:07.347076+0000 mgr.y (mgr.26252) 1715 : cluster [DBG] pgmap v2369: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:09.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:08 smithi177 bash[68594]: cluster 2024-09-13T15:23:07.347076+0000 mgr.y (mgr.26252) 1715 : cluster [DBG] pgmap v2369: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:09.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:08 smithi177 bash[68594]: cluster 2024-09-13T15:23:07.347076+0000 mgr.y (mgr.26252) 1715 : cluster [DBG] pgmap v2369: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:09.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:23:10.417 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:23:10.532 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[84576]: audit 2024-09-13T15:23:09.214805+0000 mon.a (mon.0) 2912 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.532 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[84576]: audit 2024-09-13T15:23:09.214805+0000 mon.a (mon.0) 2912 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.532 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[84576]: audit 2024-09-13T15:23:09.226532+0000 mon.a (mon.0) 2913 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.532 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[84576]: audit 2024-09-13T15:23:09.226532+0000 mon.a (mon.0) 2913 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.532 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[84576]: audit 2024-09-13T15:23:10.102506+0000 mon.a (mon.0) 2914 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.532 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[84576]: audit 2024-09-13T15:23:10.102506+0000 mon.a (mon.0) 2914 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.532 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[84576]: audit 2024-09-13T15:23:10.107595+0000 mon.a (mon.0) 2915 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.532 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[84576]: audit 2024-09-13T15:23:10.107595+0000 mon.a (mon.0) 2915 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[87147]: audit 2024-09-13T15:23:09.214805+0000 mon.a (mon.0) 2912 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[87147]: audit 2024-09-13T15:23:09.214805+0000 mon.a (mon.0) 2912 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[87147]: audit 2024-09-13T15:23:09.226532+0000 mon.a (mon.0) 2913 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[87147]: audit 2024-09-13T15:23:09.226532+0000 mon.a (mon.0) 2913 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[87147]: audit 2024-09-13T15:23:10.102506+0000 mon.a (mon.0) 2914 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[87147]: audit 2024-09-13T15:23:10.102506+0000 mon.a (mon.0) 2914 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[87147]: audit 2024-09-13T15:23:10.107595+0000 mon.a (mon.0) 2915 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.533 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:10 smithi099 bash[87147]: audit 2024-09-13T15:23:10.107595+0000 mon.a (mon.0) 2915 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:23:10 smithi177 bash[61439]: ts=2024-09-13T15:23:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:23:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:10 smithi177 bash[68594]: audit 2024-09-13T15:23:09.214805+0000 mon.a (mon.0) 2912 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:10 smithi177 bash[68594]: audit 2024-09-13T15:23:09.214805+0000 mon.a (mon.0) 2912 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:10 smithi177 bash[68594]: audit 2024-09-13T15:23:09.226532+0000 mon.a (mon.0) 2913 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:10 smithi177 bash[68594]: audit 2024-09-13T15:23:09.226532+0000 mon.a (mon.0) 2913 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:10 smithi177 bash[68594]: audit 2024-09-13T15:23:10.102506+0000 mon.a (mon.0) 2914 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:10 smithi177 bash[68594]: audit 2024-09-13T15:23:10.102506+0000 mon.a (mon.0) 2914 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:10 smithi177 bash[68594]: audit 2024-09-13T15:23:10.107595+0000 mon.a (mon.0) 2915 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:10.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:10 smithi177 bash[68594]: audit 2024-09-13T15:23:10.107595+0000 mon.a (mon.0) 2915 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[84576]: cluster 2024-09-13T15:23:09.348032+0000 mgr.y (mgr.26252) 1716 : cluster [DBG] pgmap v2370: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[84576]: cluster 2024-09-13T15:23:09.348032+0000 mgr.y (mgr.26252) 1716 : cluster [DBG] pgmap v2370: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[84576]: audit 2024-09-13T15:23:10.783176+0000 mon.a (mon.0) 2916 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:23:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[84576]: audit 2024-09-13T15:23:10.783176+0000 mon.a (mon.0) 2916 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:23:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[84576]: audit 2024-09-13T15:23:10.784240+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:23:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[84576]: audit 2024-09-13T15:23:10.784240+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:23:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[84576]: audit 2024-09-13T15:23:10.795921+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[84576]: audit 2024-09-13T15:23:10.795921+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:11.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[87147]: cluster 2024-09-13T15:23:09.348032+0000 mgr.y (mgr.26252) 1716 : cluster [DBG] pgmap v2370: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:11.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[87147]: cluster 2024-09-13T15:23:09.348032+0000 mgr.y (mgr.26252) 1716 : cluster [DBG] pgmap v2370: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:11.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[87147]: audit 2024-09-13T15:23:10.783176+0000 mon.a (mon.0) 2916 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:23:11.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[87147]: audit 2024-09-13T15:23:10.783176+0000 mon.a (mon.0) 2916 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:23:11.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[87147]: audit 2024-09-13T15:23:10.784240+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:23:11.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[87147]: audit 2024-09-13T15:23:10.784240+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:23:11.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[87147]: audit 2024-09-13T15:23:10.795921+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:11.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:11 smithi099 bash[87147]: audit 2024-09-13T15:23:10.795921+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:11.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:11 smithi177 bash[68594]: cluster 2024-09-13T15:23:09.348032+0000 mgr.y (mgr.26252) 1716 : cluster [DBG] pgmap v2370: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:11 smithi177 bash[68594]: cluster 2024-09-13T15:23:09.348032+0000 mgr.y (mgr.26252) 1716 : cluster [DBG] pgmap v2370: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:11 smithi177 bash[68594]: audit 2024-09-13T15:23:10.783176+0000 mon.a (mon.0) 2916 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:23:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:11 smithi177 bash[68594]: audit 2024-09-13T15:23:10.783176+0000 mon.a (mon.0) 2916 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:23:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:11 smithi177 bash[68594]: audit 2024-09-13T15:23:10.784240+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:23:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:11 smithi177 bash[68594]: audit 2024-09-13T15:23:10.784240+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:23:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:11 smithi177 bash[68594]: audit 2024-09-13T15:23:10.795921+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:11.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:11 smithi177 bash[68594]: audit 2024-09-13T15:23:10.795921+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:23:13.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:13 smithi099 bash[84576]: cluster 2024-09-13T15:23:11.348927+0000 mgr.y (mgr.26252) 1717 : cluster [DBG] pgmap v2371: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:13.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:13 smithi099 bash[84576]: cluster 2024-09-13T15:23:11.348927+0000 mgr.y (mgr.26252) 1717 : cluster [DBG] pgmap v2371: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:13.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:13 smithi099 bash[87147]: cluster 2024-09-13T15:23:11.348927+0000 mgr.y (mgr.26252) 1717 : cluster [DBG] pgmap v2371: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:13.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:13 smithi099 bash[87147]: cluster 2024-09-13T15:23:11.348927+0000 mgr.y (mgr.26252) 1717 : cluster [DBG] pgmap v2371: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:13 smithi177 bash[68594]: cluster 2024-09-13T15:23:11.348927+0000 mgr.y (mgr.26252) 1717 : cluster [DBG] pgmap v2371: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:13 smithi177 bash[68594]: cluster 2024-09-13T15:23:11.348927+0000 mgr.y (mgr.26252) 1717 : cluster [DBG] pgmap v2371: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:14.633 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:23:15.245 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T15:23:15.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[84576]: cluster 2024-09-13T15:23:13.349632+0000 mgr.y (mgr.26252) 1718 : cluster [DBG] pgmap v2372: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:15.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[84576]: cluster 2024-09-13T15:23:13.349632+0000 mgr.y (mgr.26252) 1718 : cluster [DBG] pgmap v2372: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:15.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[84576]: audit 2024-09-13T15:23:15.052476+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.99:0/1641270149' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[84576]: audit 2024-09-13T15:23:15.052476+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.99:0/1641270149' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[84576]: audit 2024-09-13T15:23:15.052995+0000 mon.a (mon.0) 2919 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.566 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[84576]: audit 2024-09-13T15:23:15.052995+0000 mon.a (mon.0) 2919 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[87147]: cluster 2024-09-13T15:23:13.349632+0000 mgr.y (mgr.26252) 1718 : cluster [DBG] pgmap v2372: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:15.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[87147]: cluster 2024-09-13T15:23:13.349632+0000 mgr.y (mgr.26252) 1718 : cluster [DBG] pgmap v2372: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:15.566 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[87147]: audit 2024-09-13T15:23:15.052476+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.99:0/1641270149' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[87147]: audit 2024-09-13T15:23:15.052476+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.99:0/1641270149' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[87147]: audit 2024-09-13T15:23:15.052995+0000 mon.a (mon.0) 2919 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.567 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:15 smithi099 bash[87147]: audit 2024-09-13T15:23:15.052995+0000 mon.a (mon.0) 2919 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:15 smithi177 bash[68594]: cluster 2024-09-13T15:23:13.349632+0000 mgr.y (mgr.26252) 1718 : cluster [DBG] pgmap v2372: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:15.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:15 smithi177 bash[68594]: cluster 2024-09-13T15:23:13.349632+0000 mgr.y (mgr.26252) 1718 : cluster [DBG] pgmap v2372: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:15.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:15 smithi177 bash[68594]: audit 2024-09-13T15:23:15.052476+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.99:0/1641270149' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:15 smithi177 bash[68594]: audit 2024-09-13T15:23:15.052476+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.99:0/1641270149' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:15 smithi177 bash[68594]: audit 2024-09-13T15:23:15.052995+0000 mon.a (mon.0) 2919 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:15.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:15 smithi177 bash[68594]: audit 2024-09-13T15:23:15.052995+0000 mon.a (mon.0) 2919 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:16.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 5 1 2024-09-13T15:23:16.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:16 smithi177 bash[68594]: audit 2024-09-13T15:23:15.236376+0000 mon.a (mon.0) 2920 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:16.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:16 smithi177 bash[68594]: audit 2024-09-13T15:23:15.236376+0000 mon.a (mon.0) 2920 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:16.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:16 smithi177 bash[68594]: cluster 2024-09-13T15:23:15.243774+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2024-09-13T15:23:16.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:16 smithi177 bash[68594]: cluster 2024-09-13T15:23:15.243774+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2024-09-13T15:23:16.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:16 smithi099 bash[84576]: audit 2024-09-13T15:23:15.236376+0000 mon.a (mon.0) 2920 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:16.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:16 smithi099 bash[84576]: audit 2024-09-13T15:23:15.236376+0000 mon.a (mon.0) 2920 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:16.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:16 smithi099 bash[84576]: cluster 2024-09-13T15:23:15.243774+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2024-09-13T15:23:16.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:16 smithi099 bash[84576]: cluster 2024-09-13T15:23:15.243774+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2024-09-13T15:23:16.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:16 smithi099 bash[87147]: audit 2024-09-13T15:23:15.236376+0000 mon.a (mon.0) 2920 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:16.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:16 smithi099 bash[87147]: audit 2024-09-13T15:23:15.236376+0000 mon.a (mon.0) 2920 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:16.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:16 smithi099 bash[87147]: cluster 2024-09-13T15:23:15.243774+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2024-09-13T15:23:16.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:16 smithi099 bash[87147]: cluster 2024-09-13T15:23:15.243774+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2024-09-13T15:23:17.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:23:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:23:17] "GET /metrics HTTP/1.1" 200 34332 "" "Prometheus/2.51.0" 2024-09-13T15:23:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:17 smithi099 bash[84576]: cluster 2024-09-13T15:23:15.350040+0000 mgr.y (mgr.26252) 1719 : cluster [DBG] pgmap v2374: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:17 smithi099 bash[84576]: cluster 2024-09-13T15:23:15.350040+0000 mgr.y (mgr.26252) 1719 : cluster [DBG] pgmap v2374: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:17 smithi099 bash[87147]: cluster 2024-09-13T15:23:15.350040+0000 mgr.y (mgr.26252) 1719 : cluster [DBG] pgmap v2374: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:17 smithi099 bash[87147]: cluster 2024-09-13T15:23:15.350040+0000 mgr.y (mgr.26252) 1719 : cluster [DBG] pgmap v2374: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:17.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:17 smithi177 bash[68594]: cluster 2024-09-13T15:23:15.350040+0000 mgr.y (mgr.26252) 1719 : cluster [DBG] pgmap v2374: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:17.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:17 smithi177 bash[68594]: cluster 2024-09-13T15:23:15.350040+0000 mgr.y (mgr.26252) 1719 : cluster [DBG] pgmap v2374: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:18.657 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:23:18.657 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:53829 46.6 46:6eaa05fc:::benchmark_data_smithi099_233884_object53828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:50.811735+0000", 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 447.84368726299999, 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.48090934099999999, 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.658 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 53829 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811735+0000", 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811735+0000", 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811734+0000", 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811796+0000", 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2223000000000004e-05 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811811+0000", 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4396000000000001e-05 2024-09-13T15:23:18.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811817+0000", 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8290000000000002e-06 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811864+0000", 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6749999999999998e-05 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.811975+0000", 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110584 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.812070+0000", 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5697000000000002e-05 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.813604+0000", 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015338890000000001 2024-09-13T15:23:18.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.814008+0000", 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00040333300000000002 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.292548+0000", 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.478540309 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.292599+0000", 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1319000000000001e-05 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.292645+0000", 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5453e-05 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:88714 46.5 e1106/1099 46:ab9030a0:::benchmark_data_smithi099_233884_object88713:head v 1106'5491, mlcod=1106'5487)", 2024-09-13T15:23:18.664 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:30.090741+0000", 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 408.56468167399998, 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.524941883, 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090741+0000", 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090741+0000", 2024-09-13T15:23:18.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090740+0000", 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090803+0000", 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2986999999999999e-05 2024-09-13T15:23:18.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090813+0000", 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4220000000000006e-06 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090818+0000", 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9590000000000003e-06 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.090870+0000", 2024-09-13T15:23:18.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2339999999999997e-05 2024-09-13T15:23:18.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585133+0000", 2024-09-13T15:23:18.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.494263491 2024-09-13T15:23:18.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.615617+0000", 2024-09-13T15:23:18.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.030484003999999999 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.615683+0000", 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5455000000000003e-05 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:88711 46.e e1106/1099 46:768b3877:::benchmark_data_smithi099_233884_object88710:head v 1106'5512, mlcod=1106'5509)", 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:30.092814+0000", 2024-09-13T15:23:18.669 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 408.56260879199999, 2024-09-13T15:23:18.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50584367299999999, 2024-09-13T15:23:18.670 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.670 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.670 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.092814+0000", 2024-09-13T15:23:18.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.092814+0000", 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.092813+0000", 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.092886+0000", 2024-09-13T15:23:18.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3215999999999998e-05 2024-09-13T15:23:18.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585069+0000", 2024-09-13T15:23:18.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.492183174 2024-09-13T15:23:18.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585100+0000", 2024-09-13T15:23:18.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0576e-05 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585401+0000", 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000301434 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585471+0000", 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9888000000000006e-05 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.598592+0000", 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013120427 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.598657+0000", 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5864000000000005e-05 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:88715 46.5 e1106/1099 46:ad92bd9f:::benchmark_data_smithi099_233884_object88714:head v 1106'5492, mlcod=1106'5487)", 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:30.093743+0000", 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 408.56167929100002, 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.521982485, 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.093743+0000", 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.093743+0000", 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.093742+0000", 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.093855+0000", 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011267199999999999 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585142+0000", 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49128675799999999 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.585161+0000", 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8848999999999999e-05 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.591914+0000", 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067532299999999998 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.591962+0000", 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8696000000000003e-05 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.615696+0000", 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023733204000000001 2024-09-13T15:23:18.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:30.615726+0000", 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0085000000000001e-05 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89298 46.6 46:618b79f4:::benchmark_data_smithi099_233884_object89297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.716825+0000", 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 406.93859715299999, 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.033399011, 2024-09-13T15:23:18.679 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89298 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716825+0000", 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716825+0000", 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716824+0000", 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716876+0000", 2024-09-13T15:23:18.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2182000000000003e-05 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716888+0000", 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1572e-05 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716893+0000", 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9559999999999996e-06 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.974099+0000", 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25720631100000002 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.974111+0000", 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1948e-05 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.750142+0000", 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.77603100599999997 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.750224+0000", 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2245999999999998e-05 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89316 46.16 46:6f82b5f7:::benchmark_data_smithi099_233884_object89315:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.744659+0000", 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 406.910763234, 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0223355249999999, 2024-09-13T15:23:18.684 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89316 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744659+0000", 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744659+0000", 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744657+0000", 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744719+0000", 2024-09-13T15:23:18.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2253000000000002e-05 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744729+0000", 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0032e-05 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.744735+0000", 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3159999999999996e-06 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.974133+0000", 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.229398243 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.974138+0000", 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9359999999999998e-06 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.759984+0000", 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.78584600699999996 2024-09-13T15:23:18.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.760180+0000", 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019581500000000001 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.760329+0000", 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014914099999999999 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.762793+0000", 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024643849999999999 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.763201+0000", 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00040765000000000002 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.766794+0000", 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035933530000000001 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.690 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.766873+0000", 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8777999999999997e-05 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.766995+0000", 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012185599999999999 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.691 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:106833 46.6 46:67c3ca66:::benchmark_data_smithi099_233884_object4420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:55.476414+0000", 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.179008904, 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.54527072499999996, 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 106833 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476414+0000", 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476414+0000", 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476413+0000", 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476418+0000", 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4539999999999998e-06 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476428+0000", 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5019999999999998e-06 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476432+0000", 2024-09-13T15:23:18.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0369999999999996e-06 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476464+0000", 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2116000000000002e-05 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476537+0000", 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3589000000000003e-05 2024-09-13T15:23:18.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476599+0000", 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1511000000000001e-05 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.014653+0000", 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53805408799999999 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.015286+0000", 2024-09-13T15:23:18.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00063307900000000004 2024-09-13T15:23:18.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:18.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.021378+0000", 2024-09-13T15:23:18.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0060921739999999997 2024-09-13T15:23:18.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.021598+0000", 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000219805 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.021684+0000", 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6452000000000005e-05 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:106834 46.6 46:6537e07b:::benchmark_data_smithi099_233884_object4421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:55.476755+0000", 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.178667265, 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.550843686, 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 106834 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476755+0000", 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476755+0000", 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476754+0000", 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.476759+0000", 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3239999999999996e-06 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.011921+0000", 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53516227000000005 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.011949+0000", 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8e-05 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.015361+0000", 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034122660000000002 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.015696+0000", 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033479700000000001 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.016024+0000", 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032819400000000002 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.022670+0000", 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0066458239999999998 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.025977+0000", 2024-09-13T15:23:18.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033068469999999999 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.027421+0000", 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001443843 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.027521+0000", 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9921999999999996e-05 2024-09-13T15:23:18.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.027599+0000", 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8373000000000003e-05 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop_reply(client.46109.0:106833 46.6 e1117/1108 ondisk, result = 0)", 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:55.481564+0000", 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.17385823299998, 2024-09-13T15:23:18.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53309143999999997, 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.481564+0000", 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.481564+0000", 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.481563+0000", 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.481570+0000", 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5270000000000004e-06 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.011927+0000", 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53035680299999999 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.011946+0000", 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9164000000000002e-05 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.014630+0000", 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026842749999999999 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.014643+0000", 2024-09-13T15:23:18.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2838e-05 2024-09-13T15:23:18.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.014656+0000", 2024-09-13T15:23:18.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2927e-05 2024-09-13T15:23:18.712 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.712 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.712 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:106854 46.e e1117/1099 46:721a13d8:::benchmark_data_smithi099_233884_object4441:head v 1117'6661, mlcod=1117'6659)", 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:55.510672+0000", 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.14475085300001, 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51700354100000001, 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.510672+0000", 2024-09-13T15:23:18.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.510672+0000", 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.510671+0000", 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:55.510681+0000", 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0917999999999999e-05 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.011944+0000", 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50126216499999998 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.011963+0000", 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9428000000000001e-05 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.014691+0000", 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027280849999999999 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.017667+0000", 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029754569999999999 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.027610+0000", 2024-09-13T15:23:18.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0099434789999999999 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:56.027675+0000", 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5217999999999998e-05 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:118865 46.6 46:64f49ea3:::benchmark_data_smithi099_233884_object16452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1122)", 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:17:14.342560+0000", 2024-09-13T15:23:18.717 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 364.31286296399998, 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43666500699999999, 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 118865 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.342560+0000", 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.342560+0000", 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.342558+0000", 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:18.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.342565+0000", 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7479999999999996e-06 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.342578+0000", 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3541e-05 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.342586+0000", 2024-09-13T15:23:18.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2280000000000001e-06 2024-09-13T15:23:18.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.342626+0000", 2024-09-13T15:23:18.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0488999999999998e-05 2024-09-13T15:23:18.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.342753+0000", 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012651399999999999 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.342865+0000", 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011236 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.344884+0000", 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020195059999999999 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.350504+0000", 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056196780000000003 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.779142+0000", 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.42863763799999999 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.779188+0000", 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6368000000000001e-05 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.779225+0000", 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6516000000000001e-05 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:118875 46.6 46:65dedaa9:::benchmark_data_smithi099_233884_object16462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1122)", 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:17:14.365164+0000", 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 364.29025809000001, 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.419219385, 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.724 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 118875 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.365164+0000", 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.365164+0000", 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.365162+0000", 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999971 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.365176+0000", 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4423000000000001e-05 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.365215+0000", 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8732000000000001e-05 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.365231+0000", 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6623e-05 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.365342+0000", 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110229 2024-09-13T15:23:18.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.365456+0000", 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011395699999999999 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.365570+0000", 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000114666 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:18.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.367614+0000", 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020441589999999998 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.368835+0000", 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001220685 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.784292+0000", 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.415456885 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.784343+0000", 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0559000000000002e-05 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:17:14.784384+0000", 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1329999999999999e-05 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.730 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:167688 46.2 e1130/1108 46:4177f5f0:::benchmark_data_smithi099_233884_object65275:head v 1130'7993, mlcod=1130'7991)", 2024-09-13T15:23:18.731 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:15.331314+0000", 2024-09-13T15:23:18.731 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 303.32410906600001, 2024-09-13T15:23:18.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.063329677000000001, 2024-09-13T15:23:18.731 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.731 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.731 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.331314+0000", 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.331314+0000", 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.331313+0000", 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.331322+0000", 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3300000000000005e-06 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.331332+0000", 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5429999999999999e-06 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.331336+0000", 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5419999999999999e-06 2024-09-13T15:23:18.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.332421+0000", 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010848819999999999 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.332441+0000", 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0321000000000001e-05 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.338089+0000", 2024-09-13T15:23:18.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056478320000000002 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394643+0000", 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.056553990999999998 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:167691 46.1 e1130/1108 46:8396d1a2:::benchmark_data_smithi099_233884_object65278:head v 1130'8087, mlcod=1130'8085)", 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:15.337363+0000", 2024-09-13T15:23:18.735 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 303.31805920699998, 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.069275861999999994, 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.337363+0000", 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.337363+0000", 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.337362+0000", 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.337370+0000", 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3839999999999999e-06 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.337382+0000", 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2367e-05 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394591+0000", 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.057209152999999999 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394711+0000", 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120031 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394762+0000", 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0521e-05 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.406209+0000", 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011446952 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.406639+0000", 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00043040899999999998 2024-09-13T15:23:18.739 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:167696 46.e e1130/1099 46:75475f00:::benchmark_data_smithi099_233884_object65283:head v 1130'10421, mlcod=1130'10418)", 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:15.338159+0000", 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 303.31726354400001, 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.074707143000000004, 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.338159+0000", 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.338159+0000", 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.338158+0000", 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.338166+0000", 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.396e-06 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394602+0000", 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.056435998000000001 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394612+0000", 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0189e-05 2024-09-13T15:23:18.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394694+0000", 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1743000000000007e-05 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394721+0000", 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7835e-05 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.412634+0000", 2024-09-13T15:23:18.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017912575999999999 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.412866+0000", 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023224199999999999 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46109.0:167690 46.5 e1130/1108 46:a60ea2bd:::benchmark_data_smithi099_233884_object65277:head v 1130'7926, mlcod=1130'7924)", 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:15.339865+0000", 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 303.31555783699997, 2024-09-13T15:23:18.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.06262152, 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.339865+0000", 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.339865+0000", 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.339864+0000", 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.339872+0000", 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2910000000000007e-06 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394576+0000", 2024-09-13T15:23:18.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.054703947000000003 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394596+0000", 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9723000000000001e-05 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394664+0000", 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7669999999999994e-05 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.394699+0000", 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5577000000000002e-05 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.402307+0000", 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0076075960000000003 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:15.402486+0000", 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017955 2024-09-13T15:23:18.748 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:173455 46.16 46:6dd2dfb2:::benchmark_data_smithi099_233884_object71042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:22.035116+0000", 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 296.62030668099999, 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.062596133999999998, 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.749 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 173455 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.035116+0000", 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.035116+0000", 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.035115+0000", 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.035120+0000", 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6130000000000003e-06 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.035133+0000", 2024-09-13T15:23:18.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2993000000000001e-05 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.035139+0000", 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5589999999999996e-06 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.035185+0000", 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6749999999999998e-05 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.035304+0000", 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000118712 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.035414+0000", 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110147 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.037875+0000", 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024609969999999999 2024-09-13T15:23:18.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.041878+0000", 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0040022729999999998 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.097568+0000", 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.055690293000000002 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.754 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.097641+0000", 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2869999999999999e-05 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:22.097712+0000", 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1332000000000001e-05 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:184492 46.16 46:6931773b:::benchmark_data_smithi099_233884_object82079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:23:18.755 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:36.565279+0000", 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 282.09014382100003, 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.483143404, 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 184492 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.565279+0000", 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.565279+0000", 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.565277+0000", 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:23:18.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.565286+0000", 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0899999999999994e-06 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.565313+0000", 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6497000000000001e-05 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.565323+0000", 2024-09-13T15:23:18.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0855e-05 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.565396+0000", 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3095000000000004e-05 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.565562+0000", 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016553 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:23:18.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.570583+0000", 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050210899999999998 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.573712+0000", 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031291589999999998 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.585025+0000", 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011312809 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.048213+0000", 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.46318805899999999 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.048307+0000", 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3746999999999995e-05 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.048422+0000", 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011532 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:184509 46.6 46:64608131:::benchmark_data_smithi099_233884_object82096:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:36.588218+0000", 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 282.06720466500002, 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.47171628999999998, 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 184509 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.762 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.588218+0000", 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.588218+0000", 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.588216+0000", 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:23:18.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.588226+0000", 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8579999999999995e-06 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.588241+0000", 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5335000000000001e-05 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.588248+0000", 2024-09-13T15:23:18.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7549999999999997e-06 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.588307+0000", 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8619000000000002e-05 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.588414+0000", 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010736800000000001 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.588910+0000", 2024-09-13T15:23:18.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000495792 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.592046+0000", 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031358990000000002 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.592054+0000", 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7770000000000002e-06 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.059758+0000", 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.46770455 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.059857+0000", 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9284999999999996e-05 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.059934+0000", 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6804000000000005e-05 2024-09-13T15:23:18.767 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:184517 46.6 46:64498f8a:::benchmark_data_smithi099_233884_object82104:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:36.596913+0000", 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 282.05850921899997, 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.46350879499999997, 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:18.768 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 184517 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.596913+0000", 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:18.769 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.596913+0000", 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.596912+0000", 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.596918+0000", 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5390000000000004e-06 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.770 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:18.771 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.596932+0000", 2024-09-13T15:23:18.771 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3903000000000001e-05 2024-09-13T15:23:18.771 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.771 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.771 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:18.771 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.596939+0000", 2024-09-13T15:23:18.771 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2430000000000002e-06 2024-09-13T15:23:18.771 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.771 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.771 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.597002+0000", 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2961999999999998e-05 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.597118+0000", 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116877 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:23:18.772 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.597649+0000", 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00053034200000000005 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.600325+0000", 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026765780000000002 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.600468+0000", 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000142359 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.773 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.060003+0000", 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45953522499999999 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.060345+0000", 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00034193500000000003 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.060422+0000", 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7328000000000002e-05 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.774 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.775 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.775 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:18.775 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:18.775 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:23:19.412 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:23:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:19 smithi099 bash[84576]: cluster 2024-09-13T15:23:17.350923+0000 mgr.y (mgr.26252) 1720 : cluster [DBG] pgmap v2375: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:19 smithi099 bash[84576]: cluster 2024-09-13T15:23:17.350923+0000 mgr.y (mgr.26252) 1720 : cluster [DBG] pgmap v2375: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:19 smithi099 bash[84576]: audit 2024-09-13T15:23:18.330067+0000 mon.a (mon.0) 2922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:19 smithi099 bash[84576]: audit 2024-09-13T15:23:18.330067+0000 mon.a (mon.0) 2922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:19 smithi099 bash[87147]: cluster 2024-09-13T15:23:17.350923+0000 mgr.y (mgr.26252) 1720 : cluster [DBG] pgmap v2375: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:19 smithi099 bash[87147]: cluster 2024-09-13T15:23:17.350923+0000 mgr.y (mgr.26252) 1720 : cluster [DBG] pgmap v2375: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:19 smithi099 bash[87147]: audit 2024-09-13T15:23:18.330067+0000 mon.a (mon.0) 2922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:19 smithi099 bash[87147]: audit 2024-09-13T15:23:18.330067+0000 mon.a (mon.0) 2922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:19 smithi177 bash[68594]: cluster 2024-09-13T15:23:17.350923+0000 mgr.y (mgr.26252) 1720 : cluster [DBG] pgmap v2375: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:19 smithi177 bash[68594]: cluster 2024-09-13T15:23:17.350923+0000 mgr.y (mgr.26252) 1720 : cluster [DBG] pgmap v2375: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:19 smithi177 bash[68594]: audit 2024-09-13T15:23:18.330067+0000 mon.a (mon.0) 2922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:19 smithi177 bash[68594]: audit 2024-09-13T15:23:18.330067+0000 mon.a (mon.0) 2922 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:20.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:23:20 smithi177 bash[61439]: ts=2024-09-13T15:23:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:23:21.017 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:23:21.476 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:21 smithi099 bash[84576]: cluster 2024-09-13T15:23:19.351819+0000 mgr.y (mgr.26252) 1721 : cluster [DBG] pgmap v2376: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:21.476 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:21 smithi099 bash[87147]: cluster 2024-09-13T15:23:19.351819+0000 mgr.y (mgr.26252) 1721 : cluster [DBG] pgmap v2376: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:21.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:21 smithi177 bash[68594]: cluster 2024-09-13T15:23:19.351819+0000 mgr.y (mgr.26252) 1721 : cluster [DBG] pgmap v2376: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:21.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:21 smithi177 bash[68594]: cluster 2024-09-13T15:23:19.351819+0000 mgr.y (mgr.26252) 1721 : cluster [DBG] pgmap v2376: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:21.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:21 smithi099 bash[84576]: cluster 2024-09-13T15:23:19.351819+0000 mgr.y (mgr.26252) 1721 : cluster [DBG] pgmap v2376: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:21.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:21 smithi099 bash[87147]: cluster 2024-09-13T15:23:19.351819+0000 mgr.y (mgr.26252) 1721 : cluster [DBG] pgmap v2376: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:23.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:23 smithi099 bash[87147]: cluster 2024-09-13T15:23:21.352576+0000 mgr.y (mgr.26252) 1722 : cluster [DBG] pgmap v2377: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:23.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:23 smithi099 bash[87147]: cluster 2024-09-13T15:23:21.352576+0000 mgr.y (mgr.26252) 1722 : cluster [DBG] pgmap v2377: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:23.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:23 smithi099 bash[84576]: cluster 2024-09-13T15:23:21.352576+0000 mgr.y (mgr.26252) 1722 : cluster [DBG] pgmap v2377: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:23.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:23 smithi099 bash[84576]: cluster 2024-09-13T15:23:21.352576+0000 mgr.y (mgr.26252) 1722 : cluster [DBG] pgmap v2377: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:23.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:23 smithi177 bash[68594]: cluster 2024-09-13T15:23:21.352576+0000 mgr.y (mgr.26252) 1722 : cluster [DBG] pgmap v2377: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:23.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:23 smithi177 bash[68594]: cluster 2024-09-13T15:23:21.352576+0000 mgr.y (mgr.26252) 1722 : cluster [DBG] pgmap v2377: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:24.214 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:23:25.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:25 smithi099 bash[84576]: cluster 2024-09-13T15:23:23.353544+0000 mgr.y (mgr.26252) 1723 : cluster [DBG] pgmap v2378: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:25.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:25 smithi099 bash[84576]: cluster 2024-09-13T15:23:23.353544+0000 mgr.y (mgr.26252) 1723 : cluster [DBG] pgmap v2378: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:25 smithi099 bash[87147]: cluster 2024-09-13T15:23:23.353544+0000 mgr.y (mgr.26252) 1723 : cluster [DBG] pgmap v2378: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:25 smithi099 bash[87147]: cluster 2024-09-13T15:23:23.353544+0000 mgr.y (mgr.26252) 1723 : cluster [DBG] pgmap v2378: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:25 smithi177 bash[68594]: cluster 2024-09-13T15:23:23.353544+0000 mgr.y (mgr.26252) 1723 : cluster [DBG] pgmap v2378: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:25 smithi177 bash[68594]: cluster 2024-09-13T15:23:23.353544+0000 mgr.y (mgr.26252) 1723 : cluster [DBG] pgmap v2378: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:26.285 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:26 smithi177 bash[68594]: audit 2024-09-13T15:23:25.526923+0000 mon.c (mon.2) 136 : audit [INF] from='client.? 172.21.15.99:0/1404678147' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.285 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:26 smithi177 bash[68594]: audit 2024-09-13T15:23:25.526923+0000 mon.c (mon.2) 136 : audit [INF] from='client.? 172.21.15.99:0/1404678147' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.291 INFO:teuthology.orchestra.run.smithi099.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-09-13T15:23:26.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:26 smithi099 bash[84576]: audit 2024-09-13T15:23:25.526923+0000 mon.c (mon.2) 136 : audit [INF] from='client.? 172.21.15.99:0/1404678147' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:26 smithi099 bash[84576]: audit 2024-09-13T15:23:25.526923+0000 mon.c (mon.2) 136 : audit [INF] from='client.? 172.21.15.99:0/1404678147' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:26 smithi099 bash[84576]: audit 2024-09-13T15:23:25.527420+0000 mon.a (mon.0) 2923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:26 smithi099 bash[84576]: audit 2024-09-13T15:23:25.527420+0000 mon.a (mon.0) 2923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:26 smithi099 bash[87147]: audit 2024-09-13T15:23:25.526923+0000 mon.c (mon.2) 136 : audit [INF] from='client.? 172.21.15.99:0/1404678147' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:26 smithi099 bash[87147]: audit 2024-09-13T15:23:25.526923+0000 mon.c (mon.2) 136 : audit [INF] from='client.? 172.21.15.99:0/1404678147' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:26 smithi099 bash[87147]: audit 2024-09-13T15:23:25.527420+0000 mon.a (mon.0) 2923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:26 smithi099 bash[87147]: audit 2024-09-13T15:23:25.527420+0000 mon.a (mon.0) 2923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:26 smithi177 bash[68594]: audit 2024-09-13T15:23:25.527420+0000 mon.a (mon.0) 2923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:26.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:26 smithi177 bash[68594]: audit 2024-09-13T15:23:25.527420+0000 mon.a (mon.0) 2923 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:23:27.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:23:27] "GET /metrics HTTP/1.1" 200 34331 "" "Prometheus/2.51.0" 2024-09-13T15:23:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[84576]: cluster 2024-09-13T15:23:25.354048+0000 mgr.y (mgr.26252) 1724 : cluster [DBG] pgmap v2379: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[84576]: cluster 2024-09-13T15:23:25.354048+0000 mgr.y (mgr.26252) 1724 : cluster [DBG] pgmap v2379: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[84576]: audit 2024-09-13T15:23:26.280806+0000 mon.a (mon.0) 2924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[84576]: audit 2024-09-13T15:23:26.280806+0000 mon.a (mon.0) 2924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[84576]: cluster 2024-09-13T15:23:26.289324+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2024-09-13T15:23:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[84576]: cluster 2024-09-13T15:23:26.289324+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2024-09-13T15:23:27.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[87147]: cluster 2024-09-13T15:23:25.354048+0000 mgr.y (mgr.26252) 1724 : cluster [DBG] pgmap v2379: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:27.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[87147]: cluster 2024-09-13T15:23:25.354048+0000 mgr.y (mgr.26252) 1724 : cluster [DBG] pgmap v2379: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:27.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[87147]: audit 2024-09-13T15:23:26.280806+0000 mon.a (mon.0) 2924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:27.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[87147]: audit 2024-09-13T15:23:26.280806+0000 mon.a (mon.0) 2924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:27.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[87147]: cluster 2024-09-13T15:23:26.289324+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2024-09-13T15:23:27.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:27 smithi099 bash[87147]: cluster 2024-09-13T15:23:26.289324+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2024-09-13T15:23:27.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T15:23:27.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:27 smithi177 bash[68594]: cluster 2024-09-13T15:23:25.354048+0000 mgr.y (mgr.26252) 1724 : cluster [DBG] pgmap v2379: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:27.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:27 smithi177 bash[68594]: cluster 2024-09-13T15:23:25.354048+0000 mgr.y (mgr.26252) 1724 : cluster [DBG] pgmap v2379: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:27.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:27 smithi177 bash[68594]: audit 2024-09-13T15:23:26.280806+0000 mon.a (mon.0) 2924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:27.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:27 smithi177 bash[68594]: audit 2024-09-13T15:23:26.280806+0000 mon.a (mon.0) 2924 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:23:27.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:27 smithi177 bash[68594]: cluster 2024-09-13T15:23:26.289324+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2024-09-13T15:23:27.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:27 smithi177 bash[68594]: cluster 2024-09-13T15:23:26.289324+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2024-09-13T15:23:28.290 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:23:28.291 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:23:28.291 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2024-09-13T15:23:28.291 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:23:29.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:23:29.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:29 smithi177 bash[68594]: cluster 2024-09-13T15:23:27.354690+0000 mgr.y (mgr.26252) 1725 : cluster [DBG] pgmap v2381: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:29.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:29 smithi177 bash[68594]: cluster 2024-09-13T15:23:27.354690+0000 mgr.y (mgr.26252) 1725 : cluster [DBG] pgmap v2381: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:29.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:29 smithi099 bash[84576]: cluster 2024-09-13T15:23:27.354690+0000 mgr.y (mgr.26252) 1725 : cluster [DBG] pgmap v2381: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:29.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:29 smithi099 bash[84576]: cluster 2024-09-13T15:23:27.354690+0000 mgr.y (mgr.26252) 1725 : cluster [DBG] pgmap v2381: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:29.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:29 smithi099 bash[87147]: cluster 2024-09-13T15:23:27.354690+0000 mgr.y (mgr.26252) 1725 : cluster [DBG] pgmap v2381: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:29.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:29 smithi099 bash[87147]: cluster 2024-09-13T15:23:27.354690+0000 mgr.y (mgr.26252) 1725 : cluster [DBG] pgmap v2381: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:30.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:23:30 smithi177 bash[61439]: ts=2024-09-13T15:23:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:23:31.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:31 smithi099 bash[84576]: cluster 2024-09-13T15:23:29.355552+0000 mgr.y (mgr.26252) 1726 : cluster [DBG] pgmap v2382: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:31.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:31 smithi099 bash[84576]: cluster 2024-09-13T15:23:29.355552+0000 mgr.y (mgr.26252) 1726 : cluster [DBG] pgmap v2382: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:31.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:31 smithi099 bash[87147]: cluster 2024-09-13T15:23:29.355552+0000 mgr.y (mgr.26252) 1726 : cluster [DBG] pgmap v2382: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:31.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:31 smithi099 bash[87147]: cluster 2024-09-13T15:23:29.355552+0000 mgr.y (mgr.26252) 1726 : cluster [DBG] pgmap v2382: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:31.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:31 smithi177 bash[68594]: cluster 2024-09-13T15:23:29.355552+0000 mgr.y (mgr.26252) 1726 : cluster [DBG] pgmap v2382: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:31.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:31 smithi177 bash[68594]: cluster 2024-09-13T15:23:29.355552+0000 mgr.y (mgr.26252) 1726 : cluster [DBG] pgmap v2382: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:32.117 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:23:33.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:33 smithi099 bash[84576]: cluster 2024-09-13T15:23:31.356319+0000 mgr.y (mgr.26252) 1727 : cluster [DBG] pgmap v2383: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:33.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:33 smithi099 bash[84576]: cluster 2024-09-13T15:23:31.356319+0000 mgr.y (mgr.26252) 1727 : cluster [DBG] pgmap v2383: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:33.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:33 smithi099 bash[87147]: cluster 2024-09-13T15:23:31.356319+0000 mgr.y (mgr.26252) 1727 : cluster [DBG] pgmap v2383: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:33.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:33 smithi099 bash[87147]: cluster 2024-09-13T15:23:31.356319+0000 mgr.y (mgr.26252) 1727 : cluster [DBG] pgmap v2383: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:33.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:33 smithi177 bash[68594]: cluster 2024-09-13T15:23:31.356319+0000 mgr.y (mgr.26252) 1727 : cluster [DBG] pgmap v2383: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:33.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:33 smithi177 bash[68594]: cluster 2024-09-13T15:23:31.356319+0000 mgr.y (mgr.26252) 1727 : cluster [DBG] pgmap v2383: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:33.843 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:23:34.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:34 smithi099 bash[84576]: audit 2024-09-13T15:23:33.330609+0000 mon.a (mon.0) 2926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:34.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:34 smithi099 bash[84576]: audit 2024-09-13T15:23:33.330609+0000 mon.a (mon.0) 2926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:34 smithi099 bash[87147]: audit 2024-09-13T15:23:33.330609+0000 mon.a (mon.0) 2926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:34.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:34 smithi099 bash[87147]: audit 2024-09-13T15:23:33.330609+0000 mon.a (mon.0) 2926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:34.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:34 smithi177 bash[68594]: audit 2024-09-13T15:23:33.330609+0000 mon.a (mon.0) 2926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:34.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:34 smithi177 bash[68594]: audit 2024-09-13T15:23:33.330609+0000 mon.a (mon.0) 2926 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:35.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:35 smithi099 bash[84576]: cluster 2024-09-13T15:23:33.362842+0000 mgr.y (mgr.26252) 1728 : cluster [DBG] pgmap v2384: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:35.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:35 smithi099 bash[84576]: cluster 2024-09-13T15:23:33.362842+0000 mgr.y (mgr.26252) 1728 : cluster [DBG] pgmap v2384: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:35.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:35 smithi099 bash[87147]: cluster 2024-09-13T15:23:33.362842+0000 mgr.y (mgr.26252) 1728 : cluster [DBG] pgmap v2384: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:35.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:35 smithi099 bash[87147]: cluster 2024-09-13T15:23:33.362842+0000 mgr.y (mgr.26252) 1728 : cluster [DBG] pgmap v2384: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:35.598 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:35 smithi177 bash[68594]: cluster 2024-09-13T15:23:33.362842+0000 mgr.y (mgr.26252) 1728 : cluster [DBG] pgmap v2384: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:35.598 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:35 smithi177 bash[68594]: cluster 2024-09-13T15:23:33.362842+0000 mgr.y (mgr.26252) 1728 : cluster [DBG] pgmap v2384: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:37.323 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T15:23:37.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:23:37] "GET /metrics HTTP/1.1" 200 34330 "" "Prometheus/2.51.0" 2024-09-13T15:23:37.577 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:23:37.577 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:23:37.577 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:23:37.577 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:23:37.578 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:23:37.586 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:37 smithi177 bash[68594]: cluster 2024-09-13T15:23:35.363626+0000 mgr.y (mgr.26252) 1729 : cluster [DBG] pgmap v2385: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:37.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:37 smithi177 bash[68594]: cluster 2024-09-13T15:23:35.363626+0000 mgr.y (mgr.26252) 1729 : cluster [DBG] pgmap v2385: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:37.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:37 smithi177 bash[68594]: audit 2024-09-13T15:23:36.735236+0000 mon.c (mon.2) 137 : audit [INF] from='client.? 172.21.15.99:0/3023046248' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:37 smithi177 bash[68594]: audit 2024-09-13T15:23:36.735236+0000 mon.c (mon.2) 137 : audit [INF] from='client.? 172.21.15.99:0/3023046248' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:37 smithi177 bash[68594]: audit 2024-09-13T15:23:36.735697+0000 mon.a (mon.0) 2927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:37 smithi177 bash[68594]: audit 2024-09-13T15:23:36.735697+0000 mon.a (mon.0) 2927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[84576]: cluster 2024-09-13T15:23:35.363626+0000 mgr.y (mgr.26252) 1729 : cluster [DBG] pgmap v2385: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:37.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[84576]: cluster 2024-09-13T15:23:35.363626+0000 mgr.y (mgr.26252) 1729 : cluster [DBG] pgmap v2385: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:37.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[84576]: audit 2024-09-13T15:23:36.735236+0000 mon.c (mon.2) 137 : audit [INF] from='client.? 172.21.15.99:0/3023046248' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[84576]: audit 2024-09-13T15:23:36.735236+0000 mon.c (mon.2) 137 : audit [INF] from='client.? 172.21.15.99:0/3023046248' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[84576]: audit 2024-09-13T15:23:36.735697+0000 mon.a (mon.0) 2927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.660 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[84576]: audit 2024-09-13T15:23:36.735697+0000 mon.a (mon.0) 2927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.661 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[87147]: cluster 2024-09-13T15:23:35.363626+0000 mgr.y (mgr.26252) 1729 : cluster [DBG] pgmap v2385: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:37.661 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[87147]: cluster 2024-09-13T15:23:35.363626+0000 mgr.y (mgr.26252) 1729 : cluster [DBG] pgmap v2385: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:37.661 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[87147]: audit 2024-09-13T15:23:36.735236+0000 mon.c (mon.2) 137 : audit [INF] from='client.? 172.21.15.99:0/3023046248' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.661 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[87147]: audit 2024-09-13T15:23:36.735236+0000 mon.c (mon.2) 137 : audit [INF] from='client.? 172.21.15.99:0/3023046248' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.661 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[87147]: audit 2024-09-13T15:23:36.735697+0000 mon.a (mon.0) 2927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:37.661 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:37 smithi099 bash[87147]: audit 2024-09-13T15:23:36.735697+0000 mon.a (mon.0) 2927 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:38.380 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 6 1 2024-09-13T15:23:38.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:23:38.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:38 smithi177 bash[68594]: audit 2024-09-13T15:23:37.314829+0000 mon.a (mon.0) 2928 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:38.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:38 smithi177 bash[68594]: audit 2024-09-13T15:23:37.314829+0000 mon.a (mon.0) 2928 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:38.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:38 smithi177 bash[68594]: cluster 2024-09-13T15:23:37.321896+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2024-09-13T15:23:38.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:38 smithi177 bash[68594]: cluster 2024-09-13T15:23:37.321896+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2024-09-13T15:23:38.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:38 smithi099 bash[84576]: audit 2024-09-13T15:23:37.314829+0000 mon.a (mon.0) 2928 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:38.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:38 smithi099 bash[84576]: audit 2024-09-13T15:23:37.314829+0000 mon.a (mon.0) 2928 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:38.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:38 smithi099 bash[84576]: cluster 2024-09-13T15:23:37.321896+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2024-09-13T15:23:38.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:38 smithi099 bash[84576]: cluster 2024-09-13T15:23:37.321896+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2024-09-13T15:23:38.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:38 smithi099 bash[87147]: audit 2024-09-13T15:23:37.314829+0000 mon.a (mon.0) 2928 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:38.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:38 smithi099 bash[87147]: audit 2024-09-13T15:23:37.314829+0000 mon.a (mon.0) 2928 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:38.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:38 smithi099 bash[87147]: cluster 2024-09-13T15:23:37.321896+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2024-09-13T15:23:38.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:38 smithi099 bash[87147]: cluster 2024-09-13T15:23:37.321896+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2024-09-13T15:23:39.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:39 smithi177 bash[68594]: cluster 2024-09-13T15:23:37.364163+0000 mgr.y (mgr.26252) 1730 : cluster [DBG] pgmap v2387: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:39.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:39 smithi177 bash[68594]: cluster 2024-09-13T15:23:37.364163+0000 mgr.y (mgr.26252) 1730 : cluster [DBG] pgmap v2387: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:39.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:39 smithi099 bash[84576]: cluster 2024-09-13T15:23:37.364163+0000 mgr.y (mgr.26252) 1730 : cluster [DBG] pgmap v2387: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:39.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:39 smithi099 bash[84576]: cluster 2024-09-13T15:23:37.364163+0000 mgr.y (mgr.26252) 1730 : cluster [DBG] pgmap v2387: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:39.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:39 smithi099 bash[87147]: cluster 2024-09-13T15:23:37.364163+0000 mgr.y (mgr.26252) 1730 : cluster [DBG] pgmap v2387: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:39.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:39 smithi099 bash[87147]: cluster 2024-09-13T15:23:37.364163+0000 mgr.y (mgr.26252) 1730 : cluster [DBG] pgmap v2387: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:40.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:23:40 smithi177 bash[61439]: ts=2024-09-13T15:23:40.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:23:41.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:41 smithi177 bash[68594]: cluster 2024-09-13T15:23:39.365004+0000 mgr.y (mgr.26252) 1731 : cluster [DBG] pgmap v2388: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:41.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:41 smithi177 bash[68594]: cluster 2024-09-13T15:23:39.365004+0000 mgr.y (mgr.26252) 1731 : cluster [DBG] pgmap v2388: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:41.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:41 smithi099 bash[84576]: cluster 2024-09-13T15:23:39.365004+0000 mgr.y (mgr.26252) 1731 : cluster [DBG] pgmap v2388: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:41.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:41 smithi099 bash[84576]: cluster 2024-09-13T15:23:39.365004+0000 mgr.y (mgr.26252) 1731 : cluster [DBG] pgmap v2388: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:41.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:41 smithi099 bash[87147]: cluster 2024-09-13T15:23:39.365004+0000 mgr.y (mgr.26252) 1731 : cluster [DBG] pgmap v2388: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:41.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:41 smithi099 bash[87147]: cluster 2024-09-13T15:23:39.365004+0000 mgr.y (mgr.26252) 1731 : cluster [DBG] pgmap v2388: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:42.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:42 smithi099 bash[84576]: cluster 2024-09-13T15:23:41.365768+0000 mgr.y (mgr.26252) 1732 : cluster [DBG] pgmap v2389: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:42.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:42 smithi099 bash[84576]: cluster 2024-09-13T15:23:41.365768+0000 mgr.y (mgr.26252) 1732 : cluster [DBG] pgmap v2389: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:42.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:42 smithi099 bash[87147]: cluster 2024-09-13T15:23:41.365768+0000 mgr.y (mgr.26252) 1732 : cluster [DBG] pgmap v2389: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:42.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:42 smithi099 bash[87147]: cluster 2024-09-13T15:23:41.365768+0000 mgr.y (mgr.26252) 1732 : cluster [DBG] pgmap v2389: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:42 smithi177 bash[68594]: cluster 2024-09-13T15:23:41.365768+0000 mgr.y (mgr.26252) 1732 : cluster [DBG] pgmap v2389: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:42 smithi177 bash[68594]: cluster 2024-09-13T15:23:41.365768+0000 mgr.y (mgr.26252) 1732 : cluster [DBG] pgmap v2389: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:43.174 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:23:43.220 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:23:44.655 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:44 smithi099 bash[84576]: cluster 2024-09-13T15:23:43.366739+0000 mgr.y (mgr.26252) 1733 : cluster [DBG] pgmap v2390: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:44.656 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:44 smithi099 bash[84576]: cluster 2024-09-13T15:23:43.366739+0000 mgr.y (mgr.26252) 1733 : cluster [DBG] pgmap v2390: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:44.656 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:44 smithi099 bash[87147]: cluster 2024-09-13T15:23:43.366739+0000 mgr.y (mgr.26252) 1733 : cluster [DBG] pgmap v2390: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:44.656 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:44 smithi099 bash[87147]: cluster 2024-09-13T15:23:43.366739+0000 mgr.y (mgr.26252) 1733 : cluster [DBG] pgmap v2390: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:44.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:44 smithi177 bash[68594]: cluster 2024-09-13T15:23:43.366739+0000 mgr.y (mgr.26252) 1733 : cluster [DBG] pgmap v2390: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:44.670 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:44 smithi177 bash[68594]: cluster 2024-09-13T15:23:43.366739+0000 mgr.y (mgr.26252) 1733 : cluster [DBG] pgmap v2390: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:46.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:46 smithi099 bash[84576]: cluster 2024-09-13T15:23:45.367668+0000 mgr.y (mgr.26252) 1734 : cluster [DBG] pgmap v2391: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:46.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:46 smithi099 bash[84576]: cluster 2024-09-13T15:23:45.367668+0000 mgr.y (mgr.26252) 1734 : cluster [DBG] pgmap v2391: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:46.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:46 smithi099 bash[87147]: cluster 2024-09-13T15:23:45.367668+0000 mgr.y (mgr.26252) 1734 : cluster [DBG] pgmap v2391: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:46.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:46 smithi099 bash[87147]: cluster 2024-09-13T15:23:45.367668+0000 mgr.y (mgr.26252) 1734 : cluster [DBG] pgmap v2391: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:46.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:46 smithi177 bash[68594]: cluster 2024-09-13T15:23:45.367668+0000 mgr.y (mgr.26252) 1734 : cluster [DBG] pgmap v2391: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:46.826 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:46 smithi177 bash[68594]: cluster 2024-09-13T15:23:45.367668+0000 mgr.y (mgr.26252) 1734 : cluster [DBG] pgmap v2391: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:46.994 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89300 46.1 46:840ba033:::benchmark_data_smithi099_233884_object89299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.716451+0000", 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 435.27501203600002, 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0350055899999999, 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:23:46.995 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89300 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716451+0000", 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716451+0000", 2024-09-13T15:23:46.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716450+0000", 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716493+0000", 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2806000000000003e-05 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716504+0000", 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1316e-05 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716509+0000", 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5249999999999999e-06 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.730921+0000", 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01441251 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:46.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.730926+0000", 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3419999999999998e-06 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.946922+0000", 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21599571000000001 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.946944+0000", 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2225000000000002e-05 2024-09-13T15:23:46.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.751367+0000", 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.80442297799999996 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.751457+0000", 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9672000000000005e-05 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89302 46.1 46:805bb9a7:::benchmark_data_smithi099_233884_object89301:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.716517+0000", 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 435.274946941, 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.04309641, 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89302 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716517+0000", 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716517+0000", 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716516+0000", 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:23:47.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716552+0000", 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5787000000000001e-05 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716557+0000", 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6070000000000001e-06 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.716560+0000", 2024-09-13T15:23:47.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.506e-06 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.730994+0000", 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014433931000000001 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting_for_map not empty", 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.730999+0000", 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8930000000000004e-06 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.947040+0000", 2024-09-13T15:23:47.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.216041016 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.947065+0000", 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5528e-05 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.751490+0000", 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.80442414200000001 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.752216+0000", 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00072603299999999995 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.759337+0000", 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0071212940000000002 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.759613+0000", 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00027605300000000001 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.006 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:95378 46.10 46.3da6d890 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1110)", 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:37.761038+0000", 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 429.23042572600002, 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99772587599999996, 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 95378 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.007 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.761038+0000", 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.761038+0000", 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.761036+0000", 2024-09-13T15:23:47.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.761075+0000", 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8546e-05 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.761086+0000", 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0539e-05 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.761090+0000", 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8010000000000003e-06 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.761241+0000", 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015058399999999999 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.761248+0000", 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1420000000000004e-06 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:38.758737+0000", 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.997488818 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:38.758764+0000", 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6766000000000001e-05 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:95397 46.10 46.d9b90f70 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1110)", 2024-09-13T15:23:47.011 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:37.767126+0000", 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 429.22433773400002, 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99172891799999996, 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 95397 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.767126+0000", 2024-09-13T15:23:47.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.767126+0000", 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.767119+0000", 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999938 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.767165+0000", 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5525999999999997e-05 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.767176+0000", 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0866000000000001e-05 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.767181+0000", 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5110000000000003e-06 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.767221+0000", 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9860000000000001e-05 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:37.767227+0000", 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9290000000000003e-06 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:38.758837+0000", 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99160979000000005 2024-09-13T15:23:47.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:38.758855+0000", 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7859e-05 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:101870 46.12 46:4cfce207:::benchmark_data_smithi099_233884_object101869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:47.819423+0000", 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 419.17204017, 2024-09-13T15:23:47.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.323729358, 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 101870 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.819423+0000", 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.819423+0000", 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.819422+0000", 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.819460+0000", 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8711999999999998e-05 2024-09-13T15:23:47.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.819476+0000", 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5715000000000001e-05 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.819483+0000", 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7379999999999997e-06 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.819559+0000", 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5673000000000005e-05 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.819572+0000", 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3764999999999999e-05 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665563+0000", 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.84599076799999995 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665576+0000", 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2880999999999999e-05 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.112270+0000", 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.446693655 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.112388+0000", 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011797 2024-09-13T15:23:47.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.112494+0000", 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010671300000000001 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.117194+0000", 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046994660000000002 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141676+0000", 2024-09-13T15:23:47.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024482074 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.142033+0000", 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00035701999999999999 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.142074+0000", 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1227999999999999e-05 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.143153+0000", 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010785230000000001 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:101956 46.12 46:4e07397f:::benchmark_data_smithi099_233884_object101955:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:47.918323+0000", 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 419.07314069, 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.225981972, 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.024 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 101956 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.918323+0000", 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.918323+0000", 2024-09-13T15:23:47.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.918322+0000", 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.918424+0000", 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000101981 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.918433+0000", 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1789999999999997e-06 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.918438+0000", 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8339999999999998e-06 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.919946+0000", 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015079220000000001 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.919959+0000", 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2724999999999999e-05 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665600+0000", 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.74564140800000001 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665608+0000", 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6199999999999999e-06 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.114174+0000", 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4485666189999999 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.114306+0000", 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000132051 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.114475+0000", 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016860200000000001 2024-09-13T15:23:47.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.118327+0000", 2024-09-13T15:23:47.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003852135 2024-09-13T15:23:47.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141701+0000", 2024-09-13T15:23:47.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023374045 2024-09-13T15:23:47.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.143206+0000", 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001504654 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.143267+0000", 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1618000000000005e-05 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.144305+0000", 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00103725 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:101996 46.12 46:49479a64:::benchmark_data_smithi099_233884_object101995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:47.958250+0000", 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 419.03321325899998, 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1870399360000001, 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.032 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 101996 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.958250+0000", 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.958250+0000", 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.958249+0000", 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.958284+0000", 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5040000000000003e-05 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.958293+0000", 2024-09-13T15:23:47.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3530000000000001e-06 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.958298+0000", 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1649999999999997e-06 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.958360+0000", 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1346000000000005e-05 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:47.958369+0000", 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5249999999999994e-06 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665627+0000", 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.707257786 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665643+0000", 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6211000000000001e-05 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.115790+0000", 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.450146221 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.115919+0000", 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000129252 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.116072+0000", 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015295400000000001 2024-09-13T15:23:47.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.124212+0000", 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0081399769999999996 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141722+0000", 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017510477 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.144400+0000", 2024-09-13T15:23:47.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026780150000000002 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.144444+0000", 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3940999999999998e-05 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.145290+0000", 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00084599899999999995 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102096 46.12 46:49c14e4a:::benchmark_data_smithi099_233884_object102095:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.069295+0000", 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.92216801799998, 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0769053999999998, 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102096 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.069295+0000", 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.069295+0000", 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.069294+0000", 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:47.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.069362+0000", 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7960000000000007e-05 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.069375+0000", 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2809999999999999e-05 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.069381+0000", 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8769999999999998e-06 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.071589+0000", 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022079920000000002 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.071603+0000", 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4287000000000001e-05 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665664+0000", 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.59406071999999999 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665671+0000", 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8709999999999997e-06 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.117229+0000", 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4515571780000001 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.117402+0000", 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017299200000000001 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.117519+0000", 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000117242 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.127882+0000", 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010363033000000001 2024-09-13T15:23:47.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141777+0000", 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013894814 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.145329+0000", 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035518820000000001 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.145362+0000", 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3507000000000003e-05 2024-09-13T15:23:47.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.146201+0000", 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00083872199999999997 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102099 46.12 46:4d025d91:::benchmark_data_smithi099_233884_object102098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.071668+0000", 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.91979521600001, 2024-09-13T15:23:47.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.075572561, 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102099 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.071668+0000", 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.071668+0000", 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.071667+0000", 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.071708+0000", 2024-09-13T15:23:47.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1204000000000001e-05 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.071721+0000", 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3025e-05 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.071727+0000", 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.066e-06 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.071791+0000", 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3737000000000003e-05 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.071803+0000", 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1732999999999999e-05 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665692+0000", 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.59388960899999998 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665700+0000", 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7619999999999995e-06 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.118371+0000", 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.452671201 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.118606+0000", 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023492399999999999 2024-09-13T15:23:47.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.118730+0000", 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012364900000000001 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.127894+0000", 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0091644129999999997 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141830+0000", 2024-09-13T15:23:47.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013935866 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.146244+0000", 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044135040000000004 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.146278+0000", 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4004000000000002e-05 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.147241+0000", 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00096322300000000003 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102131 46.12 46:4b27c74c:::benchmark_data_smithi099_233884_object102130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.100708+0000", 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.89075584699998, 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0478729059999998, 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.055 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102131 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.100708+0000", 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.100708+0000", 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.100706+0000", 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.100733+0000", 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6611e-05 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.100744+0000", 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1228e-05 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.100749+0000", 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6929999999999998e-06 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.100812+0000", 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3212000000000004e-05 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.100824+0000", 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2146e-05 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665740+0000", 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.56491620799999998 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665751+0000", 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0455000000000001e-05 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.120241+0000", 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4544902099999999 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.120384+0000", 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014295599999999999 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.120571+0000", 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018665399999999999 2024-09-13T15:23:47.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.127905+0000", 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073342019999999997 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141859+0000", 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013954223999999999 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.147313+0000", 2024-09-13T15:23:47.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054539139999999998 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.147373+0000", 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0381999999999999e-05 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.148581+0000", 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012070780000000001 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102217 46.12 46:493de5ea:::benchmark_data_smithi099_233884_object102216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.198199+0000", 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.79326482099998, 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9540407069999999, 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102217 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.063 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.198199+0000", 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.198199+0000", 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.198197+0000", 2024-09-13T15:23:47.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.198251+0000", 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4438000000000003e-05 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.198267+0000", 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5435e-05 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.198274+0000", 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0709999999999997e-06 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.198324+0000", 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0207999999999998e-05 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.198332+0000", 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3930000000000008e-06 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665783+0000", 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.46745088800000001 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665791+0000", 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7230000000000004e-06 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.121774+0000", 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4559830489999999 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.122083+0000", 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000308719 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.122334+0000", 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00025165799999999998 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.132504+0000", 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010169518000000001 2024-09-13T15:23:47.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141884+0000", 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0093804800000000001 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.150848+0000", 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0089636590000000006 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.150891+0000", 2024-09-13T15:23:47.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2484000000000002e-05 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.152239+0000", 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001348744 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102223 46.12 46:4e632ce0:::benchmark_data_smithi099_233884_object102222:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.206290+0000", 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.78517383600001, 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.950806383, 2024-09-13T15:23:47.070 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102223 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.206290+0000", 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.206290+0000", 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.206288+0000", 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.206335+0000", 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6984999999999998e-05 2024-09-13T15:23:47.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.206348+0000", 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3446e-05 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.206354+0000", 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0579999999999997e-06 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.206404+0000", 2024-09-13T15:23:47.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0288999999999999e-05 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.206413+0000", 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.721e-06 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665815+0000", 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45940290700000003 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665822+0000", 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8859999999999998e-06 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.124272+0000", 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4584498240000001 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.124407+0000", 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000134361 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.124516+0000", 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109386 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.132525+0000", 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0080091929999999995 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141909+0000", 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0093842530000000004 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.155627+0000", 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013717627 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.155676+0000", 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8943000000000002e-05 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.157096+0000", 2024-09-13T15:23:47.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001420078 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102227 46.12 46:4bac42e3:::benchmark_data_smithi099_233884_object102226:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.210155+0000", 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.78130836399998, 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.948247187, 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.078 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102227 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.210155+0000", 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.210155+0000", 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.210154+0000", 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.210204+0000", 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9725999999999998e-05 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.210217+0000", 2024-09-13T15:23:47.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3549e-05 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.210223+0000", 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5550000000000002e-06 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.210265+0000", 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2348999999999997e-05 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.210273+0000", 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9109999999999993e-06 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665850+0000", 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45557719099999999 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665857+0000", 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7610000000000002e-06 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.125714+0000", 2024-09-13T15:23:47.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4598563330000001 2024-09-13T15:23:47.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.125926+0000", 2024-09-13T15:23:47.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00021240199999999999 2024-09-13T15:23:47.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.126177+0000", 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000251222 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.135996+0000", 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0098184220000000003 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141934+0000", 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059386819999999998 2024-09-13T15:23:47.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.157168+0000", 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015233672 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.157238+0000", 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9603000000000004e-05 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.158402+0000", 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011647859999999999 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102250 46.12 46:4f64808a:::benchmark_data_smithi099_233884_object102249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.236953+0000", 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.75451051900001, 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9224855139999999, 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.086 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102250 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.236953+0000", 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.236953+0000", 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.236952+0000", 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.237014+0000", 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2360000000000006e-05 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.237023+0000", 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9779999999999994e-06 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.237029+0000", 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5500000000000002e-06 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.237080+0000", 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0832999999999999e-05 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.237088+0000", 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2889999999999998e-06 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665878+0000", 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.42879011500000003 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665885+0000", 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2479999999999999e-06 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.127960+0000", 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.462074361 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.128125+0000", 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016484 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.128321+0000", 2024-09-13T15:23:47.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019674199999999999 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.136001+0000", 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0076793089999999996 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141959+0000", 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059584690000000001 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.158457+0000", 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016497689999999999 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.158495+0000", 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7764e-05 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.159438+0000", 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00094389599999999995 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.093 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102253 46.12 46:4b9b187d:::benchmark_data_smithi099_233884_object102252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.240439+0000", 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.75102453800002, 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9200422509999999, 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102253 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.240439+0000", 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.240439+0000", 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.240438+0000", 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:23:47.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.240489+0000", 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1143999999999997e-05 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.240502+0000", 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3232e-05 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.240509+0000", 2024-09-13T15:23:47.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4659999999999996e-06 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.240567+0000", 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8134000000000003e-05 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.240577+0000", 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0052e-05 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665910+0000", 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.42533357999999999 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665917+0000", 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7020000000000004e-06 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.129454+0000", 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4635369979999999 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.129750+0000", 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000295846 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.130124+0000", 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00037399699999999999 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.139819+0000", 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0096949759999999992 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141983+0000", 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021646090000000001 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.159498+0000", 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017514661000000001 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.159560+0000", 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1451999999999994e-05 2024-09-13T15:23:47.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.160481+0000", 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00092163300000000004 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102308 46.12 46:4a740572:::benchmark_data_smithi099_233884_object102307:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.321938+0000", 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.66952502999999, 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.839595842, 2024-09-13T15:23:47.101 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102308 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.321938+0000", 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.321938+0000", 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.321938+0000", 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.322013+0000", 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5215000000000004e-05 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.322025+0000", 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.256e-05 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.322031+0000", 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6019999999999999e-06 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.322091+0000", 2024-09-13T15:23:47.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.05e-05 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.322099+0000", 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6299999999999998e-06 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665940+0000", 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34384098899999999 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665948+0000", 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4309999999999997e-06 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.132556+0000", 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4666085170000001 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.132719+0000", 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000162807 2024-09-13T15:23:47.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.132915+0000", 2024-09-13T15:23:47.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019585499999999999 2024-09-13T15:23:47.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.139828+0000", 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0069137540000000003 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.142008+0000", 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002179673 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.160545+0000", 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018537257000000001 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.160593+0000", 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7565000000000003e-05 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.161534+0000", 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00094130400000000001 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.109 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102314 46.12 46:49d5fe23:::benchmark_data_smithi099_233884_object102313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.331923+0000", 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.65954089500002, 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.830757988, 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.110 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102314 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.331923+0000", 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.331923+0000", 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.331922+0000", 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.332005+0000", 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3609999999999994e-05 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.332016+0000", 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0821e-05 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.332020+0000", 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3590000000000001e-06 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.332073+0000", 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3127999999999997e-05 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.332087+0000", 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3156000000000001e-05 2024-09-13T15:23:47.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665969+0000", 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.333882081 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665976+0000", 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2150000000000004e-06 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.134471+0000", 2024-09-13T15:23:47.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.468495122 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.134628+0000", 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015691699999999999 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.134834+0000", 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000206286 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.139837+0000", 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050032109999999996 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.144345+0000", 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045077069999999997 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.161699+0000", 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017353818999999999 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.161737+0000", 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8300999999999998e-05 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.162681+0000", 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00094331700000000005 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.117 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102360 46.12 46:4c79dcba:::benchmark_data_smithi099_233884_object102359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.412299+0000", 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.579164096, 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.751199291, 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102360 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.412299+0000", 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.412299+0000", 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.412298+0000", 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.412387+0000", 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9667999999999996e-05 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.412400+0000", 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2761e-05 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.412409+0000", 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4710000000000007e-06 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.412666+0000", 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00025703999999999999 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.412678+0000", 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2627e-05 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.665996+0000", 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25331732899999998 2024-09-13T15:23:47.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.666003+0000", 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1760000000000004e-06 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.136021+0000", 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.470017946 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.136107+0000", 2024-09-13T15:23:47.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6327000000000002e-05 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.136207+0000", 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9580999999999994e-05 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141644+0000", 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054371640000000004 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.144375+0000", 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002731242 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.162724+0000", 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018349394000000002 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.162759+0000", 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4885000000000002e-05 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.163499+0000", 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00073937500000000001 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102362 46.12 46:4c41b9f8:::benchmark_data_smithi099_233884_object102361:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.413435+0000", 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.578028948, 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7509199090000001, 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.125 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102362 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.413435+0000", 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.413435+0000", 2024-09-13T15:23:47.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.413434+0000", 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.413490+0000", 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6651999999999999e-05 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.413502+0000", 2024-09-13T15:23:47.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1423999999999999e-05 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.413508+0000", 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1770000000000003e-06 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.413539+0000", 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1439000000000002e-05 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.413545+0000", 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5840000000000002e-06 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.666026+0000", 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.252481338 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.666034+0000", 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8250000000000005e-06 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.137250+0000", 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.471215921 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.137351+0000", 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000101445 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.137452+0000", 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000101028 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141648+0000", 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041953229999999999 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.148640+0000", 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0069920709999999999 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.163538+0000", 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014898095 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.163574+0000", 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6026999999999999e-05 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.164354+0000", 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00078050000000000005 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.132 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:102410 46.12 46:4abeeab8:::benchmark_data_smithi099_233884_object102409:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1115)", 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:48.598914+0000", 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 418.39254927000002, 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5662508800000001, 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102410 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:23:47.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.598914+0000", 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.598914+0000", 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.598913+0000", 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.598967+0000", 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3971999999999998e-05 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.598979+0000", 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1980999999999999e-05 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.598984+0000", 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0379999999999999e-06 2024-09-13T15:23:47.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.599069+0000", 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4572000000000003e-05 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.599088+0000", 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9208e-05 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.666205+0000", 2024-09-13T15:23:47.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.067117305000000002 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:48.666214+0000", 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0920000000000004e-06 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.138588+0000", 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4723736760000001 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:23:47.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.138730+0000", 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000141963 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.138830+0000", 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100302 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.141681+0000", 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028510509999999998 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.148689+0000", 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0070079950000000004 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.164389+0000", 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015699984 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.164420+0000", 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0243000000000002e-05 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:23:47.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:23:47.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:23:47.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:50.165165+0000", 2024-09-13T15:23:47.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00074550099999999998 2024-09-13T15:23:47.140 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.140 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.140 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.140 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:23:47.140 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:23:47.140 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:23:47.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:23:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:23:47] "GET /metrics HTTP/1.1" 200 34330 "" "Prometheus/2.51.0" 2024-09-13T15:23:47.416 INFO:teuthology.orchestra.run.smithi099.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-13T15:23:47.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:47 smithi177 bash[68594]: audit 2024-09-13T15:23:47.281980+0000 mon.a (mon.0) 2930 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:47.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:47 smithi177 bash[68594]: audit 2024-09-13T15:23:47.281980+0000 mon.a (mon.0) 2930 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:47.725 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:47 smithi099 bash[84576]: audit 2024-09-13T15:23:47.281980+0000 mon.a (mon.0) 2930 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:47.725 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:47 smithi099 bash[84576]: audit 2024-09-13T15:23:47.281980+0000 mon.a (mon.0) 2930 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:47.725 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:47 smithi099 bash[87147]: audit 2024-09-13T15:23:47.281980+0000 mon.a (mon.0) 2930 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:47.725 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:47 smithi099 bash[87147]: audit 2024-09-13T15:23:47.281980+0000 mon.a (mon.0) 2930 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:23:47.844 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:23:48.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T15:23:48.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[84576]: cluster 2024-09-13T15:23:47.368616+0000 mgr.y (mgr.26252) 1735 : cluster [DBG] pgmap v2392: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:48.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[84576]: cluster 2024-09-13T15:23:47.368616+0000 mgr.y (mgr.26252) 1735 : cluster [DBG] pgmap v2392: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:48.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[84576]: audit 2024-09-13T15:23:47.415791+0000 mon.a (mon.0) 2931 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:48.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[84576]: audit 2024-09-13T15:23:47.415791+0000 mon.a (mon.0) 2931 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:48.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[84576]: cluster 2024-09-13T15:23:47.422010+0000 mon.a (mon.0) 2932 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2024-09-13T15:23:48.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[84576]: cluster 2024-09-13T15:23:47.422010+0000 mon.a (mon.0) 2932 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2024-09-13T15:23:48.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[84576]: audit 2024-09-13T15:23:48.331073+0000 mon.a (mon.0) 2933 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:48.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[84576]: audit 2024-09-13T15:23:48.331073+0000 mon.a (mon.0) 2933 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:48.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[87147]: cluster 2024-09-13T15:23:47.368616+0000 mgr.y (mgr.26252) 1735 : cluster [DBG] pgmap v2392: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:48.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[87147]: cluster 2024-09-13T15:23:47.368616+0000 mgr.y (mgr.26252) 1735 : cluster [DBG] pgmap v2392: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:48.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[87147]: audit 2024-09-13T15:23:47.415791+0000 mon.a (mon.0) 2931 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:48.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[87147]: audit 2024-09-13T15:23:47.415791+0000 mon.a (mon.0) 2931 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:48.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[87147]: cluster 2024-09-13T15:23:47.422010+0000 mon.a (mon.0) 2932 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2024-09-13T15:23:48.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[87147]: cluster 2024-09-13T15:23:47.422010+0000 mon.a (mon.0) 2932 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2024-09-13T15:23:48.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[87147]: audit 2024-09-13T15:23:48.331073+0000 mon.a (mon.0) 2933 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:48.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:48 smithi099 bash[87147]: audit 2024-09-13T15:23:48.331073+0000 mon.a (mon.0) 2933 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:48.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:48 smithi177 bash[68594]: cluster 2024-09-13T15:23:47.368616+0000 mgr.y (mgr.26252) 1735 : cluster [DBG] pgmap v2392: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:48.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:48 smithi177 bash[68594]: cluster 2024-09-13T15:23:47.368616+0000 mgr.y (mgr.26252) 1735 : cluster [DBG] pgmap v2392: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:48.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:48 smithi177 bash[68594]: audit 2024-09-13T15:23:47.415791+0000 mon.a (mon.0) 2931 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:48.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:48 smithi177 bash[68594]: audit 2024-09-13T15:23:47.415791+0000 mon.a (mon.0) 2931 : audit [INF] from='client.? 172.21.15.99:0/3655347580' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:23:48.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:48 smithi177 bash[68594]: cluster 2024-09-13T15:23:47.422010+0000 mon.a (mon.0) 2932 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2024-09-13T15:23:48.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:48 smithi177 bash[68594]: cluster 2024-09-13T15:23:47.422010+0000 mon.a (mon.0) 2932 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2024-09-13T15:23:48.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:48 smithi177 bash[68594]: audit 2024-09-13T15:23:48.331073+0000 mon.a (mon.0) 2933 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:48.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:48 smithi177 bash[68594]: audit 2024-09-13T15:23:48.331073+0000 mon.a (mon.0) 2933 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:23:50.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:23:50 smithi177 bash[61439]: ts=2024-09-13T15:23:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:23:50.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:50 smithi177 bash[68594]: cluster 2024-09-13T15:23:49.369486+0000 mgr.y (mgr.26252) 1736 : cluster [DBG] pgmap v2394: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:50.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:50 smithi177 bash[68594]: cluster 2024-09-13T15:23:49.369486+0000 mgr.y (mgr.26252) 1736 : cluster [DBG] pgmap v2394: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:50.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:50 smithi099 bash[84576]: cluster 2024-09-13T15:23:49.369486+0000 mgr.y (mgr.26252) 1736 : cluster [DBG] pgmap v2394: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:50.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:50 smithi099 bash[84576]: cluster 2024-09-13T15:23:49.369486+0000 mgr.y (mgr.26252) 1736 : cluster [DBG] pgmap v2394: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:50 smithi099 bash[87147]: cluster 2024-09-13T15:23:49.369486+0000 mgr.y (mgr.26252) 1736 : cluster [DBG] pgmap v2394: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:50.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:50 smithi099 bash[87147]: cluster 2024-09-13T15:23:49.369486+0000 mgr.y (mgr.26252) 1736 : cluster [DBG] pgmap v2394: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:52.633 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:23:52.736 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:52 smithi177 bash[68594]: cluster 2024-09-13T15:23:51.370273+0000 mgr.y (mgr.26252) 1737 : cluster [DBG] pgmap v2395: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:52.736 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:52 smithi177 bash[68594]: cluster 2024-09-13T15:23:51.370273+0000 mgr.y (mgr.26252) 1737 : cluster [DBG] pgmap v2395: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:52.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:52 smithi099 bash[84576]: cluster 2024-09-13T15:23:51.370273+0000 mgr.y (mgr.26252) 1737 : cluster [DBG] pgmap v2395: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:52.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:52 smithi099 bash[84576]: cluster 2024-09-13T15:23:51.370273+0000 mgr.y (mgr.26252) 1737 : cluster [DBG] pgmap v2395: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:52.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:52 smithi099 bash[87147]: cluster 2024-09-13T15:23:51.370273+0000 mgr.y (mgr.26252) 1737 : cluster [DBG] pgmap v2395: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:52.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:52 smithi099 bash[87147]: cluster 2024-09-13T15:23:51.370273+0000 mgr.y (mgr.26252) 1737 : cluster [DBG] pgmap v2395: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:53.180 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:23:54.695 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:54 smithi099 bash[84576]: cluster 2024-09-13T15:23:53.371165+0000 mgr.y (mgr.26252) 1738 : cluster [DBG] pgmap v2396: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:54.695 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:54 smithi099 bash[84576]: cluster 2024-09-13T15:23:53.371165+0000 mgr.y (mgr.26252) 1738 : cluster [DBG] pgmap v2396: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:54.696 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:54 smithi099 bash[87147]: cluster 2024-09-13T15:23:53.371165+0000 mgr.y (mgr.26252) 1738 : cluster [DBG] pgmap v2396: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:54.696 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:54 smithi099 bash[87147]: cluster 2024-09-13T15:23:53.371165+0000 mgr.y (mgr.26252) 1738 : cluster [DBG] pgmap v2396: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:54.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:54 smithi177 bash[68594]: cluster 2024-09-13T15:23:53.371165+0000 mgr.y (mgr.26252) 1738 : cluster [DBG] pgmap v2396: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:54.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:54 smithi177 bash[68594]: cluster 2024-09-13T15:23:53.371165+0000 mgr.y (mgr.26252) 1738 : cluster [DBG] pgmap v2396: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:56.498 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:23:56.498 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:23:56.498 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2024-09-13T15:23:56.498 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:23:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:56 smithi177 bash[68594]: cluster 2024-09-13T15:23:55.372032+0000 mgr.y (mgr.26252) 1739 : cluster [DBG] pgmap v2397: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:56.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:56 smithi177 bash[68594]: cluster 2024-09-13T15:23:55.372032+0000 mgr.y (mgr.26252) 1739 : cluster [DBG] pgmap v2397: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:56.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:56 smithi099 bash[84576]: cluster 2024-09-13T15:23:55.372032+0000 mgr.y (mgr.26252) 1739 : cluster [DBG] pgmap v2397: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:56.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:56 smithi099 bash[84576]: cluster 2024-09-13T15:23:55.372032+0000 mgr.y (mgr.26252) 1739 : cluster [DBG] pgmap v2397: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:56 smithi099 bash[87147]: cluster 2024-09-13T15:23:55.372032+0000 mgr.y (mgr.26252) 1739 : cluster [DBG] pgmap v2397: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:56.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:56 smithi099 bash[87147]: cluster 2024-09-13T15:23:55.372032+0000 mgr.y (mgr.26252) 1739 : cluster [DBG] pgmap v2397: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:57.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:23:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:23:57] "GET /metrics HTTP/1.1" 200 34334 "" "Prometheus/2.51.0" 2024-09-13T15:23:57.352 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:23:58.452 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T15:23:58.781 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:58 smithi099 bash[84576]: cluster 2024-09-13T15:23:57.372888+0000 mgr.y (mgr.26252) 1740 : cluster [DBG] pgmap v2398: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:58.781 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:58 smithi099 bash[84576]: cluster 2024-09-13T15:23:57.372888+0000 mgr.y (mgr.26252) 1740 : cluster [DBG] pgmap v2398: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:58.781 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:58 smithi099 bash[84576]: audit 2024-09-13T15:23:57.481787+0000 mon.a (mon.0) 2934 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:23:58.781 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:58 smithi099 bash[84576]: audit 2024-09-13T15:23:57.481787+0000 mon.a (mon.0) 2934 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:23:58.781 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:58 smithi099 bash[87147]: cluster 2024-09-13T15:23:57.372888+0000 mgr.y (mgr.26252) 1740 : cluster [DBG] pgmap v2398: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:58.781 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:58 smithi099 bash[87147]: cluster 2024-09-13T15:23:57.372888+0000 mgr.y (mgr.26252) 1740 : cluster [DBG] pgmap v2398: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:58.782 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:58 smithi099 bash[87147]: audit 2024-09-13T15:23:57.481787+0000 mon.a (mon.0) 2934 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:23:58.782 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:58 smithi099 bash[87147]: audit 2024-09-13T15:23:57.481787+0000 mon.a (mon.0) 2934 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:23:58.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:58 smithi177 bash[68594]: cluster 2024-09-13T15:23:57.372888+0000 mgr.y (mgr.26252) 1740 : cluster [DBG] pgmap v2398: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:58.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:58 smithi177 bash[68594]: cluster 2024-09-13T15:23:57.372888+0000 mgr.y (mgr.26252) 1740 : cluster [DBG] pgmap v2398: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:23:58.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:58 smithi177 bash[68594]: audit 2024-09-13T15:23:57.481787+0000 mon.a (mon.0) 2934 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:23:58.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:58 smithi177 bash[68594]: audit 2024-09-13T15:23:57.481787+0000 mon.a (mon.0) 2934 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:23:59.535 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 7 1 2024-09-13T15:23:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:59 smithi099 bash[84576]: audit 2024-09-13T15:23:58.451270+0000 mon.a (mon.0) 2935 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:23:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:59 smithi099 bash[84576]: audit 2024-09-13T15:23:58.451270+0000 mon.a (mon.0) 2935 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:23:59.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:59 smithi099 bash[84576]: cluster 2024-09-13T15:23:58.458575+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2024-09-13T15:23:59.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:23:59 smithi099 bash[84576]: cluster 2024-09-13T15:23:58.458575+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2024-09-13T15:23:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:59 smithi099 bash[87147]: audit 2024-09-13T15:23:58.451270+0000 mon.a (mon.0) 2935 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:23:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:59 smithi099 bash[87147]: audit 2024-09-13T15:23:58.451270+0000 mon.a (mon.0) 2935 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:23:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:59 smithi099 bash[87147]: cluster 2024-09-13T15:23:58.458575+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2024-09-13T15:23:59.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:23:59 smithi099 bash[87147]: cluster 2024-09-13T15:23:58.458575+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2024-09-13T15:23:59.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:59 smithi177 bash[68594]: audit 2024-09-13T15:23:58.451270+0000 mon.a (mon.0) 2935 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:23:59.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:59 smithi177 bash[68594]: audit 2024-09-13T15:23:58.451270+0000 mon.a (mon.0) 2935 : audit [INF] from='client.? 172.21.15.99:0/3575059108' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:23:59.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:59 smithi177 bash[68594]: cluster 2024-09-13T15:23:58.458575+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2024-09-13T15:23:59.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:23:59 smithi177 bash[68594]: cluster 2024-09-13T15:23:58.458575+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2024-09-13T15:24:00.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:00 smithi177 bash[68594]: cluster 2024-09-13T15:23:59.373734+0000 mgr.y (mgr.26252) 1741 : cluster [DBG] pgmap v2400: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:00.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:00 smithi177 bash[68594]: cluster 2024-09-13T15:23:59.373734+0000 mgr.y (mgr.26252) 1741 : cluster [DBG] pgmap v2400: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:00.613 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:24:00 smithi177 bash[61439]: ts=2024-09-13T15:24:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:24:00.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:00 smithi099 bash[84576]: cluster 2024-09-13T15:23:59.373734+0000 mgr.y (mgr.26252) 1741 : cluster [DBG] pgmap v2400: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:00.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:00 smithi099 bash[84576]: cluster 2024-09-13T15:23:59.373734+0000 mgr.y (mgr.26252) 1741 : cluster [DBG] pgmap v2400: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:00 smithi099 bash[87147]: cluster 2024-09-13T15:23:59.373734+0000 mgr.y (mgr.26252) 1741 : cluster [DBG] pgmap v2400: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:00.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:00 smithi099 bash[87147]: cluster 2024-09-13T15:23:59.373734+0000 mgr.y (mgr.26252) 1741 : cluster [DBG] pgmap v2400: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:02.140 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:24:02.695 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:02 smithi177 bash[68594]: cluster 2024-09-13T15:24:01.374489+0000 mgr.y (mgr.26252) 1742 : cluster [DBG] pgmap v2401: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:02.695 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:02 smithi177 bash[68594]: cluster 2024-09-13T15:24:01.374489+0000 mgr.y (mgr.26252) 1742 : cluster [DBG] pgmap v2401: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:02.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:02 smithi099 bash[84576]: cluster 2024-09-13T15:24:01.374489+0000 mgr.y (mgr.26252) 1742 : cluster [DBG] pgmap v2401: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:02.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:02 smithi099 bash[84576]: cluster 2024-09-13T15:24:01.374489+0000 mgr.y (mgr.26252) 1742 : cluster [DBG] pgmap v2401: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:02.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:02 smithi099 bash[87147]: cluster 2024-09-13T15:24:01.374489+0000 mgr.y (mgr.26252) 1742 : cluster [DBG] pgmap v2401: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:02.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:02 smithi099 bash[87147]: cluster 2024-09-13T15:24:01.374489+0000 mgr.y (mgr.26252) 1742 : cluster [DBG] pgmap v2401: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:03.741 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:03 smithi177 bash[68594]: audit 2024-09-13T15:24:03.331335+0000 mon.a (mon.0) 2937 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:03.741 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:03 smithi177 bash[68594]: audit 2024-09-13T15:24:03.331335+0000 mon.a (mon.0) 2937 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:03.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:03 smithi099 bash[84576]: audit 2024-09-13T15:24:03.331335+0000 mon.a (mon.0) 2937 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:03.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:03 smithi099 bash[84576]: audit 2024-09-13T15:24:03.331335+0000 mon.a (mon.0) 2937 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:03.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:03 smithi099 bash[87147]: audit 2024-09-13T15:24:03.331335+0000 mon.a (mon.0) 2937 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:03.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:03 smithi099 bash[87147]: audit 2024-09-13T15:24:03.331335+0000 mon.a (mon.0) 2937 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:04.317 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:24:04.776 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:04 smithi177 bash[68594]: cluster 2024-09-13T15:24:03.375395+0000 mgr.y (mgr.26252) 1743 : cluster [DBG] pgmap v2402: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:04.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:04 smithi177 bash[68594]: cluster 2024-09-13T15:24:03.375395+0000 mgr.y (mgr.26252) 1743 : cluster [DBG] pgmap v2402: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:04.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:04 smithi099 bash[84576]: cluster 2024-09-13T15:24:03.375395+0000 mgr.y (mgr.26252) 1743 : cluster [DBG] pgmap v2402: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:04.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:04 smithi099 bash[84576]: cluster 2024-09-13T15:24:03.375395+0000 mgr.y (mgr.26252) 1743 : cluster [DBG] pgmap v2402: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:04.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:04 smithi099 bash[87147]: cluster 2024-09-13T15:24:03.375395+0000 mgr.y (mgr.26252) 1743 : cluster [DBG] pgmap v2402: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:04.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:04 smithi099 bash[87147]: cluster 2024-09-13T15:24:03.375395+0000 mgr.y (mgr.26252) 1743 : cluster [DBG] pgmap v2402: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:06.158 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:24:06.159 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2024-09-13T15:24:06.159 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2024-09-13T15:24:06.159 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2024-09-13T15:24:06.159 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:24:06.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:06 smithi099 bash[84576]: cluster 2024-09-13T15:24:05.376279+0000 mgr.y (mgr.26252) 1744 : cluster [DBG] pgmap v2403: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:06.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:06 smithi099 bash[84576]: cluster 2024-09-13T15:24:05.376279+0000 mgr.y (mgr.26252) 1744 : cluster [DBG] pgmap v2403: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:06.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:06 smithi099 bash[87147]: cluster 2024-09-13T15:24:05.376279+0000 mgr.y (mgr.26252) 1744 : cluster [DBG] pgmap v2403: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:06.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:06 smithi099 bash[87147]: cluster 2024-09-13T15:24:05.376279+0000 mgr.y (mgr.26252) 1744 : cluster [DBG] pgmap v2403: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:06.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:06 smithi177 bash[68594]: cluster 2024-09-13T15:24:05.376279+0000 mgr.y (mgr.26252) 1744 : cluster [DBG] pgmap v2403: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:06.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:06 smithi177 bash[68594]: cluster 2024-09-13T15:24:05.376279+0000 mgr.y (mgr.26252) 1744 : cluster [DBG] pgmap v2403: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:07.000 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:24:07.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:24:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:24:07] "GET /metrics HTTP/1.1" 200 34333 "" "Prometheus/2.51.0" 2024-09-13T15:24:08.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:08 smithi099 bash[84576]: cluster 2024-09-13T15:24:07.377584+0000 mgr.y (mgr.26252) 1745 : cluster [DBG] pgmap v2404: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:08.791 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:08 smithi099 bash[84576]: cluster 2024-09-13T15:24:07.377584+0000 mgr.y (mgr.26252) 1745 : cluster [DBG] pgmap v2404: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:08.791 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:08 smithi099 bash[87147]: cluster 2024-09-13T15:24:07.377584+0000 mgr.y (mgr.26252) 1745 : cluster [DBG] pgmap v2404: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:08.791 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:08 smithi099 bash[87147]: cluster 2024-09-13T15:24:07.377584+0000 mgr.y (mgr.26252) 1745 : cluster [DBG] pgmap v2404: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:08.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:08 smithi177 bash[68594]: cluster 2024-09-13T15:24:07.377584+0000 mgr.y (mgr.26252) 1745 : cluster [DBG] pgmap v2404: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:08.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:08 smithi177 bash[68594]: cluster 2024-09-13T15:24:07.377584+0000 mgr.y (mgr.26252) 1745 : cluster [DBG] pgmap v2404: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:09.506 INFO:teuthology.orchestra.run.smithi099.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-09-13T15:24:09.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:09 smithi099 bash[84576]: audit 2024-09-13T15:24:09.146981+0000 mon.c (mon.2) 138 : audit [INF] from='client.? 172.21.15.99:0/2817749668' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:09 smithi099 bash[84576]: audit 2024-09-13T15:24:09.146981+0000 mon.c (mon.2) 138 : audit [INF] from='client.? 172.21.15.99:0/2817749668' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:09 smithi099 bash[84576]: audit 2024-09-13T15:24:09.147717+0000 mon.a (mon.0) 2938 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:09 smithi099 bash[84576]: audit 2024-09-13T15:24:09.147717+0000 mon.a (mon.0) 2938 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:09 smithi099 bash[87147]: audit 2024-09-13T15:24:09.146981+0000 mon.c (mon.2) 138 : audit [INF] from='client.? 172.21.15.99:0/2817749668' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:09 smithi099 bash[87147]: audit 2024-09-13T15:24:09.146981+0000 mon.c (mon.2) 138 : audit [INF] from='client.? 172.21.15.99:0/2817749668' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:09 smithi099 bash[87147]: audit 2024-09-13T15:24:09.147717+0000 mon.a (mon.0) 2938 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:09 smithi099 bash[87147]: audit 2024-09-13T15:24:09.147717+0000 mon.a (mon.0) 2938 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:09 smithi177 bash[68594]: audit 2024-09-13T15:24:09.146981+0000 mon.c (mon.2) 138 : audit [INF] from='client.? 172.21.15.99:0/2817749668' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:09 smithi177 bash[68594]: audit 2024-09-13T15:24:09.146981+0000 mon.c (mon.2) 138 : audit [INF] from='client.? 172.21.15.99:0/2817749668' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:09 smithi177 bash[68594]: audit 2024-09-13T15:24:09.147717+0000 mon.a (mon.0) 2938 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:09.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:09 smithi177 bash[68594]: audit 2024-09-13T15:24:09.147717+0000 mon.a (mon.0) 2938 : audit [INF] from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:24:10.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T15:24:10.499 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:24:10 smithi177 bash[61439]: ts=2024-09-13T15:24:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:24:10.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[84576]: cluster 2024-09-13T15:24:09.378638+0000 mgr.y (mgr.26252) 1746 : cluster [DBG] pgmap v2405: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:10.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[84576]: cluster 2024-09-13T15:24:09.378638+0000 mgr.y (mgr.26252) 1746 : cluster [DBG] pgmap v2405: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:10.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[84576]: audit 2024-09-13T15:24:09.496176+0000 mon.a (mon.0) 2939 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:24:10.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[84576]: audit 2024-09-13T15:24:09.496176+0000 mon.a (mon.0) 2939 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:24:10.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[84576]: cluster 2024-09-13T15:24:09.505606+0000 mon.a (mon.0) 2940 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2024-09-13T15:24:10.828 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[84576]: cluster 2024-09-13T15:24:09.505606+0000 mon.a (mon.0) 2940 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2024-09-13T15:24:10.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[87147]: cluster 2024-09-13T15:24:09.378638+0000 mgr.y (mgr.26252) 1746 : cluster [DBG] pgmap v2405: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:10.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[87147]: cluster 2024-09-13T15:24:09.378638+0000 mgr.y (mgr.26252) 1746 : cluster [DBG] pgmap v2405: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:10.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[87147]: audit 2024-09-13T15:24:09.496176+0000 mon.a (mon.0) 2939 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:24:10.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[87147]: audit 2024-09-13T15:24:09.496176+0000 mon.a (mon.0) 2939 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:24:10.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[87147]: cluster 2024-09-13T15:24:09.505606+0000 mon.a (mon.0) 2940 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2024-09-13T15:24:10.829 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:10 smithi099 bash[87147]: cluster 2024-09-13T15:24:09.505606+0000 mon.a (mon.0) 2940 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2024-09-13T15:24:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:10 smithi177 bash[68594]: cluster 2024-09-13T15:24:09.378638+0000 mgr.y (mgr.26252) 1746 : cluster [DBG] pgmap v2405: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:10.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:10 smithi177 bash[68594]: cluster 2024-09-13T15:24:09.378638+0000 mgr.y (mgr.26252) 1746 : cluster [DBG] pgmap v2405: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:10 smithi177 bash[68594]: audit 2024-09-13T15:24:09.496176+0000 mon.a (mon.0) 2939 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:24:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:10 smithi177 bash[68594]: audit 2024-09-13T15:24:09.496176+0000 mon.a (mon.0) 2939 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:24:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:10 smithi177 bash[68594]: cluster 2024-09-13T15:24:09.505606+0000 mon.a (mon.0) 2940 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2024-09-13T15:24:10.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:10 smithi177 bash[68594]: cluster 2024-09-13T15:24:09.505606+0000 mon.a (mon.0) 2940 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2024-09-13T15:24:11.788 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:24:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:11 smithi099 bash[84576]: audit 2024-09-13T15:24:10.930101+0000 mon.a (mon.0) 2941 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:24:11.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:11 smithi099 bash[84576]: audit 2024-09-13T15:24:10.930101+0000 mon.a (mon.0) 2941 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:24:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:11 smithi099 bash[87147]: audit 2024-09-13T15:24:10.930101+0000 mon.a (mon.0) 2941 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:24:11.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:11 smithi099 bash[87147]: audit 2024-09-13T15:24:10.930101+0000 mon.a (mon.0) 2941 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:24:11.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:11 smithi177 bash[68594]: audit 2024-09-13T15:24:10.930101+0000 mon.a (mon.0) 2941 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:24:11.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:11 smithi177 bash[68594]: audit 2024-09-13T15:24:10.930101+0000 mon.a (mon.0) 2941 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:24:12.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[84576]: cluster 2024-09-13T15:24:11.379391+0000 mgr.y (mgr.26252) 1747 : cluster [DBG] pgmap v2407: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:12.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[84576]: cluster 2024-09-13T15:24:11.379391+0000 mgr.y (mgr.26252) 1747 : cluster [DBG] pgmap v2407: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:12.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[84576]: audit 2024-09-13T15:24:11.526742+0000 mon.a (mon.0) 2942 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[84576]: audit 2024-09-13T15:24:11.526742+0000 mon.a (mon.0) 2942 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[84576]: audit 2024-09-13T15:24:11.531843+0000 mon.a (mon.0) 2943 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[84576]: audit 2024-09-13T15:24:11.531843+0000 mon.a (mon.0) 2943 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[87147]: cluster 2024-09-13T15:24:11.379391+0000 mgr.y (mgr.26252) 1747 : cluster [DBG] pgmap v2407: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:12.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[87147]: cluster 2024-09-13T15:24:11.379391+0000 mgr.y (mgr.26252) 1747 : cluster [DBG] pgmap v2407: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:12.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[87147]: audit 2024-09-13T15:24:11.526742+0000 mon.a (mon.0) 2942 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[87147]: audit 2024-09-13T15:24:11.526742+0000 mon.a (mon.0) 2942 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[87147]: audit 2024-09-13T15:24:11.531843+0000 mon.a (mon.0) 2943 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:12 smithi099 bash[87147]: audit 2024-09-13T15:24:11.531843+0000 mon.a (mon.0) 2943 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:12 smithi177 bash[68594]: cluster 2024-09-13T15:24:11.379391+0000 mgr.y (mgr.26252) 1747 : cluster [DBG] pgmap v2407: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:12.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:12 smithi177 bash[68594]: cluster 2024-09-13T15:24:11.379391+0000 mgr.y (mgr.26252) 1747 : cluster [DBG] pgmap v2407: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:12.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:12 smithi177 bash[68594]: audit 2024-09-13T15:24:11.526742+0000 mon.a (mon.0) 2942 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:12 smithi177 bash[68594]: audit 2024-09-13T15:24:11.526742+0000 mon.a (mon.0) 2942 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:12 smithi177 bash[68594]: audit 2024-09-13T15:24:11.531843+0000 mon.a (mon.0) 2943 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:12.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:12 smithi177 bash[68594]: audit 2024-09-13T15:24:11.531843+0000 mon.a (mon.0) 2943 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:14.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:14 smithi099 bash[84576]: cluster 2024-09-13T15:24:13.380365+0000 mgr.y (mgr.26252) 1748 : cluster [DBG] pgmap v2408: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:14.826 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:14 smithi099 bash[84576]: cluster 2024-09-13T15:24:13.380365+0000 mgr.y (mgr.26252) 1748 : cluster [DBG] pgmap v2408: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:14.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:14 smithi099 bash[87147]: cluster 2024-09-13T15:24:13.380365+0000 mgr.y (mgr.26252) 1748 : cluster [DBG] pgmap v2408: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:14.827 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:14 smithi099 bash[87147]: cluster 2024-09-13T15:24:13.380365+0000 mgr.y (mgr.26252) 1748 : cluster [DBG] pgmap v2408: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:14.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:14 smithi177 bash[68594]: cluster 2024-09-13T15:24:13.380365+0000 mgr.y (mgr.26252) 1748 : cluster [DBG] pgmap v2408: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:14.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:14 smithi177 bash[68594]: cluster 2024-09-13T15:24:13.380365+0000 mgr.y (mgr.26252) 1748 : cluster [DBG] pgmap v2408: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:15.263 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:53806 46.4 46:278b9455:::benchmark_data_smithi099_233884_object53805:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:50.790196+0000", 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 505.00218090099997, 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49713887099999998, 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.795 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 53806 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790196+0000", 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.796 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790196+0000", 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790195+0000", 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790234+0000", 2024-09-13T15:24:15.797 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9099e-05 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790249+0000", 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5415000000000001e-05 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790256+0000", 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3280000000000002e-06 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.798 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790308+0000", 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2525000000000003e-05 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790440+0000", 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013153899999999999 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.790550+0000", 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110309 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.799 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.792375+0000", 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018250899999999999 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.793986+0000", 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00161035 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.287210+0000", 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49322427800000002 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.800 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.287290+0000", 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9741999999999996e-05 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.287335+0000", 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5661000000000002e-05 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.801 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:53818 46.4 46:2783c1b4:::benchmark_data_smithi099_233884_object53817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:15:50.799624+0000", 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 504.99275317399997, 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49978747800000001, 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 53818 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.802 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799624+0000", 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799624+0000", 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799622+0000", 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.803 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799668+0000", 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5992000000000003e-05 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799684+0000", 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5778e-05 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799691+0000", 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6150000000000002e-06 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.804 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799741+0000", 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9752000000000001e-05 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.799871+0000", 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013068199999999999 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.800012+0000", 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014106700000000001 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.805 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.802376+0000", 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023633299999999999 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:50.804500+0000", 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021246860000000002 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.299299+0000", 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49479895400000001 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.806 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.299357+0000", 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7238000000000003e-05 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:15:51.299412+0000", 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4997000000000002e-05 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.807 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89299 46.4 46:24ece472:::benchmark_data_smithi099_233884_object89298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.721651+0000", 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 464.07072664999998, 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.00749683, 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89299 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.808 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721651+0000", 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721651+0000", 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721649+0000", 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.809 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721697+0000", 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7987e-05 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721705+0000", 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8619999999999992e-06 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721710+0000", 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7389999999999999e-06 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.810 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721807+0000", 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6679000000000001e-05 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721817+0000", 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0407e-05 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930714+0000", 2024-09-13T15:24:15.811 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.20889722199999999 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930724+0000", 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1300000000000007e-06 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.729055+0000", 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.79833184800000001 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.812 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.729147+0000", 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1933000000000002e-05 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89306 46.14 46:29c5ea78:::benchmark_data_smithi099_233884_object89305:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.721727+0000", 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 464.07065032200001, 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1242219280000001, 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.813 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89306 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721727+0000", 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721727+0000", 2024-09-13T15:24:15.814 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.815 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.815 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.815 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.815 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721726+0000", 2024-09-13T15:24:15.815 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:24:15.815 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721765+0000", 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8800000000000001e-05 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721774+0000", 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3260000000000002e-06 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.721780+0000", 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5010000000000004e-06 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930834+0000", 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.20905432500000001 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930840+0000", 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4720000000000003e-06 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.817 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.845819+0000", 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91497909300000002 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.845949+0000", 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012993400000000001 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:89321 46.4 46:24190b25:::benchmark_data_smithi099_233884_object89320:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:31.819689+0000", 2024-09-13T15:24:15.818 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 463.97268790599998, 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.016089185, 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89321 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819689+0000", 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.819 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819689+0000", 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819686+0000", 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999967 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819746+0000", 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0609000000000002e-05 2024-09-13T15:24:15.820 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.821 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.821 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.821 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819787+0000", 2024-09-13T15:24:15.821 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0629e-05 2024-09-13T15:24:15.821 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.821 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.821 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.821 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.819800+0000", 2024-09-13T15:24:15.821 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2469999999999999e-05 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930861+0000", 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.111061036 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:31.930867+0000", 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4110000000000001e-06 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.822 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.729174+0000", 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.79830749499999998 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.729330+0000", 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015536 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.823 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.729996+0000", 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00066596500000000002 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.733110+0000", 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031144440000000001 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.824 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.743335+0000", 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010225237 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.835687+0000", 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.092351741000000001 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.835738+0000", 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0723999999999997e-05 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.825 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:32.835778+0000", 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0571999999999997e-05 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:107356 46.4 46:27540f99:::benchmark_data_smithi099_233884_object4943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:57.940638+0000", 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 437.85173968499998, 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68978922200000004, 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.826 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 107356 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940638+0000", 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.827 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940638+0000", 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940636+0000", 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940645+0000", 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2750000000000001e-06 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.828 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940664+0000", 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8668000000000002e-05 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940671+0000", 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7510000000000008e-06 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940727+0000", 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6079999999999998e-05 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.829 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940882+0000", 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015453299999999999 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.940988+0000", 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000106291 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.942812+0000", 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018234010000000001 2024-09-13T15:24:15.830 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.967954+0000", 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025142178000000001 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630310+0000", 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.66235603499999995 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630380+0000", 2024-09-13T15:24:15.831 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0083e-05 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630427+0000", 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6864999999999999e-05 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:107364 46.a 46:5497e79b:::benchmark_data_smithi099_233884_object4951:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:57.968424+0000", 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 437.823953485, 2024-09-13T15:24:15.832 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.66201485000000004, 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 107364 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968424+0000", 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.833 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968424+0000", 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968422+0000", 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968431+0000", 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8980000000000002e-06 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.834 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968446+0000", 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4873e-05 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968452+0000", 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4679999999999997e-06 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968508+0000", 2024-09-13T15:24:15.835 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5238999999999997e-05 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968604+0000", 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6761000000000004e-05 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.968670+0000", 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5514999999999999e-05 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.836 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972366+0000", 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036965930000000002 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.973672+0000", 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013058600000000001 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630369+0000", 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.65669628599999996 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.837 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630404+0000", 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5707999999999998e-05 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630439+0000", 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4303e-05 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.838 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:107368 46.4 46:220f6261:::benchmark_data_smithi099_233884_object4955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:16:57.972723+0000", 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 437.81965384099999, 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.65779028399999995, 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 107368 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.839 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972723+0000", 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972723+0000", 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972722+0000", 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.840 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972728+0000", 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.631e-06 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972739+0000", 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0876000000000001e-05 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972745+0000", 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1480000000000003e-06 2024-09-13T15:24:15.841 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972796+0000", 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1777999999999998e-05 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972912+0000", 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000115529 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.972984+0000", 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2489000000000003e-05 2024-09-13T15:24:15.842 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.974819+0000", 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018349550000000001 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:57.975128+0000", 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00030878200000000001 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.843 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630464+0000", 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.65533615300000003 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630493+0000", 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8561999999999999e-05 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:16:58.630514+0000", 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0755e-05 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.844 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:165282 46.4 46:26580601:::benchmark_data_smithi099_233884_object62869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:11.536208+0000", 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 364.25616909299998, 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.549781466, 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 165282 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.845 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.536208+0000", 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.536208+0000", 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.536207+0000", 2024-09-13T15:24:15.846 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.536213+0000", 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3840000000000002e-06 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.536225+0000", 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2019e-05 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.847 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.536232+0000", 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3709999999999996e-06 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.536273+0000", 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1582999999999998e-05 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.536361+0000", 2024-09-13T15:24:15.848 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7517999999999998e-05 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.536754+0000", 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00039340399999999998 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.539022+0000", 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022674980000000002 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.849 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.540970+0000", 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001948291 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.085910+0000", 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.54494026600000001 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.085948+0000", 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7549000000000003e-05 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.850 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.085990+0000", 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1737999999999999e-05 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:165289 46.a 46:5e6c10a6:::benchmark_data_smithi099_233884_object62876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:11.545901+0000", 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 364.246475928, 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.55440655800000005, 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.851 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 165289 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.545901+0000", 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.545901+0000", 2024-09-13T15:24:15.852 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.545900+0000", 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.545906+0000", 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1449999999999996e-06 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.545921+0000", 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.482e-05 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.545928+0000", 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9149999999999999e-06 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.545964+0000", 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6748000000000001e-05 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.854 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.546066+0000", 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010147800000000001 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.546151+0000", 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5177000000000001e-05 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.549287+0000", 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031361150000000001 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.549412+0000", 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012515099999999999 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.100224+0000", 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.55081140399999995 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.100267+0000", 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3130000000000002e-05 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.856 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.100308+0000", 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1054999999999999e-05 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:165300 46.a 46:5579bcb5:::benchmark_data_smithi099_233884_object62887:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:11.561021+0000", 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 364.23135625200001, 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53965930699999998, 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.857 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 165300 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.561021+0000", 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.561021+0000", 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.858 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.561020+0000", 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.561025+0000", 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6799999999999998e-06 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.859 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.561039+0000", 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3436e-05 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.561044+0000", 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1630000000000004e-06 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.561091+0000", 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7181000000000001e-05 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.561234+0000", 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014243200000000001 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.561359+0000", 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012541599999999999 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.564473+0000", 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031139449999999999 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.861 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.564531+0000", 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8009e-05 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.100424+0000", 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53589263700000001 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.100567+0000", 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014320199999999999 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.100680+0000", 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011344399999999999 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:165310 46.a 46:5d339f8e:::benchmark_data_smithi099_233884_object62897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:11.584994+0000", 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 364.20738313700002, 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51585168599999998, 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.863 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 165310 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.584994+0000", 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.864 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.584994+0000", 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.584992+0000", 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.584999+0000", 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4910000000000002e-06 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.585013+0000", 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4214e-05 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.585019+0000", 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4829999999999999e-06 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.585060+0000", 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1117e-05 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.585146+0000", 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6018000000000002e-05 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.585232+0000", 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6689999999999998e-05 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.587753+0000", 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025208980000000001 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.867 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:11.587984+0000", 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023092300000000001 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.100754+0000", 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51276993900000001 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.100806+0000", 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2040999999999999e-05 2024-09-13T15:24:15.868 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:12.100846+0000", 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9610000000000002e-05 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:166903 46.4 46:261e1d28:::benchmark_data_smithi099_233884_object64490:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:14.078652+0000", 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 361.71372544600001, 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.41824199499999998, 2024-09-13T15:24:15.869 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 166903 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.078652+0000", 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.078652+0000", 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.078650+0000", 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.078657+0000", 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8079999999999999e-06 2024-09-13T15:24:15.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.078671+0000", 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3859999999999999e-05 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.078677+0000", 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0920000000000003e-06 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.872 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.078729+0000", 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2448000000000003e-05 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.078851+0000", 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000121506 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.079343+0000", 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00049174400000000001 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.873 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.085952+0000", 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006609111 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.086955+0000", 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001003002 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.496819+0000", 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40986407200000002 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.874 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.496854+0000", 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4901999999999999e-05 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.496894+0000", 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9984999999999997e-05 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:166907 46.4 46:2119ef61:::benchmark_data_smithi099_233884_object64494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:14.079309+0000", 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 361.71306825400001, 2024-09-13T15:24:15.875 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.41770248300000001, 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 166907 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.079309+0000", 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.876 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.079309+0000", 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.079308+0000", 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.079312+0000", 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6859999999999999e-06 2024-09-13T15:24:15.877 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.079390+0000", 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7991999999999995e-05 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.079395+0000", 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3789999999999997e-06 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.081429+0000", 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020341159999999999 2024-09-13T15:24:15.878 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.081535+0000", 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000105679 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.081851+0000", 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00031639199999999998 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.879 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.086324+0000", 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044730270000000001 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.097845+0000", 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011520165000000001 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.496938+0000", 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39909368699999997 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.496990+0000", 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2011000000000001e-05 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.497011+0000", 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1109000000000001e-05 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.881 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:166908 46.4 46:220fe40b:::benchmark_data_smithi099_233884_object64495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:14.080567+0000", 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 361.71181006, 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.41658350999999999, 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 166908 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.882 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.080567+0000", 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.080567+0000", 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.080566+0000", 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.883 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.080571+0000", 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3499999999999996e-06 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.080583+0000", 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2128e-05 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.080590+0000", 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7610000000000002e-06 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.884 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.083762+0000", 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003172308 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.083834+0000", 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1359999999999994e-05 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.084135+0000", 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00030156200000000001 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.885 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.086330+0000", 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021950139999999999 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.098090+0000", 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011759181 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.497072+0000", 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39898241600000001 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.497120+0000", 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7814e-05 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.497151+0000", 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0828999999999999e-05 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:166918 46.4 46:25cb8c2f:::benchmark_data_smithi099_233884_object64505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:24:15.887 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:14.092487+0000", 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 361.69989046900002, 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40480204800000003, 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 166918 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.092487+0000", 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.888 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.092487+0000", 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.092486+0000", 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.889 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.092488+0000", 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.801e-06 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.092492+0000", 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.647e-06 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.092494+0000", 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3410000000000001e-06 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.092526+0000", 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1458999999999998e-05 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.092599+0000", 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3237000000000004e-05 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.093002+0000", 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00040291699999999998 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.891 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.094865+0000", 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018629300000000001 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.098112+0000", 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032477600000000001 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.497192+0000", 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39907937900000001 2024-09-13T15:24:15.892 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.497247+0000", 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4931e-05 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:14.497289+0000", 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1987999999999998e-05 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.893 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46109.0:184504 46.a 46:5c334d58:::benchmark_data_smithi099_233884_object82091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:18:36.585997+0000", 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 339.206380642, 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.46870666799999999, 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46109", 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 184504 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.894 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.585997+0000", 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.585997+0000", 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.585995+0000", 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.586001+0000", 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2940000000000002e-06 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.586018+0000", 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6452000000000001e-05 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.586023+0000", 2024-09-13T15:24:15.896 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8149999999999996e-06 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.586074+0000", 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1702000000000001e-05 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.586208+0000", 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000133628 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.897 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.586327+0000", 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000118715 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.591499+0000", 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051718179999999999 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:36.595253+0000", 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037539779999999998 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.898 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.054395+0000", 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.459142047 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.054552+0000", 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015741900000000001 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:18:37.054703+0000", 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015121199999999999 2024-09-13T15:24:15.899 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(3.13 1161/1160 [PushReplyOp(3:cd5a64a3:::100.00000000:head)])", 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:21:14.434105+0000", 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 181.358272223, 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00044175099999999999, 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.900 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.434105+0000", 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.434105+0000", 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.434104+0000", 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.901 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.434113+0000", 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6770000000000001e-06 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.434128+0000", 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5048000000000001e-05 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.434139+0000", 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1406000000000001e-05 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.434240+0000", 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100777 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.434547+0000", 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000306812 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(3.13 1161/1160 [PushReplyOp(3:c91d4a1d:::mds0_inotable:head)])", 2024-09-13T15:24:15.903 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2024-09-13T15:21:14.436281+0000", 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 181.356096205, 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00042820100000000002, 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.436281+0000", 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.436281+0000", 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.904 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.436280+0000", 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.436288+0000", 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5190000000000003e-06 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.436305+0000", 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6951000000000001e-05 2024-09-13T15:24:15.905 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.436317+0000", 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2490999999999999e-05 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.436398+0000", 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0617000000000004e-05 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: { 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2024-09-13T15:21:14.436709+0000", 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00031115100000000002 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.906 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.907 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.907 INFO:teuthology.orchestra.run.smithi177.stdout: } 2024-09-13T15:24:15.907 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-09-13T15:24:15.907 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-09-13T15:24:16.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:24:16.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:16 smithi099 bash[84576]: cluster 2024-09-13T15:24:15.381229+0000 mgr.y (mgr.26252) 1749 : cluster [DBG] pgmap v2409: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:16.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:16 smithi099 bash[84576]: cluster 2024-09-13T15:24:15.381229+0000 mgr.y (mgr.26252) 1749 : cluster [DBG] pgmap v2409: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:16.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:16 smithi099 bash[87147]: cluster 2024-09-13T15:24:15.381229+0000 mgr.y (mgr.26252) 1749 : cluster [DBG] pgmap v2409: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:16.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:16 smithi099 bash[87147]: cluster 2024-09-13T15:24:15.381229+0000 mgr.y (mgr.26252) 1749 : cluster [DBG] pgmap v2409: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:16.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:16 smithi177 bash[68594]: cluster 2024-09-13T15:24:15.381229+0000 mgr.y (mgr.26252) 1749 : cluster [DBG] pgmap v2409: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:16.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:16 smithi177 bash[68594]: cluster 2024-09-13T15:24:15.381229+0000 mgr.y (mgr.26252) 1749 : cluster [DBG] pgmap v2409: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:17.332 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:24:17] "GET /metrics HTTP/1.1" 200 34333 "" "Prometheus/2.51.0" 2024-09-13T15:24:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:16.750507+0000 mon.a (mon.0) 2944 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:16.750507+0000 mon.a (mon.0) 2944 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:16.758497+0000 mon.a (mon.0) 2945 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:16.758497+0000 mon.a (mon.0) 2945 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:16.992410+0000 mon.a (mon.0) 2946 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:16.992410+0000 mon.a (mon.0) 2946 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:16.999341+0000 mon.a (mon.0) 2947 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:16.999341+0000 mon.a (mon.0) 2947 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:17.617118+0000 mon.a (mon.0) 2948 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:17.617118+0000 mon.a (mon.0) 2948 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:17.618517+0000 mon.a (mon.0) 2949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:17.618517+0000 mon.a (mon.0) 2949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:17.626409+0000 mon.a (mon.0) 2950 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[87147]: audit 2024-09-13T15:24:17.626409+0000 mon.a (mon.0) 2950 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:16.750507+0000 mon.a (mon.0) 2944 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:16.750507+0000 mon.a (mon.0) 2944 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:16.758497+0000 mon.a (mon.0) 2945 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.076 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:16.758497+0000 mon.a (mon.0) 2945 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:16.992410+0000 mon.a (mon.0) 2946 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:16.992410+0000 mon.a (mon.0) 2946 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:16.999341+0000 mon.a (mon.0) 2947 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:16.999341+0000 mon.a (mon.0) 2947 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:17.617118+0000 mon.a (mon.0) 2948 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:24:18.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:17.617118+0000 mon.a (mon.0) 2948 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:24:18.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:17.618517+0000 mon.a (mon.0) 2949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:24:18.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:17.618517+0000 mon.a (mon.0) 2949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:24:18.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:17.626409+0000 mon.a (mon.0) 2950 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.077 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:17 smithi099 bash[84576]: audit 2024-09-13T15:24:17.626409+0000 mon.a (mon.0) 2950 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:16.750507+0000 mon.a (mon.0) 2944 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:16.750507+0000 mon.a (mon.0) 2944 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:16.758497+0000 mon.a (mon.0) 2945 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:16.758497+0000 mon.a (mon.0) 2945 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:16.992410+0000 mon.a (mon.0) 2946 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:16.992410+0000 mon.a (mon.0) 2946 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:16.999341+0000 mon.a (mon.0) 2947 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:16.999341+0000 mon.a (mon.0) 2947 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.113 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:17.617118+0000 mon.a (mon.0) 2948 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:24:18.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:17.617118+0000 mon.a (mon.0) 2948 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:24:18.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:17.618517+0000 mon.a (mon.0) 2949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:24:18.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:17.618517+0000 mon.a (mon.0) 2949 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:24:18.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:17.626409+0000 mon.a (mon.0) 2950 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:18.114 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:17 smithi177 bash[68594]: audit 2024-09-13T15:24:17.626409+0000 mon.a (mon.0) 2950 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:24:19.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:19 smithi099 bash[84576]: cluster 2024-09-13T15:24:17.381972+0000 mgr.y (mgr.26252) 1750 : cluster [DBG] pgmap v2410: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:19.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:19 smithi099 bash[84576]: cluster 2024-09-13T15:24:17.381972+0000 mgr.y (mgr.26252) 1750 : cluster [DBG] pgmap v2410: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:19.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:19 smithi099 bash[84576]: audit 2024-09-13T15:24:18.332043+0000 mon.a (mon.0) 2951 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:19.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:19 smithi099 bash[84576]: audit 2024-09-13T15:24:18.332043+0000 mon.a (mon.0) 2951 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:19.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:19 smithi099 bash[87147]: cluster 2024-09-13T15:24:17.381972+0000 mgr.y (mgr.26252) 1750 : cluster [DBG] pgmap v2410: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:19.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:19 smithi099 bash[87147]: cluster 2024-09-13T15:24:17.381972+0000 mgr.y (mgr.26252) 1750 : cluster [DBG] pgmap v2410: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:19.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:19 smithi099 bash[87147]: audit 2024-09-13T15:24:18.332043+0000 mon.a (mon.0) 2951 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:19.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:19 smithi099 bash[87147]: audit 2024-09-13T15:24:18.332043+0000 mon.a (mon.0) 2951 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:19.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:19 smithi177 bash[68594]: cluster 2024-09-13T15:24:17.381972+0000 mgr.y (mgr.26252) 1750 : cluster [DBG] pgmap v2410: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:19.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:19 smithi177 bash[68594]: cluster 2024-09-13T15:24:17.381972+0000 mgr.y (mgr.26252) 1750 : cluster [DBG] pgmap v2410: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:19.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:19 smithi177 bash[68594]: audit 2024-09-13T15:24:18.332043+0000 mon.a (mon.0) 2951 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:19.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:19 smithi177 bash[68594]: audit 2024-09-13T15:24:18.332043+0000 mon.a (mon.0) 2951 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:20.011 INFO:teuthology.orchestra.run.smithi099.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T15:24:20.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:20 smithi099 bash[84576]: audit 2024-09-13T15:24:19.956884+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:20.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:20 smithi099 bash[84576]: audit 2024-09-13T15:24:19.956884+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:20.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:20 smithi099 bash[87147]: audit 2024-09-13T15:24:19.956884+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:20.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:20 smithi099 bash[87147]: audit 2024-09-13T15:24:19.956884+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:20.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:24:20 smithi177 bash[61439]: ts=2024-09-13T15:24:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:24:20.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:20 smithi177 bash[68594]: audit 2024-09-13T15:24:19.956884+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:20.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:20 smithi177 bash[68594]: audit 2024-09-13T15:24:19.956884+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:21.025 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 3 1 2024-09-13T15:24:21.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[84576]: cluster 2024-09-13T15:24:19.382842+0000 mgr.y (mgr.26252) 1751 : cluster [DBG] pgmap v2411: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:21.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[84576]: cluster 2024-09-13T15:24:19.382842+0000 mgr.y (mgr.26252) 1751 : cluster [DBG] pgmap v2411: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:21.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[84576]: audit 2024-09-13T15:24:20.010673+0000 mon.a (mon.0) 2953 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:21.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[84576]: audit 2024-09-13T15:24:20.010673+0000 mon.a (mon.0) 2953 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:21.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[84576]: cluster 2024-09-13T15:24:20.018815+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2024-09-13T15:24:21.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[84576]: cluster 2024-09-13T15:24:20.018815+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2024-09-13T15:24:21.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[87147]: cluster 2024-09-13T15:24:19.382842+0000 mgr.y (mgr.26252) 1751 : cluster [DBG] pgmap v2411: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:21.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[87147]: cluster 2024-09-13T15:24:19.382842+0000 mgr.y (mgr.26252) 1751 : cluster [DBG] pgmap v2411: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:21.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[87147]: audit 2024-09-13T15:24:20.010673+0000 mon.a (mon.0) 2953 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:21.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[87147]: audit 2024-09-13T15:24:20.010673+0000 mon.a (mon.0) 2953 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:21.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[87147]: cluster 2024-09-13T15:24:20.018815+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2024-09-13T15:24:21.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:21 smithi099 bash[87147]: cluster 2024-09-13T15:24:20.018815+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2024-09-13T15:24:21.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:21 smithi177 bash[68594]: cluster 2024-09-13T15:24:19.382842+0000 mgr.y (mgr.26252) 1751 : cluster [DBG] pgmap v2411: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:21.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:21 smithi177 bash[68594]: cluster 2024-09-13T15:24:19.382842+0000 mgr.y (mgr.26252) 1751 : cluster [DBG] pgmap v2411: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:21.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:21 smithi177 bash[68594]: audit 2024-09-13T15:24:20.010673+0000 mon.a (mon.0) 2953 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:21.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:21 smithi177 bash[68594]: audit 2024-09-13T15:24:20.010673+0000 mon.a (mon.0) 2953 : audit [INF] from='client.? 172.21.15.99:0/857390013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:21.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:21 smithi177 bash[68594]: cluster 2024-09-13T15:24:20.018815+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2024-09-13T15:24:21.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:21 smithi177 bash[68594]: cluster 2024-09-13T15:24:20.018815+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2024-09-13T15:24:21.428 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:24:23.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:23 smithi099 bash[84576]: cluster 2024-09-13T15:24:21.383607+0000 mgr.y (mgr.26252) 1752 : cluster [DBG] pgmap v2413: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:23.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:23 smithi099 bash[84576]: cluster 2024-09-13T15:24:21.383607+0000 mgr.y (mgr.26252) 1752 : cluster [DBG] pgmap v2413: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:23.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:23 smithi099 bash[87147]: cluster 2024-09-13T15:24:21.383607+0000 mgr.y (mgr.26252) 1752 : cluster [DBG] pgmap v2413: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:23.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:23 smithi099 bash[87147]: cluster 2024-09-13T15:24:21.383607+0000 mgr.y (mgr.26252) 1752 : cluster [DBG] pgmap v2413: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:23.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:23 smithi177 bash[68594]: cluster 2024-09-13T15:24:21.383607+0000 mgr.y (mgr.26252) 1752 : cluster [DBG] pgmap v2413: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:23.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:23 smithi177 bash[68594]: cluster 2024-09-13T15:24:21.383607+0000 mgr.y (mgr.26252) 1752 : cluster [DBG] pgmap v2413: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:25.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:25 smithi099 bash[84576]: cluster 2024-09-13T15:24:23.384557+0000 mgr.y (mgr.26252) 1753 : cluster [DBG] pgmap v2414: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:25.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:25 smithi099 bash[84576]: cluster 2024-09-13T15:24:23.384557+0000 mgr.y (mgr.26252) 1753 : cluster [DBG] pgmap v2414: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:25 smithi099 bash[87147]: cluster 2024-09-13T15:24:23.384557+0000 mgr.y (mgr.26252) 1753 : cluster [DBG] pgmap v2414: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:25.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:25 smithi099 bash[87147]: cluster 2024-09-13T15:24:23.384557+0000 mgr.y (mgr.26252) 1753 : cluster [DBG] pgmap v2414: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:25.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:25 smithi177 bash[68594]: cluster 2024-09-13T15:24:23.384557+0000 mgr.y (mgr.26252) 1753 : cluster [DBG] pgmap v2414: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:25.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:25 smithi177 bash[68594]: cluster 2024-09-13T15:24:23.384557+0000 mgr.y (mgr.26252) 1753 : cluster [DBG] pgmap v2414: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:25.572 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:24:25.612 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:24:25.612 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:24:25.612 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2024-09-13T15:24:25.612 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:24:27.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:24:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:24:27] "GET /metrics HTTP/1.1" 200 34332 "" "Prometheus/2.51.0" 2024-09-13T15:24:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:27 smithi099 bash[84576]: cluster 2024-09-13T15:24:25.385515+0000 mgr.y (mgr.26252) 1754 : cluster [DBG] pgmap v2415: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:27 smithi099 bash[84576]: cluster 2024-09-13T15:24:25.385515+0000 mgr.y (mgr.26252) 1754 : cluster [DBG] pgmap v2415: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:27 smithi099 bash[87147]: cluster 2024-09-13T15:24:25.385515+0000 mgr.y (mgr.26252) 1754 : cluster [DBG] pgmap v2415: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:27 smithi099 bash[87147]: cluster 2024-09-13T15:24:25.385515+0000 mgr.y (mgr.26252) 1754 : cluster [DBG] pgmap v2415: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:27.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T15:24:27.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:27 smithi177 bash[68594]: cluster 2024-09-13T15:24:25.385515+0000 mgr.y (mgr.26252) 1754 : cluster [DBG] pgmap v2415: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:27.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:27 smithi177 bash[68594]: cluster 2024-09-13T15:24:25.385515+0000 mgr.y (mgr.26252) 1754 : cluster [DBG] pgmap v2415: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:29.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:29 smithi099 bash[84576]: cluster 2024-09-13T15:24:27.386627+0000 mgr.y (mgr.26252) 1755 : cluster [DBG] pgmap v2416: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:29.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:29 smithi099 bash[84576]: cluster 2024-09-13T15:24:27.386627+0000 mgr.y (mgr.26252) 1755 : cluster [DBG] pgmap v2416: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:29.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:29 smithi099 bash[87147]: cluster 2024-09-13T15:24:27.386627+0000 mgr.y (mgr.26252) 1755 : cluster [DBG] pgmap v2416: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:29.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:29 smithi099 bash[87147]: cluster 2024-09-13T15:24:27.386627+0000 mgr.y (mgr.26252) 1755 : cluster [DBG] pgmap v2416: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:29.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:29 smithi177 bash[68594]: cluster 2024-09-13T15:24:27.386627+0000 mgr.y (mgr.26252) 1755 : cluster [DBG] pgmap v2416: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:29.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:29 smithi177 bash[68594]: cluster 2024-09-13T15:24:27.386627+0000 mgr.y (mgr.26252) 1755 : cluster [DBG] pgmap v2416: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:30.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:24:30 smithi177 bash[61439]: ts=2024-09-13T15:24:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:24:31.033 INFO:teuthology.orchestra.run.smithi099.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-09-13T15:24:31.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:31 smithi099 bash[87147]: cluster 2024-09-13T15:24:29.387491+0000 mgr.y (mgr.26252) 1756 : cluster [DBG] pgmap v2417: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:31.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:31 smithi099 bash[87147]: cluster 2024-09-13T15:24:29.387491+0000 mgr.y (mgr.26252) 1756 : cluster [DBG] pgmap v2417: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:31.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:31 smithi099 bash[87147]: audit 2024-09-13T15:24:30.730995+0000 mon.a (mon.0) 2955 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:31.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:31 smithi099 bash[87147]: audit 2024-09-13T15:24:30.730995+0000 mon.a (mon.0) 2955 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:31.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:31 smithi099 bash[84576]: cluster 2024-09-13T15:24:29.387491+0000 mgr.y (mgr.26252) 1756 : cluster [DBG] pgmap v2417: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:31.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:31 smithi099 bash[84576]: cluster 2024-09-13T15:24:29.387491+0000 mgr.y (mgr.26252) 1756 : cluster [DBG] pgmap v2417: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:31.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:31 smithi099 bash[84576]: audit 2024-09-13T15:24:30.730995+0000 mon.a (mon.0) 2955 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:31.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:31 smithi099 bash[84576]: audit 2024-09-13T15:24:30.730995+0000 mon.a (mon.0) 2955 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:31.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:31 smithi177 bash[68594]: cluster 2024-09-13T15:24:29.387491+0000 mgr.y (mgr.26252) 1756 : cluster [DBG] pgmap v2417: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:31.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:31 smithi177 bash[68594]: cluster 2024-09-13T15:24:29.387491+0000 mgr.y (mgr.26252) 1756 : cluster [DBG] pgmap v2417: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:31.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:31 smithi177 bash[68594]: audit 2024-09-13T15:24:30.730995+0000 mon.a (mon.0) 2955 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:31.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:31 smithi177 bash[68594]: audit 2024-09-13T15:24:30.730995+0000 mon.a (mon.0) 2955 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:24:32.033 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-09-13T15:24:32.033 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-09-13T15:24:32.033 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-09-13T15:24:32.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:32 smithi099 bash[84576]: audit 2024-09-13T15:24:31.032844+0000 mon.a (mon.0) 2956 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:32.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:32 smithi099 bash[84576]: audit 2024-09-13T15:24:31.032844+0000 mon.a (mon.0) 2956 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:32.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:32 smithi099 bash[84576]: cluster 2024-09-13T15:24:31.037945+0000 mon.a (mon.0) 2957 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2024-09-13T15:24:32.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:32 smithi099 bash[84576]: cluster 2024-09-13T15:24:31.037945+0000 mon.a (mon.0) 2957 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2024-09-13T15:24:32.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:32 smithi099 bash[87147]: audit 2024-09-13T15:24:31.032844+0000 mon.a (mon.0) 2956 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:32.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:32 smithi099 bash[87147]: audit 2024-09-13T15:24:31.032844+0000 mon.a (mon.0) 2956 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:32.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:32 smithi099 bash[87147]: cluster 2024-09-13T15:24:31.037945+0000 mon.a (mon.0) 2957 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2024-09-13T15:24:32.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:32 smithi099 bash[87147]: cluster 2024-09-13T15:24:31.037945+0000 mon.a (mon.0) 2957 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2024-09-13T15:24:32.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:32 smithi177 bash[68594]: audit 2024-09-13T15:24:31.032844+0000 mon.a (mon.0) 2956 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:32.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:32 smithi177 bash[68594]: audit 2024-09-13T15:24:31.032844+0000 mon.a (mon.0) 2956 : audit [INF] from='client.? 172.21.15.99:0/2553779806' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:24:32.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:32 smithi177 bash[68594]: cluster 2024-09-13T15:24:31.037945+0000 mon.a (mon.0) 2957 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2024-09-13T15:24:32.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:32 smithi177 bash[68594]: cluster 2024-09-13T15:24:31.037945+0000 mon.a (mon.0) 2957 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2024-09-13T15:24:32.513 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:24:33.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:33 smithi099 bash[84576]: cluster 2024-09-13T15:24:31.388130+0000 mgr.y (mgr.26252) 1757 : cluster [DBG] pgmap v2419: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:33.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:33 smithi099 bash[84576]: cluster 2024-09-13T15:24:31.388130+0000 mgr.y (mgr.26252) 1757 : cluster [DBG] pgmap v2419: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:33.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:33 smithi099 bash[87147]: cluster 2024-09-13T15:24:31.388130+0000 mgr.y (mgr.26252) 1757 : cluster [DBG] pgmap v2419: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:33.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:33 smithi099 bash[87147]: cluster 2024-09-13T15:24:31.388130+0000 mgr.y (mgr.26252) 1757 : cluster [DBG] pgmap v2419: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:33.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:33 smithi177 bash[68594]: cluster 2024-09-13T15:24:31.388130+0000 mgr.y (mgr.26252) 1757 : cluster [DBG] pgmap v2419: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:33.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:33 smithi177 bash[68594]: cluster 2024-09-13T15:24:31.388130+0000 mgr.y (mgr.26252) 1757 : cluster [DBG] pgmap v2419: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:34.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:34 smithi099 bash[84576]: audit 2024-09-13T15:24:33.332182+0000 mon.a (mon.0) 2958 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:34.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:34 smithi099 bash[84576]: audit 2024-09-13T15:24:33.332182+0000 mon.a (mon.0) 2958 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:34 smithi099 bash[87147]: audit 2024-09-13T15:24:33.332182+0000 mon.a (mon.0) 2958 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:34.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:34 smithi099 bash[87147]: audit 2024-09-13T15:24:33.332182+0000 mon.a (mon.0) 2958 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:34 smithi177 bash[68594]: audit 2024-09-13T15:24:33.332182+0000 mon.a (mon.0) 2958 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:34.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:34 smithi177 bash[68594]: audit 2024-09-13T15:24:33.332182+0000 mon.a (mon.0) 2958 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:35.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:35 smithi099 bash[84576]: cluster 2024-09-13T15:24:33.389042+0000 mgr.y (mgr.26252) 1758 : cluster [DBG] pgmap v2420: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:35.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:35 smithi099 bash[84576]: cluster 2024-09-13T15:24:33.389042+0000 mgr.y (mgr.26252) 1758 : cluster [DBG] pgmap v2420: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:35.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:35 smithi099 bash[87147]: cluster 2024-09-13T15:24:33.389042+0000 mgr.y (mgr.26252) 1758 : cluster [DBG] pgmap v2420: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:35.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:35 smithi099 bash[87147]: cluster 2024-09-13T15:24:33.389042+0000 mgr.y (mgr.26252) 1758 : cluster [DBG] pgmap v2420: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:35.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:35 smithi177 bash[68594]: cluster 2024-09-13T15:24:33.389042+0000 mgr.y (mgr.26252) 1758 : cluster [DBG] pgmap v2420: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:35.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:35 smithi177 bash[68594]: cluster 2024-09-13T15:24:33.389042+0000 mgr.y (mgr.26252) 1758 : cluster [DBG] pgmap v2420: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:36.995 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:24:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2024-09-13T15:24:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2024-09-13T15:24:36.995 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2024-09-13T15:24:36.995 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:24:37.304 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:37 smithi099 bash[84576]: cluster 2024-09-13T15:24:35.389780+0000 mgr.y (mgr.26252) 1759 : cluster [DBG] pgmap v2421: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:37.305 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:37 smithi099 bash[84576]: cluster 2024-09-13T15:24:35.389780+0000 mgr.y (mgr.26252) 1759 : cluster [DBG] pgmap v2421: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:37.305 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:24:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:24:37] "GET /metrics HTTP/1.1" 200 34328 "" "Prometheus/2.51.0" 2024-09-13T15:24:37.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:37 smithi099 bash[87147]: cluster 2024-09-13T15:24:35.389780+0000 mgr.y (mgr.26252) 1759 : cluster [DBG] pgmap v2421: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:37.305 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:37 smithi099 bash[87147]: cluster 2024-09-13T15:24:35.389780+0000 mgr.y (mgr.26252) 1759 : cluster [DBG] pgmap v2421: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:37.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:37 smithi177 bash[68594]: cluster 2024-09-13T15:24:35.389780+0000 mgr.y (mgr.26252) 1759 : cluster [DBG] pgmap v2421: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:37.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:37 smithi177 bash[68594]: cluster 2024-09-13T15:24:35.389780+0000 mgr.y (mgr.26252) 1759 : cluster [DBG] pgmap v2421: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:37.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T15:24:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:39 smithi099 bash[84576]: cluster 2024-09-13T15:24:37.390763+0000 mgr.y (mgr.26252) 1760 : cluster [DBG] pgmap v2422: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:39.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:39 smithi099 bash[84576]: cluster 2024-09-13T15:24:37.390763+0000 mgr.y (mgr.26252) 1760 : cluster [DBG] pgmap v2422: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:39.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:39 smithi099 bash[87147]: cluster 2024-09-13T15:24:37.390763+0000 mgr.y (mgr.26252) 1760 : cluster [DBG] pgmap v2422: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:39.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:39 smithi099 bash[87147]: cluster 2024-09-13T15:24:37.390763+0000 mgr.y (mgr.26252) 1760 : cluster [DBG] pgmap v2422: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:39.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:39 smithi177 bash[68594]: cluster 2024-09-13T15:24:37.390763+0000 mgr.y (mgr.26252) 1760 : cluster [DBG] pgmap v2422: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:39.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:39 smithi177 bash[68594]: cluster 2024-09-13T15:24:37.390763+0000 mgr.y (mgr.26252) 1760 : cluster [DBG] pgmap v2422: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:40.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:24:40 smithi177 bash[61439]: ts=2024-09-13T15:24:40.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:24:41.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:41 smithi177 bash[68594]: cluster 2024-09-13T15:24:39.391668+0000 mgr.y (mgr.26252) 1761 : cluster [DBG] pgmap v2423: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:41.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:41 smithi177 bash[68594]: cluster 2024-09-13T15:24:39.391668+0000 mgr.y (mgr.26252) 1761 : cluster [DBG] pgmap v2423: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:41.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:41 smithi099 bash[84576]: cluster 2024-09-13T15:24:39.391668+0000 mgr.y (mgr.26252) 1761 : cluster [DBG] pgmap v2423: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:41.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:41 smithi099 bash[84576]: cluster 2024-09-13T15:24:39.391668+0000 mgr.y (mgr.26252) 1761 : cluster [DBG] pgmap v2423: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:41.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:41 smithi099 bash[87147]: cluster 2024-09-13T15:24:39.391668+0000 mgr.y (mgr.26252) 1761 : cluster [DBG] pgmap v2423: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:41.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:41 smithi099 bash[87147]: cluster 2024-09-13T15:24:39.391668+0000 mgr.y (mgr.26252) 1761 : cluster [DBG] pgmap v2423: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:42.773 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:24:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:43 smithi099 bash[84576]: cluster 2024-09-13T15:24:41.392459+0000 mgr.y (mgr.26252) 1762 : cluster [DBG] pgmap v2424: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:43.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:43 smithi099 bash[84576]: cluster 2024-09-13T15:24:41.392459+0000 mgr.y (mgr.26252) 1762 : cluster [DBG] pgmap v2424: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:43.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:43 smithi099 bash[87147]: cluster 2024-09-13T15:24:41.392459+0000 mgr.y (mgr.26252) 1762 : cluster [DBG] pgmap v2424: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:43.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:43 smithi099 bash[87147]: cluster 2024-09-13T15:24:41.392459+0000 mgr.y (mgr.26252) 1762 : cluster [DBG] pgmap v2424: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:43.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:43 smithi177 bash[68594]: cluster 2024-09-13T15:24:41.392459+0000 mgr.y (mgr.26252) 1762 : cluster [DBG] pgmap v2424: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:43.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:43 smithi177 bash[68594]: cluster 2024-09-13T15:24:41.392459+0000 mgr.y (mgr.26252) 1762 : cluster [DBG] pgmap v2424: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:45.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:45 smithi099 bash[84576]: cluster 2024-09-13T15:24:43.393387+0000 mgr.y (mgr.26252) 1763 : cluster [DBG] pgmap v2425: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:45.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:45 smithi099 bash[84576]: cluster 2024-09-13T15:24:43.393387+0000 mgr.y (mgr.26252) 1763 : cluster [DBG] pgmap v2425: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:45.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:45 smithi099 bash[87147]: cluster 2024-09-13T15:24:43.393387+0000 mgr.y (mgr.26252) 1763 : cluster [DBG] pgmap v2425: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:45.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:45 smithi099 bash[87147]: cluster 2024-09-13T15:24:43.393387+0000 mgr.y (mgr.26252) 1763 : cluster [DBG] pgmap v2425: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:45.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:45 smithi177 bash[68594]: cluster 2024-09-13T15:24:43.393387+0000 mgr.y (mgr.26252) 1763 : cluster [DBG] pgmap v2425: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:45.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:45 smithi177 bash[68594]: cluster 2024-09-13T15:24:43.393387+0000 mgr.y (mgr.26252) 1763 : cluster [DBG] pgmap v2425: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:47.216 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-09-13T15:24:47.216 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2024-09-13T15:24:47.217 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2024-09-13T15:24:47.217 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2024-09-13T15:24:47.217 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.217 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:88714 46.5 46:ab9030a0:::benchmark_data_smithi099_233884_object88713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-09-13T15:24:47.217 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:30.089675+0000", 2024-09-13T15:24:47.217 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 497.12552731199997, 2024-09-13T15:24:47.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.52749165399999998, 2024-09-13T15:24:47.218 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:24:47.218 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:24:47.218 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:47.218 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:24:47.218 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:24:47.219 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:47.219 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 88714 2024-09-13T15:24:47.219 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.219 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:24:47.219 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.220 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:24:47.220 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.089675+0000", 2024-09-13T15:24:47.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.220 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.220 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:24:47.220 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.089675+0000", 2024-09-13T15:24:47.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:24:47.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.089677+0000", 2024-09-13T15:24:47.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.728e-06 2024-09-13T15:24:47.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:24:47.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.089703+0000", 2024-09-13T15:24:47.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.584e-05 2024-09-13T15:24:47.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:24:47.223 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.089718+0000", 2024-09-13T15:24:47.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5167999999999999e-05 2024-09-13T15:24:47.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.224 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:24:47.224 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.089726+0000", 2024-09-13T15:24:47.224 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9130000000000003e-06 2024-09-13T15:24:47.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:24:47.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.089793+0000", 2024-09-13T15:24:47.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7311999999999995e-05 2024-09-13T15:24:47.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:24:47.226 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.089926+0000", 2024-09-13T15:24:47.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013346399999999999 2024-09-13T15:24:47.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.227 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2024-09-13T15:24:47.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.090050+0000", 2024-09-13T15:24:47.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123807 2024-09-13T15:24:47.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.227 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:24:47.228 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.095071+0000", 2024-09-13T15:24:47.228 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005020633 2024-09-13T15:24:47.228 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:47.228 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.600094+0000", 2024-09-13T15:24:47.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.50502297699999998 2024-09-13T15:24:47.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:47.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.617034+0000", 2024-09-13T15:24:47.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016940368000000001 2024-09-13T15:24:47.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:24:47.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.617109+0000", 2024-09-13T15:24:47.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4661999999999998e-05 2024-09-13T15:24:47.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:24:47.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:30.617167+0000", 2024-09-13T15:24:47.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7781999999999997e-05 2024-09-13T15:24:47.230 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.231 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:24:47.231 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.231 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:89305 46.15 46:a817b1b2:::benchmark_data_smithi099_233884_object89304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:24:47.231 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:31.713474+0000", 2024-09-13T15:24:47.231 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 495.50172864799998, 2024-09-13T15:24:47.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0457813140000001, 2024-09-13T15:24:47.231 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:24:47.232 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:24:47.232 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T15:24:47.232 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:24:47.232 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:24:47.232 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:47.233 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 89305 2024-09-13T15:24:47.233 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.233 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:24:47.233 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.233 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:24:47.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.713474+0000", 2024-09-13T15:24:47.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.233 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.713474+0000", 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.713475+0000", 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5319999999999999e-06 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.713515+0000", 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0197000000000002e-05 2024-09-13T15:24:47.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.713529+0000", 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4083000000000001e-05 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.713539+0000", 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0051000000000001e-05 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.035792+0000", 2024-09-13T15:24:47.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.32225292799999999 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.035805+0000", 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2276000000000001e-05 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.759102+0000", 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.72329740600000003 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.759255+0000", 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000152841 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:89314 46.5 46:a16f7ddc:::benchmark_data_smithi099_233884_object89313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:31.730381+0000", 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 495.48482121299998, 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.030356515, 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:24:47.237 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:24:47.238 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:47.238 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:24:47.238 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:24:47.238 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:47.238 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 89314 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.730381+0000", 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.730381+0000", 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:24:47.240 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.730383+0000", 2024-09-13T15:24:47.240 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8470000000000001e-06 2024-09-13T15:24:47.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.240 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.730412+0000", 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9067000000000001e-05 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.730430+0000", 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7558999999999999e-05 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.730439+0000", 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0790000000000007e-06 2024-09-13T15:24:47.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.730500+0000", 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1870000000000002e-05 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:31.730505+0000", 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4449999999999998e-06 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:24:47.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.035811+0000", 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.30530630399999997 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.035820+0000", 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3329999999999997e-06 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.755105+0000", 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71928545399999999 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.755244+0000", 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000139047 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.755388+0000", 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014406900000000001 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.759300+0000", 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039116999999999997 2024-09-13T15:24:47.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.759330+0000", 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0292000000000001e-05 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.760625+0000", 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001295392 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.760698+0000", 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2285000000000003e-05 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:32.760738+0000", 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9771999999999998e-05 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46109.0:106842 46.15 46:adf50e2b:::benchmark_data_smithi099_233884_object4429:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-13T15:24:47.246 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:16:55.489748+0000", 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 471.72545451500002, 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.528270024, 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46109", 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2892350528", 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 106842 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.489748+0000", 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.489748+0000", 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.489749+0000", 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.68e-06 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:24:47.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.489756+0000", 2024-09-13T15:24:47.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4749999999999998e-06 2024-09-13T15:24:47.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:24:47.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.489773+0000", 2024-09-13T15:24:47.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7187e-05 2024-09-13T15:24:47.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.489781+0000", 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3070000000000003e-06 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.489853+0000", 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1459999999999997e-05 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.490013+0000", 2024-09-13T15:24:47.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000160378 2024-09-13T15:24:47.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2024-09-13T15:24:47.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.490134+0000", 2024-09-13T15:24:47.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120628 2024-09-13T15:24:47.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2024-09-13T15:24:47.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.491807+0000", 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001672775 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:55.494380+0000", 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025728980000000001 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.017924+0000", 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.52354492699999999 2024-09-13T15:24:47.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2024-09-13T15:24:47.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.017981+0000", 2024-09-13T15:24:47.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6184000000000003e-05 2024-09-13T15:24:47.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:24:47.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:16:56.018018+0000", 2024-09-13T15:24:47.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7125000000000001e-05 2024-09-13T15:24:47.254 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.254 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:24:47.254 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.254 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "MOSDPGPush(1.4 1147/1146 [PushOp(1:30306672:devicehealth::main.db.0000000000000000:head, version: 346'100, data_included: [0~327680], data_size: 327680, omap_header_size: 0, omap_entries_size: 0, attrset_size: 10, recovery_info: ObjectRecoveryInfo(1:30306672:devicehealth::main.db.0000000000000000:head@346'100, size: 327680, copy_subset: [(0, 327680)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 327680, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-09-13T15:24:47.254 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:19:58.645873+0000", 2024-09-13T15:24:47.254 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 288.569328762, 2024-09-13T15:24:47.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001536422, 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:19:58.645873+0000", 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:19:58.645873+0000", 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:19:58.645874+0000", 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9299999999999997e-07 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:24:47.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:19:58.646436+0000", 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00056163100000000004 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:19:58.646456+0000", 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0500999999999999e-05 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:19:58.646470+0000", 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3848e-05 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:19:58.646591+0000", 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012036000000000001 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:19:58.646605+0000", 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4703e-05 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:19:58.647410+0000", 2024-09-13T15:24:47.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00080458600000000004 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.34125.0:102 2.1 2:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e1155)", 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2024-09-13T15:20:48.930122+0000", 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 238.28508070300001, 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00043549099999999998, 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: "continuous": false, 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2024-09-13T15:24:47.259 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.34125", 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.99:0/2775623165", 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 102 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:20:48.930122+0000", 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:20:48.930122+0000", 2024-09-13T15:24:47.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:20:48.930125+0000", 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6770000000000001e-06 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:20:48.930145+0000", 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9623000000000002e-05 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:20:48.930187+0000", 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1835999999999997e-05 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:20:48.930205+0000", 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8298999999999999e-05 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:20:48.930346+0000", 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014054399999999999 2024-09-13T15:24:47.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2024-09-13T15:24:47.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2024-09-13T15:24:47.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2024-09-13T15:24:47.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2024-09-13T15:20:48.930557+0000", 2024-09-13T15:24:47.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021151199999999999 2024-09-13T15:24:47.263 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.263 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:24:47.263 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.263 INFO:teuthology.orchestra.run.smithi099.stdout: } 2024-09-13T15:24:47.263 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-09-13T15:24:47.263 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-09-13T15:24:47.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:47 smithi099 bash[84576]: cluster 2024-09-13T15:24:45.394290+0000 mgr.y (mgr.26252) 1764 : cluster [DBG] pgmap v2426: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:47.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:47 smithi099 bash[84576]: cluster 2024-09-13T15:24:45.394290+0000 mgr.y (mgr.26252) 1764 : cluster [DBG] pgmap v2426: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:47.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:24:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:24:47] "GET /metrics HTTP/1.1" 200 34328 "" "Prometheus/2.51.0" 2024-09-13T15:24:47.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:47 smithi099 bash[87147]: cluster 2024-09-13T15:24:45.394290+0000 mgr.y (mgr.26252) 1764 : cluster [DBG] pgmap v2426: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:47.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:47 smithi099 bash[87147]: cluster 2024-09-13T15:24:45.394290+0000 mgr.y (mgr.26252) 1764 : cluster [DBG] pgmap v2426: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:47.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:47 smithi177 bash[68594]: cluster 2024-09-13T15:24:45.394290+0000 mgr.y (mgr.26252) 1764 : cluster [DBG] pgmap v2426: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:47.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:47 smithi177 bash[68594]: cluster 2024-09-13T15:24:45.394290+0000 mgr.y (mgr.26252) 1764 : cluster [DBG] pgmap v2426: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:48.220 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-09-13T15:24:48.220 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T15:24:48.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T15:24:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:49 smithi177 bash[68594]: cluster 2024-09-13T15:24:47.395385+0000 mgr.y (mgr.26252) 1765 : cluster [DBG] pgmap v2427: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:49 smithi177 bash[68594]: cluster 2024-09-13T15:24:47.395385+0000 mgr.y (mgr.26252) 1765 : cluster [DBG] pgmap v2427: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:49 smithi177 bash[68594]: audit 2024-09-13T15:24:48.332553+0000 mon.a (mon.0) 2959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:49.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:49 smithi177 bash[68594]: audit 2024-09-13T15:24:48.332553+0000 mon.a (mon.0) 2959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:49.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:49 smithi099 bash[84576]: cluster 2024-09-13T15:24:47.395385+0000 mgr.y (mgr.26252) 1765 : cluster [DBG] pgmap v2427: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:49.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:49 smithi099 bash[84576]: cluster 2024-09-13T15:24:47.395385+0000 mgr.y (mgr.26252) 1765 : cluster [DBG] pgmap v2427: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:49.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:49 smithi099 bash[84576]: audit 2024-09-13T15:24:48.332553+0000 mon.a (mon.0) 2959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:49.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:49 smithi099 bash[84576]: audit 2024-09-13T15:24:48.332553+0000 mon.a (mon.0) 2959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:49 smithi099 bash[87147]: cluster 2024-09-13T15:24:47.395385+0000 mgr.y (mgr.26252) 1765 : cluster [DBG] pgmap v2427: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:49 smithi099 bash[87147]: cluster 2024-09-13T15:24:47.395385+0000 mgr.y (mgr.26252) 1765 : cluster [DBG] pgmap v2427: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:49 smithi099 bash[87147]: audit 2024-09-13T15:24:48.332553+0000 mon.a (mon.0) 2959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:49.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:49 smithi099 bash[87147]: audit 2024-09-13T15:24:48.332553+0000 mon.a (mon.0) 2959 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:24:50.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:24:50 smithi177 bash[61439]: ts=2024-09-13T15:24:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:24:51.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:51 smithi177 bash[68594]: cluster 2024-09-13T15:24:49.396354+0000 mgr.y (mgr.26252) 1766 : cluster [DBG] pgmap v2428: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:51.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:51 smithi177 bash[68594]: cluster 2024-09-13T15:24:49.396354+0000 mgr.y (mgr.26252) 1766 : cluster [DBG] pgmap v2428: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:51 smithi099 bash[84576]: cluster 2024-09-13T15:24:49.396354+0000 mgr.y (mgr.26252) 1766 : cluster [DBG] pgmap v2428: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:51.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:51 smithi099 bash[84576]: cluster 2024-09-13T15:24:49.396354+0000 mgr.y (mgr.26252) 1766 : cluster [DBG] pgmap v2428: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:51.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:51 smithi099 bash[87147]: cluster 2024-09-13T15:24:49.396354+0000 mgr.y (mgr.26252) 1766 : cluster [DBG] pgmap v2428: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:51.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:51 smithi099 bash[87147]: cluster 2024-09-13T15:24:49.396354+0000 mgr.y (mgr.26252) 1766 : cluster [DBG] pgmap v2428: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:53.011 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:24:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:53 smithi177 bash[68594]: cluster 2024-09-13T15:24:51.397224+0000 mgr.y (mgr.26252) 1767 : cluster [DBG] pgmap v2429: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:53 smithi177 bash[68594]: cluster 2024-09-13T15:24:51.397224+0000 mgr.y (mgr.26252) 1767 : cluster [DBG] pgmap v2429: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:53.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:53 smithi099 bash[84576]: cluster 2024-09-13T15:24:51.397224+0000 mgr.y (mgr.26252) 1767 : cluster [DBG] pgmap v2429: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:53.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:53 smithi099 bash[84576]: cluster 2024-09-13T15:24:51.397224+0000 mgr.y (mgr.26252) 1767 : cluster [DBG] pgmap v2429: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:53.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:53 smithi099 bash[87147]: cluster 2024-09-13T15:24:51.397224+0000 mgr.y (mgr.26252) 1767 : cluster [DBG] pgmap v2429: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:53.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:53 smithi099 bash[87147]: cluster 2024-09-13T15:24:51.397224+0000 mgr.y (mgr.26252) 1767 : cluster [DBG] pgmap v2429: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:55.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:55 smithi099 bash[84576]: cluster 2024-09-13T15:24:53.398312+0000 mgr.y (mgr.26252) 1768 : cluster [DBG] pgmap v2430: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:55.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:55 smithi099 bash[84576]: cluster 2024-09-13T15:24:53.398312+0000 mgr.y (mgr.26252) 1768 : cluster [DBG] pgmap v2430: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:55.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:55 smithi099 bash[87147]: cluster 2024-09-13T15:24:53.398312+0000 mgr.y (mgr.26252) 1768 : cluster [DBG] pgmap v2430: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:55.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:55 smithi099 bash[87147]: cluster 2024-09-13T15:24:53.398312+0000 mgr.y (mgr.26252) 1768 : cluster [DBG] pgmap v2430: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:55.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:55 smithi177 bash[68594]: cluster 2024-09-13T15:24:53.398312+0000 mgr.y (mgr.26252) 1768 : cluster [DBG] pgmap v2430: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:55.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:55 smithi177 bash[68594]: cluster 2024-09-13T15:24:53.398312+0000 mgr.y (mgr.26252) 1768 : cluster [DBG] pgmap v2430: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:57.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:57 smithi099 bash[84576]: cluster 2024-09-13T15:24:55.399202+0000 mgr.y (mgr.26252) 1769 : cluster [DBG] pgmap v2431: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:57 smithi099 bash[84576]: cluster 2024-09-13T15:24:55.399202+0000 mgr.y (mgr.26252) 1769 : cluster [DBG] pgmap v2431: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:57.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:24:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:24:57] "GET /metrics HTTP/1.1" 200 34330 "" "Prometheus/2.51.0" 2024-09-13T15:24:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:57 smithi099 bash[87147]: cluster 2024-09-13T15:24:55.399202+0000 mgr.y (mgr.26252) 1769 : cluster [DBG] pgmap v2431: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:57.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:57 smithi099 bash[87147]: cluster 2024-09-13T15:24:55.399202+0000 mgr.y (mgr.26252) 1769 : cluster [DBG] pgmap v2431: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:57.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:57 smithi177 bash[68594]: cluster 2024-09-13T15:24:55.399202+0000 mgr.y (mgr.26252) 1769 : cluster [DBG] pgmap v2431: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:57.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:57 smithi177 bash[68594]: cluster 2024-09-13T15:24:55.399202+0000 mgr.y (mgr.26252) 1769 : cluster [DBG] pgmap v2431: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:57.370 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T15:24:57.370 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":1179,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:10:21.363918+0000","modified":"2024-09-13T15:24:31.030134+0000","last_up_change":"2024-09-13T15:05:33.041673+0000","last_in_change":"2024-09-13T15:21:13.374925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":47,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:15:37.290226+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1157","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1129","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:21:51.531836+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:24:15.786407+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:24:16.627611+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0624feaa-339b-4468-be53-582a1f04352c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":558,"up_from":575,"up_thru":1162,"down_at":558,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6803","nonce":1813984480}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6805","nonce":1813984480}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6809","nonce":1813984480}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6807","nonce":1813984480}]},"public_addr":"172.21.15.99:6803/1813984480","cluster_addr":"172.21.15.99:6805/1813984480","heartbeat_back_addr":"172.21.15.99:6809/1813984480","heartbeat_front_addr":"172.21.15.99:6807/1813984480","state":["exists","up"]},{"osd":1,"uuid":"b7b63553-e49d-4636-b3e2-43b72556a1a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":638,"up_from":642,"up_thru":1160,"down_at":639,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6811","nonce":804978824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6813","nonce":804978824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6817","nonce":804978824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6815","nonce":804978824}]},"public_addr":"172.21.15.99:6811/804978824","cluster_addr":"172.21.15.99:6813/804978824","heartbeat_back_addr":"172.21.15.99:6817/804978824","heartbeat_front_addr":"172.21.15.99:6815/804978824","state":["exists","up"]},{"osd":2,"uuid":"3b9eff6d-5f5c-4843-a1ff-815a902167ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":656,"up_from":672,"up_thru":1164,"down_at":657,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6819","nonce":129558315}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6821","nonce":129558315}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6825","nonce":129558315}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6823","nonce":129558315}]},"public_addr":"172.21.15.99:6819/129558315","cluster_addr":"172.21.15.99:6821/129558315","heartbeat_back_addr":"172.21.15.99:6825/129558315","heartbeat_front_addr":"172.21.15.99:6823/129558315","state":["exists","up"]},{"osd":3,"uuid":"686c8d3c-9cab-4920-b02b-d243324f5808","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":706,"last_clean_end":1033,"up_from":1066,"up_thru":1162,"down_at":1034,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":2771933148},{"type":"v1","addr":"172.21.15.99:6827","nonce":2771933148}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":2771933148},{"type":"v1","addr":"172.21.15.99:6829","nonce":2771933148}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":2771933148},{"type":"v1","addr":"172.21.15.99:6833","nonce":2771933148}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":2771933148},{"type":"v1","addr":"172.21.15.99:6831","nonce":2771933148}]},"public_addr":"172.21.15.99:6827/2771933148","cluster_addr":"172.21.15.99:6829/2771933148","heartbeat_back_addr":"172.21.15.99:6833/2771933148","heartbeat_front_addr":"172.21.15.99:6831/2771933148","state":["exists","up"]},{"osd":4,"uuid":"2d56712b-a6fb-4089-9583-850beed92b37","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":724,"up_from":739,"up_thru":1160,"down_at":725,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6801","nonce":2135282964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6803","nonce":2135282964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6807","nonce":2135282964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6805","nonce":2135282964}]},"public_addr":"172.21.15.177:6801/2135282964","cluster_addr":"172.21.15.177:6803/2135282964","heartbeat_back_addr":"172.21.15.177:6807/2135282964","heartbeat_front_addr":"172.21.15.177:6805/2135282964","state":["exists","up"]},{"osd":5,"uuid":"414b880c-6534-495f-a54c-f8d28ba81c73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":1160,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6809","nonce":947070587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6811","nonce":947070587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6815","nonce":947070587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6813","nonce":947070587}]},"public_addr":"172.21.15.177:6809/947070587","cluster_addr":"172.21.15.177:6811/947070587","heartbeat_back_addr":"172.21.15.177:6815/947070587","heartbeat_front_addr":"172.21.15.177:6813/947070587","state":["exists","up"]},{"osd":6,"uuid":"ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":1160,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6817","nonce":3817692594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6819","nonce":3817692594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6823","nonce":3817692594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6821","nonce":3817692594}]},"public_addr":"172.21.15.177:6817/3817692594","cluster_addr":"172.21.15.177:6819/3817692594","heartbeat_back_addr":"172.21.15.177:6823/3817692594","heartbeat_front_addr":"172.21.15.177:6821/3817692594","state":["exists","up"]},{"osd":7,"uuid":"ad2c2977-fd6d-42f8-962e-907c2bf3204b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":1160,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6825","nonce":1984676946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6827","nonce":1984676946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6831","nonce":1984676946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6829","nonce":1984676946}]},"public_addr":"172.21.15.177:6825/1984676946","cluster_addr":"172.21.15.177:6827/1984676946","heartbeat_back_addr":"172.21.15.177:6831/1984676946","heartbeat_front_addr":"172.21.15.177:6829/1984676946","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:46.390607+0000","dead_epoch":557},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:38.398637+0000","dead_epoch":638},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:15:33.420625+0000","dead_epoch":656},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:30.964755+0000","dead_epoch":1033},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:22.513614+0000","dead_epoch":724},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:14.948720+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:19:10.431389+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:20:03.826389+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.12","mappings":[{"from":1,"to":0}]},{"pgid":"4.13","mappings":[{"from":1,"to":0}]},{"pgid":"4.14","mappings":[{"from":1,"to":5}]},{"pgid":"4.19","mappings":[{"from":3,"to":5}]},{"pgid":"4.1d","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:0/3935330909":"2024-09-13T15:40:15.846422+0000","172.21.15.99:0/3556003775":"2024-09-13T15:35:26.139282+0000","172.21.15.99:0/1151643634":"2024-09-14T14:31:47.884997+0000","172.21.15.177:6834/4079938285":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/881490667":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/4242039269":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/316966644":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/2140252887":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/17074640":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/3817526904":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6835/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.177:6835/4079938285":"2024-09-14T14:30:38.246301+0000","172.21.15.99:6800/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6834/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.99:6801/1402010506":"2024-09-14T14:31:47.884997+0000","172.21.15.177:0/1330817275":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/1638158155":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/4129805635":"2024-09-14T14:11:18.714884+0000","172.21.15.177:0/3478790043":"2024-09-14T14:30:38.246301+0000","172.21.15.99:6801/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/888319381":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/181419264":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/3483428223":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6800/1402010506":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/795365709":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/1548831286":"2024-09-14T14:11:18.714884+0000","172.21.15.177:0/3423158096":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/3394219806":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/3192879133":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/1363241618":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/405712510":"2024-09-14T14:21:07.041302+0000","172.21.15.177:0/1523474123":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/313522629":"2024-09-13T15:35:27.165268+0000","172.21.15.99:6800/2887456408":"2024-09-14T14:29:03.998039+0000","172.21.15.99:6801/2887456408":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/4277277716":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6800/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:6800/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6801/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/2372940082":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6801/3775052239":"2024-09-14T14:21:07.041302+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":227,"length":1},{"begin":229,"length":1},{"begin":231,"length":1}]},{"pool":13,"snaps":[{"begin":7,"length":2}]},{"pool":15,"snaps":[{"begin":35,"length":3},{"begin":43,"length":2},{"begin":47,"length":1}]},{"pool":16,"snaps":[{"begin":22,"length":1}]},{"pool":17,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":8,"length":3}]},{"pool":47,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T15:24:58.360 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T15:24:58.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T15:24:58.386 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:58 smithi099 bash[84576]: audit 2024-09-13T15:24:57.366640+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.99:0/2722947930' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:24:58.386 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:58 smithi099 bash[84576]: audit 2024-09-13T15:24:57.366640+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.99:0/2722947930' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:24:58.387 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:58 smithi099 bash[87147]: audit 2024-09-13T15:24:57.366640+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.99:0/2722947930' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:24:58.387 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:58 smithi099 bash[87147]: audit 2024-09-13T15:24:57.366640+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.99:0/2722947930' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:24:58.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:58 smithi177 bash[68594]: audit 2024-09-13T15:24:57.366640+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.99:0/2722947930' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:24:58.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:58 smithi177 bash[68594]: audit 2024-09-13T15:24:57.366640+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.99:0/2722947930' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:24:59.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:59 smithi099 bash[84576]: cluster 2024-09-13T15:24:57.399798+0000 mgr.y (mgr.26252) 1770 : cluster [DBG] pgmap v2432: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:59.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:24:59 smithi099 bash[84576]: cluster 2024-09-13T15:24:57.399798+0000 mgr.y (mgr.26252) 1770 : cluster [DBG] pgmap v2432: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:59.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:59 smithi099 bash[87147]: cluster 2024-09-13T15:24:57.399798+0000 mgr.y (mgr.26252) 1770 : cluster [DBG] pgmap v2432: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:59.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:24:59 smithi099 bash[87147]: cluster 2024-09-13T15:24:57.399798+0000 mgr.y (mgr.26252) 1770 : cluster [DBG] pgmap v2432: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:59.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:59 smithi177 bash[68594]: cluster 2024-09-13T15:24:57.399798+0000 mgr.y (mgr.26252) 1770 : cluster [DBG] pgmap v2432: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:24:59.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:24:59 smithi177 bash[68594]: cluster 2024-09-13T15:24:57.399798+0000 mgr.y (mgr.26252) 1770 : cluster [DBG] pgmap v2432: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:00.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:25:00 smithi177 bash[61439]: ts=2024-09-13T15:25:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:25:01.483 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:01 smithi099 bash[84576]: cluster 2024-09-13T15:24:59.400714+0000 mgr.y (mgr.26252) 1771 : cluster [DBG] pgmap v2433: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:01.483 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:01 smithi099 bash[84576]: cluster 2024-09-13T15:24:59.400714+0000 mgr.y (mgr.26252) 1771 : cluster [DBG] pgmap v2433: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:01.483 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:01 smithi099 bash[87147]: cluster 2024-09-13T15:24:59.400714+0000 mgr.y (mgr.26252) 1771 : cluster [DBG] pgmap v2433: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:01.483 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:01 smithi099 bash[87147]: cluster 2024-09-13T15:24:59.400714+0000 mgr.y (mgr.26252) 1771 : cluster [DBG] pgmap v2433: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:01.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:01 smithi177 bash[68594]: cluster 2024-09-13T15:24:59.400714+0000 mgr.y (mgr.26252) 1771 : cluster [DBG] pgmap v2433: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:01.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:01 smithi177 bash[68594]: cluster 2024-09-13T15:24:59.400714+0000 mgr.y (mgr.26252) 1771 : cluster [DBG] pgmap v2433: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:03.146 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:03.550 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:03 smithi099 bash[84576]: cluster 2024-09-13T15:25:01.401479+0000 mgr.y (mgr.26252) 1772 : cluster [DBG] pgmap v2434: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:03.550 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:03 smithi099 bash[84576]: cluster 2024-09-13T15:25:01.401479+0000 mgr.y (mgr.26252) 1772 : cluster [DBG] pgmap v2434: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:03.550 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:03 smithi099 bash[87147]: cluster 2024-09-13T15:25:01.401479+0000 mgr.y (mgr.26252) 1772 : cluster [DBG] pgmap v2434: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:03.550 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:03 smithi099 bash[87147]: cluster 2024-09-13T15:25:01.401479+0000 mgr.y (mgr.26252) 1772 : cluster [DBG] pgmap v2434: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:03 smithi177 bash[68594]: cluster 2024-09-13T15:25:01.401479+0000 mgr.y (mgr.26252) 1772 : cluster [DBG] pgmap v2434: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:03.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:03 smithi177 bash[68594]: cluster 2024-09-13T15:25:01.401479+0000 mgr.y (mgr.26252) 1772 : cluster [DBG] pgmap v2434: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:04.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:04 smithi099 bash[84576]: audit 2024-09-13T15:25:03.333050+0000 mon.a (mon.0) 2960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:04.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:04 smithi099 bash[84576]: audit 2024-09-13T15:25:03.333050+0000 mon.a (mon.0) 2960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:04 smithi099 bash[87147]: audit 2024-09-13T15:25:03.333050+0000 mon.a (mon.0) 2960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:04.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:04 smithi099 bash[87147]: audit 2024-09-13T15:25:03.333050+0000 mon.a (mon.0) 2960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:04.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:04 smithi177 bash[68594]: audit 2024-09-13T15:25:03.333050+0000 mon.a (mon.0) 2960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:04.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:04 smithi177 bash[68594]: audit 2024-09-13T15:25:03.333050+0000 mon.a (mon.0) 2960 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:05.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:05 smithi099 bash[84576]: cluster 2024-09-13T15:25:03.402496+0000 mgr.y (mgr.26252) 1773 : cluster [DBG] pgmap v2435: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:05.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:05 smithi099 bash[84576]: cluster 2024-09-13T15:25:03.402496+0000 mgr.y (mgr.26252) 1773 : cluster [DBG] pgmap v2435: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:05.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:05 smithi099 bash[87147]: cluster 2024-09-13T15:25:03.402496+0000 mgr.y (mgr.26252) 1773 : cluster [DBG] pgmap v2435: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:05.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:05 smithi099 bash[87147]: cluster 2024-09-13T15:25:03.402496+0000 mgr.y (mgr.26252) 1773 : cluster [DBG] pgmap v2435: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:05 smithi177 bash[68594]: cluster 2024-09-13T15:25:03.402496+0000 mgr.y (mgr.26252) 1773 : cluster [DBG] pgmap v2435: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:05.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:05 smithi177 bash[68594]: cluster 2024-09-13T15:25:03.402496+0000 mgr.y (mgr.26252) 1773 : cluster [DBG] pgmap v2435: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:07.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:25:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:25:07] "GET /metrics HTTP/1.1" 200 34328 "" "Prometheus/2.51.0" 2024-09-13T15:25:07.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:07 smithi099 bash[84576]: cluster 2024-09-13T15:25:05.403394+0000 mgr.y (mgr.26252) 1774 : cluster [DBG] pgmap v2436: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:07.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:07 smithi099 bash[84576]: cluster 2024-09-13T15:25:05.403394+0000 mgr.y (mgr.26252) 1774 : cluster [DBG] pgmap v2436: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:07 smithi099 bash[87147]: cluster 2024-09-13T15:25:05.403394+0000 mgr.y (mgr.26252) 1774 : cluster [DBG] pgmap v2436: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:07.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:07 smithi099 bash[87147]: cluster 2024-09-13T15:25:05.403394+0000 mgr.y (mgr.26252) 1774 : cluster [DBG] pgmap v2436: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:07.550 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T15:25:07.550 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":1179,"fsid":"af9d2c22-71d9-11ef-bceb-c7b262605968","created":"2024-09-13T14:10:21.363918+0000","modified":"2024-09-13T15:24:31.030134+0000","last_up_change":"2024-09-13T15:05:33.041673+0000","last_in_change":"2024-09-13T15:21:13.374925+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":47,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:15:37.290226+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1157","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1129","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:21:51.531836+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:24:15.786407+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:24:16.627611+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0624feaa-339b-4468-be53-582a1f04352c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":558,"up_from":575,"up_thru":1162,"down_at":558,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6803","nonce":1813984480}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6805","nonce":1813984480}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6809","nonce":1813984480}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":1813984480},{"type":"v1","addr":"172.21.15.99:6807","nonce":1813984480}]},"public_addr":"172.21.15.99:6803/1813984480","cluster_addr":"172.21.15.99:6805/1813984480","heartbeat_back_addr":"172.21.15.99:6809/1813984480","heartbeat_front_addr":"172.21.15.99:6807/1813984480","state":["exists","up"]},{"osd":1,"uuid":"b7b63553-e49d-4636-b3e2-43b72556a1a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":638,"up_from":642,"up_thru":1160,"down_at":639,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6811","nonce":804978824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6813","nonce":804978824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6817","nonce":804978824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":804978824},{"type":"v1","addr":"172.21.15.99:6815","nonce":804978824}]},"public_addr":"172.21.15.99:6811/804978824","cluster_addr":"172.21.15.99:6813/804978824","heartbeat_back_addr":"172.21.15.99:6817/804978824","heartbeat_front_addr":"172.21.15.99:6815/804978824","state":["exists","up"]},{"osd":2,"uuid":"3b9eff6d-5f5c-4843-a1ff-815a902167ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":656,"up_from":672,"up_thru":1164,"down_at":657,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6819","nonce":129558315}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6821","nonce":129558315}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6825","nonce":129558315}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":129558315},{"type":"v1","addr":"172.21.15.99:6823","nonce":129558315}]},"public_addr":"172.21.15.99:6819/129558315","cluster_addr":"172.21.15.99:6821/129558315","heartbeat_back_addr":"172.21.15.99:6825/129558315","heartbeat_front_addr":"172.21.15.99:6823/129558315","state":["exists","up"]},{"osd":3,"uuid":"686c8d3c-9cab-4920-b02b-d243324f5808","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":706,"last_clean_end":1033,"up_from":1066,"up_thru":1162,"down_at":1034,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":2771933148},{"type":"v1","addr":"172.21.15.99:6827","nonce":2771933148}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":2771933148},{"type":"v1","addr":"172.21.15.99:6829","nonce":2771933148}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6832","nonce":2771933148},{"type":"v1","addr":"172.21.15.99:6833","nonce":2771933148}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":2771933148},{"type":"v1","addr":"172.21.15.99:6831","nonce":2771933148}]},"public_addr":"172.21.15.99:6827/2771933148","cluster_addr":"172.21.15.99:6829/2771933148","heartbeat_back_addr":"172.21.15.99:6833/2771933148","heartbeat_front_addr":"172.21.15.99:6831/2771933148","state":["exists","up"]},{"osd":4,"uuid":"2d56712b-a6fb-4089-9583-850beed92b37","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":724,"up_from":739,"up_thru":1160,"down_at":725,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6801","nonce":2135282964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6803","nonce":2135282964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6807","nonce":2135282964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2135282964},{"type":"v1","addr":"172.21.15.177:6805","nonce":2135282964}]},"public_addr":"172.21.15.177:6801/2135282964","cluster_addr":"172.21.15.177:6803/2135282964","heartbeat_back_addr":"172.21.15.177:6807/2135282964","heartbeat_front_addr":"172.21.15.177:6805/2135282964","state":["exists","up"]},{"osd":5,"uuid":"414b880c-6534-495f-a54c-f8d28ba81c73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":1160,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6809","nonce":947070587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6811","nonce":947070587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6815","nonce":947070587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":947070587},{"type":"v1","addr":"172.21.15.177:6813","nonce":947070587}]},"public_addr":"172.21.15.177:6809/947070587","cluster_addr":"172.21.15.177:6811/947070587","heartbeat_back_addr":"172.21.15.177:6815/947070587","heartbeat_front_addr":"172.21.15.177:6813/947070587","state":["exists","up"]},{"osd":6,"uuid":"ba0e2db9-d3dd-4d86-9f2b-32d15b0eed0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":1160,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6817","nonce":3817692594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6819","nonce":3817692594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6823","nonce":3817692594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":3817692594},{"type":"v1","addr":"172.21.15.177:6821","nonce":3817692594}]},"public_addr":"172.21.15.177:6817/3817692594","cluster_addr":"172.21.15.177:6819/3817692594","heartbeat_back_addr":"172.21.15.177:6823/3817692594","heartbeat_front_addr":"172.21.15.177:6821/3817692594","state":["exists","up"]},{"osd":7,"uuid":"ad2c2977-fd6d-42f8-962e-907c2bf3204b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":1160,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6825","nonce":1984676946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6827","nonce":1984676946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6831","nonce":1984676946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":1984676946},{"type":"v1","addr":"172.21.15.177:6829","nonce":1984676946}]},"public_addr":"172.21.15.177:6825/1984676946","cluster_addr":"172.21.15.177:6827/1984676946","heartbeat_back_addr":"172.21.15.177:6831/1984676946","heartbeat_front_addr":"172.21.15.177:6829/1984676946","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:13:46.390607+0000","dead_epoch":557},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:14:38.398637+0000","dead_epoch":638},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:15:33.420625+0000","dead_epoch":656},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:30.964755+0000","dead_epoch":1033},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:22.513614+0000","dead_epoch":724},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:14.948720+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:19:10.431389+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:20:03.826389+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.12","mappings":[{"from":1,"to":0}]},{"pgid":"4.13","mappings":[{"from":1,"to":0}]},{"pgid":"4.14","mappings":[{"from":1,"to":5}]},{"pgid":"4.19","mappings":[{"from":3,"to":5}]},{"pgid":"4.1d","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.99:0/3935330909":"2024-09-13T15:40:15.846422+0000","172.21.15.99:0/3556003775":"2024-09-13T15:35:26.139282+0000","172.21.15.99:0/1151643634":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6801/1402010506":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6800/1402010506":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/3817526904":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/3394219806":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/181419264":"2024-09-14T14:31:47.884997+0000","172.21.15.99:0/4129805635":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/1363241618":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/3192879133":"2024-09-14T14:11:18.714884+0000","172.21.15.99:0/3483428223":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/405712510":"2024-09-14T14:21:07.041302+0000","172.21.15.177:0/1523474123":"2024-09-14T14:30:38.246301+0000","172.21.15.99:6834/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.99:0/795365709":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/1548831286":"2024-09-14T14:11:18.714884+0000","172.21.15.177:0/3423158096":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/888319381":"2024-09-14T14:10:48.295312+0000","172.21.15.177:0/3478790043":"2024-09-14T14:30:38.246301+0000","172.21.15.99:6801/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.177:0/1330817275":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/1638158155":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/313522629":"2024-09-13T15:35:27.165268+0000","172.21.15.99:6800/2887456408":"2024-09-14T14:29:03.998039+0000","172.21.15.99:6801/2887456408":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/4277277716":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6800/2671898016":"2024-09-14T14:10:48.295312+0000","172.21.15.99:0/17074640":"2024-09-14T14:21:07.041302+0000","172.21.15.99:6835/3867979721":"2024-09-14T14:24:26.174022+0000","172.21.15.99:6800/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.177:6834/4079938285":"2024-09-14T14:30:38.246301+0000","172.21.15.99:0/2372940082":"2024-09-14T14:31:47.884997+0000","172.21.15.99:6801/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.99:6801/3775052239":"2024-09-14T14:21:07.041302+0000","172.21.15.99:0/316966644":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/2140252887":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/4242039269":"2024-09-14T14:29:03.998039+0000","172.21.15.99:0/881490667":"2024-09-14T14:10:48.295312+0000","172.21.15.99:6800/3838091368":"2024-09-14T14:11:18.714884+0000","172.21.15.177:6835/4079938285":"2024-09-14T14:30:38.246301+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":227,"length":1},{"begin":229,"length":1},{"begin":231,"length":1}]},{"pool":13,"snaps":[{"begin":7,"length":2}]},{"pool":15,"snaps":[{"begin":35,"length":3},{"begin":43,"length":2},{"begin":47,"length":1}]},{"pool":16,"snaps":[{"begin":22,"length":1}]},{"pool":17,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":8,"length":3}]},{"pool":47,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T15:25:07.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:07 smithi177 bash[68594]: cluster 2024-09-13T15:25:05.403394+0000 mgr.y (mgr.26252) 1774 : cluster [DBG] pgmap v2436: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:07.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:07 smithi177 bash[68594]: cluster 2024-09-13T15:25:05.403394+0000 mgr.y (mgr.26252) 1774 : cluster [DBG] pgmap v2436: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:08.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:08 smithi099 bash[84576]: audit 2024-09-13T15:25:07.548324+0000 mon.a (mon.0) 2961 : audit [DBG] from='client.? 172.21.15.99:0/1404368023' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:25:08.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:08 smithi099 bash[84576]: audit 2024-09-13T15:25:07.548324+0000 mon.a (mon.0) 2961 : audit [DBG] from='client.? 172.21.15.99:0/1404368023' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:25:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:08 smithi099 bash[87147]: audit 2024-09-13T15:25:07.548324+0000 mon.a (mon.0) 2961 : audit [DBG] from='client.? 172.21.15.99:0/1404368023' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:25:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:08 smithi099 bash[87147]: audit 2024-09-13T15:25:07.548324+0000 mon.a (mon.0) 2961 : audit [DBG] from='client.? 172.21.15.99:0/1404368023' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:25:08.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T15:25:08.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T15:25:08.604 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T15:25:08.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T15:25:08.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T15:25:08.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T15:25:08.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T15:25:08.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T15:25:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:08 smithi177 bash[68594]: audit 2024-09-13T15:25:07.548324+0000 mon.a (mon.0) 2961 : audit [DBG] from='client.? 172.21.15.99:0/1404368023' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:25:08.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:08 smithi177 bash[68594]: audit 2024-09-13T15:25:07.548324+0000 mon.a (mon.0) 2961 : audit [DBG] from='client.? 172.21.15.99:0/1404368023' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:25:09.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:09 smithi099 bash[84576]: cluster 2024-09-13T15:25:07.404072+0000 mgr.y (mgr.26252) 1775 : cluster [DBG] pgmap v2437: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:09.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:09 smithi099 bash[84576]: cluster 2024-09-13T15:25:07.404072+0000 mgr.y (mgr.26252) 1775 : cluster [DBG] pgmap v2437: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:09 smithi099 bash[87147]: cluster 2024-09-13T15:25:07.404072+0000 mgr.y (mgr.26252) 1775 : cluster [DBG] pgmap v2437: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:09.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:09 smithi099 bash[87147]: cluster 2024-09-13T15:25:07.404072+0000 mgr.y (mgr.26252) 1775 : cluster [DBG] pgmap v2437: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:09.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:09 smithi177 bash[68594]: cluster 2024-09-13T15:25:07.404072+0000 mgr.y (mgr.26252) 1775 : cluster [DBG] pgmap v2437: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:09.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:09 smithi177 bash[68594]: cluster 2024-09-13T15:25:07.404072+0000 mgr.y (mgr.26252) 1775 : cluster [DBG] pgmap v2437: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:10.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:25:10 smithi177 bash[61439]: ts=2024-09-13T15:25:10.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:25:11.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:11 smithi099 bash[84576]: cluster 2024-09-13T15:25:09.404887+0000 mgr.y (mgr.26252) 1776 : cluster [DBG] pgmap v2438: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:11 smithi099 bash[84576]: cluster 2024-09-13T15:25:09.404887+0000 mgr.y (mgr.26252) 1776 : cluster [DBG] pgmap v2438: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:11.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:11 smithi099 bash[87147]: cluster 2024-09-13T15:25:09.404887+0000 mgr.y (mgr.26252) 1776 : cluster [DBG] pgmap v2438: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:11.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:11 smithi099 bash[87147]: cluster 2024-09-13T15:25:09.404887+0000 mgr.y (mgr.26252) 1776 : cluster [DBG] pgmap v2438: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:11.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:11 smithi177 bash[68594]: cluster 2024-09-13T15:25:09.404887+0000 mgr.y (mgr.26252) 1776 : cluster [DBG] pgmap v2438: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:11.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:11 smithi177 bash[68594]: cluster 2024-09-13T15:25:09.404887+0000 mgr.y (mgr.26252) 1776 : cluster [DBG] pgmap v2438: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:13.544 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:13.545 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:13.545 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:13.545 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:13.547 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:13.548 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:13.549 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:13.549 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:13.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:13 smithi099 bash[84576]: cluster 2024-09-13T15:25:11.405620+0000 mgr.y (mgr.26252) 1777 : cluster [DBG] pgmap v2439: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:13.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:13 smithi099 bash[84576]: cluster 2024-09-13T15:25:11.405620+0000 mgr.y (mgr.26252) 1777 : cluster [DBG] pgmap v2439: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:13.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:13 smithi099 bash[87147]: cluster 2024-09-13T15:25:11.405620+0000 mgr.y (mgr.26252) 1777 : cluster [DBG] pgmap v2439: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:13.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:13 smithi099 bash[87147]: cluster 2024-09-13T15:25:11.405620+0000 mgr.y (mgr.26252) 1777 : cluster [DBG] pgmap v2439: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:13 smithi177 bash[68594]: cluster 2024-09-13T15:25:11.405620+0000 mgr.y (mgr.26252) 1777 : cluster [DBG] pgmap v2439: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:13.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:13 smithi177 bash[68594]: cluster 2024-09-13T15:25:11.405620+0000 mgr.y (mgr.26252) 1777 : cluster [DBG] pgmap v2439: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:15.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:15 smithi099 bash[84576]: cluster 2024-09-13T15:25:13.406549+0000 mgr.y (mgr.26252) 1778 : cluster [DBG] pgmap v2440: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:15.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:15 smithi099 bash[84576]: cluster 2024-09-13T15:25:13.406549+0000 mgr.y (mgr.26252) 1778 : cluster [DBG] pgmap v2440: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:15.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:15 smithi099 bash[87147]: cluster 2024-09-13T15:25:13.406549+0000 mgr.y (mgr.26252) 1778 : cluster [DBG] pgmap v2440: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:15.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:15 smithi099 bash[87147]: cluster 2024-09-13T15:25:13.406549+0000 mgr.y (mgr.26252) 1778 : cluster [DBG] pgmap v2440: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:15 smithi177 bash[68594]: cluster 2024-09-13T15:25:13.406549+0000 mgr.y (mgr.26252) 1778 : cluster [DBG] pgmap v2440: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:15.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:15 smithi177 bash[68594]: cluster 2024-09-13T15:25:13.406549+0000 mgr.y (mgr.26252) 1778 : cluster [DBG] pgmap v2440: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:17.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:25:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:25:17] "GET /metrics HTTP/1.1" 200 34328 "" "Prometheus/2.51.0" 2024-09-13T15:25:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:17 smithi099 bash[84576]: cluster 2024-09-13T15:25:15.407425+0000 mgr.y (mgr.26252) 1779 : cluster [DBG] pgmap v2441: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:17 smithi099 bash[84576]: cluster 2024-09-13T15:25:15.407425+0000 mgr.y (mgr.26252) 1779 : cluster [DBG] pgmap v2441: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:17 smithi099 bash[87147]: cluster 2024-09-13T15:25:15.407425+0000 mgr.y (mgr.26252) 1779 : cluster [DBG] pgmap v2441: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:17 smithi099 bash[87147]: cluster 2024-09-13T15:25:15.407425+0000 mgr.y (mgr.26252) 1779 : cluster [DBG] pgmap v2441: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:17.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:17 smithi177 bash[68594]: cluster 2024-09-13T15:25:15.407425+0000 mgr.y (mgr.26252) 1779 : cluster [DBG] pgmap v2441: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:17.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:17 smithi177 bash[68594]: cluster 2024-09-13T15:25:15.407425+0000 mgr.y (mgr.26252) 1779 : cluster [DBG] pgmap v2441: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:18.562 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:18 smithi099 bash[84576]: audit 2024-09-13T15:25:17.758014+0000 mon.a (mon.0) 2962 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:25:18.563 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:18 smithi099 bash[84576]: audit 2024-09-13T15:25:17.758014+0000 mon.a (mon.0) 2962 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:25:18.563 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:18 smithi099 bash[87147]: audit 2024-09-13T15:25:17.758014+0000 mon.a (mon.0) 2962 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:25:18.563 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:18 smithi099 bash[87147]: audit 2024-09-13T15:25:17.758014+0000 mon.a (mon.0) 2962 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:25:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:18 smithi177 bash[68594]: audit 2024-09-13T15:25:17.758014+0000 mon.a (mon.0) 2962 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:25:18.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:18 smithi177 bash[68594]: audit 2024-09-13T15:25:17.758014+0000 mon.a (mon.0) 2962 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:25:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[84576]: cluster 2024-09-13T15:25:17.408436+0000 mgr.y (mgr.26252) 1780 : cluster [DBG] pgmap v2442: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[84576]: cluster 2024-09-13T15:25:17.408436+0000 mgr.y (mgr.26252) 1780 : cluster [DBG] pgmap v2442: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[84576]: audit 2024-09-13T15:25:18.332572+0000 mon.a (mon.0) 2963 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:19.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[84576]: audit 2024-09-13T15:25:18.332572+0000 mon.a (mon.0) 2963 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[84576]: audit 2024-09-13T15:25:18.363579+0000 mon.a (mon.0) 2964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[84576]: audit 2024-09-13T15:25:18.363579+0000 mon.a (mon.0) 2964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[84576]: audit 2024-09-13T15:25:18.370879+0000 mon.a (mon.0) 2965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[84576]: audit 2024-09-13T15:25:18.370879+0000 mon.a (mon.0) 2965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[87147]: cluster 2024-09-13T15:25:17.408436+0000 mgr.y (mgr.26252) 1780 : cluster [DBG] pgmap v2442: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[87147]: cluster 2024-09-13T15:25:17.408436+0000 mgr.y (mgr.26252) 1780 : cluster [DBG] pgmap v2442: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[87147]: audit 2024-09-13T15:25:18.332572+0000 mon.a (mon.0) 2963 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[87147]: audit 2024-09-13T15:25:18.332572+0000 mon.a (mon.0) 2963 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[87147]: audit 2024-09-13T15:25:18.363579+0000 mon.a (mon.0) 2964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[87147]: audit 2024-09-13T15:25:18.363579+0000 mon.a (mon.0) 2964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[87147]: audit 2024-09-13T15:25:18.370879+0000 mon.a (mon.0) 2965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.576 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:19 smithi099 bash[87147]: audit 2024-09-13T15:25:18.370879+0000 mon.a (mon.0) 2965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:19 smithi177 bash[68594]: cluster 2024-09-13T15:25:17.408436+0000 mgr.y (mgr.26252) 1780 : cluster [DBG] pgmap v2442: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:19 smithi177 bash[68594]: cluster 2024-09-13T15:25:17.408436+0000 mgr.y (mgr.26252) 1780 : cluster [DBG] pgmap v2442: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:19 smithi177 bash[68594]: audit 2024-09-13T15:25:18.332572+0000 mon.a (mon.0) 2963 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:19 smithi177 bash[68594]: audit 2024-09-13T15:25:18.332572+0000 mon.a (mon.0) 2963 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:19.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:19 smithi177 bash[68594]: audit 2024-09-13T15:25:18.363579+0000 mon.a (mon.0) 2964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:19 smithi177 bash[68594]: audit 2024-09-13T15:25:18.363579+0000 mon.a (mon.0) 2964 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:19 smithi177 bash[68594]: audit 2024-09-13T15:25:18.370879+0000 mon.a (mon.0) 2965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:19.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:19 smithi177 bash[68594]: audit 2024-09-13T15:25:18.370879+0000 mon.a (mon.0) 2965 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:20.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:25:20 smithi177 bash[61439]: ts=2024-09-13T15:25:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:25:21.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:21 smithi177 bash[68594]: cluster 2024-09-13T15:25:19.409541+0000 mgr.y (mgr.26252) 1781 : cluster [DBG] pgmap v2443: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:21.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:21 smithi177 bash[68594]: cluster 2024-09-13T15:25:19.409541+0000 mgr.y (mgr.26252) 1781 : cluster [DBG] pgmap v2443: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:21.774 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:21 smithi099 bash[87147]: cluster 2024-09-13T15:25:19.409541+0000 mgr.y (mgr.26252) 1781 : cluster [DBG] pgmap v2443: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:21.774 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:21 smithi099 bash[87147]: cluster 2024-09-13T15:25:19.409541+0000 mgr.y (mgr.26252) 1781 : cluster [DBG] pgmap v2443: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:21.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:21 smithi099 bash[84576]: cluster 2024-09-13T15:25:19.409541+0000 mgr.y (mgr.26252) 1781 : cluster [DBG] pgmap v2443: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:21.775 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:21 smithi099 bash[84576]: cluster 2024-09-13T15:25:19.409541+0000 mgr.y (mgr.26252) 1781 : cluster [DBG] pgmap v2443: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:23.528 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:23 smithi099 bash[84576]: cluster 2024-09-13T15:25:21.410348+0000 mgr.y (mgr.26252) 1782 : cluster [DBG] pgmap v2444: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:23.528 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:23 smithi099 bash[84576]: cluster 2024-09-13T15:25:21.410348+0000 mgr.y (mgr.26252) 1782 : cluster [DBG] pgmap v2444: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:23.528 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:23 smithi099 bash[87147]: cluster 2024-09-13T15:25:21.410348+0000 mgr.y (mgr.26252) 1782 : cluster [DBG] pgmap v2444: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:23.529 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:23 smithi099 bash[87147]: cluster 2024-09-13T15:25:21.410348+0000 mgr.y (mgr.26252) 1782 : cluster [DBG] pgmap v2444: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:23 smithi177 bash[68594]: cluster 2024-09-13T15:25:21.410348+0000 mgr.y (mgr.26252) 1782 : cluster [DBG] pgmap v2444: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:23 smithi177 bash[68594]: cluster 2024-09-13T15:25:21.410348+0000 mgr.y (mgr.26252) 1782 : cluster [DBG] pgmap v2444: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:25 smithi177 bash[68594]: cluster 2024-09-13T15:25:23.411263+0000 mgr.y (mgr.26252) 1783 : cluster [DBG] pgmap v2445: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:25 smithi177 bash[68594]: cluster 2024-09-13T15:25:23.411263+0000 mgr.y (mgr.26252) 1783 : cluster [DBG] pgmap v2445: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:25.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:25 smithi177 bash[68594]: audit 2024-09-13T15:25:24.356559+0000 mon.a (mon.0) 2966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:25 smithi177 bash[68594]: audit 2024-09-13T15:25:24.356559+0000 mon.a (mon.0) 2966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:25 smithi177 bash[68594]: audit 2024-09-13T15:25:24.368942+0000 mon.a (mon.0) 2967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.613 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:25 smithi177 bash[68594]: audit 2024-09-13T15:25:24.368942+0000 mon.a (mon.0) 2967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[84576]: cluster 2024-09-13T15:25:23.411263+0000 mgr.y (mgr.26252) 1783 : cluster [DBG] pgmap v2445: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:25.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[84576]: cluster 2024-09-13T15:25:23.411263+0000 mgr.y (mgr.26252) 1783 : cluster [DBG] pgmap v2445: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:25.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[84576]: audit 2024-09-13T15:25:24.356559+0000 mon.a (mon.0) 2966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[84576]: audit 2024-09-13T15:25:24.356559+0000 mon.a (mon.0) 2966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[84576]: audit 2024-09-13T15:25:24.368942+0000 mon.a (mon.0) 2967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[84576]: audit 2024-09-13T15:25:24.368942+0000 mon.a (mon.0) 2967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[87147]: cluster 2024-09-13T15:25:23.411263+0000 mgr.y (mgr.26252) 1783 : cluster [DBG] pgmap v2445: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:25.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[87147]: cluster 2024-09-13T15:25:23.411263+0000 mgr.y (mgr.26252) 1783 : cluster [DBG] pgmap v2445: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:25.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[87147]: audit 2024-09-13T15:25:24.356559+0000 mon.a (mon.0) 2966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[87147]: audit 2024-09-13T15:25:24.356559+0000 mon.a (mon.0) 2966 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[87147]: audit 2024-09-13T15:25:24.368942+0000 mon.a (mon.0) 2967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:25.826 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:25 smithi099 bash[87147]: audit 2024-09-13T15:25:24.368942+0000 mon.a (mon.0) 2967 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:27.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:25:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:25:27] "GET /metrics HTTP/1.1" 200 34329 "" "Prometheus/2.51.0" 2024-09-13T15:25:27.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:27 smithi177 bash[68594]: cluster 2024-09-13T15:25:25.412085+0000 mgr.y (mgr.26252) 1784 : cluster [DBG] pgmap v2446: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:27.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:27 smithi177 bash[68594]: cluster 2024-09-13T15:25:25.412085+0000 mgr.y (mgr.26252) 1784 : cluster [DBG] pgmap v2446: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:27.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:27 smithi099 bash[84576]: cluster 2024-09-13T15:25:25.412085+0000 mgr.y (mgr.26252) 1784 : cluster [DBG] pgmap v2446: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:27.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:27 smithi099 bash[84576]: cluster 2024-09-13T15:25:25.412085+0000 mgr.y (mgr.26252) 1784 : cluster [DBG] pgmap v2446: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:27.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:27 smithi099 bash[87147]: cluster 2024-09-13T15:25:25.412085+0000 mgr.y (mgr.26252) 1784 : cluster [DBG] pgmap v2446: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:27.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:27 smithi099 bash[87147]: cluster 2024-09-13T15:25:25.412085+0000 mgr.y (mgr.26252) 1784 : cluster [DBG] pgmap v2446: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:28.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:28 smithi099 bash[84576]: cluster 2024-09-13T15:25:27.413156+0000 mgr.y (mgr.26252) 1785 : cluster [DBG] pgmap v2447: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:28.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:28 smithi099 bash[84576]: cluster 2024-09-13T15:25:27.413156+0000 mgr.y (mgr.26252) 1785 : cluster [DBG] pgmap v2447: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:28.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:28 smithi099 bash[87147]: cluster 2024-09-13T15:25:27.413156+0000 mgr.y (mgr.26252) 1785 : cluster [DBG] pgmap v2447: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:28.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:28 smithi099 bash[87147]: cluster 2024-09-13T15:25:27.413156+0000 mgr.y (mgr.26252) 1785 : cluster [DBG] pgmap v2447: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:28.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:28 smithi177 bash[68594]: cluster 2024-09-13T15:25:27.413156+0000 mgr.y (mgr.26252) 1785 : cluster [DBG] pgmap v2447: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:28.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:28 smithi177 bash[68594]: cluster 2024-09-13T15:25:27.413156+0000 mgr.y (mgr.26252) 1785 : cluster [DBG] pgmap v2447: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:30.470 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:25:30 smithi177 bash[61439]: ts=2024-09-13T15:25:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:25:30.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:30 smithi099 bash[84576]: cluster 2024-09-13T15:25:29.414085+0000 mgr.y (mgr.26252) 1786 : cluster [DBG] pgmap v2448: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:30.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:30 smithi099 bash[84576]: cluster 2024-09-13T15:25:29.414085+0000 mgr.y (mgr.26252) 1786 : cluster [DBG] pgmap v2448: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:30 smithi099 bash[87147]: cluster 2024-09-13T15:25:29.414085+0000 mgr.y (mgr.26252) 1786 : cluster [DBG] pgmap v2448: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:30.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:30 smithi099 bash[87147]: cluster 2024-09-13T15:25:29.414085+0000 mgr.y (mgr.26252) 1786 : cluster [DBG] pgmap v2448: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:30.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:30 smithi177 bash[68594]: cluster 2024-09-13T15:25:29.414085+0000 mgr.y (mgr.26252) 1786 : cluster [DBG] pgmap v2448: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:30.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:30 smithi177 bash[68594]: cluster 2024-09-13T15:25:29.414085+0000 mgr.y (mgr.26252) 1786 : cluster [DBG] pgmap v2448: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:32.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:32 smithi099 bash[84576]: cluster 2024-09-13T15:25:31.414965+0000 mgr.y (mgr.26252) 1787 : cluster [DBG] pgmap v2449: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:32.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:32 smithi099 bash[84576]: cluster 2024-09-13T15:25:31.414965+0000 mgr.y (mgr.26252) 1787 : cluster [DBG] pgmap v2449: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:32.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:32 smithi099 bash[87147]: cluster 2024-09-13T15:25:31.414965+0000 mgr.y (mgr.26252) 1787 : cluster [DBG] pgmap v2449: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:32.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:32 smithi099 bash[87147]: cluster 2024-09-13T15:25:31.414965+0000 mgr.y (mgr.26252) 1787 : cluster [DBG] pgmap v2449: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:32.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:32 smithi177 bash[68594]: cluster 2024-09-13T15:25:31.414965+0000 mgr.y (mgr.26252) 1787 : cluster [DBG] pgmap v2449: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:32.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:32 smithi177 bash[68594]: cluster 2024-09-13T15:25:31.414965+0000 mgr.y (mgr.26252) 1787 : cluster [DBG] pgmap v2449: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:33.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:33 smithi099 bash[84576]: audit 2024-09-13T15:25:33.332946+0000 mon.a (mon.0) 2968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:33.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:33 smithi099 bash[84576]: audit 2024-09-13T15:25:33.332946+0000 mon.a (mon.0) 2968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:33.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:33 smithi099 bash[87147]: audit 2024-09-13T15:25:33.332946+0000 mon.a (mon.0) 2968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:33.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:33 smithi099 bash[87147]: audit 2024-09-13T15:25:33.332946+0000 mon.a (mon.0) 2968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:33 smithi177 bash[68594]: audit 2024-09-13T15:25:33.332946+0000 mon.a (mon.0) 2968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:33.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:33 smithi177 bash[68594]: audit 2024-09-13T15:25:33.332946+0000 mon.a (mon.0) 2968 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:34.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:34 smithi099 bash[87147]: cluster 2024-09-13T15:25:33.415525+0000 mgr.y (mgr.26252) 1788 : cluster [DBG] pgmap v2450: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:34.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:34 smithi099 bash[87147]: cluster 2024-09-13T15:25:33.415525+0000 mgr.y (mgr.26252) 1788 : cluster [DBG] pgmap v2450: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:34.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:34 smithi099 bash[84576]: cluster 2024-09-13T15:25:33.415525+0000 mgr.y (mgr.26252) 1788 : cluster [DBG] pgmap v2450: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:34.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:34 smithi099 bash[84576]: cluster 2024-09-13T15:25:33.415525+0000 mgr.y (mgr.26252) 1788 : cluster [DBG] pgmap v2450: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:34 smithi177 bash[68594]: cluster 2024-09-13T15:25:33.415525+0000 mgr.y (mgr.26252) 1788 : cluster [DBG] pgmap v2450: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:34.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:34 smithi177 bash[68594]: cluster 2024-09-13T15:25:33.415525+0000 mgr.y (mgr.26252) 1788 : cluster [DBG] pgmap v2450: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:35.623 INFO:teuthology.orchestra.run.smithi099.stdout:3173980832291 2024-09-13T15:25:35.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T15:25:36.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:36 smithi099 bash[84576]: cluster 2024-09-13T15:25:35.416423+0000 mgr.y (mgr.26252) 1789 : cluster [DBG] pgmap v2451: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:36.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:36 smithi099 bash[84576]: cluster 2024-09-13T15:25:35.416423+0000 mgr.y (mgr.26252) 1789 : cluster [DBG] pgmap v2451: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:36.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:36 smithi099 bash[87147]: cluster 2024-09-13T15:25:35.416423+0000 mgr.y (mgr.26252) 1789 : cluster [DBG] pgmap v2451: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:36.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:36 smithi099 bash[87147]: cluster 2024-09-13T15:25:35.416423+0000 mgr.y (mgr.26252) 1789 : cluster [DBG] pgmap v2451: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:36.859 INFO:teuthology.orchestra.run.smithi099.stdout:210453398294 2024-09-13T15:25:36.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T15:25:36.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:36 smithi177 bash[68594]: cluster 2024-09-13T15:25:35.416423+0000 mgr.y (mgr.26252) 1789 : cluster [DBG] pgmap v2451: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:36.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:36 smithi177 bash[68594]: cluster 2024-09-13T15:25:35.416423+0000 mgr.y (mgr.26252) 1789 : cluster [DBG] pgmap v2451: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:37.133 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:25:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:25:37] "GET /metrics HTTP/1.1" 200 34331 "" "Prometheus/2.51.0" 2024-09-13T15:25:38.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:38 smithi099 bash[84576]: cluster 2024-09-13T15:25:37.417453+0000 mgr.y (mgr.26252) 1790 : cluster [DBG] pgmap v2452: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:38.825 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:38 smithi099 bash[84576]: cluster 2024-09-13T15:25:37.417453+0000 mgr.y (mgr.26252) 1790 : cluster [DBG] pgmap v2452: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:38.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:38 smithi099 bash[87147]: cluster 2024-09-13T15:25:37.417453+0000 mgr.y (mgr.26252) 1790 : cluster [DBG] pgmap v2452: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:38.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:38 smithi099 bash[87147]: cluster 2024-09-13T15:25:37.417453+0000 mgr.y (mgr.26252) 1790 : cluster [DBG] pgmap v2452: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:38.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:38 smithi177 bash[68594]: cluster 2024-09-13T15:25:37.417453+0000 mgr.y (mgr.26252) 1790 : cluster [DBG] pgmap v2452: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:38.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:38 smithi177 bash[68594]: cluster 2024-09-13T15:25:37.417453+0000 mgr.y (mgr.26252) 1790 : cluster [DBG] pgmap v2452: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:39.533 INFO:teuthology.orchestra.run.smithi099.stdout:2469606195796 2024-09-13T15:25:39.533 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T15:25:39.948 INFO:teuthology.orchestra.run.smithi099.stdout:158913790763 2024-09-13T15:25:39.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T15:25:40.496 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:25:40 smithi177 bash[61439]: ts=2024-09-13T15:25:40.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:25:40.782 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:40 smithi099 bash[84576]: cluster 2024-09-13T15:25:39.418392+0000 mgr.y (mgr.26252) 1791 : cluster [DBG] pgmap v2453: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:40.782 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:40 smithi099 bash[84576]: cluster 2024-09-13T15:25:39.418392+0000 mgr.y (mgr.26252) 1791 : cluster [DBG] pgmap v2453: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:40.782 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:40 smithi099 bash[87147]: cluster 2024-09-13T15:25:39.418392+0000 mgr.y (mgr.26252) 1791 : cluster [DBG] pgmap v2453: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:40.782 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:40 smithi099 bash[87147]: cluster 2024-09-13T15:25:39.418392+0000 mgr.y (mgr.26252) 1791 : cluster [DBG] pgmap v2453: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:40.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:40 smithi177 bash[68594]: cluster 2024-09-13T15:25:39.418392+0000 mgr.y (mgr.26252) 1791 : cluster [DBG] pgmap v2453: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:40.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:40 smithi177 bash[68594]: cluster 2024-09-13T15:25:39.418392+0000 mgr.y (mgr.26252) 1791 : cluster [DBG] pgmap v2453: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:41.835 INFO:teuthology.orchestra.run.smithi099.stdout:4578435137779 2024-09-13T15:25:41.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T15:25:42.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:42 smithi099 bash[84576]: cluster 2024-09-13T15:25:41.419320+0000 mgr.y (mgr.26252) 1792 : cluster [DBG] pgmap v2454: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:42.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:42 smithi099 bash[84576]: cluster 2024-09-13T15:25:41.419320+0000 mgr.y (mgr.26252) 1792 : cluster [DBG] pgmap v2454: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:42.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:42 smithi099 bash[87147]: cluster 2024-09-13T15:25:41.419320+0000 mgr.y (mgr.26252) 1792 : cluster [DBG] pgmap v2454: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:42.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:42 smithi099 bash[87147]: cluster 2024-09-13T15:25:41.419320+0000 mgr.y (mgr.26252) 1792 : cluster [DBG] pgmap v2454: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:42 smithi177 bash[68594]: cluster 2024-09-13T15:25:41.419320+0000 mgr.y (mgr.26252) 1792 : cluster [DBG] pgmap v2454: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:42.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:42 smithi177 bash[68594]: cluster 2024-09-13T15:25:41.419320+0000 mgr.y (mgr.26252) 1792 : cluster [DBG] pgmap v2454: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:43.140 INFO:teuthology.orchestra.run.smithi099.stdout:184683594531 2024-09-13T15:25:43.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T15:25:43.780 INFO:teuthology.orchestra.run.smithi099.stdout:2886218023475 2024-09-13T15:25:43.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T15:25:43.823 INFO:teuthology.orchestra.run.smithi099.stdout:2757369004605 2024-09-13T15:25:43.823 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T15:25:44.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:44 smithi099 bash[84576]: cluster 2024-09-13T15:25:43.420241+0000 mgr.y (mgr.26252) 1793 : cluster [DBG] pgmap v2455: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:44.824 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:44 smithi099 bash[84576]: cluster 2024-09-13T15:25:43.420241+0000 mgr.y (mgr.26252) 1793 : cluster [DBG] pgmap v2455: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:44.824 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:44 smithi099 bash[87147]: cluster 2024-09-13T15:25:43.420241+0000 mgr.y (mgr.26252) 1793 : cluster [DBG] pgmap v2455: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:44.825 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:44 smithi099 bash[87147]: cluster 2024-09-13T15:25:43.420241+0000 mgr.y (mgr.26252) 1793 : cluster [DBG] pgmap v2455: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:44.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:44 smithi177 bash[68594]: cluster 2024-09-13T15:25:43.420241+0000 mgr.y (mgr.26252) 1793 : cluster [DBG] pgmap v2455: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:44.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:44 smithi177 bash[68594]: cluster 2024-09-13T15:25:43.420241+0000 mgr.y (mgr.26252) 1793 : cluster [DBG] pgmap v2455: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:46.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[84576]: audit 2024-09-13T15:25:45.007977+0000 mon.a (mon.0) 2969 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[84576]: audit 2024-09-13T15:25:45.007977+0000 mon.a (mon.0) 2969 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[84576]: audit 2024-09-13T15:25:45.017648+0000 mon.a (mon.0) 2970 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[84576]: audit 2024-09-13T15:25:45.017648+0000 mon.a (mon.0) 2970 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[84576]: audit 2024-09-13T15:25:45.670981+0000 mon.a (mon.0) 2971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:25:46.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[84576]: audit 2024-09-13T15:25:45.670981+0000 mon.a (mon.0) 2971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[84576]: audit 2024-09-13T15:25:45.672070+0000 mon.a (mon.0) 2972 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[84576]: audit 2024-09-13T15:25:45.672070+0000 mon.a (mon.0) 2972 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[84576]: audit 2024-09-13T15:25:45.679667+0000 mon.a (mon.0) 2973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[84576]: audit 2024-09-13T15:25:45.679667+0000 mon.a (mon.0) 2973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[87147]: audit 2024-09-13T15:25:45.007977+0000 mon.a (mon.0) 2969 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[87147]: audit 2024-09-13T15:25:45.007977+0000 mon.a (mon.0) 2969 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[87147]: audit 2024-09-13T15:25:45.017648+0000 mon.a (mon.0) 2970 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[87147]: audit 2024-09-13T15:25:45.017648+0000 mon.a (mon.0) 2970 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[87147]: audit 2024-09-13T15:25:45.670981+0000 mon.a (mon.0) 2971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[87147]: audit 2024-09-13T15:25:45.670981+0000 mon.a (mon.0) 2971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:25:46.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[87147]: audit 2024-09-13T15:25:45.672070+0000 mon.a (mon.0) 2972 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:25:46.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[87147]: audit 2024-09-13T15:25:45.672070+0000 mon.a (mon.0) 2972 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:25:46.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[87147]: audit 2024-09-13T15:25:45.679667+0000 mon.a (mon.0) 2973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.327 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:46 smithi099 bash[87147]: audit 2024-09-13T15:25:45.679667+0000 mon.a (mon.0) 2973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:46 smithi177 bash[68594]: audit 2024-09-13T15:25:45.007977+0000 mon.a (mon.0) 2969 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:46 smithi177 bash[68594]: audit 2024-09-13T15:25:45.007977+0000 mon.a (mon.0) 2969 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:46 smithi177 bash[68594]: audit 2024-09-13T15:25:45.017648+0000 mon.a (mon.0) 2970 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:46 smithi177 bash[68594]: audit 2024-09-13T15:25:45.017648+0000 mon.a (mon.0) 2970 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:46 smithi177 bash[68594]: audit 2024-09-13T15:25:45.670981+0000 mon.a (mon.0) 2971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:25:46.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:46 smithi177 bash[68594]: audit 2024-09-13T15:25:45.670981+0000 mon.a (mon.0) 2971 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:25:46.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:46 smithi177 bash[68594]: audit 2024-09-13T15:25:45.672070+0000 mon.a (mon.0) 2972 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:25:46.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:46 smithi177 bash[68594]: audit 2024-09-13T15:25:45.672070+0000 mon.a (mon.0) 2972 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:25:46.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:46 smithi177 bash[68594]: audit 2024-09-13T15:25:45.679667+0000 mon.a (mon.0) 2973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:46 smithi177 bash[68594]: audit 2024-09-13T15:25:45.679667+0000 mon.a (mon.0) 2973 : audit [INF] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' 2024-09-13T15:25:46.522 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:46.523 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:46.523 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:46.523 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:46.526 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:47.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:25:47 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:25:47] "GET /metrics HTTP/1.1" 200 34331 "" "Prometheus/2.51.0" 2024-09-13T15:25:47.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:47 smithi099 bash[84576]: cluster 2024-09-13T15:25:45.420825+0000 mgr.y (mgr.26252) 1794 : cluster [DBG] pgmap v2456: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:47.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:47 smithi099 bash[84576]: cluster 2024-09-13T15:25:45.420825+0000 mgr.y (mgr.26252) 1794 : cluster [DBG] pgmap v2456: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:47.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:47 smithi099 bash[87147]: cluster 2024-09-13T15:25:45.420825+0000 mgr.y (mgr.26252) 1794 : cluster [DBG] pgmap v2456: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:47.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:47 smithi099 bash[87147]: cluster 2024-09-13T15:25:45.420825+0000 mgr.y (mgr.26252) 1794 : cluster [DBG] pgmap v2456: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:47.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:47 smithi177 bash[68594]: cluster 2024-09-13T15:25:45.420825+0000 mgr.y (mgr.26252) 1794 : cluster [DBG] pgmap v2456: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:47.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:47 smithi177 bash[68594]: cluster 2024-09-13T15:25:45.420825+0000 mgr.y (mgr.26252) 1794 : cluster [DBG] pgmap v2456: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:47.541 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:48.567 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:48.568 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:25:49.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:49 smithi099 bash[84576]: cluster 2024-09-13T15:25:47.421780+0000 mgr.y (mgr.26252) 1795 : cluster [DBG] pgmap v2457: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:49.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:49 smithi099 bash[84576]: cluster 2024-09-13T15:25:47.421780+0000 mgr.y (mgr.26252) 1795 : cluster [DBG] pgmap v2457: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:49.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:49 smithi099 bash[84576]: audit 2024-09-13T15:25:48.333955+0000 mon.a (mon.0) 2974 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:49.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:49 smithi099 bash[84576]: audit 2024-09-13T15:25:48.333955+0000 mon.a (mon.0) 2974 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:49.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:49 smithi099 bash[87147]: cluster 2024-09-13T15:25:47.421780+0000 mgr.y (mgr.26252) 1795 : cluster [DBG] pgmap v2457: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:49.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:49 smithi099 bash[87147]: cluster 2024-09-13T15:25:47.421780+0000 mgr.y (mgr.26252) 1795 : cluster [DBG] pgmap v2457: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:49.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:49 smithi099 bash[87147]: audit 2024-09-13T15:25:48.333955+0000 mon.a (mon.0) 2974 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:49.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:49 smithi099 bash[87147]: audit 2024-09-13T15:25:48.333955+0000 mon.a (mon.0) 2974 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:49 smithi177 bash[68594]: cluster 2024-09-13T15:25:47.421780+0000 mgr.y (mgr.26252) 1795 : cluster [DBG] pgmap v2457: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:49 smithi177 bash[68594]: cluster 2024-09-13T15:25:47.421780+0000 mgr.y (mgr.26252) 1795 : cluster [DBG] pgmap v2457: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:49 smithi177 bash[68594]: audit 2024-09-13T15:25:48.333955+0000 mon.a (mon.0) 2974 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:49.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:49 smithi177 bash[68594]: audit 2024-09-13T15:25:48.333955+0000 mon.a (mon.0) 2974 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:25:50.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:25:50 smithi177 bash[61439]: ts=2024-09-13T15:25:50.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:25:51.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:51 smithi099 bash[84576]: cluster 2024-09-13T15:25:49.422537+0000 mgr.y (mgr.26252) 1796 : cluster [DBG] pgmap v2458: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:51.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:51 smithi099 bash[84576]: cluster 2024-09-13T15:25:49.422537+0000 mgr.y (mgr.26252) 1796 : cluster [DBG] pgmap v2458: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:51.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:51 smithi099 bash[87147]: cluster 2024-09-13T15:25:49.422537+0000 mgr.y (mgr.26252) 1796 : cluster [DBG] pgmap v2458: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:51.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:51 smithi099 bash[87147]: cluster 2024-09-13T15:25:49.422537+0000 mgr.y (mgr.26252) 1796 : cluster [DBG] pgmap v2458: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:51.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:51 smithi177 bash[68594]: cluster 2024-09-13T15:25:49.422537+0000 mgr.y (mgr.26252) 1796 : cluster [DBG] pgmap v2458: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:51.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:51 smithi177 bash[68594]: cluster 2024-09-13T15:25:49.422537+0000 mgr.y (mgr.26252) 1796 : cluster [DBG] pgmap v2458: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:53.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:53 smithi099 bash[84576]: cluster 2024-09-13T15:25:51.423038+0000 mgr.y (mgr.26252) 1797 : cluster [DBG] pgmap v2459: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:53.347 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:53 smithi099 bash[84576]: cluster 2024-09-13T15:25:51.423038+0000 mgr.y (mgr.26252) 1797 : cluster [DBG] pgmap v2459: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:53.347 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:53 smithi099 bash[87147]: cluster 2024-09-13T15:25:51.423038+0000 mgr.y (mgr.26252) 1797 : cluster [DBG] pgmap v2459: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:53.347 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:53 smithi099 bash[87147]: cluster 2024-09-13T15:25:51.423038+0000 mgr.y (mgr.26252) 1797 : cluster [DBG] pgmap v2459: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:53 smithi177 bash[68594]: cluster 2024-09-13T15:25:51.423038+0000 mgr.y (mgr.26252) 1797 : cluster [DBG] pgmap v2459: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:53.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:53 smithi177 bash[68594]: cluster 2024-09-13T15:25:51.423038+0000 mgr.y (mgr.26252) 1797 : cluster [DBG] pgmap v2459: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:55.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:55 smithi099 bash[84576]: cluster 2024-09-13T15:25:53.423969+0000 mgr.y (mgr.26252) 1798 : cluster [DBG] pgmap v2460: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:55.338 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:55 smithi099 bash[84576]: cluster 2024-09-13T15:25:53.423969+0000 mgr.y (mgr.26252) 1798 : cluster [DBG] pgmap v2460: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:55.338 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:55 smithi099 bash[87147]: cluster 2024-09-13T15:25:53.423969+0000 mgr.y (mgr.26252) 1798 : cluster [DBG] pgmap v2460: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:55.338 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:55 smithi099 bash[87147]: cluster 2024-09-13T15:25:53.423969+0000 mgr.y (mgr.26252) 1798 : cluster [DBG] pgmap v2460: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:55.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:55 smithi177 bash[68594]: cluster 2024-09-13T15:25:53.423969+0000 mgr.y (mgr.26252) 1798 : cluster [DBG] pgmap v2460: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:55.364 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:55 smithi177 bash[68594]: cluster 2024-09-13T15:25:53.423969+0000 mgr.y (mgr.26252) 1798 : cluster [DBG] pgmap v2460: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:57.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:25:57 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:25:57] "GET /metrics HTTP/1.1" 200 34330 "" "Prometheus/2.51.0" 2024-09-13T15:25:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:57 smithi099 bash[84576]: cluster 2024-09-13T15:25:55.424860+0000 mgr.y (mgr.26252) 1799 : cluster [DBG] pgmap v2461: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:57.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:57 smithi099 bash[84576]: cluster 2024-09-13T15:25:55.424860+0000 mgr.y (mgr.26252) 1799 : cluster [DBG] pgmap v2461: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:57 smithi099 bash[87147]: cluster 2024-09-13T15:25:55.424860+0000 mgr.y (mgr.26252) 1799 : cluster [DBG] pgmap v2461: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:57.326 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:57 smithi099 bash[87147]: cluster 2024-09-13T15:25:55.424860+0000 mgr.y (mgr.26252) 1799 : cluster [DBG] pgmap v2461: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:57.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:57 smithi177 bash[68594]: cluster 2024-09-13T15:25:55.424860+0000 mgr.y (mgr.26252) 1799 : cluster [DBG] pgmap v2461: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:57.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:57 smithi177 bash[68594]: cluster 2024-09-13T15:25:55.424860+0000 mgr.y (mgr.26252) 1799 : cluster [DBG] pgmap v2461: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:59.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:59 smithi099 bash[84576]: cluster 2024-09-13T15:25:57.426099+0000 mgr.y (mgr.26252) 1800 : cluster [DBG] pgmap v2462: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:59.075 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:25:59 smithi099 bash[84576]: cluster 2024-09-13T15:25:57.426099+0000 mgr.y (mgr.26252) 1800 : cluster [DBG] pgmap v2462: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:59.075 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:59 smithi099 bash[87147]: cluster 2024-09-13T15:25:57.426099+0000 mgr.y (mgr.26252) 1800 : cluster [DBG] pgmap v2462: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:59.076 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:25:59 smithi099 bash[87147]: cluster 2024-09-13T15:25:57.426099+0000 mgr.y (mgr.26252) 1800 : cluster [DBG] pgmap v2462: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:59.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:59 smithi177 bash[68594]: cluster 2024-09-13T15:25:57.426099+0000 mgr.y (mgr.26252) 1800 : cluster [DBG] pgmap v2462: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:25:59.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:25:59 smithi177 bash[68594]: cluster 2024-09-13T15:25:57.426099+0000 mgr.y (mgr.26252) 1800 : cluster [DBG] pgmap v2462: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:00.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:26:00 smithi177 bash[61439]: ts=2024-09-13T15:26:00.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:26:01.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:01 smithi099 bash[84576]: cluster 2024-09-13T15:25:59.426926+0000 mgr.y (mgr.26252) 1801 : cluster [DBG] pgmap v2463: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:01.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:01 smithi099 bash[84576]: cluster 2024-09-13T15:25:59.426926+0000 mgr.y (mgr.26252) 1801 : cluster [DBG] pgmap v2463: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:01.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:01 smithi099 bash[87147]: cluster 2024-09-13T15:25:59.426926+0000 mgr.y (mgr.26252) 1801 : cluster [DBG] pgmap v2463: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:01.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:01 smithi099 bash[87147]: cluster 2024-09-13T15:25:59.426926+0000 mgr.y (mgr.26252) 1801 : cluster [DBG] pgmap v2463: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:01.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:01 smithi177 bash[68594]: cluster 2024-09-13T15:25:59.426926+0000 mgr.y (mgr.26252) 1801 : cluster [DBG] pgmap v2463: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:01.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:01 smithi177 bash[68594]: cluster 2024-09-13T15:25:59.426926+0000 mgr.y (mgr.26252) 1801 : cluster [DBG] pgmap v2463: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:03.102 INFO:teuthology.orchestra.run.smithi099.stdout:2469606195801 2024-09-13T15:26:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:03 smithi177 bash[68594]: cluster 2024-09-13T15:26:01.427769+0000 mgr.y (mgr.26252) 1802 : cluster [DBG] pgmap v2464: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:03.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:03 smithi177 bash[68594]: cluster 2024-09-13T15:26:01.427769+0000 mgr.y (mgr.26252) 1802 : cluster [DBG] pgmap v2464: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:03.409 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:03 smithi099 bash[84576]: cluster 2024-09-13T15:26:01.427769+0000 mgr.y (mgr.26252) 1802 : cluster [DBG] pgmap v2464: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:03.409 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:03 smithi099 bash[84576]: cluster 2024-09-13T15:26:01.427769+0000 mgr.y (mgr.26252) 1802 : cluster [DBG] pgmap v2464: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:03.409 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:03 smithi099 bash[87147]: cluster 2024-09-13T15:26:01.427769+0000 mgr.y (mgr.26252) 1802 : cluster [DBG] pgmap v2464: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:03.409 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:03 smithi099 bash[87147]: cluster 2024-09-13T15:26:01.427769+0000 mgr.y (mgr.26252) 1802 : cluster [DBG] pgmap v2464: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:04.324 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:04 smithi099 bash[87147]: audit 2024-09-13T15:26:03.102600+0000 mon.a (mon.0) 2975 : audit [DBG] from='client.? 172.21.15.99:0/83063943' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:26:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:04 smithi099 bash[87147]: audit 2024-09-13T15:26:03.102600+0000 mon.a (mon.0) 2975 : audit [DBG] from='client.? 172.21.15.99:0/83063943' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:26:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:04 smithi099 bash[87147]: audit 2024-09-13T15:26:03.334454+0000 mon.a (mon.0) 2976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:04.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:04 smithi099 bash[87147]: audit 2024-09-13T15:26:03.334454+0000 mon.a (mon.0) 2976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:04 smithi099 bash[84576]: audit 2024-09-13T15:26:03.102600+0000 mon.a (mon.0) 2975 : audit [DBG] from='client.? 172.21.15.99:0/83063943' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:26:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:04 smithi099 bash[84576]: audit 2024-09-13T15:26:03.102600+0000 mon.a (mon.0) 2975 : audit [DBG] from='client.? 172.21.15.99:0/83063943' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:26:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:04 smithi099 bash[84576]: audit 2024-09-13T15:26:03.334454+0000 mon.a (mon.0) 2976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:04.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:04 smithi099 bash[84576]: audit 2024-09-13T15:26:03.334454+0000 mon.a (mon.0) 2976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:04.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:04 smithi177 bash[68594]: audit 2024-09-13T15:26:03.102600+0000 mon.a (mon.0) 2975 : audit [DBG] from='client.? 172.21.15.99:0/83063943' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:26:04.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:04 smithi177 bash[68594]: audit 2024-09-13T15:26:03.102600+0000 mon.a (mon.0) 2975 : audit [DBG] from='client.? 172.21.15.99:0/83063943' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:26:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:04 smithi177 bash[68594]: audit 2024-09-13T15:26:03.334454+0000 mon.a (mon.0) 2976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:04.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:04 smithi177 bash[68594]: audit 2024-09-13T15:26:03.334454+0000 mon.a (mon.0) 2976 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:05.189 INFO:teuthology.orchestra.run.smithi099.stdout:210453398300 2024-09-13T15:26:05.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:05 smithi099 bash[84576]: cluster 2024-09-13T15:26:03.428694+0000 mgr.y (mgr.26252) 1803 : cluster [DBG] pgmap v2465: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:05.202 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:05 smithi099 bash[84576]: cluster 2024-09-13T15:26:03.428694+0000 mgr.y (mgr.26252) 1803 : cluster [DBG] pgmap v2465: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:05.202 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:05 smithi099 bash[87147]: cluster 2024-09-13T15:26:03.428694+0000 mgr.y (mgr.26252) 1803 : cluster [DBG] pgmap v2465: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:05.203 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:05 smithi099 bash[87147]: cluster 2024-09-13T15:26:03.428694+0000 mgr.y (mgr.26252) 1803 : cluster [DBG] pgmap v2465: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:05.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:05 smithi177 bash[68594]: cluster 2024-09-13T15:26:03.428694+0000 mgr.y (mgr.26252) 1803 : cluster [DBG] pgmap v2465: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:05.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:05 smithi177 bash[68594]: cluster 2024-09-13T15:26:03.428694+0000 mgr.y (mgr.26252) 1803 : cluster [DBG] pgmap v2465: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:06.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:06 smithi099 bash[84576]: audit 2024-09-13T15:26:05.189534+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.99:0/3367159051' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:26:06.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:06 smithi099 bash[84576]: audit 2024-09-13T15:26:05.189534+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.99:0/3367159051' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:26:06.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:06 smithi099 bash[87147]: audit 2024-09-13T15:26:05.189534+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.99:0/3367159051' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:26:06.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:06 smithi099 bash[87147]: audit 2024-09-13T15:26:05.189534+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.99:0/3367159051' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:26:06.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:06 smithi177 bash[68594]: audit 2024-09-13T15:26:05.189534+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.99:0/3367159051' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:26:06.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:06 smithi177 bash[68594]: audit 2024-09-13T15:26:05.189534+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.99:0/3367159051' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:26:06.500 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2469606195796 got 2469606195801 for osd.0 2024-09-13T15:26:06.501 DEBUG:teuthology.parallel:result is None 2024-09-13T15:26:07.306 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:07 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:26:07] "GET /metrics HTTP/1.1" 200 34332 "" "Prometheus/2.51.0" 2024-09-13T15:26:07.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:07 smithi099 bash[84576]: cluster 2024-09-13T15:26:05.429462+0000 mgr.y (mgr.26252) 1804 : cluster [DBG] pgmap v2466: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:07.307 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:07 smithi099 bash[84576]: cluster 2024-09-13T15:26:05.429462+0000 mgr.y (mgr.26252) 1804 : cluster [DBG] pgmap v2466: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:07.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:07 smithi099 bash[87147]: cluster 2024-09-13T15:26:05.429462+0000 mgr.y (mgr.26252) 1804 : cluster [DBG] pgmap v2466: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:07.307 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:07 smithi099 bash[87147]: cluster 2024-09-13T15:26:05.429462+0000 mgr.y (mgr.26252) 1804 : cluster [DBG] pgmap v2466: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:07.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:07 smithi177 bash[68594]: cluster 2024-09-13T15:26:05.429462+0000 mgr.y (mgr.26252) 1804 : cluster [DBG] pgmap v2466: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:07.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:07 smithi177 bash[68594]: cluster 2024-09-13T15:26:05.429462+0000 mgr.y (mgr.26252) 1804 : cluster [DBG] pgmap v2466: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:07.368 INFO:teuthology.orchestra.run.smithi099.stdout:158913790769 2024-09-13T15:26:07.614 INFO:teuthology.orchestra.run.smithi099.stdout:4578435137784 2024-09-13T15:26:08.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:08 smithi177 bash[68594]: audit 2024-09-13T15:26:07.368744+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.99:0/22281825' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:26:08.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:08 smithi177 bash[68594]: audit 2024-09-13T15:26:07.368744+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.99:0/22281825' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:26:08.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:08 smithi177 bash[68594]: audit 2024-09-13T15:26:07.613964+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.99:0/1376214993' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:26:08.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:08 smithi177 bash[68594]: audit 2024-09-13T15:26:07.613964+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.99:0/1376214993' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:26:08.574 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:08 smithi099 bash[87147]: audit 2024-09-13T15:26:07.368744+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.99:0/22281825' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:26:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:08 smithi099 bash[87147]: audit 2024-09-13T15:26:07.368744+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.99:0/22281825' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:26:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:08 smithi099 bash[87147]: audit 2024-09-13T15:26:07.613964+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.99:0/1376214993' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:26:08.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:08 smithi099 bash[87147]: audit 2024-09-13T15:26:07.613964+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.99:0/1376214993' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:26:08.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:08 smithi099 bash[84576]: audit 2024-09-13T15:26:07.368744+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.99:0/22281825' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:26:08.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:08 smithi099 bash[84576]: audit 2024-09-13T15:26:07.368744+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.99:0/22281825' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:26:08.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:08 smithi099 bash[84576]: audit 2024-09-13T15:26:07.613964+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.99:0/1376214993' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:26:08.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:08 smithi099 bash[84576]: audit 2024-09-13T15:26:07.613964+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.99:0/1376214993' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:26:09.277 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:09 smithi099 bash[84576]: cluster 2024-09-13T15:26:07.430065+0000 mgr.y (mgr.26252) 1805 : cluster [DBG] pgmap v2467: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:09.277 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:09 smithi099 bash[84576]: cluster 2024-09-13T15:26:07.430065+0000 mgr.y (mgr.26252) 1805 : cluster [DBG] pgmap v2467: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:09.277 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:09 smithi099 bash[87147]: cluster 2024-09-13T15:26:07.430065+0000 mgr.y (mgr.26252) 1805 : cluster [DBG] pgmap v2467: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:09.277 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:09 smithi099 bash[87147]: cluster 2024-09-13T15:26:07.430065+0000 mgr.y (mgr.26252) 1805 : cluster [DBG] pgmap v2467: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:09.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:09 smithi177 bash[68594]: cluster 2024-09-13T15:26:07.430065+0000 mgr.y (mgr.26252) 1805 : cluster [DBG] pgmap v2467: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:09.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:09 smithi177 bash[68594]: cluster 2024-09-13T15:26:07.430065+0000 mgr.y (mgr.26252) 1805 : cluster [DBG] pgmap v2467: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:09.618 INFO:teuthology.orchestra.run.smithi099.stdout:184683594536 2024-09-13T15:26:09.681 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453398294 got 210453398300 for osd.7 2024-09-13T15:26:09.681 DEBUG:teuthology.parallel:result is None 2024-09-13T15:26:09.883 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790763 got 158913790769 for osd.5 2024-09-13T15:26:09.883 DEBUG:teuthology.parallel:result is None 2024-09-13T15:26:09.920 INFO:teuthology.orchestra.run.smithi099.stdout:3173980832299 2024-09-13T15:26:10.316 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:10 smithi099 bash[84576]: audit 2024-09-13T15:26:09.618470+0000 mon.a (mon.0) 2977 : audit [DBG] from='client.? 172.21.15.99:0/2014935400' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:26:10.316 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:10 smithi099 bash[84576]: audit 2024-09-13T15:26:09.618470+0000 mon.a (mon.0) 2977 : audit [DBG] from='client.? 172.21.15.99:0/2014935400' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:26:10.316 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:10 smithi099 bash[84576]: audit 2024-09-13T15:26:09.920549+0000 mon.a (mon.0) 2978 : audit [DBG] from='client.? 172.21.15.99:0/1849340848' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:26:10.316 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:10 smithi099 bash[84576]: audit 2024-09-13T15:26:09.920549+0000 mon.a (mon.0) 2978 : audit [DBG] from='client.? 172.21.15.99:0/1849340848' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:26:10.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:10 smithi099 bash[87147]: audit 2024-09-13T15:26:09.618470+0000 mon.a (mon.0) 2977 : audit [DBG] from='client.? 172.21.15.99:0/2014935400' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:26:10.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:10 smithi099 bash[87147]: audit 2024-09-13T15:26:09.618470+0000 mon.a (mon.0) 2977 : audit [DBG] from='client.? 172.21.15.99:0/2014935400' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:26:10.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:10 smithi099 bash[87147]: audit 2024-09-13T15:26:09.920549+0000 mon.a (mon.0) 2978 : audit [DBG] from='client.? 172.21.15.99:0/1849340848' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:26:10.317 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:10 smithi099 bash[87147]: audit 2024-09-13T15:26:09.920549+0000 mon.a (mon.0) 2978 : audit [DBG] from='client.? 172.21.15.99:0/1849340848' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:26:10.359 INFO:teuthology.orchestra.run.smithi099.stdout:2886218023481 2024-09-13T15:26:10.362 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:26:10 smithi177 bash[61439]: ts=2024-09-13T15:26:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:26:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:10 smithi177 bash[68594]: audit 2024-09-13T15:26:09.618470+0000 mon.a (mon.0) 2977 : audit [DBG] from='client.? 172.21.15.99:0/2014935400' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:26:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:10 smithi177 bash[68594]: audit 2024-09-13T15:26:09.618470+0000 mon.a (mon.0) 2977 : audit [DBG] from='client.? 172.21.15.99:0/2014935400' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:26:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:10 smithi177 bash[68594]: audit 2024-09-13T15:26:09.920549+0000 mon.a (mon.0) 2978 : audit [DBG] from='client.? 172.21.15.99:0/1849340848' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:26:10.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:10 smithi177 bash[68594]: audit 2024-09-13T15:26:09.920549+0000 mon.a (mon.0) 2978 : audit [DBG] from='client.? 172.21.15.99:0/1849340848' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:26:11.122 INFO:teuthology.orchestra.run.smithi099.stdout:2757369004611 2024-09-13T15:26:11.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:11 smithi177 bash[68594]: cluster 2024-09-13T15:26:09.430491+0000 mgr.y (mgr.26252) 1806 : cluster [DBG] pgmap v2468: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:11.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:11 smithi177 bash[68594]: cluster 2024-09-13T15:26:09.430491+0000 mgr.y (mgr.26252) 1806 : cluster [DBG] pgmap v2468: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:11.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:11 smithi177 bash[68594]: audit 2024-09-13T15:26:10.359928+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.99:0/1207627255' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:26:11.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:11 smithi177 bash[68594]: audit 2024-09-13T15:26:10.359928+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.99:0/1207627255' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:26:11.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:11 smithi099 bash[84576]: cluster 2024-09-13T15:26:09.430491+0000 mgr.y (mgr.26252) 1806 : cluster [DBG] pgmap v2468: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:11.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:11 smithi099 bash[84576]: cluster 2024-09-13T15:26:09.430491+0000 mgr.y (mgr.26252) 1806 : cluster [DBG] pgmap v2468: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:11 smithi099 bash[84576]: audit 2024-09-13T15:26:10.359928+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.99:0/1207627255' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:26:11.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:11 smithi099 bash[84576]: audit 2024-09-13T15:26:10.359928+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.99:0/1207627255' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:26:11.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:11 smithi099 bash[87147]: cluster 2024-09-13T15:26:09.430491+0000 mgr.y (mgr.26252) 1806 : cluster [DBG] pgmap v2468: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:11.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:11 smithi099 bash[87147]: cluster 2024-09-13T15:26:09.430491+0000 mgr.y (mgr.26252) 1806 : cluster [DBG] pgmap v2468: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:11.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:11 smithi099 bash[87147]: audit 2024-09-13T15:26:10.359928+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.99:0/1207627255' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:26:11.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:11 smithi099 bash[87147]: audit 2024-09-13T15:26:10.359928+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.99:0/1207627255' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:26:11.990 INFO:tasks.cephadm.ceph_manager.ceph:need seq 4578435137779 got 4578435137784 for osd.3 2024-09-13T15:26:11.990 DEBUG:teuthology.parallel:result is None 2024-09-13T15:26:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:12 smithi099 bash[84576]: audit 2024-09-13T15:26:11.122112+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.99:0/1589997443' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:26:12.324 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:12 smithi099 bash[84576]: audit 2024-09-13T15:26:11.122112+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.99:0/1589997443' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:26:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:12 smithi099 bash[87147]: audit 2024-09-13T15:26:11.122112+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.99:0/1589997443' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:26:12.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:12 smithi099 bash[87147]: audit 2024-09-13T15:26:11.122112+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.99:0/1589997443' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:26:12.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:12 smithi177 bash[68594]: audit 2024-09-13T15:26:11.122112+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.99:0/1589997443' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:26:12.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:12 smithi177 bash[68594]: audit 2024-09-13T15:26:11.122112+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.99:0/1589997443' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:26:12.843 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683594531 got 184683594536 for osd.6 2024-09-13T15:26:12.844 DEBUG:teuthology.parallel:result is None 2024-09-13T15:26:13.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:13 smithi177 bash[68594]: cluster 2024-09-13T15:26:11.431144+0000 mgr.y (mgr.26252) 1807 : cluster [DBG] pgmap v2469: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:13.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:13 smithi177 bash[68594]: cluster 2024-09-13T15:26:11.431144+0000 mgr.y (mgr.26252) 1807 : cluster [DBG] pgmap v2469: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:13.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:13 smithi099 bash[84576]: cluster 2024-09-13T15:26:11.431144+0000 mgr.y (mgr.26252) 1807 : cluster [DBG] pgmap v2469: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:13.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:13 smithi099 bash[84576]: cluster 2024-09-13T15:26:11.431144+0000 mgr.y (mgr.26252) 1807 : cluster [DBG] pgmap v2469: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:13.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:13 smithi099 bash[87147]: cluster 2024-09-13T15:26:11.431144+0000 mgr.y (mgr.26252) 1807 : cluster [DBG] pgmap v2469: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:13.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:13 smithi099 bash[87147]: cluster 2024-09-13T15:26:11.431144+0000 mgr.y (mgr.26252) 1807 : cluster [DBG] pgmap v2469: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:14.382 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2886218023475 got 2886218023481 for osd.2 2024-09-13T15:26:14.383 DEBUG:teuthology.parallel:result is None 2024-09-13T15:26:14.428 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2757369004605 got 2757369004611 for osd.1 2024-09-13T15:26:14.428 DEBUG:teuthology.parallel:result is None 2024-09-13T15:26:14.439 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3173980832291 got 3173980832299 for osd.4 2024-09-13T15:26:14.439 DEBUG:teuthology.parallel:result is None 2024-09-13T15:26:14.439 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T15:26:14.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:26:15.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:15 smithi177 bash[68594]: cluster 2024-09-13T15:26:13.432026+0000 mgr.y (mgr.26252) 1808 : cluster [DBG] pgmap v2470: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:15.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:15 smithi177 bash[68594]: cluster 2024-09-13T15:26:13.432026+0000 mgr.y (mgr.26252) 1808 : cluster [DBG] pgmap v2470: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:15.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:15 smithi099 bash[84576]: cluster 2024-09-13T15:26:13.432026+0000 mgr.y (mgr.26252) 1808 : cluster [DBG] pgmap v2470: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:15.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:15 smithi099 bash[84576]: cluster 2024-09-13T15:26:13.432026+0000 mgr.y (mgr.26252) 1808 : cluster [DBG] pgmap v2470: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:15.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:15 smithi099 bash[87147]: cluster 2024-09-13T15:26:13.432026+0000 mgr.y (mgr.26252) 1808 : cluster [DBG] pgmap v2470: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:15.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:15 smithi099 bash[87147]: cluster 2024-09-13T15:26:13.432026+0000 mgr.y (mgr.26252) 1808 : cluster [DBG] pgmap v2470: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:17.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:17 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:26:17] "GET /metrics HTTP/1.1" 200 34332 "" "Prometheus/2.51.0" 2024-09-13T15:26:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:17 smithi099 bash[84576]: cluster 2024-09-13T15:26:15.432962+0000 mgr.y (mgr.26252) 1809 : cluster [DBG] pgmap v2471: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:17.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:17 smithi099 bash[84576]: cluster 2024-09-13T15:26:15.432962+0000 mgr.y (mgr.26252) 1809 : cluster [DBG] pgmap v2471: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:17 smithi099 bash[87147]: cluster 2024-09-13T15:26:15.432962+0000 mgr.y (mgr.26252) 1809 : cluster [DBG] pgmap v2471: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:17.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:17 smithi099 bash[87147]: cluster 2024-09-13T15:26:15.432962+0000 mgr.y (mgr.26252) 1809 : cluster [DBG] pgmap v2471: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:17.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:17 smithi177 bash[68594]: cluster 2024-09-13T15:26:15.432962+0000 mgr.y (mgr.26252) 1809 : cluster [DBG] pgmap v2471: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:17.443 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:17 smithi177 bash[68594]: cluster 2024-09-13T15:26:15.432962+0000 mgr.y (mgr.26252) 1809 : cluster [DBG] pgmap v2471: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:19.275 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:26:19.371 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:19 smithi099 bash[84576]: cluster 2024-09-13T15:26:17.434016+0000 mgr.y (mgr.26252) 1810 : cluster [DBG] pgmap v2472: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:19.371 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:19 smithi099 bash[84576]: cluster 2024-09-13T15:26:17.434016+0000 mgr.y (mgr.26252) 1810 : cluster [DBG] pgmap v2472: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:19.371 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:19 smithi099 bash[84576]: audit 2024-09-13T15:26:18.334931+0000 mon.a (mon.0) 2979 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:19.371 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:19 smithi099 bash[84576]: audit 2024-09-13T15:26:18.334931+0000 mon.a (mon.0) 2979 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:19.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:19 smithi099 bash[87147]: cluster 2024-09-13T15:26:17.434016+0000 mgr.y (mgr.26252) 1810 : cluster [DBG] pgmap v2472: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:19.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:19 smithi099 bash[87147]: cluster 2024-09-13T15:26:17.434016+0000 mgr.y (mgr.26252) 1810 : cluster [DBG] pgmap v2472: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:19.371 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:19 smithi099 bash[87147]: audit 2024-09-13T15:26:18.334931+0000 mon.a (mon.0) 2979 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:19.372 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:19 smithi099 bash[87147]: audit 2024-09-13T15:26:18.334931+0000 mon.a (mon.0) 2979 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:19 smithi177 bash[68594]: cluster 2024-09-13T15:26:17.434016+0000 mgr.y (mgr.26252) 1810 : cluster [DBG] pgmap v2472: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:19 smithi177 bash[68594]: cluster 2024-09-13T15:26:17.434016+0000 mgr.y (mgr.26252) 1810 : cluster [DBG] pgmap v2472: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:19 smithi177 bash[68594]: audit 2024-09-13T15:26:18.334931+0000 mon.a (mon.0) 2979 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:19.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:19 smithi177 bash[68594]: audit 2024-09-13T15:26:18.334931+0000 mon.a (mon.0) 2979 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:20.613 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:26:20 smithi177 bash[61439]: ts=2024-09-13T15:26:20.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:26:21.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:21 smithi099 bash[84576]: cluster 2024-09-13T15:26:19.434890+0000 mgr.y (mgr.26252) 1811 : cluster [DBG] pgmap v2473: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:21.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:21 smithi099 bash[84576]: cluster 2024-09-13T15:26:19.434890+0000 mgr.y (mgr.26252) 1811 : cluster [DBG] pgmap v2473: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:21.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:21 smithi099 bash[87147]: cluster 2024-09-13T15:26:19.434890+0000 mgr.y (mgr.26252) 1811 : cluster [DBG] pgmap v2473: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:21.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:21 smithi099 bash[87147]: cluster 2024-09-13T15:26:19.434890+0000 mgr.y (mgr.26252) 1811 : cluster [DBG] pgmap v2473: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:21.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:21 smithi177 bash[68594]: cluster 2024-09-13T15:26:19.434890+0000 mgr.y (mgr.26252) 1811 : cluster [DBG] pgmap v2473: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:21.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:21 smithi177 bash[68594]: cluster 2024-09-13T15:26:19.434890+0000 mgr.y (mgr.26252) 1811 : cluster [DBG] pgmap v2473: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:23.475 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T15:26:23.476 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T15:26:23.487 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:23 smithi099 bash[84576]: cluster 2024-09-13T15:26:21.435648+0000 mgr.y (mgr.26252) 1812 : cluster [DBG] pgmap v2474: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:23.487 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:23 smithi099 bash[84576]: cluster 2024-09-13T15:26:21.435648+0000 mgr.y (mgr.26252) 1812 : cluster [DBG] pgmap v2474: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:23.487 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:23 smithi099 bash[87147]: cluster 2024-09-13T15:26:21.435648+0000 mgr.y (mgr.26252) 1812 : cluster [DBG] pgmap v2474: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:23.487 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:23 smithi099 bash[87147]: cluster 2024-09-13T15:26:21.435648+0000 mgr.y (mgr.26252) 1812 : cluster [DBG] pgmap v2474: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:23 smithi177 bash[68594]: cluster 2024-09-13T15:26:21.435648+0000 mgr.y (mgr.26252) 1812 : cluster [DBG] pgmap v2474: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:23.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:23 smithi177 bash[68594]: cluster 2024-09-13T15:26:21.435648+0000 mgr.y (mgr.26252) 1812 : cluster [DBG] pgmap v2474: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:24.364 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":2475,"stamp":"2024-09-13T15:26:23.435929+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":26,"num_read_kb":28,"num_write":63,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":280346,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1164,"ondisk_log_size":1164,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6525268,"kb_used_data":105204,"kb_used_omap":397,"kb_used_meta":6419570,"kb_avail":743435948,"statfs":{"total":767960285184,"available":761278410752,"internally_reserved":0,"allocated":107728896,"data_stored":67176105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":406940,"internal_metadata":6573640292},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005228"},"pg_stats":[{"pgid":"3.1f","version":"65'5","reported_seq":52,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040019+0000","last_change":"2024-09-13T15:21:41.414612+0000","last_active":"2024-09-13T15:24:31.040019+0000","last_peered":"2024-09-13T15:24:31.040019+0000","last_clean":"2024-09-13T15:24:31.040019+0000","last_became_active":"2024-09-13T15:21:41.406892+0000","last_became_peered":"2024-09-13T15:21:41.406892+0000","last_unstale":"2024-09-13T15:24:31.040019+0000","last_undegraded":"2024-09-13T15:24:31.040019+0000","last_fullsized":"2024-09-13T15:24:31.040019+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:42.810175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2850,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":495,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802154+0000","last_change":"2024-09-13T15:08:52.180022+0000","last_active":"2024-09-13T15:24:31.802154+0000","last_peered":"2024-09-13T15:24:31.802154+0000","last_clean":"2024-09-13T15:24:31.802154+0000","last_became_active":"2024-09-13T15:08:52.179546+0000","last_became_peered":"2024-09-13T15:08:52.179546+0000","last_unstale":"2024-09-13T15:24:31.802154+0000","last_undegraded":"2024-09-13T15:24:31.802154+0000","last_fullsized":"2024-09-13T15:24:31.802154+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:15:47.462895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1505,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.038659+0000","last_change":"2024-09-13T15:21:14.418247+0000","last_active":"2024-09-13T15:24:31.038659+0000","last_peered":"2024-09-13T15:24:31.038659+0000","last_clean":"2024-09-13T15:24:31.038659+0000","last_became_active":"2024-09-13T15:21:14.417022+0000","last_became_peered":"2024-09-13T15:21:14.417022+0000","last_unstale":"2024-09-13T15:24:31.038659+0000","last_undegraded":"2024-09-13T15:24:31.038659+0000","last_fullsized":"2024-09-13T15:24:31.038659+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:53.441136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.017895+0000","last_change":"2024-09-13T15:21:14.426470+0000","last_active":"2024-09-13T15:24:20.017895+0000","last_peered":"2024-09-13T15:24:20.017895+0000","last_clean":"2024-09-13T15:24:20.017895+0000","last_became_active":"2024-09-13T15:21:14.425617+0000","last_became_peered":"2024-09-13T15:21:14.425617+0000","last_unstale":"2024-09-13T15:24:20.017895+0000","last_undegraded":"2024-09-13T15:24:20.017895+0000","last_fullsized":"2024-09-13T15:24:20.017895+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:11.377197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1192,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018567+0000","last_change":"2024-09-13T15:08:52.205951+0000","last_active":"2024-09-13T15:24:20.018567+0000","last_peered":"2024-09-13T15:24:20.018567+0000","last_clean":"2024-09-13T15:24:20.018567+0000","last_became_active":"2024-09-13T15:08:52.183627+0000","last_became_peered":"2024-09-13T15:08:52.183627+0000","last_unstale":"2024-09-13T15:24:20.018567+0000","last_undegraded":"2024-09-13T15:24:20.018567+0000","last_fullsized":"2024-09-13T15:24:20.018567+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:14:14.282531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":478,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802966+0000","last_change":"2024-09-13T15:05:34.086551+0000","last_active":"2024-09-13T15:24:31.802966+0000","last_peered":"2024-09-13T15:24:31.802966+0000","last_clean":"2024-09-13T15:24:31.802966+0000","last_became_active":"2024-09-13T15:05:34.086077+0000","last_became_peered":"2024-09-13T15:05:34.086077+0000","last_unstale":"2024-09-13T15:24:31.802966+0000","last_undegraded":"2024-09-13T15:24:31.802966+0000","last_fullsized":"2024-09-13T15:24:31.802966+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:33:47.917777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1209,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018621+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T15:24:20.018621+0000","last_peered":"2024-09-13T15:24:20.018621+0000","last_clean":"2024-09-13T15:24:20.018621+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T15:24:20.018621+0000","last_undegraded":"2024-09-13T15:24:20.018621+0000","last_fullsized":"2024-09-13T15:24:20.018621+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":492,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803572+0000","last_change":"2024-09-13T15:05:34.075607+0000","last_active":"2024-09-13T15:24:31.803572+0000","last_peered":"2024-09-13T15:24:31.803572+0000","last_clean":"2024-09-13T15:24:31.803572+0000","last_became_active":"2024-09-13T15:05:34.075079+0000","last_became_peered":"2024-09-13T15:05:34.075079+0000","last_unstale":"2024-09-13T15:24:31.803572+0000","last_undegraded":"2024-09-13T15:24:31.803572+0000","last_fullsized":"2024-09-13T15:24:31.803572+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:05.384206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":45,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040610+0000","last_change":"2024-09-13T15:21:41.415390+0000","last_active":"2024-09-13T15:24:31.040610+0000","last_peered":"2024-09-13T15:24:31.040610+0000","last_clean":"2024-09-13T15:24:31.040610+0000","last_became_active":"2024-09-13T15:21:41.414869+0000","last_became_peered":"2024-09-13T15:21:41.414869+0000","last_unstale":"2024-09-13T15:24:31.040610+0000","last_undegraded":"2024-09-13T15:24:31.040610+0000","last_fullsized":"2024-09-13T15:24:31.040610+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:42:01.612211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":50,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.801933+0000","last_change":"2024-09-13T15:21:14.428062+0000","last_active":"2024-09-13T15:24:31.801933+0000","last_peered":"2024-09-13T15:24:31.801933+0000","last_clean":"2024-09-13T15:24:31.801933+0000","last_became_active":"2024-09-13T15:21:14.424173+0000","last_became_peered":"2024-09-13T15:21:14.424173+0000","last_unstale":"2024-09-13T15:24:31.801933+0000","last_undegraded":"2024-09-13T15:24:31.801933+0000","last_fullsized":"2024-09-13T15:24:31.801933+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:25.248892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":495,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802806+0000","last_change":"2024-09-13T15:21:14.421106+0000","last_active":"2024-09-13T15:24:31.802806+0000","last_peered":"2024-09-13T15:24:31.802806+0000","last_clean":"2024-09-13T15:24:31.802806+0000","last_became_active":"2024-09-13T15:21:14.420054+0000","last_became_peered":"2024-09-13T15:21:14.420054+0000","last_unstale":"2024-09-13T15:24:31.802806+0000","last_undegraded":"2024-09-13T15:24:31.802806+0000","last_fullsized":"2024-09-13T15:24:31.802806+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:23.045562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":41,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.801693+0000","last_change":"2024-09-13T15:21:55.779597+0000","last_active":"2024-09-13T15:24:31.801693+0000","last_peered":"2024-09-13T15:24:31.801693+0000","last_clean":"2024-09-13T15:24:31.801693+0000","last_became_active":"2024-09-13T15:21:55.778864+0000","last_became_peered":"2024-09-13T15:21:55.778864+0000","last_unstale":"2024-09-13T15:24:31.801693+0000","last_undegraded":"2024-09-13T15:24:31.801693+0000","last_fullsized":"2024-09-13T15:24:31.801693+0000","mapping_epoch":1164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:01.376294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":382,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.326398+0000","last_change":"2024-09-13T15:05:34.071133+0000","last_active":"2024-09-13T15:24:31.326398+0000","last_peered":"2024-09-13T15:24:31.326398+0000","last_clean":"2024-09-13T15:24:31.326398+0000","last_became_active":"2024-09-13T15:05:34.070519+0000","last_became_peered":"2024-09-13T15:05:34.070519+0000","last_unstale":"2024-09-13T15:24:31.326398+0000","last_undegraded":"2024-09-13T15:24:31.326398+0000","last_fullsized":"2024-09-13T15:24:31.326398+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:02.617403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1767,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040292+0000","last_change":"2024-09-13T15:18:41.619980+0000","last_active":"2024-09-13T15:24:31.040292+0000","last_peered":"2024-09-13T15:24:31.040292+0000","last_clean":"2024-09-13T15:24:31.040292+0000","last_became_active":"2024-09-13T15:18:41.619445+0000","last_became_peered":"2024-09-13T15:18:41.619445+0000","last_unstale":"2024-09-13T15:24:31.040292+0000","last_undegraded":"2024-09-13T15:24:31.040292+0000","last_fullsized":"2024-09-13T15:24:31.040292+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:48.792190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1566,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.037671+0000","last_change":"2024-09-13T15:21:41.488339+0000","last_active":"2024-09-13T15:24:31.037671+0000","last_peered":"2024-09-13T15:24:31.037671+0000","last_clean":"2024-09-13T15:24:31.037671+0000","last_became_active":"2024-09-13T15:21:41.487177+0000","last_became_peered":"2024-09-13T15:21:41.487177+0000","last_unstale":"2024-09-13T15:24:31.037671+0000","last_undegraded":"2024-09-13T15:24:31.037671+0000","last_fullsized":"2024-09-13T15:24:31.037671+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:45.259145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":118,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.933620+0000","last_change":"2024-09-13T15:18:41.606623+0000","last_active":"2024-09-13T15:24:20.933620+0000","last_peered":"2024-09-13T15:24:20.933620+0000","last_clean":"2024-09-13T15:24:20.933620+0000","last_became_active":"2024-09-13T15:18:41.606144+0000","last_became_peered":"2024-09-13T15:18:41.606144+0000","last_unstale":"2024-09-13T15:24:20.933620+0000","last_undegraded":"2024-09-13T15:24:20.933620+0000","last_fullsized":"2024-09-13T15:24:20.933620+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:12.837936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1532,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.036703+0000","last_change":"2024-09-13T15:08:52.170737+0000","last_active":"2024-09-13T15:24:31.036703+0000","last_peered":"2024-09-13T15:24:31.036703+0000","last_clean":"2024-09-13T15:24:31.036703+0000","last_became_active":"2024-09-13T15:08:52.170217+0000","last_became_peered":"2024-09-13T15:08:52.170217+0000","last_unstale":"2024-09-13T15:24:31.036703+0000","last_undegraded":"2024-09-13T15:24:31.036703+0000","last_fullsized":"2024-09-13T15:24:31.036703+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:17.688936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1760,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.041059+0000","last_change":"2024-09-13T15:05:34.087376+0000","last_active":"2024-09-13T15:24:31.041059+0000","last_peered":"2024-09-13T15:24:31.041059+0000","last_clean":"2024-09-13T15:24:31.041059+0000","last_became_active":"2024-09-13T15:05:34.087046+0000","last_became_peered":"2024-09-13T15:05:34.087046+0000","last_unstale":"2024-09-13T15:24:31.041059+0000","last_undegraded":"2024-09-13T15:24:31.041059+0000","last_fullsized":"2024-09-13T15:24:31.041059+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:47.801431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1472,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.036996+0000","last_change":"2024-09-13T15:08:52.174279+0000","last_active":"2024-09-13T15:24:31.036996+0000","last_peered":"2024-09-13T15:24:31.036996+0000","last_clean":"2024-09-13T15:24:31.036996+0000","last_became_active":"2024-09-13T15:08:52.173877+0000","last_became_peered":"2024-09-13T15:08:52.173877+0000","last_unstale":"2024-09-13T15:24:31.036996+0000","last_undegraded":"2024-09-13T15:24:31.036996+0000","last_fullsized":"2024-09-13T15:24:31.036996+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:16.648765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1153,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018275+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T15:24:20.018275+0000","last_peered":"2024-09-13T15:24:20.018275+0000","last_clean":"2024-09-13T15:24:20.018275+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T15:24:20.018275+0000","last_undegraded":"2024-09-13T15:24:20.018275+0000","last_fullsized":"2024-09-13T15:24:20.018275+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1207,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558559+0000","last_change":"2024-09-13T15:05:34.089020+0000","last_active":"2024-09-13T15:24:20.558559+0000","last_peered":"2024-09-13T15:24:20.558559+0000","last_clean":"2024-09-13T15:24:20.558559+0000","last_became_active":"2024-09-13T15:05:34.088440+0000","last_became_peered":"2024-09-13T15:05:34.088440+0000","last_unstale":"2024-09-13T15:24:20.558559+0000","last_undegraded":"2024-09-13T15:24:20.558559+0000","last_fullsized":"2024-09-13T15:24:20.558559+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:24.826746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":162,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.039293+0000","last_change":"2024-09-13T15:21:14.416181+0000","last_active":"2024-09-13T15:24:31.039293+0000","last_peered":"2024-09-13T15:24:31.039293+0000","last_clean":"2024-09-13T15:24:31.039293+0000","last_became_active":"2024-09-13T15:21:14.415217+0000","last_became_peered":"2024-09-13T15:21:14.415217+0000","last_unstale":"2024-09-13T15:24:31.039293+0000","last_undegraded":"2024-09-13T15:24:31.039293+0000","last_fullsized":"2024-09-13T15:24:31.039293+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:06:59.197995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":512,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803250+0000","last_change":"2024-09-13T15:08:52.203370+0000","last_active":"2024-09-13T15:24:31.803250+0000","last_peered":"2024-09-13T15:24:31.803250+0000","last_clean":"2024-09-13T15:24:31.803250+0000","last_became_active":"2024-09-13T15:08:52.179887+0000","last_became_peered":"2024-09-13T15:08:52.179887+0000","last_unstale":"2024-09-13T15:24:31.803250+0000","last_undegraded":"2024-09-13T15:24:31.803250+0000","last_fullsized":"2024-09-13T15:24:31.803250+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:57.293764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":503,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802277+0000","last_change":"2024-09-13T15:20:55.395006+0000","last_active":"2024-09-13T15:24:31.802277+0000","last_peered":"2024-09-13T15:24:31.802277+0000","last_clean":"2024-09-13T15:24:31.802277+0000","last_became_active":"2024-09-13T15:20:55.393901+0000","last_became_peered":"2024-09-13T15:20:55.393901+0000","last_unstale":"2024-09-13T15:24:31.802277+0000","last_undegraded":"2024-09-13T15:24:31.802277+0000","last_fullsized":"2024-09-13T15:24:31.802277+0000","mapping_epoch":1157,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:04.690653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"346'100","reported_seq":99,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.038014+0000","last_change":"2024-09-13T15:20:09.715321+0000","last_active":"2024-09-13T15:24:31.038014+0000","last_peered":"2024-09-13T15:24:31.038014+0000","last_clean":"2024-09-13T15:24:31.038014+0000","last_became_active":"2024-09-13T15:19:58.641197+0000","last_became_peered":"2024-09-13T15:19:58.641197+0000","last_unstale":"2024-09-13T15:24:31.038014+0000","last_undegraded":"2024-09-13T15:24:31.038014+0000","last_fullsized":"2024-09-13T15:24:31.038014+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:09.715239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:09.715239+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:39.018561+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":484,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802811+0000","last_change":"2024-09-13T15:08:52.188050+0000","last_active":"2024-09-13T15:24:31.802811+0000","last_peered":"2024-09-13T15:24:31.802811+0000","last_clean":"2024-09-13T15:24:31.802811+0000","last_became_active":"2024-09-13T15:08:52.187522+0000","last_became_peered":"2024-09-13T15:08:52.187522+0000","last_unstale":"2024-09-13T15:24:31.802811+0000","last_undegraded":"2024-09-13T15:24:31.802811+0000","last_fullsized":"2024-09-13T15:24:31.802811+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:29.314964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":128,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.933445+0000","last_change":"2024-09-13T15:21:14.415297+0000","last_active":"2024-09-13T15:24:20.933445+0000","last_peered":"2024-09-13T15:24:20.933445+0000","last_clean":"2024-09-13T15:24:20.933445+0000","last_became_active":"2024-09-13T15:21:14.414292+0000","last_became_peered":"2024-09-13T15:21:14.414292+0000","last_unstale":"2024-09-13T15:24:20.933445+0000","last_undegraded":"2024-09-13T15:24:20.933445+0000","last_fullsized":"2024-09-13T15:24:20.933445+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.468824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1812,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040452+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T15:24:31.040452+0000","last_peered":"2024-09-13T15:24:31.040452+0000","last_clean":"2024-09-13T15:24:31.040452+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T15:24:31.040452+0000","last_undegraded":"2024-09-13T15:24:31.040452+0000","last_fullsized":"2024-09-13T15:24:31.040452+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"346'100","reported_seq":101,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803167+0000","last_change":"2024-09-13T15:21:14.420881+0000","last_active":"2024-09-13T15:24:31.803167+0000","last_peered":"2024-09-13T15:24:31.803167+0000","last_clean":"2024-09-13T15:24:31.803167+0000","last_became_active":"2024-09-13T15:21:14.419843+0000","last_became_peered":"2024-09-13T15:21:14.419843+0000","last_unstale":"2024-09-13T15:24:31.803167+0000","last_undegraded":"2024-09-13T15:24:31.803167+0000","last_fullsized":"2024-09-13T15:24:31.803167+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:09.523750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:09.523750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:33.221965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1474,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.036988+0000","last_change":"2024-09-13T15:21:41.415212+0000","last_active":"2024-09-13T15:24:31.036988+0000","last_peered":"2024-09-13T15:24:31.036988+0000","last_clean":"2024-09-13T15:24:31.036988+0000","last_became_active":"2024-09-13T15:21:41.414028+0000","last_became_peered":"2024-09-13T15:21:41.414028+0000","last_unstale":"2024-09-13T15:24:31.036988+0000","last_undegraded":"2024-09-13T15:24:31.036988+0000","last_fullsized":"2024-09-13T15:24:31.036988+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:46.313434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":387,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327766+0000","last_change":"2024-09-13T15:08:52.186362+0000","last_active":"2024-09-13T15:24:31.327766+0000","last_peered":"2024-09-13T15:24:31.327766+0000","last_clean":"2024-09-13T15:24:31.327766+0000","last_became_active":"2024-09-13T15:08:52.185731+0000","last_became_peered":"2024-09-13T15:08:52.185731+0000","last_unstale":"2024-09-13T15:24:31.327766+0000","last_undegraded":"2024-09-13T15:24:31.327766+0000","last_fullsized":"2024-09-13T15:24:31.327766+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:26.408919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1462,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040561+0000","last_change":"2024-09-13T15:05:34.085917+0000","last_active":"2024-09-13T15:24:31.040561+0000","last_peered":"2024-09-13T15:24:31.040561+0000","last_clean":"2024-09-13T15:24:31.040561+0000","last_became_active":"2024-09-13T15:05:34.085156+0000","last_became_peered":"2024-09-13T15:05:34.085156+0000","last_unstale":"2024-09-13T15:24:31.040561+0000","last_undegraded":"2024-09-13T15:24:31.040561+0000","last_fullsized":"2024-09-13T15:24:31.040561+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:33.482732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"346'100","reported_seq":1227,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.933434+0000","last_change":"2024-09-13T15:20:10.562576+0000","last_active":"2024-09-13T15:24:20.933434+0000","last_peered":"2024-09-13T15:24:20.933434+0000","last_clean":"2024-09-13T15:24:20.933434+0000","last_became_active":"2024-09-13T15:19:58.639558+0000","last_became_peered":"2024-09-13T15:19:58.639558+0000","last_unstale":"2024-09-13T15:24:20.933434+0000","last_undegraded":"2024-09-13T15:24:20.933434+0000","last_fullsized":"2024-09-13T15:24:20.933434+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:10.562423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:10.562423+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:54.624883+0000","scrub_duration":0.0039412570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1742,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040089+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T15:24:31.040089+0000","last_peered":"2024-09-13T15:24:31.040089+0000","last_clean":"2024-09-13T15:24:31.040089+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T15:24:31.040089+0000","last_undegraded":"2024-09-13T15:24:31.040089+0000","last_fullsized":"2024-09-13T15:24:31.040089+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1185,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558834+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T15:24:20.558834+0000","last_peered":"2024-09-13T15:24:20.558834+0000","last_clean":"2024-09-13T15:24:20.558834+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T15:24:20.558834+0000","last_undegraded":"2024-09-13T15:24:20.558834+0000","last_fullsized":"2024-09-13T15:24:20.558834+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":388,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327276+0000","last_change":"2024-09-13T15:21:14.420902+0000","last_active":"2024-09-13T15:24:31.327276+0000","last_peered":"2024-09-13T15:24:31.327276+0000","last_clean":"2024-09-13T15:24:31.327276+0000","last_became_active":"2024-09-13T15:21:14.419966+0000","last_became_peered":"2024-09-13T15:21:14.419966+0000","last_unstale":"2024-09-13T15:24:31.327276+0000","last_undegraded":"2024-09-13T15:24:31.327276+0000","last_fullsized":"2024-09-13T15:24:31.327276+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:07:52.638527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"346'100","reported_seq":1227,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.933448+0000","last_change":"2024-09-13T15:20:14.516263+0000","last_active":"2024-09-13T15:24:20.933448+0000","last_peered":"2024-09-13T15:24:20.933448+0000","last_clean":"2024-09-13T15:24:20.933448+0000","last_became_active":"2024-09-13T15:19:58.638300+0000","last_became_peered":"2024-09-13T15:19:58.638300+0000","last_unstale":"2024-09-13T15:24:20.933448+0000","last_undegraded":"2024-09-13T15:24:20.933448+0000","last_fullsized":"2024-09-13T15:24:20.933448+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:14.516167+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:14.516167+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:46:37.405030+0000","scrub_duration":0.0023068670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":374,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327122+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T15:24:31.327122+0000","last_peered":"2024-09-13T15:24:31.327122+0000","last_clean":"2024-09-13T15:24:31.327122+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T15:24:31.327122+0000","last_undegraded":"2024-09-13T15:24:31.327122+0000","last_fullsized":"2024-09-13T15:24:31.327122+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":374,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327442+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T15:24:31.327442+0000","last_peered":"2024-09-13T15:24:31.327442+0000","last_clean":"2024-09-13T15:24:31.327442+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T15:24:31.327442+0000","last_undegraded":"2024-09-13T15:24:31.327442+0000","last_fullsized":"2024-09-13T15:24:31.327442+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1187,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018683+0000","last_change":"2024-09-13T15:21:14.425885+0000","last_active":"2024-09-13T15:24:20.018683+0000","last_peered":"2024-09-13T15:24:20.018683+0000","last_clean":"2024-09-13T15:24:20.018683+0000","last_became_active":"2024-09-13T15:21:14.424951+0000","last_became_peered":"2024-09-13T15:21:14.424951+0000","last_unstale":"2024-09-13T15:24:20.018683+0000","last_undegraded":"2024-09-13T15:24:20.018683+0000","last_fullsized":"2024-09-13T15:24:20.018683+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:05.236686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"346'100","reported_seq":62,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.017913+0000","last_change":"2024-09-13T15:21:14.425980+0000","last_active":"2024-09-13T15:24:20.017913+0000","last_peered":"2024-09-13T15:24:20.017913+0000","last_clean":"2024-09-13T15:24:20.017913+0000","last_became_active":"2024-09-13T15:21:14.424979+0000","last_became_peered":"2024-09-13T15:21:14.424979+0000","last_unstale":"2024-09-13T15:24:20.017913+0000","last_undegraded":"2024-09-13T15:24:20.017913+0000","last_fullsized":"2024-09-13T15:24:20.017913+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:06.725832+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:06.725832+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:03:39.021814+0000","scrub_duration":0.0036607340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":41,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018133+0000","last_change":"2024-09-13T15:21:14.430198+0000","last_active":"2024-09-13T15:24:20.018133+0000","last_peered":"2024-09-13T15:24:20.018133+0000","last_clean":"2024-09-13T15:24:20.018133+0000","last_became_active":"2024-09-13T15:21:14.429446+0000","last_became_peered":"2024-09-13T15:21:14.429446+0000","last_unstale":"2024-09-13T15:24:20.018133+0000","last_undegraded":"2024-09-13T15:24:20.018133+0000","last_fullsized":"2024-09-13T15:24:20.018133+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:48.415699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":484,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803653+0000","last_change":"2024-09-13T15:08:52.188254+0000","last_active":"2024-09-13T15:24:31.803653+0000","last_peered":"2024-09-13T15:24:31.803653+0000","last_clean":"2024-09-13T15:24:31.803653+0000","last_became_active":"2024-09-13T15:08:52.187535+0000","last_became_peered":"2024-09-13T15:08:52.187535+0000","last_unstale":"2024-09-13T15:24:31.803653+0000","last_undegraded":"2024-09-13T15:24:31.803653+0000","last_fullsized":"2024-09-13T15:24:31.803653+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:32.024558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1206,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018655+0000","last_change":"2024-09-13T15:08:52.209457+0000","last_active":"2024-09-13T15:24:20.018655+0000","last_peered":"2024-09-13T15:24:20.018655+0000","last_clean":"2024-09-13T15:24:20.018655+0000","last_became_active":"2024-09-13T15:08:52.171595+0000","last_became_peered":"2024-09-13T15:08:52.171595+0000","last_unstale":"2024-09-13T15:24:20.018655+0000","last_undegraded":"2024-09-13T15:24:20.018655+0000","last_fullsized":"2024-09-13T15:24:20.018655+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:05:42.960465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"346'100","reported_seq":1315,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.557919+0000","last_change":"2024-09-13T15:19:58.639631+0000","last_active":"2024-09-13T15:24:20.557919+0000","last_peered":"2024-09-13T15:24:20.557919+0000","last_clean":"2024-09-13T15:24:20.557919+0000","last_became_active":"2024-09-13T15:19:58.639023+0000","last_became_peered":"2024-09-13T15:19:58.639023+0000","last_unstale":"2024-09-13T15:24:20.557919+0000","last_undegraded":"2024-09-13T15:24:20.557919+0000","last_fullsized":"2024-09-13T15:24:20.557919+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:18:17.216949+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:18:17.216949+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:10.007605+0000","scrub_duration":0.011279501000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":373,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327709+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T15:24:31.327709+0000","last_peered":"2024-09-13T15:24:31.327709+0000","last_clean":"2024-09-13T15:24:31.327709+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T15:24:31.327709+0000","last_undegraded":"2024-09-13T15:24:31.327709+0000","last_fullsized":"2024-09-13T15:24:31.327709+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":402,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.326932+0000","last_change":"2024-09-13T15:21:14.414672+0000","last_active":"2024-09-13T15:24:31.326932+0000","last_peered":"2024-09-13T15:24:31.326932+0000","last_clean":"2024-09-13T15:24:31.326932+0000","last_became_active":"2024-09-13T15:21:14.413645+0000","last_became_peered":"2024-09-13T15:21:14.413645+0000","last_unstale":"2024-09-13T15:24:31.326932+0000","last_undegraded":"2024-09-13T15:24:31.326932+0000","last_fullsized":"2024-09-13T15:24:31.326932+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:18.095504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":50,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.801790+0000","last_change":"2024-09-13T15:21:14.412552+0000","last_active":"2024-09-13T15:24:31.801790+0000","last_peered":"2024-09-13T15:24:31.801790+0000","last_clean":"2024-09-13T15:24:31.801790+0000","last_became_active":"2024-09-13T15:21:14.411158+0000","last_became_peered":"2024-09-13T15:21:14.411158+0000","last_unstale":"2024-09-13T15:24:31.801790+0000","last_undegraded":"2024-09-13T15:24:31.801790+0000","last_fullsized":"2024-09-13T15:24:31.801790+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:08.123906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"346'100","reported_seq":94,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.326930+0000","last_change":"2024-09-13T15:20:10.181623+0000","last_active":"2024-09-13T15:24:31.326930+0000","last_peered":"2024-09-13T15:24:31.326930+0000","last_clean":"2024-09-13T15:24:31.326930+0000","last_became_active":"2024-09-13T15:19:58.637274+0000","last_became_peered":"2024-09-13T15:19:58.637274+0000","last_unstale":"2024-09-13T15:24:31.326930+0000","last_undegraded":"2024-09-13T15:24:31.326930+0000","last_fullsized":"2024-09-13T15:24:31.326930+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:10.181561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:10.181561+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:33.160752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":46,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040231+0000","last_change":"2024-09-13T15:21:41.488014+0000","last_active":"2024-09-13T15:24:31.040231+0000","last_peered":"2024-09-13T15:24:31.040231+0000","last_clean":"2024-09-13T15:24:31.040231+0000","last_became_active":"2024-09-13T15:21:41.487059+0000","last_became_peered":"2024-09-13T15:21:41.487059+0000","last_unstale":"2024-09-13T15:24:31.040231+0000","last_undegraded":"2024-09-13T15:24:31.040231+0000","last_fullsized":"2024-09-13T15:24:31.040231+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:20:34.460599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1789,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.039154+0000","last_change":"2024-09-13T15:05:34.087817+0000","last_active":"2024-09-13T15:24:31.039154+0000","last_peered":"2024-09-13T15:24:31.039154+0000","last_clean":"2024-09-13T15:24:31.039154+0000","last_became_active":"2024-09-13T15:05:34.087450+0000","last_became_peered":"2024-09-13T15:05:34.087450+0000","last_unstale":"2024-09-13T15:24:31.039154+0000","last_undegraded":"2024-09-13T15:24:31.039154+0000","last_fullsized":"2024-09-13T15:24:31.039154+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:47.661935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1185,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.557975+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T15:24:20.557975+0000","last_peered":"2024-09-13T15:24:20.557975+0000","last_clean":"2024-09-13T15:24:20.557975+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T15:24:20.557975+0000","last_undegraded":"2024-09-13T15:24:20.557975+0000","last_fullsized":"2024-09-13T15:24:20.557975+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":108,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.933461+0000","last_change":"2024-09-13T15:08:52.188028+0000","last_active":"2024-09-13T15:24:20.933461+0000","last_peered":"2024-09-13T15:24:20.933461+0000","last_clean":"2024-09-13T15:24:20.933461+0000","last_became_active":"2024-09-13T15:08:52.187424+0000","last_became_peered":"2024-09-13T15:08:52.187424+0000","last_unstale":"2024-09-13T15:24:20.933461+0000","last_undegraded":"2024-09-13T15:24:20.933461+0000","last_fullsized":"2024-09-13T15:24:20.933461+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:31.790876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1475,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.037226+0000","last_change":"2024-09-13T15:08:52.170510+0000","last_active":"2024-09-13T15:24:31.037226+0000","last_peered":"2024-09-13T15:24:31.037226+0000","last_clean":"2024-09-13T15:24:31.037226+0000","last_became_active":"2024-09-13T15:08:52.170141+0000","last_became_peered":"2024-09-13T15:08:52.170141+0000","last_unstale":"2024-09-13T15:24:31.037226+0000","last_undegraded":"2024-09-13T15:24:31.037226+0000","last_fullsized":"2024-09-13T15:24:31.037226+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:20.674380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1174,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018371+0000","last_change":"2024-09-13T15:21:14.446858+0000","last_active":"2024-09-13T15:24:20.018371+0000","last_peered":"2024-09-13T15:24:20.018371+0000","last_clean":"2024-09-13T15:24:20.018371+0000","last_became_active":"2024-09-13T15:21:14.419856+0000","last_became_peered":"2024-09-13T15:21:14.419856+0000","last_unstale":"2024-09-13T15:24:20.018371+0000","last_undegraded":"2024-09-13T15:24:20.018371+0000","last_fullsized":"2024-09-13T15:24:20.018371+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:26:11.634373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1306,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558121+0000","last_change":"2024-09-13T15:17:58.733763+0000","last_active":"2024-09-13T15:24:20.558121+0000","last_peered":"2024-09-13T15:24:20.558121+0000","last_clean":"2024-09-13T15:24:20.558121+0000","last_became_active":"2024-09-13T15:17:58.326073+0000","last_became_peered":"2024-09-13T15:17:58.326073+0000","last_unstale":"2024-09-13T15:24:20.558121+0000","last_undegraded":"2024-09-13T15:24:20.558121+0000","last_fullsized":"2024-09-13T15:24:20.558121+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:17:58.733680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:17:58.733680+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:09:09.783840+0000","scrub_duration":0.006408459,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"346'100","reported_seq":62,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.017948+0000","last_change":"2024-09-13T15:21:14.420905+0000","last_active":"2024-09-13T15:24:20.017948+0000","last_peered":"2024-09-13T15:24:20.017948+0000","last_clean":"2024-09-13T15:24:20.017948+0000","last_became_active":"2024-09-13T15:21:14.419866+0000","last_became_peered":"2024-09-13T15:21:14.419866+0000","last_unstale":"2024-09-13T15:24:20.017948+0000","last_undegraded":"2024-09-13T15:24:20.017948+0000","last_fullsized":"2024-09-13T15:24:20.017948+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:09.661080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:09.661080+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:30.770615+0000","scrub_duration":0.0033469020000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":383,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327789+0000","last_change":"2024-09-13T15:05:34.078805+0000","last_active":"2024-09-13T15:24:31.327789+0000","last_peered":"2024-09-13T15:24:31.327789+0000","last_clean":"2024-09-13T15:24:31.327789+0000","last_became_active":"2024-09-13T15:05:34.078180+0000","last_became_peered":"2024-09-13T15:05:34.078180+0000","last_unstale":"2024-09-13T15:24:31.327789+0000","last_undegraded":"2024-09-13T15:24:31.327789+0000","last_fullsized":"2024-09-13T15:24:31.327789+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:47.240906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1523,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.037260+0000","last_change":"2024-09-13T15:05:34.076149+0000","last_active":"2024-09-13T15:24:31.037260+0000","last_peered":"2024-09-13T15:24:31.037260+0000","last_clean":"2024-09-13T15:24:31.037260+0000","last_became_active":"2024-09-13T15:05:34.075623+0000","last_became_peered":"2024-09-13T15:05:34.075623+0000","last_unstale":"2024-09-13T15:24:31.037260+0000","last_undegraded":"2024-09-13T15:24:31.037260+0000","last_fullsized":"2024-09-13T15:24:31.037260+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:23.608262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":497,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803537+0000","last_change":"2024-09-13T15:08:52.189786+0000","last_active":"2024-09-13T15:24:31.803537+0000","last_peered":"2024-09-13T15:24:31.803537+0000","last_clean":"2024-09-13T15:24:31.803537+0000","last_became_active":"2024-09-13T15:08:52.188997+0000","last_became_peered":"2024-09-13T15:08:52.188997+0000","last_unstale":"2024-09-13T15:24:31.803537+0000","last_undegraded":"2024-09-13T15:24:31.803537+0000","last_fullsized":"2024-09-13T15:24:31.803537+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:08.932986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":483,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802626+0000","last_change":"2024-09-13T15:21:14.426533+0000","last_active":"2024-09-13T15:24:31.802626+0000","last_peered":"2024-09-13T15:24:31.802626+0000","last_clean":"2024-09-13T15:24:31.802626+0000","last_became_active":"2024-09-13T15:21:14.425534+0000","last_became_peered":"2024-09-13T15:21:14.425534+0000","last_unstale":"2024-09-13T15:24:31.802626+0000","last_undegraded":"2024-09-13T15:24:31.802626+0000","last_fullsized":"2024-09-13T15:24:31.802626+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:00:07.710512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"346'100","reported_seq":71,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.017608+0000","last_change":"2024-09-13T15:21:14.452344+0000","last_active":"2024-09-13T15:24:20.017608+0000","last_peered":"2024-09-13T15:24:20.017608+0000","last_clean":"2024-09-13T15:24:20.017608+0000","last_became_active":"2024-09-13T15:21:14.422433+0000","last_became_peered":"2024-09-13T15:21:14.422433+0000","last_unstale":"2024-09-13T15:24:20.017608+0000","last_undegraded":"2024-09-13T15:24:20.017608+0000","last_fullsized":"2024-09-13T15:24:20.017608+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:08.705948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:08.705948+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:27:01.156043+0000","scrub_duration":0.0041351900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":497,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803790+0000","last_change":"2024-09-13T15:21:14.420492+0000","last_active":"2024-09-13T15:24:31.803790+0000","last_peered":"2024-09-13T15:24:31.803790+0000","last_clean":"2024-09-13T15:24:31.803790+0000","last_became_active":"2024-09-13T15:21:14.419542+0000","last_became_peered":"2024-09-13T15:21:14.419542+0000","last_unstale":"2024-09-13T15:24:31.803790+0000","last_undegraded":"2024-09-13T15:24:31.803790+0000","last_fullsized":"2024-09-13T15:24:31.803790+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:37:55.394189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":108,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.932046+0000","last_change":"2024-09-13T15:08:52.182709+0000","last_active":"2024-09-13T15:24:20.932046+0000","last_peered":"2024-09-13T15:24:20.932046+0000","last_clean":"2024-09-13T15:24:20.932046+0000","last_became_active":"2024-09-13T15:08:52.181906+0000","last_became_peered":"2024-09-13T15:08:52.181906+0000","last_unstale":"2024-09-13T15:24:20.932046+0000","last_undegraded":"2024-09-13T15:24:20.932046+0000","last_fullsized":"2024-09-13T15:24:20.932046+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:36.317662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"346'100","reported_seq":1325,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.557751+0000","last_change":"2024-09-13T15:21:14.408213+0000","last_active":"2024-09-13T15:24:20.557751+0000","last_peered":"2024-09-13T15:24:20.557751+0000","last_clean":"2024-09-13T15:24:20.557751+0000","last_became_active":"2024-09-13T15:21:14.406975+0000","last_became_peered":"2024-09-13T15:21:14.406975+0000","last_unstale":"2024-09-13T15:24:20.557751+0000","last_undegraded":"2024-09-13T15:24:20.557751+0000","last_fullsized":"2024-09-13T15:24:20.557751+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:18:04.969997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:18:04.969997+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:09.481540+0000","scrub_duration":0.0031202019999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":494,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803172+0000","last_change":"2024-09-13T15:08:52.182885+0000","last_active":"2024-09-13T15:24:31.803172+0000","last_peered":"2024-09-13T15:24:31.803172+0000","last_clean":"2024-09-13T15:24:31.803172+0000","last_became_active":"2024-09-13T15:08:52.180608+0000","last_became_peered":"2024-09-13T15:08:52.180608+0000","last_unstale":"2024-09-13T15:24:31.803172+0000","last_undegraded":"2024-09-13T15:24:31.803172+0000","last_fullsized":"2024-09-13T15:24:31.803172+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:12.517462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1507,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.037687+0000","last_change":"2024-09-13T15:05:34.085458+0000","last_active":"2024-09-13T15:24:31.037687+0000","last_peered":"2024-09-13T15:24:31.037687+0000","last_clean":"2024-09-13T15:24:31.037687+0000","last_became_active":"2024-09-13T15:05:34.084748+0000","last_became_peered":"2024-09-13T15:05:34.084748+0000","last_unstale":"2024-09-13T15:24:31.037687+0000","last_undegraded":"2024-09-13T15:24:31.037687+0000","last_fullsized":"2024-09-13T15:24:31.037687+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:21.898959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1804,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040437+0000","last_change":"2024-09-13T15:18:41.638220+0000","last_active":"2024-09-13T15:24:31.040437+0000","last_peered":"2024-09-13T15:24:31.040437+0000","last_clean":"2024-09-13T15:24:31.040437+0000","last_became_active":"2024-09-13T15:18:41.637822+0000","last_became_peered":"2024-09-13T15:18:41.637822+0000","last_unstale":"2024-09-13T15:24:31.040437+0000","last_undegraded":"2024-09-13T15:24:31.040437+0000","last_fullsized":"2024-09-13T15:24:31.040437+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:59:28.421185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1170,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018444+0000","last_change":"2024-09-13T15:08:52.175569+0000","last_active":"2024-09-13T15:24:20.018444+0000","last_peered":"2024-09-13T15:24:20.018444+0000","last_clean":"2024-09-13T15:24:20.018444+0000","last_became_active":"2024-09-13T15:08:52.174891+0000","last_became_peered":"2024-09-13T15:08:52.174891+0000","last_unstale":"2024-09-13T15:24:20.018444+0000","last_undegraded":"2024-09-13T15:24:20.018444+0000","last_fullsized":"2024-09-13T15:24:20.018444+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.933115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":108,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.932075+0000","last_change":"2024-09-13T15:08:52.188074+0000","last_active":"2024-09-13T15:24:20.932075+0000","last_peered":"2024-09-13T15:24:20.932075+0000","last_clean":"2024-09-13T15:24:20.932075+0000","last_became_active":"2024-09-13T15:08:52.186649+0000","last_became_peered":"2024-09-13T15:08:52.186649+0000","last_unstale":"2024-09-13T15:24:20.932075+0000","last_undegraded":"2024-09-13T15:24:20.932075+0000","last_fullsized":"2024-09-13T15:24:20.932075+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:23.478250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":1149,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.557608+0000","last_change":"2024-09-13T15:08:52.208388+0000","last_active":"2024-09-13T15:24:20.557608+0000","last_peered":"2024-09-13T15:24:20.557608+0000","last_clean":"2024-09-13T15:24:20.557608+0000","last_became_active":"2024-09-13T15:08:52.192253+0000","last_became_peered":"2024-09-13T15:08:52.192253+0000","last_unstale":"2024-09-13T15:24:20.557608+0000","last_undegraded":"2024-09-13T15:24:20.557608+0000","last_fullsized":"2024-09-13T15:24:20.557608+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:07.508774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":494,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802908+0000","last_change":"2024-09-13T15:05:34.072278+0000","last_active":"2024-09-13T15:24:31.802908+0000","last_peered":"2024-09-13T15:24:31.802908+0000","last_clean":"2024-09-13T15:24:31.802908+0000","last_became_active":"2024-09-13T15:05:34.071723+0000","last_became_peered":"2024-09-13T15:05:34.071723+0000","last_unstale":"2024-09-13T15:24:31.802908+0000","last_undegraded":"2024-09-13T15:24:31.802908+0000","last_fullsized":"2024-09-13T15:24:31.802908+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:02.130890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1212,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558972+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T15:24:20.558972+0000","last_peered":"2024-09-13T15:24:20.558972+0000","last_clean":"2024-09-13T15:24:20.558972+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T15:24:20.558972+0000","last_undegraded":"2024-09-13T15:24:20.558972+0000","last_fullsized":"2024-09-13T15:24:20.558972+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1135,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.017859+0000","last_change":"2024-09-13T15:08:52.176830+0000","last_active":"2024-09-13T15:24:20.017859+0000","last_peered":"2024-09-13T15:24:20.017859+0000","last_clean":"2024-09-13T15:24:20.017859+0000","last_became_active":"2024-09-13T15:08:52.176303+0000","last_became_peered":"2024-09-13T15:08:52.176303+0000","last_unstale":"2024-09-13T15:24:20.017859+0000","last_undegraded":"2024-09-13T15:24:20.017859+0000","last_fullsized":"2024-09-13T15:24:20.017859+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:59.553257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1183,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.559037+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T15:24:20.559037+0000","last_peered":"2024-09-13T15:24:20.559037+0000","last_clean":"2024-09-13T15:24:20.559037+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T15:24:20.559037+0000","last_undegraded":"2024-09-13T15:24:20.559037+0000","last_fullsized":"2024-09-13T15:24:20.559037+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":118,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.932066+0000","last_change":"2024-09-13T15:08:52.200695+0000","last_active":"2024-09-13T15:24:20.932066+0000","last_peered":"2024-09-13T15:24:20.932066+0000","last_clean":"2024-09-13T15:24:20.932066+0000","last_became_active":"2024-09-13T15:08:52.183473+0000","last_became_peered":"2024-09-13T15:08:52.183473+0000","last_unstale":"2024-09-13T15:24:20.932066+0000","last_undegraded":"2024-09-13T15:24:20.932066+0000","last_fullsized":"2024-09-13T15:24:20.932066+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:42.810003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1520,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040263+0000","last_change":"2024-09-13T15:05:34.074069+0000","last_active":"2024-09-13T15:24:31.040263+0000","last_peered":"2024-09-13T15:24:31.040263+0000","last_clean":"2024-09-13T15:24:31.040263+0000","last_became_active":"2024-09-13T15:05:34.073282+0000","last_became_peered":"2024-09-13T15:05:34.073282+0000","last_unstale":"2024-09-13T15:24:31.040263+0000","last_undegraded":"2024-09-13T15:24:31.040263+0000","last_fullsized":"2024-09-13T15:24:31.040263+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:14.307582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1176,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558123+0000","last_change":"2024-09-13T15:08:52.192368+0000","last_active":"2024-09-13T15:24:20.558123+0000","last_peered":"2024-09-13T15:24:20.558123+0000","last_clean":"2024-09-13T15:24:20.558123+0000","last_became_active":"2024-09-13T15:08:52.191668+0000","last_became_peered":"2024-09-13T15:08:52.191668+0000","last_unstale":"2024-09-13T15:24:20.558123+0000","last_undegraded":"2024-09-13T15:24:20.558123+0000","last_fullsized":"2024-09-13T15:24:20.558123+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:27.410222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1760,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040500+0000","last_change":"2024-09-13T15:05:34.087636+0000","last_active":"2024-09-13T15:24:31.040500+0000","last_peered":"2024-09-13T15:24:31.040500+0000","last_clean":"2024-09-13T15:24:31.040500+0000","last_became_active":"2024-09-13T15:05:34.087255+0000","last_became_peered":"2024-09-13T15:05:34.087255+0000","last_unstale":"2024-09-13T15:24:31.040500+0000","last_undegraded":"2024-09-13T15:24:31.040500+0000","last_fullsized":"2024-09-13T15:24:31.040500+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:36.313581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1761,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.039800+0000","last_change":"2024-09-13T15:05:34.089162+0000","last_active":"2024-09-13T15:24:31.039800+0000","last_peered":"2024-09-13T15:24:31.039800+0000","last_clean":"2024-09-13T15:24:31.039800+0000","last_became_active":"2024-09-13T15:05:34.088727+0000","last_became_peered":"2024-09-13T15:05:34.088727+0000","last_unstale":"2024-09-13T15:24:31.039800+0000","last_undegraded":"2024-09-13T15:24:31.039800+0000","last_fullsized":"2024-09-13T15:24:31.039800+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:29.209082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1149,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018483+0000","last_change":"2024-09-13T15:05:34.073363+0000","last_active":"2024-09-13T15:24:20.018483+0000","last_peered":"2024-09-13T15:24:20.018483+0000","last_clean":"2024-09-13T15:24:20.018483+0000","last_became_active":"2024-09-13T15:05:34.072906+0000","last_became_peered":"2024-09-13T15:05:34.072906+0000","last_unstale":"2024-09-13T15:24:20.018483+0000","last_undegraded":"2024-09-13T15:24:20.018483+0000","last_fullsized":"2024-09-13T15:24:20.018483+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:27.966654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1221,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558516+0000","last_change":"2024-09-13T15:21:14.437266+0000","last_active":"2024-09-13T15:24:20.558516+0000","last_peered":"2024-09-13T15:24:20.558516+0000","last_clean":"2024-09-13T15:24:20.558516+0000","last_became_active":"2024-09-13T15:21:14.419722+0000","last_became_peered":"2024-09-13T15:21:14.419722+0000","last_unstale":"2024-09-13T15:24:20.558516+0000","last_undegraded":"2024-09-13T15:24:20.558516+0000","last_fullsized":"2024-09-13T15:24:20.558516+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:42:51.958824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1531,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.038543+0000","last_change":"2024-09-13T15:05:34.075192+0000","last_active":"2024-09-13T15:24:31.038543+0000","last_peered":"2024-09-13T15:24:31.038543+0000","last_clean":"2024-09-13T15:24:31.038543+0000","last_became_active":"2024-09-13T15:05:34.073315+0000","last_became_peered":"2024-09-13T15:05:34.073315+0000","last_unstale":"2024-09-13T15:24:31.038543+0000","last_undegraded":"2024-09-13T15:24:31.038543+0000","last_fullsized":"2024-09-13T15:24:31.038543+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:38.699775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":26,"num_read_kb":28,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":17582,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398304,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909344,"kb_used_data":13268,"kb_used_omap":1,"kb_used_meta":896062,"kb_avail":92835808,"statfs":{"total":95995035648,"available":95063867392,"internally_reserved":0,"allocated":13586432,"data_stored":8506440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917567832},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594539,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900760,"kb_used_data":12940,"kb_used_omap":1,"kb_used_meta":887806,"kb_avail":92844392,"statfs":{"total":95995035648,"available":95072657408,"internally_reserved":0,"allocated":13250560,"data_stored":8178625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":909113688},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790773,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903764,"kb_used_data":13512,"kb_used_omap":1,"kb_used_meta":890238,"kb_avail":92841388,"statfs":{"total":95995035648,"available":95069581312,"internally_reserved":0,"allocated":13836288,"data_stored":8764811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":911604056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980832302,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":747228,"kb_used_data":12944,"kb_used_omap":76,"kb_used_meta":734195,"kb_avail":92997924,"statfs":{"total":95995035648,"available":95229874176,"internally_reserved":0,"allocated":13254656,"data_stored":8178702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":751816512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1066,"seq":4578435137788,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":387340,"kb_used_data":13504,"kb_used_omap":121,"kb_used_meta":373702,"kb_avail":93357812,"statfs":{"total":95995035648,"available":95598399488,"internally_reserved":0,"allocated":13828096,"data_stored":8764278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124093,"internal_metadata":382671683},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023484,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":915472,"kb_used_data":13188,"kb_used_omap":71,"kb_used_meta":902200,"kb_avail":92829680,"statfs":{"total":95995035648,"available":95057592320,"internally_reserved":0,"allocated":13504512,"data_stored":8437120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":923853472},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004613,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":794184,"kb_used_data":12924,"kb_used_omap":66,"kb_used_meta":781181,"kb_avail":92950968,"statfs":{"total":95995035648,"available":95181791232,"internally_reserved":0,"allocated":13234176,"data_stored":8173839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":799929514},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195806,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":967176,"kb_used_data":12924,"kb_used_omap":56,"kb_used_meta":954183,"kb_avail":92777976,"statfs":{"total":95995035648,"available":95004647424,"internally_reserved":0,"allocated":13234176,"data_stored":8172290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":977083535},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:26:24.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:26:25.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:25 smithi099 bash[84576]: cluster 2024-09-13T15:26:23.436532+0000 mgr.y (mgr.26252) 1813 : cluster [DBG] pgmap v2475: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:25.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:25 smithi099 bash[84576]: cluster 2024-09-13T15:26:23.436532+0000 mgr.y (mgr.26252) 1813 : cluster [DBG] pgmap v2475: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:25.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:25 smithi099 bash[84576]: audit 2024-09-13T15:26:23.461373+0000 mgr.y (mgr.26252) 1814 : audit [DBG] from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:26:25.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:25 smithi099 bash[84576]: audit 2024-09-13T15:26:23.461373+0000 mgr.y (mgr.26252) 1814 : audit [DBG] from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:26:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:25 smithi099 bash[87147]: cluster 2024-09-13T15:26:23.436532+0000 mgr.y (mgr.26252) 1813 : cluster [DBG] pgmap v2475: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:25 smithi099 bash[87147]: cluster 2024-09-13T15:26:23.436532+0000 mgr.y (mgr.26252) 1813 : cluster [DBG] pgmap v2475: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:25 smithi099 bash[87147]: audit 2024-09-13T15:26:23.461373+0000 mgr.y (mgr.26252) 1814 : audit [DBG] from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:26:25.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:25 smithi099 bash[87147]: audit 2024-09-13T15:26:23.461373+0000 mgr.y (mgr.26252) 1814 : audit [DBG] from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:26:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:25 smithi177 bash[68594]: cluster 2024-09-13T15:26:23.436532+0000 mgr.y (mgr.26252) 1813 : cluster [DBG] pgmap v2475: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:25 smithi177 bash[68594]: cluster 2024-09-13T15:26:23.436532+0000 mgr.y (mgr.26252) 1813 : cluster [DBG] pgmap v2475: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:25 smithi177 bash[68594]: audit 2024-09-13T15:26:23.461373+0000 mgr.y (mgr.26252) 1814 : audit [DBG] from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:26:25.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:25 smithi177 bash[68594]: audit 2024-09-13T15:26:23.461373+0000 mgr.y (mgr.26252) 1814 : audit [DBG] from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:26:27.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:27 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:26:27] "GET /metrics HTTP/1.1" 200 34332 "" "Prometheus/2.51.0" 2024-09-13T15:26:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:27 smithi099 bash[84576]: cluster 2024-09-13T15:26:25.437394+0000 mgr.y (mgr.26252) 1815 : cluster [DBG] pgmap v2476: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:27.325 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:27 smithi099 bash[84576]: cluster 2024-09-13T15:26:25.437394+0000 mgr.y (mgr.26252) 1815 : cluster [DBG] pgmap v2476: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:27 smithi099 bash[87147]: cluster 2024-09-13T15:26:25.437394+0000 mgr.y (mgr.26252) 1815 : cluster [DBG] pgmap v2476: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:27.325 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:27 smithi099 bash[87147]: cluster 2024-09-13T15:26:25.437394+0000 mgr.y (mgr.26252) 1815 : cluster [DBG] pgmap v2476: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:27.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:27 smithi177 bash[68594]: cluster 2024-09-13T15:26:25.437394+0000 mgr.y (mgr.26252) 1815 : cluster [DBG] pgmap v2476: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:27.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:27 smithi177 bash[68594]: cluster 2024-09-13T15:26:25.437394+0000 mgr.y (mgr.26252) 1815 : cluster [DBG] pgmap v2476: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:29.187 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:26:29.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:29 smithi099 bash[84576]: cluster 2024-09-13T15:26:27.438393+0000 mgr.y (mgr.26252) 1816 : cluster [DBG] pgmap v2477: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:29.575 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:29 smithi099 bash[84576]: cluster 2024-09-13T15:26:27.438393+0000 mgr.y (mgr.26252) 1816 : cluster [DBG] pgmap v2477: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:29.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:29 smithi099 bash[87147]: cluster 2024-09-13T15:26:27.438393+0000 mgr.y (mgr.26252) 1816 : cluster [DBG] pgmap v2477: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:29.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:29 smithi099 bash[87147]: cluster 2024-09-13T15:26:27.438393+0000 mgr.y (mgr.26252) 1816 : cluster [DBG] pgmap v2477: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:29.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:29 smithi177 bash[68594]: cluster 2024-09-13T15:26:27.438393+0000 mgr.y (mgr.26252) 1816 : cluster [DBG] pgmap v2477: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:29.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:29 smithi177 bash[68594]: cluster 2024-09-13T15:26:27.438393+0000 mgr.y (mgr.26252) 1816 : cluster [DBG] pgmap v2477: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:30.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:26:30 smithi177 bash[61439]: ts=2024-09-13T15:26:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:26:31.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:31 smithi099 bash[84576]: cluster 2024-09-13T15:26:29.439285+0000 mgr.y (mgr.26252) 1817 : cluster [DBG] pgmap v2478: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:31.574 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:31 smithi099 bash[84576]: cluster 2024-09-13T15:26:29.439285+0000 mgr.y (mgr.26252) 1817 : cluster [DBG] pgmap v2478: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:31.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:31 smithi099 bash[87147]: cluster 2024-09-13T15:26:29.439285+0000 mgr.y (mgr.26252) 1817 : cluster [DBG] pgmap v2478: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:31.575 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:31 smithi099 bash[87147]: cluster 2024-09-13T15:26:29.439285+0000 mgr.y (mgr.26252) 1817 : cluster [DBG] pgmap v2478: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:31.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:31 smithi177 bash[68594]: cluster 2024-09-13T15:26:29.439285+0000 mgr.y (mgr.26252) 1817 : cluster [DBG] pgmap v2478: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:31.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:31 smithi177 bash[68594]: cluster 2024-09-13T15:26:29.439285+0000 mgr.y (mgr.26252) 1817 : cluster [DBG] pgmap v2478: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:33.484 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-09-13T15:26:33.485 INFO:teuthology.orchestra.run.smithi099.stderr:dumped all 2024-09-13T15:26:33.496 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:33 smithi099 bash[84576]: cluster 2024-09-13T15:26:31.440181+0000 mgr.y (mgr.26252) 1818 : cluster [DBG] pgmap v2479: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:33.496 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:33 smithi099 bash[84576]: cluster 2024-09-13T15:26:31.440181+0000 mgr.y (mgr.26252) 1818 : cluster [DBG] pgmap v2479: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:33.496 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:33 smithi099 bash[87147]: cluster 2024-09-13T15:26:31.440181+0000 mgr.y (mgr.26252) 1818 : cluster [DBG] pgmap v2479: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:33.496 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:33 smithi099 bash[87147]: cluster 2024-09-13T15:26:31.440181+0000 mgr.y (mgr.26252) 1818 : cluster [DBG] pgmap v2479: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:33.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:33 smithi177 bash[68594]: cluster 2024-09-13T15:26:31.440181+0000 mgr.y (mgr.26252) 1818 : cluster [DBG] pgmap v2479: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:33.612 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:33 smithi177 bash[68594]: cluster 2024-09-13T15:26:31.440181+0000 mgr.y (mgr.26252) 1818 : cluster [DBG] pgmap v2479: 83 pgs: 83 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-09-13T15:26:34.464 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:34 smithi099 bash[84576]: audit 2024-09-13T15:26:33.335134+0000 mon.a (mon.0) 2980 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:34.464 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:34 smithi099 bash[84576]: audit 2024-09-13T15:26:33.335134+0000 mon.a (mon.0) 2980 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:34.465 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:34 smithi099 bash[87147]: audit 2024-09-13T15:26:33.335134+0000 mon.a (mon.0) 2980 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:34.465 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:34 smithi099 bash[87147]: audit 2024-09-13T15:26:33.335134+0000 mon.a (mon.0) 2980 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:34.465 INFO:teuthology.orchestra.run.smithi099.stdout:{"pg_ready":true,"pg_map":{"version":2480,"stamp":"2024-09-13T15:26:33.440520+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":26,"num_read_kb":28,"num_write":63,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":280346,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1164,"ondisk_log_size":1164,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6525268,"kb_used_data":105204,"kb_used_omap":397,"kb_used_meta":6419570,"kb_avail":743435948,"statfs":{"total":767960285184,"available":761278410752,"internally_reserved":0,"allocated":107728896,"data_stored":67176105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":406940,"internal_metadata":6573640292},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005333"},"pg_stats":[{"pgid":"3.1f","version":"65'5","reported_seq":52,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040019+0000","last_change":"2024-09-13T15:21:41.414612+0000","last_active":"2024-09-13T15:24:31.040019+0000","last_peered":"2024-09-13T15:24:31.040019+0000","last_clean":"2024-09-13T15:24:31.040019+0000","last_became_active":"2024-09-13T15:21:41.406892+0000","last_became_peered":"2024-09-13T15:21:41.406892+0000","last_unstale":"2024-09-13T15:24:31.040019+0000","last_undegraded":"2024-09-13T15:24:31.040019+0000","last_fullsized":"2024-09-13T15:24:31.040019+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:42.810175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2850,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":496,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802240+0000","last_change":"2024-09-13T15:08:52.180022+0000","last_active":"2024-09-13T15:24:31.802240+0000","last_peered":"2024-09-13T15:24:31.802240+0000","last_clean":"2024-09-13T15:24:31.802240+0000","last_became_active":"2024-09-13T15:08:52.179546+0000","last_became_peered":"2024-09-13T15:08:52.179546+0000","last_unstale":"2024-09-13T15:24:31.802240+0000","last_undegraded":"2024-09-13T15:24:31.802240+0000","last_fullsized":"2024-09-13T15:24:31.802240+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:15:47.462895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"65'2","reported_seq":1505,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.038659+0000","last_change":"2024-09-13T15:21:14.418247+0000","last_active":"2024-09-13T15:24:31.038659+0000","last_peered":"2024-09-13T15:24:31.038659+0000","last_clean":"2024-09-13T15:24:31.038659+0000","last_became_active":"2024-09-13T15:21:14.417022+0000","last_became_peered":"2024-09-13T15:21:14.417022+0000","last_unstale":"2024-09-13T15:24:31.038659+0000","last_undegraded":"2024-09-13T15:24:31.038659+0000","last_fullsized":"2024-09-13T15:24:31.038659+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:53.441136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.017895+0000","last_change":"2024-09-13T15:21:14.426470+0000","last_active":"2024-09-13T15:24:20.017895+0000","last_peered":"2024-09-13T15:24:20.017895+0000","last_clean":"2024-09-13T15:24:20.017895+0000","last_became_active":"2024-09-13T15:21:14.425617+0000","last_became_peered":"2024-09-13T15:21:14.425617+0000","last_unstale":"2024-09-13T15:24:20.017895+0000","last_undegraded":"2024-09-13T15:24:20.017895+0000","last_fullsized":"2024-09-13T15:24:20.017895+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:11.377197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"64'2","reported_seq":1192,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018567+0000","last_change":"2024-09-13T15:08:52.205951+0000","last_active":"2024-09-13T15:24:20.018567+0000","last_peered":"2024-09-13T15:24:20.018567+0000","last_clean":"2024-09-13T15:24:20.018567+0000","last_became_active":"2024-09-13T15:08:52.183627+0000","last_became_peered":"2024-09-13T15:08:52.183627+0000","last_unstale":"2024-09-13T15:24:20.018567+0000","last_undegraded":"2024-09-13T15:24:20.018567+0000","last_fullsized":"2024-09-13T15:24:20.018567+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:14:14.282531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":478,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802966+0000","last_change":"2024-09-13T15:05:34.086551+0000","last_active":"2024-09-13T15:24:31.802966+0000","last_peered":"2024-09-13T15:24:31.802966+0000","last_clean":"2024-09-13T15:24:31.802966+0000","last_became_active":"2024-09-13T15:05:34.086077+0000","last_became_peered":"2024-09-13T15:05:34.086077+0000","last_unstale":"2024-09-13T15:24:31.802966+0000","last_undegraded":"2024-09-13T15:24:31.802966+0000","last_fullsized":"2024-09-13T15:24:31.802966+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:33:47.917777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"64'2","reported_seq":1209,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018621+0000","last_change":"2024-09-13T14:52:43.608079+0000","last_active":"2024-09-13T15:24:20.018621+0000","last_peered":"2024-09-13T15:24:20.018621+0000","last_clean":"2024-09-13T15:24:20.018621+0000","last_became_active":"2024-09-13T14:52:43.592770+0000","last_became_peered":"2024-09-13T14:52:43.592770+0000","last_unstale":"2024-09-13T15:24:20.018621+0000","last_undegraded":"2024-09-13T15:24:20.018621+0000","last_fullsized":"2024-09-13T15:24:20.018621+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:34:41.702080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":492,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803572+0000","last_change":"2024-09-13T15:05:34.075607+0000","last_active":"2024-09-13T15:24:31.803572+0000","last_peered":"2024-09-13T15:24:31.803572+0000","last_clean":"2024-09-13T15:24:31.803572+0000","last_became_active":"2024-09-13T15:05:34.075079+0000","last_became_peered":"2024-09-13T15:05:34.075079+0000","last_unstale":"2024-09-13T15:24:31.803572+0000","last_undegraded":"2024-09-13T15:24:31.803572+0000","last_fullsized":"2024-09-13T15:24:31.803572+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:47:05.384206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"65'2","reported_seq":45,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040610+0000","last_change":"2024-09-13T15:21:41.415390+0000","last_active":"2024-09-13T15:24:31.040610+0000","last_peered":"2024-09-13T15:24:31.040610+0000","last_clean":"2024-09-13T15:24:31.040610+0000","last_became_active":"2024-09-13T15:21:41.414869+0000","last_became_peered":"2024-09-13T15:21:41.414869+0000","last_unstale":"2024-09-13T15:24:31.040610+0000","last_undegraded":"2024-09-13T15:24:31.040610+0000","last_fullsized":"2024-09-13T15:24:31.040610+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:42:01.612211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":50,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.801933+0000","last_change":"2024-09-13T15:21:14.428062+0000","last_active":"2024-09-13T15:24:31.801933+0000","last_peered":"2024-09-13T15:24:31.801933+0000","last_clean":"2024-09-13T15:24:31.801933+0000","last_became_active":"2024-09-13T15:21:14.424173+0000","last_became_peered":"2024-09-13T15:21:14.424173+0000","last_unstale":"2024-09-13T15:24:31.801933+0000","last_undegraded":"2024-09-13T15:24:31.801933+0000","last_fullsized":"2024-09-13T15:24:31.801933+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:25.248892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"65'2","reported_seq":495,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802806+0000","last_change":"2024-09-13T15:21:14.421106+0000","last_active":"2024-09-13T15:24:31.802806+0000","last_peered":"2024-09-13T15:24:31.802806+0000","last_clean":"2024-09-13T15:24:31.802806+0000","last_became_active":"2024-09-13T15:21:14.420054+0000","last_became_peered":"2024-09-13T15:21:14.420054+0000","last_unstale":"2024-09-13T15:24:31.802806+0000","last_undegraded":"2024-09-13T15:24:31.802806+0000","last_fullsized":"2024-09-13T15:24:31.802806+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:23.045562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":41,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.801693+0000","last_change":"2024-09-13T15:21:55.779597+0000","last_active":"2024-09-13T15:24:31.801693+0000","last_peered":"2024-09-13T15:24:31.801693+0000","last_clean":"2024-09-13T15:24:31.801693+0000","last_became_active":"2024-09-13T15:21:55.778864+0000","last_became_peered":"2024-09-13T15:21:55.778864+0000","last_unstale":"2024-09-13T15:24:31.801693+0000","last_undegraded":"2024-09-13T15:24:31.801693+0000","last_fullsized":"2024-09-13T15:24:31.801693+0000","mapping_epoch":1164,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:01.376294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":382,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.326398+0000","last_change":"2024-09-13T15:05:34.071133+0000","last_active":"2024-09-13T15:24:31.326398+0000","last_peered":"2024-09-13T15:24:31.326398+0000","last_clean":"2024-09-13T15:24:31.326398+0000","last_became_active":"2024-09-13T15:05:34.070519+0000","last_became_peered":"2024-09-13T15:05:34.070519+0000","last_unstale":"2024-09-13T15:24:31.326398+0000","last_undegraded":"2024-09-13T15:24:31.326398+0000","last_fullsized":"2024-09-13T15:24:31.326398+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:56:02.617403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1768,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040332+0000","last_change":"2024-09-13T15:18:41.619980+0000","last_active":"2024-09-13T15:24:31.040332+0000","last_peered":"2024-09-13T15:24:31.040332+0000","last_clean":"2024-09-13T15:24:31.040332+0000","last_became_active":"2024-09-13T15:18:41.619445+0000","last_became_peered":"2024-09-13T15:18:41.619445+0000","last_unstale":"2024-09-13T15:24:31.040332+0000","last_undegraded":"2024-09-13T15:24:31.040332+0000","last_fullsized":"2024-09-13T15:24:31.040332+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:29:48.792190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1566,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.037671+0000","last_change":"2024-09-13T15:21:41.488339+0000","last_active":"2024-09-13T15:24:31.037671+0000","last_peered":"2024-09-13T15:24:31.037671+0000","last_clean":"2024-09-13T15:24:31.037671+0000","last_became_active":"2024-09-13T15:21:41.487177+0000","last_became_peered":"2024-09-13T15:21:41.487177+0000","last_unstale":"2024-09-13T15:24:31.037671+0000","last_undegraded":"2024-09-13T15:24:31.037671+0000","last_fullsized":"2024-09-13T15:24:31.037671+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:45.259145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":118,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.933620+0000","last_change":"2024-09-13T15:18:41.606623+0000","last_active":"2024-09-13T15:24:20.933620+0000","last_peered":"2024-09-13T15:24:20.933620+0000","last_clean":"2024-09-13T15:24:20.933620+0000","last_became_active":"2024-09-13T15:18:41.606144+0000","last_became_peered":"2024-09-13T15:18:41.606144+0000","last_unstale":"2024-09-13T15:24:20.933620+0000","last_undegraded":"2024-09-13T15:24:20.933620+0000","last_fullsized":"2024-09-13T15:24:20.933620+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:54:12.837936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1533,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.036842+0000","last_change":"2024-09-13T15:08:52.170737+0000","last_active":"2024-09-13T15:24:31.036842+0000","last_peered":"2024-09-13T15:24:31.036842+0000","last_clean":"2024-09-13T15:24:31.036842+0000","last_became_active":"2024-09-13T15:08:52.170217+0000","last_became_peered":"2024-09-13T15:08:52.170217+0000","last_unstale":"2024-09-13T15:24:31.036842+0000","last_undegraded":"2024-09-13T15:24:31.036842+0000","last_fullsized":"2024-09-13T15:24:31.036842+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:17.688936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1760,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.041059+0000","last_change":"2024-09-13T15:05:34.087376+0000","last_active":"2024-09-13T15:24:31.041059+0000","last_peered":"2024-09-13T15:24:31.041059+0000","last_clean":"2024-09-13T15:24:31.041059+0000","last_became_active":"2024-09-13T15:05:34.087046+0000","last_became_peered":"2024-09-13T15:05:34.087046+0000","last_unstale":"2024-09-13T15:24:31.041059+0000","last_undegraded":"2024-09-13T15:24:31.041059+0000","last_fullsized":"2024-09-13T15:24:31.041059+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:47.801431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1472,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.036996+0000","last_change":"2024-09-13T15:08:52.174279+0000","last_active":"2024-09-13T15:24:31.036996+0000","last_peered":"2024-09-13T15:24:31.036996+0000","last_clean":"2024-09-13T15:24:31.036996+0000","last_became_active":"2024-09-13T15:08:52.173877+0000","last_became_peered":"2024-09-13T15:08:52.173877+0000","last_unstale":"2024-09-13T15:24:31.036996+0000","last_undegraded":"2024-09-13T15:24:31.036996+0000","last_fullsized":"2024-09-13T15:24:31.036996+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:16.648765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1153,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018275+0000","last_change":"2024-09-13T14:52:43.594174+0000","last_active":"2024-09-13T15:24:20.018275+0000","last_peered":"2024-09-13T15:24:20.018275+0000","last_clean":"2024-09-13T15:24:20.018275+0000","last_became_active":"2024-09-13T14:52:43.593210+0000","last_became_peered":"2024-09-13T14:52:43.593210+0000","last_unstale":"2024-09-13T15:24:20.018275+0000","last_undegraded":"2024-09-13T15:24:20.018275+0000","last_fullsized":"2024-09-13T15:24:20.018275+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:02:00.154601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"64'3","reported_seq":1207,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558559+0000","last_change":"2024-09-13T15:05:34.089020+0000","last_active":"2024-09-13T15:24:20.558559+0000","last_peered":"2024-09-13T15:24:20.558559+0000","last_clean":"2024-09-13T15:24:20.558559+0000","last_became_active":"2024-09-13T15:05:34.088440+0000","last_became_peered":"2024-09-13T15:05:34.088440+0000","last_unstale":"2024-09-13T15:24:20.558559+0000","last_undegraded":"2024-09-13T15:24:20.558559+0000","last_fullsized":"2024-09-13T15:24:20.558559+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:24.826746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":162,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.039293+0000","last_change":"2024-09-13T15:21:14.416181+0000","last_active":"2024-09-13T15:24:31.039293+0000","last_peered":"2024-09-13T15:24:31.039293+0000","last_clean":"2024-09-13T15:24:31.039293+0000","last_became_active":"2024-09-13T15:21:14.415217+0000","last_became_peered":"2024-09-13T15:21:14.415217+0000","last_unstale":"2024-09-13T15:24:31.039293+0000","last_undegraded":"2024-09-13T15:24:31.039293+0000","last_fullsized":"2024-09-13T15:24:31.039293+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:06:59.197995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"428'23","reported_seq":512,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803250+0000","last_change":"2024-09-13T15:08:52.203370+0000","last_active":"2024-09-13T15:24:31.803250+0000","last_peered":"2024-09-13T15:24:31.803250+0000","last_clean":"2024-09-13T15:24:31.803250+0000","last_became_active":"2024-09-13T15:08:52.179887+0000","last_became_peered":"2024-09-13T15:08:52.179887+0000","last_unstale":"2024-09-13T15:24:31.803250+0000","last_undegraded":"2024-09-13T15:24:31.803250+0000","last_fullsized":"2024-09-13T15:24:31.803250+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:57.293764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":503,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802277+0000","last_change":"2024-09-13T15:20:55.395006+0000","last_active":"2024-09-13T15:24:31.802277+0000","last_peered":"2024-09-13T15:24:31.802277+0000","last_clean":"2024-09-13T15:24:31.802277+0000","last_became_active":"2024-09-13T15:20:55.393901+0000","last_became_peered":"2024-09-13T15:20:55.393901+0000","last_unstale":"2024-09-13T15:24:31.802277+0000","last_undegraded":"2024-09-13T15:24:31.802277+0000","last_fullsized":"2024-09-13T15:24:31.802277+0000","mapping_epoch":1157,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:04.690653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"346'100","reported_seq":99,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.038014+0000","last_change":"2024-09-13T15:20:09.715321+0000","last_active":"2024-09-13T15:24:31.038014+0000","last_peered":"2024-09-13T15:24:31.038014+0000","last_clean":"2024-09-13T15:24:31.038014+0000","last_became_active":"2024-09-13T15:19:58.641197+0000","last_became_peered":"2024-09-13T15:19:58.641197+0000","last_unstale":"2024-09-13T15:24:31.038014+0000","last_undegraded":"2024-09-13T15:24:31.038014+0000","last_fullsized":"2024-09-13T15:24:31.038014+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:09.715239+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:09.715239+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:39.018561+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":484,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802811+0000","last_change":"2024-09-13T15:08:52.188050+0000","last_active":"2024-09-13T15:24:31.802811+0000","last_peered":"2024-09-13T15:24:31.802811+0000","last_clean":"2024-09-13T15:24:31.802811+0000","last_became_active":"2024-09-13T15:08:52.187522+0000","last_became_peered":"2024-09-13T15:08:52.187522+0000","last_unstale":"2024-09-13T15:24:31.802811+0000","last_undegraded":"2024-09-13T15:24:31.802811+0000","last_fullsized":"2024-09-13T15:24:31.802811+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:09:29.314964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":128,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.933445+0000","last_change":"2024-09-13T15:21:14.415297+0000","last_active":"2024-09-13T15:24:20.933445+0000","last_peered":"2024-09-13T15:24:20.933445+0000","last_clean":"2024-09-13T15:24:20.933445+0000","last_became_active":"2024-09-13T15:21:14.414292+0000","last_became_peered":"2024-09-13T15:21:14.414292+0000","last_unstale":"2024-09-13T15:24:20.933445+0000","last_undegraded":"2024-09-13T15:24:20.933445+0000","last_fullsized":"2024-09-13T15:24:20.933445+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:19:27.468824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033738000000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"65'1","reported_seq":1812,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040452+0000","last_change":"2024-09-13T14:52:43.606993+0000","last_active":"2024-09-13T15:24:31.040452+0000","last_peered":"2024-09-13T15:24:31.040452+0000","last_clean":"2024-09-13T15:24:31.040452+0000","last_became_active":"2024-09-13T14:52:43.595207+0000","last_became_peered":"2024-09-13T14:52:43.595207+0000","last_unstale":"2024-09-13T15:24:31.040452+0000","last_undegraded":"2024-09-13T15:24:31.040452+0000","last_fullsized":"2024-09-13T15:24:31.040452+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:49.919286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"346'100","reported_seq":101,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803167+0000","last_change":"2024-09-13T15:21:14.420881+0000","last_active":"2024-09-13T15:24:31.803167+0000","last_peered":"2024-09-13T15:24:31.803167+0000","last_clean":"2024-09-13T15:24:31.803167+0000","last_became_active":"2024-09-13T15:21:14.419843+0000","last_became_peered":"2024-09-13T15:21:14.419843+0000","last_unstale":"2024-09-13T15:24:31.803167+0000","last_undegraded":"2024-09-13T15:24:31.803167+0000","last_fullsized":"2024-09-13T15:24:31.803167+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:09.523750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:09.523750+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:33.221965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1474,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.036988+0000","last_change":"2024-09-13T15:21:41.415212+0000","last_active":"2024-09-13T15:24:31.036988+0000","last_peered":"2024-09-13T15:24:31.036988+0000","last_clean":"2024-09-13T15:24:31.036988+0000","last_became_active":"2024-09-13T15:21:41.414028+0000","last_became_peered":"2024-09-13T15:21:41.414028+0000","last_unstale":"2024-09-13T15:24:31.036988+0000","last_undegraded":"2024-09-13T15:24:31.036988+0000","last_fullsized":"2024-09-13T15:24:31.036988+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:46.313434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":387,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327766+0000","last_change":"2024-09-13T15:08:52.186362+0000","last_active":"2024-09-13T15:24:31.327766+0000","last_peered":"2024-09-13T15:24:31.327766+0000","last_clean":"2024-09-13T15:24:31.327766+0000","last_became_active":"2024-09-13T15:08:52.185731+0000","last_became_peered":"2024-09-13T15:08:52.185731+0000","last_unstale":"2024-09-13T15:24:31.327766+0000","last_undegraded":"2024-09-13T15:24:31.327766+0000","last_fullsized":"2024-09-13T15:24:31.327766+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:26.408919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"64'2","reported_seq":1462,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040561+0000","last_change":"2024-09-13T15:05:34.085917+0000","last_active":"2024-09-13T15:24:31.040561+0000","last_peered":"2024-09-13T15:24:31.040561+0000","last_clean":"2024-09-13T15:24:31.040561+0000","last_became_active":"2024-09-13T15:05:34.085156+0000","last_became_peered":"2024-09-13T15:05:34.085156+0000","last_unstale":"2024-09-13T15:24:31.040561+0000","last_undegraded":"2024-09-13T15:24:31.040561+0000","last_fullsized":"2024-09-13T15:24:31.040561+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:33.482732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"346'100","reported_seq":1227,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.933434+0000","last_change":"2024-09-13T15:20:10.562576+0000","last_active":"2024-09-13T15:24:20.933434+0000","last_peered":"2024-09-13T15:24:20.933434+0000","last_clean":"2024-09-13T15:24:20.933434+0000","last_became_active":"2024-09-13T15:19:58.639558+0000","last_became_peered":"2024-09-13T15:19:58.639558+0000","last_unstale":"2024-09-13T15:24:20.933434+0000","last_undegraded":"2024-09-13T15:24:20.933434+0000","last_fullsized":"2024-09-13T15:24:20.933434+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:10.562423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:10.562423+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:54.624883+0000","scrub_duration":0.0039412570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1742,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040089+0000","last_change":"2024-09-13T14:36:09.374948+0000","last_active":"2024-09-13T15:24:31.040089+0000","last_peered":"2024-09-13T15:24:31.040089+0000","last_clean":"2024-09-13T15:24:31.040089+0000","last_became_active":"2024-09-13T14:36:09.374591+0000","last_became_peered":"2024-09-13T14:36:09.374591+0000","last_unstale":"2024-09-13T15:24:31.040089+0000","last_undegraded":"2024-09-13T15:24:31.040089+0000","last_fullsized":"2024-09-13T15:24:31.040089+0000","mapping_epoch":575,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:12.701942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1185,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558834+0000","last_change":"2024-09-13T14:47:46.329504+0000","last_active":"2024-09-13T15:24:20.558834+0000","last_peered":"2024-09-13T15:24:20.558834+0000","last_clean":"2024-09-13T15:24:20.558834+0000","last_became_active":"2024-09-13T14:47:46.329149+0000","last_became_peered":"2024-09-13T14:47:46.329149+0000","last_unstale":"2024-09-13T15:24:20.558834+0000","last_undegraded":"2024-09-13T15:24:20.558834+0000","last_fullsized":"2024-09-13T15:24:20.558834+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:21.890588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069009099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":388,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327276+0000","last_change":"2024-09-13T15:21:14.420902+0000","last_active":"2024-09-13T15:24:31.327276+0000","last_peered":"2024-09-13T15:24:31.327276+0000","last_clean":"2024-09-13T15:24:31.327276+0000","last_became_active":"2024-09-13T15:21:14.419966+0000","last_became_peered":"2024-09-13T15:21:14.419966+0000","last_unstale":"2024-09-13T15:24:31.327276+0000","last_undegraded":"2024-09-13T15:24:31.327276+0000","last_fullsized":"2024-09-13T15:24:31.327276+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:07:52.638527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"346'100","reported_seq":1227,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.933448+0000","last_change":"2024-09-13T15:20:14.516263+0000","last_active":"2024-09-13T15:24:20.933448+0000","last_peered":"2024-09-13T15:24:20.933448+0000","last_clean":"2024-09-13T15:24:20.933448+0000","last_became_active":"2024-09-13T15:19:58.638300+0000","last_became_peered":"2024-09-13T15:19:58.638300+0000","last_unstale":"2024-09-13T15:24:20.933448+0000","last_undegraded":"2024-09-13T15:24:20.933448+0000","last_fullsized":"2024-09-13T15:24:20.933448+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:14.516167+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:14.516167+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:46:37.405030+0000","scrub_duration":0.0023068670000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":374,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327122+0000","last_change":"2024-09-13T14:52:43.592296+0000","last_active":"2024-09-13T15:24:31.327122+0000","last_peered":"2024-09-13T15:24:31.327122+0000","last_clean":"2024-09-13T15:24:31.327122+0000","last_became_active":"2024-09-13T14:52:43.591933+0000","last_became_peered":"2024-09-13T14:52:43.591933+0000","last_unstale":"2024-09-13T15:24:31.327122+0000","last_undegraded":"2024-09-13T15:24:31.327122+0000","last_fullsized":"2024-09-13T15:24:31.327122+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:42:11.866459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":374,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327442+0000","last_change":"2024-09-13T14:52:43.583656+0000","last_active":"2024-09-13T15:24:31.327442+0000","last_peered":"2024-09-13T15:24:31.327442+0000","last_clean":"2024-09-13T15:24:31.327442+0000","last_became_active":"2024-09-13T14:52:43.582012+0000","last_became_peered":"2024-09-13T14:52:43.582012+0000","last_unstale":"2024-09-13T15:24:31.327442+0000","last_undegraded":"2024-09-13T15:24:31.327442+0000","last_fullsized":"2024-09-13T15:24:31.327442+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:18:34.685411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096378200000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1187,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018683+0000","last_change":"2024-09-13T15:21:14.425885+0000","last_active":"2024-09-13T15:24:20.018683+0000","last_peered":"2024-09-13T15:24:20.018683+0000","last_clean":"2024-09-13T15:24:20.018683+0000","last_became_active":"2024-09-13T15:21:14.424951+0000","last_became_peered":"2024-09-13T15:21:14.424951+0000","last_unstale":"2024-09-13T15:24:20.018683+0000","last_undegraded":"2024-09-13T15:24:20.018683+0000","last_fullsized":"2024-09-13T15:24:20.018683+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:05.236686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"346'100","reported_seq":62,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.017913+0000","last_change":"2024-09-13T15:21:14.425980+0000","last_active":"2024-09-13T15:24:20.017913+0000","last_peered":"2024-09-13T15:24:20.017913+0000","last_clean":"2024-09-13T15:24:20.017913+0000","last_became_active":"2024-09-13T15:21:14.424979+0000","last_became_peered":"2024-09-13T15:21:14.424979+0000","last_unstale":"2024-09-13T15:24:20.017913+0000","last_undegraded":"2024-09-13T15:24:20.017913+0000","last_fullsized":"2024-09-13T15:24:20.017913+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:06.725832+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:06.725832+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:03:39.021814+0000","scrub_duration":0.0036607340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":41,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018133+0000","last_change":"2024-09-13T15:21:14.430198+0000","last_active":"2024-09-13T15:24:20.018133+0000","last_peered":"2024-09-13T15:24:20.018133+0000","last_clean":"2024-09-13T15:24:20.018133+0000","last_became_active":"2024-09-13T15:21:14.429446+0000","last_became_peered":"2024-09-13T15:21:14.429446+0000","last_unstale":"2024-09-13T15:24:20.018133+0000","last_undegraded":"2024-09-13T15:24:20.018133+0000","last_fullsized":"2024-09-13T15:24:20.018133+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:00:48.415699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":484,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803653+0000","last_change":"2024-09-13T15:08:52.188254+0000","last_active":"2024-09-13T15:24:31.803653+0000","last_peered":"2024-09-13T15:24:31.803653+0000","last_clean":"2024-09-13T15:24:31.803653+0000","last_became_active":"2024-09-13T15:08:52.187535+0000","last_became_peered":"2024-09-13T15:08:52.187535+0000","last_unstale":"2024-09-13T15:24:31.803653+0000","last_undegraded":"2024-09-13T15:24:31.803653+0000","last_fullsized":"2024-09-13T15:24:31.803653+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:32.024558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1206,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018655+0000","last_change":"2024-09-13T15:08:52.209457+0000","last_active":"2024-09-13T15:24:20.018655+0000","last_peered":"2024-09-13T15:24:20.018655+0000","last_clean":"2024-09-13T15:24:20.018655+0000","last_became_active":"2024-09-13T15:08:52.171595+0000","last_became_peered":"2024-09-13T15:08:52.171595+0000","last_unstale":"2024-09-13T15:24:20.018655+0000","last_undegraded":"2024-09-13T15:24:20.018655+0000","last_fullsized":"2024-09-13T15:24:20.018655+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:05:42.960465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010281509999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"346'100","reported_seq":1315,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.557919+0000","last_change":"2024-09-13T15:19:58.639631+0000","last_active":"2024-09-13T15:24:20.557919+0000","last_peered":"2024-09-13T15:24:20.557919+0000","last_clean":"2024-09-13T15:24:20.557919+0000","last_became_active":"2024-09-13T15:19:58.639023+0000","last_became_peered":"2024-09-13T15:19:58.639023+0000","last_unstale":"2024-09-13T15:24:20.557919+0000","last_undegraded":"2024-09-13T15:24:20.557919+0000","last_fullsized":"2024-09-13T15:24:20.557919+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:18:17.216949+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:18:17.216949+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:10.007605+0000","scrub_duration":0.011279501000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":373,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327709+0000","last_change":"2024-09-13T14:52:43.587894+0000","last_active":"2024-09-13T15:24:31.327709+0000","last_peered":"2024-09-13T15:24:31.327709+0000","last_clean":"2024-09-13T15:24:31.327709+0000","last_became_active":"2024-09-13T14:52:43.587453+0000","last_became_peered":"2024-09-13T14:52:43.587453+0000","last_unstale":"2024-09-13T15:24:31.327709+0000","last_undegraded":"2024-09-13T15:24:31.327709+0000","last_fullsized":"2024-09-13T15:24:31.327709+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:46.911208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":402,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.326932+0000","last_change":"2024-09-13T15:21:14.414672+0000","last_active":"2024-09-13T15:24:31.326932+0000","last_peered":"2024-09-13T15:24:31.326932+0000","last_clean":"2024-09-13T15:24:31.326932+0000","last_became_active":"2024-09-13T15:21:14.413645+0000","last_became_peered":"2024-09-13T15:21:14.413645+0000","last_unstale":"2024-09-13T15:24:31.326932+0000","last_undegraded":"2024-09-13T15:24:31.326932+0000","last_fullsized":"2024-09-13T15:24:31.326932+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:18.095504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":50,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.801790+0000","last_change":"2024-09-13T15:21:14.412552+0000","last_active":"2024-09-13T15:24:31.801790+0000","last_peered":"2024-09-13T15:24:31.801790+0000","last_clean":"2024-09-13T15:24:31.801790+0000","last_became_active":"2024-09-13T15:21:14.411158+0000","last_became_peered":"2024-09-13T15:21:14.411158+0000","last_unstale":"2024-09-13T15:24:31.801790+0000","last_undegraded":"2024-09-13T15:24:31.801790+0000","last_fullsized":"2024-09-13T15:24:31.801790+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:08.123906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010279060000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"346'100","reported_seq":94,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.326930+0000","last_change":"2024-09-13T15:20:10.181623+0000","last_active":"2024-09-13T15:24:31.326930+0000","last_peered":"2024-09-13T15:24:31.326930+0000","last_clean":"2024-09-13T15:24:31.326930+0000","last_became_active":"2024-09-13T15:19:58.637274+0000","last_became_peered":"2024-09-13T15:19:58.637274+0000","last_unstale":"2024-09-13T15:24:31.326930+0000","last_undegraded":"2024-09-13T15:24:31.326930+0000","last_fullsized":"2024-09-13T15:24:31.326930+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":3,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:10.181561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:10.181561+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:33.160752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":46,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040231+0000","last_change":"2024-09-13T15:21:41.488014+0000","last_active":"2024-09-13T15:24:31.040231+0000","last_peered":"2024-09-13T15:24:31.040231+0000","last_clean":"2024-09-13T15:24:31.040231+0000","last_became_active":"2024-09-13T15:21:41.487059+0000","last_became_peered":"2024-09-13T15:21:41.487059+0000","last_unstale":"2024-09-13T15:24:31.040231+0000","last_undegraded":"2024-09-13T15:24:31.040231+0000","last_fullsized":"2024-09-13T15:24:31.040231+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:20:34.460599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1789,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.039154+0000","last_change":"2024-09-13T15:05:34.087817+0000","last_active":"2024-09-13T15:24:31.039154+0000","last_peered":"2024-09-13T15:24:31.039154+0000","last_clean":"2024-09-13T15:24:31.039154+0000","last_became_active":"2024-09-13T15:05:34.087450+0000","last_became_peered":"2024-09-13T15:05:34.087450+0000","last_unstale":"2024-09-13T15:24:31.039154+0000","last_undegraded":"2024-09-13T15:24:31.039154+0000","last_fullsized":"2024-09-13T15:24:31.039154+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:57:47.661935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1185,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.557975+0000","last_change":"2024-09-13T14:52:43.588879+0000","last_active":"2024-09-13T15:24:20.557975+0000","last_peered":"2024-09-13T15:24:20.557975+0000","last_clean":"2024-09-13T15:24:20.557975+0000","last_became_active":"2024-09-13T14:52:43.587895+0000","last_became_peered":"2024-09-13T14:52:43.587895+0000","last_unstale":"2024-09-13T15:24:20.557975+0000","last_undegraded":"2024-09-13T15:24:20.557975+0000","last_fullsized":"2024-09-13T15:24:20.557975+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:21:35.022336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064098099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":108,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.933461+0000","last_change":"2024-09-13T15:08:52.188028+0000","last_active":"2024-09-13T15:24:20.933461+0000","last_peered":"2024-09-13T15:24:20.933461+0000","last_clean":"2024-09-13T15:24:20.933461+0000","last_became_active":"2024-09-13T15:08:52.187424+0000","last_became_peered":"2024-09-13T15:08:52.187424+0000","last_unstale":"2024-09-13T15:24:20.933461+0000","last_undegraded":"2024-09-13T15:24:20.933461+0000","last_fullsized":"2024-09-13T15:24:20.933461+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:31.790876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1476,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.037280+0000","last_change":"2024-09-13T15:08:52.170510+0000","last_active":"2024-09-13T15:24:31.037280+0000","last_peered":"2024-09-13T15:24:31.037280+0000","last_clean":"2024-09-13T15:24:31.037280+0000","last_became_active":"2024-09-13T15:08:52.170141+0000","last_became_peered":"2024-09-13T15:08:52.170141+0000","last_unstale":"2024-09-13T15:24:31.037280+0000","last_undegraded":"2024-09-13T15:24:31.037280+0000","last_fullsized":"2024-09-13T15:24:31.037280+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:20.674380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1174,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018371+0000","last_change":"2024-09-13T15:21:14.446858+0000","last_active":"2024-09-13T15:24:20.018371+0000","last_peered":"2024-09-13T15:24:20.018371+0000","last_clean":"2024-09-13T15:24:20.018371+0000","last_became_active":"2024-09-13T15:21:14.419856+0000","last_became_peered":"2024-09-13T15:21:14.419856+0000","last_unstale":"2024-09-13T15:24:20.018371+0000","last_undegraded":"2024-09-13T15:24:20.018371+0000","last_fullsized":"2024-09-13T15:24:20.018371+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:21:51.837728+0000","last_clean_scrub_stamp":"2024-09-13T14:21:51.837728+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:26:11.634373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039056600000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"346'100","reported_seq":1306,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558121+0000","last_change":"2024-09-13T15:17:58.733763+0000","last_active":"2024-09-13T15:24:20.558121+0000","last_peered":"2024-09-13T15:24:20.558121+0000","last_clean":"2024-09-13T15:24:20.558121+0000","last_became_active":"2024-09-13T15:17:58.326073+0000","last_became_peered":"2024-09-13T15:17:58.326073+0000","last_unstale":"2024-09-13T15:24:20.558121+0000","last_undegraded":"2024-09-13T15:24:20.558121+0000","last_fullsized":"2024-09-13T15:24:20.558121+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:17:58.733680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:17:58.733680+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:09:09.783840+0000","scrub_duration":0.006408459,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"346'100","reported_seq":62,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.017948+0000","last_change":"2024-09-13T15:21:14.420905+0000","last_active":"2024-09-13T15:24:20.017948+0000","last_peered":"2024-09-13T15:24:20.017948+0000","last_clean":"2024-09-13T15:24:20.017948+0000","last_became_active":"2024-09-13T15:21:14.419866+0000","last_became_peered":"2024-09-13T15:21:14.419866+0000","last_unstale":"2024-09-13T15:24:20.017948+0000","last_undegraded":"2024-09-13T15:24:20.017948+0000","last_fullsized":"2024-09-13T15:24:20.017948+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:09.661080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:09.661080+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:30.770615+0000","scrub_duration":0.0033469020000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":383,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.327789+0000","last_change":"2024-09-13T15:05:34.078805+0000","last_active":"2024-09-13T15:24:31.327789+0000","last_peered":"2024-09-13T15:24:31.327789+0000","last_clean":"2024-09-13T15:24:31.327789+0000","last_became_active":"2024-09-13T15:05:34.078180+0000","last_became_peered":"2024-09-13T15:05:34.078180+0000","last_unstale":"2024-09-13T15:24:31.327789+0000","last_undegraded":"2024-09-13T15:24:31.327789+0000","last_fullsized":"2024-09-13T15:24:31.327789+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:09:47.240906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1523,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.037260+0000","last_change":"2024-09-13T15:05:34.076149+0000","last_active":"2024-09-13T15:24:31.037260+0000","last_peered":"2024-09-13T15:24:31.037260+0000","last_clean":"2024-09-13T15:24:31.037260+0000","last_became_active":"2024-09-13T15:05:34.075623+0000","last_became_peered":"2024-09-13T15:05:34.075623+0000","last_unstale":"2024-09-13T15:24:31.037260+0000","last_undegraded":"2024-09-13T15:24:31.037260+0000","last_fullsized":"2024-09-13T15:24:31.037260+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:23.608262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":497,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803537+0000","last_change":"2024-09-13T15:08:52.189786+0000","last_active":"2024-09-13T15:24:31.803537+0000","last_peered":"2024-09-13T15:24:31.803537+0000","last_clean":"2024-09-13T15:24:31.803537+0000","last_became_active":"2024-09-13T15:08:52.188997+0000","last_became_peered":"2024-09-13T15:08:52.188997+0000","last_unstale":"2024-09-13T15:24:31.803537+0000","last_undegraded":"2024-09-13T15:24:31.803537+0000","last_fullsized":"2024-09-13T15:24:31.803537+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:08.932986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":483,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802626+0000","last_change":"2024-09-13T15:21:14.426533+0000","last_active":"2024-09-13T15:24:31.802626+0000","last_peered":"2024-09-13T15:24:31.802626+0000","last_clean":"2024-09-13T15:24:31.802626+0000","last_became_active":"2024-09-13T15:21:14.425534+0000","last_became_peered":"2024-09-13T15:21:14.425534+0000","last_unstale":"2024-09-13T15:24:31.802626+0000","last_undegraded":"2024-09-13T15:24:31.802626+0000","last_fullsized":"2024-09-13T15:24:31.802626+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:00:07.710512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"346'100","reported_seq":71,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.017608+0000","last_change":"2024-09-13T15:21:14.452344+0000","last_active":"2024-09-13T15:24:20.017608+0000","last_peered":"2024-09-13T15:24:20.017608+0000","last_clean":"2024-09-13T15:24:20.017608+0000","last_became_active":"2024-09-13T15:21:14.422433+0000","last_became_peered":"2024-09-13T15:21:14.422433+0000","last_unstale":"2024-09-13T15:24:20.017608+0000","last_undegraded":"2024-09-13T15:24:20.017608+0000","last_fullsized":"2024-09-13T15:24:20.017608+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:20:08.705948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:20:08.705948+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:27:01.156043+0000","scrub_duration":0.0041351900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":497,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803790+0000","last_change":"2024-09-13T15:21:14.420492+0000","last_active":"2024-09-13T15:24:31.803790+0000","last_peered":"2024-09-13T15:24:31.803790+0000","last_clean":"2024-09-13T15:24:31.803790+0000","last_became_active":"2024-09-13T15:21:14.419542+0000","last_became_peered":"2024-09-13T15:21:14.419542+0000","last_unstale":"2024-09-13T15:24:31.803790+0000","last_undegraded":"2024-09-13T15:24:31.803790+0000","last_fullsized":"2024-09-13T15:24:31.803790+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:37:55.394189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":108,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.932046+0000","last_change":"2024-09-13T15:08:52.182709+0000","last_active":"2024-09-13T15:24:20.932046+0000","last_peered":"2024-09-13T15:24:20.932046+0000","last_clean":"2024-09-13T15:24:20.932046+0000","last_became_active":"2024-09-13T15:08:52.181906+0000","last_became_peered":"2024-09-13T15:08:52.181906+0000","last_unstale":"2024-09-13T15:24:20.932046+0000","last_undegraded":"2024-09-13T15:24:20.932046+0000","last_fullsized":"2024-09-13T15:24:20.932046+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:36.317662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"346'100","reported_seq":1325,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.557751+0000","last_change":"2024-09-13T15:21:14.408213+0000","last_active":"2024-09-13T15:24:20.557751+0000","last_peered":"2024-09-13T15:24:20.557751+0000","last_clean":"2024-09-13T15:24:20.557751+0000","last_became_active":"2024-09-13T15:21:14.406975+0000","last_became_peered":"2024-09-13T15:21:14.406975+0000","last_unstale":"2024-09-13T15:24:20.557751+0000","last_undegraded":"2024-09-13T15:24:20.557751+0000","last_fullsized":"2024-09-13T15:24:20.557751+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":4,"last_scrub":"346'100","last_scrub_stamp":"2024-09-13T15:18:04.969997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:15:37.340490+0000","last_clean_scrub_stamp":"2024-09-13T15:18:04.969997+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:09.481540+0000","scrub_duration":0.0031202019999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":494,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.803172+0000","last_change":"2024-09-13T15:08:52.182885+0000","last_active":"2024-09-13T15:24:31.803172+0000","last_peered":"2024-09-13T15:24:31.803172+0000","last_clean":"2024-09-13T15:24:31.803172+0000","last_became_active":"2024-09-13T15:08:52.180608+0000","last_became_peered":"2024-09-13T15:08:52.180608+0000","last_unstale":"2024-09-13T15:24:31.803172+0000","last_undegraded":"2024-09-13T15:24:31.803172+0000","last_fullsized":"2024-09-13T15:24:31.803172+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:12.517462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"65'3","reported_seq":1507,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.037687+0000","last_change":"2024-09-13T15:05:34.085458+0000","last_active":"2024-09-13T15:24:31.037687+0000","last_peered":"2024-09-13T15:24:31.037687+0000","last_clean":"2024-09-13T15:24:31.037687+0000","last_became_active":"2024-09-13T15:05:34.084748+0000","last_became_peered":"2024-09-13T15:05:34.084748+0000","last_unstale":"2024-09-13T15:24:31.037687+0000","last_undegraded":"2024-09-13T15:24:31.037687+0000","last_fullsized":"2024-09-13T15:24:31.037687+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:21.898959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1804,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040437+0000","last_change":"2024-09-13T15:18:41.638220+0000","last_active":"2024-09-13T15:24:31.040437+0000","last_peered":"2024-09-13T15:24:31.040437+0000","last_clean":"2024-09-13T15:24:31.040437+0000","last_became_active":"2024-09-13T15:18:41.637822+0000","last_became_peered":"2024-09-13T15:18:41.637822+0000","last_unstale":"2024-09-13T15:24:31.040437+0000","last_undegraded":"2024-09-13T15:24:31.040437+0000","last_fullsized":"2024-09-13T15:24:31.040437+0000","mapping_epoch":1132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:59:28.421185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1170,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018444+0000","last_change":"2024-09-13T15:08:52.175569+0000","last_active":"2024-09-13T15:24:20.018444+0000","last_peered":"2024-09-13T15:24:20.018444+0000","last_clean":"2024-09-13T15:24:20.018444+0000","last_became_active":"2024-09-13T15:08:52.174891+0000","last_became_peered":"2024-09-13T15:08:52.174891+0000","last_unstale":"2024-09-13T15:24:20.018444+0000","last_undegraded":"2024-09-13T15:24:20.018444+0000","last_fullsized":"2024-09-13T15:24:20.018444+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:16.933115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":108,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.932075+0000","last_change":"2024-09-13T15:08:52.188074+0000","last_active":"2024-09-13T15:24:20.932075+0000","last_peered":"2024-09-13T15:24:20.932075+0000","last_clean":"2024-09-13T15:24:20.932075+0000","last_became_active":"2024-09-13T15:08:52.186649+0000","last_became_peered":"2024-09-13T15:08:52.186649+0000","last_unstale":"2024-09-13T15:24:20.932075+0000","last_undegraded":"2024-09-13T15:24:20.932075+0000","last_fullsized":"2024-09-13T15:24:20.932075+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:38:23.478250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"64'1","reported_seq":1149,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.557608+0000","last_change":"2024-09-13T15:08:52.208388+0000","last_active":"2024-09-13T15:24:20.557608+0000","last_peered":"2024-09-13T15:24:20.557608+0000","last_clean":"2024-09-13T15:24:20.557608+0000","last_became_active":"2024-09-13T15:08:52.192253+0000","last_became_peered":"2024-09-13T15:08:52.192253+0000","last_unstale":"2024-09-13T15:24:20.557608+0000","last_undegraded":"2024-09-13T15:24:20.557608+0000","last_fullsized":"2024-09-13T15:24:20.557608+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:07.508774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":494,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.802908+0000","last_change":"2024-09-13T15:05:34.072278+0000","last_active":"2024-09-13T15:24:31.802908+0000","last_peered":"2024-09-13T15:24:31.802908+0000","last_clean":"2024-09-13T15:24:31.802908+0000","last_became_active":"2024-09-13T15:05:34.071723+0000","last_became_peered":"2024-09-13T15:05:34.071723+0000","last_unstale":"2024-09-13T15:24:31.802908+0000","last_undegraded":"2024-09-13T15:24:31.802908+0000","last_fullsized":"2024-09-13T15:24:31.802908+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:02.130890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"64'2","reported_seq":1212,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558972+0000","last_change":"2024-09-13T14:52:43.610257+0000","last_active":"2024-09-13T15:24:20.558972+0000","last_peered":"2024-09-13T15:24:20.558972+0000","last_clean":"2024-09-13T15:24:20.558972+0000","last_became_active":"2024-09-13T14:52:43.596361+0000","last_became_peered":"2024-09-13T14:52:43.596361+0000","last_unstale":"2024-09-13T15:24:20.558972+0000","last_undegraded":"2024-09-13T15:24:20.558972+0000","last_fullsized":"2024-09-13T15:24:20.558972+0000","mapping_epoch":1005,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1006,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:38:46.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1135,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.017859+0000","last_change":"2024-09-13T15:08:52.176830+0000","last_active":"2024-09-13T15:24:20.017859+0000","last_peered":"2024-09-13T15:24:20.017859+0000","last_clean":"2024-09-13T15:24:20.017859+0000","last_became_active":"2024-09-13T15:08:52.176303+0000","last_became_peered":"2024-09-13T15:08:52.176303+0000","last_unstale":"2024-09-13T15:24:20.017859+0000","last_undegraded":"2024-09-13T15:24:20.017859+0000","last_fullsized":"2024-09-13T15:24:20.017859+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:59.553257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"64'1","reported_seq":1183,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.559037+0000","last_change":"2024-09-13T14:47:46.351450+0000","last_active":"2024-09-13T15:24:20.559037+0000","last_peered":"2024-09-13T15:24:20.559037+0000","last_clean":"2024-09-13T15:24:20.559037+0000","last_became_active":"2024-09-13T14:47:46.329933+0000","last_became_peered":"2024-09-13T14:47:46.329933+0000","last_unstale":"2024-09-13T15:24:20.559037+0000","last_undegraded":"2024-09-13T15:24:20.559037+0000","last_fullsized":"2024-09-13T15:24:20.559037+0000","mapping_epoch":958,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":959,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:17.778115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"64'8","reported_seq":118,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.932066+0000","last_change":"2024-09-13T15:08:52.200695+0000","last_active":"2024-09-13T15:24:20.932066+0000","last_peered":"2024-09-13T15:24:20.932066+0000","last_clean":"2024-09-13T15:24:20.932066+0000","last_became_active":"2024-09-13T15:08:52.183473+0000","last_became_peered":"2024-09-13T15:08:52.183473+0000","last_unstale":"2024-09-13T15:24:20.932066+0000","last_undegraded":"2024-09-13T15:24:20.932066+0000","last_fullsized":"2024-09-13T15:24:20.932066+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:42.810003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1520,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040263+0000","last_change":"2024-09-13T15:05:34.074069+0000","last_active":"2024-09-13T15:24:31.040263+0000","last_peered":"2024-09-13T15:24:31.040263+0000","last_clean":"2024-09-13T15:24:31.040263+0000","last_became_active":"2024-09-13T15:05:34.073282+0000","last_became_peered":"2024-09-13T15:05:34.073282+0000","last_unstale":"2024-09-13T15:24:31.040263+0000","last_undegraded":"2024-09-13T15:24:31.040263+0000","last_fullsized":"2024-09-13T15:24:31.040263+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:14.307582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1176,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558123+0000","last_change":"2024-09-13T15:08:52.192368+0000","last_active":"2024-09-13T15:24:20.558123+0000","last_peered":"2024-09-13T15:24:20.558123+0000","last_clean":"2024-09-13T15:24:20.558123+0000","last_became_active":"2024-09-13T15:08:52.191668+0000","last_became_peered":"2024-09-13T15:08:52.191668+0000","last_unstale":"2024-09-13T15:24:20.558123+0000","last_undegraded":"2024-09-13T15:24:20.558123+0000","last_fullsized":"2024-09-13T15:24:20.558123+0000","mapping_epoch":1080,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1081,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:34:27.410222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1760,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.040500+0000","last_change":"2024-09-13T15:05:34.087636+0000","last_active":"2024-09-13T15:24:31.040500+0000","last_peered":"2024-09-13T15:24:31.040500+0000","last_clean":"2024-09-13T15:24:31.040500+0000","last_became_active":"2024-09-13T15:05:34.087255+0000","last_became_peered":"2024-09-13T15:05:34.087255+0000","last_unstale":"2024-09-13T15:24:31.040500+0000","last_undegraded":"2024-09-13T15:24:31.040500+0000","last_fullsized":"2024-09-13T15:24:31.040500+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:28:36.313581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1761,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.039800+0000","last_change":"2024-09-13T15:05:34.089162+0000","last_active":"2024-09-13T15:24:31.039800+0000","last_peered":"2024-09-13T15:24:31.039800+0000","last_clean":"2024-09-13T15:24:31.039800+0000","last_became_active":"2024-09-13T15:05:34.088727+0000","last_became_peered":"2024-09-13T15:05:34.088727+0000","last_unstale":"2024-09-13T15:24:31.039800+0000","last_undegraded":"2024-09-13T15:24:31.039800+0000","last_fullsized":"2024-09-13T15:24:31.039800+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:55:29.209082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1149,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.018483+0000","last_change":"2024-09-13T15:05:34.073363+0000","last_active":"2024-09-13T15:24:20.018483+0000","last_peered":"2024-09-13T15:24:20.018483+0000","last_clean":"2024-09-13T15:24:20.018483+0000","last_became_active":"2024-09-13T15:05:34.072906+0000","last_became_peered":"2024-09-13T15:05:34.072906+0000","last_unstale":"2024-09-13T15:24:20.018483+0000","last_undegraded":"2024-09-13T15:24:20.018483+0000","last_fullsized":"2024-09-13T15:24:20.018483+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:27.966654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"64'2","reported_seq":1221,"reported_epoch":1178,"state":"active+clean","last_fresh":"2024-09-13T15:24:20.558516+0000","last_change":"2024-09-13T15:21:14.437266+0000","last_active":"2024-09-13T15:24:20.558516+0000","last_peered":"2024-09-13T15:24:20.558516+0000","last_clean":"2024-09-13T15:24:20.558516+0000","last_became_active":"2024-09-13T15:21:14.419722+0000","last_became_peered":"2024-09-13T15:21:14.419722+0000","last_unstale":"2024-09-13T15:24:20.558516+0000","last_undegraded":"2024-09-13T15:24:20.558516+0000","last_fullsized":"2024-09-13T15:24:20.558516+0000","mapping_epoch":1160,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:16.175691+0000","last_clean_scrub_stamp":"2024-09-13T14:24:16.175691+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:42:51.958824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1531,"reported_epoch":1179,"state":"active+clean","last_fresh":"2024-09-13T15:24:31.038543+0000","last_change":"2024-09-13T15:05:34.075192+0000","last_active":"2024-09-13T15:24:31.038543+0000","last_peered":"2024-09-13T15:24:31.038543+0000","last_clean":"2024-09-13T15:24:31.038543+0000","last_became_active":"2024-09-13T15:05:34.073315+0000","last_became_peered":"2024-09-13T15:05:34.073315+0000","last_unstale":"2024-09-13T15:24:31.038543+0000","last_undegraded":"2024-09-13T15:24:31.038543+0000","last_fullsized":"2024-09-13T15:24:31.038543+0000","mapping_epoch":1066,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1067,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:24:17.188743+0000","last_clean_scrub_stamp":"2024-09-13T14:24:17.188743+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:35:38.699775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":26,"num_read_kb":28,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":17582,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5715,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398306,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909344,"kb_used_data":13268,"kb_used_omap":1,"kb_used_meta":896062,"kb_avail":92835808,"statfs":{"total":95995035648,"available":95063867392,"internally_reserved":0,"allocated":13586432,"data_stored":8506440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917567832},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594541,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900760,"kb_used_data":12940,"kb_used_omap":1,"kb_used_meta":887806,"kb_avail":92844392,"statfs":{"total":95995035648,"available":95072657408,"internally_reserved":0,"allocated":13250560,"data_stored":8178625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":909113688},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790775,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903764,"kb_used_data":13512,"kb_used_omap":1,"kb_used_meta":890238,"kb_avail":92841388,"statfs":{"total":95995035648,"available":95069581312,"internally_reserved":0,"allocated":13836288,"data_stored":8764811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":911604056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":739,"seq":3173980832304,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":747228,"kb_used_data":12944,"kb_used_omap":76,"kb_used_meta":734195,"kb_avail":92997924,"statfs":{"total":95995035648,"available":95229874176,"internally_reserved":0,"allocated":13254656,"data_stored":8178702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78016,"internal_metadata":751816512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1066,"seq":4578435137790,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":387340,"kb_used_data":13504,"kb_used_omap":121,"kb_used_meta":373702,"kb_avail":93357812,"statfs":{"total":95995035648,"available":95598399488,"internally_reserved":0,"allocated":13828096,"data_stored":8764278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124093,"internal_metadata":382671683},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":672,"seq":2886218023486,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":915472,"kb_used_data":13188,"kb_used_omap":71,"kb_used_meta":902200,"kb_avail":92829680,"statfs":{"total":95995035648,"available":95057592320,"internally_reserved":0,"allocated":13504512,"data_stored":8437120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73056,"internal_metadata":923853472},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":642,"seq":2757369004615,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":794184,"kb_used_data":12924,"kb_used_omap":66,"kb_used_meta":781181,"kb_avail":92950968,"statfs":{"total":95995035648,"available":95181791232,"internally_reserved":0,"allocated":13234176,"data_stored":8173839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68438,"internal_metadata":799929514},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":575,"seq":2469606195808,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":967176,"kb_used_data":12924,"kb_used_omap":56,"kb_used_meta":954183,"kb_avail":92777976,"statfs":{"total":95995035648,"available":95004647424,"internally_reserved":0,"allocated":13234176,"data_stored":8172290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58225,"internal_metadata":977083535},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1089,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4626,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:26:34.469 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T15:26:34.469 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi099 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-13T15:26:34.646 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=ba6b3b2c3b7c40ff8d28c7d9a04059ea Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi099 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-13T15:26:34.649 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T15:26:34.659 INFO:tasks.cephadm:Teardown begin 2024-09-13T15:26:34.660 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:26:34.675 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:34 smithi177 bash[68594]: audit 2024-09-13T15:26:33.335134+0000 mon.a (mon.0) 2980 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:34.676 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:34 smithi177 bash[68594]: audit 2024-09-13T15:26:33.335134+0000 mon.a (mon.0) 2980 : audit [DBG] from='mgr.26252 172.21.15.99:0/717998643' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:26:34.688 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:26:34.731 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T15:26:34.731 DEBUG:teuthology.orchestra.run.smithi099:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T15:26:34.754 DEBUG:teuthology.orchestra.run.smithi177:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T15:26:34.776 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T15:26:34.777 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-13T15:26:34.777 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.a 2024-09-13T15:26:35.066 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:34 smithi099 bash[76324]: [13/Sep/2024:15:26:34] ENGINE Bus STOPPING 2024-09-13T15:26:35.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:34 smithi099 systemd[1]: Stopping Ceph mon.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:26:35.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:34 smithi099 bash[84576]: debug 2024-09-13T15:26:34.939+0000 7f068e3dd640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T15:26:35.067 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:34 smithi099 bash[84576]: debug 2024-09-13T15:26:34.939+0000 7f068e3dd640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-13T15:26:35.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:35 smithi099 bash[76324]: [13/Sep/2024:15:26:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T15:26:35.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:35 smithi099 bash[76324]: [13/Sep/2024:15:26:35] ENGINE Bus STOPPED 2024-09-13T15:26:35.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:35 smithi099 bash[76324]: [13/Sep/2024:15:26:35] ENGINE Bus STARTING 2024-09-13T15:26:35.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:35 smithi099 bash[76324]: [13/Sep/2024:15:26:35] ENGINE Serving on http://:::9283 2024-09-13T15:26:35.325 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:35 smithi099 bash[76324]: [13/Sep/2024:15:26:35] ENGINE Bus STARTED 2024-09-13T15:26:36.372 INFO:journalctl@ceph.mon.a.smithi099.stdout:Sep 13 15:26:36 smithi099 bash[263341]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-a 2024-09-13T15:26:36.377 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.a.service' 2024-09-13T15:26:36.478 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:26:36.479 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-13T15:26:36.479 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-13T15:26:36.479 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.c 2024-09-13T15:26:36.640 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:36 smithi099 systemd[1]: Stopping Ceph mon.c for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:26:36.970 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:36 smithi099 bash[87147]: debug 2024-09-13T15:26:36.635+0000 7ff50c44a640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T15:26:36.971 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:36 smithi099 bash[87147]: debug 2024-09-13T15:26:36.635+0000 7ff50c44a640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-13T15:26:36.971 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:36 smithi099 bash[76324]: [13/Sep/2024:15:26:36] ENGINE Bus STOPPING 2024-09-13T15:26:36.971 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:36 smithi099 bash[76324]: [13/Sep/2024:15:26:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T15:26:36.971 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:36 smithi099 bash[76324]: [13/Sep/2024:15:26:36] ENGINE Bus STOPPED 2024-09-13T15:26:36.971 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:36 smithi099 bash[76324]: [13/Sep/2024:15:26:36] ENGINE Bus STARTING 2024-09-13T15:26:36.972 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:36 smithi099 bash[76324]: [13/Sep/2024:15:26:36] ENGINE Serving on http://:::9283 2024-09-13T15:26:36.972 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:36 smithi099 bash[76324]: [13/Sep/2024:15:26:36] ENGINE Bus STARTED 2024-09-13T15:26:37.324 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:37 smithi099 bash[76324]: ::ffff:172.21.15.177 - - [13/Sep/2024:15:26:37] "GET /metrics HTTP/1.1" 200 34332 "" "Prometheus/2.51.0" 2024-09-13T15:26:37.922 INFO:journalctl@ceph.mon.c.smithi099.stdout:Sep 13 15:26:37 smithi099 bash[263451]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-c 2024-09-13T15:26:37.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.c.service' 2024-09-13T15:26:38.017 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:26:38.017 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-13T15:26:38.018 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-13T15:26:38.018 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.b 2024-09-13T15:26:38.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:38 smithi177 systemd[1]: Stopping Ceph mon.b for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:26:38.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:38 smithi177 bash[68594]: debug 2024-09-13T15:26:38.250+0000 7f67f0013640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T15:26:38.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:38 smithi177 bash[68594]: debug 2024-09-13T15:26:38.250+0000 7f67f0013640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-13T15:26:39.338 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.b.service' 2024-09-13T15:26:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:39 smithi177 bash[177696]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-b 2024-09-13T15:26:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:39 smithi177 bash[177760]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mon-b 2024-09-13T15:26:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:39 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T15:26:39.356 INFO:journalctl@ceph.mon.b.smithi177.stdout:Sep 13 15:26:39 smithi177 systemd[1]: Stopped Ceph mon.b for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:26:39.382 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:26:39.382 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-13T15:26:39.382 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-13T15:26:39.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.y 2024-09-13T15:26:39.802 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:39 smithi099 systemd[1]: Stopping Ceph mgr.y for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:26:39.802 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:39 smithi099 bash[76324]: debug 2024-09-13T15:26:39.527+0000 7fd5b984d640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T15:26:40.612 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:26:40 smithi177 bash[61439]: ts=2024-09-13T15:26:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"af9d2c22-71d9-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T15:26:40.735 INFO:journalctl@ceph.mgr.y.smithi099.stdout:Sep 13 15:26:40 smithi099 bash[263559]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mgr-y 2024-09-13T15:26:40.741 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-13T15:26:40.816 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:26:40.816 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-13T15:26:40.817 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-13T15:26:40.817 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.x 2024-09-13T15:26:41.112 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 15:26:40 smithi177 systemd[1]: Stopping Ceph mgr.x for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:26:41.997 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 15:26:41 smithi177 bash[177822]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-mgr-x 2024-09-13T15:26:41.998 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Sep 13 15:26:41 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:26:42.000 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-13T15:26:42.018 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:26:42.019 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-13T15:26:42.019 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T15:26:42.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.0 2024-09-13T15:26:42.356 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 15:26:42 smithi099 systemd[1]: Stopping Ceph osd.0 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:26:42.357 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 15:26:42 smithi099 bash[94102]: debug 2024-09-13T15:26:42.167+0000 7fdfeb9a9640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T15:26:42.357 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 15:26:42 smithi099 bash[94102]: debug 2024-09-13T15:26:42.167+0000 7fdfeb9a9640 -1 osd.0 1179 *** Got signal Terminated *** 2024-09-13T15:26:42.357 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 15:26:42 smithi099 bash[94102]: debug 2024-09-13T15:26:42.167+0000 7fdfeb9a9640 -1 osd.0 1179 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:26:48.824 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 15:26:48 smithi099 bash[263657]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-0 2024-09-13T15:26:48.824 INFO:journalctl@ceph.osd.0.smithi099.stdout:Sep 13 15:26:48 smithi099 bash[263721]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-0 2024-09-13T15:26:51.538 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T15:26:51.634 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:26:51.634 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T15:26:51.634 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T15:26:51.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.1 2024-09-13T15:26:51.844 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 15:26:51 smithi099 systemd[1]: Stopping Ceph osd.1 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:26:51.845 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 15:26:51 smithi099 bash[100499]: debug 2024-09-13T15:26:51.810+0000 7f69959ef640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T15:26:51.845 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 15:26:51 smithi099 bash[100499]: debug 2024-09-13T15:26:51.810+0000 7f69959ef640 -1 osd.1 1179 *** Got signal Terminated *** 2024-09-13T15:26:51.845 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 15:26:51 smithi099 bash[100499]: debug 2024-09-13T15:26:51.810+0000 7f69959ef640 -1 osd.1 1179 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:26:58.504 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 15:26:58 smithi099 bash[263853]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-1 2024-09-13T15:26:58.504 INFO:journalctl@ceph.osd.1.smithi099.stdout:Sep 13 15:26:58 smithi099 bash[263918]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-1 2024-09-13T15:27:00.911 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T15:27:01.023 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:27:01.023 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T15:27:01.023 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T15:27:01.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.2 2024-09-13T15:27:01.184 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 15:27:01 smithi099 systemd[1]: Stopping Ceph osd.2 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:27:01.184 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 15:27:01 smithi099 bash[105859]: debug 2024-09-13T15:27:01.166+0000 7f394182c640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T15:27:01.185 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 15:27:01 smithi099 bash[105859]: debug 2024-09-13T15:27:01.166+0000 7f394182c640 -1 osd.2 1179 *** Got signal Terminated *** 2024-09-13T15:27:01.185 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 15:27:01 smithi099 bash[105859]: debug 2024-09-13T15:27:01.166+0000 7f394182c640 -1 osd.2 1179 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:27:07.548 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 15:27:07 smithi099 bash[264047]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-2 2024-09-13T15:27:07.824 INFO:journalctl@ceph.osd.2.smithi099.stdout:Sep 13 15:27:07 smithi099 bash[264109]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-2 2024-09-13T15:27:07.862 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:07 smithi177 bash[61439]: ts=2024-09-13T15:27:07.366Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.99:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.99:8765: connect: connection refused" 2024-09-13T15:27:07.863 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:07 smithi177 bash[61439]: ts=2024-09-13T15:27:07.369Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.99:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.99:8765: connect: connection refused" 2024-09-13T15:27:07.863 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:07 smithi177 bash[61439]: ts=2024-09-13T15:27:07.371Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.99:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.99:8765: connect: connection refused" 2024-09-13T15:27:07.863 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:07 smithi177 bash[61439]: ts=2024-09-13T15:27:07.372Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.99:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.99:8765: connect: connection refused" 2024-09-13T15:27:07.863 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:07 smithi177 bash[61439]: ts=2024-09-13T15:27:07.371Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.99:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.99:8765: connect: connection refused" 2024-09-13T15:27:07.863 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:07 smithi177 bash[61439]: ts=2024-09-13T15:27:07.372Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.99:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.99:8765: connect: connection refused" 2024-09-13T15:27:09.748 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T15:27:09.825 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:27:09.826 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T15:27:09.826 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T15:27:09.826 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.3 2024-09-13T15:27:09.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:09 smithi177 bash[38978]: debug 2024-09-13T15:27:09.425+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:10.002 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:09 smithi099 systemd[1]: Stopping Ceph osd.3 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:27:10.324 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:10 smithi099 bash[193561]: debug 2024-09-13T15:27:09.998+0000 7f58a83a8640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T15:27:10.324 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:10 smithi099 bash[193561]: debug 2024-09-13T15:27:09.998+0000 7f58a83a8640 -1 osd.3 1179 *** Got signal Terminated *** 2024-09-13T15:27:10.325 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:10 smithi099 bash[193561]: debug 2024-09-13T15:27:09.998+0000 7f58a83a8640 -1 osd.3 1179 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:27:10.325 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:10 smithi099 bash[193561]: debug 2024-09-13T15:27:10.054+0000 7f58a09b5640 -1 osd.3 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.199431+0000 front 2024-09-13T15:26:43.199553+0000 (oldest deadline 2024-09-13T15:27:09.099529+0000) 2024-09-13T15:27:10.834 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:10 smithi177 bash[38978]: debug 2024-09-13T15:27:10.473+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:10.835 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:10 smithi177 bash[33283]: debug 2024-09-13T15:27:10.649+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:11.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:10 smithi177 bash[74346]: debug 2024-09-13T15:27:10.833+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:11.324 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:11 smithi099 bash[193561]: debug 2024-09-13T15:27:11.030+0000 7f58a09b5640 -1 osd.3 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.199431+0000 front 2024-09-13T15:26:43.199553+0000 (oldest deadline 2024-09-13T15:27:09.099529+0000) 2024-09-13T15:27:11.777 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:11 smithi177 bash[38978]: debug 2024-09-13T15:27:11.513+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:11.777 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:11 smithi177 bash[33283]: debug 2024-09-13T15:27:11.597+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:12.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:11 smithi177 bash[74346]: debug 2024-09-13T15:27:11.793+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:12.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:11 smithi177 bash[44725]: debug 2024-09-13T15:27:11.773+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:12.324 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:12 smithi099 bash[193561]: debug 2024-09-13T15:27:12.002+0000 7f58a09b5640 -1 osd.3 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.199431+0000 front 2024-09-13T15:26:43.199553+0000 (oldest deadline 2024-09-13T15:27:09.099529+0000) 2024-09-13T15:27:12.762 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:12 smithi177 bash[38978]: debug 2024-09-13T15:27:12.509+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:12.763 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:12 smithi177 bash[33283]: debug 2024-09-13T15:27:12.569+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:13.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:12 smithi177 bash[74346]: debug 2024-09-13T15:27:12.777+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:13.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:12 smithi177 bash[44725]: debug 2024-09-13T15:27:12.757+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:13.324 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:13 smithi099 bash[193561]: debug 2024-09-13T15:27:13.010+0000 7f58a09b5640 -1 osd.3 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.199431+0000 front 2024-09-13T15:26:43.199553+0000 (oldest deadline 2024-09-13T15:27:09.099529+0000) 2024-09-13T15:27:13.785 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:13 smithi177 bash[38978]: debug 2024-09-13T15:27:13.533+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:13.785 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:13 smithi177 bash[33283]: debug 2024-09-13T15:27:13.553+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:13.785 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:13 smithi177 bash[44725]: debug 2024-09-13T15:27:13.733+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:14.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:13 smithi177 bash[74346]: debug 2024-09-13T15:27:13.785+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:14.324 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:13 smithi099 bash[193561]: debug 2024-09-13T15:27:13.994+0000 7f58a09b5640 -1 osd.3 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.199431+0000 front 2024-09-13T15:26:43.199553+0000 (oldest deadline 2024-09-13T15:27:09.099529+0000) 2024-09-13T15:27:14.776 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:14 smithi177 bash[33283]: debug 2024-09-13T15:27:14.549+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:14.776 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:14 smithi177 bash[38978]: debug 2024-09-13T15:27:14.501+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:14.776 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:14 smithi177 bash[44725]: debug 2024-09-13T15:27:14.725+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:15.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:14 smithi177 bash[74346]: debug 2024-09-13T15:27:14.773+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:15.324 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:14 smithi099 bash[193561]: debug 2024-09-13T15:27:14.986+0000 7f58a09b5640 -1 osd.3 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.199431+0000 front 2024-09-13T15:26:43.199553+0000 (oldest deadline 2024-09-13T15:27:09.099529+0000) 2024-09-13T15:27:15.749 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:15 smithi177 bash[33283]: debug 2024-09-13T15:27:15.581+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:15.749 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:15 smithi177 bash[44725]: debug 2024-09-13T15:27:15.685+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:15.750 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:15 smithi177 bash[38978]: debug 2024-09-13T15:27:15.453+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:16.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:15 smithi177 bash[74346]: debug 2024-09-13T15:27:15.745+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:16.574 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:16 smithi099 bash[264239]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-3 2024-09-13T15:27:16.574 INFO:journalctl@ceph.osd.3.smithi099.stdout:Sep 13 15:27:16 smithi099 bash[264302]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-3 2024-09-13T15:27:16.692 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:16 smithi177 bash[33283]: debug 2024-09-13T15:27:16.605+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:16.693 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:16 smithi177 bash[38978]: debug 2024-09-13T15:27:16.433+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:17.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:16 smithi177 bash[44725]: debug 2024-09-13T15:27:16.689+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:17.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:16 smithi177 bash[74346]: debug 2024-09-13T15:27:16.785+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:17.834 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:17 smithi177 bash[44725]: debug 2024-09-13T15:27:17.648+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:17.835 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:17 smithi177 bash[33283]: debug 2024-09-13T15:27:17.568+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:17.835 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:17 smithi177 bash[38978]: debug 2024-09-13T15:27:17.420+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:18.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:17 smithi177 bash[74346]: debug 2024-09-13T15:27:17.828+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:18.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T15:27:18.701 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:27:18.702 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T15:27:18.702 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T15:27:18.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.4 2024-09-13T15:27:18.713 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:18 smithi177 bash[33283]: debug 2024-09-13T15:27:18.604+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:18.713 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:18 smithi177 bash[38978]: debug 2024-09-13T15:27:18.436+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:18.714 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:18 smithi177 bash[44725]: debug 2024-09-13T15:27:18.676+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:18.714 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:18 smithi177 bash[44725]: debug 2024-09-13T15:27:18.676+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:19.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:18 smithi177 systemd[1]: Stopping Ceph osd.4 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:27:19.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:18 smithi177 bash[74346]: debug 2024-09-13T15:27:18.780+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:19.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:18 smithi177 bash[74346]: debug 2024-09-13T15:27:18.844+0000 7f6b6c9f0640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T15:27:19.113 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:18 smithi177 bash[74346]: debug 2024-09-13T15:27:18.844+0000 7f6b6c9f0640 -1 osd.4 1179 *** Got signal Terminated *** 2024-09-13T15:27:19.113 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:18 smithi177 bash[74346]: debug 2024-09-13T15:27:18.844+0000 7f6b6c9f0640 -1 osd.4 1179 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:27:19.774 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:19 smithi177 bash[33283]: debug 2024-09-13T15:27:19.576+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:20.039 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:19 smithi177 bash[38978]: debug 2024-09-13T15:27:19.468+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:20.039 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:19 smithi177 bash[38978]: debug 2024-09-13T15:27:19.468+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:20.039 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:19 smithi177 bash[44725]: debug 2024-09-13T15:27:19.708+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:20.039 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:19 smithi177 bash[44725]: debug 2024-09-13T15:27:19.708+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:20.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:19 smithi177 bash[74346]: debug 2024-09-13T15:27:19.772+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:20.112 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:19 smithi177 bash[74346]: debug 2024-09-13T15:27:19.772+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.995171+0000 front 2024-09-13T15:26:55.994979+0000 (oldest deadline 2024-09-13T15:27:18.894789+0000) 2024-09-13T15:27:20.862 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:20 smithi177 bash[74346]: debug 2024-09-13T15:27:20.756+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:20.862 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:20 smithi177 bash[74346]: debug 2024-09-13T15:27:20.756+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.995171+0000 front 2024-09-13T15:26:55.994979+0000 (oldest deadline 2024-09-13T15:27:18.894789+0000) 2024-09-13T15:27:20.862 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:20 smithi177 bash[33283]: debug 2024-09-13T15:27:20.620+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:20.863 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:20 smithi177 bash[33283]: debug 2024-09-13T15:27:20.620+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:20.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:20 smithi177 bash[38978]: debug 2024-09-13T15:27:20.504+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:20.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:20 smithi177 bash[38978]: debug 2024-09-13T15:27:20.504+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:20.863 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:20 smithi177 bash[44725]: debug 2024-09-13T15:27:20.692+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:20.863 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:20 smithi177 bash[44725]: debug 2024-09-13T15:27:20.692+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:21.862 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:21 smithi177 bash[74346]: debug 2024-09-13T15:27:21.712+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:21.862 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:21 smithi177 bash[74346]: debug 2024-09-13T15:27:21.712+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.995171+0000 front 2024-09-13T15:26:55.994979+0000 (oldest deadline 2024-09-13T15:27:18.894789+0000) 2024-09-13T15:27:21.862 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:21 smithi177 bash[33283]: debug 2024-09-13T15:27:21.600+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:21.862 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:21 smithi177 bash[33283]: debug 2024-09-13T15:27:21.600+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:21.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:21 smithi177 bash[38978]: debug 2024-09-13T15:27:21.556+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:21.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:21 smithi177 bash[38978]: debug 2024-09-13T15:27:21.556+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:21.863 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:21 smithi177 bash[44725]: debug 2024-09-13T15:27:21.736+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:21.863 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:21 smithi177 bash[44725]: debug 2024-09-13T15:27:21.736+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:22.862 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:22 smithi177 bash[33283]: debug 2024-09-13T15:27:22.552+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:22.862 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:22 smithi177 bash[33283]: debug 2024-09-13T15:27:22.552+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:22.862 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:22 smithi177 bash[74346]: debug 2024-09-13T15:27:22.748+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:22.862 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:22 smithi177 bash[74346]: debug 2024-09-13T15:27:22.748+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.995171+0000 front 2024-09-13T15:26:55.994979+0000 (oldest deadline 2024-09-13T15:27:18.894789+0000) 2024-09-13T15:27:22.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:22 smithi177 bash[38978]: debug 2024-09-13T15:27:22.552+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:22.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:22 smithi177 bash[38978]: debug 2024-09-13T15:27:22.552+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:22.863 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:22 smithi177 bash[44725]: debug 2024-09-13T15:27:22.736+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:22.863 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:22 smithi177 bash[44725]: debug 2024-09-13T15:27:22.736+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:23.800 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:23 smithi177 bash[33283]: debug 2024-09-13T15:27:23.580+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:23.800 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:23 smithi177 bash[33283]: debug 2024-09-13T15:27:23.580+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:23.800 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:23 smithi177 bash[44725]: debug 2024-09-13T15:27:23.700+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:23.801 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:23 smithi177 bash[44725]: debug 2024-09-13T15:27:23.700+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:23.801 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:23 smithi177 bash[38978]: debug 2024-09-13T15:27:23.544+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:23.801 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:23 smithi177 bash[38978]: debug 2024-09-13T15:27:23.544+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:24.113 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:23 smithi177 bash[74346]: debug 2024-09-13T15:27:23.796+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:45.493569+0000 front 2024-09-13T15:26:45.493541+0000 (oldest deadline 2024-09-13T15:27:10.193228+0000) 2024-09-13T15:27:24.113 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:23 smithi177 bash[74346]: debug 2024-09-13T15:27:23.796+0000 7f6b64ffd640 -1 osd.4 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.995171+0000 front 2024-09-13T15:26:55.994979+0000 (oldest deadline 2024-09-13T15:27:18.894789+0000) 2024-09-13T15:27:24.862 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:24 smithi177 bash[44725]: debug 2024-09-13T15:27:24.736+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:24.862 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:24 smithi177 bash[44725]: debug 2024-09-13T15:27:24.736+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:24.863 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:24 smithi177 bash[33283]: debug 2024-09-13T15:27:24.616+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:24.863 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:24 smithi177 bash[33283]: debug 2024-09-13T15:27:24.616+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:24.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:24 smithi177 bash[38978]: debug 2024-09-13T15:27:24.588+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:24.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:24 smithi177 bash[38978]: debug 2024-09-13T15:27:24.588+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:25.595 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:25 smithi177 bash[178277]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-4 2024-09-13T15:27:25.595 INFO:journalctl@ceph.osd.4.smithi177.stdout:Sep 13 15:27:25 smithi177 bash[178341]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-4 2024-09-13T15:27:25.862 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:25 smithi177 bash[44725]: debug 2024-09-13T15:27:25.780+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:25.862 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:25 smithi177 bash[44725]: debug 2024-09-13T15:27:25.780+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:25.862 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:25 smithi177 bash[33283]: debug 2024-09-13T15:27:25.660+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:25.863 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:25 smithi177 bash[33283]: debug 2024-09-13T15:27:25.660+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:25.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:25 smithi177 bash[38978]: debug 2024-09-13T15:27:25.592+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:25.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:25 smithi177 bash[38978]: debug 2024-09-13T15:27:25.592+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:26.862 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:26 smithi177 bash[44725]: debug 2024-09-13T15:27:26.804+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:26.862 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:26 smithi177 bash[44725]: debug 2024-09-13T15:27:26.804+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:26.863 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:26 smithi177 bash[33283]: debug 2024-09-13T15:27:26.700+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:26.863 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:26 smithi177 bash[33283]: debug 2024-09-13T15:27:26.700+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:26.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:26 smithi177 bash[38978]: debug 2024-09-13T15:27:26.588+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:26.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:26 smithi177 bash[38978]: debug 2024-09-13T15:27:26.588+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:27.630 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:27 smithi177 bash[38978]: debug 2024-09-13T15:27:27.628+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:27.630 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:27 smithi177 bash[38978]: debug 2024-09-13T15:27:27.628+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:27.630 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:27 smithi177 bash[38978]: debug 2024-09-13T15:27:27.628+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:28.018 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:27 smithi177 bash[33283]: debug 2024-09-13T15:27:27.728+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:28.019 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:27 smithi177 bash[33283]: debug 2024-09-13T15:27:27.728+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:28.019 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:27 smithi177 bash[44725]: debug 2024-09-13T15:27:27.836+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:28.019 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:27 smithi177 bash[44725]: debug 2024-09-13T15:27:27.836+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:28.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T15:27:28.091 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:27:28.092 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T15:27:28.092 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T15:27:28.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.5 2024-09-13T15:27:28.294 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:28 smithi177 systemd[1]: Stopping Ceph osd.5 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:27:28.612 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:28 smithi177 bash[33283]: debug 2024-09-13T15:27:28.292+0000 7f46c2b32640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T15:27:28.612 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:28 smithi177 bash[33283]: debug 2024-09-13T15:27:28.292+0000 7f46c2b32640 -1 osd.5 1179 *** Got signal Terminated *** 2024-09-13T15:27:28.612 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:28 smithi177 bash[33283]: debug 2024-09-13T15:27:28.292+0000 7f46c2b32640 -1 osd.5 1179 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:27:28.886 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:28 smithi177 bash[33283]: debug 2024-09-13T15:27:28.772+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:28.886 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:28 smithi177 bash[33283]: debug 2024-09-13T15:27:28.772+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:28.886 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:28 smithi177 bash[38978]: debug 2024-09-13T15:27:28.612+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:28.886 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:28 smithi177 bash[38978]: debug 2024-09-13T15:27:28.612+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:28.887 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:28 smithi177 bash[38978]: debug 2024-09-13T15:27:28.612+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:29.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:28 smithi177 bash[44725]: debug 2024-09-13T15:27:28.884+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:29.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:28 smithi177 bash[44725]: debug 2024-09-13T15:27:28.884+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:30.112 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:29 smithi177 bash[33283]: debug 2024-09-13T15:27:29.736+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:30.112 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:29 smithi177 bash[33283]: debug 2024-09-13T15:27:29.736+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:30.112 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:29 smithi177 bash[38978]: debug 2024-09-13T15:27:29.644+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:30.113 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:29 smithi177 bash[38978]: debug 2024-09-13T15:27:29.644+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:30.113 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:29 smithi177 bash[38978]: debug 2024-09-13T15:27:29.644+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:30.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:29 smithi177 bash[44725]: debug 2024-09-13T15:27:29.852+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:30.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:29 smithi177 bash[44725]: debug 2024-09-13T15:27:29.852+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:30.862 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:30 smithi177 bash[33283]: debug 2024-09-13T15:27:30.728+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:30.862 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:30 smithi177 bash[33283]: debug 2024-09-13T15:27:30.728+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:30.862 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:30 smithi177 bash[33283]: debug 2024-09-13T15:27:30.728+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:05.692170+0000 front 2024-09-13T15:27:05.692367+0000 (oldest deadline 2024-09-13T15:27:30.391777+0000) 2024-09-13T15:27:30.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:30 smithi177 bash[38978]: debug 2024-09-13T15:27:30.596+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:30.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:30 smithi177 bash[38978]: debug 2024-09-13T15:27:30.596+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:30.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:30 smithi177 bash[38978]: debug 2024-09-13T15:27:30.596+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:31.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:30 smithi177 bash[44725]: debug 2024-09-13T15:27:30.864+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:31.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:30 smithi177 bash[44725]: debug 2024-09-13T15:27:30.864+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:31.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:30 smithi177 bash[44725]: debug 2024-09-13T15:27:30.864+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:31.835 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:31 smithi177 bash[33283]: debug 2024-09-13T15:27:31.696+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:31.836 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:31 smithi177 bash[33283]: debug 2024-09-13T15:27:31.696+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:31.836 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:31 smithi177 bash[33283]: debug 2024-09-13T15:27:31.696+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:05.692170+0000 front 2024-09-13T15:27:05.692367+0000 (oldest deadline 2024-09-13T15:27:30.391777+0000) 2024-09-13T15:27:31.836 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:31 smithi177 bash[38978]: debug 2024-09-13T15:27:31.560+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:31.837 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:31 smithi177 bash[38978]: debug 2024-09-13T15:27:31.560+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:31.837 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:31 smithi177 bash[38978]: debug 2024-09-13T15:27:31.560+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:32.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:31 smithi177 bash[44725]: debug 2024-09-13T15:27:31.832+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:32.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:31 smithi177 bash[44725]: debug 2024-09-13T15:27:31.832+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:32.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:31 smithi177 bash[44725]: debug 2024-09-13T15:27:31.832+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:32.859 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:32 smithi177 bash[33283]: debug 2024-09-13T15:27:32.672+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:43.988847+0000 front 2024-09-13T15:26:43.988694+0000 (oldest deadline 2024-09-13T15:27:09.888544+0000) 2024-09-13T15:27:32.859 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:32 smithi177 bash[33283]: debug 2024-09-13T15:27:32.672+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:55.190129+0000 front 2024-09-13T15:26:55.190036+0000 (oldest deadline 2024-09-13T15:27:19.889924+0000) 2024-09-13T15:27:32.859 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:32 smithi177 bash[33283]: debug 2024-09-13T15:27:32.672+0000 7f46c094d640 -1 osd.5 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:05.692170+0000 front 2024-09-13T15:27:05.692367+0000 (oldest deadline 2024-09-13T15:27:30.391777+0000) 2024-09-13T15:27:32.859 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:32 smithi177 bash[38978]: debug 2024-09-13T15:27:32.528+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:32.859 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:32 smithi177 bash[38978]: debug 2024-09-13T15:27:32.528+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:32.860 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:32 smithi177 bash[38978]: debug 2024-09-13T15:27:32.528+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:33.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:32 smithi177 bash[44725]: debug 2024-09-13T15:27:32.856+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:33.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:32 smithi177 bash[44725]: debug 2024-09-13T15:27:32.856+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:33.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:32 smithi177 bash[44725]: debug 2024-09-13T15:27:32.856+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:33.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:33 smithi177 bash[38978]: debug 2024-09-13T15:27:33.480+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:33.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:33 smithi177 bash[38978]: debug 2024-09-13T15:27:33.480+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:33.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:33 smithi177 bash[38978]: debug 2024-09-13T15:27:33.480+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:34.272 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:33 smithi177 bash[44725]: debug 2024-09-13T15:27:33.864+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:34.273 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:33 smithi177 bash[44725]: debug 2024-09-13T15:27:33.864+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:34.273 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:33 smithi177 bash[44725]: debug 2024-09-13T15:27:33.864+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:34.612 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:34 smithi177 bash[38978]: debug 2024-09-13T15:27:34.484+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:34.612 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:34 smithi177 bash[38978]: debug 2024-09-13T15:27:34.484+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:34.612 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:34 smithi177 bash[38978]: debug 2024-09-13T15:27:34.484+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:35.112 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:34 smithi177 bash[178471]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-5 2024-09-13T15:27:35.112 INFO:journalctl@ceph.osd.5.smithi177.stdout:Sep 13 15:27:35 smithi177 bash[178535]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-5 2024-09-13T15:27:35.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:34 smithi177 bash[44725]: debug 2024-09-13T15:27:34.892+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:35.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:34 smithi177 bash[44725]: debug 2024-09-13T15:27:34.892+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:35.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:34 smithi177 bash[44725]: debug 2024-09-13T15:27:34.892+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:35.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:35 smithi177 bash[38978]: debug 2024-09-13T15:27:35.444+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:35.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:35 smithi177 bash[38978]: debug 2024-09-13T15:27:35.444+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:35.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:35 smithi177 bash[38978]: debug 2024-09-13T15:27:35.444+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:36.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:35 smithi177 bash[44725]: debug 2024-09-13T15:27:35.940+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:36.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:35 smithi177 bash[44725]: debug 2024-09-13T15:27:35.940+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:36.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:35 smithi177 bash[44725]: debug 2024-09-13T15:27:35.940+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:36.363 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:35 smithi177 bash[44725]: debug 2024-09-13T15:27:35.940+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:36.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:36 smithi177 bash[38978]: debug 2024-09-13T15:27:36.488+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:36.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:36 smithi177 bash[38978]: debug 2024-09-13T15:27:36.492+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:36.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:36 smithi177 bash[38978]: debug 2024-09-13T15:27:36.492+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:37.129 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:36 smithi177 bash[44725]: debug 2024-09-13T15:27:36.892+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:37.130 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:36 smithi177 bash[44725]: debug 2024-09-13T15:27:36.892+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:37.130 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:36 smithi177 bash[44725]: debug 2024-09-13T15:27:36.892+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:37.130 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:36 smithi177 bash[44725]: debug 2024-09-13T15:27:36.892+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:37.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T15:27:37.663 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:27:37.663 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T15:27:37.663 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T15:27:37.663 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.6 2024-09-13T15:27:37.847 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[38978]: debug 2024-09-13T15:27:37.524+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:37.847 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[38978]: debug 2024-09-13T15:27:37.524+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:37.847 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[38978]: debug 2024-09-13T15:27:37.524+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:37.847 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[38978]: debug 2024-09-13T15:27:37.524+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:13.811180+0000 front 2024-09-13T15:27:13.811250+0000 (oldest deadline 2024-09-13T15:27:37.310916+0000) 2024-09-13T15:27:37.847 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:37 smithi177 systemd[1]: Stopping Ceph osd.6 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:27:37.847 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[38978]: debug 2024-09-13T15:27:37.844+0000 7f57eb2b5640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T15:27:37.913 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[38978]: debug 2024-09-13T15:27:37.844+0000 7f57eb2b5640 -1 osd.6 1179 *** Got signal Terminated *** 2024-09-13T15:27:37.913 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[38978]: debug 2024-09-13T15:27:37.844+0000 7f57eb2b5640 -1 osd.6 1179 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:27:38.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[44725]: debug 2024-09-13T15:27:37.912+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:38.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[44725]: debug 2024-09-13T15:27:37.912+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:38.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[44725]: debug 2024-09-13T15:27:37.912+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:38.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:37 smithi177 bash[44725]: debug 2024-09-13T15:27:37.912+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:38.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:38 smithi177 bash[38978]: debug 2024-09-13T15:27:38.488+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:38.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:38 smithi177 bash[38978]: debug 2024-09-13T15:27:38.488+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:38.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:38 smithi177 bash[38978]: debug 2024-09-13T15:27:38.488+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:38.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:38 smithi177 bash[38978]: debug 2024-09-13T15:27:38.488+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:13.811180+0000 front 2024-09-13T15:27:13.811250+0000 (oldest deadline 2024-09-13T15:27:37.310916+0000) 2024-09-13T15:27:39.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:38 smithi177 bash[44725]: debug 2024-09-13T15:27:38.868+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:39.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:38 smithi177 bash[44725]: debug 2024-09-13T15:27:38.868+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:39.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:38 smithi177 bash[44725]: debug 2024-09-13T15:27:38.868+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:39.363 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:38 smithi177 bash[44725]: debug 2024-09-13T15:27:38.868+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:39.850 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:39 smithi177 bash[38978]: debug 2024-09-13T15:27:39.456+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:39.850 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:39 smithi177 bash[38978]: debug 2024-09-13T15:27:39.456+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:39.850 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:39 smithi177 bash[38978]: debug 2024-09-13T15:27:39.456+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:39.850 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:39 smithi177 bash[38978]: debug 2024-09-13T15:27:39.456+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:13.811180+0000 front 2024-09-13T15:27:13.811250+0000 (oldest deadline 2024-09-13T15:27:37.310916+0000) 2024-09-13T15:27:40.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:39 smithi177 bash[44725]: debug 2024-09-13T15:27:39.848+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:40.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:39 smithi177 bash[44725]: debug 2024-09-13T15:27:39.848+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:40.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:39 smithi177 bash[44725]: debug 2024-09-13T15:27:39.848+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:40.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:39 smithi177 bash[44725]: debug 2024-09-13T15:27:39.848+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:40.825 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:40 smithi177 bash[38978]: debug 2024-09-13T15:27:40.484+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:40.826 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:40 smithi177 bash[38978]: debug 2024-09-13T15:27:40.484+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:40.826 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:40 smithi177 bash[38978]: debug 2024-09-13T15:27:40.484+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:40.826 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:40 smithi177 bash[38978]: debug 2024-09-13T15:27:40.484+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:13.811180+0000 front 2024-09-13T15:27:13.811250+0000 (oldest deadline 2024-09-13T15:27:37.310916+0000) 2024-09-13T15:27:41.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:40 smithi177 bash[44725]: debug 2024-09-13T15:27:40.824+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:41.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:40 smithi177 bash[44725]: debug 2024-09-13T15:27:40.824+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:41.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:40 smithi177 bash[44725]: debug 2024-09-13T15:27:40.824+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:41.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:40 smithi177 bash[44725]: debug 2024-09-13T15:27:40.824+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:41.822 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:41 smithi177 bash[38978]: debug 2024-09-13T15:27:41.496+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:41.822 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:41 smithi177 bash[38978]: debug 2024-09-13T15:27:41.496+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:41.822 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:41 smithi177 bash[38978]: debug 2024-09-13T15:27:41.496+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:41.823 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:41 smithi177 bash[38978]: debug 2024-09-13T15:27:41.496+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:13.811180+0000 front 2024-09-13T15:27:13.811250+0000 (oldest deadline 2024-09-13T15:27:37.310916+0000) 2024-09-13T15:27:42.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:41 smithi177 bash[44725]: debug 2024-09-13T15:27:41.820+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:42.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:41 smithi177 bash[44725]: debug 2024-09-13T15:27:41.820+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:42.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:41 smithi177 bash[44725]: debug 2024-09-13T15:27:41.820+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:42.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:41 smithi177 bash[44725]: debug 2024-09-13T15:27:41.820+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:42.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:42 smithi177 bash[38978]: debug 2024-09-13T15:27:42.504+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:46.406183+0000 front 2024-09-13T15:26:46.406265+0000 (oldest deadline 2024-09-13T15:27:08.706070+0000) 2024-09-13T15:27:42.862 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:42 smithi177 bash[38978]: debug 2024-09-13T15:27:42.504+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:54.607376+0000 front 2024-09-13T15:26:54.607541+0000 (oldest deadline 2024-09-13T15:27:19.307372+0000) 2024-09-13T15:27:42.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:42 smithi177 bash[38978]: debug 2024-09-13T15:27:42.504+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:01.609201+0000 front 2024-09-13T15:27:01.609093+0000 (oldest deadline 2024-09-13T15:27:27.508688+0000) 2024-09-13T15:27:42.863 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:42 smithi177 bash[38978]: debug 2024-09-13T15:27:42.504+0000 7f57e48c7640 -1 osd.6 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:13.811180+0000 front 2024-09-13T15:27:13.811250+0000 (oldest deadline 2024-09-13T15:27:37.310916+0000) 2024-09-13T15:27:43.361 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:42 smithi177 bash[44725]: debug 2024-09-13T15:27:42.864+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:43.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:42 smithi177 bash[44725]: debug 2024-09-13T15:27:42.864+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:43.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:42 smithi177 bash[44725]: debug 2024-09-13T15:27:42.864+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:43.362 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:42 smithi177 bash[44725]: debug 2024-09-13T15:27:42.864+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:44.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:43 smithi177 bash[44725]: debug 2024-09-13T15:27:43.820+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:44.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:43 smithi177 bash[44725]: debug 2024-09-13T15:27:43.820+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:44.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:43 smithi177 bash[44725]: debug 2024-09-13T15:27:43.820+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:44.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:43 smithi177 bash[44725]: debug 2024-09-13T15:27:43.820+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:44.496 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:44 smithi177 bash[178666]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-6 2024-09-13T15:27:44.496 INFO:journalctl@ceph.osd.6.smithi177.stdout:Sep 13 15:27:44 smithi177 bash[178734]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-6 2024-09-13T15:27:45.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:44 smithi177 bash[44725]: debug 2024-09-13T15:27:44.776+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:45.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:44 smithi177 bash[44725]: debug 2024-09-13T15:27:44.776+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:45.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:44 smithi177 bash[44725]: debug 2024-09-13T15:27:44.776+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:45.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:44 smithi177 bash[44725]: debug 2024-09-13T15:27:44.776+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:45.922 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:45 smithi177 bash[44725]: debug 2024-09-13T15:27:45.808+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:45.922 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:45 smithi177 bash[44725]: debug 2024-09-13T15:27:45.808+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:45.922 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:45 smithi177 bash[44725]: debug 2024-09-13T15:27:45.808+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:45.922 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:45 smithi177 bash[44725]: debug 2024-09-13T15:27:45.808+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:46.627 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T15:27:46.654 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:27:46.655 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T15:27:46.655 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T15:27:46.655 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.7 2024-09-13T15:27:47.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:46 smithi177 systemd[1]: Stopping Ceph osd.7 for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:27:47.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:46 smithi177 bash[44725]: debug 2024-09-13T15:27:46.840+0000 7fc124e94640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T15:27:47.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:46 smithi177 bash[44725]: debug 2024-09-13T15:27:46.840+0000 7fc124e94640 -1 osd.7 1179 *** Got signal Terminated *** 2024-09-13T15:27:47.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:46 smithi177 bash[44725]: debug 2024-09-13T15:27:46.840+0000 7fc124e94640 -1 osd.7 1179 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:27:47.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:46 smithi177 bash[44725]: debug 2024-09-13T15:27:46.844+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:47.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:46 smithi177 bash[44725]: debug 2024-09-13T15:27:46.844+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:47.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:46 smithi177 bash[44725]: debug 2024-09-13T15:27:46.844+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:47.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:46 smithi177 bash[44725]: debug 2024-09-13T15:27:46.844+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:48.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:47 smithi177 bash[44725]: debug 2024-09-13T15:27:47.844+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:48.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:47 smithi177 bash[44725]: debug 2024-09-13T15:27:47.844+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:48.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:47 smithi177 bash[44725]: debug 2024-09-13T15:27:47.844+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:48.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:47 smithi177 bash[44725]: debug 2024-09-13T15:27:47.844+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:48.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:47 smithi177 bash[44725]: debug 2024-09-13T15:27:47.844+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.177:6804 osd.4 since back 2024-09-13T15:27:22.537819+0000 front 2024-09-13T15:27:22.537970+0000 (oldest deadline 2024-09-13T15:27:47.237560+0000) 2024-09-13T15:27:49.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:48 smithi177 bash[44725]: debug 2024-09-13T15:27:48.796+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:49.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:48 smithi177 bash[44725]: debug 2024-09-13T15:27:48.796+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:49.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:48 smithi177 bash[44725]: debug 2024-09-13T15:27:48.796+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:49.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:48 smithi177 bash[44725]: debug 2024-09-13T15:27:48.796+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:49.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:48 smithi177 bash[44725]: debug 2024-09-13T15:27:48.796+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.177:6804 osd.4 since back 2024-09-13T15:27:22.537819+0000 front 2024-09-13T15:27:22.537970+0000 (oldest deadline 2024-09-13T15:27:47.237560+0000) 2024-09-13T15:27:50.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:49 smithi177 bash[44725]: debug 2024-09-13T15:27:49.752+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:50.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:49 smithi177 bash[44725]: debug 2024-09-13T15:27:49.752+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:50.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:49 smithi177 bash[44725]: debug 2024-09-13T15:27:49.752+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:50.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:49 smithi177 bash[44725]: debug 2024-09-13T15:27:49.752+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:50.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:49 smithi177 bash[44725]: debug 2024-09-13T15:27:49.752+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.177:6804 osd.4 since back 2024-09-13T15:27:22.537819+0000 front 2024-09-13T15:27:22.537970+0000 (oldest deadline 2024-09-13T15:27:47.237560+0000) 2024-09-13T15:27:51.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:50 smithi177 bash[44725]: debug 2024-09-13T15:27:50.720+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:51.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:50 smithi177 bash[44725]: debug 2024-09-13T15:27:50.720+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:51.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:50 smithi177 bash[44725]: debug 2024-09-13T15:27:50.720+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:51.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:50 smithi177 bash[44725]: debug 2024-09-13T15:27:50.720+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:51.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:50 smithi177 bash[44725]: debug 2024-09-13T15:27:50.720+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.177:6804 osd.4 since back 2024-09-13T15:27:22.537819+0000 front 2024-09-13T15:27:22.537970+0000 (oldest deadline 2024-09-13T15:27:47.237560+0000) 2024-09-13T15:27:52.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:51 smithi177 bash[44725]: debug 2024-09-13T15:27:51.699+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6806 osd.0 since back 2024-09-13T15:26:47.029965+0000 front 2024-09-13T15:26:47.030107+0000 (oldest deadline 2024-09-13T15:27:11.130222+0000) 2024-09-13T15:27:52.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:51 smithi177 bash[44725]: debug 2024-09-13T15:27:51.699+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6814 osd.1 since back 2024-09-13T15:26:53.332712+0000 front 2024-09-13T15:26:53.332739+0000 (oldest deadline 2024-09-13T15:27:18.632271+0000) 2024-09-13T15:27:52.112 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:51 smithi177 bash[44725]: debug 2024-09-13T15:27:51.699+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6822 osd.2 since back 2024-09-13T15:27:04.434178+0000 front 2024-09-13T15:27:04.434485+0000 (oldest deadline 2024-09-13T15:27:30.334063+0000) 2024-09-13T15:27:52.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:51 smithi177 bash[44725]: debug 2024-09-13T15:27:51.699+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.99:6830 osd.3 since back 2024-09-13T15:27:10.335010+0000 front 2024-09-13T15:27:10.335085+0000 (oldest deadline 2024-09-13T15:27:35.635043+0000) 2024-09-13T15:27:52.113 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:51 smithi177 bash[44725]: debug 2024-09-13T15:27:51.699+0000 7fc11e4a6640 -1 osd.7 1179 heartbeat_check: no reply from 172.21.15.177:6804 osd.4 since back 2024-09-13T15:27:22.537819+0000 front 2024-09-13T15:27:22.537970+0000 (oldest deadline 2024-09-13T15:27:47.237560+0000) 2024-09-13T15:27:53.611 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:53 smithi177 bash[178865]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-7 2024-09-13T15:27:53.612 INFO:journalctl@ceph.osd.7.smithi177.stdout:Sep 13 15:27:53 smithi177 bash[178930]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-osd-7 2024-09-13T15:27:55.561 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T15:27:55.589 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:27:55.590 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T15:27:55.590 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-13T15:27:55.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@prometheus.a 2024-09-13T15:27:55.829 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 systemd[1]: Stopping Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:27:55.829 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.765Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T15:27:55.830 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.766Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-13T15:27:55.830 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.766Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-13T15:27:55.830 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.766Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T15:27:55.830 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.766Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-13T15:27:55.830 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.766Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-13T15:27:55.830 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.766Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T15:27:55.830 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.766Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-13T15:27:55.830 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.766Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-13T15:27:55.831 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.769Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T15:27:55.831 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.769Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-13T15:27:55.831 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:55 smithi177 bash[61439]: ts=2024-09-13T15:27:55.769Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-13T15:27:56.701 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-13T15:27:56.781 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:56 smithi177 bash[179059]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T15:27:56.781 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:56 smithi177 bash[179119]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T15:27:56.781 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:56 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T15:27:56.782 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Sep 13 15:27:56 smithi177 systemd[1]: Stopped Ceph prometheus.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:27:56.794 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:27:56.794 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-13T15:27:56.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T15:28:02.168 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:01 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:02.169 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:01 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:02.480 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:02 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:02.480 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:02 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:02.480 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:02 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:02.481 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:02 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:02.755 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:02 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:02.755 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:02 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:03.074 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:02 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:03.074 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:02 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:03.371 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:03 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:03.372 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:03 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:03.382 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:03 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:03.382 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:03 smithi099 bash[75088]: ts=2024-09-13T15:28:03.249Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephMgrPrometheusModuleInactive\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-09-13T15:28:03.382 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:03 smithi099 bash[75088]: ts=2024-09-13T15:28:03.251Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephMgrPrometheusModuleInactive\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.177:8443: connect: connection refused" 2024-09-13T15:28:03.382 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:03 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:03.740 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:03 smithi099 systemd[1]: Stopping Ceph node-exporter.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:28:04.467 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:04 smithi099 bash[264658]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T15:28:04.467 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:04 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:28:04.467 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:04 smithi099 bash[264719]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T15:28:04.468 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:04 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T15:28:04.468 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:04 smithi099 systemd[1]: Stopped Ceph node-exporter.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:28:04.795 INFO:journalctl@ceph.node-exporter.a.smithi099.stdout:Sep 13 15:28:04 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:04.796 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:04 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:04.796 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:04 smithi099 systemd[1]: Stopping Ceph alertmanager.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:28:05.077 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:04 smithi099 bash[75088]: ts=2024-09-13T15:28:04.795Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T15:28:05.740 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:05 smithi099 bash[264781]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T15:28:05.740 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:05 smithi099 bash[264842]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T15:28:05.740 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:05 smithi099 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T15:28:05.740 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:05 smithi099 systemd[1]: Stopped Ceph alertmanager.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:28:06.074 INFO:journalctl@ceph.alertmanager.a.smithi099.stdout:Sep 13 15:28:05 smithi099 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:11.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T15:28:16.785 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:16 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:16.786 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:16 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:17.084 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:16 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:17.084 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:16 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:17.362 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:17 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:17.362 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:17 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:17.712 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:17 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:17.713 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:17 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:17.713 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:17 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:17.713 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:17 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:17.977 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:17 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:17.977 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:17 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:18.303 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:18 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:18.303 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:18 smithi177 systemd[1]: Stopping Ceph node-exporter.b for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:28:18.304 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:18 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:19.318 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:18 smithi177 bash[179385]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T15:28:19.319 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:19 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:28:19.319 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:19 smithi177 bash[179446]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T15:28:19.319 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:19 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T15:28:19.319 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:19 smithi177 systemd[1]: Stopped Ceph node-exporter.b for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:28:19.581 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Sep 13 15:28:19 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:19.581 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:19 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:19.582 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:19 smithi177 systemd[1]: Stopping Ceph grafana.a for af9d2c22-71d9-11ef-bceb-c7b262605968... 2024-09-13T15:28:20.361 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:20 smithi177 bash[179509]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-grafana-a 2024-09-13T15:28:20.362 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:20 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:28:20.362 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:20 smithi177 bash[179570]: Error response from daemon: No such container: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968-grafana-a 2024-09-13T15:28:20.362 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:20 smithi177 systemd[1]: ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T15:28:20.362 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:20 smithi177 systemd[1]: Stopped Ceph grafana.a for af9d2c22-71d9-11ef-bceb-c7b262605968. 2024-09-13T15:28:20.862 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Sep 13 15:28:20 smithi177 systemd[1]: /etc/systemd/system/ceph-af9d2c22-71d9-11ef-bceb-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T15:28:30.879 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:28:30.894 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:28:30.907 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T15:28:30.909 DEBUG:teuthology.misc:Transferring archived files from smithi099:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903545/remote/smithi099/crash 2024-09-13T15:28:30.910 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/crash -- . 2024-09-13T15:28:30.963 INFO:teuthology.orchestra.run.smithi099.stderr:tar: /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T15:28:30.963 INFO:teuthology.orchestra.run.smithi099.stderr:tar: Error is not recoverable: exiting now 2024-09-13T15:28:30.965 DEBUG:teuthology.misc:Transferring archived files from smithi177:/var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903545/remote/smithi177/crash 2024-09-13T15:28:30.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo tar c -f - -C /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/crash -- . 2024-09-13T15:28:30.990 INFO:teuthology.orchestra.run.smithi177.stderr:tar: /var/lib/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T15:28:30.990 INFO:teuthology.orchestra.run.smithi177.stderr:tar: Error is not recoverable: exiting now 2024-09-13T15:28:30.991 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T15:28:30.991 DEBUG:teuthology.orchestra.run.smithi099:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v PG_AVAILABILITY | head -n 1 2024-09-13T15:28:31.112 INFO:teuthology.orchestra.run.smithi099.stdout:1726237466.1740053 mon.a (mon.0) 784 : cluster [WRN] Replacing daemon mds.foo.smithi099.yyrtld as rank 0 with standby daemon mds.foo.smithi177.outtkv 2024-09-13T15:28:31.115 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-13T15:28:31.115 INFO:tasks.cephadm:Compressing logs... 2024-09-13T15:28:31.115 DEBUG:teuthology.orchestra.run.smithi099:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T15:28:31.117 DEBUG:teuthology.orchestra.run.smithi177:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T15:28:31.125 INFO:teuthology.orchestra.run.smithi099.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-09-13T15:28:31.134 INFO:teuthology.orchestra.run.smithi177.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-09-13T15:28:31.146 INFO:teuthology.orchestra.run.smithi177.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T15:28:31.147 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T15:28:31.151 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-13T15:28:31.152 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T15:28:31.152 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T15:28:31.152 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mgr.y.log: /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-13T15:28:31.154 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi099.yyrtld.log 2024-09-13T15:28:31.154 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T15:28:31.154 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mon.a.log: /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi099.yyrtld.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T15:28:31.155 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T15:28:31.155 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log: /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.log 2024-09-13T15:28:31.155 INFO:teuthology.orchestra.run.smithi099.stderr: 86.7% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi099.yyrtld.log.gz 2024-09-13T15:28:31.155 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T15:28:31.157 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-13T15:28:31.157 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T15:28:31.160 INFO:teuthology.orchestra.run.smithi177.stderr:gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T15:28:31.160 INFO:teuthology.orchestra.run.smithi177.stderr:gzip -5 --verbose --/var/log/ceph/cephadm.log: /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T15:28:31.161 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi177.outtkv.log 2024-09-13T15:28:31.161 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T15:28:31.161 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi177.outtkv.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T15:28:31.161 INFO:teuthology.orchestra.run.smithi177.stderr: 81.9% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mds.foo.smithi177.outtkv.log.gz 2024-09-13T15:28:31.161 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-13T15:28:31.161 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.log 2024-09-13T15:28:31.161 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T15:28:31.161 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T15:28:31.162 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-13T15:28:31.162 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T15:28:31.237 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mon.c.log: /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.audit.log: 90.8% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T15:28:31.311 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mon.b.log: /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.4.log: 84.1% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T15:28:31.351 INFO:teuthology.orchestra.run.smithi177.stderr: 91.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T15:28:31.364 INFO:teuthology.orchestra.run.smithi177.stderr: 90.0% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T15:28:31.405 INFO:teuthology.orchestra.run.smithi177.stderr: 91.4% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T15:28:31.447 INFO:teuthology.orchestra.run.smithi177.stderr: 87.0% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T15:28:31.594 INFO:teuthology.orchestra.run.smithi099.stderr: 92.6% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.log.gz 91.4% 2024-09-13T15:28:31.594 INFO:teuthology.orchestra.run.smithi099.stderr: -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T15:28:31.625 INFO:teuthology.orchestra.run.smithi099.stderr: 94.0% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T15:28:31.650 INFO:teuthology.orchestra.run.smithi099.stderr: 93.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T15:28:31.828 INFO:teuthology.orchestra.run.smithi177.stderr: 90.3% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-13T15:28:39.246 INFO:teuthology.orchestra.run.smithi099.stderr: 89.8% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-13T15:28:42.269 INFO:teuthology.orchestra.run.smithi177.stderr: 92.5% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-13T15:28:56.913 INFO:teuthology.orchestra.run.smithi099.stderr: 92.3% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-13T15:29:10.363 INFO:teuthology.orchestra.run.smithi099.stderr: 90.5% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-13T15:39:35.151 INFO:teuthology.orchestra.run.smithi177.stderr: 93.6% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T15:40:37.652 INFO:teuthology.orchestra.run.smithi177.stderr: 93.6% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T15:41:00.652 INFO:teuthology.orchestra.run.smithi177.stderr: 93.9% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T15:41:10.631 INFO:teuthology.orchestra.run.smithi177.stderr: 93.6% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T15:41:10.632 INFO:teuthology.orchestra.run.smithi177.stderr: 2024-09-13T15:41:10.632 INFO:teuthology.orchestra.run.smithi177.stderr:real 12m39.507s 2024-09-13T15:41:10.632 INFO:teuthology.orchestra.run.smithi177.stderr:user 13m38.441s 2024-09-13T15:41:10.632 INFO:teuthology.orchestra.run.smithi177.stderr:sys 0m39.190s 2024-09-13T15:47:08.548 INFO:teuthology.orchestra.run.smithi099.stderr: 93.3% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T15:47:32.863 INFO:teuthology.orchestra.run.smithi099.stderr: 93.2% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T15:50:49.147 INFO:teuthology.orchestra.run.smithi099.stderr: 92.6% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T15:50:49.848 INFO:teuthology.orchestra.run.smithi099.stderr: 92.6% -- replaced with /var/log/ceph/af9d2c22-71d9-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T15:50:49.850 INFO:teuthology.orchestra.run.smithi099.stderr: 2024-09-13T15:50:49.850 INFO:teuthology.orchestra.run.smithi099.stderr:real 22m18.732s 2024-09-13T15:50:49.850 INFO:teuthology.orchestra.run.smithi099.stderr:user 19m31.514s 2024-09-13T15:50:49.850 INFO:teuthology.orchestra.run.smithi099.stderr:sys 0m55.272s 2024-09-13T15:50:49.851 INFO:tasks.cephadm:Archiving logs... 2024-09-13T15:50:49.856 DEBUG:teuthology.misc:Transferring archived files from smithi099:/var/log/ceph to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903545/remote/smithi099/log 2024-09-13T15:50:49.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T15:51:51.178 DEBUG:teuthology.misc:Transferring archived files from smithi177:/var/log/ceph to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903545/remote/smithi177/log 2024-09-13T15:51:51.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T15:52:31.287 INFO:tasks.cephadm:Removing cluster... 2024-09-13T15:52:31.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 --force 2024-09-13T15:52:33.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid af9d2c22-71d9-11ef-bceb-c7b262605968 --force 2024-09-13T15:52:36.014 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T15:52:36.014 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T15:52:36.032 DEBUG:teuthology.orchestra.run.smithi177:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T15:52:36.038 INFO:tasks.cephadm:Teardown complete 2024-09-13T15:52:36.038 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-13T15:52:36.050 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-13T15:52:36.051 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T15:52:36.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T15:52:36.210 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T15:52:36.220 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T15:52:36.220 DEBUG:teuthology.orchestra.run.smithi099:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T15:52:36.223 DEBUG:teuthology.orchestra.run.smithi177:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T15:52:36.254 INFO:teuthology.orchestra.run.smithi099.stdout: remote refid st t when poll reach delay offset jitter 2024-09-13T15:52:36.254 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================== 2024-09-13T15:52:36.254 INFO:teuthology.orchestra.run.smithi099.stdout:*hv01.front.sepi 67.205.162.81 3 u 10 256 377 0.188 -2.033 0.214 2024-09-13T15:52:36.254 INFO:teuthology.orchestra.run.smithi099.stdout:+hv02.front.sepi 63.231.80.2 3 u 33 256 377 0.206 -1.304 0.396 2024-09-13T15:52:36.254 INFO:teuthology.orchestra.run.smithi099.stdout:+hv03.front.sepi 74.6.168.72 3 u 28 256 377 0.176 -2.244 0.225 2024-09-13T15:52:36.255 INFO:teuthology.orchestra.run.smithi099.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-09-13T15:52:36.267 INFO:teuthology.orchestra.run.smithi177.stdout: remote refid st t when poll reach delay offset jitter 2024-09-13T15:52:36.267 INFO:teuthology.orchestra.run.smithi177.stdout:============================================================================== 2024-09-13T15:52:36.267 INFO:teuthology.orchestra.run.smithi177.stdout:*hv01.front.sepi 67.205.162.81 3 u 89 128 377 0.115 -1.563 0.376 2024-09-13T15:52:36.267 INFO:teuthology.orchestra.run.smithi177.stdout:+hv02.front.sepi 63.231.80.2 3 u 20 128 377 0.088 -0.591 0.442 2024-09-13T15:52:36.268 INFO:teuthology.orchestra.run.smithi177.stdout:+hv03.front.sepi 74.6.168.72 3 u 23 128 377 0.169 -1.839 0.316 2024-09-13T15:52:36.268 INFO:teuthology.orchestra.run.smithi177.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-09-13T15:52:36.269 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T15:52:36.279 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T15:52:36.280 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T15:52:36.290 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T15:52:36.304 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T15:52:36.317 INFO:teuthology.task.internal:Duration was 6782.794966 seconds 2024-09-13T15:52:36.318 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T15:52:36.331 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T15:52:36.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T15:52:36.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T15:52:36.461 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T15:52:36.461 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi099.front.sepia.ceph.com 2024-09-13T15:52:36.462 DEBUG:teuthology.orchestra.run.smithi099:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T15:52:36.488 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi177.front.sepia.ceph.com 2024-09-13T15:52:36.488 DEBUG:teuthology.orchestra.run.smithi177:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T15:52:36.511 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T15:52:36.512 DEBUG:teuthology.orchestra.run.smithi099:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T15:52:36.537 DEBUG:teuthology.orchestra.run.smithi177:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T15:52:36.800 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T15:52:36.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T15:52:36.802 DEBUG:teuthology.orchestra.run.smithi177:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T15:52:37.670 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T15:52:37.683 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T15:52:37.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T15:52:37.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T15:52:37.760 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T15:52:37.770 DEBUG:teuthology.orchestra.run.smithi099:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:52:37.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:52:37.804 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern = core 2024-09-13T15:52:37.825 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern = core 2024-09-13T15:52:37.848 DEBUG:teuthology.orchestra.run.smithi099:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:52:37.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:52:37.896 DEBUG:teuthology.orchestra.run.smithi177:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:52:37.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:52:37.901 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T15:52:37.911 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T15:52:37.913 DEBUG:teuthology.misc:Transferring archived files from smithi099:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903545/remote/smithi099 2024-09-13T15:52:37.913 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T15:52:37.995 DEBUG:teuthology.misc:Transferring archived files from smithi177:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903545/remote/smithi177 2024-09-13T15:52:37.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T15:52:38.033 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T15:52:38.033 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T15:52:38.036 DEBUG:teuthology.orchestra.run.smithi177:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T15:52:38.072 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T15:52:38.084 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T15:52:38.084 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T15:52:38.118 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T15:52:38.118 DEBUG:teuthology.orchestra.run.smithi099:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T15:52:38.121 DEBUG:teuthology.orchestra.run.smithi177:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T15:52:38.125 INFO:teuthology.orchestra.run.smithi099.stdout: 658628 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 15:52 /home/ubuntu/cephtest 2024-09-13T15:52:38.125 INFO:teuthology.orchestra.run.smithi099.stdout: 1051844 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 13 15:20 /home/ubuntu/cephtest/mnt.0 2024-09-13T15:52:38.125 INFO:teuthology.orchestra.run.smithi099.stdout: 1051845 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 13 15:20 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T15:52:38.125 INFO:teuthology.orchestra.run.smithi099.stdout: 1714000 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 13 15:20 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T15:52:38.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:52:38.126 INFO:teuthology.orchestra.run.smithi099.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-13T15:52:38.126 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi099 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T15:52:38.133 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T15:52:38.143 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T15:52:38.156 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi099 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T15:52:38.159 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 6782.794965982437 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi099 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_akupczyk@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=ba6b3b2c3b7c40ff8d28c7d9a04059ea status: fail success: false 2024-09-13T15:52:38.160 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T15:52:38.186 INFO:teuthology.orchestra.run.smithi177.stdout: 655475 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 15:52 /home/ubuntu/cephtest 2024-09-13T15:52:38.270 INFO:teuthology.run:FAIL